@ainyc/canonry 4.77.0 → 4.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +18 -0
- package/assets/assets/{BacklinksPage-CwXveumn.js → BacklinksPage-dRc62jAY.js} +1 -1
- package/assets/assets/{ChartPrimitives-DntKGI5J.js → ChartPrimitives-D2_IvTkk.js} +1 -1
- package/assets/assets/{ProjectPage-CVudiU8X.js → ProjectPage-DSuvRUIf.js} +1 -1
- package/assets/assets/{RunRow-DMtYXaxG.js → RunRow-C0MA3yuQ.js} +1 -1
- package/assets/assets/{RunsPage-Cz-YlucO.js → RunsPage-4uxTYgGy.js} +1 -1
- package/assets/assets/{SettingsPage-BCuG3C-0.js → SettingsPage-3-SLhcJ7.js} +1 -1
- package/assets/assets/{TrafficPage-DV8X47wa.js → TrafficPage-DZ50qwme.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BmYhK9jm.js → TrafficSourceDetailPage-CzK5TMFp.js} +1 -1
- package/assets/assets/{arrow-left-CUmHyNnF.js → arrow-left-BaZIkAXX.js} +1 -1
- package/assets/assets/{extract-error-message-DFjy9_zi.js → extract-error-message-cpvfuFqW.js} +1 -1
- package/assets/assets/{index-D9smxU6R.js → index-EnY_OBRd.js} +70 -70
- package/assets/assets/{trash-2-B_UtEEm8.js → trash-2-JpcztiS5.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-XI6YSTGE.js → chunk-2QBSRHSN.js} +187 -1
- package/dist/{chunk-KPN22EWK.js → chunk-AVN6Q6LM.js} +138 -2
- package/dist/{chunk-BPZWX7YI.js → chunk-CXIGHPBE.js} +1006 -324
- package/dist/{chunk-FB43IMZT.js → chunk-LCABGFYN.js} +724 -286
- package/dist/cli.js +372 -148
- package/dist/index.d.ts +17 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-C76ZRMF5.js → intelligence-service-ZWW3I3NL.js} +2 -2
- package/dist/mcp.js +9 -3
- package/package.json +11 -10
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
loadConfig,
|
|
10
10
|
loadConfigRaw,
|
|
11
11
|
saveConfigPatch
|
|
12
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-2QBSRHSN.js";
|
|
13
13
|
import {
|
|
14
14
|
CC_CACHE_DIR,
|
|
15
15
|
DUCKDB_SPEC,
|
|
@@ -17,6 +17,11 @@ import {
|
|
|
17
17
|
GSC_DATA_LAG_DAYS,
|
|
18
18
|
IntelligenceService,
|
|
19
19
|
PLUGIN_DIR,
|
|
20
|
+
adsAdGroups,
|
|
21
|
+
adsAds,
|
|
22
|
+
adsCampaigns,
|
|
23
|
+
adsConnections,
|
|
24
|
+
adsInsightsDaily,
|
|
20
25
|
agentMemory,
|
|
21
26
|
agentSessions,
|
|
22
27
|
aiUserFetchEventsHourly,
|
|
@@ -97,7 +102,7 @@ import {
|
|
|
97
102
|
siteAuditPages,
|
|
98
103
|
siteAuditSnapshots,
|
|
99
104
|
usageCounters
|
|
100
|
-
} from "./chunk-
|
|
105
|
+
} from "./chunk-CXIGHPBE.js";
|
|
101
106
|
import {
|
|
102
107
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
103
108
|
AGENT_PROVIDER_IDS,
|
|
@@ -134,6 +139,7 @@ import {
|
|
|
134
139
|
contentActionLabel,
|
|
135
140
|
contentBriefDtoSchema,
|
|
136
141
|
determineAnswerMentioned,
|
|
142
|
+
dollarsToMicros,
|
|
137
143
|
effectiveBrandNames,
|
|
138
144
|
effectiveDomains,
|
|
139
145
|
factorStatusFromScore,
|
|
@@ -149,7 +155,7 @@ import {
|
|
|
149
155
|
validationError,
|
|
150
156
|
winnabilityClassLabel,
|
|
151
157
|
withRetry
|
|
152
|
-
} from "./chunk-
|
|
158
|
+
} from "./chunk-AVN6Q6LM.js";
|
|
153
159
|
|
|
154
160
|
// src/telemetry.ts
|
|
155
161
|
import crypto from "crypto";
|
|
@@ -437,11 +443,11 @@ function checkLatestVersionForServer(opts) {
|
|
|
437
443
|
|
|
438
444
|
// src/server.ts
|
|
439
445
|
import { createRequire as createRequire3 } from "module";
|
|
440
|
-
import
|
|
446
|
+
import crypto19 from "crypto";
|
|
441
447
|
import fs8 from "fs";
|
|
442
448
|
import path9 from "path";
|
|
443
449
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
444
|
-
import { and as and13, eq as
|
|
450
|
+
import { and as and13, eq as eq19 } from "drizzle-orm";
|
|
445
451
|
import Fastify from "fastify";
|
|
446
452
|
import os5 from "os";
|
|
447
453
|
|
|
@@ -1836,6 +1842,7 @@ var CDPProviderError = class extends Error {
|
|
|
1836
1842
|
this.code = code;
|
|
1837
1843
|
this.name = "CDPProviderError";
|
|
1838
1844
|
}
|
|
1845
|
+
code;
|
|
1839
1846
|
};
|
|
1840
1847
|
|
|
1841
1848
|
// ../provider-cdp/src/connection.ts
|
|
@@ -3044,6 +3051,8 @@ var ProviderExecutionGate = class {
|
|
|
3044
3051
|
this.maxConcurrency = maxConcurrency;
|
|
3045
3052
|
this.maxPerMinute = maxPerMinute;
|
|
3046
3053
|
}
|
|
3054
|
+
maxConcurrency;
|
|
3055
|
+
maxPerMinute;
|
|
3047
3056
|
window = [];
|
|
3048
3057
|
waiters = [];
|
|
3049
3058
|
rateLimitChain = Promise.resolve();
|
|
@@ -4074,12 +4083,361 @@ function monthKey(m) {
|
|
|
4074
4083
|
return `${m.year}-${String(m.month).padStart(2, "0")}`;
|
|
4075
4084
|
}
|
|
4076
4085
|
|
|
4077
|
-
// src/
|
|
4086
|
+
// src/ads-sync.ts
|
|
4078
4087
|
import crypto6 from "crypto";
|
|
4079
|
-
import { eq as eq4
|
|
4088
|
+
import { eq as eq4 } from "drizzle-orm";
|
|
4089
|
+
|
|
4090
|
+
// ../integration-openai-ads/src/constants.ts
|
|
4091
|
+
var OPENAI_ADS_API_BASE = "https://api.ads.openai.com/v1";
|
|
4092
|
+
var OPENAI_ADS_REQUEST_TIMEOUT_MS = 3e4;
|
|
4093
|
+
var OPENAI_ADS_MAX_PAGES = 100;
|
|
4094
|
+
|
|
4095
|
+
// ../integration-openai-ads/src/types.ts
|
|
4096
|
+
var OpenAiAdsApiError = class extends Error {
|
|
4097
|
+
status;
|
|
4098
|
+
code;
|
|
4099
|
+
constructor(message, status, code = null) {
|
|
4100
|
+
super(message);
|
|
4101
|
+
this.name = "OpenAiAdsApiError";
|
|
4102
|
+
this.status = status;
|
|
4103
|
+
this.code = code;
|
|
4104
|
+
}
|
|
4105
|
+
};
|
|
4106
|
+
function parseErrorEnvelope(body) {
|
|
4107
|
+
try {
|
|
4108
|
+
const parsed = JSON.parse(body);
|
|
4109
|
+
return {
|
|
4110
|
+
message: parsed.error?.message ?? null,
|
|
4111
|
+
code: parsed.error?.code ?? null
|
|
4112
|
+
};
|
|
4113
|
+
} catch {
|
|
4114
|
+
return { message: null, code: null };
|
|
4115
|
+
}
|
|
4116
|
+
}
|
|
4117
|
+
|
|
4118
|
+
// ../integration-openai-ads/src/ads-client.ts
|
|
4119
|
+
function validateApiKey(apiKey) {
|
|
4120
|
+
if (!apiKey || typeof apiKey !== "string" || apiKey.trim().length === 0) {
|
|
4121
|
+
throw new OpenAiAdsApiError("API key is required and must be a non-empty string", 400);
|
|
4122
|
+
}
|
|
4123
|
+
}
|
|
4124
|
+
function validateId(value, label) {
|
|
4125
|
+
if (!value || typeof value !== "string" || value.trim().length === 0) {
|
|
4126
|
+
throw new OpenAiAdsApiError(`${label} is required and must be a non-empty string`, 400);
|
|
4127
|
+
}
|
|
4128
|
+
}
|
|
4129
|
+
function adsClientLog(level, action, ctx) {
|
|
4130
|
+
const entry = {
|
|
4131
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4132
|
+
level,
|
|
4133
|
+
module: "OpenAiAdsClient",
|
|
4134
|
+
action,
|
|
4135
|
+
...ctx
|
|
4136
|
+
};
|
|
4137
|
+
if (entry.apiKey) entry.apiKey = "***";
|
|
4138
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
4139
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
4140
|
+
}
|
|
4141
|
+
function buildUrl(path10, queryPairs) {
|
|
4142
|
+
const qs = queryPairs.join("&");
|
|
4143
|
+
return qs ? `${OPENAI_ADS_API_BASE}/${path10}?${qs}` : `${OPENAI_ADS_API_BASE}/${path10}`;
|
|
4144
|
+
}
|
|
4145
|
+
async function adsFetch(apiKey, path10, queryPairs = []) {
|
|
4146
|
+
const url = buildUrl(path10, queryPairs);
|
|
4147
|
+
const res = await fetch(url, {
|
|
4148
|
+
method: "GET",
|
|
4149
|
+
headers: {
|
|
4150
|
+
Authorization: `Bearer ${apiKey}`,
|
|
4151
|
+
"Content-Type": "application/json"
|
|
4152
|
+
},
|
|
4153
|
+
signal: AbortSignal.timeout(OPENAI_ADS_REQUEST_TIMEOUT_MS)
|
|
4154
|
+
});
|
|
4155
|
+
if (res.status === 401 || res.status === 403) {
|
|
4156
|
+
const { code } = parseErrorEnvelope(await res.text());
|
|
4157
|
+
adsClientLog("error", "http.auth-failed", { path: path10, httpStatus: res.status, code });
|
|
4158
|
+
throw new OpenAiAdsApiError("OpenAI Ads API key is invalid or unauthorized", res.status, code);
|
|
4159
|
+
}
|
|
4160
|
+
if (res.status === 429) {
|
|
4161
|
+
const { code } = parseErrorEnvelope(await res.text());
|
|
4162
|
+
adsClientLog("error", "http.rate-limited", { path: path10, httpStatus: 429, code });
|
|
4163
|
+
throw new OpenAiAdsApiError("OpenAI Ads API rate limit exceeded", 429, code);
|
|
4164
|
+
}
|
|
4165
|
+
if (!res.ok) {
|
|
4166
|
+
const body = await res.text();
|
|
4167
|
+
const { message, code } = parseErrorEnvelope(body);
|
|
4168
|
+
adsClientLog("error", "http.error", { path: path10, httpStatus: res.status, code });
|
|
4169
|
+
const detail = message ?? (body.length <= 500 ? body : `${body.slice(0, 500)}... [truncated]`);
|
|
4170
|
+
throw new OpenAiAdsApiError(`OpenAI Ads API error (${res.status}): ${detail}`, res.status, code);
|
|
4171
|
+
}
|
|
4172
|
+
const text = await res.text();
|
|
4173
|
+
try {
|
|
4174
|
+
return JSON.parse(text);
|
|
4175
|
+
} catch {
|
|
4176
|
+
throw new OpenAiAdsApiError("OpenAI Ads API returned invalid JSON", 502);
|
|
4177
|
+
}
|
|
4178
|
+
}
|
|
4179
|
+
async function fetchAllPages(apiKey, path10, queryPairs) {
|
|
4180
|
+
const items = [];
|
|
4181
|
+
let after = null;
|
|
4182
|
+
for (let page = 0; page < OPENAI_ADS_MAX_PAGES; page++) {
|
|
4183
|
+
const pairs = after ? [...queryPairs, `after=${encodeURIComponent(after)}`] : [...queryPairs];
|
|
4184
|
+
const response = await adsFetch(apiKey, path10, pairs);
|
|
4185
|
+
items.push(...response.data);
|
|
4186
|
+
if (!response.has_more || !response.last_id) {
|
|
4187
|
+
return items;
|
|
4188
|
+
}
|
|
4189
|
+
after = response.last_id;
|
|
4190
|
+
}
|
|
4191
|
+
adsClientLog("error", "pagination.cap-reached", { path: path10, pages: OPENAI_ADS_MAX_PAGES, items: items.length });
|
|
4192
|
+
return items;
|
|
4193
|
+
}
|
|
4194
|
+
function insightsPairs(opts) {
|
|
4195
|
+
return (opts?.fields ?? []).map((field) => `fields[]=${encodeURIComponent(field)}`);
|
|
4196
|
+
}
|
|
4197
|
+
async function getAdAccount(apiKey) {
|
|
4198
|
+
validateApiKey(apiKey);
|
|
4199
|
+
return adsFetch(apiKey, "ad_account");
|
|
4200
|
+
}
|
|
4201
|
+
async function listCampaigns(apiKey) {
|
|
4202
|
+
validateApiKey(apiKey);
|
|
4203
|
+
return fetchAllPages(apiKey, "campaigns", []);
|
|
4204
|
+
}
|
|
4205
|
+
async function listAdGroups(apiKey, campaignId) {
|
|
4206
|
+
validateApiKey(apiKey);
|
|
4207
|
+
validateId(campaignId, "Campaign id");
|
|
4208
|
+
return fetchAllPages(apiKey, "ad_groups", [`campaign_id=${encodeURIComponent(campaignId)}`]);
|
|
4209
|
+
}
|
|
4210
|
+
async function listAds(apiKey, adGroupId) {
|
|
4211
|
+
validateApiKey(apiKey);
|
|
4212
|
+
validateId(adGroupId, "Ad group id");
|
|
4213
|
+
return fetchAllPages(apiKey, "ads", [`ad_group_id=${encodeURIComponent(adGroupId)}`]);
|
|
4214
|
+
}
|
|
4215
|
+
async function getCampaignInsights(apiKey, campaignId, opts) {
|
|
4216
|
+
validateApiKey(apiKey);
|
|
4217
|
+
validateId(campaignId, "Campaign id");
|
|
4218
|
+
return fetchAllPages(
|
|
4219
|
+
apiKey,
|
|
4220
|
+
`campaigns/${encodeURIComponent(campaignId)}/insights`,
|
|
4221
|
+
insightsPairs(opts)
|
|
4222
|
+
);
|
|
4223
|
+
}
|
|
4224
|
+
async function getAdGroupInsights(apiKey, adGroupId, opts) {
|
|
4225
|
+
validateApiKey(apiKey);
|
|
4226
|
+
validateId(adGroupId, "Ad group id");
|
|
4227
|
+
return fetchAllPages(
|
|
4228
|
+
apiKey,
|
|
4229
|
+
`ad_groups/${encodeURIComponent(adGroupId)}/insights`,
|
|
4230
|
+
insightsPairs(opts)
|
|
4231
|
+
);
|
|
4232
|
+
}
|
|
4233
|
+
|
|
4234
|
+
// src/ads-config.ts
|
|
4235
|
+
function ensureConnections7(config) {
|
|
4236
|
+
if (!config.openaiAds) config.openaiAds = {};
|
|
4237
|
+
if (!config.openaiAds.connections) config.openaiAds.connections = [];
|
|
4238
|
+
return config.openaiAds.connections;
|
|
4239
|
+
}
|
|
4240
|
+
function getOpenAiAdsConnection(config, projectName) {
|
|
4241
|
+
return (config.openaiAds?.connections ?? []).find((c) => c.projectName === projectName);
|
|
4242
|
+
}
|
|
4243
|
+
function upsertOpenAiAdsConnection(config, connection) {
|
|
4244
|
+
const connections = ensureConnections7(config);
|
|
4245
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
4246
|
+
if (index === -1) {
|
|
4247
|
+
connections.push(connection);
|
|
4248
|
+
return connection;
|
|
4249
|
+
}
|
|
4250
|
+
connections[index] = connection;
|
|
4251
|
+
return connection;
|
|
4252
|
+
}
|
|
4253
|
+
function removeOpenAiAdsConnection(config, projectName) {
|
|
4254
|
+
const connections = config.openaiAds?.connections;
|
|
4255
|
+
if (!connections?.length) return false;
|
|
4256
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
4257
|
+
if (next.length === connections.length) return false;
|
|
4258
|
+
if (!config.openaiAds) return false;
|
|
4259
|
+
config.openaiAds.connections = next;
|
|
4260
|
+
if (next.length === 0) {
|
|
4261
|
+
delete config.openaiAds;
|
|
4262
|
+
}
|
|
4263
|
+
return true;
|
|
4264
|
+
}
|
|
4265
|
+
|
|
4266
|
+
// src/ads-sync.ts
|
|
4267
|
+
var log4 = createLogger("AdsSync");
|
|
4268
|
+
var CAMPAIGN_INSIGHT_FIELDS = ["campaign.impressions", "campaign.clicks", "campaign.spend", "metadata.readable_time"];
|
|
4269
|
+
var AD_GROUP_INSIGHT_FIELDS = ["ad_group.impressions", "ad_group.clicks", "ad_group.spend", "metadata.readable_time"];
|
|
4270
|
+
function toInsightUpserts(level, entityId, rows) {
|
|
4271
|
+
const upserts = [];
|
|
4272
|
+
for (const row of rows) {
|
|
4273
|
+
if (!row.readable_time) {
|
|
4274
|
+
log4.warn("insights.row-missing-date", { level, entityId, rowId: row.id });
|
|
4275
|
+
continue;
|
|
4276
|
+
}
|
|
4277
|
+
upserts.push({
|
|
4278
|
+
level,
|
|
4279
|
+
entityId,
|
|
4280
|
+
date: row.readable_time,
|
|
4281
|
+
impressions: row.impressions ?? 0,
|
|
4282
|
+
clicks: row.clicks ?? 0,
|
|
4283
|
+
spendMicros: dollarsToMicros(row.spend ?? 0)
|
|
4284
|
+
});
|
|
4285
|
+
}
|
|
4286
|
+
return upserts;
|
|
4287
|
+
}
|
|
4288
|
+
async function executeAdsSync(db, runId, projectId, opts) {
|
|
4289
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4290
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq4(runs.id, runId)).run();
|
|
4291
|
+
try {
|
|
4292
|
+
const project = db.select().from(projects).where(eq4(projects.id, projectId)).get();
|
|
4293
|
+
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
4294
|
+
const connRow = db.select().from(adsConnections).where(eq4(adsConnections.projectId, projectId)).get();
|
|
4295
|
+
if (!connRow) {
|
|
4296
|
+
throw new Error('No ads connection found for this project. Run "canonry ads connect" first.');
|
|
4297
|
+
}
|
|
4298
|
+
const cfgConn = getOpenAiAdsConnection(opts.config, project.name);
|
|
4299
|
+
if (!cfgConn?.apiKey) {
|
|
4300
|
+
throw new Error('No OpenAI Ads API key in the local Canonry config. Run "canonry ads connect" first.');
|
|
4301
|
+
}
|
|
4302
|
+
const apiKey = cfgConn.apiKey;
|
|
4303
|
+
log4.info("sync.start", { runId, projectId, adAccountId: connRow.adAccountId });
|
|
4304
|
+
const account = await getAdAccount(apiKey);
|
|
4305
|
+
const campaigns = await listCampaigns(apiKey);
|
|
4306
|
+
const errors = /* @__PURE__ */ new Map();
|
|
4307
|
+
const adGroupsByCampaign = /* @__PURE__ */ new Map();
|
|
4308
|
+
const adsByGroup = /* @__PURE__ */ new Map();
|
|
4309
|
+
const insightUpserts = [];
|
|
4310
|
+
const syncedCampaigns = [];
|
|
4311
|
+
for (const campaign of campaigns) {
|
|
4312
|
+
try {
|
|
4313
|
+
const [adGroups, campaignInsights] = await Promise.all([
|
|
4314
|
+
listAdGroups(apiKey, campaign.id),
|
|
4315
|
+
getCampaignInsights(apiKey, campaign.id, { fields: CAMPAIGN_INSIGHT_FIELDS })
|
|
4316
|
+
]);
|
|
4317
|
+
const groupResults = await Promise.all(adGroups.map(async (group) => ({
|
|
4318
|
+
group,
|
|
4319
|
+
ads: await listAds(apiKey, group.id),
|
|
4320
|
+
insights: await getAdGroupInsights(apiKey, group.id, { fields: AD_GROUP_INSIGHT_FIELDS })
|
|
4321
|
+
})));
|
|
4322
|
+
syncedCampaigns.push(campaign);
|
|
4323
|
+
adGroupsByCampaign.set(campaign.id, adGroups);
|
|
4324
|
+
insightUpserts.push(...toInsightUpserts("campaign", campaign.id, campaignInsights));
|
|
4325
|
+
for (const { group, ads, insights: insights2 } of groupResults) {
|
|
4326
|
+
adsByGroup.set(group.id, ads);
|
|
4327
|
+
insightUpserts.push(...toInsightUpserts("ad_group", group.id, insights2));
|
|
4328
|
+
}
|
|
4329
|
+
} catch (err) {
|
|
4330
|
+
errors.set(campaign.name, err instanceof Error ? err.message : String(err));
|
|
4331
|
+
log4.error("campaign.failed", { runId, campaignId: campaign.id, error: err instanceof Error ? err.message : String(err) });
|
|
4332
|
+
}
|
|
4333
|
+
}
|
|
4334
|
+
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4335
|
+
db.transaction((tx) => {
|
|
4336
|
+
tx.delete(adsCampaigns).where(eq4(adsCampaigns.projectId, projectId)).run();
|
|
4337
|
+
for (const campaign of syncedCampaigns) {
|
|
4338
|
+
tx.insert(adsCampaigns).values({
|
|
4339
|
+
id: campaign.id,
|
|
4340
|
+
projectId,
|
|
4341
|
+
name: campaign.name,
|
|
4342
|
+
status: campaign.status,
|
|
4343
|
+
biddingType: campaign.bidding_type,
|
|
4344
|
+
dailySpendLimitMicros: campaign.budget?.daily_spend_limit_micros ?? null,
|
|
4345
|
+
lifetimeSpendLimitMicros: campaign.budget?.lifetime_spend_limit_micros ?? null,
|
|
4346
|
+
targeting: campaign.targeting,
|
|
4347
|
+
upstreamCreatedAt: campaign.created_at,
|
|
4348
|
+
upstreamUpdatedAt: campaign.updated_at,
|
|
4349
|
+
syncRunId: runId,
|
|
4350
|
+
syncedAt: insertNow
|
|
4351
|
+
}).run();
|
|
4352
|
+
for (const group of adGroupsByCampaign.get(campaign.id) ?? []) {
|
|
4353
|
+
tx.insert(adsAdGroups).values({
|
|
4354
|
+
id: group.id,
|
|
4355
|
+
projectId,
|
|
4356
|
+
campaignId: campaign.id,
|
|
4357
|
+
name: group.name,
|
|
4358
|
+
status: group.status,
|
|
4359
|
+
billingEventType: group.bidding_config?.billing_event_type ?? null,
|
|
4360
|
+
maxBidMicros: group.bidding_config?.max_bid_micros ?? null,
|
|
4361
|
+
contextHints: group.context_hints,
|
|
4362
|
+
upstreamCreatedAt: group.created_at,
|
|
4363
|
+
upstreamUpdatedAt: group.updated_at,
|
|
4364
|
+
syncRunId: runId,
|
|
4365
|
+
syncedAt: insertNow
|
|
4366
|
+
}).run();
|
|
4367
|
+
for (const ad of adsByGroup.get(group.id) ?? []) {
|
|
4368
|
+
tx.insert(adsAds).values({
|
|
4369
|
+
id: ad.id,
|
|
4370
|
+
projectId,
|
|
4371
|
+
adGroupId: group.id,
|
|
4372
|
+
name: ad.name,
|
|
4373
|
+
status: ad.status,
|
|
4374
|
+
creative: ad.creative,
|
|
4375
|
+
reviewStatus: ad.review_status ?? ad.review?.status ?? null,
|
|
4376
|
+
upstreamCreatedAt: ad.created_at,
|
|
4377
|
+
upstreamUpdatedAt: ad.updated_at,
|
|
4378
|
+
syncRunId: runId,
|
|
4379
|
+
syncedAt: insertNow
|
|
4380
|
+
}).run();
|
|
4381
|
+
}
|
|
4382
|
+
}
|
|
4383
|
+
}
|
|
4384
|
+
for (const upsert of insightUpserts) {
|
|
4385
|
+
tx.insert(adsInsightsDaily).values({
|
|
4386
|
+
id: crypto6.randomUUID(),
|
|
4387
|
+
projectId,
|
|
4388
|
+
...upsert,
|
|
4389
|
+
syncRunId: runId
|
|
4390
|
+
}).onConflictDoUpdate({
|
|
4391
|
+
target: [adsInsightsDaily.projectId, adsInsightsDaily.level, adsInsightsDaily.entityId, adsInsightsDaily.date],
|
|
4392
|
+
set: {
|
|
4393
|
+
impressions: upsert.impressions,
|
|
4394
|
+
clicks: upsert.clicks,
|
|
4395
|
+
spendMicros: upsert.spendMicros,
|
|
4396
|
+
syncRunId: runId
|
|
4397
|
+
}
|
|
4398
|
+
}).run();
|
|
4399
|
+
}
|
|
4400
|
+
tx.update(adsConnections).set({
|
|
4401
|
+
adAccountId: account.id,
|
|
4402
|
+
displayName: account.name,
|
|
4403
|
+
currencyCode: account.currency_code,
|
|
4404
|
+
timezone: account.timezone,
|
|
4405
|
+
status: account.status,
|
|
4406
|
+
lastSyncedAt: insertNow,
|
|
4407
|
+
updatedAt: insertNow
|
|
4408
|
+
}).where(eq4(adsConnections.projectId, projectId)).run();
|
|
4409
|
+
});
|
|
4410
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4411
|
+
if (errors.size === 0) {
|
|
4412
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq4(runs.id, runId)).run();
|
|
4413
|
+
} else if (syncedCampaigns.length > 0) {
|
|
4414
|
+
db.update(runs).set({
|
|
4415
|
+
status: "partial",
|
|
4416
|
+
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4417
|
+
finishedAt
|
|
4418
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4419
|
+
} else {
|
|
4420
|
+
db.update(runs).set({
|
|
4421
|
+
status: "failed",
|
|
4422
|
+
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4423
|
+
finishedAt
|
|
4424
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4425
|
+
}
|
|
4426
|
+
log4.info("sync.done", { runId, projectId, campaigns: syncedCampaigns.length, insightRows: insightUpserts.length, failed: errors.size });
|
|
4427
|
+
} catch (err) {
|
|
4428
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4429
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
4430
|
+
log4.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
4431
|
+
throw err;
|
|
4432
|
+
}
|
|
4433
|
+
}
|
|
4434
|
+
|
|
4435
|
+
// src/gsc-inspect-sitemap.ts
|
|
4436
|
+
import crypto7 from "crypto";
|
|
4437
|
+
import { eq as eq5, and as and4 } from "drizzle-orm";
|
|
4080
4438
|
|
|
4081
4439
|
// src/sitemap-parser.ts
|
|
4082
|
-
var
|
|
4440
|
+
var log5 = createLogger("SitemapParser");
|
|
4083
4441
|
var LOC_REGEX = /<loc>([^<]+)<\/loc>/gi;
|
|
4084
4442
|
var SITEMAP_TAG_REGEX = /<sitemap>[\s\S]*?<\/sitemap>/gi;
|
|
4085
4443
|
async function validateSitemapUrl(url) {
|
|
@@ -4121,7 +4479,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4121
4479
|
res = await fetch(url);
|
|
4122
4480
|
} catch (err) {
|
|
4123
4481
|
if (!isChild) throw err;
|
|
4124
|
-
|
|
4482
|
+
log5.warn("child-sitemap.fetch-failed", {
|
|
4125
4483
|
url,
|
|
4126
4484
|
error: err instanceof Error ? err.message : String(err)
|
|
4127
4485
|
});
|
|
@@ -4131,7 +4489,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4131
4489
|
if (!isChild) {
|
|
4132
4490
|
throw new Error(`Failed to fetch sitemap at ${url}: ${res.status} ${res.statusText}`);
|
|
4133
4491
|
}
|
|
4134
|
-
|
|
4492
|
+
log5.warn("child-sitemap.http-error", { url, status: res.status, statusText: res.statusText });
|
|
4135
4493
|
return;
|
|
4136
4494
|
}
|
|
4137
4495
|
let xml;
|
|
@@ -4139,7 +4497,7 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4139
4497
|
xml = await readSitemapBody(res);
|
|
4140
4498
|
} catch (err) {
|
|
4141
4499
|
if (!isChild) throw err;
|
|
4142
|
-
|
|
4500
|
+
log5.warn("child-sitemap.parse-failed", {
|
|
4143
4501
|
url,
|
|
4144
4502
|
error: err instanceof Error ? err.message : String(err)
|
|
4145
4503
|
});
|
|
@@ -4175,16 +4533,16 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
4175
4533
|
}
|
|
4176
4534
|
|
|
4177
4535
|
// src/gsc-inspect-sitemap.ts
|
|
4178
|
-
var
|
|
4536
|
+
var log6 = createLogger("InspectSitemap");
|
|
4179
4537
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
4180
4538
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4181
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4539
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq5(runs.id, runId)).run();
|
|
4182
4540
|
try {
|
|
4183
4541
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
4184
4542
|
if (!googleClientId || !googleClientSecret) {
|
|
4185
4543
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
4186
4544
|
}
|
|
4187
|
-
const project = db.select().from(projects).where(
|
|
4545
|
+
const project = db.select().from(projects).where(eq5(projects.id, projectId)).get();
|
|
4188
4546
|
if (!project) {
|
|
4189
4547
|
throw new Error(`Project not found: ${projectId}`);
|
|
4190
4548
|
}
|
|
@@ -4209,9 +4567,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4209
4567
|
saveConfigPatch(opts.config);
|
|
4210
4568
|
}
|
|
4211
4569
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
4212
|
-
|
|
4570
|
+
log6.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
4213
4571
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
4214
|
-
|
|
4572
|
+
log6.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
4215
4573
|
if (urls.length === 0) {
|
|
4216
4574
|
throw new Error("No URLs found in sitemap");
|
|
4217
4575
|
}
|
|
@@ -4226,7 +4584,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4226
4584
|
const rich = ir.richResultsResult;
|
|
4227
4585
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4228
4586
|
db.insert(gscUrlInspections).values({
|
|
4229
|
-
id:
|
|
4587
|
+
id: crypto7.randomUUID(),
|
|
4230
4588
|
projectId,
|
|
4231
4589
|
syncRunId: runId,
|
|
4232
4590
|
url: pageUrl,
|
|
@@ -4243,16 +4601,16 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4243
4601
|
inspectedAt,
|
|
4244
4602
|
createdAt: inspectedAt
|
|
4245
4603
|
}).run();
|
|
4246
|
-
|
|
4604
|
+
log6.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${index + 1}/${urls.length}` });
|
|
4247
4605
|
},
|
|
4248
4606
|
onError: (pageUrl, err) => {
|
|
4249
|
-
|
|
4607
|
+
log6.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
4250
4608
|
}
|
|
4251
4609
|
},
|
|
4252
4610
|
{
|
|
4253
4611
|
log: {
|
|
4254
|
-
info: (action, ctx) =>
|
|
4255
|
-
error: (action, ctx) =>
|
|
4612
|
+
info: (action, ctx) => log6.info(action, { runId, projectId, ...ctx }),
|
|
4613
|
+
error: (action, ctx) => log6.error(action, { runId, projectId, ...ctx })
|
|
4256
4614
|
}
|
|
4257
4615
|
}
|
|
4258
4616
|
);
|
|
@@ -4262,7 +4620,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4262
4620
|
`URL inspection aborted after ${INSPECT_FAILFAST_THRESHOLD} consecutive rate/access failures (likely GSC URL Inspection quota exhaustion or property access loss). Last error: ${detail}`
|
|
4263
4621
|
);
|
|
4264
4622
|
}
|
|
4265
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
4623
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq5(gscUrlInspections.projectId, projectId)).all();
|
|
4266
4624
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
4267
4625
|
for (const row of allInspections) {
|
|
4268
4626
|
const existing = latestByUrl.get(row.url);
|
|
@@ -4283,9 +4641,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4283
4641
|
}
|
|
4284
4642
|
}
|
|
4285
4643
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
4286
|
-
db.delete(gscCoverageSnapshots).where(and4(
|
|
4644
|
+
db.delete(gscCoverageSnapshots).where(and4(eq5(gscCoverageSnapshots.projectId, projectId), eq5(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
4287
4645
|
db.insert(gscCoverageSnapshots).values({
|
|
4288
|
-
id:
|
|
4646
|
+
id: crypto7.randomUUID(),
|
|
4289
4647
|
projectId,
|
|
4290
4648
|
syncRunId: runId,
|
|
4291
4649
|
date: snapshotDate,
|
|
@@ -4295,20 +4653,20 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
4295
4653
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4296
4654
|
}).run();
|
|
4297
4655
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
4298
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4299
|
-
|
|
4656
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4657
|
+
log6.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
4300
4658
|
} catch (err) {
|
|
4301
4659
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4302
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4303
|
-
|
|
4660
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
4661
|
+
log6.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
4304
4662
|
throw err;
|
|
4305
4663
|
}
|
|
4306
4664
|
}
|
|
4307
4665
|
|
|
4308
4666
|
// src/bing-inspect-sitemap.ts
|
|
4309
|
-
import
|
|
4310
|
-
import { eq as
|
|
4311
|
-
var
|
|
4667
|
+
import crypto8 from "crypto";
|
|
4668
|
+
import { eq as eq6, desc as desc2 } from "drizzle-orm";
|
|
4669
|
+
var log7 = createLogger("BingInspectSitemap");
|
|
4312
4670
|
function parseBingDate(value) {
|
|
4313
4671
|
if (!value) return null;
|
|
4314
4672
|
const match = /\/Date\((-?\d+)(?:[-+]\d+)?\)\//.exec(value);
|
|
@@ -4325,9 +4683,9 @@ function isBlockingIssueType(issueType) {
|
|
|
4325
4683
|
}
|
|
4326
4684
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
4327
4685
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4328
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
4686
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq6(runs.id, runId)).run();
|
|
4329
4687
|
try {
|
|
4330
|
-
const project = db.select().from(projects).where(
|
|
4688
|
+
const project = db.select().from(projects).where(eq6(projects.id, projectId)).get();
|
|
4331
4689
|
if (!project) {
|
|
4332
4690
|
throw new Error(`Project not found: ${projectId}`);
|
|
4333
4691
|
}
|
|
@@ -4339,16 +4697,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4339
4697
|
throw new Error('No Bing site configured. Run "canonry bing set-site <project> <url>" first.');
|
|
4340
4698
|
}
|
|
4341
4699
|
const sitemapUrl = opts.sitemapUrl ?? `https://${project.canonicalDomain}/sitemap.xml`;
|
|
4342
|
-
|
|
4700
|
+
log7.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
4343
4701
|
const sitemapUrls = await fetchAndParseSitemap(sitemapUrl);
|
|
4344
|
-
|
|
4702
|
+
log7.info("sitemap.parsed", { runId, projectId, urlCount: sitemapUrls.length, sitemapUrl });
|
|
4345
4703
|
if (sitemapUrls.length === 0) {
|
|
4346
4704
|
throw new Error("No URLs found in sitemap");
|
|
4347
4705
|
}
|
|
4348
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
4706
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq6(bingUrlInspections.projectId, projectId)).all();
|
|
4349
4707
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
4350
4708
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
4351
|
-
|
|
4709
|
+
log7.info("sitemap.diff", {
|
|
4352
4710
|
runId,
|
|
4353
4711
|
projectId,
|
|
4354
4712
|
sitemapTotal: sitemapUrls.length,
|
|
@@ -4363,9 +4721,9 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4363
4721
|
blockedUrls.add(issue.Url);
|
|
4364
4722
|
}
|
|
4365
4723
|
}
|
|
4366
|
-
|
|
4724
|
+
log7.info("crawl-issues.loaded", { runId, projectId, blockedCount: blockedUrls.size });
|
|
4367
4725
|
} catch (err) {
|
|
4368
|
-
|
|
4726
|
+
log7.warn("crawl-issues.lookup-failed", {
|
|
4369
4727
|
runId,
|
|
4370
4728
|
projectId,
|
|
4371
4729
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -4394,7 +4752,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4394
4752
|
derivedInIndex = false;
|
|
4395
4753
|
}
|
|
4396
4754
|
db.insert(bingUrlInspections).values({
|
|
4397
|
-
id:
|
|
4755
|
+
id: crypto8.randomUUID(),
|
|
4398
4756
|
projectId,
|
|
4399
4757
|
url: pageUrl,
|
|
4400
4758
|
httpCode,
|
|
@@ -4409,7 +4767,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4409
4767
|
discoveryDate
|
|
4410
4768
|
}).run();
|
|
4411
4769
|
inspected++;
|
|
4412
|
-
|
|
4770
|
+
log7.info("inspect.url-done", {
|
|
4413
4771
|
runId,
|
|
4414
4772
|
projectId,
|
|
4415
4773
|
url: pageUrl,
|
|
@@ -4417,7 +4775,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4417
4775
|
});
|
|
4418
4776
|
} catch (err) {
|
|
4419
4777
|
errors++;
|
|
4420
|
-
|
|
4778
|
+
log7.error("inspect.url-failed", {
|
|
4421
4779
|
runId,
|
|
4422
4780
|
projectId,
|
|
4423
4781
|
url: pageUrl,
|
|
@@ -4428,7 +4786,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4428
4786
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
4429
4787
|
}
|
|
4430
4788
|
}
|
|
4431
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
4789
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq6(bingUrlInspections.projectId, projectId)).orderBy(desc2(bingUrlInspections.inspectedAt)).all();
|
|
4432
4790
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
4433
4791
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
4434
4792
|
for (const row of allInspections) {
|
|
@@ -4452,7 +4810,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4452
4810
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
4453
4811
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4454
4812
|
db.insert(bingCoverageSnapshots).values({
|
|
4455
|
-
id:
|
|
4813
|
+
id: crypto8.randomUUID(),
|
|
4456
4814
|
projectId,
|
|
4457
4815
|
syncRunId: runId,
|
|
4458
4816
|
date: snapshotDate,
|
|
@@ -4471,8 +4829,8 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4471
4829
|
}
|
|
4472
4830
|
}).run();
|
|
4473
4831
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
4474
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4475
|
-
|
|
4832
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
4833
|
+
log7.info("inspect.completed", {
|
|
4476
4834
|
runId,
|
|
4477
4835
|
projectId,
|
|
4478
4836
|
inspected,
|
|
@@ -4485,16 +4843,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
4485
4843
|
});
|
|
4486
4844
|
} catch (err) {
|
|
4487
4845
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4488
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4489
|
-
|
|
4846
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
4847
|
+
log7.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
4490
4848
|
throw err;
|
|
4491
4849
|
}
|
|
4492
4850
|
}
|
|
4493
4851
|
|
|
4494
4852
|
// src/coverage-refresh.ts
|
|
4495
|
-
import
|
|
4496
|
-
import { and as and5, desc as desc3, eq as
|
|
4497
|
-
var
|
|
4853
|
+
import crypto9 from "crypto";
|
|
4854
|
+
import { and as and5, desc as desc3, eq as eq7, inArray as inArray3 } from "drizzle-orm";
|
|
4855
|
+
var log8 = createLogger("CoverageRefresh");
|
|
4498
4856
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
4499
4857
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
4500
4858
|
RunStatuses.queued,
|
|
@@ -4504,7 +4862,7 @@ var ACTIVE_OR_DONE_STATUSES = [
|
|
|
4504
4862
|
];
|
|
4505
4863
|
var defaultDeps = { executeInspectSitemap };
|
|
4506
4864
|
async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps, nowMs = Date.now()) {
|
|
4507
|
-
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(
|
|
4865
|
+
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(eq7(projects.id, projectId)).get();
|
|
4508
4866
|
if (!project) return null;
|
|
4509
4867
|
const { clientId, clientSecret } = getGoogleAuthConfig(config);
|
|
4510
4868
|
if (!clientId || !clientSecret) return null;
|
|
@@ -4512,19 +4870,19 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4512
4870
|
if (!conn?.refreshToken || !conn.propertyId) return null;
|
|
4513
4871
|
const recent = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
4514
4872
|
and5(
|
|
4515
|
-
|
|
4516
|
-
|
|
4873
|
+
eq7(runs.projectId, projectId),
|
|
4874
|
+
eq7(runs.kind, RunKinds["inspect-sitemap"]),
|
|
4517
4875
|
inArray3(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
4518
4876
|
)
|
|
4519
4877
|
).orderBy(desc3(runs.createdAt)).limit(1).get();
|
|
4520
4878
|
if (recent) {
|
|
4521
4879
|
const ageMs = nowMs - Date.parse(recent.createdAt);
|
|
4522
4880
|
if (Number.isFinite(ageMs) && ageMs < COVERAGE_REFRESH_MIN_INTERVAL_MS) {
|
|
4523
|
-
|
|
4881
|
+
log8.info("skip.recent", { projectId, ageMs });
|
|
4524
4882
|
return null;
|
|
4525
4883
|
}
|
|
4526
4884
|
}
|
|
4527
|
-
const runId =
|
|
4885
|
+
const runId = crypto9.randomUUID();
|
|
4528
4886
|
db.insert(runs).values({
|
|
4529
4887
|
id: runId,
|
|
4530
4888
|
projectId,
|
|
@@ -4533,11 +4891,11 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4533
4891
|
trigger: RunTriggers.scheduled,
|
|
4534
4892
|
createdAt: new Date(nowMs).toISOString()
|
|
4535
4893
|
}).run();
|
|
4536
|
-
|
|
4894
|
+
log8.info("refresh.start", { projectId, runId });
|
|
4537
4895
|
try {
|
|
4538
4896
|
await deps.executeInspectSitemap(db, runId, projectId, { config });
|
|
4539
4897
|
} catch (err) {
|
|
4540
|
-
|
|
4898
|
+
log8.error("refresh.failed", {
|
|
4541
4899
|
projectId,
|
|
4542
4900
|
runId,
|
|
4543
4901
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -4547,10 +4905,10 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
4547
4905
|
}
|
|
4548
4906
|
|
|
4549
4907
|
// src/commoncrawl-sync.ts
|
|
4550
|
-
import
|
|
4908
|
+
import crypto10 from "crypto";
|
|
4551
4909
|
import path4 from "path";
|
|
4552
|
-
import { and as and6, eq as
|
|
4553
|
-
var
|
|
4910
|
+
import { and as and6, eq as eq8, sql as sql3 } from "drizzle-orm";
|
|
4911
|
+
var log9 = createLogger("CommonCrawlSync");
|
|
4554
4912
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
4555
4913
|
function defaultDeps2() {
|
|
4556
4914
|
return {
|
|
@@ -4575,7 +4933,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4575
4933
|
phaseDetail: "downloading vertices + edges",
|
|
4576
4934
|
updatedAt: downloadStartedAt,
|
|
4577
4935
|
error: null
|
|
4578
|
-
}).where(
|
|
4936
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
4579
4937
|
const paths = ccReleasePaths(release);
|
|
4580
4938
|
const releaseCacheDir = path4.join(deps.cacheDir, release);
|
|
4581
4939
|
const vertexPath = path4.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -4598,7 +4956,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4598
4956
|
vertexSha256: vertex.sha256,
|
|
4599
4957
|
edgesSha256: edges.sha256,
|
|
4600
4958
|
updatedAt: downloadFinishedAt
|
|
4601
|
-
}).where(
|
|
4959
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
4602
4960
|
const allProjects = db.select().from(projects).all();
|
|
4603
4961
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
4604
4962
|
let rows = [];
|
|
@@ -4614,15 +4972,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4614
4972
|
}
|
|
4615
4973
|
const queriedAt = deps.now().toISOString();
|
|
4616
4974
|
db.transaction((tx) => {
|
|
4617
|
-
tx.delete(backlinkDomains).where(
|
|
4618
|
-
tx.delete(backlinkSummaries).where(
|
|
4975
|
+
tx.delete(backlinkDomains).where(eq8(backlinkDomains.releaseSyncId, syncId)).run();
|
|
4976
|
+
tx.delete(backlinkSummaries).where(eq8(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
4619
4977
|
const expanded = [];
|
|
4620
4978
|
for (const r of rows) {
|
|
4621
4979
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
4622
4980
|
if (!projectIds) continue;
|
|
4623
4981
|
for (const projectId of projectIds) {
|
|
4624
4982
|
expanded.push({
|
|
4625
|
-
id:
|
|
4983
|
+
id: crypto10.randomUUID(),
|
|
4626
4984
|
projectId,
|
|
4627
4985
|
releaseSyncId: syncId,
|
|
4628
4986
|
release,
|
|
@@ -4642,7 +5000,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4642
5000
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
4643
5001
|
const summary = computeSummary(projectRows);
|
|
4644
5002
|
tx.insert(backlinkSummaries).values({
|
|
4645
|
-
id:
|
|
5003
|
+
id: crypto10.randomUUID(),
|
|
4646
5004
|
projectId: p.id,
|
|
4647
5005
|
releaseSyncId: syncId,
|
|
4648
5006
|
release,
|
|
@@ -4674,8 +5032,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4674
5032
|
domainsDiscovered: rows.length,
|
|
4675
5033
|
updatedAt: finishedAt,
|
|
4676
5034
|
error: null
|
|
4677
|
-
}).where(
|
|
4678
|
-
|
|
5035
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
5036
|
+
log9.info("sync.completed", {
|
|
4679
5037
|
syncId,
|
|
4680
5038
|
release,
|
|
4681
5039
|
projectsProcessed: allProjects.length,
|
|
@@ -4687,7 +5045,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4687
5045
|
try {
|
|
4688
5046
|
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
4689
5047
|
} catch (err) {
|
|
4690
|
-
|
|
5048
|
+
log9.error("auto-extract.enqueue-failed", {
|
|
4691
5049
|
syncId,
|
|
4692
5050
|
release,
|
|
4693
5051
|
projectId: p.id,
|
|
@@ -4704,8 +5062,8 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
4704
5062
|
error: errorMsg,
|
|
4705
5063
|
phaseDetail: null,
|
|
4706
5064
|
updatedAt: finishedAt
|
|
4707
|
-
}).where(
|
|
4708
|
-
|
|
5065
|
+
}).where(eq8(ccReleaseSyncs.id, syncId)).run();
|
|
5066
|
+
log9.error("sync.failed", { syncId, release, error: errorMsg });
|
|
4709
5067
|
throw err;
|
|
4710
5068
|
}
|
|
4711
5069
|
}
|
|
@@ -4738,10 +5096,10 @@ function computeSummary(rows) {
|
|
|
4738
5096
|
}
|
|
4739
5097
|
|
|
4740
5098
|
// src/backlink-extract.ts
|
|
4741
|
-
import
|
|
5099
|
+
import crypto11 from "crypto";
|
|
4742
5100
|
import fs3 from "fs";
|
|
4743
|
-
import { and as and7, desc as desc4, eq as
|
|
4744
|
-
var
|
|
5101
|
+
import { and as and7, desc as desc4, eq as eq9 } from "drizzle-orm";
|
|
5102
|
+
var log10 = createLogger("BacklinkExtract");
|
|
4745
5103
|
function defaultDeps3() {
|
|
4746
5104
|
return {
|
|
4747
5105
|
queryBacklinks,
|
|
@@ -4752,13 +5110,13 @@ function defaultDeps3() {
|
|
|
4752
5110
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
4753
5111
|
const deps = { ...defaultDeps3(), ...opts.deps };
|
|
4754
5112
|
const startedAt = deps.now().toISOString();
|
|
4755
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5113
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq9(runs.id, runId)).run();
|
|
4756
5114
|
try {
|
|
4757
|
-
const project = db.select().from(projects).where(
|
|
5115
|
+
const project = db.select().from(projects).where(eq9(projects.id, projectId)).get();
|
|
4758
5116
|
if (!project) {
|
|
4759
5117
|
throw new Error(`Project not found: ${projectId}`);
|
|
4760
5118
|
}
|
|
4761
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
5119
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq9(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq9(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc4(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
4762
5120
|
if (!sync) {
|
|
4763
5121
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
4764
5122
|
}
|
|
@@ -4786,11 +5144,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4786
5144
|
const targetDomain = project.canonicalDomain;
|
|
4787
5145
|
db.transaction((tx) => {
|
|
4788
5146
|
tx.delete(backlinkDomains).where(
|
|
4789
|
-
and7(
|
|
5147
|
+
and7(eq9(backlinkDomains.projectId, projectId), eq9(backlinkDomains.release, release))
|
|
4790
5148
|
).run();
|
|
4791
5149
|
if (rows.length > 0) {
|
|
4792
5150
|
const values = rows.map((r) => ({
|
|
4793
|
-
id:
|
|
5151
|
+
id: crypto11.randomUUID(),
|
|
4794
5152
|
projectId,
|
|
4795
5153
|
releaseSyncId: syncId,
|
|
4796
5154
|
release,
|
|
@@ -4803,7 +5161,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4803
5161
|
}
|
|
4804
5162
|
const summary = computeSummary2(rows);
|
|
4805
5163
|
tx.insert(backlinkSummaries).values({
|
|
4806
|
-
id:
|
|
5164
|
+
id: crypto11.randomUUID(),
|
|
4807
5165
|
projectId,
|
|
4808
5166
|
releaseSyncId: syncId,
|
|
4809
5167
|
release,
|
|
@@ -4826,8 +5184,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4826
5184
|
}).run();
|
|
4827
5185
|
});
|
|
4828
5186
|
const finishedAt = deps.now().toISOString();
|
|
4829
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
4830
|
-
|
|
5187
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq9(runs.id, runId)).run();
|
|
5188
|
+
log10.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
4831
5189
|
} catch (err) {
|
|
4832
5190
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4833
5191
|
const finishedAt = deps.now().toISOString();
|
|
@@ -4835,8 +5193,8 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
4835
5193
|
status: RunStatuses.failed,
|
|
4836
5194
|
error: errorMsg,
|
|
4837
5195
|
finishedAt
|
|
4838
|
-
}).where(
|
|
4839
|
-
|
|
5196
|
+
}).where(eq9(runs.id, runId)).run();
|
|
5197
|
+
log10.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
4840
5198
|
throw err;
|
|
4841
5199
|
}
|
|
4842
5200
|
}
|
|
@@ -4856,18 +5214,18 @@ function computeSummary2(rows) {
|
|
|
4856
5214
|
}
|
|
4857
5215
|
|
|
4858
5216
|
// src/discovery-run.ts
|
|
4859
|
-
import
|
|
4860
|
-
import { and as and8, eq as
|
|
4861
|
-
var
|
|
5217
|
+
import crypto12 from "crypto";
|
|
5218
|
+
import { and as and8, eq as eq10 } from "drizzle-orm";
|
|
5219
|
+
var log11 = createLogger("DiscoveryRun");
|
|
4862
5220
|
var DEFAULT_SEED_COUNT = 30;
|
|
4863
5221
|
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
4864
5222
|
async function executeDiscoveryRun(opts) {
|
|
4865
5223
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4866
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5224
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq10(runs.id, opts.runId)).run();
|
|
4867
5225
|
try {
|
|
4868
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
5226
|
+
const projectRow = opts.db.select().from(projects).where(eq10(projects.id, opts.projectId)).get();
|
|
4869
5227
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
4870
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5228
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq10(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
4871
5229
|
const canonicalDomains = effectiveDomains({
|
|
4872
5230
|
canonicalDomain: projectRow.canonicalDomain,
|
|
4873
5231
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -4897,8 +5255,8 @@ async function executeDiscoveryRun(opts) {
|
|
|
4897
5255
|
seedProvider: result.seedProvider,
|
|
4898
5256
|
result
|
|
4899
5257
|
});
|
|
4900
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4901
|
-
|
|
5258
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq10(runs.id, opts.runId)).run();
|
|
5259
|
+
log11.info("discovery.completed", {
|
|
4902
5260
|
runId: opts.runId,
|
|
4903
5261
|
sessionId: opts.sessionId,
|
|
4904
5262
|
buckets: result.buckets,
|
|
@@ -4906,13 +5264,13 @@ async function executeDiscoveryRun(opts) {
|
|
|
4906
5264
|
});
|
|
4907
5265
|
} catch (err) {
|
|
4908
5266
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4909
|
-
|
|
5267
|
+
log11.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
4910
5268
|
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
4911
5269
|
opts.db.update(runs).set({
|
|
4912
5270
|
status: RunStatuses.failed,
|
|
4913
5271
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4914
5272
|
error: errorMsg
|
|
4915
|
-
}).where(
|
|
5273
|
+
}).where(eq10(runs.id, opts.runId)).run();
|
|
4916
5274
|
}
|
|
4917
5275
|
}
|
|
4918
5276
|
function buildDefaultDeps(registry) {
|
|
@@ -5118,12 +5476,12 @@ function writeDiscoveryInsight(db, input) {
|
|
|
5118
5476
|
});
|
|
5119
5477
|
db.transaction((tx) => {
|
|
5120
5478
|
tx.update(insights).set({ dismissed: true }).where(and8(
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5479
|
+
eq10(insights.projectId, input.projectId),
|
|
5480
|
+
eq10(insights.type, "discovery.basket-divergence"),
|
|
5481
|
+
eq10(insights.dismissed, false)
|
|
5124
5482
|
)).run();
|
|
5125
5483
|
tx.insert(insights).values({
|
|
5126
|
-
id:
|
|
5484
|
+
id: crypto12.randomUUID(),
|
|
5127
5485
|
projectId: input.projectId,
|
|
5128
5486
|
runId: input.runId,
|
|
5129
5487
|
type: "discovery.basket-divergence",
|
|
@@ -5159,10 +5517,10 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
5159
5517
|
}
|
|
5160
5518
|
|
|
5161
5519
|
// src/execute-site-audit.ts
|
|
5162
|
-
import
|
|
5163
|
-
import { eq as
|
|
5520
|
+
import crypto13 from "crypto";
|
|
5521
|
+
import { eq as eq11 } from "drizzle-orm";
|
|
5164
5522
|
import { runSitemapAudit } from "@ainyc/aeo-audit";
|
|
5165
|
-
var
|
|
5523
|
+
var log12 = createLogger("SiteAudit");
|
|
5166
5524
|
var SITE_AUDIT_DEFAULT_PAGE_LIMIT = 500;
|
|
5167
5525
|
var SITE_AUDIT_MAX_PAGE_LIMIT = 2e3;
|
|
5168
5526
|
function toHomepageUrl(canonicalDomain) {
|
|
@@ -5223,15 +5581,15 @@ function computeFactorAverages(pages) {
|
|
|
5223
5581
|
}
|
|
5224
5582
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
5225
5583
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5226
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
5584
|
+
db.update(runs).set({ status: "running", startedAt }).where(eq11(runs.id, runId)).run();
|
|
5227
5585
|
try {
|
|
5228
|
-
const project = db.select().from(projects).where(
|
|
5586
|
+
const project = db.select().from(projects).where(eq11(projects.id, projectId)).get();
|
|
5229
5587
|
if (!project) {
|
|
5230
5588
|
throw new Error(`Project not found: ${projectId}`);
|
|
5231
5589
|
}
|
|
5232
5590
|
const homepageUrl = toHomepageUrl(project.canonicalDomain);
|
|
5233
5591
|
const limit = clampSiteAuditLimit(opts.limit);
|
|
5234
|
-
|
|
5592
|
+
log12.info("start", { runId, projectId, homepageUrl, sitemapUrl: opts.sitemapUrl ?? null, limit });
|
|
5235
5593
|
await assertSiteAuditUrlAllowed(homepageUrl, "canonicalDomain");
|
|
5236
5594
|
if (opts.sitemapUrl) await assertSiteAuditUrlAllowed(opts.sitemapUrl, "sitemapUrl");
|
|
5237
5595
|
const report = await runSitemapAudit(homepageUrl, { sitemapUrl: opts.sitemapUrl, limit });
|
|
@@ -5239,7 +5597,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5239
5597
|
const pagesErrored = report.pages.filter((page) => page.status === "error").length;
|
|
5240
5598
|
const auditable = report.pagesDiscovered - report.pagesSkipped;
|
|
5241
5599
|
if (auditable > report.pagesAudited) {
|
|
5242
|
-
|
|
5600
|
+
log12.info("truncated", {
|
|
5243
5601
|
runId,
|
|
5244
5602
|
projectId,
|
|
5245
5603
|
auditable,
|
|
@@ -5258,7 +5616,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5258
5616
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5259
5617
|
db.transaction((tx) => {
|
|
5260
5618
|
tx.insert(siteAuditSnapshots).values({
|
|
5261
|
-
id:
|
|
5619
|
+
id: crypto13.randomUUID(),
|
|
5262
5620
|
projectId,
|
|
5263
5621
|
runId,
|
|
5264
5622
|
sitemapUrl: report.sitemapUrl,
|
|
@@ -5287,7 +5645,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5287
5645
|
}).run();
|
|
5288
5646
|
for (const page of report.pages) {
|
|
5289
5647
|
tx.insert(siteAuditPages).values({
|
|
5290
|
-
id:
|
|
5648
|
+
id: crypto13.randomUUID(),
|
|
5291
5649
|
projectId,
|
|
5292
5650
|
runId,
|
|
5293
5651
|
url: page.url,
|
|
@@ -5298,9 +5656,9 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5298
5656
|
createdAt: finishedAt
|
|
5299
5657
|
}).run();
|
|
5300
5658
|
}
|
|
5301
|
-
tx.update(runs).set({ status, finishedAt }).where(
|
|
5659
|
+
tx.update(runs).set({ status, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
5302
5660
|
});
|
|
5303
|
-
|
|
5661
|
+
log12.info("completed", {
|
|
5304
5662
|
runId,
|
|
5305
5663
|
projectId,
|
|
5306
5664
|
status,
|
|
@@ -5310,14 +5668,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
5310
5668
|
});
|
|
5311
5669
|
} catch (err) {
|
|
5312
5670
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5313
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5314
|
-
|
|
5671
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq11(runs.id, runId)).run();
|
|
5672
|
+
log12.error("failed", { runId, projectId, error: errorMsg });
|
|
5315
5673
|
throw err;
|
|
5316
5674
|
}
|
|
5317
5675
|
}
|
|
5318
5676
|
|
|
5319
5677
|
// src/commands/backfill.ts
|
|
5320
|
-
import { and as and9, eq as
|
|
5678
|
+
import { and as and9, eq as eq12, inArray as inArray4, isNull, sql as sql4 } from "drizzle-orm";
|
|
5321
5679
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
5322
5680
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
5323
5681
|
const config = loadConfig();
|
|
@@ -5325,7 +5683,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5325
5683
|
migrate(db);
|
|
5326
5684
|
const projectFilter = opts?.project?.trim();
|
|
5327
5685
|
const isDryRun = opts?.dryRun === true;
|
|
5328
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
5686
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq12(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
5329
5687
|
let examined = 0;
|
|
5330
5688
|
let updated = 0;
|
|
5331
5689
|
let wouldUpdate = 0;
|
|
@@ -5334,9 +5692,9 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5334
5692
|
let providerErrors = 0;
|
|
5335
5693
|
if (scopedProjects.length > 0) {
|
|
5336
5694
|
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and9(
|
|
5337
|
-
|
|
5695
|
+
eq12(runs.kind, RunKinds["answer-visibility"]),
|
|
5338
5696
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
5339
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
5697
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq12(runs.kind, RunKinds["answer-visibility"])).all();
|
|
5340
5698
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
5341
5699
|
for (const run of runRows) {
|
|
5342
5700
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -5344,7 +5702,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5344
5702
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
5345
5703
|
}
|
|
5346
5704
|
for (const project of scopedProjects) {
|
|
5347
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5705
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq12(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
5348
5706
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
5349
5707
|
if (runIds.length === 0) continue;
|
|
5350
5708
|
const projectDomains = effectiveDomains({
|
|
@@ -5432,7 +5790,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
5432
5790
|
} else {
|
|
5433
5791
|
db.transaction((tx) => {
|
|
5434
5792
|
for (const update of pendingUpdates) {
|
|
5435
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
5793
|
+
tx.update(querySnapshots).set(update.patch).where(eq12(querySnapshots.id, update.id)).run();
|
|
5436
5794
|
}
|
|
5437
5795
|
});
|
|
5438
5796
|
updated += pendingUpdates.length;
|
|
@@ -5481,7 +5839,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
5481
5839
|
function backfillNormalizedPaths(db, opts) {
|
|
5482
5840
|
const baseConditions = [];
|
|
5483
5841
|
if (opts?.projectId) {
|
|
5484
|
-
baseConditions.push(
|
|
5842
|
+
baseConditions.push(eq12(gaTrafficSnapshots.projectId, opts.projectId));
|
|
5485
5843
|
}
|
|
5486
5844
|
const rows = db.select({
|
|
5487
5845
|
id: gaTrafficSnapshots.id,
|
|
@@ -5502,7 +5860,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
5502
5860
|
unchanged++;
|
|
5503
5861
|
continue;
|
|
5504
5862
|
}
|
|
5505
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
5863
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq12(gaTrafficSnapshots.id, row.id)).run();
|
|
5506
5864
|
updated++;
|
|
5507
5865
|
}
|
|
5508
5866
|
});
|
|
@@ -5516,7 +5874,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
5516
5874
|
const projectFilter = opts?.project?.trim();
|
|
5517
5875
|
let projectId;
|
|
5518
5876
|
if (projectFilter) {
|
|
5519
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
5877
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq12(projects.name, projectFilter)).get();
|
|
5520
5878
|
if (!project) {
|
|
5521
5879
|
const result2 = {
|
|
5522
5880
|
project: projectFilter,
|
|
@@ -5553,7 +5911,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
5553
5911
|
function backfillAiReferralPaths(db, opts) {
|
|
5554
5912
|
const baseConditions = [];
|
|
5555
5913
|
if (opts?.projectId) {
|
|
5556
|
-
baseConditions.push(
|
|
5914
|
+
baseConditions.push(eq12(gaAiReferrals.projectId, opts.projectId));
|
|
5557
5915
|
}
|
|
5558
5916
|
const rows = db.select({
|
|
5559
5917
|
id: gaAiReferrals.id,
|
|
@@ -5574,7 +5932,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
5574
5932
|
unchanged++;
|
|
5575
5933
|
continue;
|
|
5576
5934
|
}
|
|
5577
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
5935
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq12(gaAiReferrals.id, row.id)).run();
|
|
5578
5936
|
updated++;
|
|
5579
5937
|
}
|
|
5580
5938
|
});
|
|
@@ -5588,7 +5946,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
5588
5946
|
const projectFilter = opts?.project?.trim();
|
|
5589
5947
|
let projectId;
|
|
5590
5948
|
if (projectFilter) {
|
|
5591
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
5949
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq12(projects.name, projectFilter)).get();
|
|
5592
5950
|
if (!project) {
|
|
5593
5951
|
const result2 = {
|
|
5594
5952
|
project: projectFilter,
|
|
@@ -5624,10 +5982,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
5624
5982
|
}
|
|
5625
5983
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
5626
5984
|
const isDryRun = opts?.dryRun === true;
|
|
5627
|
-
const project = db.select().from(projects).where(
|
|
5985
|
+
const project = db.select().from(projects).where(eq12(projects.id, projectId)).get();
|
|
5628
5986
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
5629
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
5630
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(and9(
|
|
5987
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq12(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
5988
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and9(eq12(runs.kind, RunKinds["answer-visibility"]), eq12(runs.projectId, projectId))).all();
|
|
5631
5989
|
const runIds = runRows.map((r) => r.id);
|
|
5632
5990
|
let examined = 0;
|
|
5633
5991
|
let updated = 0;
|
|
@@ -5699,7 +6057,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
5699
6057
|
} else {
|
|
5700
6058
|
db.transaction((tx) => {
|
|
5701
6059
|
for (const update of pendingUpdates) {
|
|
5702
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
6060
|
+
tx.update(querySnapshots).set(update.patch).where(eq12(querySnapshots.id, update.id)).run();
|
|
5703
6061
|
}
|
|
5704
6062
|
});
|
|
5705
6063
|
updated += pendingUpdates.length;
|
|
@@ -5714,7 +6072,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
5714
6072
|
migrate(db);
|
|
5715
6073
|
const projectFilter = opts?.project?.trim();
|
|
5716
6074
|
const isDryRun = opts?.dryRun === true;
|
|
5717
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6075
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq12(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
5718
6076
|
let examined = 0;
|
|
5719
6077
|
let updated = 0;
|
|
5720
6078
|
let wouldUpdate = 0;
|
|
@@ -5774,7 +6132,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
5774
6132
|
return result;
|
|
5775
6133
|
}
|
|
5776
6134
|
async function backfillInsightsCommand(project, opts) {
|
|
5777
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
6135
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-ZWW3I3NL.js");
|
|
5778
6136
|
const config = loadConfig();
|
|
5779
6137
|
const db = createClient(config.database);
|
|
5780
6138
|
migrate(db);
|
|
@@ -5933,7 +6291,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5933
6291
|
const config = loadConfig();
|
|
5934
6292
|
const db = createClient(config.database);
|
|
5935
6293
|
migrate(db);
|
|
5936
|
-
const project = db.select().from(projects).where(
|
|
6294
|
+
const project = db.select().from(projects).where(eq12(projects.name, opts.project)).get();
|
|
5937
6295
|
if (!project) {
|
|
5938
6296
|
throw new Error(`Project "${opts.project}" not found`);
|
|
5939
6297
|
}
|
|
@@ -5945,7 +6303,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5945
6303
|
`);
|
|
5946
6304
|
}
|
|
5947
6305
|
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and9(
|
|
5948
|
-
|
|
6306
|
+
eq12(auditLog.projectId, project.id),
|
|
5949
6307
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
5950
6308
|
)).orderBy(auditLog.createdAt).all();
|
|
5951
6309
|
const history = replayQueryAuditLog(events);
|
|
@@ -5953,8 +6311,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5953
6311
|
runId: runs.id,
|
|
5954
6312
|
createdAt: runs.createdAt,
|
|
5955
6313
|
location: runs.location
|
|
5956
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
5957
|
-
|
|
6314
|
+
}).from(runs).innerJoin(querySnapshots, eq12(querySnapshots.runId, runs.id)).where(and9(
|
|
6315
|
+
eq12(runs.projectId, project.id),
|
|
5958
6316
|
isNull(querySnapshots.queryId),
|
|
5959
6317
|
isNull(querySnapshots.queryText)
|
|
5960
6318
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -5977,7 +6335,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
5977
6335
|
createdAt: querySnapshots.createdAt,
|
|
5978
6336
|
answerText: querySnapshots.answerText
|
|
5979
6337
|
}).from(querySnapshots).where(and9(
|
|
5980
|
-
|
|
6338
|
+
eq12(querySnapshots.runId, run.runId),
|
|
5981
6339
|
isNull(querySnapshots.queryId),
|
|
5982
6340
|
isNull(querySnapshots.queryText)
|
|
5983
6341
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -6043,7 +6401,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
6043
6401
|
if (!isDryRun && updates.length > 0) {
|
|
6044
6402
|
db.transaction((tx) => {
|
|
6045
6403
|
for (const u of updates) {
|
|
6046
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
6404
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq12(querySnapshots.id, u.id)).run();
|
|
6047
6405
|
}
|
|
6048
6406
|
});
|
|
6049
6407
|
}
|
|
@@ -6117,7 +6475,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6117
6475
|
const projectFilter = opts?.project?.trim();
|
|
6118
6476
|
const isDryRun = opts?.dryRun === true;
|
|
6119
6477
|
const isJson = isMachineFormat(opts?.format);
|
|
6120
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6478
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq12(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6121
6479
|
if (scopedProjects.length === 0) {
|
|
6122
6480
|
if (projectFilter && !isJson) {
|
|
6123
6481
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -6143,7 +6501,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6143
6501
|
byBot: {}
|
|
6144
6502
|
};
|
|
6145
6503
|
const unknownCountRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(and9(
|
|
6146
|
-
|
|
6504
|
+
eq12(rawEventSamples.eventType, "unknown"),
|
|
6147
6505
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6148
6506
|
)).get();
|
|
6149
6507
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -6156,7 +6514,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6156
6514
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
6157
6515
|
status: rawEventSamples.status
|
|
6158
6516
|
}).from(rawEventSamples).where(and9(
|
|
6159
|
-
|
|
6517
|
+
eq12(rawEventSamples.eventType, "unknown"),
|
|
6160
6518
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6161
6519
|
)).all();
|
|
6162
6520
|
result.examined = unknownSamples.length;
|
|
@@ -6195,7 +6553,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6195
6553
|
result.reclassified++;
|
|
6196
6554
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
6197
6555
|
if (isDryRun) continue;
|
|
6198
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
6556
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq12(rawEventSamples.id, snap.id)).run();
|
|
6199
6557
|
const tsHour = new Date(snap.ts);
|
|
6200
6558
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
6201
6559
|
if (userFetch) {
|
|
@@ -6260,7 +6618,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
6260
6618
|
}
|
|
6261
6619
|
if (!isDryRun) {
|
|
6262
6620
|
const afterRow = db.select({ n: sql4`count(*)` }).from(rawEventSamples).where(and9(
|
|
6263
|
-
|
|
6621
|
+
eq12(rawEventSamples.eventType, "unknown"),
|
|
6264
6622
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
6265
6623
|
)).get();
|
|
6266
6624
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -6295,7 +6653,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
6295
6653
|
}
|
|
6296
6654
|
|
|
6297
6655
|
// src/commands/skills.ts
|
|
6298
|
-
import
|
|
6656
|
+
import crypto14 from "crypto";
|
|
6299
6657
|
import fs4 from "fs";
|
|
6300
6658
|
import os4 from "os";
|
|
6301
6659
|
import path5 from "path";
|
|
@@ -6350,7 +6708,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
6350
6708
|
return out.sort();
|
|
6351
6709
|
}
|
|
6352
6710
|
function sha256File(filePath) {
|
|
6353
|
-
return
|
|
6711
|
+
return crypto14.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
6354
6712
|
}
|
|
6355
6713
|
function readSkillManifest(skillDir) {
|
|
6356
6714
|
try {
|
|
@@ -6673,10 +7031,10 @@ var ProviderRegistry = class {
|
|
|
6673
7031
|
};
|
|
6674
7032
|
|
|
6675
7033
|
// src/scheduler.ts
|
|
6676
|
-
import
|
|
7034
|
+
import crypto15 from "crypto";
|
|
6677
7035
|
import cron from "node-cron";
|
|
6678
|
-
import { and as and10, eq as
|
|
6679
|
-
var
|
|
7036
|
+
import { and as and10, eq as eq13, inArray as inArray5 } from "drizzle-orm";
|
|
7037
|
+
var log13 = createLogger("Scheduler");
|
|
6680
7038
|
function taskKey(projectId, kind) {
|
|
6681
7039
|
return `${projectId}::${kind}`;
|
|
6682
7040
|
}
|
|
@@ -6690,16 +7048,16 @@ var Scheduler = class {
|
|
|
6690
7048
|
}
|
|
6691
7049
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
6692
7050
|
start() {
|
|
6693
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
7051
|
+
const allSchedules = this.db.select().from(schedules).where(eq13(schedules.enabled, true)).all();
|
|
6694
7052
|
for (const schedule of allSchedules) {
|
|
6695
7053
|
const missedRunAt = schedule.nextRunAt;
|
|
6696
7054
|
this.registerCronTask(schedule);
|
|
6697
7055
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
6698
|
-
|
|
7056
|
+
log13.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
6699
7057
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
6700
7058
|
}
|
|
6701
7059
|
}
|
|
6702
|
-
|
|
7060
|
+
log13.info("started", { scheduleCount: allSchedules.length });
|
|
6703
7061
|
}
|
|
6704
7062
|
/** Stop all cron tasks for graceful shutdown. */
|
|
6705
7063
|
stop() {
|
|
@@ -6720,7 +7078,7 @@ var Scheduler = class {
|
|
|
6720
7078
|
this.stopTask(key, existing, "Stopped");
|
|
6721
7079
|
this.tasks.delete(key);
|
|
6722
7080
|
}
|
|
6723
|
-
const schedule = this.db.select().from(schedules).where(and10(
|
|
7081
|
+
const schedule = this.db.select().from(schedules).where(and10(eq13(schedules.projectId, projectId), eq13(schedules.kind, kind))).get();
|
|
6724
7082
|
if (schedule && schedule.enabled) {
|
|
6725
7083
|
this.registerCronTask(schedule);
|
|
6726
7084
|
}
|
|
@@ -6743,13 +7101,13 @@ var Scheduler = class {
|
|
|
6743
7101
|
stopTask(key, task, verb) {
|
|
6744
7102
|
void task.stop();
|
|
6745
7103
|
void task.destroy();
|
|
6746
|
-
|
|
7104
|
+
log13.info(`task.${verb.toLowerCase()}`, { key });
|
|
6747
7105
|
}
|
|
6748
7106
|
registerCronTask(schedule) {
|
|
6749
7107
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
6750
7108
|
const kind = schedule.kind;
|
|
6751
7109
|
if (!cron.validate(cronExpr)) {
|
|
6752
|
-
|
|
7110
|
+
log13.error("cron.invalid", { projectId, kind, cronExpr });
|
|
6753
7111
|
return;
|
|
6754
7112
|
}
|
|
6755
7113
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -6761,51 +7119,51 @@ var Scheduler = class {
|
|
|
6761
7119
|
this.db.update(schedules).set({
|
|
6762
7120
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
6763
7121
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6764
|
-
}).where(
|
|
7122
|
+
}).where(eq13(schedules.id, scheduleId)).run();
|
|
6765
7123
|
const label = schedule.preset ?? cronExpr;
|
|
6766
|
-
|
|
7124
|
+
log13.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
6767
7125
|
}
|
|
6768
7126
|
triggerRun(scheduleId, projectId, kind) {
|
|
6769
7127
|
try {
|
|
6770
7128
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6771
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
7129
|
+
const currentSchedule = this.db.select().from(schedules).where(eq13(schedules.id, scheduleId)).get();
|
|
6772
7130
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
6773
|
-
|
|
7131
|
+
log13.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
6774
7132
|
this.remove(projectId, kind);
|
|
6775
7133
|
return;
|
|
6776
7134
|
}
|
|
6777
7135
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
6778
|
-
const project = this.db.select().from(projects).where(
|
|
7136
|
+
const project = this.db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
6779
7137
|
if (!project) {
|
|
6780
|
-
|
|
7138
|
+
log13.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
6781
7139
|
this.remove(projectId, kind);
|
|
6782
7140
|
return;
|
|
6783
7141
|
}
|
|
6784
7142
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
6785
7143
|
const sourceId = currentSchedule.sourceId;
|
|
6786
7144
|
if (!sourceId) {
|
|
6787
|
-
|
|
7145
|
+
log13.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
6788
7146
|
return;
|
|
6789
7147
|
}
|
|
6790
7148
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
6791
|
-
|
|
7149
|
+
log13.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
6792
7150
|
return;
|
|
6793
7151
|
}
|
|
6794
7152
|
this.db.update(schedules).set({
|
|
6795
7153
|
lastRunAt: now,
|
|
6796
7154
|
nextRunAt,
|
|
6797
7155
|
updatedAt: now
|
|
6798
|
-
}).where(
|
|
6799
|
-
|
|
7156
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7157
|
+
log13.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
6800
7158
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
6801
7159
|
return;
|
|
6802
7160
|
}
|
|
6803
7161
|
if (kind === SchedulableRunKinds["gbp-sync"]) {
|
|
6804
7162
|
if (!this.callbacks.onGbpSyncRequested) {
|
|
6805
|
-
|
|
7163
|
+
log13.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
6806
7164
|
return;
|
|
6807
7165
|
}
|
|
6808
|
-
const runId2 =
|
|
7166
|
+
const runId2 = crypto15.randomUUID();
|
|
6809
7167
|
this.db.insert(runs).values({
|
|
6810
7168
|
id: runId2,
|
|
6811
7169
|
projectId,
|
|
@@ -6818,55 +7176,88 @@ var Scheduler = class {
|
|
|
6818
7176
|
lastRunAt: now,
|
|
6819
7177
|
nextRunAt,
|
|
6820
7178
|
updatedAt: now
|
|
6821
|
-
}).where(
|
|
6822
|
-
|
|
7179
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7180
|
+
log13.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
6823
7181
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
6824
7182
|
return;
|
|
6825
7183
|
}
|
|
7184
|
+
if (kind === SchedulableRunKinds["ads-sync"]) {
|
|
7185
|
+
if (!this.callbacks.onAdsSyncRequested) {
|
|
7186
|
+
log13.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
7187
|
+
return;
|
|
7188
|
+
}
|
|
7189
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and10(
|
|
7190
|
+
eq13(runs.projectId, projectId),
|
|
7191
|
+
eq13(runs.kind, RunKinds["ads-sync"]),
|
|
7192
|
+
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
7193
|
+
)).get();
|
|
7194
|
+
if (activeAdsRun) {
|
|
7195
|
+
log13.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
7196
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7197
|
+
return;
|
|
7198
|
+
}
|
|
7199
|
+
const runId2 = crypto15.randomUUID();
|
|
7200
|
+
this.db.insert(runs).values({
|
|
7201
|
+
id: runId2,
|
|
7202
|
+
projectId,
|
|
7203
|
+
kind: RunKinds["ads-sync"],
|
|
7204
|
+
status: RunStatuses.queued,
|
|
7205
|
+
trigger: RunTriggers.scheduled,
|
|
7206
|
+
createdAt: now
|
|
7207
|
+
}).run();
|
|
7208
|
+
this.db.update(schedules).set({
|
|
7209
|
+
lastRunAt: now,
|
|
7210
|
+
nextRunAt,
|
|
7211
|
+
updatedAt: now
|
|
7212
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7213
|
+
log13.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
7214
|
+
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
7215
|
+
return;
|
|
7216
|
+
}
|
|
6826
7217
|
if (kind === SchedulableRunKinds["data-refresh"]) {
|
|
6827
7218
|
if (!this.callbacks.onDataRefreshRequested) {
|
|
6828
|
-
|
|
7219
|
+
log13.warn("data-refresh.no-callback", { scheduleId, projectId, msg: "host did not register onDataRefreshRequested" });
|
|
6829
7220
|
return;
|
|
6830
7221
|
}
|
|
6831
7222
|
this.db.update(schedules).set({
|
|
6832
7223
|
lastRunAt: now,
|
|
6833
7224
|
nextRunAt,
|
|
6834
7225
|
updatedAt: now
|
|
6835
|
-
}).where(
|
|
6836
|
-
|
|
7226
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7227
|
+
log13.info("data-refresh.triggered", { projectName: project.name });
|
|
6837
7228
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
6838
7229
|
return;
|
|
6839
7230
|
}
|
|
6840
7231
|
if (kind === SchedulableRunKinds["backlinks-sync"]) {
|
|
6841
7232
|
if (!this.callbacks.onBacklinksSyncRequested) {
|
|
6842
|
-
|
|
7233
|
+
log13.warn("backlinks-sync.no-callback", { scheduleId, projectId, msg: "host did not register onBacklinksSyncRequested" });
|
|
6843
7234
|
return;
|
|
6844
7235
|
}
|
|
6845
7236
|
this.db.update(schedules).set({
|
|
6846
7237
|
lastRunAt: now,
|
|
6847
7238
|
nextRunAt,
|
|
6848
7239
|
updatedAt: now
|
|
6849
|
-
}).where(
|
|
6850
|
-
|
|
7240
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7241
|
+
log13.info("backlinks-sync.triggered", { projectName: project.name });
|
|
6851
7242
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
6852
7243
|
return;
|
|
6853
7244
|
}
|
|
6854
7245
|
if (kind === SchedulableRunKinds["site-audit"]) {
|
|
6855
7246
|
if (!this.callbacks.onSiteAuditRequested) {
|
|
6856
|
-
|
|
7247
|
+
log13.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
6857
7248
|
return;
|
|
6858
7249
|
}
|
|
6859
7250
|
const active = this.db.select({ id: runs.id }).from(runs).where(and10(
|
|
6860
|
-
|
|
6861
|
-
|
|
7251
|
+
eq13(runs.projectId, projectId),
|
|
7252
|
+
eq13(runs.kind, RunKinds["site-audit"]),
|
|
6862
7253
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
6863
7254
|
)).get();
|
|
6864
7255
|
if (active) {
|
|
6865
|
-
|
|
6866
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
7256
|
+
log13.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
7257
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
6867
7258
|
return;
|
|
6868
7259
|
}
|
|
6869
|
-
const runId2 =
|
|
7260
|
+
const runId2 = crypto15.randomUUID();
|
|
6870
7261
|
this.db.insert(runs).values({
|
|
6871
7262
|
id: runId2,
|
|
6872
7263
|
projectId,
|
|
@@ -6879,8 +7270,8 @@ var Scheduler = class {
|
|
|
6879
7270
|
lastRunAt: now,
|
|
6880
7271
|
nextRunAt,
|
|
6881
7272
|
updatedAt: now
|
|
6882
|
-
}).where(
|
|
6883
|
-
|
|
7273
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
7274
|
+
log13.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
6884
7275
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
6885
7276
|
return;
|
|
6886
7277
|
}
|
|
@@ -6889,7 +7280,7 @@ var Scheduler = class {
|
|
|
6889
7280
|
if (project.defaultLocation) {
|
|
6890
7281
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
6891
7282
|
if (!loc) {
|
|
6892
|
-
|
|
7283
|
+
log13.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
6893
7284
|
return;
|
|
6894
7285
|
}
|
|
6895
7286
|
resolvedLocation = loc;
|
|
@@ -6903,11 +7294,11 @@ var Scheduler = class {
|
|
|
6903
7294
|
location: locationLabel
|
|
6904
7295
|
});
|
|
6905
7296
|
if (queueResult.conflict) {
|
|
6906
|
-
|
|
7297
|
+
log13.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
6907
7298
|
this.db.update(schedules).set({
|
|
6908
7299
|
nextRunAt,
|
|
6909
7300
|
updatedAt: now
|
|
6910
|
-
}).where(
|
|
7301
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
6911
7302
|
return;
|
|
6912
7303
|
}
|
|
6913
7304
|
const runId = queueResult.runId;
|
|
@@ -6915,43 +7306,44 @@ var Scheduler = class {
|
|
|
6915
7306
|
lastRunAt: now,
|
|
6916
7307
|
nextRunAt,
|
|
6917
7308
|
updatedAt: now
|
|
6918
|
-
}).where(
|
|
7309
|
+
}).where(eq13(schedules.id, currentSchedule.id)).run();
|
|
6919
7310
|
const scheduleProviders = currentSchedule.providers;
|
|
6920
7311
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
6921
|
-
|
|
7312
|
+
log13.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
6922
7313
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
6923
7314
|
} catch (err) {
|
|
6924
|
-
|
|
7315
|
+
log13.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
6925
7316
|
}
|
|
6926
7317
|
}
|
|
6927
7318
|
};
|
|
6928
7319
|
|
|
6929
7320
|
// src/data-refresh.ts
|
|
6930
|
-
var
|
|
7321
|
+
var log14 = createLogger("DataRefresh");
|
|
6931
7322
|
async function refreshAllIntegrations(client, projectName) {
|
|
6932
7323
|
const integrations = [
|
|
6933
7324
|
{ name: "gsc", run: () => client.gscSync(projectName, {}) },
|
|
6934
7325
|
{ name: "bing", run: () => client.bingInspectSitemap(projectName, {}) },
|
|
6935
7326
|
{ name: "ga", run: () => client.gaSync(projectName, { days: 30 }) },
|
|
6936
|
-
{ name: "gbp", run: () => client.triggerGbpSync(projectName, {}) }
|
|
7327
|
+
{ name: "gbp", run: () => client.triggerGbpSync(projectName, {}) },
|
|
7328
|
+
{ name: "ads", run: () => client.triggerAdsSync(projectName) }
|
|
6937
7329
|
];
|
|
6938
7330
|
const results = await Promise.allSettled(integrations.map((i) => i.run()));
|
|
6939
7331
|
results.forEach((result, idx) => {
|
|
6940
7332
|
const integration = integrations[idx].name;
|
|
6941
7333
|
if (result.status === "fulfilled") {
|
|
6942
|
-
|
|
7334
|
+
log14.info("integration.refreshed", { projectName, integration });
|
|
6943
7335
|
} else {
|
|
6944
7336
|
const reason = result.reason;
|
|
6945
7337
|
const message = reason instanceof Error ? reason.message : String(reason);
|
|
6946
|
-
|
|
7338
|
+
log14.warn("integration.refresh-failed", { projectName, integration, error: message });
|
|
6947
7339
|
}
|
|
6948
7340
|
});
|
|
6949
7341
|
}
|
|
6950
7342
|
|
|
6951
7343
|
// src/notifier.ts
|
|
6952
|
-
import { eq as
|
|
6953
|
-
import
|
|
6954
|
-
var
|
|
7344
|
+
import { eq as eq14, desc as desc5, and as and11, inArray as inArray6, or } from "drizzle-orm";
|
|
7345
|
+
import crypto16 from "crypto";
|
|
7346
|
+
var log15 = createLogger("Notifier");
|
|
6955
7347
|
var Notifier = class {
|
|
6956
7348
|
db;
|
|
6957
7349
|
serverUrl;
|
|
@@ -6961,26 +7353,26 @@ var Notifier = class {
|
|
|
6961
7353
|
}
|
|
6962
7354
|
/** Called after a run completes (success, partial, or failed). */
|
|
6963
7355
|
async onRunCompleted(runId, projectId) {
|
|
6964
|
-
|
|
6965
|
-
const notifs = this.db.select().from(notifications).where(
|
|
7356
|
+
log15.info("run.completed", { runId, projectId });
|
|
7357
|
+
const notifs = this.db.select().from(notifications).where(eq14(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
6966
7358
|
if (notifs.length === 0) {
|
|
6967
|
-
|
|
7359
|
+
log15.info("notifications.none-enabled", { projectId });
|
|
6968
7360
|
return;
|
|
6969
7361
|
}
|
|
6970
|
-
|
|
6971
|
-
const run = this.db.select().from(runs).where(
|
|
7362
|
+
log15.info("notifications.found", { projectId, count: notifs.length });
|
|
7363
|
+
const run = this.db.select().from(runs).where(eq14(runs.id, runId)).get();
|
|
6972
7364
|
if (!run) {
|
|
6973
|
-
|
|
7365
|
+
log15.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
6974
7366
|
return;
|
|
6975
7367
|
}
|
|
6976
|
-
const project = this.db.select().from(projects).where(
|
|
7368
|
+
const project = this.db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
6977
7369
|
if (!project) {
|
|
6978
|
-
|
|
7370
|
+
log15.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
6979
7371
|
return;
|
|
6980
7372
|
}
|
|
6981
7373
|
const transitions = this.computeTransitions(runId, projectId);
|
|
6982
7374
|
const events = [];
|
|
6983
|
-
|
|
7375
|
+
log15.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
6984
7376
|
if (run.status === "completed" || run.status === "partial") {
|
|
6985
7377
|
events.push("run.completed");
|
|
6986
7378
|
}
|
|
@@ -6996,7 +7388,7 @@ var Notifier = class {
|
|
|
6996
7388
|
if (!config.url) continue;
|
|
6997
7389
|
const subscribedEvents = config.events;
|
|
6998
7390
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
6999
|
-
|
|
7391
|
+
log15.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
7000
7392
|
if (matchingEvents.length === 0) continue;
|
|
7001
7393
|
for (const event of matchingEvents) {
|
|
7002
7394
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -7020,11 +7412,11 @@ var Notifier = class {
|
|
|
7020
7412
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
7021
7413
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
7022
7414
|
if (insightEvents.length === 0) return;
|
|
7023
|
-
const notifs = this.db.select().from(notifications).where(
|
|
7415
|
+
const notifs = this.db.select().from(notifications).where(eq14(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
7024
7416
|
if (notifs.length === 0) return;
|
|
7025
|
-
const run = this.db.select().from(runs).where(
|
|
7417
|
+
const run = this.db.select().from(runs).where(eq14(runs.id, runId)).get();
|
|
7026
7418
|
if (!run) return;
|
|
7027
|
-
const project = this.db.select().from(projects).where(
|
|
7419
|
+
const project = this.db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
7028
7420
|
if (!project) return;
|
|
7029
7421
|
for (const notif of notifs) {
|
|
7030
7422
|
const config = notif.config;
|
|
@@ -7054,12 +7446,12 @@ var Notifier = class {
|
|
|
7054
7446
|
}
|
|
7055
7447
|
}
|
|
7056
7448
|
computeTransitions(runId, projectId) {
|
|
7057
|
-
const thisRun = this.db.select().from(runs).where(
|
|
7449
|
+
const thisRun = this.db.select().from(runs).where(eq14(runs.id, runId)).get();
|
|
7058
7450
|
if (!thisRun) return [];
|
|
7059
7451
|
const groupSiblings = this.db.select().from(runs).where(and11(
|
|
7060
|
-
|
|
7061
|
-
|
|
7062
|
-
|
|
7452
|
+
eq14(runs.projectId, projectId),
|
|
7453
|
+
eq14(runs.kind, thisRun.kind),
|
|
7454
|
+
eq14(runs.createdAt, thisRun.createdAt)
|
|
7063
7455
|
)).all();
|
|
7064
7456
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
7065
7457
|
if (stillPending) return [];
|
|
@@ -7075,7 +7467,7 @@ var Notifier = class {
|
|
|
7075
7467
|
return candidate.id > best.id ? candidate : best;
|
|
7076
7468
|
});
|
|
7077
7469
|
if (winner.id !== runId) return [];
|
|
7078
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
7470
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq14(projects.id, projectId)).get();
|
|
7079
7471
|
const locationCount = Math.max(
|
|
7080
7472
|
1,
|
|
7081
7473
|
(projectLocations?.locations ?? []).length
|
|
@@ -7083,9 +7475,9 @@ var Notifier = class {
|
|
|
7083
7475
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
7084
7476
|
const recentRuns = this.db.select().from(runs).where(
|
|
7085
7477
|
and11(
|
|
7086
|
-
|
|
7087
|
-
|
|
7088
|
-
or(
|
|
7478
|
+
eq14(runs.projectId, projectId),
|
|
7479
|
+
eq14(runs.kind, thisRun.kind),
|
|
7480
|
+
or(eq14(runs.status, "completed"), eq14(runs.status, "partial"))
|
|
7089
7481
|
)
|
|
7090
7482
|
).orderBy(desc5(runs.createdAt), desc5(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
7091
7483
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -7102,7 +7494,7 @@ var Notifier = class {
|
|
|
7102
7494
|
provider: querySnapshots.provider,
|
|
7103
7495
|
location: querySnapshots.location,
|
|
7104
7496
|
citationState: querySnapshots.citationState
|
|
7105
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
7497
|
+
}).from(querySnapshots).leftJoin(queries, eq14(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
7106
7498
|
const previousSnapshots = this.db.select({
|
|
7107
7499
|
queryId: querySnapshots.queryId,
|
|
7108
7500
|
provider: querySnapshots.provider,
|
|
@@ -7135,23 +7527,23 @@ var Notifier = class {
|
|
|
7135
7527
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
7136
7528
|
const targetCheck = await resolveWebhookTarget(url);
|
|
7137
7529
|
if (!targetCheck.ok) {
|
|
7138
|
-
|
|
7530
|
+
log15.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
7139
7531
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
7140
7532
|
return;
|
|
7141
7533
|
}
|
|
7142
|
-
|
|
7534
|
+
log15.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
7143
7535
|
const maxRetries = 3;
|
|
7144
7536
|
const delays = [1e3, 4e3, 16e3];
|
|
7145
7537
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
7146
7538
|
try {
|
|
7147
7539
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
7148
7540
|
if (response.status >= 200 && response.status < 300) {
|
|
7149
|
-
|
|
7541
|
+
log15.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
7150
7542
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
7151
7543
|
return;
|
|
7152
7544
|
}
|
|
7153
7545
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
7154
|
-
|
|
7546
|
+
log15.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
7155
7547
|
if (attempt === maxRetries - 1) {
|
|
7156
7548
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
7157
7549
|
}
|
|
@@ -7159,7 +7551,7 @@ var Notifier = class {
|
|
|
7159
7551
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
7160
7552
|
if (attempt === maxRetries - 1) {
|
|
7161
7553
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
7162
|
-
|
|
7554
|
+
log15.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
7163
7555
|
}
|
|
7164
7556
|
}
|
|
7165
7557
|
if (attempt < maxRetries - 1) {
|
|
@@ -7169,7 +7561,7 @@ var Notifier = class {
|
|
|
7169
7561
|
}
|
|
7170
7562
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
7171
7563
|
this.db.insert(auditLog).values({
|
|
7172
|
-
id:
|
|
7564
|
+
id: crypto16.randomUUID(),
|
|
7173
7565
|
projectId,
|
|
7174
7566
|
actor: "scheduler",
|
|
7175
7567
|
action: `notification.${status}`,
|
|
@@ -7182,8 +7574,8 @@ var Notifier = class {
|
|
|
7182
7574
|
};
|
|
7183
7575
|
|
|
7184
7576
|
// src/run-coordinator.ts
|
|
7185
|
-
import { eq as
|
|
7186
|
-
var
|
|
7577
|
+
import { eq as eq15 } from "drizzle-orm";
|
|
7578
|
+
var log16 = createLogger("RunCoordinator");
|
|
7187
7579
|
var RunCoordinator = class {
|
|
7188
7580
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
7189
7581
|
this.db = db;
|
|
@@ -7192,11 +7584,16 @@ var RunCoordinator = class {
|
|
|
7192
7584
|
this.onInsightsGenerated = onInsightsGenerated;
|
|
7193
7585
|
this.onAeroEvent = onAeroEvent;
|
|
7194
7586
|
}
|
|
7587
|
+
db;
|
|
7588
|
+
notifier;
|
|
7589
|
+
intelligenceService;
|
|
7590
|
+
onInsightsGenerated;
|
|
7591
|
+
onAeroEvent;
|
|
7195
7592
|
async onRunCompleted(runId, projectId) {
|
|
7196
|
-
const runRow = this.db.select().from(runs).where(
|
|
7593
|
+
const runRow = this.db.select().from(runs).where(eq15(runs.id, runId)).get();
|
|
7197
7594
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
7198
7595
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
7199
|
-
|
|
7596
|
+
log16.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
7200
7597
|
return;
|
|
7201
7598
|
}
|
|
7202
7599
|
let insightCount = 0;
|
|
@@ -7213,12 +7610,12 @@ var RunCoordinator = class {
|
|
|
7213
7610
|
try {
|
|
7214
7611
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
7215
7612
|
} catch (err) {
|
|
7216
|
-
|
|
7613
|
+
log16.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7217
7614
|
}
|
|
7218
7615
|
}
|
|
7219
7616
|
}
|
|
7220
7617
|
} catch (err) {
|
|
7221
|
-
|
|
7618
|
+
log16.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7222
7619
|
}
|
|
7223
7620
|
} else if (kind === RunKinds["gbp-sync"]) {
|
|
7224
7621
|
try {
|
|
@@ -7231,17 +7628,17 @@ var RunCoordinator = class {
|
|
|
7231
7628
|
try {
|
|
7232
7629
|
await this.onInsightsGenerated(runId, projectId, analysisResultFromInsights(gbpInsights));
|
|
7233
7630
|
} catch (err) {
|
|
7234
|
-
|
|
7631
|
+
log16.error("gbp-insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7235
7632
|
}
|
|
7236
7633
|
}
|
|
7237
7634
|
} catch (err) {
|
|
7238
|
-
|
|
7635
|
+
log16.error("gbp-intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7239
7636
|
}
|
|
7240
7637
|
}
|
|
7241
7638
|
try {
|
|
7242
7639
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
7243
7640
|
} catch (err) {
|
|
7244
|
-
|
|
7641
|
+
log16.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7245
7642
|
}
|
|
7246
7643
|
if (this.onAeroEvent) {
|
|
7247
7644
|
try {
|
|
@@ -7254,7 +7651,7 @@ var RunCoordinator = class {
|
|
|
7254
7651
|
};
|
|
7255
7652
|
await this.onAeroEvent(ctx);
|
|
7256
7653
|
} catch (err) {
|
|
7257
|
-
|
|
7654
|
+
log16.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
7258
7655
|
}
|
|
7259
7656
|
}
|
|
7260
7657
|
}
|
|
@@ -7269,7 +7666,7 @@ var RunCoordinator = class {
|
|
|
7269
7666
|
* so the Aero queue is never starved of a follow-up.
|
|
7270
7667
|
*/
|
|
7271
7668
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
7272
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
7669
|
+
const session = this.db.select().from(discoverySessions).where(eq15(discoverySessions.runId, runId)).get();
|
|
7273
7670
|
const competitorMap = session ? session.competitorMap : [];
|
|
7274
7671
|
return {
|
|
7275
7672
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -7309,8 +7706,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
7309
7706
|
}
|
|
7310
7707
|
|
|
7311
7708
|
// src/agent/session-registry.ts
|
|
7312
|
-
import
|
|
7313
|
-
import { eq as
|
|
7709
|
+
import crypto18 from "crypto";
|
|
7710
|
+
import { eq as eq17 } from "drizzle-orm";
|
|
7314
7711
|
|
|
7315
7712
|
// src/agent/session.ts
|
|
7316
7713
|
import fs7 from "fs";
|
|
@@ -7698,8 +8095,8 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
7698
8095
|
}
|
|
7699
8096
|
|
|
7700
8097
|
// src/agent/memory-store.ts
|
|
7701
|
-
import
|
|
7702
|
-
import { and as and12, desc as desc6, eq as
|
|
8098
|
+
import crypto17 from "crypto";
|
|
8099
|
+
import { and as and12, desc as desc6, eq as eq16, like, sql as sql5 } from "drizzle-orm";
|
|
7703
8100
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
7704
8101
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
7705
8102
|
function rowToDto(row) {
|
|
@@ -7713,7 +8110,7 @@ function rowToDto(row) {
|
|
|
7713
8110
|
};
|
|
7714
8111
|
}
|
|
7715
8112
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
7716
|
-
const query = db.select().from(agentMemory).where(
|
|
8113
|
+
const query = db.select().from(agentMemory).where(eq16(agentMemory.projectId, projectId)).orderBy(desc6(agentMemory.updatedAt));
|
|
7717
8114
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
7718
8115
|
return rows.map(rowToDto);
|
|
7719
8116
|
}
|
|
@@ -7727,7 +8124,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
7727
8124
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
7728
8125
|
}
|
|
7729
8126
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7730
|
-
const id =
|
|
8127
|
+
const id = crypto17.randomUUID();
|
|
7731
8128
|
db.insert(agentMemory).values({
|
|
7732
8129
|
id,
|
|
7733
8130
|
projectId: args.projectId,
|
|
@@ -7744,12 +8141,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
7744
8141
|
updatedAt: now
|
|
7745
8142
|
}
|
|
7746
8143
|
}).run();
|
|
7747
|
-
const row = db.select().from(agentMemory).where(and12(
|
|
8144
|
+
const row = db.select().from(agentMemory).where(and12(eq16(agentMemory.projectId, args.projectId), eq16(agentMemory.key, args.key))).get();
|
|
7748
8145
|
if (!row) throw new Error("memory upsert produced no row");
|
|
7749
8146
|
return rowToDto(row);
|
|
7750
8147
|
}
|
|
7751
8148
|
function deleteMemoryEntry(db, projectId, key) {
|
|
7752
|
-
const result = db.delete(agentMemory).where(and12(
|
|
8149
|
+
const result = db.delete(agentMemory).where(and12(eq16(agentMemory.projectId, projectId), eq16(agentMemory.key, key))).run();
|
|
7753
8150
|
const changes = result.changes ?? 0;
|
|
7754
8151
|
return changes > 0;
|
|
7755
8152
|
}
|
|
@@ -7764,7 +8161,7 @@ function writeCompactionNote(db, args) {
|
|
|
7764
8161
|
}
|
|
7765
8162
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7766
8163
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
7767
|
-
const id =
|
|
8164
|
+
const id = crypto17.randomUUID();
|
|
7768
8165
|
let inserted;
|
|
7769
8166
|
db.transaction((tx) => {
|
|
7770
8167
|
tx.insert(agentMemory).values({
|
|
@@ -7779,7 +8176,7 @@ function writeCompactionNote(db, args) {
|
|
|
7779
8176
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
7780
8177
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
7781
8178
|
and12(
|
|
7782
|
-
|
|
8179
|
+
eq16(agentMemory.projectId, args.projectId),
|
|
7783
8180
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
7784
8181
|
)
|
|
7785
8182
|
).orderBy(desc6(agentMemory.updatedAt)).all();
|
|
@@ -7787,7 +8184,7 @@ function writeCompactionNote(db, args) {
|
|
|
7787
8184
|
if (stale.length > 0) {
|
|
7788
8185
|
tx.delete(agentMemory).where(sql5`${agentMemory.id} IN (${sql5.join(stale.map((s) => sql5`${s}`), sql5`, `)})`).run();
|
|
7789
8186
|
}
|
|
7790
|
-
const row = tx.select().from(agentMemory).where(and12(
|
|
8187
|
+
const row = tx.select().from(agentMemory).where(and12(eq16(agentMemory.projectId, args.projectId), eq16(agentMemory.key, key))).get();
|
|
7791
8188
|
if (row) inserted = rowToDto(row);
|
|
7792
8189
|
});
|
|
7793
8190
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -7920,7 +8317,7 @@ async function compactMessages(args) {
|
|
|
7920
8317
|
}
|
|
7921
8318
|
|
|
7922
8319
|
// src/agent/session-registry.ts
|
|
7923
|
-
var
|
|
8320
|
+
var log17 = createLogger("SessionRegistry");
|
|
7924
8321
|
var MAX_HYDRATE_NOTES = 20;
|
|
7925
8322
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
7926
8323
|
function escapeMemoryFragment(value) {
|
|
@@ -7969,7 +8366,7 @@ var SessionRegistry = class {
|
|
|
7969
8366
|
modelProvider: effectiveProvider,
|
|
7970
8367
|
modelId: effectiveModelId,
|
|
7971
8368
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
7972
|
-
}).where(
|
|
8369
|
+
}).where(eq17(agentSessions.projectId, projectId)).run();
|
|
7973
8370
|
}
|
|
7974
8371
|
const agent2 = createAeroSession({
|
|
7975
8372
|
projectName,
|
|
@@ -8147,13 +8544,13 @@ ${lines.join("\n")}
|
|
|
8147
8544
|
agent.state.messages = result.messages;
|
|
8148
8545
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
8149
8546
|
this.save(projectName);
|
|
8150
|
-
|
|
8547
|
+
log17.info("compaction.completed", {
|
|
8151
8548
|
projectName,
|
|
8152
8549
|
removedCount: result.removedCount,
|
|
8153
8550
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
8154
8551
|
});
|
|
8155
8552
|
} catch (err) {
|
|
8156
|
-
|
|
8553
|
+
log17.error("compaction.failed", {
|
|
8157
8554
|
projectName,
|
|
8158
8555
|
error: err instanceof Error ? err.message : String(err)
|
|
8159
8556
|
});
|
|
@@ -8183,7 +8580,7 @@ ${lines.join("\n")}
|
|
|
8183
8580
|
modelProvider: nextProvider,
|
|
8184
8581
|
modelId: nextModelId,
|
|
8185
8582
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8186
|
-
}).where(
|
|
8583
|
+
}).where(eq17(agentSessions.projectId, projectId)).run();
|
|
8187
8584
|
}
|
|
8188
8585
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
8189
8586
|
save(projectName) {
|
|
@@ -8250,7 +8647,7 @@ ${lines.join("\n")}
|
|
|
8250
8647
|
await agent.prompt(msgs);
|
|
8251
8648
|
this.save(projectName);
|
|
8252
8649
|
} catch (err) {
|
|
8253
|
-
|
|
8650
|
+
log17.error("drain.failed", {
|
|
8254
8651
|
projectName,
|
|
8255
8652
|
error: err instanceof Error ? err.message : String(err)
|
|
8256
8653
|
});
|
|
@@ -8345,17 +8742,17 @@ ${lines.join("\n")}
|
|
|
8345
8742
|
return id;
|
|
8346
8743
|
}
|
|
8347
8744
|
tryResolveProjectId(projectName) {
|
|
8348
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
8745
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq17(projects.name, projectName)).get();
|
|
8349
8746
|
return row?.id;
|
|
8350
8747
|
}
|
|
8351
8748
|
loadRow(projectId) {
|
|
8352
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
8749
|
+
const row = this.opts.db.select().from(agentSessions).where(eq17(agentSessions.projectId, projectId)).get();
|
|
8353
8750
|
return row ?? null;
|
|
8354
8751
|
}
|
|
8355
8752
|
insertRow(params) {
|
|
8356
8753
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8357
8754
|
this.opts.db.insert(agentSessions).values({
|
|
8358
|
-
id:
|
|
8755
|
+
id: crypto18.randomUUID(),
|
|
8359
8756
|
projectId: params.projectId,
|
|
8360
8757
|
systemPrompt: params.systemPrompt,
|
|
8361
8758
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -8368,14 +8765,14 @@ ${lines.join("\n")}
|
|
|
8368
8765
|
}
|
|
8369
8766
|
updateRow(projectId, patch) {
|
|
8370
8767
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8371
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
8768
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq17(agentSessions.projectId, projectId)).run();
|
|
8372
8769
|
}
|
|
8373
8770
|
};
|
|
8374
8771
|
|
|
8375
8772
|
// src/agent/agent-routes.ts
|
|
8376
|
-
import { eq as
|
|
8773
|
+
import { eq as eq18 } from "drizzle-orm";
|
|
8377
8774
|
function resolveProject(db, name) {
|
|
8378
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
8775
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq18(projects.name, name)).get();
|
|
8379
8776
|
if (!row) throw notFound("project", name);
|
|
8380
8777
|
return row;
|
|
8381
8778
|
}
|
|
@@ -8384,7 +8781,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
8384
8781
|
"/projects/:name/agent/transcript",
|
|
8385
8782
|
async (request) => {
|
|
8386
8783
|
const project = resolveProject(opts.db, request.params.name);
|
|
8387
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
8784
|
+
const row = opts.db.select().from(agentSessions).where(eq18(agentSessions.projectId, project.id)).get();
|
|
8388
8785
|
if (!row) {
|
|
8389
8786
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
8390
8787
|
}
|
|
@@ -8408,7 +8805,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
8408
8805
|
async (request) => {
|
|
8409
8806
|
const project = resolveProject(opts.db, request.params.name);
|
|
8410
8807
|
opts.sessionRegistry.reset(project.name);
|
|
8411
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
8808
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(agentSessions.projectId, project.id)).run();
|
|
8412
8809
|
return { status: "reset" };
|
|
8413
8810
|
}
|
|
8414
8811
|
);
|
|
@@ -8848,7 +9245,7 @@ function formatAuditFactorScore(factor) {
|
|
|
8848
9245
|
}
|
|
8849
9246
|
|
|
8850
9247
|
// src/snapshot-service.ts
|
|
8851
|
-
var
|
|
9248
|
+
var log18 = createLogger("Snapshot");
|
|
8852
9249
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
8853
9250
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
8854
9251
|
var ProviderExecutionGate2 = class {
|
|
@@ -8856,6 +9253,8 @@ var ProviderExecutionGate2 = class {
|
|
|
8856
9253
|
this.maxConcurrency = maxConcurrency;
|
|
8857
9254
|
this.maxPerMinute = maxPerMinute;
|
|
8858
9255
|
}
|
|
9256
|
+
maxConcurrency;
|
|
9257
|
+
maxPerMinute;
|
|
8859
9258
|
window = [];
|
|
8860
9259
|
waiters = [];
|
|
8861
9260
|
rateLimitChain = Promise.resolve();
|
|
@@ -8917,6 +9316,7 @@ var SnapshotService = class {
|
|
|
8917
9316
|
constructor(registry) {
|
|
8918
9317
|
this.registry = registry;
|
|
8919
9318
|
}
|
|
9319
|
+
registry;
|
|
8920
9320
|
async createReport(input) {
|
|
8921
9321
|
const companyName = input.companyName.trim();
|
|
8922
9322
|
const domain = normalizeDomain(input.domain);
|
|
@@ -8991,7 +9391,7 @@ var SnapshotService = class {
|
|
|
8991
9391
|
return mapAuditReport(report);
|
|
8992
9392
|
} catch (err) {
|
|
8993
9393
|
const message = err instanceof Error ? err.message : String(err);
|
|
8994
|
-
|
|
9394
|
+
log18.warn("audit.failed", { homepageUrl, error: message });
|
|
8995
9395
|
return {
|
|
8996
9396
|
url: homepageUrl,
|
|
8997
9397
|
finalUrl: homepageUrl,
|
|
@@ -9020,7 +9420,7 @@ var SnapshotService = class {
|
|
|
9020
9420
|
queries: parsedQueries
|
|
9021
9421
|
};
|
|
9022
9422
|
} catch (err) {
|
|
9023
|
-
|
|
9423
|
+
log18.warn("profile.generation-failed", {
|
|
9024
9424
|
domain: ctx.domain,
|
|
9025
9425
|
provider: ctx.analysisProvider.adapter.name,
|
|
9026
9426
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -9162,7 +9562,7 @@ var SnapshotService = class {
|
|
|
9162
9562
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
9163
9563
|
};
|
|
9164
9564
|
} catch (err) {
|
|
9165
|
-
|
|
9565
|
+
log18.warn("response.analysis-failed", {
|
|
9166
9566
|
provider: ctx.analysisProvider.adapter.name,
|
|
9167
9567
|
error: err instanceof Error ? err.message : String(err)
|
|
9168
9568
|
});
|
|
@@ -9444,7 +9844,7 @@ function clipText(value, length) {
|
|
|
9444
9844
|
// src/server.ts
|
|
9445
9845
|
var _require3 = createRequire3(import.meta.url);
|
|
9446
9846
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
9447
|
-
var
|
|
9847
|
+
var log19 = createLogger("Server");
|
|
9448
9848
|
var DEFAULT_QUOTA = {
|
|
9449
9849
|
maxConcurrency: 2,
|
|
9450
9850
|
maxRequestsPerMinute: 10,
|
|
@@ -9479,14 +9879,14 @@ function summarizeProviderConfig(config) {
|
|
|
9479
9879
|
};
|
|
9480
9880
|
}
|
|
9481
9881
|
function hashApiKey(key) {
|
|
9482
|
-
return
|
|
9882
|
+
return crypto19.createHash("sha256").update(key).digest("hex");
|
|
9483
9883
|
}
|
|
9484
9884
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
9485
9885
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
9486
9886
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
9487
9887
|
function hashDashboardPassword(password) {
|
|
9488
|
-
const salt =
|
|
9489
|
-
const derived =
|
|
9888
|
+
const salt = crypto19.randomBytes(16);
|
|
9889
|
+
const derived = crypto19.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
9490
9890
|
N: DASHBOARD_SCRYPT_COST,
|
|
9491
9891
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
9492
9892
|
});
|
|
@@ -9507,18 +9907,18 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
9507
9907
|
} catch {
|
|
9508
9908
|
return { ok: false, needsRehash: false };
|
|
9509
9909
|
}
|
|
9510
|
-
const derived =
|
|
9910
|
+
const derived = crypto19.scryptSync(password, salt, expected.length, {
|
|
9511
9911
|
N: DASHBOARD_SCRYPT_COST,
|
|
9512
9912
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
9513
9913
|
});
|
|
9514
9914
|
if (derived.length !== expected.length) return { ok: false, needsRehash: false };
|
|
9515
|
-
return { ok:
|
|
9915
|
+
return { ok: crypto19.timingSafeEqual(derived, expected), needsRehash: false };
|
|
9516
9916
|
}
|
|
9517
9917
|
if (/^[a-f0-9]{64}$/i.test(storedHash)) {
|
|
9518
9918
|
const candidate = Buffer.from(hashApiKey(password), "hex");
|
|
9519
9919
|
const expected = Buffer.from(storedHash, "hex");
|
|
9520
9920
|
if (candidate.length !== expected.length) return { ok: false, needsRehash: false };
|
|
9521
|
-
const ok =
|
|
9921
|
+
const ok = crypto19.timingSafeEqual(candidate, expected);
|
|
9522
9922
|
return { ok, needsRehash: ok };
|
|
9523
9923
|
}
|
|
9524
9924
|
return { ok: false, needsRehash: false };
|
|
@@ -9577,7 +9977,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
9577
9977
|
}
|
|
9578
9978
|
if (migratedGoogle > 0) {
|
|
9579
9979
|
saveConfigPatch({ google: config.google });
|
|
9580
|
-
|
|
9980
|
+
log19.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
9581
9981
|
}
|
|
9582
9982
|
let migratedGa4 = 0;
|
|
9583
9983
|
for (const row of rows.ga4) {
|
|
@@ -9595,7 +9995,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
9595
9995
|
}
|
|
9596
9996
|
if (migratedGa4 > 0) {
|
|
9597
9997
|
saveConfigPatch({ ga4: config.ga4 });
|
|
9598
|
-
|
|
9998
|
+
log19.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
9599
9999
|
}
|
|
9600
10000
|
}
|
|
9601
10001
|
function isLoopbackBindHost(host) {
|
|
@@ -9634,11 +10034,11 @@ async function createServer(opts) {
|
|
|
9634
10034
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
9635
10035
|
dropLegacyCredentialColumns(opts.db);
|
|
9636
10036
|
} catch (err) {
|
|
9637
|
-
|
|
10037
|
+
log19.warn("credentials.migration.failed", {
|
|
9638
10038
|
error: err instanceof Error ? err.message : String(err)
|
|
9639
10039
|
});
|
|
9640
10040
|
}
|
|
9641
|
-
|
|
10041
|
+
log19.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
9642
10042
|
const p = providers[k];
|
|
9643
10043
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
9644
10044
|
}) });
|
|
@@ -9687,7 +10087,7 @@ async function createServer(opts) {
|
|
|
9687
10087
|
intelligenceService,
|
|
9688
10088
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
9689
10089
|
async (ctx) => {
|
|
9690
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
10090
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq19(projects.id, ctx.projectId)).get();
|
|
9691
10091
|
if (!project) return;
|
|
9692
10092
|
let content;
|
|
9693
10093
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -9730,11 +10130,41 @@ async function createServer(opts) {
|
|
|
9730
10130
|
app.log.error({ runId, err }, "GBP sync failed");
|
|
9731
10131
|
});
|
|
9732
10132
|
};
|
|
10133
|
+
const runAdsSync = (runId, projectId) => {
|
|
10134
|
+
executeAdsSync(opts.db, runId, projectId, { config: opts.config }).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
10135
|
+
app.log.error({ runId, err }, "Ads sync failed");
|
|
10136
|
+
});
|
|
10137
|
+
};
|
|
9733
10138
|
const runSiteAudit = (runId, projectId, auditOpts) => {
|
|
9734
10139
|
executeSiteAudit(opts.db, runId, projectId, auditOpts ?? {}).then(() => runCoordinator.onRunCompleted(runId, projectId)).catch((err) => {
|
|
9735
10140
|
app.log.error({ runId, err }, "Site audit failed");
|
|
9736
10141
|
});
|
|
9737
10142
|
};
|
|
10143
|
+
const adsCredentialStore = {
|
|
10144
|
+
getConnection: (projectName) => {
|
|
10145
|
+
return getOpenAiAdsConnection(opts.config, projectName);
|
|
10146
|
+
},
|
|
10147
|
+
upsertConnection: (connection) => {
|
|
10148
|
+
const updated = upsertOpenAiAdsConnection(opts.config, connection);
|
|
10149
|
+
saveConfigPatch(opts.config);
|
|
10150
|
+
return updated;
|
|
10151
|
+
},
|
|
10152
|
+
removeConnection: (projectName) => {
|
|
10153
|
+
const removed = removeOpenAiAdsConnection(opts.config, projectName);
|
|
10154
|
+
if (removed) saveConfigPatch(opts.config);
|
|
10155
|
+
return removed;
|
|
10156
|
+
}
|
|
10157
|
+
};
|
|
10158
|
+
const verifyAdsAccount = async (apiKey) => {
|
|
10159
|
+
const account = await getAdAccount(apiKey);
|
|
10160
|
+
return {
|
|
10161
|
+
id: account.id,
|
|
10162
|
+
name: account.name,
|
|
10163
|
+
status: account.status,
|
|
10164
|
+
currencyCode: account.currency_code ?? null,
|
|
10165
|
+
timezone: account.timezone ?? null
|
|
10166
|
+
};
|
|
10167
|
+
};
|
|
9738
10168
|
const scheduler = new Scheduler(opts.db, {
|
|
9739
10169
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
9740
10170
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
@@ -9749,6 +10179,9 @@ async function createServer(opts) {
|
|
|
9749
10179
|
onGbpSyncRequested: (runId, projectId) => {
|
|
9750
10180
|
runGbpSync(runId, projectId);
|
|
9751
10181
|
},
|
|
10182
|
+
onAdsSyncRequested: (runId, projectId) => {
|
|
10183
|
+
runAdsSync(runId, projectId);
|
|
10184
|
+
},
|
|
9752
10185
|
onDataRefreshRequested: (projectName) => {
|
|
9753
10186
|
void refreshAllIntegrations(aeroClient, projectName);
|
|
9754
10187
|
},
|
|
@@ -9760,8 +10193,8 @@ async function createServer(opts) {
|
|
|
9760
10193
|
});
|
|
9761
10194
|
if (!probed) return;
|
|
9762
10195
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(and13(
|
|
9763
|
-
|
|
9764
|
-
|
|
10196
|
+
eq19(ccReleaseSyncs.release, probed.release),
|
|
10197
|
+
eq19(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
9765
10198
|
)).limit(1).get();
|
|
9766
10199
|
if (alreadySynced) {
|
|
9767
10200
|
app.log.info({ projectName, release: probed.release }, "Scheduled backlinks sync: already up to date, skipping");
|
|
@@ -9894,7 +10327,7 @@ async function createServer(opts) {
|
|
|
9894
10327
|
return removed;
|
|
9895
10328
|
}
|
|
9896
10329
|
};
|
|
9897
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
10330
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto19.randomBytes(32).toString("hex");
|
|
9898
10331
|
const googleConnectionStore = {
|
|
9899
10332
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
9900
10333
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -9940,11 +10373,11 @@ async function createServer(opts) {
|
|
|
9940
10373
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
9941
10374
|
if (opts.config.apiKey) {
|
|
9942
10375
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
9943
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
10376
|
+
const existing = opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, keyHash)).get();
|
|
9944
10377
|
if (!existing) {
|
|
9945
10378
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
9946
10379
|
opts.db.insert(apiKeys).values({
|
|
9947
|
-
id: `key_${
|
|
10380
|
+
id: `key_${crypto19.randomBytes(8).toString("hex")}`,
|
|
9948
10381
|
name: "default",
|
|
9949
10382
|
keyHash,
|
|
9950
10383
|
keyPrefix: prefix,
|
|
@@ -9968,7 +10401,7 @@ async function createServer(opts) {
|
|
|
9968
10401
|
};
|
|
9969
10402
|
const createSession = (apiKeyId) => {
|
|
9970
10403
|
pruneExpiredSessions();
|
|
9971
|
-
const sessionId =
|
|
10404
|
+
const sessionId = crypto19.randomBytes(32).toString("hex");
|
|
9972
10405
|
sessions.set(sessionId, {
|
|
9973
10406
|
apiKeyId,
|
|
9974
10407
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -9992,7 +10425,7 @@ async function createServer(opts) {
|
|
|
9992
10425
|
};
|
|
9993
10426
|
const getDefaultApiKey = () => {
|
|
9994
10427
|
if (!opts.config.apiKey) return void 0;
|
|
9995
|
-
return opts.db.select().from(apiKeys).where(
|
|
10428
|
+
return opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
9996
10429
|
};
|
|
9997
10430
|
const createPasswordSession = (reply) => {
|
|
9998
10431
|
const key = getDefaultApiKey();
|
|
@@ -10013,7 +10446,7 @@ async function createServer(opts) {
|
|
|
10013
10446
|
if (!header) return false;
|
|
10014
10447
|
const parts = header.split(" ");
|
|
10015
10448
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
10016
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
10449
|
+
const key = opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
10017
10450
|
return Boolean(key && !key.revokedAt);
|
|
10018
10451
|
};
|
|
10019
10452
|
app.get(apiPrefix + "/session", async (request, reply) => {
|
|
@@ -10067,12 +10500,12 @@ async function createServer(opts) {
|
|
|
10067
10500
|
return reply.send({ authenticated: true });
|
|
10068
10501
|
}
|
|
10069
10502
|
if (apiKey) {
|
|
10070
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
10503
|
+
const key = opts.db.select().from(apiKeys).where(eq19(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
10071
10504
|
if (!key || key.revokedAt) {
|
|
10072
10505
|
const err2 = authInvalid();
|
|
10073
10506
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
10074
10507
|
}
|
|
10075
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10508
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(apiKeys.id, key.id)).run();
|
|
10076
10509
|
const sessionId = createSession(key.id);
|
|
10077
10510
|
reply.header("set-cookie", serializeSessionCookie({
|
|
10078
10511
|
name: SESSION_COOKIE_NAME,
|
|
@@ -10205,6 +10638,11 @@ async function createServer(opts) {
|
|
|
10205
10638
|
onGbpSyncRequested: (runId, projectId, syncOpts) => {
|
|
10206
10639
|
runGbpSync(runId, projectId, syncOpts);
|
|
10207
10640
|
},
|
|
10641
|
+
adsCredentialStore,
|
|
10642
|
+
verifyAdsAccount,
|
|
10643
|
+
onAdsSyncRequested: (runId, projectId) => {
|
|
10644
|
+
runAdsSync(runId, projectId);
|
|
10645
|
+
},
|
|
10208
10646
|
getBacklinksStatus: () => ({
|
|
10209
10647
|
duckdbInstalled: isDuckdbInstalled(),
|
|
10210
10648
|
duckdbVersion: readInstalledVersion() ?? void 0,
|
|
@@ -10226,7 +10664,7 @@ async function createServer(opts) {
|
|
|
10226
10664
|
deps: {
|
|
10227
10665
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
10228
10666
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10229
|
-
const runId =
|
|
10667
|
+
const runId = crypto19.randomUUID();
|
|
10230
10668
|
opts.db.insert(runs).values({
|
|
10231
10669
|
id: runId,
|
|
10232
10670
|
projectId,
|
|
@@ -10312,7 +10750,7 @@ async function createServer(opts) {
|
|
|
10312
10750
|
...inspectOpts,
|
|
10313
10751
|
config: opts.config
|
|
10314
10752
|
}).then(() => {
|
|
10315
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
10753
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq19(runs.id, runId)).get();
|
|
10316
10754
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
10317
10755
|
return maybeRefreshGscCoverage(opts.db, opts.config, projectId);
|
|
10318
10756
|
}
|
|
@@ -10400,7 +10838,7 @@ async function createServer(opts) {
|
|
|
10400
10838
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
10401
10839
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
10402
10840
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
10403
|
-
id:
|
|
10841
|
+
id: crypto19.randomUUID(),
|
|
10404
10842
|
projectId,
|
|
10405
10843
|
actor: "api",
|
|
10406
10844
|
action: existing ? "provider.updated" : "provider.created",
|