@canonry/canonry 4.129.1 → 4.131.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/assets/assets/{AuditHistoryPanel-wJqs99Ak.js → AuditHistoryPanel-yJXZMO5t.js} +1 -1
- package/assets/assets/BacklinksPage-6VTLbAYx.js +1 -0
- package/assets/assets/{ChartPrimitives-DSr5ldMf.js → ChartPrimitives-D5YGiDLi.js} +1 -1
- package/assets/assets/{HistoryPage-Ca-GGnTI.js → HistoryPage-BxdTM6-R.js} +1 -1
- package/assets/assets/ProjectPage-BJKxZPRO.js +8 -0
- package/assets/assets/{RunRow-O9TbOdQY.js → RunRow-DTrAO7tE.js} +1 -1
- package/assets/assets/{RunsPage-C1Sqd7AN.js → RunsPage-CHU0DQGl.js} +1 -1
- package/assets/assets/{SettingsPage-DPFh62BV.js → SettingsPage-bOIwju6t.js} +1 -1
- package/assets/assets/{TrafficPage-5vKoQyoP.js → TrafficPage-BKvuR482.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-D17oADs8.js → TrafficSourceDetailPage-3-iUTgQg.js} +1 -1
- package/assets/assets/{arrow-left-De5P5AIE.js → arrow-left-BXSfGhr6.js} +1 -1
- package/assets/assets/{extract-error-message-BqMd-fQ6.js → extract-error-message-DEAY62LS.js} +1 -1
- package/assets/assets/index-CfjIYitV.js +210 -0
- package/assets/assets/{index-jymsMBpZ.css → index-JhX-wfQO.css} +1 -1
- package/assets/assets/trash-2-O1wo9IX1.js +1 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-K4FDDCYG.js → chunk-3WBAJKRR.js} +2746 -2626
- package/dist/{chunk-GYAKRVE2.js → chunk-5GQ6FF5R.js} +508 -355
- package/dist/{chunk-WWZE7VDQ.js → chunk-73HPY6XQ.js} +118 -1
- package/dist/{chunk-LPEE5AYY.js → chunk-ZXVWL5HX.js} +956 -301
- package/dist/cli.js +432 -182
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-2QBAGT5X.js → intelligence-service-YJKQQYJQ.js} +2 -2
- package/dist/mcp.js +5 -5
- package/package.json +10 -10
- package/assets/assets/BacklinksPage-BBDExYo_.js +0 -1
- package/assets/assets/ProjectPage-BuaSDSwf.js +0 -7
- package/assets/assets/index-C1ABEgkr.js +0 -210
- package/assets/assets/trash-2-6ZpABj9G.js +0 -1
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
loadConfig,
|
|
11
11
|
loadConfigRaw,
|
|
12
12
|
saveConfigPatch
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-73HPY6XQ.js";
|
|
14
14
|
import {
|
|
15
15
|
CC_CACHE_DIR,
|
|
16
16
|
DUCKDB_SPEC,
|
|
@@ -110,13 +110,15 @@ import {
|
|
|
110
110
|
readInstalledVersion,
|
|
111
111
|
redactNotificationUrl,
|
|
112
112
|
refreshAccessToken,
|
|
113
|
+
researchRunQueries,
|
|
114
|
+
researchRuns,
|
|
113
115
|
resolveWebhookTarget,
|
|
114
116
|
runs,
|
|
115
117
|
schedules,
|
|
116
118
|
siteAuditPages,
|
|
117
119
|
siteAuditSnapshots,
|
|
118
120
|
usageCounters
|
|
119
|
-
} from "./chunk-
|
|
121
|
+
} from "./chunk-ZXVWL5HX.js";
|
|
120
122
|
import {
|
|
121
123
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
122
124
|
AGENT_PROVIDER_IDS,
|
|
@@ -133,6 +135,8 @@ import {
|
|
|
133
135
|
LlmCapabilities,
|
|
134
136
|
MemorySources,
|
|
135
137
|
ProviderNames,
|
|
138
|
+
ResearchQueryStatuses,
|
|
139
|
+
ResearchRunStatuses,
|
|
136
140
|
RunKinds,
|
|
137
141
|
RunStatuses,
|
|
138
142
|
RunTriggers,
|
|
@@ -172,6 +176,7 @@ import {
|
|
|
172
176
|
isBrowserProvider,
|
|
173
177
|
isGhostTelemetryEvent,
|
|
174
178
|
isRetryableHttpError,
|
|
179
|
+
mapWithConcurrency,
|
|
175
180
|
normalizeIdTokens,
|
|
176
181
|
normalizeServedModel,
|
|
177
182
|
normalizeUrlPath,
|
|
@@ -186,7 +191,7 @@ import {
|
|
|
186
191
|
validationError,
|
|
187
192
|
winnabilityClassLabel,
|
|
188
193
|
withRetry
|
|
189
|
-
} from "./chunk-
|
|
194
|
+
} from "./chunk-3WBAJKRR.js";
|
|
190
195
|
|
|
191
196
|
// src/telemetry.ts
|
|
192
197
|
import crypto from "crypto";
|
|
@@ -478,11 +483,11 @@ function checkLatestVersionForServer(opts) {
|
|
|
478
483
|
|
|
479
484
|
// src/server.ts
|
|
480
485
|
import { createRequire as createRequire3 } from "module";
|
|
481
|
-
import
|
|
486
|
+
import crypto23 from "crypto";
|
|
482
487
|
import fs8 from "fs";
|
|
483
488
|
import path9 from "path";
|
|
484
489
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
485
|
-
import { and as
|
|
490
|
+
import { and as and16, eq as eq22 } from "drizzle-orm";
|
|
486
491
|
import Fastify from "fastify";
|
|
487
492
|
import os5 from "os";
|
|
488
493
|
|
|
@@ -3093,11 +3098,11 @@ function removeWordpressConnection(config, projectName) {
|
|
|
3093
3098
|
}
|
|
3094
3099
|
|
|
3095
3100
|
// src/job-runner.ts
|
|
3096
|
-
import
|
|
3101
|
+
import crypto4 from "crypto";
|
|
3097
3102
|
import fs2 from "fs";
|
|
3098
3103
|
import path3 from "path";
|
|
3099
3104
|
import os3 from "os";
|
|
3100
|
-
import { and, eq, inArray, sql } from "drizzle-orm";
|
|
3105
|
+
import { and as and2, eq as eq2, inArray, sql as sql2 } from "drizzle-orm";
|
|
3101
3106
|
|
|
3102
3107
|
// src/run-telemetry.ts
|
|
3103
3108
|
import crypto2 from "crypto";
|
|
@@ -3139,14 +3144,7 @@ function buildRunCompletedProps(input) {
|
|
|
3139
3144
|
return props;
|
|
3140
3145
|
}
|
|
3141
3146
|
|
|
3142
|
-
// src/
|
|
3143
|
-
var log = createLogger("JobRunner");
|
|
3144
|
-
var RunCancelledError = class extends Error {
|
|
3145
|
-
constructor(runId) {
|
|
3146
|
-
super(`Run ${runId} was cancelled`);
|
|
3147
|
-
this.name = "RunCancelledError";
|
|
3148
|
-
}
|
|
3149
|
-
};
|
|
3147
|
+
// src/provider-execution-gate.ts
|
|
3150
3148
|
var ProviderExecutionGate = class {
|
|
3151
3149
|
constructor(maxConcurrency, maxPerMinute) {
|
|
3152
3150
|
this.maxConcurrency = maxConcurrency;
|
|
@@ -3172,15 +3170,12 @@ var ProviderExecutionGate = class {
|
|
|
3172
3170
|
this.inFlight++;
|
|
3173
3171
|
return;
|
|
3174
3172
|
}
|
|
3175
|
-
await new Promise((resolve) =>
|
|
3176
|
-
this.waiters.push(resolve);
|
|
3177
|
-
});
|
|
3173
|
+
await new Promise((resolve) => this.waiters.push(resolve));
|
|
3178
3174
|
this.inFlight++;
|
|
3179
3175
|
}
|
|
3180
3176
|
release() {
|
|
3181
3177
|
this.inFlight = Math.max(0, this.inFlight - 1);
|
|
3182
|
-
|
|
3183
|
-
next?.();
|
|
3178
|
+
this.waiters.shift()?.();
|
|
3184
3179
|
}
|
|
3185
3180
|
async waitForRateLimit() {
|
|
3186
3181
|
let releaseChain;
|
|
@@ -3192,18 +3187,11 @@ var ProviderExecutionGate = class {
|
|
|
3192
3187
|
try {
|
|
3193
3188
|
const now = Date.now();
|
|
3194
3189
|
const windowStart = now - 6e4;
|
|
3195
|
-
while (this.window.length > 0 && this.window[0] < windowStart)
|
|
3196
|
-
this.window.shift();
|
|
3197
|
-
}
|
|
3190
|
+
while (this.window.length > 0 && this.window[0] < windowStart) this.window.shift();
|
|
3198
3191
|
if (this.window.length >= this.maxPerMinute) {
|
|
3199
|
-
|
|
3200
|
-
const
|
|
3201
|
-
|
|
3202
|
-
const nowAfterWait = Date.now();
|
|
3203
|
-
const newWindowStart = nowAfterWait - 6e4;
|
|
3204
|
-
while (this.window.length > 0 && this.window[0] < newWindowStart) {
|
|
3205
|
-
this.window.shift();
|
|
3206
|
-
}
|
|
3192
|
+
await new Promise((resolve) => setTimeout(resolve, this.window[0] + 6e4 - now + 50));
|
|
3193
|
+
const nextWindowStart = Date.now() - 6e4;
|
|
3194
|
+
while (this.window.length > 0 && this.window[0] < nextWindowStart) this.window.shift();
|
|
3207
3195
|
}
|
|
3208
3196
|
this.window.push(Date.now());
|
|
3209
3197
|
} finally {
|
|
@@ -3211,6 +3199,51 @@ var ProviderExecutionGate = class {
|
|
|
3211
3199
|
}
|
|
3212
3200
|
}
|
|
3213
3201
|
};
|
|
3202
|
+
|
|
3203
|
+
// src/usage-quota.ts
|
|
3204
|
+
import crypto3 from "crypto";
|
|
3205
|
+
import { and, eq, lte, sql } from "drizzle-orm";
|
|
3206
|
+
function getCurrentUsageDay() {
|
|
3207
|
+
return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
3208
|
+
}
|
|
3209
|
+
function reserveDailyQueryQuota(db, input) {
|
|
3210
|
+
if (input.count <= 0) return { reserved: true, used: 0 };
|
|
3211
|
+
if (input.count > input.limit) return { reserved: false, used: 0 };
|
|
3212
|
+
return db.transaction((tx) => {
|
|
3213
|
+
const reserveExisting = () => tx.update(usageCounters).set({ count: sql`${usageCounters.count} + ${input.count}`, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and(
|
|
3214
|
+
eq(usageCounters.scope, input.scope),
|
|
3215
|
+
eq(usageCounters.period, input.period),
|
|
3216
|
+
eq(usageCounters.metric, "queries"),
|
|
3217
|
+
lte(usageCounters.count, input.limit - input.count)
|
|
3218
|
+
)).run();
|
|
3219
|
+
if (reserveExisting().changes === 1) return { reserved: true, used: 0 };
|
|
3220
|
+
const inserted = tx.insert(usageCounters).values({
|
|
3221
|
+
id: crypto3.randomUUID(),
|
|
3222
|
+
scope: input.scope,
|
|
3223
|
+
period: input.period,
|
|
3224
|
+
metric: "queries",
|
|
3225
|
+
count: input.count,
|
|
3226
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3227
|
+
}).onConflictDoNothing().run();
|
|
3228
|
+
if (inserted.changes === 1) return { reserved: true, used: 0 };
|
|
3229
|
+
if (reserveExisting().changes === 1) return { reserved: true, used: 0 };
|
|
3230
|
+
const row = tx.select({ count: usageCounters.count }).from(usageCounters).where(and(eq(usageCounters.scope, input.scope), eq(usageCounters.period, input.period), eq(usageCounters.metric, "queries"))).get();
|
|
3231
|
+
return { reserved: false, used: row?.count ?? 0 };
|
|
3232
|
+
});
|
|
3233
|
+
}
|
|
3234
|
+
function releaseDailyQueryQuota(db, input) {
|
|
3235
|
+
if (input.count <= 0) return;
|
|
3236
|
+
db.update(usageCounters).set({ count: sql`MAX(0, ${usageCounters.count} - ${input.count})`, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and(eq(usageCounters.scope, input.scope), eq(usageCounters.period, input.period), eq(usageCounters.metric, "queries"))).run();
|
|
3237
|
+
}
|
|
3238
|
+
|
|
3239
|
+
// src/job-runner.ts
|
|
3240
|
+
var log = createLogger("JobRunner");
|
|
3241
|
+
var RunCancelledError = class extends Error {
|
|
3242
|
+
constructor(runId) {
|
|
3243
|
+
super(`Run ${runId} was cancelled`);
|
|
3244
|
+
this.name = "RunCancelledError";
|
|
3245
|
+
}
|
|
3246
|
+
};
|
|
3214
3247
|
async function runWithConcurrency(items, limit, worker) {
|
|
3215
3248
|
if (items.length === 0) return;
|
|
3216
3249
|
const cap = Math.max(1, Math.min(limit, items.length));
|
|
@@ -3288,7 +3321,7 @@ var JobRunner = class {
|
|
|
3288
3321
|
if (stale.length === 0) return;
|
|
3289
3322
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3290
3323
|
for (const run of stale) {
|
|
3291
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
3324
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq2(runs.id, run.id)).run();
|
|
3292
3325
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
3293
3326
|
}
|
|
3294
3327
|
}
|
|
@@ -3303,6 +3336,7 @@ var JobRunner = class {
|
|
|
3303
3336
|
let runTrigger;
|
|
3304
3337
|
let canonicalDomain;
|
|
3305
3338
|
const providerDispatchCounts = /* @__PURE__ */ new Map();
|
|
3339
|
+
const providerReservations = /* @__PURE__ */ new Map();
|
|
3306
3340
|
try {
|
|
3307
3341
|
const existingRun = this.getRunState(runId);
|
|
3308
3342
|
if (!existingRun) {
|
|
@@ -3322,10 +3356,10 @@ var JobRunner = class {
|
|
|
3322
3356
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
3323
3357
|
}
|
|
3324
3358
|
if (existingRun.status === "queued") {
|
|
3325
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
3359
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and2(eq2(runs.id, runId), eq2(runs.status, "queued"))).run();
|
|
3326
3360
|
}
|
|
3327
3361
|
this.throwIfRunCancelled(runId);
|
|
3328
|
-
const project = this.db.select().from(projects).where(
|
|
3362
|
+
const project = this.db.select().from(projects).where(eq2(projects.id, projectId)).get();
|
|
3329
3363
|
if (!project) {
|
|
3330
3364
|
throw new Error(`Project ${projectId} not found`);
|
|
3331
3365
|
}
|
|
@@ -3350,8 +3384,8 @@ var JobRunner = class {
|
|
|
3350
3384
|
}
|
|
3351
3385
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
3352
3386
|
const scopedQueryNames = existingRun.queries;
|
|
3353
|
-
projectQueries = scopedQueryNames ? this.db.select().from(queries).where(
|
|
3354
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
3387
|
+
projectQueries = scopedQueryNames ? this.db.select().from(queries).where(and2(eq2(queries.projectId, projectId), inArray(queries.query, scopedQueryNames))).all() : this.db.select().from(queries).where(eq2(queries.projectId, projectId)).all();
|
|
3388
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq2(competitors.projectId, projectId)).all();
|
|
3355
3389
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
3356
3390
|
const allDomains = effectiveDomains({
|
|
3357
3391
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -3373,13 +3407,14 @@ var JobRunner = class {
|
|
|
3373
3407
|
const todayPeriod = getCurrentUsageDay();
|
|
3374
3408
|
for (const p of activeProviders) {
|
|
3375
3409
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
3376
|
-
const providerUsage = this.db.select().from(usageCounters).where(eq(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
3377
3410
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
3378
|
-
|
|
3411
|
+
const quota = reserveDailyQueryQuota(this.db, { scope: providerScope, period: todayPeriod, count: queriesPerProvider, limit });
|
|
3412
|
+
if (!quota.reserved) {
|
|
3379
3413
|
throw new Error(
|
|
3380
|
-
`Daily quota exceeded for ${p.adapter.name}: ${
|
|
3414
|
+
`Daily quota exceeded for ${p.adapter.name}: ${quota.used} queries used today, limit is ${limit}. This run needs ${queriesPerProvider} more.`
|
|
3381
3415
|
);
|
|
3382
3416
|
}
|
|
3417
|
+
providerReservations.set(p.adapter.name, { scope: providerScope, period: todayPeriod, reserved: queriesPerProvider });
|
|
3383
3418
|
}
|
|
3384
3419
|
const executionGates = /* @__PURE__ */ new Map();
|
|
3385
3420
|
for (const provider of activeProviders) {
|
|
@@ -3434,7 +3469,7 @@ var JobRunner = class {
|
|
|
3434
3469
|
);
|
|
3435
3470
|
let screenshotRelPath = null;
|
|
3436
3471
|
if (raw.screenshotPath && fs2.existsSync(raw.screenshotPath)) {
|
|
3437
|
-
const snapshotId =
|
|
3472
|
+
const snapshotId = crypto4.randomUUID();
|
|
3438
3473
|
const screenshotDir = path3.join(os3.homedir(), ".canonry", "screenshots", runId);
|
|
3439
3474
|
if (!fs2.existsSync(screenshotDir)) fs2.mkdirSync(screenshotDir, { recursive: true });
|
|
3440
3475
|
const destPath = path3.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -3467,7 +3502,7 @@ var JobRunner = class {
|
|
|
3467
3502
|
}).run();
|
|
3468
3503
|
} else {
|
|
3469
3504
|
this.db.insert(querySnapshots).values({
|
|
3470
|
-
id:
|
|
3505
|
+
id: crypto4.randomUUID(),
|
|
3471
3506
|
runId,
|
|
3472
3507
|
queryId: q.id,
|
|
3473
3508
|
queryText: q.query,
|
|
@@ -3523,14 +3558,14 @@ var JobRunner = class {
|
|
|
3523
3558
|
const someFailed = providerErrors.size > 0;
|
|
3524
3559
|
if (allFailed) {
|
|
3525
3560
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
3526
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
3561
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq2(runs.id, runId)).run();
|
|
3527
3562
|
} else if (someFailed) {
|
|
3528
3563
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
3529
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
3564
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq2(runs.id, runId)).run();
|
|
3530
3565
|
} else {
|
|
3531
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
3566
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq2(runs.id, runId)).run();
|
|
3532
3567
|
}
|
|
3533
|
-
this.flushProviderUsage(
|
|
3568
|
+
this.flushProviderUsage(providerDispatchCounts, providerReservations);
|
|
3534
3569
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
3535
3570
|
const failureCode = providerErrors.size > 0 ? classifyProviderErrors(providerErrors) : void 0;
|
|
3536
3571
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
@@ -3565,7 +3600,7 @@ var JobRunner = class {
|
|
|
3565
3600
|
...canonicalDomain ? { canonicalDomain } : {}
|
|
3566
3601
|
};
|
|
3567
3602
|
if (err instanceof RunCancelledError || this.isRunCancelled(runId)) {
|
|
3568
|
-
this.flushProviderUsage(
|
|
3603
|
+
this.flushProviderUsage(providerDispatchCounts, providerReservations);
|
|
3569
3604
|
this.handleCancelledRun(runId, projectId, startTime, executionContext);
|
|
3570
3605
|
return;
|
|
3571
3606
|
}
|
|
@@ -3574,8 +3609,8 @@ var JobRunner = class {
|
|
|
3574
3609
|
status: "failed",
|
|
3575
3610
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3576
3611
|
error: errorMessage
|
|
3577
|
-
}).where(
|
|
3578
|
-
this.flushProviderUsage(
|
|
3612
|
+
}).where(eq2(runs.id, runId)).run();
|
|
3613
|
+
this.flushProviderUsage(providerDispatchCounts, providerReservations);
|
|
3579
3614
|
const abortReason = classifyRunAbortReason(errorMessage);
|
|
3580
3615
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
3581
3616
|
if (abortReason) {
|
|
@@ -3619,7 +3654,7 @@ var JobRunner = class {
|
|
|
3619
3654
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3620
3655
|
const period = now.slice(0, 10);
|
|
3621
3656
|
this.db.insert(usageCounters).values({
|
|
3622
|
-
id:
|
|
3657
|
+
id: crypto4.randomUUID(),
|
|
3623
3658
|
scope,
|
|
3624
3659
|
period,
|
|
3625
3660
|
metric,
|
|
@@ -3627,14 +3662,15 @@ var JobRunner = class {
|
|
|
3627
3662
|
updatedAt: now
|
|
3628
3663
|
}).onConflictDoUpdate({
|
|
3629
3664
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
3630
|
-
set: { count:
|
|
3665
|
+
set: { count: sql2`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
3631
3666
|
}).run();
|
|
3632
3667
|
}
|
|
3633
|
-
flushProviderUsage(
|
|
3634
|
-
for (const [providerName,
|
|
3635
|
-
|
|
3636
|
-
this.
|
|
3668
|
+
flushProviderUsage(providerDispatchCounts, providerReservations) {
|
|
3669
|
+
for (const [providerName, reservation] of providerReservations.entries()) {
|
|
3670
|
+
const dispatched = providerDispatchCounts.get(providerName) ?? 0;
|
|
3671
|
+
releaseDailyQueryQuota(this.db, { scope: reservation.scope, period: reservation.period, count: Math.max(0, reservation.reserved - dispatched) });
|
|
3637
3672
|
}
|
|
3673
|
+
providerReservations.clear();
|
|
3638
3674
|
}
|
|
3639
3675
|
getRunState(runId) {
|
|
3640
3676
|
return this.db.select({
|
|
@@ -3643,7 +3679,7 @@ var JobRunner = class {
|
|
|
3643
3679
|
error: runs.error,
|
|
3644
3680
|
trigger: runs.trigger,
|
|
3645
3681
|
queries: runs.queries
|
|
3646
|
-
}).from(runs).where(
|
|
3682
|
+
}).from(runs).where(eq2(runs.id, runId)).get();
|
|
3647
3683
|
}
|
|
3648
3684
|
isRunCancelled(runId) {
|
|
3649
3685
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -3659,7 +3695,7 @@ var JobRunner = class {
|
|
|
3659
3695
|
this.db.update(runs).set({
|
|
3660
3696
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3661
3697
|
error: currentRun.error ?? "Cancelled by user"
|
|
3662
|
-
}).where(
|
|
3698
|
+
}).where(eq2(runs.id, runId)).run();
|
|
3663
3699
|
}
|
|
3664
3700
|
trackEvent(
|
|
3665
3701
|
"run.completed",
|
|
@@ -3682,9 +3718,6 @@ var JobRunner = class {
|
|
|
3682
3718
|
}
|
|
3683
3719
|
}
|
|
3684
3720
|
};
|
|
3685
|
-
function getCurrentUsageDay() {
|
|
3686
|
-
return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
3687
|
-
}
|
|
3688
3721
|
function buildPhases(input) {
|
|
3689
3722
|
const total_ms = Date.now() - input.startTime;
|
|
3690
3723
|
if (input.providerCallStart === void 0) {
|
|
@@ -3696,8 +3729,8 @@ function buildPhases(input) {
|
|
|
3696
3729
|
}
|
|
3697
3730
|
|
|
3698
3731
|
// src/gsc-sync.ts
|
|
3699
|
-
import
|
|
3700
|
-
import { eq as
|
|
3732
|
+
import crypto5 from "crypto";
|
|
3733
|
+
import { eq as eq3, and as and3, sql as sql3 } from "drizzle-orm";
|
|
3701
3734
|
|
|
3702
3735
|
// src/gsc-inspect-paced.ts
|
|
3703
3736
|
var INSPECT_BASE_DELAY_MS = 1e3;
|
|
@@ -3774,13 +3807,13 @@ function daysAgo(n) {
|
|
|
3774
3807
|
}
|
|
3775
3808
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
3776
3809
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3777
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
3810
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq3(runs.id, runId)).run();
|
|
3778
3811
|
try {
|
|
3779
3812
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
3780
3813
|
if (!googleClientId || !googleClientSecret) {
|
|
3781
3814
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
3782
3815
|
}
|
|
3783
|
-
const project = db.select().from(projects).where(
|
|
3816
|
+
const project = db.select().from(projects).where(eq3(projects.id, projectId)).get();
|
|
3784
3817
|
if (!project) {
|
|
3785
3818
|
throw new Error(`Project not found: ${projectId}`);
|
|
3786
3819
|
}
|
|
@@ -3815,10 +3848,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3815
3848
|
});
|
|
3816
3849
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
3817
3850
|
db.delete(gscSearchData).where(
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3851
|
+
and3(
|
|
3852
|
+
eq3(gscSearchData.projectId, projectId),
|
|
3853
|
+
sql3`${gscSearchData.date} >= ${startDate}`,
|
|
3854
|
+
sql3`${gscSearchData.date} <= ${endDate}`
|
|
3822
3855
|
)
|
|
3823
3856
|
).run();
|
|
3824
3857
|
const batchSize = 500;
|
|
@@ -3828,7 +3861,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3828
3861
|
for (const row of batch) {
|
|
3829
3862
|
const [query, page, country, device, date] = row.keys;
|
|
3830
3863
|
db.insert(gscSearchData).values({
|
|
3831
|
-
id:
|
|
3864
|
+
id: crypto5.randomUUID(),
|
|
3832
3865
|
projectId,
|
|
3833
3866
|
syncRunId: runId,
|
|
3834
3867
|
date: date ?? "",
|
|
@@ -3850,17 +3883,17 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3850
3883
|
dimensions: ["date"]
|
|
3851
3884
|
});
|
|
3852
3885
|
db.delete(gscDailyTotals).where(
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3886
|
+
and3(
|
|
3887
|
+
eq3(gscDailyTotals.projectId, projectId),
|
|
3888
|
+
sql3`${gscDailyTotals.date} >= ${startDate}`,
|
|
3889
|
+
sql3`${gscDailyTotals.date} <= ${endDate}`
|
|
3857
3890
|
)
|
|
3858
3891
|
).run();
|
|
3859
3892
|
const dailyTotalsNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
3860
3893
|
for (const row of totalRows) {
|
|
3861
3894
|
const [date] = row.keys;
|
|
3862
3895
|
db.insert(gscDailyTotals).values({
|
|
3863
|
-
id:
|
|
3896
|
+
id: crypto5.randomUUID(),
|
|
3864
3897
|
projectId,
|
|
3865
3898
|
date: date ?? "",
|
|
3866
3899
|
clicks: row.clicks,
|
|
@@ -3876,10 +3909,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3876
3909
|
dimensions: ["date", "query"]
|
|
3877
3910
|
});
|
|
3878
3911
|
db.delete(gscQueryDailyTotals).where(
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3912
|
+
and3(
|
|
3913
|
+
eq3(gscQueryDailyTotals.projectId, projectId),
|
|
3914
|
+
sql3`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
3915
|
+
sql3`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
3883
3916
|
)
|
|
3884
3917
|
).run();
|
|
3885
3918
|
const queryTotalsNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -3887,7 +3920,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3887
3920
|
const [date, query] = row.keys;
|
|
3888
3921
|
if (!date || !query) continue;
|
|
3889
3922
|
db.insert(gscQueryDailyTotals).values({
|
|
3890
|
-
id:
|
|
3923
|
+
id: crypto5.randomUUID(),
|
|
3891
3924
|
projectId,
|
|
3892
3925
|
date,
|
|
3893
3926
|
query,
|
|
@@ -3920,7 +3953,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3920
3953
|
const rich = ir.richResultsResult;
|
|
3921
3954
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3922
3955
|
db.insert(gscUrlInspections).values({
|
|
3923
|
-
id:
|
|
3956
|
+
id: crypto5.randomUUID(),
|
|
3924
3957
|
projectId,
|
|
3925
3958
|
syncRunId: runId,
|
|
3926
3959
|
url: pageUrl,
|
|
@@ -3957,7 +3990,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3957
3990
|
note: "URL inspection stopped early after sustained rate/access failures; search-analytics data was still saved"
|
|
3958
3991
|
});
|
|
3959
3992
|
}
|
|
3960
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
3993
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq3(gscUrlInspections.projectId, projectId)).all();
|
|
3961
3994
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
3962
3995
|
for (const row of allInspections) {
|
|
3963
3996
|
const existing = latestByUrl.get(row.url);
|
|
@@ -3978,9 +4011,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3978
4011
|
}
|
|
3979
4012
|
}
|
|
3980
4013
|
const snapshotDate = formatDate(/* @__PURE__ */ new Date());
|
|
3981
|
-
db.delete(gscCoverageSnapshots).where(
|
|
4014
|
+
db.delete(gscCoverageSnapshots).where(and3(eq3(gscCoverageSnapshots.projectId, projectId), eq3(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
3982
4015
|
db.insert(gscCoverageSnapshots).values({
|
|
3983
|
-
id:
|
|
4016
|
+
id: crypto5.randomUUID(),
|
|
3984
4017
|
projectId,
|
|
3985
4018
|
syncRunId: runId,
|
|
3986
4019
|
date: snapshotDate,
|
|
@@ -3989,19 +4022,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
3989
4022
|
reasonBreakdown: reasonCounts,
|
|
3990
4023
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3991
4024
|
}).run();
|
|
3992
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4025
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq3(runs.id, runId)).run();
|
|
3993
4026
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
3994
4027
|
} catch (err) {
|
|
3995
4028
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
3996
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4029
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq3(runs.id, runId)).run();
|
|
3997
4030
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
3998
4031
|
throw err;
|
|
3999
4032
|
}
|
|
4000
4033
|
}
|
|
4001
4034
|
|
|
4002
4035
|
// src/gbp-sync.ts
|
|
4003
|
-
import
|
|
4004
|
-
import { eq as
|
|
4036
|
+
import crypto6 from "crypto";
|
|
4037
|
+
import { eq as eq4, and as and4, desc, inArray as inArray2, lt } from "drizzle-orm";
|
|
4005
4038
|
var MS_PER_DAY = 864e5;
|
|
4006
4039
|
var log3 = createLogger("GbpSync");
|
|
4007
4040
|
var LOCATION_CONCURRENCY = 4;
|
|
@@ -4022,13 +4055,13 @@ function monthMinus(n) {
|
|
|
4022
4055
|
}
|
|
4023
4056
|
async function executeGbpSync(db, runId, projectId, opts) {
|
|
4024
4057
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4025
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4058
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq4(runs.id, runId)).run();
|
|
4026
4059
|
try {
|
|
4027
4060
|
const { clientId, clientSecret } = getGoogleAuthConfig(opts.config);
|
|
4028
4061
|
if (!clientId || !clientSecret) {
|
|
4029
4062
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
4030
4063
|
}
|
|
4031
|
-
const project = db.select().from(projects).where(
|
|
4064
|
+
const project = db.select().from(projects).where(eq4(projects.id, projectId)).get();
|
|
4032
4065
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
4033
4066
|
const conn = getGoogleConnection(opts.config, project.canonicalDomain, "gbp");
|
|
4034
4067
|
if (!conn || !conn.refreshToken) {
|
|
@@ -4046,7 +4079,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4046
4079
|
});
|
|
4047
4080
|
saveConfigPatch(opts.config);
|
|
4048
4081
|
}
|
|
4049
|
-
let locationRows = db.select().from(gbpLocations).where(
|
|
4082
|
+
let locationRows = db.select().from(gbpLocations).where(and4(eq4(gbpLocations.projectId, projectId), eq4(gbpLocations.selected, true))).all();
|
|
4050
4083
|
if (opts.locationNames?.length) {
|
|
4051
4084
|
const wanted = new Set(opts.locationNames);
|
|
4052
4085
|
locationRows = locationRows.filter((l) => wanted.has(l.locationName));
|
|
@@ -4096,15 +4129,15 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4096
4129
|
})))
|
|
4097
4130
|
]);
|
|
4098
4131
|
const lodgingHash = lodging ? hashLodging(lodging) : null;
|
|
4099
|
-
const latestLodging = lodging ? db.select().from(gbpLodgingSnapshots).where(
|
|
4132
|
+
const latestLodging = lodging ? db.select().from(gbpLodgingSnapshots).where(and4(eq4(gbpLodgingSnapshots.projectId, projectId), eq4(gbpLodgingSnapshots.locationName, loc.locationName))).orderBy(desc(gbpLodgingSnapshots.syncedAt)).limit(1).get() : void 0;
|
|
4100
4133
|
const lodgingChanged = lodging !== null && latestLodging?.contentHash !== lodgingHash;
|
|
4101
4134
|
const attributesHash = hashAttributes(attributes);
|
|
4102
|
-
const latestAttributes = db.select().from(gbpAttributesSnapshots).where(
|
|
4135
|
+
const latestAttributes = db.select().from(gbpAttributesSnapshots).where(and4(eq4(gbpAttributesSnapshots.projectId, projectId), eq4(gbpAttributesSnapshots.locationName, loc.locationName))).orderBy(desc(gbpAttributesSnapshots.syncedAt)).limit(1).get();
|
|
4103
4136
|
const attributesChanged = latestAttributes?.contentHash !== attributesHash;
|
|
4104
4137
|
let placeToWrite = null;
|
|
4105
4138
|
let placeToTouch = null;
|
|
4106
4139
|
if (placesTier !== "off" && placesApiKey && lodging !== null && loc.placeId) {
|
|
4107
|
-
const latestPlace = db.select().from(gbpPlaceDetails).where(
|
|
4140
|
+
const latestPlace = db.select().from(gbpPlaceDetails).where(and4(eq4(gbpPlaceDetails.projectId, projectId), eq4(gbpPlaceDetails.locationName, loc.locationName))).orderBy(desc(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
4108
4141
|
const ageDays = latestPlace ? (Date.now() - new Date(latestPlace.syncedAt).getTime()) / MS_PER_DAY : Infinity;
|
|
4109
4142
|
if (ageDays >= placesRefreshDays) {
|
|
4110
4143
|
try {
|
|
@@ -4122,10 +4155,10 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4122
4155
|
}
|
|
4123
4156
|
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4124
4157
|
db.transaction((tx) => {
|
|
4125
|
-
tx.delete(gbpDailyMetrics).where(
|
|
4158
|
+
tx.delete(gbpDailyMetrics).where(and4(eq4(gbpDailyMetrics.projectId, projectId), eq4(gbpDailyMetrics.locationName, loc.locationName))).run();
|
|
4126
4159
|
for (const row of metricRows) {
|
|
4127
4160
|
tx.insert(gbpDailyMetrics).values({
|
|
4128
|
-
id:
|
|
4161
|
+
id: crypto6.randomUUID(),
|
|
4129
4162
|
projectId,
|
|
4130
4163
|
locationName: loc.locationName,
|
|
4131
4164
|
date: row.date,
|
|
@@ -4134,10 +4167,10 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4134
4167
|
syncRunId: runId
|
|
4135
4168
|
}).run();
|
|
4136
4169
|
}
|
|
4137
|
-
tx.delete(gbpKeywordImpressions).where(
|
|
4170
|
+
tx.delete(gbpKeywordImpressions).where(and4(eq4(gbpKeywordImpressions.projectId, projectId), eq4(gbpKeywordImpressions.locationName, loc.locationName))).run();
|
|
4138
4171
|
for (const row of keywordRows) {
|
|
4139
4172
|
tx.insert(gbpKeywordImpressions).values({
|
|
4140
|
-
id:
|
|
4173
|
+
id: crypto6.randomUUID(),
|
|
4141
4174
|
projectId,
|
|
4142
4175
|
locationName: loc.locationName,
|
|
4143
4176
|
periodStart: monthKey(keywordsStart),
|
|
@@ -4150,15 +4183,15 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4150
4183
|
}
|
|
4151
4184
|
const fetchedMonths = monthlyKeywordResults.map((r) => r.month);
|
|
4152
4185
|
if (fetchedMonths.length > 0) {
|
|
4153
|
-
tx.delete(gbpKeywordMonthly).where(
|
|
4154
|
-
|
|
4155
|
-
|
|
4186
|
+
tx.delete(gbpKeywordMonthly).where(and4(
|
|
4187
|
+
eq4(gbpKeywordMonthly.projectId, projectId),
|
|
4188
|
+
eq4(gbpKeywordMonthly.locationName, loc.locationName),
|
|
4156
4189
|
inArray2(gbpKeywordMonthly.month, fetchedMonths)
|
|
4157
4190
|
)).run();
|
|
4158
4191
|
for (const { month, rows } of monthlyKeywordResults) {
|
|
4159
4192
|
for (const row of rows) {
|
|
4160
4193
|
tx.insert(gbpKeywordMonthly).values({
|
|
4161
|
-
id:
|
|
4194
|
+
id: crypto6.randomUUID(),
|
|
4162
4195
|
projectId,
|
|
4163
4196
|
locationName: loc.locationName,
|
|
4164
4197
|
month,
|
|
@@ -4170,16 +4203,16 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4170
4203
|
}).run();
|
|
4171
4204
|
}
|
|
4172
4205
|
}
|
|
4173
|
-
tx.delete(gbpKeywordMonthly).where(
|
|
4174
|
-
|
|
4175
|
-
|
|
4206
|
+
tx.delete(gbpKeywordMonthly).where(and4(
|
|
4207
|
+
eq4(gbpKeywordMonthly.projectId, projectId),
|
|
4208
|
+
eq4(gbpKeywordMonthly.locationName, loc.locationName),
|
|
4176
4209
|
lt(gbpKeywordMonthly.month, keywordRetentionCutoff)
|
|
4177
4210
|
)).run();
|
|
4178
4211
|
}
|
|
4179
|
-
tx.delete(gbpPlaceActions).where(
|
|
4212
|
+
tx.delete(gbpPlaceActions).where(and4(eq4(gbpPlaceActions.projectId, projectId), eq4(gbpPlaceActions.locationName, loc.locationName))).run();
|
|
4180
4213
|
for (const row of placeActionRows) {
|
|
4181
4214
|
tx.insert(gbpPlaceActions).values({
|
|
4182
|
-
id:
|
|
4215
|
+
id: crypto6.randomUUID(),
|
|
4183
4216
|
projectId,
|
|
4184
4217
|
locationName: loc.locationName,
|
|
4185
4218
|
placeActionLinkName: row.placeActionLinkName,
|
|
@@ -4192,7 +4225,7 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4192
4225
|
}
|
|
4193
4226
|
if (lodging !== null && lodgingChanged) {
|
|
4194
4227
|
tx.insert(gbpLodgingSnapshots).values({
|
|
4195
|
-
id:
|
|
4228
|
+
id: crypto6.randomUUID(),
|
|
4196
4229
|
projectId,
|
|
4197
4230
|
locationName: loc.locationName,
|
|
4198
4231
|
contentHash: lodgingHash,
|
|
@@ -4202,11 +4235,11 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4202
4235
|
syncRunId: runId
|
|
4203
4236
|
}).run();
|
|
4204
4237
|
} else if (lodging !== null && latestLodging) {
|
|
4205
|
-
tx.update(gbpLodgingSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(
|
|
4238
|
+
tx.update(gbpLodgingSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(eq4(gbpLodgingSnapshots.id, latestLodging.id)).run();
|
|
4206
4239
|
}
|
|
4207
4240
|
if (attributesChanged) {
|
|
4208
4241
|
tx.insert(gbpAttributesSnapshots).values({
|
|
4209
|
-
id:
|
|
4242
|
+
id: crypto6.randomUUID(),
|
|
4210
4243
|
projectId,
|
|
4211
4244
|
locationName: loc.locationName,
|
|
4212
4245
|
contentHash: attributesHash,
|
|
@@ -4216,11 +4249,11 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4216
4249
|
syncRunId: runId
|
|
4217
4250
|
}).run();
|
|
4218
4251
|
} else if (latestAttributes) {
|
|
4219
|
-
tx.update(gbpAttributesSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(
|
|
4252
|
+
tx.update(gbpAttributesSnapshots).set({ syncedAt: insertNow, syncRunId: runId }).where(eq4(gbpAttributesSnapshots.id, latestAttributes.id)).run();
|
|
4220
4253
|
}
|
|
4221
4254
|
if (placeToWrite) {
|
|
4222
4255
|
tx.insert(gbpPlaceDetails).values({
|
|
4223
|
-
id:
|
|
4256
|
+
id: crypto6.randomUUID(),
|
|
4224
4257
|
projectId,
|
|
4225
4258
|
locationName: loc.locationName,
|
|
4226
4259
|
placeId: placeToWrite.placeId,
|
|
@@ -4231,9 +4264,9 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4231
4264
|
syncRunId: runId
|
|
4232
4265
|
}).run();
|
|
4233
4266
|
} else if (placeToTouch) {
|
|
4234
|
-
tx.update(gbpPlaceDetails).set({ syncedAt: insertNow, syncRunId: runId }).where(
|
|
4267
|
+
tx.update(gbpPlaceDetails).set({ syncedAt: insertNow, syncRunId: runId }).where(eq4(gbpPlaceDetails.id, placeToTouch)).run();
|
|
4235
4268
|
}
|
|
4236
|
-
tx.update(gbpLocations).set({ syncedAt: insertNow, updatedAt: insertNow }).where(
|
|
4269
|
+
tx.update(gbpLocations).set({ syncedAt: insertNow, updatedAt: insertNow }).where(eq4(gbpLocations.id, loc.id)).run();
|
|
4237
4270
|
});
|
|
4238
4271
|
okCount++;
|
|
4239
4272
|
} catch (err) {
|
|
@@ -4244,24 +4277,24 @@ async function executeGbpSync(db, runId, projectId, opts) {
|
|
|
4244
4277
|
}
|
|
4245
4278
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4246
4279
|
if (errors.size === 0) {
|
|
4247
|
-
db.update(runs).set({ status: "completed", finishedAt }).where(
|
|
4280
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq4(runs.id, runId)).run();
|
|
4248
4281
|
} else if (okCount > 0) {
|
|
4249
4282
|
db.update(runs).set({
|
|
4250
4283
|
status: "partial",
|
|
4251
4284
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4252
4285
|
finishedAt
|
|
4253
|
-
}).where(
|
|
4286
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4254
4287
|
} else {
|
|
4255
4288
|
db.update(runs).set({
|
|
4256
4289
|
status: "failed",
|
|
4257
4290
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
4258
4291
|
finishedAt
|
|
4259
|
-
}).where(
|
|
4292
|
+
}).where(eq4(runs.id, runId)).run();
|
|
4260
4293
|
}
|
|
4261
4294
|
log3.info("sync.done", { runId, projectId, ok: okCount, failed: errors.size });
|
|
4262
4295
|
} catch (err) {
|
|
4263
4296
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
4264
|
-
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
4297
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq4(runs.id, runId)).run();
|
|
4265
4298
|
log3.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
4266
4299
|
throw err;
|
|
4267
4300
|
}
|
|
@@ -4299,7 +4332,7 @@ async function refreshSelectedLocationProfiles(db, projectId, accessToken, locat
|
|
|
4299
4332
|
...profile,
|
|
4300
4333
|
updatedAt
|
|
4301
4334
|
};
|
|
4302
|
-
db.update(gbpLocations).set(update).where(
|
|
4335
|
+
db.update(gbpLocations).set(update).where(and4(eq4(gbpLocations.projectId, projectId), eq4(gbpLocations.id, row.id))).run();
|
|
4303
4336
|
refreshedRows.set(row.id, { ...row, ...update });
|
|
4304
4337
|
}
|
|
4305
4338
|
}
|
|
@@ -4307,8 +4340,8 @@ async function refreshSelectedLocationProfiles(db, projectId, accessToken, locat
|
|
|
4307
4340
|
}
|
|
4308
4341
|
|
|
4309
4342
|
// src/ads-sync.ts
|
|
4310
|
-
import
|
|
4311
|
-
import { eq as
|
|
4343
|
+
import crypto7 from "crypto";
|
|
4344
|
+
import { eq as eq5 } from "drizzle-orm";
|
|
4312
4345
|
|
|
4313
4346
|
// ../integration-openai-ads/src/constants.ts
|
|
4314
4347
|
var OPENAI_ADS_API_BASE = "https://api.ads.openai.com/v1";
|
|
@@ -4873,11 +4906,11 @@ function toInsightUpserts(level, entityId, rows) {
|
|
|
4873
4906
|
}
|
|
4874
4907
|
async function executeAdsSync(db, runId, projectId, opts) {
|
|
4875
4908
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
4876
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
4909
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq5(runs.id, runId)).run();
|
|
4877
4910
|
try {
|
|
4878
|
-
const project = db.select().from(projects).where(
|
|
4911
|
+
const project = db.select().from(projects).where(eq5(projects.id, projectId)).get();
|
|
4879
4912
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
4880
|
-
const connRow = db.select().from(adsConnections).where(
|
|
4913
|
+
const connRow = db.select().from(adsConnections).where(eq5(adsConnections.projectId, projectId)).get();
|
|
4881
4914
|
if (!connRow) {
|
|
4882
4915
|
throw new Error('No ads connection found for this project. Run "canonry ads connect" first.');
|
|
4883
4916
|
}
|
|
@@ -4929,7 +4962,7 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
4929
4962
|
);
|
|
4930
4963
|
const insertNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
4931
4964
|
db.transaction((tx) => {
|
|
4932
|
-
tx.delete(adsCampaigns).where(
|
|
4965
|
+
tx.delete(adsCampaigns).where(eq5(adsCampaigns.projectId, projectId)).run();
|
|
4933
4966
|
for (const campaign of syncedCampaigns) {
|
|
4934
4967
|
tx.insert(adsCampaigns).values({
|
|
4935
4968
|
id: campaign.id,
|
|
@@ -4984,7 +5017,7 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
4984
5017
|
}
|
|
4985
5018
|
for (const upsert of insightUpserts) {
|
|
4986
5019
|
tx.insert(adsInsightsDaily).values({
|
|
4987
|
-
id:
|
|
5020
|
+
id: crypto7.randomUUID(),
|
|
4988
5021
|
projectId,
|
|
4989
5022
|
...upsert,
|
|
4990
5023
|
syncRunId: runId
|
|
@@ -5011,36 +5044,36 @@ async function executeAdsSync(db, runId, projectId, opts) {
|
|
|
5011
5044
|
conversionTrackingConfigured,
|
|
5012
5045
|
lastSyncedAt: insertNow,
|
|
5013
5046
|
updatedAt: insertNow
|
|
5014
|
-
}).where(
|
|
5047
|
+
}).where(eq5(adsConnections.projectId, projectId)).run();
|
|
5015
5048
|
});
|
|
5016
5049
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5017
5050
|
if (errors.size === 0) {
|
|
5018
|
-
db.update(runs).set({ status: "completed", finishedAt }).where(
|
|
5051
|
+
db.update(runs).set({ status: "completed", finishedAt }).where(eq5(runs.id, runId)).run();
|
|
5019
5052
|
} else if (syncedCampaigns.length > 0) {
|
|
5020
5053
|
db.update(runs).set({
|
|
5021
5054
|
status: "partial",
|
|
5022
5055
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
5023
5056
|
finishedAt
|
|
5024
|
-
}).where(
|
|
5057
|
+
}).where(eq5(runs.id, runId)).run();
|
|
5025
5058
|
} else {
|
|
5026
5059
|
db.update(runs).set({
|
|
5027
5060
|
status: "failed",
|
|
5028
5061
|
error: serializeRunError(buildRunErrorFromMessages(errors)),
|
|
5029
5062
|
finishedAt
|
|
5030
|
-
}).where(
|
|
5063
|
+
}).where(eq5(runs.id, runId)).run();
|
|
5031
5064
|
}
|
|
5032
5065
|
log4.info("sync.done", { runId, projectId, campaigns: syncedCampaigns.length, insightRows: insightUpserts.length, failed: errors.size });
|
|
5033
5066
|
} catch (err) {
|
|
5034
5067
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5035
|
-
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5068
|
+
db.update(runs).set({ status: "failed", error: serializeRunError({ message: errorMsg }), finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq5(runs.id, runId)).run();
|
|
5036
5069
|
log4.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
5037
5070
|
throw err;
|
|
5038
5071
|
}
|
|
5039
5072
|
}
|
|
5040
5073
|
|
|
5041
5074
|
// src/gsc-inspect-sitemap.ts
|
|
5042
|
-
import
|
|
5043
|
-
import { eq as
|
|
5075
|
+
import crypto8 from "crypto";
|
|
5076
|
+
import { eq as eq6, and as and5 } from "drizzle-orm";
|
|
5044
5077
|
|
|
5045
5078
|
// src/sitemap-parser.ts
|
|
5046
5079
|
var log5 = createLogger("SitemapParser");
|
|
@@ -5142,13 +5175,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
5142
5175
|
var log6 = createLogger("InspectSitemap");
|
|
5143
5176
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
5144
5177
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
5145
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
5178
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq6(runs.id, runId)).run();
|
|
5146
5179
|
try {
|
|
5147
5180
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
5148
5181
|
if (!googleClientId || !googleClientSecret) {
|
|
5149
5182
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
5150
5183
|
}
|
|
5151
|
-
const project = db.select().from(projects).where(
|
|
5184
|
+
const project = db.select().from(projects).where(eq6(projects.id, projectId)).get();
|
|
5152
5185
|
if (!project) {
|
|
5153
5186
|
throw new Error(`Project not found: ${projectId}`);
|
|
5154
5187
|
}
|
|
@@ -5190,7 +5223,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5190
5223
|
const rich = ir.richResultsResult;
|
|
5191
5224
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5192
5225
|
db.insert(gscUrlInspections).values({
|
|
5193
|
-
id:
|
|
5226
|
+
id: crypto8.randomUUID(),
|
|
5194
5227
|
projectId,
|
|
5195
5228
|
syncRunId: runId,
|
|
5196
5229
|
url: pageUrl,
|
|
@@ -5226,7 +5259,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5226
5259
|
`URL inspection aborted after ${INSPECT_FAILFAST_THRESHOLD} consecutive rate/access failures (likely GSC URL Inspection quota exhaustion or property access loss). Last error: ${detail}`
|
|
5227
5260
|
);
|
|
5228
5261
|
}
|
|
5229
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
5262
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq6(gscUrlInspections.projectId, projectId)).all();
|
|
5230
5263
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
5231
5264
|
for (const row of allInspections) {
|
|
5232
5265
|
const existing = latestByUrl.get(row.url);
|
|
@@ -5247,9 +5280,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5247
5280
|
}
|
|
5248
5281
|
}
|
|
5249
5282
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
5250
|
-
db.delete(gscCoverageSnapshots).where(
|
|
5283
|
+
db.delete(gscCoverageSnapshots).where(and5(eq6(gscCoverageSnapshots.projectId, projectId), eq6(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
5251
5284
|
db.insert(gscCoverageSnapshots).values({
|
|
5252
|
-
id:
|
|
5285
|
+
id: crypto8.randomUUID(),
|
|
5253
5286
|
projectId,
|
|
5254
5287
|
syncRunId: runId,
|
|
5255
5288
|
date: snapshotDate,
|
|
@@ -5259,19 +5292,19 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
5259
5292
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
5260
5293
|
}).run();
|
|
5261
5294
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
5262
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5295
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
5263
5296
|
log6.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
5264
5297
|
} catch (err) {
|
|
5265
5298
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5266
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5299
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq6(runs.id, runId)).run();
|
|
5267
5300
|
log6.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
5268
5301
|
throw err;
|
|
5269
5302
|
}
|
|
5270
5303
|
}
|
|
5271
5304
|
|
|
5272
5305
|
// src/bing-inspect-sitemap.ts
|
|
5273
|
-
import
|
|
5274
|
-
import { eq as
|
|
5306
|
+
import crypto9 from "crypto";
|
|
5307
|
+
import { eq as eq7, desc as desc2 } from "drizzle-orm";
|
|
5275
5308
|
var log7 = createLogger("BingInspectSitemap");
|
|
5276
5309
|
function parseBingDate(value) {
|
|
5277
5310
|
if (!value) return null;
|
|
@@ -5289,9 +5322,9 @@ function isBlockingIssueType(issueType) {
|
|
|
5289
5322
|
}
|
|
5290
5323
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
5291
5324
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5292
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5325
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq7(runs.id, runId)).run();
|
|
5293
5326
|
try {
|
|
5294
|
-
const project = db.select().from(projects).where(
|
|
5327
|
+
const project = db.select().from(projects).where(eq7(projects.id, projectId)).get();
|
|
5295
5328
|
if (!project) {
|
|
5296
5329
|
throw new Error(`Project not found: ${projectId}`);
|
|
5297
5330
|
}
|
|
@@ -5309,7 +5342,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5309
5342
|
if (sitemapUrls.length === 0) {
|
|
5310
5343
|
throw new Error("No URLs found in sitemap");
|
|
5311
5344
|
}
|
|
5312
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
5345
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq7(bingUrlInspections.projectId, projectId)).all();
|
|
5313
5346
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
5314
5347
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
5315
5348
|
log7.info("sitemap.diff", {
|
|
@@ -5358,7 +5391,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5358
5391
|
derivedInIndex = false;
|
|
5359
5392
|
}
|
|
5360
5393
|
db.insert(bingUrlInspections).values({
|
|
5361
|
-
id:
|
|
5394
|
+
id: crypto9.randomUUID(),
|
|
5362
5395
|
projectId,
|
|
5363
5396
|
url: pageUrl,
|
|
5364
5397
|
httpCode,
|
|
@@ -5392,7 +5425,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5392
5425
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
5393
5426
|
}
|
|
5394
5427
|
}
|
|
5395
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
5428
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq7(bingUrlInspections.projectId, projectId)).orderBy(desc2(bingUrlInspections.inspectedAt)).all();
|
|
5396
5429
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
5397
5430
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
5398
5431
|
for (const row of allInspections) {
|
|
@@ -5416,7 +5449,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5416
5449
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
5417
5450
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
5418
5451
|
db.insert(bingCoverageSnapshots).values({
|
|
5419
|
-
id:
|
|
5452
|
+
id: crypto9.randomUUID(),
|
|
5420
5453
|
projectId,
|
|
5421
5454
|
syncRunId: runId,
|
|
5422
5455
|
date: snapshotDate,
|
|
@@ -5435,7 +5468,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5435
5468
|
}
|
|
5436
5469
|
}).run();
|
|
5437
5470
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
5438
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5471
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
5439
5472
|
log7.info("inspect.completed", {
|
|
5440
5473
|
runId,
|
|
5441
5474
|
projectId,
|
|
@@ -5449,15 +5482,15 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
5449
5482
|
});
|
|
5450
5483
|
} catch (err) {
|
|
5451
5484
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5452
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
5485
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq7(runs.id, runId)).run();
|
|
5453
5486
|
log7.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
5454
5487
|
throw err;
|
|
5455
5488
|
}
|
|
5456
5489
|
}
|
|
5457
5490
|
|
|
5458
5491
|
// src/coverage-refresh.ts
|
|
5459
|
-
import
|
|
5460
|
-
import { and as
|
|
5492
|
+
import crypto10 from "crypto";
|
|
5493
|
+
import { and as and6, desc as desc3, eq as eq8, inArray as inArray3 } from "drizzle-orm";
|
|
5461
5494
|
var log8 = createLogger("CoverageRefresh");
|
|
5462
5495
|
var COVERAGE_REFRESH_MIN_INTERVAL_MS = 60 * 60 * 1e3;
|
|
5463
5496
|
var ACTIVE_OR_DONE_STATUSES = [
|
|
@@ -5468,16 +5501,16 @@ var ACTIVE_OR_DONE_STATUSES = [
|
|
|
5468
5501
|
];
|
|
5469
5502
|
var defaultDeps = { executeInspectSitemap };
|
|
5470
5503
|
async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps, nowMs = Date.now()) {
|
|
5471
|
-
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(
|
|
5504
|
+
const project = db.select({ canonicalDomain: projects.canonicalDomain }).from(projects).where(eq8(projects.id, projectId)).get();
|
|
5472
5505
|
if (!project) return null;
|
|
5473
5506
|
const { clientId, clientSecret } = getGoogleAuthConfig(config);
|
|
5474
5507
|
if (!clientId || !clientSecret) return null;
|
|
5475
5508
|
const conn = getGoogleConnection(config, project.canonicalDomain, "gsc");
|
|
5476
5509
|
if (!conn?.refreshToken || !conn.propertyId) return null;
|
|
5477
5510
|
const recent = db.select({ createdAt: runs.createdAt }).from(runs).where(
|
|
5478
|
-
|
|
5479
|
-
|
|
5480
|
-
|
|
5511
|
+
and6(
|
|
5512
|
+
eq8(runs.projectId, projectId),
|
|
5513
|
+
eq8(runs.kind, RunKinds["inspect-sitemap"]),
|
|
5481
5514
|
inArray3(runs.status, ACTIVE_OR_DONE_STATUSES)
|
|
5482
5515
|
)
|
|
5483
5516
|
).orderBy(desc3(runs.createdAt)).limit(1).get();
|
|
@@ -5488,7 +5521,7 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
5488
5521
|
return null;
|
|
5489
5522
|
}
|
|
5490
5523
|
}
|
|
5491
|
-
const runId =
|
|
5524
|
+
const runId = crypto10.randomUUID();
|
|
5492
5525
|
db.insert(runs).values({
|
|
5493
5526
|
id: runId,
|
|
5494
5527
|
projectId,
|
|
@@ -5511,9 +5544,9 @@ async function maybeRefreshGscCoverage(db, config, projectId, deps = defaultDeps
|
|
|
5511
5544
|
}
|
|
5512
5545
|
|
|
5513
5546
|
// src/commoncrawl-sync.ts
|
|
5514
|
-
import
|
|
5547
|
+
import crypto11 from "crypto";
|
|
5515
5548
|
import path4 from "path";
|
|
5516
|
-
import { and as
|
|
5549
|
+
import { and as and7, eq as eq9, sql as sql4 } from "drizzle-orm";
|
|
5517
5550
|
var log9 = createLogger("CommonCrawlSync");
|
|
5518
5551
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
5519
5552
|
function defaultDeps2() {
|
|
@@ -5539,7 +5572,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
5539
5572
|
phaseDetail: "downloading vertices + edges",
|
|
5540
5573
|
updatedAt: downloadStartedAt,
|
|
5541
5574
|
error: null
|
|
5542
|
-
}).where(
|
|
5575
|
+
}).where(eq9(ccReleaseSyncs.id, syncId)).run();
|
|
5543
5576
|
const paths = ccReleasePaths(release);
|
|
5544
5577
|
const releaseCacheDir = path4.join(deps.cacheDir, release);
|
|
5545
5578
|
const vertexPath = path4.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -5562,7 +5595,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
5562
5595
|
vertexSha256: vertex.sha256,
|
|
5563
5596
|
edgesSha256: edges.sha256,
|
|
5564
5597
|
updatedAt: downloadFinishedAt
|
|
5565
|
-
}).where(
|
|
5598
|
+
}).where(eq9(ccReleaseSyncs.id, syncId)).run();
|
|
5566
5599
|
const allProjects = db.select().from(projects).all();
|
|
5567
5600
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
5568
5601
|
let rows = [];
|
|
@@ -5578,15 +5611,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
5578
5611
|
}
|
|
5579
5612
|
const queriedAt = deps.now().toISOString();
|
|
5580
5613
|
db.transaction((tx) => {
|
|
5581
|
-
tx.delete(backlinkDomains).where(
|
|
5582
|
-
tx.delete(backlinkSummaries).where(
|
|
5614
|
+
tx.delete(backlinkDomains).where(eq9(backlinkDomains.releaseSyncId, syncId)).run();
|
|
5615
|
+
tx.delete(backlinkSummaries).where(eq9(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
5583
5616
|
const expanded = [];
|
|
5584
5617
|
for (const r of rows) {
|
|
5585
5618
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
5586
5619
|
if (!projectIds) continue;
|
|
5587
5620
|
for (const projectId of projectIds) {
|
|
5588
5621
|
expanded.push({
|
|
5589
|
-
id:
|
|
5622
|
+
id: crypto11.randomUUID(),
|
|
5590
5623
|
projectId,
|
|
5591
5624
|
releaseSyncId: syncId,
|
|
5592
5625
|
release,
|
|
@@ -5606,7 +5639,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
5606
5639
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
5607
5640
|
const summary = computeSummary(projectRows);
|
|
5608
5641
|
tx.insert(backlinkSummaries).values({
|
|
5609
|
-
id:
|
|
5642
|
+
id: crypto11.randomUUID(),
|
|
5610
5643
|
projectId: p.id,
|
|
5611
5644
|
releaseSyncId: syncId,
|
|
5612
5645
|
source: BacklinkSources.commoncrawl,
|
|
@@ -5639,7 +5672,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
5639
5672
|
domainsDiscovered: rows.length,
|
|
5640
5673
|
updatedAt: finishedAt,
|
|
5641
5674
|
error: null
|
|
5642
|
-
}).where(
|
|
5675
|
+
}).where(eq9(ccReleaseSyncs.id, syncId)).run();
|
|
5643
5676
|
log9.info("sync.completed", {
|
|
5644
5677
|
syncId,
|
|
5645
5678
|
release,
|
|
@@ -5669,7 +5702,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
5669
5702
|
error: errorMsg,
|
|
5670
5703
|
phaseDetail: null,
|
|
5671
5704
|
updatedAt: finishedAt
|
|
5672
|
-
}).where(
|
|
5705
|
+
}).where(eq9(ccReleaseSyncs.id, syncId)).run();
|
|
5673
5706
|
log9.error("sync.failed", { syncId, release, error: errorMsg });
|
|
5674
5707
|
throw err;
|
|
5675
5708
|
}
|
|
@@ -5703,9 +5736,9 @@ function computeSummary(rows) {
|
|
|
5703
5736
|
}
|
|
5704
5737
|
|
|
5705
5738
|
// src/backlink-extract.ts
|
|
5706
|
-
import
|
|
5739
|
+
import crypto12 from "crypto";
|
|
5707
5740
|
import fs3 from "fs";
|
|
5708
|
-
import { and as
|
|
5741
|
+
import { and as and8, desc as desc4, eq as eq10 } from "drizzle-orm";
|
|
5709
5742
|
var log10 = createLogger("BacklinkExtract");
|
|
5710
5743
|
function defaultDeps3() {
|
|
5711
5744
|
return {
|
|
@@ -5717,13 +5750,13 @@ function defaultDeps3() {
|
|
|
5717
5750
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
5718
5751
|
const deps = { ...defaultDeps3(), ...opts.deps };
|
|
5719
5752
|
const startedAt = deps.now().toISOString();
|
|
5720
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5753
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq10(runs.id, runId)).run();
|
|
5721
5754
|
try {
|
|
5722
|
-
const project = db.select().from(projects).where(
|
|
5755
|
+
const project = db.select().from(projects).where(eq10(projects.id, projectId)).get();
|
|
5723
5756
|
if (!project) {
|
|
5724
5757
|
throw new Error(`Project not found: ${projectId}`);
|
|
5725
5758
|
}
|
|
5726
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
5759
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq10(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq10(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc4(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
5727
5760
|
if (!sync) {
|
|
5728
5761
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
5729
5762
|
}
|
|
@@ -5751,15 +5784,15 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
5751
5784
|
const targetDomain = project.canonicalDomain;
|
|
5752
5785
|
db.transaction((tx) => {
|
|
5753
5786
|
tx.delete(backlinkDomains).where(
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5787
|
+
and8(
|
|
5788
|
+
eq10(backlinkDomains.projectId, projectId),
|
|
5789
|
+
eq10(backlinkDomains.source, BacklinkSources.commoncrawl),
|
|
5790
|
+
eq10(backlinkDomains.release, release)
|
|
5758
5791
|
)
|
|
5759
5792
|
).run();
|
|
5760
5793
|
if (rows.length > 0) {
|
|
5761
5794
|
const values = rows.map((r) => ({
|
|
5762
|
-
id:
|
|
5795
|
+
id: crypto12.randomUUID(),
|
|
5763
5796
|
projectId,
|
|
5764
5797
|
releaseSyncId: syncId,
|
|
5765
5798
|
source: BacklinkSources.commoncrawl,
|
|
@@ -5773,7 +5806,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
5773
5806
|
}
|
|
5774
5807
|
const summary = computeSummary2(rows);
|
|
5775
5808
|
tx.insert(backlinkSummaries).values({
|
|
5776
|
-
id:
|
|
5809
|
+
id: crypto12.randomUUID(),
|
|
5777
5810
|
projectId,
|
|
5778
5811
|
releaseSyncId: syncId,
|
|
5779
5812
|
source: BacklinkSources.commoncrawl,
|
|
@@ -5797,7 +5830,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
5797
5830
|
}).run();
|
|
5798
5831
|
});
|
|
5799
5832
|
const finishedAt = deps.now().toISOString();
|
|
5800
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
5833
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq10(runs.id, runId)).run();
|
|
5801
5834
|
log10.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
5802
5835
|
} catch (err) {
|
|
5803
5836
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -5806,7 +5839,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
5806
5839
|
status: RunStatuses.failed,
|
|
5807
5840
|
error: errorMsg,
|
|
5808
5841
|
finishedAt
|
|
5809
|
-
}).where(
|
|
5842
|
+
}).where(eq10(runs.id, runId)).run();
|
|
5810
5843
|
log10.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
5811
5844
|
throw err;
|
|
5812
5845
|
}
|
|
@@ -5816,8 +5849,8 @@ function computeSummary2(rows) {
|
|
|
5816
5849
|
}
|
|
5817
5850
|
|
|
5818
5851
|
// src/bing-backlinks-sync.ts
|
|
5819
|
-
import
|
|
5820
|
-
import { and as
|
|
5852
|
+
import crypto13 from "crypto";
|
|
5853
|
+
import { and as and9, eq as eq11 } from "drizzle-orm";
|
|
5821
5854
|
var log11 = createLogger("BingBacklinkSync");
|
|
5822
5855
|
var DEFAULT_MAX_PAGES = 200;
|
|
5823
5856
|
var LINK_COUNTS_MAX_PAGES = 50;
|
|
@@ -5850,9 +5883,9 @@ function defaultDeps4() {
|
|
|
5850
5883
|
async function executeBingBacklinkSync(db, runId, projectId, opts) {
|
|
5851
5884
|
const deps = { ...defaultDeps4(), ...opts.deps };
|
|
5852
5885
|
const startedAt = deps.now().toISOString();
|
|
5853
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
5886
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq11(runs.id, runId)).run();
|
|
5854
5887
|
try {
|
|
5855
|
-
const project = db.select().from(projects).where(
|
|
5888
|
+
const project = db.select().from(projects).where(eq11(projects.id, projectId)).get();
|
|
5856
5889
|
if (!project) throw new Error(`Project not found: ${projectId}`);
|
|
5857
5890
|
const conn = opts.resolveConnection(project.canonicalDomain);
|
|
5858
5891
|
if (!conn) throw new Error(`No Bing Webmaster connection for ${project.canonicalDomain}`);
|
|
@@ -5896,22 +5929,22 @@ async function executeBingBacklinkSync(db, runId, projectId, opts) {
|
|
|
5896
5929
|
const release = bingReleaseId(deps.now());
|
|
5897
5930
|
const targetDomain = project.canonicalDomain;
|
|
5898
5931
|
const source = BacklinkSources["bing-webmaster"];
|
|
5899
|
-
const existing = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5932
|
+
const existing = db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and9(
|
|
5933
|
+
eq11(backlinkSummaries.projectId, projectId),
|
|
5934
|
+
eq11(backlinkSummaries.source, source),
|
|
5935
|
+
eq11(backlinkSummaries.release, release)
|
|
5903
5936
|
)).get();
|
|
5904
5937
|
const preserveExisting = pageFailures > 0 && !!existing;
|
|
5905
5938
|
if (!preserveExisting) {
|
|
5906
5939
|
db.transaction((tx) => {
|
|
5907
|
-
tx.delete(backlinkDomains).where(
|
|
5908
|
-
|
|
5909
|
-
|
|
5910
|
-
|
|
5940
|
+
tx.delete(backlinkDomains).where(and9(
|
|
5941
|
+
eq11(backlinkDomains.projectId, projectId),
|
|
5942
|
+
eq11(backlinkDomains.source, source),
|
|
5943
|
+
eq11(backlinkDomains.release, release)
|
|
5911
5944
|
)).run();
|
|
5912
5945
|
if (rows.length > 0) {
|
|
5913
5946
|
tx.insert(backlinkDomains).values(rows.map((r) => ({
|
|
5914
|
-
id:
|
|
5947
|
+
id: crypto13.randomUUID(),
|
|
5915
5948
|
projectId,
|
|
5916
5949
|
releaseSyncId: null,
|
|
5917
5950
|
source,
|
|
@@ -5923,7 +5956,7 @@ async function executeBingBacklinkSync(db, runId, projectId, opts) {
|
|
|
5923
5956
|
}))).run();
|
|
5924
5957
|
}
|
|
5925
5958
|
tx.insert(backlinkSummaries).values({
|
|
5926
|
-
id:
|
|
5959
|
+
id: crypto13.randomUUID(),
|
|
5927
5960
|
projectId,
|
|
5928
5961
|
releaseSyncId: null,
|
|
5929
5962
|
source,
|
|
@@ -5949,20 +5982,20 @@ async function executeBingBacklinkSync(db, runId, projectId, opts) {
|
|
|
5949
5982
|
const finishedAt = deps.now().toISOString();
|
|
5950
5983
|
const status = pageFailures > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
5951
5984
|
const error = preserveExisting ? `Kept existing ${release} snapshot; ${pageFailures} of ${targetPages.length} inbound-link page fetches failed` : pageFailures > 0 ? `${pageFailures} of ${targetPages.length} inbound-link page fetches failed` : null;
|
|
5952
|
-
db.update(runs).set({ status, error, finishedAt }).where(
|
|
5985
|
+
db.update(runs).set({ status, error, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
5953
5986
|
log11.info("bing-sync.completed", { runId, projectId, release, rows: rows.length, status, pageFailures, preserveExisting });
|
|
5954
5987
|
} catch (err) {
|
|
5955
5988
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
5956
5989
|
const finishedAt = deps.now().toISOString();
|
|
5957
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt }).where(
|
|
5990
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt }).where(eq11(runs.id, runId)).run();
|
|
5958
5991
|
log11.error("bing-sync.failed", { runId, projectId, error: errorMsg });
|
|
5959
5992
|
throw err;
|
|
5960
5993
|
}
|
|
5961
5994
|
}
|
|
5962
5995
|
|
|
5963
5996
|
// src/discovery-run.ts
|
|
5964
|
-
import
|
|
5965
|
-
import { and as
|
|
5997
|
+
import crypto14 from "crypto";
|
|
5998
|
+
import { and as and10, eq as eq12 } from "drizzle-orm";
|
|
5966
5999
|
var log12 = createLogger("DiscoveryRun");
|
|
5967
6000
|
var DEFAULT_SEED_COUNT = 30;
|
|
5968
6001
|
var EMBED_RETRY_MAX_RETRIES = 3;
|
|
@@ -6004,11 +6037,11 @@ async function embedWithRetry(fn, opts = {}) {
|
|
|
6004
6037
|
var QUERIES_PER_INTENT_BUCKET = 6;
|
|
6005
6038
|
async function executeDiscoveryRun(opts) {
|
|
6006
6039
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6007
|
-
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
6040
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq12(runs.id, opts.runId)).run();
|
|
6008
6041
|
try {
|
|
6009
|
-
const projectRow = opts.db.select().from(projects).where(
|
|
6042
|
+
const projectRow = opts.db.select().from(projects).where(eq12(projects.id, opts.projectId)).get();
|
|
6010
6043
|
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
6011
|
-
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6044
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq12(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
6012
6045
|
const canonicalDomains = effectiveDomains({
|
|
6013
6046
|
canonicalDomain: projectRow.canonicalDomain,
|
|
6014
6047
|
ownedDomains: projectRow.ownedDomains
|
|
@@ -6046,7 +6079,7 @@ async function executeDiscoveryRun(opts) {
|
|
|
6046
6079
|
seedProvider: result.seedProvider,
|
|
6047
6080
|
result
|
|
6048
6081
|
});
|
|
6049
|
-
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6082
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq12(runs.id, opts.runId)).run();
|
|
6050
6083
|
log12.info("discovery.completed", {
|
|
6051
6084
|
runId: opts.runId,
|
|
6052
6085
|
sessionId: opts.sessionId,
|
|
@@ -6061,7 +6094,7 @@ async function executeDiscoveryRun(opts) {
|
|
|
6061
6094
|
status: RunStatuses.failed,
|
|
6062
6095
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6063
6096
|
error: errorMsg
|
|
6064
|
-
}).where(
|
|
6097
|
+
}).where(eq12(runs.id, opts.runId)).run();
|
|
6065
6098
|
}
|
|
6066
6099
|
}
|
|
6067
6100
|
function buildDefaultDeps(registry) {
|
|
@@ -6310,13 +6343,13 @@ function writeDiscoveryInsight(db, input) {
|
|
|
6310
6343
|
totalProbes
|
|
6311
6344
|
});
|
|
6312
6345
|
db.transaction((tx) => {
|
|
6313
|
-
tx.update(insights).set({ dismissed: true }).where(
|
|
6314
|
-
|
|
6315
|
-
|
|
6316
|
-
|
|
6346
|
+
tx.update(insights).set({ dismissed: true }).where(and10(
|
|
6347
|
+
eq12(insights.projectId, input.projectId),
|
|
6348
|
+
eq12(insights.type, "discovery.basket-divergence"),
|
|
6349
|
+
eq12(insights.dismissed, false)
|
|
6317
6350
|
)).run();
|
|
6318
6351
|
tx.insert(insights).values({
|
|
6319
|
-
id:
|
|
6352
|
+
id: crypto14.randomUUID(),
|
|
6320
6353
|
projectId: input.projectId,
|
|
6321
6354
|
runId: input.runId,
|
|
6322
6355
|
type: "discovery.basket-divergence",
|
|
@@ -6352,8 +6385,8 @@ function buildDiscoveryInsightTitle(input) {
|
|
|
6352
6385
|
}
|
|
6353
6386
|
|
|
6354
6387
|
// src/execute-site-audit.ts
|
|
6355
|
-
import
|
|
6356
|
-
import { eq as
|
|
6388
|
+
import crypto15 from "crypto";
|
|
6389
|
+
import { eq as eq13 } from "drizzle-orm";
|
|
6357
6390
|
import { runSitemapAudit } from "@ainyc/aeo-audit";
|
|
6358
6391
|
var log13 = createLogger("SiteAudit");
|
|
6359
6392
|
var SITE_AUDIT_DEFAULT_PAGE_LIMIT = 500;
|
|
@@ -6416,9 +6449,9 @@ function computeFactorAverages(pages) {
|
|
|
6416
6449
|
}
|
|
6417
6450
|
async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
6418
6451
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6419
|
-
db.update(runs).set({ status: "running", startedAt }).where(
|
|
6452
|
+
db.update(runs).set({ status: "running", startedAt }).where(eq13(runs.id, runId)).run();
|
|
6420
6453
|
try {
|
|
6421
|
-
const project = db.select().from(projects).where(
|
|
6454
|
+
const project = db.select().from(projects).where(eq13(projects.id, projectId)).get();
|
|
6422
6455
|
if (!project) {
|
|
6423
6456
|
throw new Error(`Project not found: ${projectId}`);
|
|
6424
6457
|
}
|
|
@@ -6451,7 +6484,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6451
6484
|
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
6452
6485
|
db.transaction((tx) => {
|
|
6453
6486
|
tx.insert(siteAuditSnapshots).values({
|
|
6454
|
-
id:
|
|
6487
|
+
id: crypto15.randomUUID(),
|
|
6455
6488
|
projectId,
|
|
6456
6489
|
runId,
|
|
6457
6490
|
sitemapUrl: report.sitemapUrl,
|
|
@@ -6480,7 +6513,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6480
6513
|
}).run();
|
|
6481
6514
|
for (const page of report.pages) {
|
|
6482
6515
|
tx.insert(siteAuditPages).values({
|
|
6483
|
-
id:
|
|
6516
|
+
id: crypto15.randomUUID(),
|
|
6484
6517
|
projectId,
|
|
6485
6518
|
runId,
|
|
6486
6519
|
url: page.url,
|
|
@@ -6491,7 +6524,7 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6491
6524
|
createdAt: finishedAt
|
|
6492
6525
|
}).run();
|
|
6493
6526
|
}
|
|
6494
|
-
tx.update(runs).set({ status, finishedAt }).where(
|
|
6527
|
+
tx.update(runs).set({ status, finishedAt }).where(eq13(runs.id, runId)).run();
|
|
6495
6528
|
});
|
|
6496
6529
|
log13.info("completed", {
|
|
6497
6530
|
runId,
|
|
@@ -6503,14 +6536,14 @@ async function executeSiteAudit(db, runId, projectId, opts = {}) {
|
|
|
6503
6536
|
});
|
|
6504
6537
|
} catch (err) {
|
|
6505
6538
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
6506
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
6539
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq13(runs.id, runId)).run();
|
|
6507
6540
|
log13.error("failed", { runId, projectId, error: errorMsg });
|
|
6508
6541
|
throw err;
|
|
6509
6542
|
}
|
|
6510
6543
|
}
|
|
6511
6544
|
|
|
6512
6545
|
// src/commands/backfill.ts
|
|
6513
|
-
import { and as
|
|
6546
|
+
import { and as and11, eq as eq14, inArray as inArray4, isNull, sql as sql5 } from "drizzle-orm";
|
|
6514
6547
|
var SNAPSHOT_BATCH_SIZE = 500;
|
|
6515
6548
|
async function backfillAnswerVisibilityCommand(opts) {
|
|
6516
6549
|
const config = loadConfig();
|
|
@@ -6518,7 +6551,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6518
6551
|
migrate(db);
|
|
6519
6552
|
const projectFilter = opts?.project?.trim();
|
|
6520
6553
|
const isDryRun = opts?.dryRun === true;
|
|
6521
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6554
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq14(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6522
6555
|
let examined = 0;
|
|
6523
6556
|
let updated = 0;
|
|
6524
6557
|
let wouldUpdate = 0;
|
|
@@ -6526,10 +6559,10 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6526
6559
|
let reparsed = 0;
|
|
6527
6560
|
let providerErrors = 0;
|
|
6528
6561
|
if (scopedProjects.length > 0) {
|
|
6529
|
-
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
6530
|
-
|
|
6562
|
+
const runRows = projectFilter ? db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(and11(
|
|
6563
|
+
eq14(runs.kind, RunKinds["answer-visibility"]),
|
|
6531
6564
|
inArray4(runs.projectId, scopedProjects.map((project) => project.id))
|
|
6532
|
-
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(
|
|
6565
|
+
)).all() : db.select({ id: runs.id, projectId: runs.projectId }).from(runs).where(eq14(runs.kind, RunKinds["answer-visibility"])).all();
|
|
6533
6566
|
const runIdsByProject = /* @__PURE__ */ new Map();
|
|
6534
6567
|
for (const run of runRows) {
|
|
6535
6568
|
const existing = runIdsByProject.get(run.projectId);
|
|
@@ -6537,7 +6570,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6537
6570
|
else runIdsByProject.set(run.projectId, [run.id]);
|
|
6538
6571
|
}
|
|
6539
6572
|
for (const project of scopedProjects) {
|
|
6540
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6573
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq14(competitors.projectId, project.id)).all().map((row) => row.domain);
|
|
6541
6574
|
const runIds = runIdsByProject.get(project.id) ?? [];
|
|
6542
6575
|
if (runIds.length === 0) continue;
|
|
6543
6576
|
const projectDomains = effectiveDomains({
|
|
@@ -6625,7 +6658,7 @@ async function backfillAnswerVisibilityCommand(opts) {
|
|
|
6625
6658
|
} else {
|
|
6626
6659
|
db.transaction((tx) => {
|
|
6627
6660
|
for (const update of pendingUpdates) {
|
|
6628
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
6661
|
+
tx.update(querySnapshots).set(update.patch).where(eq14(querySnapshots.id, update.id)).run();
|
|
6629
6662
|
}
|
|
6630
6663
|
});
|
|
6631
6664
|
updated += pendingUpdates.length;
|
|
@@ -6674,13 +6707,13 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
6674
6707
|
function backfillNormalizedPaths(db, opts) {
|
|
6675
6708
|
const baseConditions = [];
|
|
6676
6709
|
if (opts?.projectId) {
|
|
6677
|
-
baseConditions.push(
|
|
6710
|
+
baseConditions.push(eq14(gaTrafficSnapshots.projectId, opts.projectId));
|
|
6678
6711
|
}
|
|
6679
6712
|
const rows = db.select({
|
|
6680
6713
|
id: gaTrafficSnapshots.id,
|
|
6681
6714
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
6682
6715
|
landingPageNormalized: gaTrafficSnapshots.landingPageNormalized
|
|
6683
|
-
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ?
|
|
6716
|
+
}).from(gaTrafficSnapshots).where(baseConditions.length > 0 ? and11(...baseConditions) : void 0).all();
|
|
6684
6717
|
let updated = 0;
|
|
6685
6718
|
let unchanged = 0;
|
|
6686
6719
|
if (rows.length > 0) {
|
|
@@ -6695,7 +6728,7 @@ function backfillNormalizedPaths(db, opts) {
|
|
|
6695
6728
|
unchanged++;
|
|
6696
6729
|
continue;
|
|
6697
6730
|
}
|
|
6698
|
-
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(
|
|
6731
|
+
tx.update(gaTrafficSnapshots).set({ landingPageNormalized: next }).where(eq14(gaTrafficSnapshots.id, row.id)).run();
|
|
6699
6732
|
updated++;
|
|
6700
6733
|
}
|
|
6701
6734
|
});
|
|
@@ -6709,7 +6742,7 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
6709
6742
|
const projectFilter = opts?.project?.trim();
|
|
6710
6743
|
let projectId;
|
|
6711
6744
|
if (projectFilter) {
|
|
6712
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
6745
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq14(projects.name, projectFilter)).get();
|
|
6713
6746
|
if (!project) {
|
|
6714
6747
|
const result2 = {
|
|
6715
6748
|
project: projectFilter,
|
|
@@ -6746,13 +6779,13 @@ async function backfillNormalizedPathsCommand(opts) {
|
|
|
6746
6779
|
function backfillAiReferralPaths(db, opts) {
|
|
6747
6780
|
const baseConditions = [];
|
|
6748
6781
|
if (opts?.projectId) {
|
|
6749
|
-
baseConditions.push(
|
|
6782
|
+
baseConditions.push(eq14(gaAiReferrals.projectId, opts.projectId));
|
|
6750
6783
|
}
|
|
6751
6784
|
const rows = db.select({
|
|
6752
6785
|
id: gaAiReferrals.id,
|
|
6753
6786
|
landingPage: gaAiReferrals.landingPage,
|
|
6754
6787
|
landingPageNormalized: gaAiReferrals.landingPageNormalized
|
|
6755
|
-
}).from(gaAiReferrals).where(baseConditions.length > 0 ?
|
|
6788
|
+
}).from(gaAiReferrals).where(baseConditions.length > 0 ? and11(...baseConditions) : void 0).all();
|
|
6756
6789
|
let updated = 0;
|
|
6757
6790
|
let unchanged = 0;
|
|
6758
6791
|
if (rows.length > 0) {
|
|
@@ -6767,7 +6800,7 @@ function backfillAiReferralPaths(db, opts) {
|
|
|
6767
6800
|
unchanged++;
|
|
6768
6801
|
continue;
|
|
6769
6802
|
}
|
|
6770
|
-
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(
|
|
6803
|
+
tx.update(gaAiReferrals).set({ landingPageNormalized: next }).where(eq14(gaAiReferrals.id, row.id)).run();
|
|
6771
6804
|
updated++;
|
|
6772
6805
|
}
|
|
6773
6806
|
});
|
|
@@ -6781,7 +6814,7 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
6781
6814
|
const projectFilter = opts?.project?.trim();
|
|
6782
6815
|
let projectId;
|
|
6783
6816
|
if (projectFilter) {
|
|
6784
|
-
const project = db.select({ id: projects.id }).from(projects).where(
|
|
6817
|
+
const project = db.select({ id: projects.id }).from(projects).where(eq14(projects.name, projectFilter)).get();
|
|
6785
6818
|
if (!project) {
|
|
6786
6819
|
const result2 = {
|
|
6787
6820
|
project: projectFilter,
|
|
@@ -6817,10 +6850,10 @@ async function backfillAiReferralPathsCommand(opts) {
|
|
|
6817
6850
|
}
|
|
6818
6851
|
function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
6819
6852
|
const isDryRun = opts?.dryRun === true;
|
|
6820
|
-
const project = db.select().from(projects).where(
|
|
6853
|
+
const project = db.select().from(projects).where(eq14(projects.id, projectId)).get();
|
|
6821
6854
|
if (!project) return { examined: 0, updated: 0, mentioned: 0 };
|
|
6822
|
-
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(
|
|
6823
|
-
const runRows = db.select({ id: runs.id }).from(runs).where(
|
|
6855
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq14(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
6856
|
+
const runRows = db.select({ id: runs.id }).from(runs).where(and11(eq14(runs.kind, RunKinds["answer-visibility"]), eq14(runs.projectId, projectId))).all();
|
|
6824
6857
|
const runIds = runRows.map((r) => r.id);
|
|
6825
6858
|
let examined = 0;
|
|
6826
6859
|
let updated = 0;
|
|
@@ -6892,7 +6925,7 @@ function backfillProjectAnswerMentions(db, projectId, opts) {
|
|
|
6892
6925
|
} else {
|
|
6893
6926
|
db.transaction((tx) => {
|
|
6894
6927
|
for (const update of pendingUpdates) {
|
|
6895
|
-
tx.update(querySnapshots).set(update.patch).where(
|
|
6928
|
+
tx.update(querySnapshots).set(update.patch).where(eq14(querySnapshots.id, update.id)).run();
|
|
6896
6929
|
}
|
|
6897
6930
|
});
|
|
6898
6931
|
updated += pendingUpdates.length;
|
|
@@ -6907,7 +6940,7 @@ async function backfillAnswerMentionsCommand(opts) {
|
|
|
6907
6940
|
migrate(db);
|
|
6908
6941
|
const projectFilter = opts?.project?.trim();
|
|
6909
6942
|
const isDryRun = opts?.dryRun === true;
|
|
6910
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
6943
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq14(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
6911
6944
|
let examined = 0;
|
|
6912
6945
|
let updated = 0;
|
|
6913
6946
|
let wouldUpdate = 0;
|
|
@@ -6967,7 +7000,7 @@ function readStoredGroundingSources(rawResponse) {
|
|
|
6967
7000
|
return result;
|
|
6968
7001
|
}
|
|
6969
7002
|
async function backfillInsightsCommand(project, opts) {
|
|
6970
|
-
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-
|
|
7003
|
+
const { IntelligenceService: IntelligenceService2 } = await import("./intelligence-service-YJKQQYJQ.js");
|
|
6971
7004
|
const config = loadConfig();
|
|
6972
7005
|
const db = createClient(config.database);
|
|
6973
7006
|
migrate(db);
|
|
@@ -7126,7 +7159,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7126
7159
|
const config = loadConfig();
|
|
7127
7160
|
const db = createClient(config.database);
|
|
7128
7161
|
migrate(db);
|
|
7129
|
-
const project = db.select().from(projects).where(
|
|
7162
|
+
const project = db.select().from(projects).where(eq14(projects.name, opts.project)).get();
|
|
7130
7163
|
if (!project) {
|
|
7131
7164
|
throw new Error(`Project "${opts.project}" not found`);
|
|
7132
7165
|
}
|
|
@@ -7137,8 +7170,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7137
7170
|
process.stderr.write(`Recovering orphan snapshot attribution for "${project.name}"${mode}...
|
|
7138
7171
|
`);
|
|
7139
7172
|
}
|
|
7140
|
-
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(
|
|
7141
|
-
|
|
7173
|
+
const events = db.select({ createdAt: auditLog.createdAt, action: auditLog.action, diff: auditLog.diff }).from(auditLog).where(and11(
|
|
7174
|
+
eq14(auditLog.projectId, project.id),
|
|
7142
7175
|
inArray4(auditLog.action, ["keywords.appended", "keywords.deleted", "queries.appended", "queries.deleted", "queries.replaced"])
|
|
7143
7176
|
)).orderBy(auditLog.createdAt).all();
|
|
7144
7177
|
const history = replayQueryAuditLog(events);
|
|
@@ -7146,8 +7179,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7146
7179
|
runId: runs.id,
|
|
7147
7180
|
createdAt: runs.createdAt,
|
|
7148
7181
|
location: runs.location
|
|
7149
|
-
}).from(runs).innerJoin(querySnapshots,
|
|
7150
|
-
|
|
7182
|
+
}).from(runs).innerJoin(querySnapshots, eq14(querySnapshots.runId, runs.id)).where(and11(
|
|
7183
|
+
eq14(runs.projectId, project.id),
|
|
7151
7184
|
isNull(querySnapshots.queryId),
|
|
7152
7185
|
isNull(querySnapshots.queryText)
|
|
7153
7186
|
)).groupBy(runs.id).orderBy(runs.createdAt).all();
|
|
@@ -7169,8 +7202,8 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7169
7202
|
provider: querySnapshots.provider,
|
|
7170
7203
|
createdAt: querySnapshots.createdAt,
|
|
7171
7204
|
answerText: querySnapshots.answerText
|
|
7172
|
-
}).from(querySnapshots).where(
|
|
7173
|
-
|
|
7205
|
+
}).from(querySnapshots).where(and11(
|
|
7206
|
+
eq14(querySnapshots.runId, run.runId),
|
|
7174
7207
|
isNull(querySnapshots.queryId),
|
|
7175
7208
|
isNull(querySnapshots.queryText)
|
|
7176
7209
|
)).orderBy(querySnapshots.provider, querySnapshots.createdAt).all();
|
|
@@ -7236,7 +7269,7 @@ async function backfillSnapshotAttributionCommand(opts) {
|
|
|
7236
7269
|
if (!isDryRun && updates.length > 0) {
|
|
7237
7270
|
db.transaction((tx) => {
|
|
7238
7271
|
for (const u of updates) {
|
|
7239
|
-
tx.update(querySnapshots).set({ queryText: u.queryText }).where(
|
|
7272
|
+
tx.update(querySnapshots).set({ queryText: u.queryText }).where(eq14(querySnapshots.id, u.id)).run();
|
|
7240
7273
|
}
|
|
7241
7274
|
});
|
|
7242
7275
|
}
|
|
@@ -7310,7 +7343,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7310
7343
|
const projectFilter = opts?.project?.trim();
|
|
7311
7344
|
const isDryRun = opts?.dryRun === true;
|
|
7312
7345
|
const isJson = isMachineFormat(opts?.format);
|
|
7313
|
-
const scopedProjects = projectFilter ? db.select().from(projects).where(
|
|
7346
|
+
const scopedProjects = projectFilter ? db.select().from(projects).where(eq14(projects.name, projectFilter)).all() : db.select().from(projects).all();
|
|
7314
7347
|
if (scopedProjects.length === 0) {
|
|
7315
7348
|
if (projectFilter && !isJson) {
|
|
7316
7349
|
process.stderr.write(`No project named "${projectFilter}".
|
|
@@ -7335,8 +7368,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7335
7368
|
dryRun: isDryRun,
|
|
7336
7369
|
byBot: {}
|
|
7337
7370
|
};
|
|
7338
|
-
const unknownCountRow = db.select({ n:
|
|
7339
|
-
|
|
7371
|
+
const unknownCountRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(and11(
|
|
7372
|
+
eq14(rawEventSamples.eventType, "unknown"),
|
|
7340
7373
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7341
7374
|
)).get();
|
|
7342
7375
|
result.unknownBefore = Number(unknownCountRow?.n ?? 0);
|
|
@@ -7348,8 +7381,8 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7348
7381
|
userAgent: rawEventSamples.userAgent,
|
|
7349
7382
|
pathNormalized: rawEventSamples.pathNormalized,
|
|
7350
7383
|
status: rawEventSamples.status
|
|
7351
|
-
}).from(rawEventSamples).where(
|
|
7352
|
-
|
|
7384
|
+
}).from(rawEventSamples).where(and11(
|
|
7385
|
+
eq14(rawEventSamples.eventType, "unknown"),
|
|
7353
7386
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7354
7387
|
)).all();
|
|
7355
7388
|
result.examined = unknownSamples.length;
|
|
@@ -7388,7 +7421,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7388
7421
|
result.reclassified++;
|
|
7389
7422
|
result.byBot[classified.botId] = (result.byBot[classified.botId] ?? 0) + 1;
|
|
7390
7423
|
if (isDryRun) continue;
|
|
7391
|
-
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(
|
|
7424
|
+
db.update(rawEventSamples).set({ eventType: userFetch ? TrafficEventKinds["ai-user-fetch"] : TrafficEventKinds.crawler }).where(eq14(rawEventSamples.id, snap.id)).run();
|
|
7392
7425
|
const tsHour = new Date(snap.ts);
|
|
7393
7426
|
tsHour.setUTCMinutes(0, 0, 0);
|
|
7394
7427
|
if (userFetch) {
|
|
@@ -7416,7 +7449,7 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7416
7449
|
aiUserFetchEventsHourly.status
|
|
7417
7450
|
],
|
|
7418
7451
|
set: {
|
|
7419
|
-
hits:
|
|
7452
|
+
hits: sql5`${aiUserFetchEventsHourly.hits} + 1`,
|
|
7420
7453
|
updatedAt: now
|
|
7421
7454
|
}
|
|
7422
7455
|
}).run();
|
|
@@ -7445,15 +7478,15 @@ async function backfillTrafficClassificationCommand(opts) {
|
|
|
7445
7478
|
crawlerEventsHourly.status
|
|
7446
7479
|
],
|
|
7447
7480
|
set: {
|
|
7448
|
-
hits:
|
|
7481
|
+
hits: sql5`${crawlerEventsHourly.hits} + 1`,
|
|
7449
7482
|
updatedAt: now
|
|
7450
7483
|
}
|
|
7451
7484
|
}).run();
|
|
7452
7485
|
}
|
|
7453
7486
|
}
|
|
7454
7487
|
if (!isDryRun) {
|
|
7455
|
-
const afterRow = db.select({ n:
|
|
7456
|
-
|
|
7488
|
+
const afterRow = db.select({ n: sql5`count(*)` }).from(rawEventSamples).where(and11(
|
|
7489
|
+
eq14(rawEventSamples.eventType, "unknown"),
|
|
7457
7490
|
inArray4(rawEventSamples.projectId, projectIds)
|
|
7458
7491
|
)).get();
|
|
7459
7492
|
result.unknownAfter = Number(afterRow?.n ?? 0);
|
|
@@ -7488,7 +7521,7 @@ No DB writes performed. Re-run without --dry-run to apply.`);
|
|
|
7488
7521
|
}
|
|
7489
7522
|
|
|
7490
7523
|
// src/commands/skills.ts
|
|
7491
|
-
import
|
|
7524
|
+
import crypto16 from "crypto";
|
|
7492
7525
|
import fs4 from "fs";
|
|
7493
7526
|
import os4 from "os";
|
|
7494
7527
|
import path5 from "path";
|
|
@@ -7543,7 +7576,7 @@ function walkRelative(dir, prefix = "") {
|
|
|
7543
7576
|
return out.sort();
|
|
7544
7577
|
}
|
|
7545
7578
|
function sha256File(filePath) {
|
|
7546
|
-
return
|
|
7579
|
+
return crypto16.createHash("sha256").update(fs4.readFileSync(filePath)).digest("hex");
|
|
7547
7580
|
}
|
|
7548
7581
|
function readSkillManifest(skillDir) {
|
|
7549
7582
|
try {
|
|
@@ -7885,9 +7918,9 @@ var ProviderRegistry = class {
|
|
|
7885
7918
|
};
|
|
7886
7919
|
|
|
7887
7920
|
// src/scheduler.ts
|
|
7888
|
-
import
|
|
7921
|
+
import crypto17 from "crypto";
|
|
7889
7922
|
import cron from "node-cron";
|
|
7890
|
-
import { and as
|
|
7923
|
+
import { and as and12, eq as eq15, inArray as inArray5 } from "drizzle-orm";
|
|
7891
7924
|
var log14 = createLogger("Scheduler");
|
|
7892
7925
|
function taskKey(projectId, kind) {
|
|
7893
7926
|
return `${projectId}::${kind}`;
|
|
@@ -7902,7 +7935,7 @@ var Scheduler = class {
|
|
|
7902
7935
|
}
|
|
7903
7936
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
7904
7937
|
start() {
|
|
7905
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
7938
|
+
const allSchedules = this.db.select().from(schedules).where(eq15(schedules.enabled, true)).all();
|
|
7906
7939
|
for (const schedule of allSchedules) {
|
|
7907
7940
|
const missedRunAt = schedule.nextRunAt;
|
|
7908
7941
|
this.registerCronTask(schedule);
|
|
@@ -7932,7 +7965,7 @@ var Scheduler = class {
|
|
|
7932
7965
|
this.stopTask(key, existing, "Stopped");
|
|
7933
7966
|
this.tasks.delete(key);
|
|
7934
7967
|
}
|
|
7935
|
-
const schedule = this.db.select().from(schedules).where(
|
|
7968
|
+
const schedule = this.db.select().from(schedules).where(and12(eq15(schedules.projectId, projectId), eq15(schedules.kind, kind))).get();
|
|
7936
7969
|
if (schedule && schedule.enabled) {
|
|
7937
7970
|
this.registerCronTask(schedule);
|
|
7938
7971
|
}
|
|
@@ -7973,21 +8006,21 @@ var Scheduler = class {
|
|
|
7973
8006
|
this.db.update(schedules).set({
|
|
7974
8007
|
nextRunAt: nextRunFromCron(cronExpr, timezone),
|
|
7975
8008
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
7976
|
-
}).where(
|
|
8009
|
+
}).where(eq15(schedules.id, scheduleId)).run();
|
|
7977
8010
|
const label = schedule.preset ?? cronExpr;
|
|
7978
8011
|
log14.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
7979
8012
|
}
|
|
7980
8013
|
triggerRun(scheduleId, projectId, kind) {
|
|
7981
8014
|
try {
|
|
7982
8015
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7983
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
8016
|
+
const currentSchedule = this.db.select().from(schedules).where(eq15(schedules.id, scheduleId)).get();
|
|
7984
8017
|
if (!currentSchedule || !currentSchedule.enabled) {
|
|
7985
8018
|
log14.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
7986
8019
|
this.remove(projectId, kind);
|
|
7987
8020
|
return;
|
|
7988
8021
|
}
|
|
7989
8022
|
const nextRunAt = nextRunFromCron(currentSchedule.cronExpr, currentSchedule.timezone);
|
|
7990
|
-
const project = this.db.select().from(projects).where(
|
|
8023
|
+
const project = this.db.select().from(projects).where(eq15(projects.id, projectId)).get();
|
|
7991
8024
|
if (!project) {
|
|
7992
8025
|
log14.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
7993
8026
|
this.remove(projectId, kind);
|
|
@@ -8007,7 +8040,7 @@ var Scheduler = class {
|
|
|
8007
8040
|
lastRunAt: now,
|
|
8008
8041
|
nextRunAt,
|
|
8009
8042
|
updatedAt: now
|
|
8010
|
-
}).where(
|
|
8043
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8011
8044
|
log14.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
8012
8045
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
8013
8046
|
return;
|
|
@@ -8017,7 +8050,7 @@ var Scheduler = class {
|
|
|
8017
8050
|
log14.warn("gbp-sync.no-callback", { scheduleId, projectId, msg: "host did not register onGbpSyncRequested" });
|
|
8018
8051
|
return;
|
|
8019
8052
|
}
|
|
8020
|
-
const runId2 =
|
|
8053
|
+
const runId2 = crypto17.randomUUID();
|
|
8021
8054
|
this.db.insert(runs).values({
|
|
8022
8055
|
id: runId2,
|
|
8023
8056
|
projectId,
|
|
@@ -8030,7 +8063,7 @@ var Scheduler = class {
|
|
|
8030
8063
|
lastRunAt: now,
|
|
8031
8064
|
nextRunAt,
|
|
8032
8065
|
updatedAt: now
|
|
8033
|
-
}).where(
|
|
8066
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8034
8067
|
log14.info("gbp-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8035
8068
|
this.callbacks.onGbpSyncRequested(runId2, projectId);
|
|
8036
8069
|
return;
|
|
@@ -8040,17 +8073,17 @@ var Scheduler = class {
|
|
|
8040
8073
|
log14.warn("ads-sync.no-callback", { scheduleId, projectId, msg: "host did not register onAdsSyncRequested" });
|
|
8041
8074
|
return;
|
|
8042
8075
|
}
|
|
8043
|
-
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(
|
|
8044
|
-
|
|
8045
|
-
|
|
8076
|
+
const activeAdsRun = this.db.select({ id: runs.id }).from(runs).where(and12(
|
|
8077
|
+
eq15(runs.projectId, projectId),
|
|
8078
|
+
eq15(runs.kind, RunKinds["ads-sync"]),
|
|
8046
8079
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8047
8080
|
)).get();
|
|
8048
8081
|
if (activeAdsRun) {
|
|
8049
8082
|
log14.info("ads-sync.skipped-active", { projectName: project.name, activeRunId: activeAdsRun.id });
|
|
8050
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
8083
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8051
8084
|
return;
|
|
8052
8085
|
}
|
|
8053
|
-
const runId2 =
|
|
8086
|
+
const runId2 = crypto17.randomUUID();
|
|
8054
8087
|
this.db.insert(runs).values({
|
|
8055
8088
|
id: runId2,
|
|
8056
8089
|
projectId,
|
|
@@ -8063,7 +8096,7 @@ var Scheduler = class {
|
|
|
8063
8096
|
lastRunAt: now,
|
|
8064
8097
|
nextRunAt,
|
|
8065
8098
|
updatedAt: now
|
|
8066
|
-
}).where(
|
|
8099
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8067
8100
|
log14.info("ads-sync.triggered", { runId: runId2, projectName: project.name });
|
|
8068
8101
|
this.callbacks.onAdsSyncRequested(runId2, projectId);
|
|
8069
8102
|
return;
|
|
@@ -8077,7 +8110,7 @@ var Scheduler = class {
|
|
|
8077
8110
|
lastRunAt: now,
|
|
8078
8111
|
nextRunAt,
|
|
8079
8112
|
updatedAt: now
|
|
8080
|
-
}).where(
|
|
8113
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8081
8114
|
log14.info("data-refresh.triggered", { projectName: project.name });
|
|
8082
8115
|
this.callbacks.onDataRefreshRequested(project.name);
|
|
8083
8116
|
return;
|
|
@@ -8091,7 +8124,7 @@ var Scheduler = class {
|
|
|
8091
8124
|
lastRunAt: now,
|
|
8092
8125
|
nextRunAt,
|
|
8093
8126
|
updatedAt: now
|
|
8094
|
-
}).where(
|
|
8127
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8095
8128
|
log14.info("backlinks-sync.triggered", { projectName: project.name });
|
|
8096
8129
|
this.callbacks.onBacklinksSyncRequested(project.name);
|
|
8097
8130
|
return;
|
|
@@ -8101,17 +8134,17 @@ var Scheduler = class {
|
|
|
8101
8134
|
log14.warn("site-audit.no-callback", { scheduleId, projectId, msg: "host did not register onSiteAuditRequested" });
|
|
8102
8135
|
return;
|
|
8103
8136
|
}
|
|
8104
|
-
const active = this.db.select({ id: runs.id }).from(runs).where(
|
|
8105
|
-
|
|
8106
|
-
|
|
8137
|
+
const active = this.db.select({ id: runs.id }).from(runs).where(and12(
|
|
8138
|
+
eq15(runs.projectId, projectId),
|
|
8139
|
+
eq15(runs.kind, RunKinds["site-audit"]),
|
|
8107
8140
|
inArray5(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
8108
8141
|
)).get();
|
|
8109
8142
|
if (active) {
|
|
8110
8143
|
log14.info("site-audit.skipped-active", { projectName: project.name, activeRunId: active.id });
|
|
8111
|
-
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(
|
|
8144
|
+
this.db.update(schedules).set({ nextRunAt, updatedAt: now }).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8112
8145
|
return;
|
|
8113
8146
|
}
|
|
8114
|
-
const runId2 =
|
|
8147
|
+
const runId2 = crypto17.randomUUID();
|
|
8115
8148
|
this.db.insert(runs).values({
|
|
8116
8149
|
id: runId2,
|
|
8117
8150
|
projectId,
|
|
@@ -8124,7 +8157,7 @@ var Scheduler = class {
|
|
|
8124
8157
|
lastRunAt: now,
|
|
8125
8158
|
nextRunAt,
|
|
8126
8159
|
updatedAt: now
|
|
8127
|
-
}).where(
|
|
8160
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8128
8161
|
log14.info("site-audit.triggered", { runId: runId2, projectName: project.name });
|
|
8129
8162
|
this.callbacks.onSiteAuditRequested(runId2, projectId);
|
|
8130
8163
|
return;
|
|
@@ -8152,7 +8185,7 @@ var Scheduler = class {
|
|
|
8152
8185
|
this.db.update(schedules).set({
|
|
8153
8186
|
nextRunAt,
|
|
8154
8187
|
updatedAt: now
|
|
8155
|
-
}).where(
|
|
8188
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8156
8189
|
return;
|
|
8157
8190
|
}
|
|
8158
8191
|
const runId = queueResult.runId;
|
|
@@ -8160,7 +8193,7 @@ var Scheduler = class {
|
|
|
8160
8193
|
lastRunAt: now,
|
|
8161
8194
|
nextRunAt,
|
|
8162
8195
|
updatedAt: now
|
|
8163
|
-
}).where(
|
|
8196
|
+
}).where(eq15(schedules.id, currentSchedule.id)).run();
|
|
8164
8197
|
const scheduleProviders = currentSchedule.providers;
|
|
8165
8198
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
8166
8199
|
log14.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -8195,8 +8228,8 @@ async function refreshAllIntegrations(client, projectName) {
|
|
|
8195
8228
|
}
|
|
8196
8229
|
|
|
8197
8230
|
// src/notifier.ts
|
|
8198
|
-
import { eq as
|
|
8199
|
-
import
|
|
8231
|
+
import { eq as eq16, desc as desc5, and as and13, inArray as inArray6, or } from "drizzle-orm";
|
|
8232
|
+
import crypto18 from "crypto";
|
|
8200
8233
|
var log16 = createLogger("Notifier");
|
|
8201
8234
|
var Notifier = class {
|
|
8202
8235
|
db;
|
|
@@ -8208,18 +8241,18 @@ var Notifier = class {
|
|
|
8208
8241
|
/** Called after a run completes (success, partial, or failed). */
|
|
8209
8242
|
async onRunCompleted(runId, projectId) {
|
|
8210
8243
|
log16.info("run.completed", { runId, projectId });
|
|
8211
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8244
|
+
const notifs = this.db.select().from(notifications).where(eq16(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8212
8245
|
if (notifs.length === 0) {
|
|
8213
8246
|
log16.info("notifications.none-enabled", { projectId });
|
|
8214
8247
|
return;
|
|
8215
8248
|
}
|
|
8216
8249
|
log16.info("notifications.found", { projectId, count: notifs.length });
|
|
8217
|
-
const run = this.db.select().from(runs).where(
|
|
8250
|
+
const run = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8218
8251
|
if (!run) {
|
|
8219
8252
|
log16.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
8220
8253
|
return;
|
|
8221
8254
|
}
|
|
8222
|
-
const project = this.db.select().from(projects).where(
|
|
8255
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8223
8256
|
if (!project) {
|
|
8224
8257
|
log16.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
8225
8258
|
return;
|
|
@@ -8266,11 +8299,11 @@ var Notifier = class {
|
|
|
8266
8299
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
8267
8300
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
8268
8301
|
if (insightEvents.length === 0) return;
|
|
8269
|
-
const notifs = this.db.select().from(notifications).where(
|
|
8302
|
+
const notifs = this.db.select().from(notifications).where(eq16(notifications.projectId, projectId)).all().filter((n) => n.enabled);
|
|
8270
8303
|
if (notifs.length === 0) return;
|
|
8271
|
-
const run = this.db.select().from(runs).where(
|
|
8304
|
+
const run = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8272
8305
|
if (!run) return;
|
|
8273
|
-
const project = this.db.select().from(projects).where(
|
|
8306
|
+
const project = this.db.select().from(projects).where(eq16(projects.id, projectId)).get();
|
|
8274
8307
|
if (!project) return;
|
|
8275
8308
|
for (const notif of notifs) {
|
|
8276
8309
|
const config = notif.config;
|
|
@@ -8300,12 +8333,12 @@ var Notifier = class {
|
|
|
8300
8333
|
}
|
|
8301
8334
|
}
|
|
8302
8335
|
computeTransitions(runId, projectId) {
|
|
8303
|
-
const thisRun = this.db.select().from(runs).where(
|
|
8336
|
+
const thisRun = this.db.select().from(runs).where(eq16(runs.id, runId)).get();
|
|
8304
8337
|
if (!thisRun) return [];
|
|
8305
|
-
const groupSiblings = this.db.select().from(runs).where(
|
|
8306
|
-
|
|
8307
|
-
|
|
8308
|
-
|
|
8338
|
+
const groupSiblings = this.db.select().from(runs).where(and13(
|
|
8339
|
+
eq16(runs.projectId, projectId),
|
|
8340
|
+
eq16(runs.kind, thisRun.kind),
|
|
8341
|
+
eq16(runs.createdAt, thisRun.createdAt)
|
|
8309
8342
|
)).all();
|
|
8310
8343
|
const stillPending = groupSiblings.some((r) => r.status === "queued" || r.status === "running");
|
|
8311
8344
|
if (stillPending) return [];
|
|
@@ -8321,17 +8354,17 @@ var Notifier = class {
|
|
|
8321
8354
|
return candidate.id > best.id ? candidate : best;
|
|
8322
8355
|
});
|
|
8323
8356
|
if (winner.id !== runId) return [];
|
|
8324
|
-
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
8357
|
+
const projectLocations = this.db.select({ locations: projects.locations }).from(projects).where(eq16(projects.id, projectId)).get();
|
|
8325
8358
|
const locationCount = Math.max(
|
|
8326
8359
|
1,
|
|
8327
8360
|
(projectLocations?.locations ?? []).length
|
|
8328
8361
|
);
|
|
8329
8362
|
const RECENT_FETCH_LIMIT = Math.max(8, locationCount * 4);
|
|
8330
8363
|
const recentRuns = this.db.select().from(runs).where(
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
8334
|
-
or(
|
|
8364
|
+
and13(
|
|
8365
|
+
eq16(runs.projectId, projectId),
|
|
8366
|
+
eq16(runs.kind, thisRun.kind),
|
|
8367
|
+
or(eq16(runs.status, "completed"), eq16(runs.status, "partial"))
|
|
8335
8368
|
)
|
|
8336
8369
|
).orderBy(desc5(runs.createdAt), desc5(runs.id)).limit(RECENT_FETCH_LIMIT).all();
|
|
8337
8370
|
const groups = groupRunsByCreatedAt(recentRuns);
|
|
@@ -8348,7 +8381,7 @@ var Notifier = class {
|
|
|
8348
8381
|
provider: querySnapshots.provider,
|
|
8349
8382
|
location: querySnapshots.location,
|
|
8350
8383
|
citationState: querySnapshots.citationState
|
|
8351
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
8384
|
+
}).from(querySnapshots).leftJoin(queries, eq16(querySnapshots.queryId, queries.id)).where(inArray6(querySnapshots.runId, currentRunIds)).all();
|
|
8352
8385
|
const previousSnapshots = this.db.select({
|
|
8353
8386
|
queryId: querySnapshots.queryId,
|
|
8354
8387
|
provider: querySnapshots.provider,
|
|
@@ -8415,7 +8448,7 @@ var Notifier = class {
|
|
|
8415
8448
|
}
|
|
8416
8449
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
8417
8450
|
this.db.insert(auditLog).values({
|
|
8418
|
-
id:
|
|
8451
|
+
id: crypto18.randomUUID(),
|
|
8419
8452
|
projectId,
|
|
8420
8453
|
actor: "scheduler",
|
|
8421
8454
|
action: `notification.${status}`,
|
|
@@ -8428,7 +8461,7 @@ var Notifier = class {
|
|
|
8428
8461
|
};
|
|
8429
8462
|
|
|
8430
8463
|
// src/run-coordinator.ts
|
|
8431
|
-
import { eq as
|
|
8464
|
+
import { eq as eq17 } from "drizzle-orm";
|
|
8432
8465
|
var log17 = createLogger("RunCoordinator");
|
|
8433
8466
|
var RunCoordinator = class {
|
|
8434
8467
|
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
@@ -8444,7 +8477,7 @@ var RunCoordinator = class {
|
|
|
8444
8477
|
onInsightsGenerated;
|
|
8445
8478
|
onAeroEvent;
|
|
8446
8479
|
async onRunCompleted(runId, projectId) {
|
|
8447
|
-
const runRow = this.db.select().from(runs).where(
|
|
8480
|
+
const runRow = this.db.select().from(runs).where(eq17(runs.id, runId)).get();
|
|
8448
8481
|
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
8449
8482
|
if (runRow?.trigger === RunTriggers.probe) {
|
|
8450
8483
|
log17.info("probe.skip-side-effects", { runId, projectId, kind });
|
|
@@ -8520,7 +8553,7 @@ var RunCoordinator = class {
|
|
|
8520
8553
|
* so the Aero queue is never starved of a follow-up.
|
|
8521
8554
|
*/
|
|
8522
8555
|
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
8523
|
-
const session = this.db.select().from(discoverySessions).where(
|
|
8556
|
+
const session = this.db.select().from(discoverySessions).where(eq17(discoverySessions.runId, runId)).get();
|
|
8524
8557
|
const competitorMap = session ? session.competitorMap : [];
|
|
8525
8558
|
return {
|
|
8526
8559
|
kind: RunKinds["aeo-discover-probe"],
|
|
@@ -8562,8 +8595,8 @@ function analysisResultFromInsights(insights2) {
|
|
|
8562
8595
|
}
|
|
8563
8596
|
|
|
8564
8597
|
// src/agent/session-registry.ts
|
|
8565
|
-
import
|
|
8566
|
-
import { eq as
|
|
8598
|
+
import crypto22 from "crypto";
|
|
8599
|
+
import { eq as eq19 } from "drizzle-orm";
|
|
8567
8600
|
|
|
8568
8601
|
// src/agent/session.ts
|
|
8569
8602
|
import fs7 from "fs";
|
|
@@ -9301,7 +9334,7 @@ function buildAeroStateTools(ctx, opts = {}) {
|
|
|
9301
9334
|
}
|
|
9302
9335
|
|
|
9303
9336
|
// src/agent/llm-usage.ts
|
|
9304
|
-
import
|
|
9337
|
+
import crypto19 from "crypto";
|
|
9305
9338
|
var AeroLlmUsageFeatures = {
|
|
9306
9339
|
turn: "aero.turn"
|
|
9307
9340
|
};
|
|
@@ -9320,7 +9353,7 @@ function recordLlmUsageEvent(args) {
|
|
|
9320
9353
|
const usage = args.message.usage;
|
|
9321
9354
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9322
9355
|
args.db.insert(llmUsageEvents).values({
|
|
9323
|
-
id:
|
|
9356
|
+
id: crypto19.randomUUID(),
|
|
9324
9357
|
projectId: args.projectId,
|
|
9325
9358
|
runId: args.runId,
|
|
9326
9359
|
agentSessionId: args.agentSessionId,
|
|
@@ -9396,7 +9429,7 @@ function splitAeroAnthropicSystemCachePayload(payload, model) {
|
|
|
9396
9429
|
}
|
|
9397
9430
|
|
|
9398
9431
|
// src/agent/tool-usage.ts
|
|
9399
|
-
import
|
|
9432
|
+
import crypto20 from "crypto";
|
|
9400
9433
|
var AeroToolEventStatuses = {
|
|
9401
9434
|
success: "success",
|
|
9402
9435
|
error: "error"
|
|
@@ -9459,7 +9492,7 @@ function createAeroToolUsageHooks(args) {
|
|
|
9459
9492
|
function recordAgentToolEvent(args) {
|
|
9460
9493
|
try {
|
|
9461
9494
|
args.db.insert(agentToolEvents).values({
|
|
9462
|
-
id:
|
|
9495
|
+
id: crypto20.randomUUID(),
|
|
9463
9496
|
projectId: args.projectId,
|
|
9464
9497
|
agentSessionId: args.agentSessionId,
|
|
9465
9498
|
toolCallId: args.toolCallId,
|
|
@@ -9686,8 +9719,8 @@ async function loadExternalMcpTools(servers, opts = {}) {
|
|
|
9686
9719
|
}
|
|
9687
9720
|
|
|
9688
9721
|
// src/agent/memory-store.ts
|
|
9689
|
-
import
|
|
9690
|
-
import { and as
|
|
9722
|
+
import crypto21 from "crypto";
|
|
9723
|
+
import { and as and14, desc as desc6, eq as eq18, like, sql as sql6 } from "drizzle-orm";
|
|
9691
9724
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
9692
9725
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
9693
9726
|
function rowToDto(row) {
|
|
@@ -9701,7 +9734,7 @@ function rowToDto(row) {
|
|
|
9701
9734
|
};
|
|
9702
9735
|
}
|
|
9703
9736
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
9704
|
-
const query = db.select().from(agentMemory).where(
|
|
9737
|
+
const query = db.select().from(agentMemory).where(eq18(agentMemory.projectId, projectId)).orderBy(desc6(agentMemory.updatedAt));
|
|
9705
9738
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
9706
9739
|
return rows.map(rowToDto);
|
|
9707
9740
|
}
|
|
@@ -9715,7 +9748,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
9715
9748
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
9716
9749
|
}
|
|
9717
9750
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9718
|
-
const id =
|
|
9751
|
+
const id = crypto21.randomUUID();
|
|
9719
9752
|
db.insert(agentMemory).values({
|
|
9720
9753
|
id,
|
|
9721
9754
|
projectId: args.projectId,
|
|
@@ -9732,12 +9765,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
9732
9765
|
updatedAt: now
|
|
9733
9766
|
}
|
|
9734
9767
|
}).run();
|
|
9735
|
-
const row = db.select().from(agentMemory).where(
|
|
9768
|
+
const row = db.select().from(agentMemory).where(and14(eq18(agentMemory.projectId, args.projectId), eq18(agentMemory.key, args.key))).get();
|
|
9736
9769
|
if (!row) throw new Error("memory upsert produced no row");
|
|
9737
9770
|
return rowToDto(row);
|
|
9738
9771
|
}
|
|
9739
9772
|
function deleteMemoryEntry(db, projectId, key) {
|
|
9740
|
-
const result = db.delete(agentMemory).where(
|
|
9773
|
+
const result = db.delete(agentMemory).where(and14(eq18(agentMemory.projectId, projectId), eq18(agentMemory.key, key))).run();
|
|
9741
9774
|
const changes = result.changes ?? 0;
|
|
9742
9775
|
return changes > 0;
|
|
9743
9776
|
}
|
|
@@ -9752,7 +9785,7 @@ function writeCompactionNote(db, args) {
|
|
|
9752
9785
|
}
|
|
9753
9786
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9754
9787
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
9755
|
-
const id =
|
|
9788
|
+
const id = crypto21.randomUUID();
|
|
9756
9789
|
let inserted;
|
|
9757
9790
|
db.transaction((tx) => {
|
|
9758
9791
|
tx.insert(agentMemory).values({
|
|
@@ -9766,16 +9799,16 @@ function writeCompactionNote(db, args) {
|
|
|
9766
9799
|
}).run();
|
|
9767
9800
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
9768
9801
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
9769
|
-
|
|
9770
|
-
|
|
9802
|
+
and14(
|
|
9803
|
+
eq18(agentMemory.projectId, args.projectId),
|
|
9771
9804
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
9772
9805
|
)
|
|
9773
9806
|
).orderBy(desc6(agentMemory.updatedAt)).all();
|
|
9774
9807
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
9775
9808
|
if (stale.length > 0) {
|
|
9776
|
-
tx.delete(agentMemory).where(
|
|
9809
|
+
tx.delete(agentMemory).where(sql6`${agentMemory.id} IN (${sql6.join(stale.map((s) => sql6`${s}`), sql6`, `)})`).run();
|
|
9777
9810
|
}
|
|
9778
|
-
const row = tx.select().from(agentMemory).where(
|
|
9811
|
+
const row = tx.select().from(agentMemory).where(and14(eq18(agentMemory.projectId, args.projectId), eq18(agentMemory.key, key))).get();
|
|
9779
9812
|
if (row) inserted = rowToDto(row);
|
|
9780
9813
|
});
|
|
9781
9814
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -10002,7 +10035,7 @@ var SessionRegistry = class {
|
|
|
10002
10035
|
modelProvider: effectiveProvider,
|
|
10003
10036
|
modelId: effectiveModelId,
|
|
10004
10037
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10005
|
-
}).where(
|
|
10038
|
+
}).where(eq19(agentSessions.projectId, projectId)).run();
|
|
10006
10039
|
}
|
|
10007
10040
|
const agent2 = createAeroSession({
|
|
10008
10041
|
projectName,
|
|
@@ -10031,7 +10064,7 @@ var SessionRegistry = class {
|
|
|
10031
10064
|
}
|
|
10032
10065
|
const { provider, modelId } = resolveSessionProviderAndModel(this.opts.config, preferences);
|
|
10033
10066
|
const systemPrompt = loadAeroSystemPrompt();
|
|
10034
|
-
const sessionId =
|
|
10067
|
+
const sessionId = crypto22.randomUUID();
|
|
10035
10068
|
const agent = createAeroSession({
|
|
10036
10069
|
projectName,
|
|
10037
10070
|
client: this.opts.client,
|
|
@@ -10236,7 +10269,7 @@ ${lines.join("\n")}
|
|
|
10236
10269
|
modelProvider: nextProvider,
|
|
10237
10270
|
modelId: nextModelId,
|
|
10238
10271
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
10239
|
-
}).where(
|
|
10272
|
+
}).where(eq19(agentSessions.projectId, projectId)).run();
|
|
10240
10273
|
}
|
|
10241
10274
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
10242
10275
|
save(projectName) {
|
|
@@ -10400,17 +10433,17 @@ ${lines.join("\n")}
|
|
|
10400
10433
|
return id;
|
|
10401
10434
|
}
|
|
10402
10435
|
tryResolveProjectId(projectName) {
|
|
10403
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
10436
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq19(projects.name, projectName)).get();
|
|
10404
10437
|
return row?.id;
|
|
10405
10438
|
}
|
|
10406
10439
|
loadRow(projectId) {
|
|
10407
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
10440
|
+
const row = this.opts.db.select().from(agentSessions).where(eq19(agentSessions.projectId, projectId)).get();
|
|
10408
10441
|
return row ?? null;
|
|
10409
10442
|
}
|
|
10410
10443
|
insertRow(params) {
|
|
10411
10444
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10412
10445
|
this.opts.db.insert(agentSessions).values({
|
|
10413
|
-
id: params.id ??
|
|
10446
|
+
id: params.id ?? crypto22.randomUUID(),
|
|
10414
10447
|
projectId: params.projectId,
|
|
10415
10448
|
systemPrompt: params.systemPrompt,
|
|
10416
10449
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -10423,14 +10456,14 @@ ${lines.join("\n")}
|
|
|
10423
10456
|
}
|
|
10424
10457
|
updateRow(projectId, patch) {
|
|
10425
10458
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
10426
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
10459
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq19(agentSessions.projectId, projectId)).run();
|
|
10427
10460
|
}
|
|
10428
10461
|
};
|
|
10429
10462
|
|
|
10430
10463
|
// src/agent/agent-routes.ts
|
|
10431
|
-
import { eq as
|
|
10464
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
10432
10465
|
function resolveProject(db, name) {
|
|
10433
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
10466
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq20(projects.name, name)).get();
|
|
10434
10467
|
if (!row) throw notFound("project", name);
|
|
10435
10468
|
return row;
|
|
10436
10469
|
}
|
|
@@ -10439,7 +10472,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
10439
10472
|
"/projects/:name/agent/transcript",
|
|
10440
10473
|
async (request) => {
|
|
10441
10474
|
const project = resolveProject(opts.db, request.params.name);
|
|
10442
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
10475
|
+
const row = opts.db.select().from(agentSessions).where(eq20(agentSessions.projectId, project.id)).get();
|
|
10443
10476
|
if (!row) {
|
|
10444
10477
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
10445
10478
|
}
|
|
@@ -10463,7 +10496,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
10463
10496
|
async (request) => {
|
|
10464
10497
|
const project = resolveProject(opts.db, request.params.name);
|
|
10465
10498
|
opts.sessionRegistry.reset(project.name);
|
|
10466
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10499
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(agentSessions.projectId, project.id)).run();
|
|
10467
10500
|
return { status: "reset" };
|
|
10468
10501
|
}
|
|
10469
10502
|
);
|
|
@@ -11502,6 +11535,117 @@ function clipText(value, length) {
|
|
|
11502
11535
|
return `${value.slice(0, length - 3)}...`;
|
|
11503
11536
|
}
|
|
11504
11537
|
|
|
11538
|
+
// src/research-runner.ts
|
|
11539
|
+
import { and as and15, eq as eq21, inArray as inArray7, sql as sql7 } from "drizzle-orm";
|
|
11540
|
+
var unfinishedResearchQueryStatuses = [ResearchQueryStatuses.queued, ResearchQueryStatuses.running];
|
|
11541
|
+
function finalResearchRunStatus(completed, failed) {
|
|
11542
|
+
if (failed === 0) return ResearchRunStatuses.completed;
|
|
11543
|
+
return completed > 0 ? ResearchRunStatuses.partial : ResearchRunStatuses.failed;
|
|
11544
|
+
}
|
|
11545
|
+
async function executeResearchRun(db, registry, runId, projectId) {
|
|
11546
|
+
const run = db.select().from(researchRuns).where(and15(eq21(researchRuns.id, runId), eq21(researchRuns.projectId, projectId))).get();
|
|
11547
|
+
if (!run || run.status !== ResearchRunStatuses.queued) return;
|
|
11548
|
+
const project = db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
11549
|
+
if (!project) return;
|
|
11550
|
+
const provider = registry.get(run.provider);
|
|
11551
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11552
|
+
const claim = db.update(researchRuns).set({ status: ResearchRunStatuses.running, startedAt: now }).where(and15(eq21(researchRuns.id, runId), eq21(researchRuns.status, ResearchRunStatuses.queued))).run();
|
|
11553
|
+
if (claim.changes !== 1) return;
|
|
11554
|
+
let reserved = 0;
|
|
11555
|
+
let dispatched = 0;
|
|
11556
|
+
let reservation;
|
|
11557
|
+
let fatalError;
|
|
11558
|
+
const workerPersistenceErrors = [];
|
|
11559
|
+
try {
|
|
11560
|
+
if (!provider || isBrowserProvider(run.provider)) {
|
|
11561
|
+
throw new Error("Configured API provider is unavailable.");
|
|
11562
|
+
}
|
|
11563
|
+
const period = getCurrentUsageDay();
|
|
11564
|
+
const scope = `${projectId}:${run.provider}`;
|
|
11565
|
+
const quota = reserveDailyQueryQuota(db, {
|
|
11566
|
+
scope,
|
|
11567
|
+
period,
|
|
11568
|
+
count: run.totalQueries,
|
|
11569
|
+
limit: provider.config.quotaPolicy.maxRequestsPerDay
|
|
11570
|
+
});
|
|
11571
|
+
if (!quota.reserved) {
|
|
11572
|
+
throw new Error(`Daily quota exceeded for ${run.provider}: ${quota.used} queries used today, limit is ${provider.config.quotaPolicy.maxRequestsPerDay}. This batch needs ${run.totalQueries} more.`);
|
|
11573
|
+
}
|
|
11574
|
+
reserved = run.totalQueries;
|
|
11575
|
+
reservation = { scope, period };
|
|
11576
|
+
const rows = db.select().from(researchRunQueries).where(eq21(researchRunQueries.researchRunId, runId)).orderBy(researchRunQueries.position).all();
|
|
11577
|
+
const competitorDomains = db.select({ domain: competitors.domain }).from(competitors).where(eq21(competitors.projectId, projectId)).all().map((row) => row.domain);
|
|
11578
|
+
const domains = effectiveDomains(project);
|
|
11579
|
+
const brands = effectiveBrandNames(project);
|
|
11580
|
+
const config = { ...provider.config, model: run.resolvedModel };
|
|
11581
|
+
const gate = new ProviderExecutionGate(provider.config.quotaPolicy.maxConcurrency, provider.config.quotaPolicy.maxRequestsPerMinute);
|
|
11582
|
+
await mapWithConcurrency(rows, Math.max(1, provider.config.quotaPolicy.maxConcurrency), async (row) => {
|
|
11583
|
+
try {
|
|
11584
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11585
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.running, startedAt }).where(and15(eq21(researchRunQueries.id, row.id), eq21(researchRunQueries.status, ResearchQueryStatuses.queued))).run();
|
|
11586
|
+
const raw = await gate.run(async () => {
|
|
11587
|
+
dispatched++;
|
|
11588
|
+
return provider.adapter.executeTrackedQuery({ query: row.queryText, canonicalDomains: domains, competitorDomains, ...run.location ? { location: run.location } : {} }, config);
|
|
11589
|
+
});
|
|
11590
|
+
const normalized = provider.adapter.normalizeResult(raw);
|
|
11591
|
+
const completed = db.update(researchRunQueries).set({
|
|
11592
|
+
status: ResearchQueryStatuses.completed,
|
|
11593
|
+
servedModel: raw.servedModel ?? null,
|
|
11594
|
+
answerText: normalized.answerText,
|
|
11595
|
+
groundingSources: normalized.groundingSources,
|
|
11596
|
+
citedDomains: normalized.citedDomains,
|
|
11597
|
+
searchQueries: normalized.searchQueries,
|
|
11598
|
+
answerMentioned: determineAnswerMentioned(normalized.answerText, brands, domains),
|
|
11599
|
+
citationState: determineCitationState(normalized, domains),
|
|
11600
|
+
rawResponse: raw.rawResponse,
|
|
11601
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11602
|
+
}).where(and15(eq21(researchRunQueries.id, row.id), eq21(researchRunQueries.status, ResearchQueryStatuses.running))).run();
|
|
11603
|
+
if (completed.changes === 1) incrementResearchProgress(db, runId, "completedQueries");
|
|
11604
|
+
} catch (error) {
|
|
11605
|
+
try {
|
|
11606
|
+
markResearchQueryFailed(db, runId, row.id, error);
|
|
11607
|
+
} catch (persistenceError) {
|
|
11608
|
+
workerPersistenceErrors.push(persistenceError instanceof Error ? persistenceError.message : String(persistenceError));
|
|
11609
|
+
}
|
|
11610
|
+
}
|
|
11611
|
+
});
|
|
11612
|
+
if (workerPersistenceErrors.length > 0) {
|
|
11613
|
+
fatalError = `Failed to persist one or more research query results: ${workerPersistenceErrors[0]}`;
|
|
11614
|
+
}
|
|
11615
|
+
} catch (error) {
|
|
11616
|
+
fatalError = error instanceof Error ? error.message : String(error);
|
|
11617
|
+
} finally {
|
|
11618
|
+
if (fatalError) markUnfinishedResearchQueriesFailed(db, runId, fatalError);
|
|
11619
|
+
finalizeResearchRun(db, runId, fatalError);
|
|
11620
|
+
if (reservation && reserved > dispatched) {
|
|
11621
|
+
releaseDailyQueryQuota(db, { ...reservation, count: reserved - dispatched });
|
|
11622
|
+
}
|
|
11623
|
+
}
|
|
11624
|
+
}
|
|
11625
|
+
function incrementResearchProgress(db, runId, column) {
|
|
11626
|
+
db.update(researchRuns).set({ [column]: sql7`${researchRuns[column]} + 1` }).where(eq21(researchRuns.id, runId)).run();
|
|
11627
|
+
}
|
|
11628
|
+
function markResearchQueryFailed(db, runId, queryId, error) {
|
|
11629
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
11630
|
+
const failed = db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error: message, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and15(eq21(researchRunQueries.id, queryId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
11631
|
+
if (failed.changes === 1) incrementResearchProgress(db, runId, "failedQueries");
|
|
11632
|
+
}
|
|
11633
|
+
function markUnfinishedResearchQueriesFailed(db, runId, error) {
|
|
11634
|
+
db.update(researchRunQueries).set({ status: ResearchQueryStatuses.failed, error, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(and15(eq21(researchRunQueries.researchRunId, runId), inArray7(researchRunQueries.status, unfinishedResearchQueryStatuses))).run();
|
|
11635
|
+
}
|
|
11636
|
+
function finalizeResearchRun(db, runId, fatalError) {
|
|
11637
|
+
const rows = db.select({ status: researchRunQueries.status }).from(researchRunQueries).where(eq21(researchRunQueries.researchRunId, runId)).all();
|
|
11638
|
+
const completed = rows.filter((row) => row.status === ResearchQueryStatuses.completed).length;
|
|
11639
|
+
const failed = rows.filter((row) => row.status === ResearchQueryStatuses.failed).length;
|
|
11640
|
+
db.update(researchRuns).set({
|
|
11641
|
+
status: finalResearchRunStatus(completed, failed),
|
|
11642
|
+
completedQueries: completed,
|
|
11643
|
+
failedQueries: failed,
|
|
11644
|
+
error: fatalError ?? (failed === rows.length ? "Every research query failed." : null),
|
|
11645
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
11646
|
+
}).where(eq21(researchRuns.id, runId)).run();
|
|
11647
|
+
}
|
|
11648
|
+
|
|
11505
11649
|
// src/server.ts
|
|
11506
11650
|
var _require3 = createRequire3(import.meta.url);
|
|
11507
11651
|
var { version: PKG_VERSION2 } = _require3("../package.json");
|
|
@@ -11548,14 +11692,14 @@ function summarizeProviderConfig(config) {
|
|
|
11548
11692
|
};
|
|
11549
11693
|
}
|
|
11550
11694
|
function hashApiKey(key) {
|
|
11551
|
-
return
|
|
11695
|
+
return crypto23.createHash("sha256").update(key).digest("hex");
|
|
11552
11696
|
}
|
|
11553
11697
|
var DASHBOARD_SCRYPT_KEYLEN = 64;
|
|
11554
11698
|
var DASHBOARD_SCRYPT_COST = 1 << 15;
|
|
11555
11699
|
var DASHBOARD_SCRYPT_MAXMEM = 64 * 1024 * 1024;
|
|
11556
11700
|
function hashDashboardPassword(password) {
|
|
11557
|
-
const salt =
|
|
11558
|
-
const derived =
|
|
11701
|
+
const salt = crypto23.randomBytes(16);
|
|
11702
|
+
const derived = crypto23.scryptSync(password, salt, DASHBOARD_SCRYPT_KEYLEN, {
|
|
11559
11703
|
N: DASHBOARD_SCRYPT_COST,
|
|
11560
11704
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
11561
11705
|
});
|
|
@@ -11576,14 +11720,14 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
11576
11720
|
} catch {
|
|
11577
11721
|
return { ok: false, needsRehash: false };
|
|
11578
11722
|
}
|
|
11579
|
-
const derived =
|
|
11723
|
+
const derived = crypto23.scryptSync(password, salt, expected.length, {
|
|
11580
11724
|
N: DASHBOARD_SCRYPT_COST,
|
|
11581
11725
|
maxmem: DASHBOARD_SCRYPT_MAXMEM
|
|
11582
11726
|
});
|
|
11583
11727
|
if (derived.length !== expected.length)
|
|
11584
11728
|
return { ok: false, needsRehash: false };
|
|
11585
11729
|
return {
|
|
11586
|
-
ok:
|
|
11730
|
+
ok: crypto23.timingSafeEqual(derived, expected),
|
|
11587
11731
|
needsRehash: false
|
|
11588
11732
|
};
|
|
11589
11733
|
}
|
|
@@ -11592,7 +11736,7 @@ function verifyDashboardPassword(password, storedHash) {
|
|
|
11592
11736
|
const expected = Buffer.from(storedHash, "hex");
|
|
11593
11737
|
if (candidate.length !== expected.length)
|
|
11594
11738
|
return { ok: false, needsRehash: false };
|
|
11595
|
-
const ok =
|
|
11739
|
+
const ok = crypto23.timingSafeEqual(candidate, expected);
|
|
11596
11740
|
return { ok, needsRehash: ok };
|
|
11597
11741
|
}
|
|
11598
11742
|
return { ok: false, needsRehash: false };
|
|
@@ -11793,7 +11937,7 @@ async function createServer(opts) {
|
|
|
11793
11937
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
11794
11938
|
async (ctx) => {
|
|
11795
11939
|
if (!sessionRegistry) return;
|
|
11796
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
11940
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq22(projects.id, ctx.projectId)).get();
|
|
11797
11941
|
if (!project) return;
|
|
11798
11942
|
let content;
|
|
11799
11943
|
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
@@ -12094,9 +12238,9 @@ async function createServer(opts) {
|
|
|
12094
12238
|
});
|
|
12095
12239
|
if (!probed) return;
|
|
12096
12240
|
const alreadySynced = opts.db.select().from(ccReleaseSyncs).where(
|
|
12097
|
-
|
|
12098
|
-
|
|
12099
|
-
|
|
12241
|
+
and16(
|
|
12242
|
+
eq22(ccReleaseSyncs.release, probed.release),
|
|
12243
|
+
eq22(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)
|
|
12100
12244
|
)
|
|
12101
12245
|
).limit(1).get();
|
|
12102
12246
|
if (alreadySynced) {
|
|
@@ -12117,7 +12261,7 @@ async function createServer(opts) {
|
|
|
12117
12261
|
);
|
|
12118
12262
|
});
|
|
12119
12263
|
})();
|
|
12120
|
-
const project = opts.db.select().from(projects).where(
|
|
12264
|
+
const project = opts.db.select().from(projects).where(eq22(projects.name, projectName)).get();
|
|
12121
12265
|
if (project && bingConnectionStore.getConnection(project.canonicalDomain)) {
|
|
12122
12266
|
aeroClient.backlinksBingSync(projectName).catch((err) => {
|
|
12123
12267
|
app.log.error(
|
|
@@ -12263,7 +12407,7 @@ async function createServer(opts) {
|
|
|
12263
12407
|
return removed;
|
|
12264
12408
|
}
|
|
12265
12409
|
};
|
|
12266
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
12410
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto23.randomBytes(32).toString("hex");
|
|
12267
12411
|
const googleConnectionStore = {
|
|
12268
12412
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
12269
12413
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -12336,11 +12480,11 @@ async function createServer(opts) {
|
|
|
12336
12480
|
const googlePublicUrl = resolveGooglePublicUrl(opts.config, basePath);
|
|
12337
12481
|
if (opts.config.apiKey) {
|
|
12338
12482
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
12339
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
12483
|
+
const existing = opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, keyHash)).get();
|
|
12340
12484
|
if (!existing) {
|
|
12341
12485
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
12342
12486
|
opts.db.insert(apiKeys).values({
|
|
12343
|
-
id: `key_${
|
|
12487
|
+
id: `key_${crypto23.randomBytes(8).toString("hex")}`,
|
|
12344
12488
|
name: "default",
|
|
12345
12489
|
keyHash,
|
|
12346
12490
|
keyPrefix: prefix,
|
|
@@ -12364,7 +12508,7 @@ async function createServer(opts) {
|
|
|
12364
12508
|
};
|
|
12365
12509
|
const createSession = (apiKeyId) => {
|
|
12366
12510
|
pruneExpiredSessions();
|
|
12367
|
-
const sessionId =
|
|
12511
|
+
const sessionId = crypto23.randomBytes(32).toString("hex");
|
|
12368
12512
|
sessions.set(sessionId, {
|
|
12369
12513
|
apiKeyId,
|
|
12370
12514
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -12388,7 +12532,7 @@ async function createServer(opts) {
|
|
|
12388
12532
|
};
|
|
12389
12533
|
const getDefaultApiKey = () => {
|
|
12390
12534
|
if (!opts.config.apiKey) return void 0;
|
|
12391
|
-
return opts.db.select().from(apiKeys).where(
|
|
12535
|
+
return opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
12392
12536
|
};
|
|
12393
12537
|
const createPasswordSession = (reply) => {
|
|
12394
12538
|
const key = getDefaultApiKey();
|
|
@@ -12412,7 +12556,7 @@ async function createServer(opts) {
|
|
|
12412
12556
|
if (!header) return false;
|
|
12413
12557
|
const parts = header.split(" ");
|
|
12414
12558
|
if (parts.length !== 2 || parts[0] !== "Bearer") return false;
|
|
12415
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
12559
|
+
const key = opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, hashApiKey(parts[1]))).get();
|
|
12416
12560
|
return Boolean(key && !key.revokedAt);
|
|
12417
12561
|
};
|
|
12418
12562
|
app.get(apiPrefix + "/session", async (request, reply) => {
|
|
@@ -12489,12 +12633,12 @@ async function createServer(opts) {
|
|
|
12489
12633
|
return reply.send({ authenticated: true });
|
|
12490
12634
|
}
|
|
12491
12635
|
if (apiKey) {
|
|
12492
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
12636
|
+
const key = opts.db.select().from(apiKeys).where(eq22(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
12493
12637
|
if (!key || key.revokedAt) {
|
|
12494
12638
|
const err2 = authInvalid();
|
|
12495
12639
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
12496
12640
|
}
|
|
12497
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
12641
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq22(apiKeys.id, key.id)).run();
|
|
12498
12642
|
const sessionId = createSession(key.id);
|
|
12499
12643
|
reply.header(
|
|
12500
12644
|
"set-cookie",
|
|
@@ -12554,6 +12698,11 @@ async function createServer(opts) {
|
|
|
12554
12698
|
const briefContentRecommendation = createRecommendationBriefSynthesizer({
|
|
12555
12699
|
config: opts.config
|
|
12556
12700
|
});
|
|
12701
|
+
const dispatchResearchRun = (runId, projectId) => {
|
|
12702
|
+
executeResearchRun(opts.db, registry, runId, projectId).catch((err) => {
|
|
12703
|
+
app.log.error({ runId, err }, "Research run failed");
|
|
12704
|
+
});
|
|
12705
|
+
};
|
|
12557
12706
|
await app.register(apiRoutes, {
|
|
12558
12707
|
db: opts.db,
|
|
12559
12708
|
routePrefix: apiPrefix,
|
|
@@ -12677,7 +12826,7 @@ async function createServer(opts) {
|
|
|
12677
12826
|
deps: {
|
|
12678
12827
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
12679
12828
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
12680
|
-
const runId =
|
|
12829
|
+
const runId = crypto23.randomUUID();
|
|
12681
12830
|
opts.db.insert(runs).values({
|
|
12682
12831
|
id: runId,
|
|
12683
12832
|
projectId,
|
|
@@ -12732,6 +12881,7 @@ async function createServer(opts) {
|
|
|
12732
12881
|
app.log.error({ runId: input.runId, err }, "Discovery run failed");
|
|
12733
12882
|
});
|
|
12734
12883
|
},
|
|
12884
|
+
onResearchRunRequested: dispatchResearchRun,
|
|
12735
12885
|
// Read issued search queries (fan-out) back out of a stored probe payload.
|
|
12736
12886
|
// Discovery is Gemini-only today, so the Gemini extractor handles every
|
|
12737
12887
|
// probe; the provider arg lets a future multi-provider discovery dispatch.
|
|
@@ -12808,7 +12958,7 @@ async function createServer(opts) {
|
|
|
12808
12958
|
...inspectOpts,
|
|
12809
12959
|
config: opts.config
|
|
12810
12960
|
}).then(() => {
|
|
12811
|
-
const finished = opts.db.select({ status: runs.status }).from(runs).where(
|
|
12961
|
+
const finished = opts.db.select({ status: runs.status }).from(runs).where(eq22(runs.id, runId)).get();
|
|
12812
12962
|
if (finished?.status === RunStatuses.completed || finished?.status === RunStatuses.partial) {
|
|
12813
12963
|
return maybeRefreshGscCoverage(opts.db, opts.config, projectId);
|
|
12814
12964
|
}
|
|
@@ -12901,7 +13051,7 @@ async function createServer(opts) {
|
|
|
12901
13051
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12902
13052
|
opts.db.insert(auditLog).values(
|
|
12903
13053
|
targetProjectIds.map((projectId) => ({
|
|
12904
|
-
id:
|
|
13054
|
+
id: crypto23.randomUUID(),
|
|
12905
13055
|
projectId,
|
|
12906
13056
|
actor: "api",
|
|
12907
13057
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -13152,6 +13302,9 @@ async function createServer(opts) {
|
|
|
13152
13302
|
}
|
|
13153
13303
|
checkLatestVersionForServer();
|
|
13154
13304
|
scheduler.start();
|
|
13305
|
+
for (const run of opts.db.select({ id: researchRuns.id, projectId: researchRuns.projectId }).from(researchRuns).where(eq22(researchRuns.status, ResearchRunStatuses.queued)).all()) {
|
|
13306
|
+
dispatchResearchRun(run.id, run.projectId);
|
|
13307
|
+
}
|
|
13155
13308
|
app.addHook("onClose", async () => {
|
|
13156
13309
|
scheduler.stop();
|
|
13157
13310
|
});
|