@ainyc/canonry 2.2.1 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/canonry-setup/SKILL.md +7 -7
- package/assets/assets/{index-J73csS93.js → index-C_pxQt0X.js} +139 -139
- package/assets/index.html +1 -1
- package/dist/{chunk-TAII35VC.js → chunk-CW6CAPBQ.js} +114 -1
- package/dist/{chunk-2QNWFP6R.js → chunk-JXOUZ6JH.js} +1601 -339
- package/dist/cli.js +576 -249
- package/dist/index.js +2 -2
- package/dist/{intelligence-service-C5LAYDFM.js → intelligence-service-232P7625.js} +1 -1
- package/package.json +6 -5
|
@@ -4,8 +4,11 @@ import {
|
|
|
4
4
|
agentSessions,
|
|
5
5
|
apiKeys,
|
|
6
6
|
auditLog,
|
|
7
|
+
backlinkDomains,
|
|
8
|
+
backlinkSummaries,
|
|
7
9
|
bingCoverageSnapshots,
|
|
8
10
|
bingUrlInspections,
|
|
11
|
+
ccReleaseSyncs,
|
|
9
12
|
competitors,
|
|
10
13
|
createLogger,
|
|
11
14
|
dropLegacyCredentialColumns,
|
|
@@ -27,7 +30,7 @@ import {
|
|
|
27
30
|
runs,
|
|
28
31
|
schedules,
|
|
29
32
|
usageCounters
|
|
30
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-CW6CAPBQ.js";
|
|
31
34
|
|
|
32
35
|
// src/config.ts
|
|
33
36
|
import fs from "fs";
|
|
@@ -292,6 +295,11 @@ function usageError(displayMessage, options) {
|
|
|
292
295
|
details: options?.details
|
|
293
296
|
});
|
|
294
297
|
}
|
|
298
|
+
function isEndpointMissing(err) {
|
|
299
|
+
if (!(err instanceof CliError)) return false;
|
|
300
|
+
const status = err.details?.httpStatus;
|
|
301
|
+
return status === 404 || status === 405;
|
|
302
|
+
}
|
|
295
303
|
function printCliError(err, format) {
|
|
296
304
|
if (format === "json") {
|
|
297
305
|
if (err instanceof CliError) {
|
|
@@ -337,12 +345,12 @@ function printCliError(err, format) {
|
|
|
337
345
|
}
|
|
338
346
|
|
|
339
347
|
// src/server.ts
|
|
340
|
-
import { createRequire as
|
|
341
|
-
import
|
|
342
|
-
import
|
|
343
|
-
import
|
|
348
|
+
import { createRequire as createRequire3 } from "module";
|
|
349
|
+
import crypto27 from "crypto";
|
|
350
|
+
import fs12 from "fs";
|
|
351
|
+
import path15 from "path";
|
|
344
352
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
345
|
-
import { eq as
|
|
353
|
+
import { eq as eq29 } from "drizzle-orm";
|
|
346
354
|
import Fastify from "fastify";
|
|
347
355
|
|
|
348
356
|
// ../contracts/src/config-schema.ts
|
|
@@ -611,6 +619,9 @@ function agentBusy(projectName) {
|
|
|
611
619
|
409
|
|
612
620
|
);
|
|
613
621
|
}
|
|
622
|
+
function missingDependency(message, details) {
|
|
623
|
+
return new AppError("MISSING_DEPENDENCY", message, 422, details);
|
|
624
|
+
}
|
|
614
625
|
|
|
615
626
|
// ../contracts/src/google.ts
|
|
616
627
|
import { z as z5 } from "zod";
|
|
@@ -939,7 +950,8 @@ var runKindSchema = z8.enum([
|
|
|
939
950
|
"gsc-sync",
|
|
940
951
|
"inspect-sitemap",
|
|
941
952
|
"ga-sync",
|
|
942
|
-
"bing-inspect"
|
|
953
|
+
"bing-inspect",
|
|
954
|
+
"backlink-extract"
|
|
943
955
|
]);
|
|
944
956
|
var RunKinds = runKindSchema.enum;
|
|
945
957
|
var runTriggerSchema = z8.enum(["manual", "scheduled", "config-apply"]);
|
|
@@ -987,6 +999,13 @@ var querySnapshotDtoSchema = z8.object({
|
|
|
987
999
|
location: z8.string().nullable().optional(),
|
|
988
1000
|
createdAt: z8.string()
|
|
989
1001
|
});
|
|
1002
|
+
var runDetailDtoSchema = runDtoSchema.extend({
|
|
1003
|
+
snapshots: z8.array(querySnapshotDtoSchema).optional()
|
|
1004
|
+
});
|
|
1005
|
+
var latestProjectRunDtoSchema = z8.object({
|
|
1006
|
+
totalRuns: z8.number().int().nonnegative(),
|
|
1007
|
+
run: runDetailDtoSchema.nullable()
|
|
1008
|
+
});
|
|
990
1009
|
var auditLogEntrySchema = z8.object({
|
|
991
1010
|
id: z8.string(),
|
|
992
1011
|
projectId: z8.string().nullable().optional(),
|
|
@@ -1419,6 +1438,83 @@ var agentMemoryDeleteRequestSchema = z13.object({
|
|
|
1419
1438
|
key: z13.string().min(1).max(AGENT_MEMORY_KEY_MAX_LENGTH)
|
|
1420
1439
|
});
|
|
1421
1440
|
|
|
1441
|
+
// ../contracts/src/backlinks.ts
|
|
1442
|
+
import { z as z14 } from "zod";
|
|
1443
|
+
var ccReleaseSyncStatusSchema = z14.enum(["queued", "downloading", "querying", "ready", "failed"]);
|
|
1444
|
+
var CcReleaseSyncStatuses = ccReleaseSyncStatusSchema.enum;
|
|
1445
|
+
var ccReleaseSyncDtoSchema = z14.object({
|
|
1446
|
+
id: z14.string(),
|
|
1447
|
+
release: z14.string(),
|
|
1448
|
+
status: ccReleaseSyncStatusSchema,
|
|
1449
|
+
phaseDetail: z14.string().nullable().optional(),
|
|
1450
|
+
vertexPath: z14.string().nullable().optional(),
|
|
1451
|
+
edgesPath: z14.string().nullable().optional(),
|
|
1452
|
+
vertexSha256: z14.string().nullable().optional(),
|
|
1453
|
+
edgesSha256: z14.string().nullable().optional(),
|
|
1454
|
+
vertexBytes: z14.number().int().nullable().optional(),
|
|
1455
|
+
edgesBytes: z14.number().int().nullable().optional(),
|
|
1456
|
+
projectsProcessed: z14.number().int().nullable().optional(),
|
|
1457
|
+
domainsDiscovered: z14.number().int().nullable().optional(),
|
|
1458
|
+
downloadStartedAt: z14.string().nullable().optional(),
|
|
1459
|
+
downloadFinishedAt: z14.string().nullable().optional(),
|
|
1460
|
+
queryStartedAt: z14.string().nullable().optional(),
|
|
1461
|
+
queryFinishedAt: z14.string().nullable().optional(),
|
|
1462
|
+
error: z14.string().nullable().optional(),
|
|
1463
|
+
createdAt: z14.string(),
|
|
1464
|
+
updatedAt: z14.string()
|
|
1465
|
+
});
|
|
1466
|
+
var backlinkDomainDtoSchema = z14.object({
|
|
1467
|
+
linkingDomain: z14.string(),
|
|
1468
|
+
numHosts: z14.number().int()
|
|
1469
|
+
});
|
|
1470
|
+
var backlinkSummaryDtoSchema = z14.object({
|
|
1471
|
+
projectId: z14.string(),
|
|
1472
|
+
release: z14.string(),
|
|
1473
|
+
targetDomain: z14.string(),
|
|
1474
|
+
totalLinkingDomains: z14.number().int(),
|
|
1475
|
+
totalHosts: z14.number().int(),
|
|
1476
|
+
top10HostsShare: z14.string(),
|
|
1477
|
+
queriedAt: z14.string()
|
|
1478
|
+
});
|
|
1479
|
+
var backlinkListResponseSchema = z14.object({
|
|
1480
|
+
summary: backlinkSummaryDtoSchema.nullable(),
|
|
1481
|
+
total: z14.number().int(),
|
|
1482
|
+
rows: z14.array(backlinkDomainDtoSchema)
|
|
1483
|
+
});
|
|
1484
|
+
var backlinkHistoryEntrySchema = z14.object({
|
|
1485
|
+
release: z14.string(),
|
|
1486
|
+
totalLinkingDomains: z14.number().int(),
|
|
1487
|
+
totalHosts: z14.number().int(),
|
|
1488
|
+
top10HostsShare: z14.string(),
|
|
1489
|
+
queriedAt: z14.string()
|
|
1490
|
+
});
|
|
1491
|
+
var backlinksInstallStatusDtoSchema = z14.object({
|
|
1492
|
+
duckdbInstalled: z14.boolean(),
|
|
1493
|
+
duckdbVersion: z14.string().nullable().optional(),
|
|
1494
|
+
duckdbSpec: z14.string(),
|
|
1495
|
+
pluginDir: z14.string()
|
|
1496
|
+
});
|
|
1497
|
+
var backlinksInstallResultDtoSchema = z14.object({
|
|
1498
|
+
installed: z14.boolean(),
|
|
1499
|
+
version: z14.string(),
|
|
1500
|
+
path: z14.string(),
|
|
1501
|
+
alreadyPresent: z14.boolean()
|
|
1502
|
+
});
|
|
1503
|
+
var ccAvailableReleaseSchema = z14.object({
|
|
1504
|
+
release: z14.string(),
|
|
1505
|
+
vertexUrl: z14.string(),
|
|
1506
|
+
edgesUrl: z14.string(),
|
|
1507
|
+
vertexBytes: z14.number().int().nullable(),
|
|
1508
|
+
edgesBytes: z14.number().int().nullable(),
|
|
1509
|
+
lastModified: z14.string().nullable()
|
|
1510
|
+
});
|
|
1511
|
+
var ccCachedReleaseSchema = z14.object({
|
|
1512
|
+
release: z14.string(),
|
|
1513
|
+
syncStatus: ccReleaseSyncStatusSchema.nullable(),
|
|
1514
|
+
bytes: z14.number().int(),
|
|
1515
|
+
lastUsedAt: z14.string().nullable()
|
|
1516
|
+
});
|
|
1517
|
+
|
|
1422
1518
|
// ../api-routes/src/auth.ts
|
|
1423
1519
|
import crypto2 from "crypto";
|
|
1424
1520
|
import { eq } from "drizzle-orm";
|
|
@@ -1996,7 +2092,7 @@ async function competitorRoutes(app) {
|
|
|
1996
2092
|
|
|
1997
2093
|
// ../api-routes/src/runs.ts
|
|
1998
2094
|
import crypto8 from "crypto";
|
|
1999
|
-
import { eq as eq7, asc, desc } from "drizzle-orm";
|
|
2095
|
+
import { eq as eq7, asc, desc, sql as sql2 } from "drizzle-orm";
|
|
2000
2096
|
|
|
2001
2097
|
// ../api-routes/src/run-queue.ts
|
|
2002
2098
|
import crypto7 from "crypto";
|
|
@@ -2136,6 +2232,19 @@ async function runRoutes(app, opts) {
|
|
|
2136
2232
|
const rows = limit == null ? app.db.select().from(runs).where(eq7(runs.projectId, project.id)).orderBy(asc(runs.createdAt)).all() : app.db.select().from(runs).where(eq7(runs.projectId, project.id)).orderBy(desc(runs.createdAt)).limit(limit).all().reverse();
|
|
2137
2233
|
return reply.send(rows.map(formatRun));
|
|
2138
2234
|
});
|
|
2235
|
+
app.get("/projects/:name/runs/latest", async (request, reply) => {
|
|
2236
|
+
const project = resolveProject(app.db, request.params.name);
|
|
2237
|
+
const countRow = app.db.select({ count: sql2`count(*)` }).from(runs).where(eq7(runs.projectId, project.id)).get();
|
|
2238
|
+
const totalRuns = countRow?.count ?? 0;
|
|
2239
|
+
const latestRun = app.db.select().from(runs).where(eq7(runs.projectId, project.id)).orderBy(desc(runs.createdAt)).limit(1).get();
|
|
2240
|
+
if (!latestRun) {
|
|
2241
|
+
return reply.send({ totalRuns: 0, run: null });
|
|
2242
|
+
}
|
|
2243
|
+
return reply.send({
|
|
2244
|
+
totalRuns,
|
|
2245
|
+
run: loadRunDetail(app, latestRun)
|
|
2246
|
+
});
|
|
2247
|
+
});
|
|
2139
2248
|
app.get("/runs", async (_request, reply) => {
|
|
2140
2249
|
const rows = app.db.select().from(runs).all();
|
|
2141
2250
|
return reply.send(rows.map(formatRun));
|
|
@@ -2224,55 +2333,7 @@ async function runRoutes(app, opts) {
|
|
|
2224
2333
|
app.get("/runs/:id", async (request, reply) => {
|
|
2225
2334
|
const run = app.db.select().from(runs).where(eq7(runs.id, request.params.id)).get();
|
|
2226
2335
|
if (!run) throw notFound("Run", request.params.id);
|
|
2227
|
-
|
|
2228
|
-
displayName: projects.displayName,
|
|
2229
|
-
canonicalDomain: projects.canonicalDomain,
|
|
2230
|
-
ownedDomains: projects.ownedDomains
|
|
2231
|
-
}).from(projects).where(eq7(projects.id, run.projectId)).get();
|
|
2232
|
-
const snapshots = app.db.select({
|
|
2233
|
-
id: querySnapshots.id,
|
|
2234
|
-
runId: querySnapshots.runId,
|
|
2235
|
-
keywordId: querySnapshots.keywordId,
|
|
2236
|
-
keyword: keywords.keyword,
|
|
2237
|
-
provider: querySnapshots.provider,
|
|
2238
|
-
model: querySnapshots.model,
|
|
2239
|
-
citationState: querySnapshots.citationState,
|
|
2240
|
-
answerMentioned: querySnapshots.answerMentioned,
|
|
2241
|
-
answerText: querySnapshots.answerText,
|
|
2242
|
-
citedDomains: querySnapshots.citedDomains,
|
|
2243
|
-
competitorOverlap: querySnapshots.competitorOverlap,
|
|
2244
|
-
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
2245
|
-
location: querySnapshots.location,
|
|
2246
|
-
rawResponse: querySnapshots.rawResponse,
|
|
2247
|
-
createdAt: querySnapshots.createdAt
|
|
2248
|
-
}).from(querySnapshots).leftJoin(keywords, eq7(querySnapshots.keywordId, keywords.id)).where(eq7(querySnapshots.runId, run.id)).all();
|
|
2249
|
-
return reply.send({
|
|
2250
|
-
...formatRun(run),
|
|
2251
|
-
snapshots: snapshots.map((s) => {
|
|
2252
|
-
const rawParsed = parseSnapshotRawResponse(s.rawResponse);
|
|
2253
|
-
const answerMentioned = project ? resolveSnapshotAnswerMentioned(s, project) : s.answerMentioned ?? false;
|
|
2254
|
-
return {
|
|
2255
|
-
id: s.id,
|
|
2256
|
-
runId: s.runId,
|
|
2257
|
-
keywordId: s.keywordId,
|
|
2258
|
-
keyword: s.keyword,
|
|
2259
|
-
provider: s.provider,
|
|
2260
|
-
citationState: s.citationState,
|
|
2261
|
-
answerMentioned,
|
|
2262
|
-
visibilityState: project ? resolveSnapshotVisibilityState(s, project) : answerMentioned ? "visible" : "not-visible",
|
|
2263
|
-
answerText: s.answerText,
|
|
2264
|
-
citedDomains: parseJsonColumn(s.citedDomains, []),
|
|
2265
|
-
competitorOverlap: parseJsonColumn(s.competitorOverlap, []),
|
|
2266
|
-
recommendedCompetitors: parseJsonColumn(s.recommendedCompetitors, []),
|
|
2267
|
-
matchedTerms: project ? resolveSnapshotMatchedTerms(s, project) : [],
|
|
2268
|
-
model: s.model ?? rawParsed.model,
|
|
2269
|
-
location: s.location,
|
|
2270
|
-
groundingSources: rawParsed.groundingSources,
|
|
2271
|
-
searchQueries: rawParsed.searchQueries,
|
|
2272
|
-
createdAt: s.createdAt
|
|
2273
|
-
};
|
|
2274
|
-
})
|
|
2275
|
-
});
|
|
2336
|
+
return reply.send(loadRunDetail(app, run));
|
|
2276
2337
|
});
|
|
2277
2338
|
}
|
|
2278
2339
|
function formatRun(row) {
|
|
@@ -2297,6 +2358,57 @@ function parseSnapshotRawResponse(raw) {
|
|
|
2297
2358
|
model: parsed.model ?? null
|
|
2298
2359
|
};
|
|
2299
2360
|
}
|
|
2361
|
+
function loadRunDetail(app, run) {
|
|
2362
|
+
const project = app.db.select({
|
|
2363
|
+
displayName: projects.displayName,
|
|
2364
|
+
canonicalDomain: projects.canonicalDomain,
|
|
2365
|
+
ownedDomains: projects.ownedDomains
|
|
2366
|
+
}).from(projects).where(eq7(projects.id, run.projectId)).get();
|
|
2367
|
+
const snapshots = app.db.select({
|
|
2368
|
+
id: querySnapshots.id,
|
|
2369
|
+
runId: querySnapshots.runId,
|
|
2370
|
+
keywordId: querySnapshots.keywordId,
|
|
2371
|
+
keyword: keywords.keyword,
|
|
2372
|
+
provider: querySnapshots.provider,
|
|
2373
|
+
model: querySnapshots.model,
|
|
2374
|
+
citationState: querySnapshots.citationState,
|
|
2375
|
+
answerMentioned: querySnapshots.answerMentioned,
|
|
2376
|
+
answerText: querySnapshots.answerText,
|
|
2377
|
+
citedDomains: querySnapshots.citedDomains,
|
|
2378
|
+
competitorOverlap: querySnapshots.competitorOverlap,
|
|
2379
|
+
recommendedCompetitors: querySnapshots.recommendedCompetitors,
|
|
2380
|
+
location: querySnapshots.location,
|
|
2381
|
+
rawResponse: querySnapshots.rawResponse,
|
|
2382
|
+
createdAt: querySnapshots.createdAt
|
|
2383
|
+
}).from(querySnapshots).leftJoin(keywords, eq7(querySnapshots.keywordId, keywords.id)).where(eq7(querySnapshots.runId, run.id)).all();
|
|
2384
|
+
return {
|
|
2385
|
+
...formatRun(run),
|
|
2386
|
+
snapshots: snapshots.map((s) => {
|
|
2387
|
+
const rawParsed = parseSnapshotRawResponse(s.rawResponse);
|
|
2388
|
+
const answerMentioned = project ? resolveSnapshotAnswerMentioned(s, project) : s.answerMentioned ?? false;
|
|
2389
|
+
return {
|
|
2390
|
+
id: s.id,
|
|
2391
|
+
runId: s.runId,
|
|
2392
|
+
keywordId: s.keywordId,
|
|
2393
|
+
keyword: s.keyword,
|
|
2394
|
+
provider: s.provider,
|
|
2395
|
+
citationState: s.citationState,
|
|
2396
|
+
answerMentioned,
|
|
2397
|
+
visibilityState: project ? resolveSnapshotVisibilityState(s, project) : answerMentioned ? "visible" : "not-visible",
|
|
2398
|
+
answerText: s.answerText,
|
|
2399
|
+
citedDomains: parseJsonColumn(s.citedDomains, []),
|
|
2400
|
+
competitorOverlap: parseJsonColumn(s.competitorOverlap, []),
|
|
2401
|
+
recommendedCompetitors: parseJsonColumn(s.recommendedCompetitors, []),
|
|
2402
|
+
matchedTerms: project ? resolveSnapshotMatchedTerms(s, project) : [],
|
|
2403
|
+
model: s.model ?? rawParsed.model,
|
|
2404
|
+
location: s.location,
|
|
2405
|
+
groundingSources: rawParsed.groundingSources,
|
|
2406
|
+
searchQueries: rawParsed.searchQueries,
|
|
2407
|
+
createdAt: s.createdAt
|
|
2408
|
+
};
|
|
2409
|
+
})
|
|
2410
|
+
};
|
|
2411
|
+
}
|
|
2300
2412
|
|
|
2301
2413
|
// ../api-routes/src/apply.ts
|
|
2302
2414
|
import crypto10 from "crypto";
|
|
@@ -2435,7 +2547,7 @@ async function deliverWebhook(target, payload, webhookSecret) {
|
|
|
2435
2547
|
const body = JSON.stringify(payload);
|
|
2436
2548
|
const isHttps = target.url.protocol === "https:";
|
|
2437
2549
|
const port = target.url.port ? Number(target.url.port) : isHttps ? 443 : 80;
|
|
2438
|
-
const
|
|
2550
|
+
const path16 = `${target.url.pathname}${target.url.search}`;
|
|
2439
2551
|
const headers = {
|
|
2440
2552
|
"Content-Length": String(Buffer.byteLength(body)),
|
|
2441
2553
|
"Content-Type": "application/json",
|
|
@@ -2451,7 +2563,7 @@ async function deliverWebhook(target, payload, webhookSecret) {
|
|
|
2451
2563
|
headers,
|
|
2452
2564
|
hostname: target.address,
|
|
2453
2565
|
method: "POST",
|
|
2454
|
-
path:
|
|
2566
|
+
path: path16,
|
|
2455
2567
|
port,
|
|
2456
2568
|
timeout: REQUEST_TIMEOUT_MS
|
|
2457
2569
|
};
|
|
@@ -3930,6 +4042,16 @@ var routeCatalog = [
|
|
|
3930
4042
|
200: { description: "Runs returned." }
|
|
3931
4043
|
}
|
|
3932
4044
|
},
|
|
4045
|
+
{
|
|
4046
|
+
method: "get",
|
|
4047
|
+
path: "/api/v1/projects/{name}/runs/latest",
|
|
4048
|
+
summary: "Get the latest project run",
|
|
4049
|
+
tags: ["runs"],
|
|
4050
|
+
parameters: [nameParameter],
|
|
4051
|
+
responses: {
|
|
4052
|
+
200: { description: "Latest run returned." }
|
|
4053
|
+
}
|
|
4054
|
+
},
|
|
3933
4055
|
{
|
|
3934
4056
|
method: "get",
|
|
3935
4057
|
path: "/api/v1/runs",
|
|
@@ -5619,6 +5741,171 @@ var routeCatalog = [
|
|
|
5619
5741
|
200: { description: "Health history returned." },
|
|
5620
5742
|
404: { description: "Project not found." }
|
|
5621
5743
|
}
|
|
5744
|
+
},
|
|
5745
|
+
{
|
|
5746
|
+
method: "get",
|
|
5747
|
+
path: "/api/v1/backlinks/status",
|
|
5748
|
+
summary: "Get the Common Crawl DuckDB plugin install status",
|
|
5749
|
+
description: "Reports whether @duckdb/node-api is installed in the local plugin dir. Returns MISSING_DEPENDENCY (422) on deployments that cannot host the plugin (e.g. the cloud API).",
|
|
5750
|
+
tags: ["backlinks"],
|
|
5751
|
+
responses: {
|
|
5752
|
+
200: { description: "Install status returned." },
|
|
5753
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5754
|
+
}
|
|
5755
|
+
},
|
|
5756
|
+
{
|
|
5757
|
+
method: "post",
|
|
5758
|
+
path: "/api/v1/backlinks/install",
|
|
5759
|
+
summary: "Install the @duckdb/node-api plugin",
|
|
5760
|
+
description: "Idempotently installs DuckDB into the canonry plugin dir. Returns MISSING_DEPENDENCY (422) when the host cannot perform the install.",
|
|
5761
|
+
tags: ["backlinks"],
|
|
5762
|
+
responses: {
|
|
5763
|
+
200: { description: "Installed (or already present)." },
|
|
5764
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5765
|
+
}
|
|
5766
|
+
},
|
|
5767
|
+
{
|
|
5768
|
+
method: "post",
|
|
5769
|
+
path: "/api/v1/backlinks/syncs",
|
|
5770
|
+
summary: "Queue a workspace-wide Common Crawl release sync",
|
|
5771
|
+
description: "Creates a `cc_release_syncs` row and fires the sync callback. Idempotent: an existing in-flight row for the same release is returned.",
|
|
5772
|
+
tags: ["backlinks"],
|
|
5773
|
+
requestBody: {
|
|
5774
|
+
required: true,
|
|
5775
|
+
content: {
|
|
5776
|
+
"application/json": {
|
|
5777
|
+
schema: {
|
|
5778
|
+
type: "object",
|
|
5779
|
+
required: ["release"],
|
|
5780
|
+
properties: {
|
|
5781
|
+
release: stringSchema
|
|
5782
|
+
}
|
|
5783
|
+
}
|
|
5784
|
+
}
|
|
5785
|
+
}
|
|
5786
|
+
},
|
|
5787
|
+
responses: {
|
|
5788
|
+
200: { description: "Existing in-flight sync returned." },
|
|
5789
|
+
201: { description: "Sync queued." },
|
|
5790
|
+
400: { description: "Invalid release id." },
|
|
5791
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5792
|
+
}
|
|
5793
|
+
},
|
|
5794
|
+
{
|
|
5795
|
+
method: "get",
|
|
5796
|
+
path: "/api/v1/backlinks/syncs",
|
|
5797
|
+
summary: "List Common Crawl release syncs",
|
|
5798
|
+
description: "Returns syncs ordered by updatedAt DESC \u2014 re-queued rows surface ahead of untouched newer rows.",
|
|
5799
|
+
tags: ["backlinks"],
|
|
5800
|
+
responses: {
|
|
5801
|
+
200: { description: "Sync history returned." }
|
|
5802
|
+
}
|
|
5803
|
+
},
|
|
5804
|
+
{
|
|
5805
|
+
method: "get",
|
|
5806
|
+
path: "/api/v1/backlinks/syncs/latest",
|
|
5807
|
+
summary: "Get the most recently-updated Common Crawl release sync",
|
|
5808
|
+
tags: ["backlinks"],
|
|
5809
|
+
responses: {
|
|
5810
|
+
200: { description: "Latest sync returned, or null when no sync exists." }
|
|
5811
|
+
}
|
|
5812
|
+
},
|
|
5813
|
+
{
|
|
5814
|
+
method: "get",
|
|
5815
|
+
path: "/api/v1/backlinks/releases",
|
|
5816
|
+
summary: "List cached Common Crawl releases on the local filesystem",
|
|
5817
|
+
tags: ["backlinks"],
|
|
5818
|
+
responses: {
|
|
5819
|
+
200: { description: "Cached release metadata returned." }
|
|
5820
|
+
}
|
|
5821
|
+
},
|
|
5822
|
+
{
|
|
5823
|
+
method: "delete",
|
|
5824
|
+
path: "/api/v1/backlinks/cache/{release}",
|
|
5825
|
+
summary: "Prune a cached Common Crawl release",
|
|
5826
|
+
tags: ["backlinks"],
|
|
5827
|
+
parameters: [
|
|
5828
|
+
{
|
|
5829
|
+
name: "release",
|
|
5830
|
+
in: "path",
|
|
5831
|
+
required: true,
|
|
5832
|
+
description: "Release id (e.g. cc-main-2026-jan-feb-mar).",
|
|
5833
|
+
schema: stringSchema
|
|
5834
|
+
}
|
|
5835
|
+
],
|
|
5836
|
+
responses: {
|
|
5837
|
+
200: { description: "Cache pruned." },
|
|
5838
|
+
400: { description: "Invalid release id." },
|
|
5839
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5840
|
+
}
|
|
5841
|
+
},
|
|
5842
|
+
{
|
|
5843
|
+
method: "post",
|
|
5844
|
+
path: "/api/v1/projects/{name}/backlinks/extract",
|
|
5845
|
+
summary: "Extract backlinks for a single project from a cached release",
|
|
5846
|
+
description: 'Creates a `runs` row with kind="backlink-extract" and fires the extract callback. Defaults to the most recent ready release when `release` is omitted.',
|
|
5847
|
+
tags: ["backlinks"],
|
|
5848
|
+
parameters: [nameParameter],
|
|
5849
|
+
requestBody: {
|
|
5850
|
+
required: false,
|
|
5851
|
+
content: {
|
|
5852
|
+
"application/json": {
|
|
5853
|
+
schema: {
|
|
5854
|
+
type: "object",
|
|
5855
|
+
properties: {
|
|
5856
|
+
release: stringSchema
|
|
5857
|
+
}
|
|
5858
|
+
}
|
|
5859
|
+
}
|
|
5860
|
+
}
|
|
5861
|
+
},
|
|
5862
|
+
responses: {
|
|
5863
|
+
201: { description: "Extract run queued." },
|
|
5864
|
+
400: { description: "Invalid release id." },
|
|
5865
|
+
404: { description: "Project not found." },
|
|
5866
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5867
|
+
}
|
|
5868
|
+
},
|
|
5869
|
+
{
|
|
5870
|
+
method: "get",
|
|
5871
|
+
path: "/api/v1/projects/{name}/backlinks/summary",
|
|
5872
|
+
summary: "Get the latest backlink summary for a project",
|
|
5873
|
+
tags: ["backlinks"],
|
|
5874
|
+
parameters: [
|
|
5875
|
+
nameParameter,
|
|
5876
|
+
{ name: "release", in: "query", description: "Release id filter.", schema: stringSchema }
|
|
5877
|
+
],
|
|
5878
|
+
responses: {
|
|
5879
|
+
200: { description: "Summary returned, or null when no backlinks exist." },
|
|
5880
|
+
404: { description: "Project not found." }
|
|
5881
|
+
}
|
|
5882
|
+
},
|
|
5883
|
+
{
|
|
5884
|
+
method: "get",
|
|
5885
|
+
path: "/api/v1/projects/{name}/backlinks/domains",
|
|
5886
|
+
summary: "Paginate backlink domains for a project",
|
|
5887
|
+
tags: ["backlinks"],
|
|
5888
|
+
parameters: [
|
|
5889
|
+
nameParameter,
|
|
5890
|
+
{ name: "release", in: "query", description: "Release id filter.", schema: stringSchema },
|
|
5891
|
+
{ name: "limit", in: "query", description: "Max results (1-500).", schema: stringSchema },
|
|
5892
|
+
{ name: "offset", in: "query", description: "Pagination offset.", schema: stringSchema }
|
|
5893
|
+
],
|
|
5894
|
+
responses: {
|
|
5895
|
+
200: { description: "Domain list returned." },
|
|
5896
|
+
404: { description: "Project not found." }
|
|
5897
|
+
}
|
|
5898
|
+
},
|
|
5899
|
+
{
|
|
5900
|
+
method: "get",
|
|
5901
|
+
path: "/api/v1/projects/{name}/backlinks/history",
|
|
5902
|
+
summary: "Get per-release backlink summaries for a project",
|
|
5903
|
+
tags: ["backlinks"],
|
|
5904
|
+
parameters: [nameParameter],
|
|
5905
|
+
responses: {
|
|
5906
|
+
200: { description: "History returned oldest-first by queriedAt." },
|
|
5907
|
+
404: { description: "Project not found." }
|
|
5908
|
+
}
|
|
5622
5909
|
}
|
|
5623
5910
|
];
|
|
5624
5911
|
var canonryLocalRouteCatalog = [
|
|
@@ -5753,8 +6040,8 @@ async function openApiRoutes(app, opts = {}) {
|
|
|
5753
6040
|
return reply.type("application/json").send(buildOpenApiDocument(opts));
|
|
5754
6041
|
});
|
|
5755
6042
|
}
|
|
5756
|
-
function buildOperationId(method,
|
|
5757
|
-
const parts =
|
|
6043
|
+
function buildOperationId(method, path16) {
|
|
6044
|
+
const parts = path16.split("/").filter(Boolean).map((part) => {
|
|
5758
6045
|
if (part.startsWith("{") && part.endsWith("}")) {
|
|
5759
6046
|
return `by-${part.slice(1, -1)}`;
|
|
5760
6047
|
}
|
|
@@ -5918,10 +6205,9 @@ async function snapshotRoutes(app, opts) {
|
|
|
5918
6205
|
|
|
5919
6206
|
// ../api-routes/src/telemetry.ts
|
|
5920
6207
|
async function telemetryRoutes(app, opts) {
|
|
5921
|
-
app.get("/telemetry", async (
|
|
6208
|
+
app.get("/telemetry", async () => {
|
|
5922
6209
|
if (!opts.getTelemetryStatus) {
|
|
5923
|
-
|
|
5924
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
6210
|
+
throw notImplemented("Telemetry status is not available in this deployment");
|
|
5925
6211
|
}
|
|
5926
6212
|
const status = opts.getTelemetryStatus();
|
|
5927
6213
|
return {
|
|
@@ -5929,15 +6215,13 @@ async function telemetryRoutes(app, opts) {
|
|
|
5929
6215
|
anonymousId: status.anonymousId ? status.anonymousId.slice(0, 8) + "..." : void 0
|
|
5930
6216
|
};
|
|
5931
6217
|
});
|
|
5932
|
-
app.put("/telemetry", async (request
|
|
6218
|
+
app.put("/telemetry", async (request) => {
|
|
5933
6219
|
if (!opts.setTelemetryEnabled) {
|
|
5934
|
-
|
|
5935
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
6220
|
+
throw notImplemented("Telemetry configuration is not available in this deployment");
|
|
5936
6221
|
}
|
|
5937
6222
|
const { enabled } = request.body ?? {};
|
|
5938
6223
|
if (typeof enabled !== "boolean") {
|
|
5939
|
-
|
|
5940
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
6224
|
+
throw validationError("enabled (boolean) is required");
|
|
5941
6225
|
}
|
|
5942
6226
|
opts.setTelemetryEnabled(enabled);
|
|
5943
6227
|
const status = opts.getTelemetryStatus?.();
|
|
@@ -6190,7 +6474,7 @@ function formatNotification(row) {
|
|
|
6190
6474
|
|
|
6191
6475
|
// ../api-routes/src/google.ts
|
|
6192
6476
|
import crypto14 from "crypto";
|
|
6193
|
-
import { eq as eq14, and as and3, desc as desc5, sql as
|
|
6477
|
+
import { eq as eq14, and as and3, desc as desc5, sql as sql3 } from "drizzle-orm";
|
|
6194
6478
|
|
|
6195
6479
|
// ../integration-google/src/constants.ts
|
|
6196
6480
|
var GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
|
|
@@ -7313,11 +7597,11 @@ async function googleRoutes(app, opts) {
|
|
|
7313
7597
|
const { startDate, endDate, query, page, limit } = request.query;
|
|
7314
7598
|
const cutoffDate = !startDate ? windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null : null;
|
|
7315
7599
|
const conditions = [eq14(gscSearchData.projectId, project.id)];
|
|
7316
|
-
if (startDate) conditions.push(
|
|
7317
|
-
else if (cutoffDate) conditions.push(
|
|
7318
|
-
if (endDate) conditions.push(
|
|
7319
|
-
if (query) conditions.push(
|
|
7320
|
-
if (page) conditions.push(
|
|
7600
|
+
if (startDate) conditions.push(sql3`${gscSearchData.date} >= ${startDate}`);
|
|
7601
|
+
else if (cutoffDate) conditions.push(sql3`${gscSearchData.date} >= ${cutoffDate}`);
|
|
7602
|
+
if (endDate) conditions.push(sql3`${gscSearchData.date} <= ${endDate}`);
|
|
7603
|
+
if (query) conditions.push(sql3`${gscSearchData.query} LIKE ${"%" + query + "%"}`);
|
|
7604
|
+
if (page) conditions.push(sql3`${gscSearchData.page} LIKE ${"%" + page + "%"}`);
|
|
7321
7605
|
const rows = app.db.select().from(gscSearchData).where(and3(...conditions)).orderBy(desc5(gscSearchData.date)).limit(parseInt(limit ?? "500", 10)).all();
|
|
7322
7606
|
return rows.map((r) => ({
|
|
7323
7607
|
date: r.date,
|
|
@@ -8543,7 +8827,7 @@ async function cdpRoutes(app, opts) {
|
|
|
8543
8827
|
|
|
8544
8828
|
// ../api-routes/src/ga.ts
|
|
8545
8829
|
import crypto16 from "crypto";
|
|
8546
|
-
import { eq as eq17, desc as desc7, and as and6, sql as
|
|
8830
|
+
import { eq as eq17, desc as desc7, and as and6, sql as sql4 } from "drizzle-orm";
|
|
8547
8831
|
function gaLog(level, action, ctx) {
|
|
8548
8832
|
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
8549
8833
|
const stream = level === "error" ? process.stderr : process.stdout;
|
|
@@ -8785,8 +9069,8 @@ async function ga4Routes(app, opts) {
|
|
|
8785
9069
|
tx.delete(gaTrafficSnapshots).where(
|
|
8786
9070
|
and6(
|
|
8787
9071
|
eq17(gaTrafficSnapshots.projectId, project.id),
|
|
8788
|
-
|
|
8789
|
-
|
|
9072
|
+
sql4`${gaTrafficSnapshots.date} >= ${summary.periodStart}`,
|
|
9073
|
+
sql4`${gaTrafficSnapshots.date} <= ${summary.periodEnd}`
|
|
8790
9074
|
)
|
|
8791
9075
|
).run();
|
|
8792
9076
|
for (const row of rows) {
|
|
@@ -8807,8 +9091,8 @@ async function ga4Routes(app, opts) {
|
|
|
8807
9091
|
tx.delete(gaAiReferrals).where(
|
|
8808
9092
|
and6(
|
|
8809
9093
|
eq17(gaAiReferrals.projectId, project.id),
|
|
8810
|
-
|
|
8811
|
-
|
|
9094
|
+
sql4`${gaAiReferrals.date} >= ${summary.periodStart}`,
|
|
9095
|
+
sql4`${gaAiReferrals.date} <= ${summary.periodEnd}`
|
|
8812
9096
|
)
|
|
8813
9097
|
).run();
|
|
8814
9098
|
for (const row of aiReferrals) {
|
|
@@ -8830,8 +9114,8 @@ async function ga4Routes(app, opts) {
|
|
|
8830
9114
|
tx.delete(gaSocialReferrals).where(
|
|
8831
9115
|
and6(
|
|
8832
9116
|
eq17(gaSocialReferrals.projectId, project.id),
|
|
8833
|
-
|
|
8834
|
-
|
|
9117
|
+
sql4`${gaSocialReferrals.date} >= ${summary.periodStart}`,
|
|
9118
|
+
sql4`${gaSocialReferrals.date} <= ${summary.periodEnd}`
|
|
8835
9119
|
)
|
|
8836
9120
|
).run();
|
|
8837
9121
|
for (const row of socialReferrals) {
|
|
@@ -8900,15 +9184,15 @@ async function ga4Routes(app, opts) {
|
|
|
8900
9184
|
const cutoff = windowCutoff(window);
|
|
8901
9185
|
const cutoffDate = cutoff?.slice(0, 10) ?? null;
|
|
8902
9186
|
const snapshotConditions = [eq17(gaTrafficSnapshots.projectId, project.id)];
|
|
8903
|
-
if (cutoffDate) snapshotConditions.push(
|
|
9187
|
+
if (cutoffDate) snapshotConditions.push(sql4`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
8904
9188
|
const aiConditions = [eq17(gaAiReferrals.projectId, project.id)];
|
|
8905
|
-
if (cutoffDate) aiConditions.push(
|
|
9189
|
+
if (cutoffDate) aiConditions.push(sql4`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
8906
9190
|
const socialConditions = [eq17(gaSocialReferrals.projectId, project.id)];
|
|
8907
|
-
if (cutoffDate) socialConditions.push(
|
|
9191
|
+
if (cutoffDate) socialConditions.push(sql4`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
8908
9192
|
const summaryRow = cutoffDate ? app.db.select({
|
|
8909
|
-
totalSessions:
|
|
8910
|
-
totalOrganicSessions:
|
|
8911
|
-
totalUsers:
|
|
9193
|
+
totalSessions: sql4`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
9194
|
+
totalOrganicSessions: sql4`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
9195
|
+
totalUsers: sql4`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
8912
9196
|
}).from(gaTrafficSnapshots).where(and6(...snapshotConditions)).get() : app.db.select({
|
|
8913
9197
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
8914
9198
|
totalOrganicSessions: gaTrafficSummaries.totalOrganicSessions,
|
|
@@ -8920,27 +9204,27 @@ async function ga4Routes(app, opts) {
|
|
|
8920
9204
|
}).from(gaTrafficSummaries).where(eq17(gaTrafficSummaries.projectId, project.id)).get();
|
|
8921
9205
|
const rows = app.db.select({
|
|
8922
9206
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
8923
|
-
sessions:
|
|
8924
|
-
organicSessions:
|
|
8925
|
-
users:
|
|
8926
|
-
}).from(gaTrafficSnapshots).where(and6(...snapshotConditions)).groupBy(gaTrafficSnapshots.landingPage).orderBy(
|
|
9207
|
+
sessions: sql4`SUM(${gaTrafficSnapshots.sessions})`,
|
|
9208
|
+
organicSessions: sql4`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
9209
|
+
users: sql4`SUM(${gaTrafficSnapshots.users})`
|
|
9210
|
+
}).from(gaTrafficSnapshots).where(and6(...snapshotConditions)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql4`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
8927
9211
|
const aiReferrals = app.db.select({
|
|
8928
9212
|
source: gaAiReferrals.source,
|
|
8929
9213
|
medium: gaAiReferrals.medium,
|
|
8930
9214
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
8931
|
-
sessions:
|
|
8932
|
-
users:
|
|
8933
|
-
}).from(gaAiReferrals).where(and6(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).orderBy(
|
|
9215
|
+
sessions: sql4`SUM(${gaAiReferrals.sessions})`,
|
|
9216
|
+
users: sql4`SUM(${gaAiReferrals.users})`
|
|
9217
|
+
}).from(gaAiReferrals).where(and6(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).orderBy(sql4`SUM(${gaAiReferrals.sessions}) DESC`).all();
|
|
8934
9218
|
const aiDeduped = app.db.select({
|
|
8935
|
-
sessions:
|
|
8936
|
-
users:
|
|
9219
|
+
sessions: sql4`SUM(max_sessions)`,
|
|
9220
|
+
users: sql4`SUM(max_users)`
|
|
8937
9221
|
}).from(
|
|
8938
|
-
|
|
9222
|
+
sql4`(
|
|
8939
9223
|
SELECT date, source, medium,
|
|
8940
9224
|
MAX(sessions) AS max_sessions,
|
|
8941
9225
|
MAX(users) AS max_users
|
|
8942
9226
|
FROM ga_ai_referrals
|
|
8943
|
-
WHERE project_id = ${project.id}${cutoffDate ?
|
|
9227
|
+
WHERE project_id = ${project.id}${cutoffDate ? sql4` AND date >= ${cutoffDate}` : sql4``}
|
|
8944
9228
|
GROUP BY date, source, medium
|
|
8945
9229
|
)`
|
|
8946
9230
|
).get();
|
|
@@ -8948,12 +9232,12 @@ async function ga4Routes(app, opts) {
|
|
|
8948
9232
|
source: gaSocialReferrals.source,
|
|
8949
9233
|
medium: gaSocialReferrals.medium,
|
|
8950
9234
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
8951
|
-
sessions:
|
|
8952
|
-
users:
|
|
8953
|
-
}).from(gaSocialReferrals).where(and6(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
9235
|
+
sessions: sql4`SUM(${gaSocialReferrals.sessions})`,
|
|
9236
|
+
users: sql4`SUM(${gaSocialReferrals.users})`
|
|
9237
|
+
}).from(gaSocialReferrals).where(and6(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql4`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
8954
9238
|
const socialTotals = app.db.select({
|
|
8955
|
-
sessions:
|
|
8956
|
-
users:
|
|
9239
|
+
sessions: sql4`SUM(${gaSocialReferrals.sessions})`,
|
|
9240
|
+
users: sql4`SUM(${gaSocialReferrals.users})`
|
|
8957
9241
|
}).from(gaSocialReferrals).where(and6(...socialConditions)).get();
|
|
8958
9242
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq17(gaTrafficSummaries.projectId, project.id)).orderBy(desc7(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
8959
9243
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -9003,7 +9287,7 @@ async function ga4Routes(app, opts) {
|
|
|
9003
9287
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
9004
9288
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
9005
9289
|
const conditions = [eq17(gaAiReferrals.projectId, project.id)];
|
|
9006
|
-
if (cutoffDate) conditions.push(
|
|
9290
|
+
if (cutoffDate) conditions.push(sql4`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
9007
9291
|
const rows = app.db.select({
|
|
9008
9292
|
date: gaAiReferrals.date,
|
|
9009
9293
|
source: gaAiReferrals.source,
|
|
@@ -9019,7 +9303,7 @@ async function ga4Routes(app, opts) {
|
|
|
9019
9303
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
9020
9304
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
9021
9305
|
const conditions = [eq17(gaSocialReferrals.projectId, project.id)];
|
|
9022
|
-
if (cutoffDate) conditions.push(
|
|
9306
|
+
if (cutoffDate) conditions.push(sql4`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
9023
9307
|
const rows = app.db.select({
|
|
9024
9308
|
date: gaSocialReferrals.date,
|
|
9025
9309
|
source: gaSocialReferrals.source,
|
|
@@ -9040,10 +9324,10 @@ async function ga4Routes(app, opts) {
|
|
|
9040
9324
|
d.setDate(d.getDate() - n);
|
|
9041
9325
|
return fmt(d);
|
|
9042
9326
|
};
|
|
9043
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
9327
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql4`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and6(
|
|
9044
9328
|
eq17(gaSocialReferrals.projectId, project.id),
|
|
9045
|
-
|
|
9046
|
-
|
|
9329
|
+
sql4`${gaSocialReferrals.date} >= ${from}`,
|
|
9330
|
+
sql4`${gaSocialReferrals.date} < ${to}`
|
|
9047
9331
|
)).get();
|
|
9048
9332
|
const current7d = sumSocial(daysAgo2(7), fmt(today));
|
|
9049
9333
|
const prev7d = sumSocial(daysAgo2(14), daysAgo2(7));
|
|
@@ -9052,19 +9336,19 @@ async function ga4Routes(app, opts) {
|
|
|
9052
9336
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
9053
9337
|
const sourceCurrent = app.db.select({
|
|
9054
9338
|
source: gaSocialReferrals.source,
|
|
9055
|
-
sessions:
|
|
9339
|
+
sessions: sql4`SUM(${gaSocialReferrals.sessions})`
|
|
9056
9340
|
}).from(gaSocialReferrals).where(and6(
|
|
9057
9341
|
eq17(gaSocialReferrals.projectId, project.id),
|
|
9058
|
-
|
|
9059
|
-
|
|
9342
|
+
sql4`${gaSocialReferrals.date} >= ${daysAgo2(7)}`,
|
|
9343
|
+
sql4`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
9060
9344
|
)).groupBy(gaSocialReferrals.source).all();
|
|
9061
9345
|
const sourcePrev = app.db.select({
|
|
9062
9346
|
source: gaSocialReferrals.source,
|
|
9063
|
-
sessions:
|
|
9347
|
+
sessions: sql4`SUM(${gaSocialReferrals.sessions})`
|
|
9064
9348
|
}).from(gaSocialReferrals).where(and6(
|
|
9065
9349
|
eq17(gaSocialReferrals.projectId, project.id),
|
|
9066
|
-
|
|
9067
|
-
|
|
9350
|
+
sql4`${gaSocialReferrals.date} >= ${daysAgo2(14)}`,
|
|
9351
|
+
sql4`${gaSocialReferrals.date} < ${daysAgo2(7)}`
|
|
9068
9352
|
)).groupBy(gaSocialReferrals.source).all();
|
|
9069
9353
|
const prevMap = new Map(sourcePrev.map((r) => [r.source, r.sessions]));
|
|
9070
9354
|
let biggestMover = null;
|
|
@@ -9103,15 +9387,15 @@ async function ga4Routes(app, opts) {
|
|
|
9103
9387
|
return fmt(d);
|
|
9104
9388
|
};
|
|
9105
9389
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
9106
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
9107
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
9108
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
9390
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql4`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and6(eq17(gaTrafficSnapshots.projectId, project.id), sql4`${gaTrafficSnapshots.date} >= ${from}`, sql4`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
9391
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql4`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and6(eq17(gaTrafficSnapshots.projectId, project.id), sql4`${gaTrafficSnapshots.date} >= ${from}`, sql4`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
9392
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql4`COALESCE(SUM(max_sessions), 0)` }).from(sql4`(
|
|
9109
9393
|
SELECT date, source, medium, MAX(sessions) AS max_sessions
|
|
9110
9394
|
FROM ga_ai_referrals
|
|
9111
9395
|
WHERE project_id = ${project.id} AND date >= ${from} AND date < ${to}
|
|
9112
9396
|
GROUP BY date, source, medium
|
|
9113
9397
|
)`).get();
|
|
9114
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
9398
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql4`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and6(eq17(gaSocialReferrals.projectId, project.id), sql4`${gaSocialReferrals.date} >= ${from}`, sql4`${gaSocialReferrals.date} < ${to}`)).get();
|
|
9115
9399
|
const todayStr = fmt(today);
|
|
9116
9400
|
const buildTrend = (sum) => {
|
|
9117
9401
|
const c7 = sum(daysAgo2(7), todayStr)?.sessions ?? 0;
|
|
@@ -9120,18 +9404,18 @@ async function ga4Routes(app, opts) {
|
|
|
9120
9404
|
const p30 = sum(daysAgo2(60), daysAgo2(30))?.sessions ?? 0;
|
|
9121
9405
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: pct(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: pct(c30, p30) };
|
|
9122
9406
|
};
|
|
9123
|
-
const aiSourceCurrent = app.db.select({ source:
|
|
9407
|
+
const aiSourceCurrent = app.db.select({ source: sql4`source`, sessions: sql4`COALESCE(SUM(max_sessions), 0)` }).from(sql4`(
|
|
9124
9408
|
SELECT date, source, medium, MAX(sessions) AS max_sessions
|
|
9125
9409
|
FROM ga_ai_referrals
|
|
9126
9410
|
WHERE project_id = ${project.id} AND date >= ${daysAgo2(7)} AND date < ${todayStr}
|
|
9127
9411
|
GROUP BY date, source, medium
|
|
9128
|
-
)`).groupBy(
|
|
9129
|
-
const aiSourcePrev = app.db.select({ source:
|
|
9412
|
+
)`).groupBy(sql4`source`).all();
|
|
9413
|
+
const aiSourcePrev = app.db.select({ source: sql4`source`, sessions: sql4`COALESCE(SUM(max_sessions), 0)` }).from(sql4`(
|
|
9130
9414
|
SELECT date, source, medium, MAX(sessions) AS max_sessions
|
|
9131
9415
|
FROM ga_ai_referrals
|
|
9132
9416
|
WHERE project_id = ${project.id} AND date >= ${daysAgo2(14)} AND date < ${daysAgo2(7)}
|
|
9133
9417
|
GROUP BY date, source, medium
|
|
9134
|
-
)`).groupBy(
|
|
9418
|
+
)`).groupBy(sql4`source`).all();
|
|
9135
9419
|
const findBiggestMover = (current, prev) => {
|
|
9136
9420
|
const prevMap = new Map(prev.map((r) => [r.source, r.sessions]));
|
|
9137
9421
|
let mover = null;
|
|
@@ -9146,8 +9430,8 @@ async function ga4Routes(app, opts) {
|
|
|
9146
9430
|
}
|
|
9147
9431
|
return mover;
|
|
9148
9432
|
};
|
|
9149
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
9150
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
9433
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql4`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and6(eq17(gaSocialReferrals.projectId, project.id), sql4`${gaSocialReferrals.date} >= ${daysAgo2(7)}`, sql4`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
9434
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql4`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and6(eq17(gaSocialReferrals.projectId, project.id), sql4`${gaSocialReferrals.date} >= ${daysAgo2(14)}`, sql4`${gaSocialReferrals.date} < ${daysAgo2(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
9151
9435
|
return {
|
|
9152
9436
|
total: buildTrend(sumTotal),
|
|
9153
9437
|
organic: buildTrend(sumOrganic),
|
|
@@ -9162,12 +9446,12 @@ async function ga4Routes(app, opts) {
|
|
|
9162
9446
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
9163
9447
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
9164
9448
|
const conditions = [eq17(gaTrafficSnapshots.projectId, project.id)];
|
|
9165
|
-
if (cutoffDate) conditions.push(
|
|
9449
|
+
if (cutoffDate) conditions.push(sql4`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
9166
9450
|
const rows = app.db.select({
|
|
9167
9451
|
date: gaTrafficSnapshots.date,
|
|
9168
|
-
sessions:
|
|
9169
|
-
organicSessions:
|
|
9170
|
-
users:
|
|
9452
|
+
sessions: sql4`SUM(${gaTrafficSnapshots.sessions})`,
|
|
9453
|
+
organicSessions: sql4`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
9454
|
+
users: sql4`SUM(${gaTrafficSnapshots.users})`
|
|
9171
9455
|
}).from(gaTrafficSnapshots).where(and6(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
9172
9456
|
return rows.map((r) => ({
|
|
9173
9457
|
date: r.date,
|
|
@@ -9181,10 +9465,10 @@ async function ga4Routes(app, opts) {
|
|
|
9181
9465
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
9182
9466
|
const trafficPages = app.db.select({
|
|
9183
9467
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
9184
|
-
sessions:
|
|
9185
|
-
organicSessions:
|
|
9186
|
-
users:
|
|
9187
|
-
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(
|
|
9468
|
+
sessions: sql4`SUM(${gaTrafficSnapshots.sessions})`,
|
|
9469
|
+
organicSessions: sql4`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
9470
|
+
users: sql4`SUM(${gaTrafficSnapshots.users})`
|
|
9471
|
+
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql4`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
9188
9472
|
return {
|
|
9189
9473
|
pages: trafficPages.map((r) => ({
|
|
9190
9474
|
landingPage: r.landingPage,
|
|
@@ -9400,10 +9684,10 @@ function buildAuthErrorMessage(res, responseText) {
|
|
|
9400
9684
|
}
|
|
9401
9685
|
return "WordPress credentials are invalid or lack permission for this action";
|
|
9402
9686
|
}
|
|
9403
|
-
async function fetchJson(connection, siteUrl,
|
|
9687
|
+
async function fetchJson(connection, siteUrl, path16, init) {
|
|
9404
9688
|
if (siteUrl.startsWith("http:")) {
|
|
9405
9689
|
}
|
|
9406
|
-
const res = await fetch(`${normalizeSiteUrl(siteUrl)}${
|
|
9690
|
+
const res = await fetch(`${normalizeSiteUrl(siteUrl)}${path16}`, {
|
|
9407
9691
|
...init,
|
|
9408
9692
|
headers: {
|
|
9409
9693
|
"Authorization": `Basic ${encodeBasicAuth(connection.username, connection.appPassword)}`,
|
|
@@ -10885,6 +11169,566 @@ async function wordpressRoutes(app, opts) {
|
|
|
10885
11169
|
});
|
|
10886
11170
|
}
|
|
10887
11171
|
|
|
11172
|
+
// ../api-routes/src/backlinks.ts
|
|
11173
|
+
import crypto18 from "crypto";
|
|
11174
|
+
import { and as and7, asc as asc2, desc as desc8, eq as eq18, sql as sql5 } from "drizzle-orm";
|
|
11175
|
+
|
|
11176
|
+
// ../integration-commoncrawl/src/constants.ts
|
|
11177
|
+
import os3 from "os";
|
|
11178
|
+
import path3 from "path";
|
|
11179
|
+
var CC_BASE_URL = "https://data.commoncrawl.org/projects/hyperlinkgraph";
|
|
11180
|
+
var PLUGIN_DIR = path3.join(os3.homedir(), ".canonry", "plugins");
|
|
11181
|
+
var PLUGIN_PKG_JSON = path3.join(PLUGIN_DIR, "package.json");
|
|
11182
|
+
var DUCKDB_SPEC = process.env.CANONRY_DUCKDB_SPEC ?? "@duckdb/node-api@1.4.4-r.3";
|
|
11183
|
+
var CC_CACHE_DIR = process.env.CANONRY_CC_CACHE_DIR ?? path3.join(os3.homedir(), ".canonry", "cache", "commoncrawl");
|
|
11184
|
+
var RELEASE_ID_REGEX = /^cc-main-(\d{4})-(jan-feb-mar|apr-may-jun|jul-aug-sep|oct-nov-dec)$/;
|
|
11185
|
+
function ccReleasePaths(release) {
|
|
11186
|
+
const base = `${CC_BASE_URL}/${release}/domain`;
|
|
11187
|
+
const vertexFilename = `${release}-domain-vertices.txt.gz`;
|
|
11188
|
+
const edgesFilename = `${release}-domain-edges.txt.gz`;
|
|
11189
|
+
return {
|
|
11190
|
+
vertexUrl: `${base}/${vertexFilename}`,
|
|
11191
|
+
edgesUrl: `${base}/${edgesFilename}`,
|
|
11192
|
+
vertexFilename,
|
|
11193
|
+
edgesFilename
|
|
11194
|
+
};
|
|
11195
|
+
}
|
|
11196
|
+
|
|
11197
|
+
// ../integration-commoncrawl/src/reverse-domain.ts
|
|
11198
|
+
function reverseDomain(domain) {
|
|
11199
|
+
return domain.split(".").reverse().join(".");
|
|
11200
|
+
}
|
|
11201
|
+
function forwardDomain(revDomain) {
|
|
11202
|
+
return revDomain.split(".").reverse().join(".");
|
|
11203
|
+
}
|
|
11204
|
+
|
|
11205
|
+
// ../integration-commoncrawl/src/release-id.ts
|
|
11206
|
+
function isValidReleaseId(id) {
|
|
11207
|
+
return RELEASE_ID_REGEX.test(id);
|
|
11208
|
+
}
|
|
11209
|
+
|
|
11210
|
+
// ../integration-commoncrawl/src/downloader.ts
|
|
11211
|
+
import { createHash } from "crypto";
|
|
11212
|
+
import { createWriteStream } from "fs";
|
|
11213
|
+
import fs3 from "fs/promises";
|
|
11214
|
+
import path4 from "path";
|
|
11215
|
+
import { pipeline } from "stream/promises";
|
|
11216
|
+
import { Readable, Transform } from "stream";
|
|
11217
|
+
async function downloadFile(opts) {
|
|
11218
|
+
const start = Date.now();
|
|
11219
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
11220
|
+
const sidecarPath = `${opts.destPath}.sha256`;
|
|
11221
|
+
try {
|
|
11222
|
+
const stat = await fs3.stat(opts.destPath);
|
|
11223
|
+
const sidecar = await readSidecar(sidecarPath);
|
|
11224
|
+
const sha2562 = sidecar ?? await hashFile(opts.destPath);
|
|
11225
|
+
if (!sidecar) await writeSidecar(sidecarPath, sha2562);
|
|
11226
|
+
return { bytes: stat.size, sha256: sha2562, cached: true, elapsedMs: Date.now() - start };
|
|
11227
|
+
} catch {
|
|
11228
|
+
}
|
|
11229
|
+
const partialPath = `${opts.destPath}.partial`;
|
|
11230
|
+
await fs3.mkdir(path4.dirname(opts.destPath), { recursive: true });
|
|
11231
|
+
await unlinkIfExists(partialPath);
|
|
11232
|
+
const res = await fetchImpl(opts.url);
|
|
11233
|
+
if (!res.ok || !res.body) {
|
|
11234
|
+
throw new Error(`HTTP ${res.status} ${res.statusText} for ${opts.url}`);
|
|
11235
|
+
}
|
|
11236
|
+
const total = parseContentLength(res.headers.get("content-length"));
|
|
11237
|
+
const hasher = createHash("sha256");
|
|
11238
|
+
let bytes = 0;
|
|
11239
|
+
const hashAndCount = new Transform({
|
|
11240
|
+
transform(chunk, _enc, cb) {
|
|
11241
|
+
hasher.update(chunk);
|
|
11242
|
+
bytes += chunk.length;
|
|
11243
|
+
opts.onProgress?.(bytes, total);
|
|
11244
|
+
cb(null, chunk);
|
|
11245
|
+
}
|
|
11246
|
+
});
|
|
11247
|
+
await pipeline(
|
|
11248
|
+
Readable.fromWeb(res.body),
|
|
11249
|
+
hashAndCount,
|
|
11250
|
+
createWriteStream(partialPath)
|
|
11251
|
+
);
|
|
11252
|
+
const sha256 = hasher.digest("hex");
|
|
11253
|
+
await fs3.rename(partialPath, opts.destPath);
|
|
11254
|
+
await writeSidecar(sidecarPath, sha256);
|
|
11255
|
+
return { bytes, sha256, cached: false, elapsedMs: Date.now() - start };
|
|
11256
|
+
}
|
|
11257
|
+
async function hashFile(filePath) {
|
|
11258
|
+
const hasher = createHash("sha256");
|
|
11259
|
+
const handle = await fs3.open(filePath, "r");
|
|
11260
|
+
try {
|
|
11261
|
+
const stream = handle.createReadStream();
|
|
11262
|
+
for await (const chunk of stream) hasher.update(chunk);
|
|
11263
|
+
} finally {
|
|
11264
|
+
await handle.close();
|
|
11265
|
+
}
|
|
11266
|
+
return hasher.digest("hex");
|
|
11267
|
+
}
|
|
11268
|
+
async function readSidecar(sidecarPath) {
|
|
11269
|
+
try {
|
|
11270
|
+
const raw = await fs3.readFile(sidecarPath, "utf8");
|
|
11271
|
+
const trimmed = raw.trim();
|
|
11272
|
+
return /^[0-9a-f]{64}$/i.test(trimmed) ? trimmed.toLowerCase() : null;
|
|
11273
|
+
} catch {
|
|
11274
|
+
return null;
|
|
11275
|
+
}
|
|
11276
|
+
}
|
|
11277
|
+
async function writeSidecar(sidecarPath, sha256) {
|
|
11278
|
+
await fs3.writeFile(sidecarPath, `${sha256}
|
|
11279
|
+
`);
|
|
11280
|
+
}
|
|
11281
|
+
async function unlinkIfExists(p) {
|
|
11282
|
+
try {
|
|
11283
|
+
await fs3.unlink(p);
|
|
11284
|
+
} catch {
|
|
11285
|
+
}
|
|
11286
|
+
}
|
|
11287
|
+
function parseContentLength(value) {
|
|
11288
|
+
if (!value) return null;
|
|
11289
|
+
const n = Number.parseInt(value, 10);
|
|
11290
|
+
return Number.isFinite(n) ? n : null;
|
|
11291
|
+
}
|
|
11292
|
+
|
|
11293
|
+
// ../integration-commoncrawl/src/plugin-resolver.ts
|
|
11294
|
+
import fs4 from "fs";
|
|
11295
|
+
import { createRequire as createRequire2 } from "module";
|
|
11296
|
+
import path5 from "path";
|
|
11297
|
+
function pluginDirFor(pkgJson) {
|
|
11298
|
+
return path5.dirname(pkgJson);
|
|
11299
|
+
}
|
|
11300
|
+
function duckdbPkgJsonFor(pluginDir) {
|
|
11301
|
+
return path5.join(pluginDir, "node_modules", "@duckdb", "node-api", "package.json");
|
|
11302
|
+
}
|
|
11303
|
+
function loadDuckdb(opts = {}) {
|
|
11304
|
+
const pkgJson = opts.pluginPkgJson ?? PLUGIN_PKG_JSON;
|
|
11305
|
+
const pluginDir = pluginDirFor(pkgJson);
|
|
11306
|
+
const duckdbPkg = duckdbPkgJsonFor(pluginDir);
|
|
11307
|
+
if (!fs4.existsSync(duckdbPkg)) {
|
|
11308
|
+
throw missingDependency(
|
|
11309
|
+
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature.",
|
|
11310
|
+
{ pluginDir }
|
|
11311
|
+
);
|
|
11312
|
+
}
|
|
11313
|
+
try {
|
|
11314
|
+
const pluginRequire = createRequire2(duckdbPkg);
|
|
11315
|
+
return pluginRequire("@duckdb/node-api");
|
|
11316
|
+
} catch {
|
|
11317
|
+
throw missingDependency(
|
|
11318
|
+
"@duckdb/node-api is installed but failed to load. Re-run `canonry backlinks install`.",
|
|
11319
|
+
{ pluginDir }
|
|
11320
|
+
);
|
|
11321
|
+
}
|
|
11322
|
+
}
|
|
11323
|
+
function isDuckdbInstalled(opts = {}) {
|
|
11324
|
+
const pkgJson = opts.pluginPkgJson ?? PLUGIN_PKG_JSON;
|
|
11325
|
+
return fs4.existsSync(duckdbPkgJsonFor(pluginDirFor(pkgJson)));
|
|
11326
|
+
}
|
|
11327
|
+
function readInstalledVersion(opts = {}) {
|
|
11328
|
+
const pluginDir = opts.pluginPkgJson ? pluginDirFor(opts.pluginPkgJson) : PLUGIN_DIR;
|
|
11329
|
+
try {
|
|
11330
|
+
const raw = fs4.readFileSync(duckdbPkgJsonFor(pluginDir), "utf8");
|
|
11331
|
+
const pkg = JSON.parse(raw);
|
|
11332
|
+
return pkg.version ?? null;
|
|
11333
|
+
} catch {
|
|
11334
|
+
return null;
|
|
11335
|
+
}
|
|
11336
|
+
}
|
|
11337
|
+
|
|
11338
|
+
// ../integration-commoncrawl/src/plugin-installer.ts
|
|
11339
|
+
import { spawn } from "child_process";
|
|
11340
|
+
import fs5 from "fs/promises";
|
|
11341
|
+
import path6 from "path";
|
|
11342
|
+
async function installDuckdb(opts = {}) {
|
|
11343
|
+
const pluginDir = opts.pluginDir ?? PLUGIN_DIR;
|
|
11344
|
+
const pluginPkgJson = path6.join(pluginDir, "package.json");
|
|
11345
|
+
const spec = opts.spec ?? DUCKDB_SPEC;
|
|
11346
|
+
const pkgManager = opts.packageManager ?? "npm";
|
|
11347
|
+
await ensurePluginDir(pluginDir, pluginPkgJson);
|
|
11348
|
+
if (isDuckdbInstalled({ pluginPkgJson })) {
|
|
11349
|
+
const version2 = readInstalledVersion({ pluginPkgJson }) ?? "unknown";
|
|
11350
|
+
return { alreadyPresent: true, version: version2, path: pluginDir };
|
|
11351
|
+
}
|
|
11352
|
+
await runInstall(pkgManager, spec, pluginDir, opts.onLog);
|
|
11353
|
+
if (!isDuckdbInstalled({ pluginPkgJson })) {
|
|
11354
|
+
throw new Error(`${pkgManager} install completed but @duckdb/node-api still cannot be resolved from ${pluginDir}`);
|
|
11355
|
+
}
|
|
11356
|
+
const version = readInstalledVersion({ pluginPkgJson }) ?? "unknown";
|
|
11357
|
+
return { alreadyPresent: false, version, path: pluginDir };
|
|
11358
|
+
}
|
|
11359
|
+
async function ensurePluginDir(pluginDir = PLUGIN_DIR, pluginPkgJson = PLUGIN_PKG_JSON) {
|
|
11360
|
+
await fs5.mkdir(pluginDir, { recursive: true });
|
|
11361
|
+
try {
|
|
11362
|
+
await fs5.access(pluginPkgJson);
|
|
11363
|
+
} catch {
|
|
11364
|
+
const contents = JSON.stringify({ name: "canonry-plugins", private: true, dependencies: {} }, null, 2);
|
|
11365
|
+
await fs5.writeFile(pluginPkgJson, `${contents}
|
|
11366
|
+
`);
|
|
11367
|
+
}
|
|
11368
|
+
}
|
|
11369
|
+
async function runInstall(pkgManager, spec, pluginDir, onLog) {
|
|
11370
|
+
const args = pkgManager === "pnpm" ? ["add", spec, "--dir", pluginDir] : ["install", spec, "--prefix", pluginDir];
|
|
11371
|
+
await new Promise((resolve, reject) => {
|
|
11372
|
+
const child = spawn(pkgManager, args, {
|
|
11373
|
+
stdio: onLog ? ["ignore", "pipe", "pipe"] : "inherit"
|
|
11374
|
+
});
|
|
11375
|
+
if (onLog) {
|
|
11376
|
+
child.stdout?.setEncoding("utf8");
|
|
11377
|
+
child.stderr?.setEncoding("utf8");
|
|
11378
|
+
child.stdout?.on("data", (chunk) => {
|
|
11379
|
+
for (const line of chunk.split(/\r?\n/)) {
|
|
11380
|
+
if (line.length > 0) onLog(line);
|
|
11381
|
+
}
|
|
11382
|
+
});
|
|
11383
|
+
child.stderr?.on("data", (chunk) => {
|
|
11384
|
+
for (const line of chunk.split(/\r?\n/)) {
|
|
11385
|
+
if (line.length > 0) onLog(line);
|
|
11386
|
+
}
|
|
11387
|
+
});
|
|
11388
|
+
}
|
|
11389
|
+
child.on("error", reject);
|
|
11390
|
+
child.on("exit", (code) => {
|
|
11391
|
+
if (code === 0) resolve();
|
|
11392
|
+
else reject(new Error(`${pkgManager} install exited with code ${code}`));
|
|
11393
|
+
});
|
|
11394
|
+
});
|
|
11395
|
+
}
|
|
11396
|
+
|
|
11397
|
+
// ../integration-commoncrawl/src/duckdb-query.ts
|
|
11398
|
+
async function queryBacklinks(opts) {
|
|
11399
|
+
if (opts.targets.length === 0) return [];
|
|
11400
|
+
const duckdb = opts.duckdb;
|
|
11401
|
+
const reversed = opts.targets.map(reverseDomain);
|
|
11402
|
+
const targetList = reversed.map(quote).join(", ");
|
|
11403
|
+
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
11404
|
+
const sql10 = `
|
|
11405
|
+
WITH vertices AS (
|
|
11406
|
+
SELECT * FROM read_csv(
|
|
11407
|
+
${quote(opts.vertexPath)},
|
|
11408
|
+
delim=' ', header=false,
|
|
11409
|
+
columns={'id':'BIGINT','rev_domain':'VARCHAR','num_hosts':'BIGINT'}
|
|
11410
|
+
)
|
|
11411
|
+
),
|
|
11412
|
+
targets AS (
|
|
11413
|
+
SELECT v.id AS target_id, v.rev_domain AS target_rev_domain
|
|
11414
|
+
FROM vertices v
|
|
11415
|
+
WHERE v.rev_domain IN (${targetList})
|
|
11416
|
+
),
|
|
11417
|
+
inbound AS (
|
|
11418
|
+
SELECT e.from_id, e.to_id
|
|
11419
|
+
FROM read_csv(
|
|
11420
|
+
${quote(opts.edgesPath)},
|
|
11421
|
+
delim=' ', header=false,
|
|
11422
|
+
columns={'from_id':'BIGINT','to_id':'BIGINT'}
|
|
11423
|
+
) e
|
|
11424
|
+
WHERE e.to_id IN (SELECT target_id FROM targets)
|
|
11425
|
+
)
|
|
11426
|
+
SELECT
|
|
11427
|
+
t.target_rev_domain,
|
|
11428
|
+
v.rev_domain AS linking_rev_domain,
|
|
11429
|
+
v.num_hosts
|
|
11430
|
+
FROM inbound i
|
|
11431
|
+
JOIN targets t ON t.target_id = i.to_id
|
|
11432
|
+
JOIN vertices v ON v.id = i.from_id
|
|
11433
|
+
${limitClause}
|
|
11434
|
+
ORDER BY t.target_rev_domain, v.num_hosts DESC
|
|
11435
|
+
`;
|
|
11436
|
+
const instance = await duckdb.DuckDBInstance.create(":memory:");
|
|
11437
|
+
const conn = await instance.connect();
|
|
11438
|
+
let rows;
|
|
11439
|
+
try {
|
|
11440
|
+
const reader = await conn.runAndReadAll(sql10);
|
|
11441
|
+
rows = reader.getRowObjects();
|
|
11442
|
+
} finally {
|
|
11443
|
+
conn.disconnectSync?.();
|
|
11444
|
+
conn.closeSync?.();
|
|
11445
|
+
instance.closeSync?.();
|
|
11446
|
+
}
|
|
11447
|
+
return rows.map((r) => ({
|
|
11448
|
+
targetDomain: forwardDomain(String(r["target_rev_domain"])),
|
|
11449
|
+
linkingDomain: forwardDomain(String(r["linking_rev_domain"])),
|
|
11450
|
+
numHosts: Number(r["num_hosts"])
|
|
11451
|
+
}));
|
|
11452
|
+
}
|
|
11453
|
+
function quote(s) {
|
|
11454
|
+
return `'${s.replace(/'/g, "''")}'`;
|
|
11455
|
+
}
|
|
11456
|
+
|
|
11457
|
+
// ../integration-commoncrawl/src/cache.ts
|
|
11458
|
+
import fs6 from "fs";
|
|
11459
|
+
import path7 from "path";
|
|
11460
|
+
function cacheRoot(opts = {}) {
|
|
11461
|
+
return opts.cacheDir ?? CC_CACHE_DIR;
|
|
11462
|
+
}
|
|
11463
|
+
function directoryBytesAndLastUsed(dir) {
|
|
11464
|
+
let bytes = 0;
|
|
11465
|
+
let latestMtimeMs = 0;
|
|
11466
|
+
const walk = (p) => {
|
|
11467
|
+
let stat;
|
|
11468
|
+
try {
|
|
11469
|
+
stat = fs6.statSync(p);
|
|
11470
|
+
} catch {
|
|
11471
|
+
return;
|
|
11472
|
+
}
|
|
11473
|
+
if (stat.isDirectory()) {
|
|
11474
|
+
let entries;
|
|
11475
|
+
try {
|
|
11476
|
+
entries = fs6.readdirSync(p);
|
|
11477
|
+
} catch {
|
|
11478
|
+
return;
|
|
11479
|
+
}
|
|
11480
|
+
for (const e of entries) walk(path7.join(p, e));
|
|
11481
|
+
} else if (stat.isFile()) {
|
|
11482
|
+
bytes += stat.size;
|
|
11483
|
+
const mtime = Math.max(stat.mtimeMs, stat.atimeMs);
|
|
11484
|
+
if (mtime > latestMtimeMs) latestMtimeMs = mtime;
|
|
11485
|
+
}
|
|
11486
|
+
};
|
|
11487
|
+
walk(dir);
|
|
11488
|
+
return {
|
|
11489
|
+
bytes,
|
|
11490
|
+
lastUsedAt: latestMtimeMs > 0 ? new Date(latestMtimeMs).toISOString() : null
|
|
11491
|
+
};
|
|
11492
|
+
}
|
|
11493
|
+
function listCachedReleases(opts = {}) {
|
|
11494
|
+
const root = cacheRoot(opts);
|
|
11495
|
+
if (!fs6.existsSync(root)) return [];
|
|
11496
|
+
const entries = fs6.readdirSync(root, { withFileTypes: true });
|
|
11497
|
+
const result = [];
|
|
11498
|
+
for (const entry of entries) {
|
|
11499
|
+
if (!entry.isDirectory()) continue;
|
|
11500
|
+
if (!RELEASE_ID_REGEX.test(entry.name)) continue;
|
|
11501
|
+
const dir = path7.join(root, entry.name);
|
|
11502
|
+
const stats = directoryBytesAndLastUsed(dir);
|
|
11503
|
+
result.push({ release: entry.name, bytes: stats.bytes, lastUsedAt: stats.lastUsedAt });
|
|
11504
|
+
}
|
|
11505
|
+
result.sort((a, b) => (b.lastUsedAt ?? "").localeCompare(a.lastUsedAt ?? ""));
|
|
11506
|
+
return result;
|
|
11507
|
+
}
|
|
11508
|
+
function pruneCachedRelease(release, opts = {}) {
|
|
11509
|
+
if (!RELEASE_ID_REGEX.test(release)) {
|
|
11510
|
+
throw new Error(`Invalid release id: ${release}`);
|
|
11511
|
+
}
|
|
11512
|
+
const dir = path7.join(cacheRoot(opts), release);
|
|
11513
|
+
fs6.rmSync(dir, { recursive: true, force: true });
|
|
11514
|
+
}
|
|
11515
|
+
|
|
11516
|
+
// ../api-routes/src/backlinks.ts
|
|
11517
|
+
var BACKLINKS_UNSUPPORTED_MESSAGE = "Backlinks sync and install are only available from a local canonry install. Run `canonry backlinks install` locally to use this feature.";
|
|
11518
|
+
var NON_TERMINAL_SYNC_STATUSES = /* @__PURE__ */ new Set([
|
|
11519
|
+
CcReleaseSyncStatuses.queued,
|
|
11520
|
+
CcReleaseSyncStatuses.downloading,
|
|
11521
|
+
CcReleaseSyncStatuses.querying
|
|
11522
|
+
]);
|
|
11523
|
+
function mapSyncRow(row) {
|
|
11524
|
+
return {
|
|
11525
|
+
id: row.id,
|
|
11526
|
+
release: row.release,
|
|
11527
|
+
status: row.status,
|
|
11528
|
+
phaseDetail: row.phaseDetail ?? null,
|
|
11529
|
+
vertexPath: row.vertexPath ?? null,
|
|
11530
|
+
edgesPath: row.edgesPath ?? null,
|
|
11531
|
+
vertexSha256: row.vertexSha256 ?? null,
|
|
11532
|
+
edgesSha256: row.edgesSha256 ?? null,
|
|
11533
|
+
vertexBytes: row.vertexBytes ?? null,
|
|
11534
|
+
edgesBytes: row.edgesBytes ?? null,
|
|
11535
|
+
projectsProcessed: row.projectsProcessed ?? null,
|
|
11536
|
+
domainsDiscovered: row.domainsDiscovered ?? null,
|
|
11537
|
+
downloadStartedAt: row.downloadStartedAt ?? null,
|
|
11538
|
+
downloadFinishedAt: row.downloadFinishedAt ?? null,
|
|
11539
|
+
queryStartedAt: row.queryStartedAt ?? null,
|
|
11540
|
+
queryFinishedAt: row.queryFinishedAt ?? null,
|
|
11541
|
+
error: row.error ?? null,
|
|
11542
|
+
createdAt: row.createdAt,
|
|
11543
|
+
updatedAt: row.updatedAt
|
|
11544
|
+
};
|
|
11545
|
+
}
|
|
11546
|
+
function mapSummaryRow(row) {
|
|
11547
|
+
return {
|
|
11548
|
+
projectId: row.projectId,
|
|
11549
|
+
release: row.release,
|
|
11550
|
+
targetDomain: row.targetDomain,
|
|
11551
|
+
totalLinkingDomains: row.totalLinkingDomains,
|
|
11552
|
+
totalHosts: row.totalHosts,
|
|
11553
|
+
top10HostsShare: row.top10HostsShare,
|
|
11554
|
+
queriedAt: row.queriedAt
|
|
11555
|
+
};
|
|
11556
|
+
}
|
|
11557
|
+
function mapRunRow(row) {
|
|
11558
|
+
return {
|
|
11559
|
+
id: row.id,
|
|
11560
|
+
projectId: row.projectId,
|
|
11561
|
+
kind: row.kind,
|
|
11562
|
+
status: row.status,
|
|
11563
|
+
trigger: row.trigger,
|
|
11564
|
+
location: row.location ?? null,
|
|
11565
|
+
startedAt: row.startedAt ?? null,
|
|
11566
|
+
finishedAt: row.finishedAt ?? null,
|
|
11567
|
+
error: row.error ?? null,
|
|
11568
|
+
createdAt: row.createdAt
|
|
11569
|
+
};
|
|
11570
|
+
}
|
|
11571
|
+
function latestSummaryForProject(db, projectId, release) {
|
|
11572
|
+
const condition = release ? and7(eq18(backlinkSummaries.projectId, projectId), eq18(backlinkSummaries.release, release)) : eq18(backlinkSummaries.projectId, projectId);
|
|
11573
|
+
return db.select().from(backlinkSummaries).where(condition).orderBy(desc8(backlinkSummaries.queriedAt)).limit(1).get();
|
|
11574
|
+
}
|
|
11575
|
+
async function backlinksRoutes(app, opts) {
|
|
11576
|
+
app.get("/backlinks/status", async (_request, reply) => {
|
|
11577
|
+
if (!opts.getBacklinksStatus) {
|
|
11578
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11579
|
+
}
|
|
11580
|
+
return reply.send(opts.getBacklinksStatus());
|
|
11581
|
+
});
|
|
11582
|
+
app.post("/backlinks/install", async (_request, reply) => {
|
|
11583
|
+
if (!opts.onInstallBacklinks) {
|
|
11584
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11585
|
+
}
|
|
11586
|
+
const result = await opts.onInstallBacklinks();
|
|
11587
|
+
return reply.status(200).send(result);
|
|
11588
|
+
});
|
|
11589
|
+
app.post("/backlinks/syncs", async (request, reply) => {
|
|
11590
|
+
const release = request.body?.release;
|
|
11591
|
+
if (!release || !isValidReleaseId(release)) {
|
|
11592
|
+
throw validationError("Invalid release id. Expected form: cc-main-YYYY-{jan-feb-mar,apr-may-jun,jul-aug-sep,oct-nov-dec}");
|
|
11593
|
+
}
|
|
11594
|
+
if (!opts.getBacklinksStatus || !opts.onReleaseSyncRequested) {
|
|
11595
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11596
|
+
}
|
|
11597
|
+
if (!opts.getBacklinksStatus().duckdbInstalled) {
|
|
11598
|
+
throw missingDependency(
|
|
11599
|
+
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature."
|
|
11600
|
+
);
|
|
11601
|
+
}
|
|
11602
|
+
const existing = app.db.select().from(ccReleaseSyncs).where(eq18(ccReleaseSyncs.release, release)).get();
|
|
11603
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11604
|
+
if (existing) {
|
|
11605
|
+
if (NON_TERMINAL_SYNC_STATUSES.has(existing.status)) {
|
|
11606
|
+
return reply.status(200).send(mapSyncRow(existing));
|
|
11607
|
+
}
|
|
11608
|
+
app.db.update(ccReleaseSyncs).set({
|
|
11609
|
+
status: CcReleaseSyncStatuses.queued,
|
|
11610
|
+
phaseDetail: null,
|
|
11611
|
+
error: null,
|
|
11612
|
+
updatedAt: now
|
|
11613
|
+
}).where(eq18(ccReleaseSyncs.id, existing.id)).run();
|
|
11614
|
+
opts.onReleaseSyncRequested(existing.id, release);
|
|
11615
|
+
const refreshed = app.db.select().from(ccReleaseSyncs).where(eq18(ccReleaseSyncs.id, existing.id)).get();
|
|
11616
|
+
return reply.status(200).send(mapSyncRow(refreshed));
|
|
11617
|
+
}
|
|
11618
|
+
const id = crypto18.randomUUID();
|
|
11619
|
+
app.db.insert(ccReleaseSyncs).values({
|
|
11620
|
+
id,
|
|
11621
|
+
release,
|
|
11622
|
+
status: CcReleaseSyncStatuses.queued,
|
|
11623
|
+
createdAt: now,
|
|
11624
|
+
updatedAt: now
|
|
11625
|
+
}).run();
|
|
11626
|
+
opts.onReleaseSyncRequested(id, release);
|
|
11627
|
+
const inserted = app.db.select().from(ccReleaseSyncs).where(eq18(ccReleaseSyncs.id, id)).get();
|
|
11628
|
+
return reply.status(201).send(mapSyncRow(inserted));
|
|
11629
|
+
});
|
|
11630
|
+
app.get("/backlinks/syncs/latest", async (_request, reply) => {
|
|
11631
|
+
const row = app.db.select().from(ccReleaseSyncs).orderBy(desc8(ccReleaseSyncs.updatedAt)).limit(1).get();
|
|
11632
|
+
return reply.send(row ? mapSyncRow(row) : null);
|
|
11633
|
+
});
|
|
11634
|
+
app.get("/backlinks/syncs", async (_request, reply) => {
|
|
11635
|
+
const rows = app.db.select().from(ccReleaseSyncs).orderBy(desc8(ccReleaseSyncs.updatedAt)).all();
|
|
11636
|
+
return reply.send(rows.map(mapSyncRow));
|
|
11637
|
+
});
|
|
11638
|
+
app.get("/backlinks/releases", async (_request, reply) => {
|
|
11639
|
+
const releases = opts.listCachedReleases?.() ?? [];
|
|
11640
|
+
return reply.send(releases);
|
|
11641
|
+
});
|
|
11642
|
+
app.delete("/backlinks/cache/:release", async (request, reply) => {
|
|
11643
|
+
const release = request.params.release;
|
|
11644
|
+
if (!isValidReleaseId(release)) {
|
|
11645
|
+
throw validationError("Invalid release id");
|
|
11646
|
+
}
|
|
11647
|
+
if (!opts.onBacklinksPruneCache) {
|
|
11648
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11649
|
+
}
|
|
11650
|
+
opts.onBacklinksPruneCache(release);
|
|
11651
|
+
return reply.send({ ok: true });
|
|
11652
|
+
});
|
|
11653
|
+
app.post("/projects/:name/backlinks/extract", async (request, reply) => {
|
|
11654
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11655
|
+
if (!opts.getBacklinksStatus || !opts.onBacklinkExtractRequested) {
|
|
11656
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11657
|
+
}
|
|
11658
|
+
if (!opts.getBacklinksStatus().duckdbInstalled) {
|
|
11659
|
+
throw missingDependency(
|
|
11660
|
+
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature."
|
|
11661
|
+
);
|
|
11662
|
+
}
|
|
11663
|
+
const release = request.body?.release;
|
|
11664
|
+
if (release !== void 0 && !isValidReleaseId(release)) {
|
|
11665
|
+
throw validationError("Invalid release id");
|
|
11666
|
+
}
|
|
11667
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11668
|
+
const runId = crypto18.randomUUID();
|
|
11669
|
+
app.db.insert(runs).values({
|
|
11670
|
+
id: runId,
|
|
11671
|
+
projectId: project.id,
|
|
11672
|
+
kind: RunKinds["backlink-extract"],
|
|
11673
|
+
status: RunStatuses.queued,
|
|
11674
|
+
trigger: RunTriggers.manual,
|
|
11675
|
+
createdAt: now
|
|
11676
|
+
}).run();
|
|
11677
|
+
opts.onBacklinkExtractRequested(runId, project.id, release);
|
|
11678
|
+
const run = app.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
11679
|
+
return reply.status(201).send(mapRunRow(run));
|
|
11680
|
+
});
|
|
11681
|
+
app.get(
|
|
11682
|
+
"/projects/:name/backlinks/summary",
|
|
11683
|
+
async (request, reply) => {
|
|
11684
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11685
|
+
const row = latestSummaryForProject(app.db, project.id, request.query.release);
|
|
11686
|
+
return reply.send(row ? mapSummaryRow(row) : null);
|
|
11687
|
+
}
|
|
11688
|
+
);
|
|
11689
|
+
app.get("/projects/:name/backlinks/domains", async (request, reply) => {
|
|
11690
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11691
|
+
const summaryRow = latestSummaryForProject(app.db, project.id, request.query.release);
|
|
11692
|
+
const targetRelease = request.query.release ?? summaryRow?.release;
|
|
11693
|
+
if (!targetRelease) {
|
|
11694
|
+
const response2 = { summary: null, total: 0, rows: [] };
|
|
11695
|
+
return reply.send(response2);
|
|
11696
|
+
}
|
|
11697
|
+
const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
|
|
11698
|
+
const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
|
|
11699
|
+
const domainCondition = and7(
|
|
11700
|
+
eq18(backlinkDomains.projectId, project.id),
|
|
11701
|
+
eq18(backlinkDomains.release, targetRelease)
|
|
11702
|
+
);
|
|
11703
|
+
const totalRow = app.db.select({ count: sql5`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
11704
|
+
const rows = app.db.select({
|
|
11705
|
+
linkingDomain: backlinkDomains.linkingDomain,
|
|
11706
|
+
numHosts: backlinkDomains.numHosts
|
|
11707
|
+
}).from(backlinkDomains).where(domainCondition).orderBy(desc8(backlinkDomains.numHosts)).limit(limit).offset(offset).all();
|
|
11708
|
+
const response = {
|
|
11709
|
+
summary: summaryRow ? mapSummaryRow(summaryRow) : null,
|
|
11710
|
+
total: Number(totalRow?.count ?? 0),
|
|
11711
|
+
rows
|
|
11712
|
+
};
|
|
11713
|
+
return reply.send(response);
|
|
11714
|
+
});
|
|
11715
|
+
app.get(
|
|
11716
|
+
"/projects/:name/backlinks/history",
|
|
11717
|
+
async (request, reply) => {
|
|
11718
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11719
|
+
const rows = app.db.select().from(backlinkSummaries).where(eq18(backlinkSummaries.projectId, project.id)).orderBy(asc2(backlinkSummaries.queriedAt)).all();
|
|
11720
|
+
const response = rows.map((r) => ({
|
|
11721
|
+
release: r.release,
|
|
11722
|
+
totalLinkingDomains: r.totalLinkingDomains,
|
|
11723
|
+
totalHosts: r.totalHosts,
|
|
11724
|
+
top10HostsShare: r.top10HostsShare,
|
|
11725
|
+
queriedAt: r.queriedAt
|
|
11726
|
+
}));
|
|
11727
|
+
return reply.send(response);
|
|
11728
|
+
}
|
|
11729
|
+
);
|
|
11730
|
+
}
|
|
11731
|
+
|
|
10888
11732
|
// ../api-routes/src/index.ts
|
|
10889
11733
|
async function apiRoutes(app, opts) {
|
|
10890
11734
|
app.decorate("db", opts.db);
|
|
@@ -10993,6 +11837,14 @@ async function apiRoutes(app, opts) {
|
|
|
10993
11837
|
googleConnectionStore: opts.googleConnectionStore,
|
|
10994
11838
|
getGoogleAuthConfig: opts.getGoogleAuthConfig
|
|
10995
11839
|
});
|
|
11840
|
+
await api.register(backlinksRoutes, {
|
|
11841
|
+
getBacklinksStatus: opts.getBacklinksStatus,
|
|
11842
|
+
onInstallBacklinks: opts.onInstallBacklinks,
|
|
11843
|
+
onReleaseSyncRequested: opts.onReleaseSyncRequested,
|
|
11844
|
+
onBacklinkExtractRequested: opts.onBacklinkExtractRequested,
|
|
11845
|
+
onBacklinksPruneCache: opts.onBacklinksPruneCache,
|
|
11846
|
+
listCachedReleases: opts.listCachedReleases
|
|
11847
|
+
});
|
|
10996
11848
|
if (opts.registerAuthenticatedRoutes) {
|
|
10997
11849
|
await opts.registerAuthenticatedRoutes(api);
|
|
10998
11850
|
}
|
|
@@ -11000,7 +11852,7 @@ async function apiRoutes(app, opts) {
|
|
|
11000
11852
|
}
|
|
11001
11853
|
|
|
11002
11854
|
// src/server.ts
|
|
11003
|
-
import
|
|
11855
|
+
import os6 from "os";
|
|
11004
11856
|
|
|
11005
11857
|
// ../provider-gemini/src/normalize.ts
|
|
11006
11858
|
import { GoogleGenAI } from "@google/genai";
|
|
@@ -12388,8 +13240,8 @@ var localAdapter = {
|
|
|
12388
13240
|
};
|
|
12389
13241
|
|
|
12390
13242
|
// ../provider-cdp/src/adapter.ts
|
|
12391
|
-
import
|
|
12392
|
-
import
|
|
13243
|
+
import path9 from "path";
|
|
13244
|
+
import os4 from "os";
|
|
12393
13245
|
|
|
12394
13246
|
// ../provider-cdp/src/connection.ts
|
|
12395
13247
|
import CDP from "chrome-remote-interface";
|
|
@@ -12753,12 +13605,12 @@ function sleep2(ms) {
|
|
|
12753
13605
|
}
|
|
12754
13606
|
|
|
12755
13607
|
// ../provider-cdp/src/screenshot.ts
|
|
12756
|
-
import
|
|
12757
|
-
import
|
|
13608
|
+
import fs7 from "fs";
|
|
13609
|
+
import path8 from "path";
|
|
12758
13610
|
async function captureElementScreenshot(client, selector, outputPath) {
|
|
12759
|
-
const dir =
|
|
12760
|
-
if (!
|
|
12761
|
-
|
|
13611
|
+
const dir = path8.dirname(outputPath);
|
|
13612
|
+
if (!fs7.existsSync(dir)) {
|
|
13613
|
+
fs7.mkdirSync(dir, { recursive: true });
|
|
12762
13614
|
}
|
|
12763
13615
|
let clip;
|
|
12764
13616
|
try {
|
|
@@ -12792,7 +13644,7 @@ async function captureElementScreenshot(client, selector, outputPath) {
|
|
|
12792
13644
|
}
|
|
12793
13645
|
const { data } = await client.Page.captureScreenshot(screenshotParams);
|
|
12794
13646
|
const buffer = Buffer.from(data, "base64");
|
|
12795
|
-
|
|
13647
|
+
fs7.writeFileSync(outputPath, buffer);
|
|
12796
13648
|
return outputPath;
|
|
12797
13649
|
}
|
|
12798
13650
|
|
|
@@ -12853,7 +13705,7 @@ function getConnection(config) {
|
|
|
12853
13705
|
return conn;
|
|
12854
13706
|
}
|
|
12855
13707
|
function getScreenshotDir2() {
|
|
12856
|
-
return
|
|
13708
|
+
return path9.join(os4.homedir(), ".canonry", "screenshots");
|
|
12857
13709
|
}
|
|
12858
13710
|
var cdpChatgptAdapter = {
|
|
12859
13711
|
name: "cdp:chatgpt",
|
|
@@ -12917,7 +13769,7 @@ var cdpChatgptAdapter = {
|
|
|
12917
13769
|
const answerText = await target.extractAnswer(client);
|
|
12918
13770
|
const groundingSources = await target.extractCitations(client);
|
|
12919
13771
|
const screenshotId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
12920
|
-
const screenshotPath =
|
|
13772
|
+
const screenshotPath = path9.join(getScreenshotDir2(), `${screenshotId}.png`);
|
|
12921
13773
|
let capturedScreenshotPath;
|
|
12922
13774
|
try {
|
|
12923
13775
|
capturedScreenshotPath = await captureElementScreenshot(
|
|
@@ -13453,11 +14305,11 @@ function removeWordpressConnection(config, projectName) {
|
|
|
13453
14305
|
}
|
|
13454
14306
|
|
|
13455
14307
|
// src/job-runner.ts
|
|
13456
|
-
import
|
|
13457
|
-
import
|
|
13458
|
-
import
|
|
13459
|
-
import
|
|
13460
|
-
import { and as
|
|
14308
|
+
import crypto19 from "crypto";
|
|
14309
|
+
import fs8 from "fs";
|
|
14310
|
+
import path10 from "path";
|
|
14311
|
+
import os5 from "os";
|
|
14312
|
+
import { and as and8, eq as eq19, inArray as inArray3, sql as sql6 } from "drizzle-orm";
|
|
13461
14313
|
|
|
13462
14314
|
// src/citation-utils.ts
|
|
13463
14315
|
function domainMatches(domain, canonicalDomain) {
|
|
@@ -13693,7 +14545,7 @@ var JobRunner = class {
|
|
|
13693
14545
|
if (stale.length === 0) return;
|
|
13694
14546
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13695
14547
|
for (const run of stale) {
|
|
13696
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
14548
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq19(runs.id, run.id)).run();
|
|
13697
14549
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
13698
14550
|
}
|
|
13699
14551
|
}
|
|
@@ -13721,10 +14573,10 @@ var JobRunner = class {
|
|
|
13721
14573
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
13722
14574
|
}
|
|
13723
14575
|
if (existingRun.status === "queued") {
|
|
13724
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
14576
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and8(eq19(runs.id, runId), eq19(runs.status, "queued"))).run();
|
|
13725
14577
|
}
|
|
13726
14578
|
this.throwIfRunCancelled(runId);
|
|
13727
|
-
const project = this.db.select().from(projects).where(
|
|
14579
|
+
const project = this.db.select().from(projects).where(eq19(projects.id, projectId)).get();
|
|
13728
14580
|
if (!project) {
|
|
13729
14581
|
throw new Error(`Project ${projectId} not found`);
|
|
13730
14582
|
}
|
|
@@ -13744,8 +14596,8 @@ var JobRunner = class {
|
|
|
13744
14596
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
13745
14597
|
}
|
|
13746
14598
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
13747
|
-
projectKeywords = this.db.select().from(keywords).where(
|
|
13748
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
14599
|
+
projectKeywords = this.db.select().from(keywords).where(eq19(keywords.projectId, projectId)).all();
|
|
14600
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq19(competitors.projectId, projectId)).all();
|
|
13749
14601
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
13750
14602
|
const allDomains = effectiveDomains({
|
|
13751
14603
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -13761,7 +14613,7 @@ var JobRunner = class {
|
|
|
13761
14613
|
const todayPeriod = getCurrentUsageDay();
|
|
13762
14614
|
for (const p of activeProviders) {
|
|
13763
14615
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
13764
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
14616
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq19(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
13765
14617
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
13766
14618
|
if (providerUsage + queriesPerProvider > limit) {
|
|
13767
14619
|
throw new Error(
|
|
@@ -13820,12 +14672,12 @@ var JobRunner = class {
|
|
|
13820
14672
|
competitorDomains
|
|
13821
14673
|
);
|
|
13822
14674
|
let screenshotRelPath = null;
|
|
13823
|
-
if (raw.screenshotPath &&
|
|
13824
|
-
const snapshotId =
|
|
13825
|
-
const screenshotDir =
|
|
13826
|
-
if (!
|
|
13827
|
-
const destPath =
|
|
13828
|
-
|
|
14675
|
+
if (raw.screenshotPath && fs8.existsSync(raw.screenshotPath)) {
|
|
14676
|
+
const snapshotId = crypto19.randomUUID();
|
|
14677
|
+
const screenshotDir = path10.join(os5.homedir(), ".canonry", "screenshots", runId);
|
|
14678
|
+
if (!fs8.existsSync(screenshotDir)) fs8.mkdirSync(screenshotDir, { recursive: true });
|
|
14679
|
+
const destPath = path10.join(screenshotDir, `${snapshotId}.png`);
|
|
14680
|
+
fs8.renameSync(raw.screenshotPath, destPath);
|
|
13829
14681
|
screenshotRelPath = `${runId}/${snapshotId}.png`;
|
|
13830
14682
|
this.db.insert(querySnapshots).values({
|
|
13831
14683
|
id: snapshotId,
|
|
@@ -13851,7 +14703,7 @@ var JobRunner = class {
|
|
|
13851
14703
|
}).run();
|
|
13852
14704
|
} else {
|
|
13853
14705
|
this.db.insert(querySnapshots).values({
|
|
13854
|
-
id:
|
|
14706
|
+
id: crypto19.randomUUID(),
|
|
13855
14707
|
runId,
|
|
13856
14708
|
keywordId: kw.id,
|
|
13857
14709
|
provider: providerName,
|
|
@@ -13902,12 +14754,12 @@ var JobRunner = class {
|
|
|
13902
14754
|
const someFailed = providerErrors.size > 0;
|
|
13903
14755
|
if (allFailed) {
|
|
13904
14756
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
13905
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
14757
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq19(runs.id, runId)).run();
|
|
13906
14758
|
} else if (someFailed) {
|
|
13907
14759
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
13908
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
14760
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq19(runs.id, runId)).run();
|
|
13909
14761
|
} else {
|
|
13910
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14762
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(runs.id, runId)).run();
|
|
13911
14763
|
}
|
|
13912
14764
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
13913
14765
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -13942,7 +14794,7 @@ var JobRunner = class {
|
|
|
13942
14794
|
status: "failed",
|
|
13943
14795
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13944
14796
|
error: errorMessage
|
|
13945
|
-
}).where(
|
|
14797
|
+
}).where(eq19(runs.id, runId)).run();
|
|
13946
14798
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
13947
14799
|
trackEvent("run.completed", {
|
|
13948
14800
|
status: "failed",
|
|
@@ -13963,7 +14815,7 @@ var JobRunner = class {
|
|
|
13963
14815
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13964
14816
|
const period = now.slice(0, 10);
|
|
13965
14817
|
this.db.insert(usageCounters).values({
|
|
13966
|
-
id:
|
|
14818
|
+
id: crypto19.randomUUID(),
|
|
13967
14819
|
scope,
|
|
13968
14820
|
period,
|
|
13969
14821
|
metric,
|
|
@@ -13971,7 +14823,7 @@ var JobRunner = class {
|
|
|
13971
14823
|
updatedAt: now
|
|
13972
14824
|
}).onConflictDoUpdate({
|
|
13973
14825
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
13974
|
-
set: { count:
|
|
14826
|
+
set: { count: sql6`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
13975
14827
|
}).run();
|
|
13976
14828
|
}
|
|
13977
14829
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -13985,7 +14837,7 @@ var JobRunner = class {
|
|
|
13985
14837
|
status: runs.status,
|
|
13986
14838
|
finishedAt: runs.finishedAt,
|
|
13987
14839
|
error: runs.error
|
|
13988
|
-
}).from(runs).where(
|
|
14840
|
+
}).from(runs).where(eq19(runs.id, runId)).get();
|
|
13989
14841
|
}
|
|
13990
14842
|
isRunCancelled(runId) {
|
|
13991
14843
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -14001,7 +14853,7 @@ var JobRunner = class {
|
|
|
14001
14853
|
this.db.update(runs).set({
|
|
14002
14854
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14003
14855
|
error: currentRun.error ?? "Cancelled by user"
|
|
14004
|
-
}).where(
|
|
14856
|
+
}).where(eq19(runs.id, runId)).run();
|
|
14005
14857
|
}
|
|
14006
14858
|
trackEvent("run.completed", {
|
|
14007
14859
|
status: "cancelled",
|
|
@@ -14023,8 +14875,8 @@ function getCurrentUsageDay() {
|
|
|
14023
14875
|
}
|
|
14024
14876
|
|
|
14025
14877
|
// src/gsc-sync.ts
|
|
14026
|
-
import
|
|
14027
|
-
import { eq as
|
|
14878
|
+
import crypto20 from "crypto";
|
|
14879
|
+
import { eq as eq20, and as and9, sql as sql7 } from "drizzle-orm";
|
|
14028
14880
|
var log2 = createLogger("GscSync");
|
|
14029
14881
|
function formatDate2(d) {
|
|
14030
14882
|
return d.toISOString().split("T")[0];
|
|
@@ -14036,13 +14888,13 @@ function daysAgo(n) {
|
|
|
14036
14888
|
}
|
|
14037
14889
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
14038
14890
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14039
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
14891
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq20(runs.id, runId)).run();
|
|
14040
14892
|
try {
|
|
14041
14893
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
14042
14894
|
if (!googleClientId || !googleClientSecret) {
|
|
14043
14895
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
14044
14896
|
}
|
|
14045
|
-
const project = db.select().from(projects).where(
|
|
14897
|
+
const project = db.select().from(projects).where(eq20(projects.id, projectId)).get();
|
|
14046
14898
|
if (!project) {
|
|
14047
14899
|
throw new Error(`Project not found: ${projectId}`);
|
|
14048
14900
|
}
|
|
@@ -14076,10 +14928,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14076
14928
|
});
|
|
14077
14929
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
14078
14930
|
db.delete(gscSearchData).where(
|
|
14079
|
-
|
|
14080
|
-
|
|
14081
|
-
|
|
14082
|
-
|
|
14931
|
+
and9(
|
|
14932
|
+
eq20(gscSearchData.projectId, projectId),
|
|
14933
|
+
sql7`${gscSearchData.date} >= ${startDate}`,
|
|
14934
|
+
sql7`${gscSearchData.date} <= ${endDate}`
|
|
14083
14935
|
)
|
|
14084
14936
|
).run();
|
|
14085
14937
|
const batchSize = 500;
|
|
@@ -14089,7 +14941,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14089
14941
|
for (const row of batch) {
|
|
14090
14942
|
const [query, page, country, device, date] = row.keys;
|
|
14091
14943
|
db.insert(gscSearchData).values({
|
|
14092
|
-
id:
|
|
14944
|
+
id: crypto20.randomUUID(),
|
|
14093
14945
|
projectId,
|
|
14094
14946
|
syncRunId: runId,
|
|
14095
14947
|
date: date ?? "",
|
|
@@ -14123,7 +14975,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14123
14975
|
const rich = ir.richResultsResult;
|
|
14124
14976
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14125
14977
|
db.insert(gscUrlInspections).values({
|
|
14126
|
-
id:
|
|
14978
|
+
id: crypto20.randomUUID(),
|
|
14127
14979
|
projectId,
|
|
14128
14980
|
syncRunId: runId,
|
|
14129
14981
|
url: pageUrl,
|
|
@@ -14144,7 +14996,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14144
14996
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
14145
14997
|
}
|
|
14146
14998
|
}
|
|
14147
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
14999
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq20(gscUrlInspections.projectId, projectId)).all();
|
|
14148
15000
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
14149
15001
|
for (const row of allInspections) {
|
|
14150
15002
|
const existing = latestByUrl.get(row.url);
|
|
@@ -14165,9 +15017,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14165
15017
|
}
|
|
14166
15018
|
}
|
|
14167
15019
|
const snapshotDate = formatDate2(/* @__PURE__ */ new Date());
|
|
14168
|
-
db.delete(gscCoverageSnapshots).where(
|
|
15020
|
+
db.delete(gscCoverageSnapshots).where(and9(eq20(gscCoverageSnapshots.projectId, projectId), eq20(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
14169
15021
|
db.insert(gscCoverageSnapshots).values({
|
|
14170
|
-
id:
|
|
15022
|
+
id: crypto20.randomUUID(),
|
|
14171
15023
|
projectId,
|
|
14172
15024
|
syncRunId: runId,
|
|
14173
15025
|
date: snapshotDate,
|
|
@@ -14176,19 +15028,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14176
15028
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
14177
15029
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14178
15030
|
}).run();
|
|
14179
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15031
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(runs.id, runId)).run();
|
|
14180
15032
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
14181
15033
|
} catch (err) {
|
|
14182
15034
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
14183
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15035
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(runs.id, runId)).run();
|
|
14184
15036
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
14185
15037
|
throw err;
|
|
14186
15038
|
}
|
|
14187
15039
|
}
|
|
14188
15040
|
|
|
14189
15041
|
// src/gsc-inspect-sitemap.ts
|
|
14190
|
-
import
|
|
14191
|
-
import { eq as
|
|
15042
|
+
import crypto21 from "crypto";
|
|
15043
|
+
import { eq as eq21, and as and10 } from "drizzle-orm";
|
|
14192
15044
|
|
|
14193
15045
|
// src/sitemap-parser.ts
|
|
14194
15046
|
var LOC_REGEX = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
|
|
@@ -14257,13 +15109,13 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
14257
15109
|
var log3 = createLogger("InspectSitemap");
|
|
14258
15110
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
14259
15111
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14260
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
15112
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq21(runs.id, runId)).run();
|
|
14261
15113
|
try {
|
|
14262
15114
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
14263
15115
|
if (!googleClientId || !googleClientSecret) {
|
|
14264
15116
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
14265
15117
|
}
|
|
14266
|
-
const project = db.select().from(projects).where(
|
|
15118
|
+
const project = db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
14267
15119
|
if (!project) {
|
|
14268
15120
|
throw new Error(`Project not found: ${projectId}`);
|
|
14269
15121
|
}
|
|
@@ -14304,7 +15156,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14304
15156
|
const rich = ir.richResultsResult;
|
|
14305
15157
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14306
15158
|
db.insert(gscUrlInspections).values({
|
|
14307
|
-
id:
|
|
15159
|
+
id: crypto21.randomUUID(),
|
|
14308
15160
|
projectId,
|
|
14309
15161
|
syncRunId: runId,
|
|
14310
15162
|
url: pageUrl,
|
|
@@ -14331,7 +15183,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14331
15183
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
14332
15184
|
}
|
|
14333
15185
|
}
|
|
14334
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
15186
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq21(gscUrlInspections.projectId, projectId)).all();
|
|
14335
15187
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
14336
15188
|
for (const row of allInspections) {
|
|
14337
15189
|
const existing = latestByUrl.get(row.url);
|
|
@@ -14352,9 +15204,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14352
15204
|
}
|
|
14353
15205
|
}
|
|
14354
15206
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
14355
|
-
db.delete(gscCoverageSnapshots).where(
|
|
15207
|
+
db.delete(gscCoverageSnapshots).where(and10(eq21(gscCoverageSnapshots.projectId, projectId), eq21(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
14356
15208
|
db.insert(gscCoverageSnapshots).values({
|
|
14357
|
-
id:
|
|
15209
|
+
id: crypto21.randomUUID(),
|
|
14358
15210
|
projectId,
|
|
14359
15211
|
syncRunId: runId,
|
|
14360
15212
|
date: snapshotDate,
|
|
@@ -14364,16 +15216,304 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14364
15216
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14365
15217
|
}).run();
|
|
14366
15218
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
14367
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15219
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
14368
15220
|
log3.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
14369
15221
|
} catch (err) {
|
|
14370
15222
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
14371
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15223
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
14372
15224
|
log3.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
14373
15225
|
throw err;
|
|
14374
15226
|
}
|
|
14375
15227
|
}
|
|
14376
15228
|
|
|
15229
|
+
// src/commoncrawl-sync.ts
|
|
15230
|
+
import crypto22 from "crypto";
|
|
15231
|
+
import path11 from "path";
|
|
15232
|
+
import { and as and11, eq as eq22, sql as sql8 } from "drizzle-orm";
|
|
15233
|
+
var log4 = createLogger("CommonCrawlSync");
|
|
15234
|
+
var INSERT_CHUNK_SIZE = 1e4;
|
|
15235
|
+
function defaultDeps() {
|
|
15236
|
+
return {
|
|
15237
|
+
downloadFile,
|
|
15238
|
+
queryBacklinks,
|
|
15239
|
+
loadDuckdb,
|
|
15240
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
15241
|
+
cacheDir: CC_CACHE_DIR
|
|
15242
|
+
};
|
|
15243
|
+
}
|
|
15244
|
+
async function executeReleaseSync(db, syncId, opts) {
|
|
15245
|
+
const deps = { ...defaultDeps(), ...opts.deps };
|
|
15246
|
+
const release = opts.release;
|
|
15247
|
+
try {
|
|
15248
|
+
if (!isValidReleaseId(release)) {
|
|
15249
|
+
throw new Error(`Invalid release id: ${release}`);
|
|
15250
|
+
}
|
|
15251
|
+
const downloadStartedAt = deps.now().toISOString();
|
|
15252
|
+
db.update(ccReleaseSyncs).set({
|
|
15253
|
+
status: CcReleaseSyncStatuses.downloading,
|
|
15254
|
+
downloadStartedAt,
|
|
15255
|
+
phaseDetail: "downloading vertices + edges",
|
|
15256
|
+
updatedAt: downloadStartedAt,
|
|
15257
|
+
error: null
|
|
15258
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15259
|
+
const paths = ccReleasePaths(release);
|
|
15260
|
+
const releaseCacheDir = path11.join(deps.cacheDir, release);
|
|
15261
|
+
const vertexPath = path11.join(releaseCacheDir, paths.vertexFilename);
|
|
15262
|
+
const edgesPath = path11.join(releaseCacheDir, paths.edgesFilename);
|
|
15263
|
+
const [vertex, edges] = await Promise.all([
|
|
15264
|
+
deps.downloadFile({ url: paths.vertexUrl, destPath: vertexPath }),
|
|
15265
|
+
deps.downloadFile({ url: paths.edgesUrl, destPath: edgesPath })
|
|
15266
|
+
]);
|
|
15267
|
+
const downloadFinishedAt = deps.now().toISOString();
|
|
15268
|
+
const queryStartedAt = downloadFinishedAt;
|
|
15269
|
+
db.update(ccReleaseSyncs).set({
|
|
15270
|
+
status: CcReleaseSyncStatuses.querying,
|
|
15271
|
+
downloadFinishedAt,
|
|
15272
|
+
queryStartedAt,
|
|
15273
|
+
phaseDetail: "querying backlinks",
|
|
15274
|
+
vertexPath,
|
|
15275
|
+
edgesPath,
|
|
15276
|
+
vertexBytes: vertex.bytes,
|
|
15277
|
+
edgesBytes: edges.bytes,
|
|
15278
|
+
vertexSha256: vertex.sha256,
|
|
15279
|
+
edgesSha256: edges.sha256,
|
|
15280
|
+
updatedAt: downloadFinishedAt
|
|
15281
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15282
|
+
const allProjects = db.select().from(projects).all();
|
|
15283
|
+
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
15284
|
+
let rows = [];
|
|
15285
|
+
if (targets.length > 0) {
|
|
15286
|
+
const duckdb = deps.loadDuckdb();
|
|
15287
|
+
rows = await deps.queryBacklinks({ vertexPath, edgesPath, targets, duckdb });
|
|
15288
|
+
}
|
|
15289
|
+
const projectsByDomain = /* @__PURE__ */ new Map();
|
|
15290
|
+
for (const p of allProjects) {
|
|
15291
|
+
const ids = projectsByDomain.get(p.canonicalDomain) ?? [];
|
|
15292
|
+
ids.push(p.id);
|
|
15293
|
+
projectsByDomain.set(p.canonicalDomain, ids);
|
|
15294
|
+
}
|
|
15295
|
+
const queriedAt = deps.now().toISOString();
|
|
15296
|
+
db.transaction((tx) => {
|
|
15297
|
+
tx.delete(backlinkDomains).where(eq22(backlinkDomains.releaseSyncId, syncId)).run();
|
|
15298
|
+
tx.delete(backlinkSummaries).where(eq22(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
15299
|
+
const expanded = [];
|
|
15300
|
+
for (const r of rows) {
|
|
15301
|
+
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
15302
|
+
if (!projectIds) continue;
|
|
15303
|
+
for (const projectId of projectIds) {
|
|
15304
|
+
expanded.push({
|
|
15305
|
+
id: crypto22.randomUUID(),
|
|
15306
|
+
projectId,
|
|
15307
|
+
releaseSyncId: syncId,
|
|
15308
|
+
release,
|
|
15309
|
+
targetDomain: r.targetDomain,
|
|
15310
|
+
linkingDomain: r.linkingDomain,
|
|
15311
|
+
numHosts: r.numHosts,
|
|
15312
|
+
createdAt: queriedAt
|
|
15313
|
+
});
|
|
15314
|
+
}
|
|
15315
|
+
}
|
|
15316
|
+
for (let i = 0; i < expanded.length; i += INSERT_CHUNK_SIZE) {
|
|
15317
|
+
const chunk = expanded.slice(i, i + INSERT_CHUNK_SIZE);
|
|
15318
|
+
if (chunk.length > 0) tx.insert(backlinkDomains).values(chunk).run();
|
|
15319
|
+
}
|
|
15320
|
+
const rowsByProject = groupByProject(rows, projectsByDomain);
|
|
15321
|
+
for (const p of allProjects) {
|
|
15322
|
+
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
15323
|
+
const summary = computeSummary(projectRows);
|
|
15324
|
+
tx.insert(backlinkSummaries).values({
|
|
15325
|
+
id: crypto22.randomUUID(),
|
|
15326
|
+
projectId: p.id,
|
|
15327
|
+
releaseSyncId: syncId,
|
|
15328
|
+
release,
|
|
15329
|
+
targetDomain: p.canonicalDomain,
|
|
15330
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15331
|
+
totalHosts: summary.totalHosts,
|
|
15332
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15333
|
+
queriedAt,
|
|
15334
|
+
createdAt: queriedAt
|
|
15335
|
+
}).onConflictDoUpdate({
|
|
15336
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.release],
|
|
15337
|
+
set: {
|
|
15338
|
+
releaseSyncId: syncId,
|
|
15339
|
+
targetDomain: p.canonicalDomain,
|
|
15340
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15341
|
+
totalHosts: summary.totalHosts,
|
|
15342
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15343
|
+
queriedAt
|
|
15344
|
+
}
|
|
15345
|
+
}).run();
|
|
15346
|
+
}
|
|
15347
|
+
});
|
|
15348
|
+
const finishedAt = deps.now().toISOString();
|
|
15349
|
+
db.update(ccReleaseSyncs).set({
|
|
15350
|
+
status: CcReleaseSyncStatuses.ready,
|
|
15351
|
+
queryFinishedAt: finishedAt,
|
|
15352
|
+
phaseDetail: null,
|
|
15353
|
+
projectsProcessed: allProjects.length,
|
|
15354
|
+
domainsDiscovered: rows.length,
|
|
15355
|
+
updatedAt: finishedAt,
|
|
15356
|
+
error: null
|
|
15357
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15358
|
+
log4.info("sync.completed", {
|
|
15359
|
+
syncId,
|
|
15360
|
+
release,
|
|
15361
|
+
projectsProcessed: allProjects.length,
|
|
15362
|
+
domainsDiscovered: rows.length
|
|
15363
|
+
});
|
|
15364
|
+
} catch (err) {
|
|
15365
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15366
|
+
const finishedAt = deps.now().toISOString();
|
|
15367
|
+
db.update(ccReleaseSyncs).set({
|
|
15368
|
+
status: CcReleaseSyncStatuses.failed,
|
|
15369
|
+
error: errorMsg,
|
|
15370
|
+
phaseDetail: null,
|
|
15371
|
+
updatedAt: finishedAt
|
|
15372
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15373
|
+
log4.error("sync.failed", { syncId, release, error: errorMsg });
|
|
15374
|
+
throw err;
|
|
15375
|
+
}
|
|
15376
|
+
}
|
|
15377
|
+
function groupByProject(rows, projectsByDomain) {
|
|
15378
|
+
const out = /* @__PURE__ */ new Map();
|
|
15379
|
+
for (const row of rows) {
|
|
15380
|
+
const projectIds = projectsByDomain.get(row.targetDomain);
|
|
15381
|
+
if (!projectIds) continue;
|
|
15382
|
+
for (const projectId of projectIds) {
|
|
15383
|
+
const bucket = out.get(projectId) ?? [];
|
|
15384
|
+
bucket.push(row);
|
|
15385
|
+
out.set(projectId, bucket);
|
|
15386
|
+
}
|
|
15387
|
+
}
|
|
15388
|
+
return out;
|
|
15389
|
+
}
|
|
15390
|
+
function computeSummary(rows) {
|
|
15391
|
+
if (rows.length === 0) {
|
|
15392
|
+
return { totalLinkingDomains: 0, totalHosts: 0, top10HostsShare: "0" };
|
|
15393
|
+
}
|
|
15394
|
+
const sorted = [...rows].sort((a, b) => b.numHosts - a.numHosts);
|
|
15395
|
+
const totalHosts = sorted.reduce((acc, r) => acc + r.numHosts, 0);
|
|
15396
|
+
const top10Hosts = sorted.slice(0, 10).reduce((acc, r) => acc + r.numHosts, 0);
|
|
15397
|
+
const share = totalHosts > 0 ? top10Hosts / totalHosts : 0;
|
|
15398
|
+
return {
|
|
15399
|
+
totalLinkingDomains: rows.length,
|
|
15400
|
+
totalHosts,
|
|
15401
|
+
top10HostsShare: share.toFixed(6)
|
|
15402
|
+
};
|
|
15403
|
+
}
|
|
15404
|
+
|
|
15405
|
+
// src/backlink-extract.ts
|
|
15406
|
+
import crypto23 from "crypto";
|
|
15407
|
+
import { and as and12, desc as desc9, eq as eq23 } from "drizzle-orm";
|
|
15408
|
+
var log5 = createLogger("BacklinkExtract");
|
|
15409
|
+
function defaultDeps2() {
|
|
15410
|
+
return {
|
|
15411
|
+
queryBacklinks,
|
|
15412
|
+
loadDuckdb,
|
|
15413
|
+
now: () => /* @__PURE__ */ new Date()
|
|
15414
|
+
};
|
|
15415
|
+
}
|
|
15416
|
+
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
15417
|
+
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
15418
|
+
const startedAt = deps.now().toISOString();
|
|
15419
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq23(runs.id, runId)).run();
|
|
15420
|
+
try {
|
|
15421
|
+
const project = db.select().from(projects).where(eq23(projects.id, projectId)).get();
|
|
15422
|
+
if (!project) {
|
|
15423
|
+
throw new Error(`Project not found: ${projectId}`);
|
|
15424
|
+
}
|
|
15425
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq23(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq23(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc9(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
15426
|
+
if (!sync) {
|
|
15427
|
+
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
15428
|
+
}
|
|
15429
|
+
if (sync.status !== CcReleaseSyncStatuses.ready) {
|
|
15430
|
+
throw new Error(`Release ${sync.release} is not ready (status=${sync.status})`);
|
|
15431
|
+
}
|
|
15432
|
+
if (!sync.vertexPath || !sync.edgesPath) {
|
|
15433
|
+
throw new Error(`Release ${sync.release} is missing cached file paths`);
|
|
15434
|
+
}
|
|
15435
|
+
const duckdb = deps.loadDuckdb();
|
|
15436
|
+
const rows = await deps.queryBacklinks({
|
|
15437
|
+
vertexPath: sync.vertexPath,
|
|
15438
|
+
edgesPath: sync.edgesPath,
|
|
15439
|
+
targets: [project.canonicalDomain],
|
|
15440
|
+
duckdb
|
|
15441
|
+
});
|
|
15442
|
+
const queriedAt = deps.now().toISOString();
|
|
15443
|
+
const syncId = sync.id;
|
|
15444
|
+
const release = sync.release;
|
|
15445
|
+
const targetDomain = project.canonicalDomain;
|
|
15446
|
+
db.transaction((tx) => {
|
|
15447
|
+
tx.delete(backlinkDomains).where(
|
|
15448
|
+
and12(eq23(backlinkDomains.projectId, projectId), eq23(backlinkDomains.release, release))
|
|
15449
|
+
).run();
|
|
15450
|
+
if (rows.length > 0) {
|
|
15451
|
+
const values = rows.map((r) => ({
|
|
15452
|
+
id: crypto23.randomUUID(),
|
|
15453
|
+
projectId,
|
|
15454
|
+
releaseSyncId: syncId,
|
|
15455
|
+
release,
|
|
15456
|
+
targetDomain,
|
|
15457
|
+
linkingDomain: r.linkingDomain,
|
|
15458
|
+
numHosts: r.numHosts,
|
|
15459
|
+
createdAt: queriedAt
|
|
15460
|
+
}));
|
|
15461
|
+
tx.insert(backlinkDomains).values(values).run();
|
|
15462
|
+
}
|
|
15463
|
+
const summary = computeSummary2(rows);
|
|
15464
|
+
tx.insert(backlinkSummaries).values({
|
|
15465
|
+
id: crypto23.randomUUID(),
|
|
15466
|
+
projectId,
|
|
15467
|
+
releaseSyncId: syncId,
|
|
15468
|
+
release,
|
|
15469
|
+
targetDomain,
|
|
15470
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15471
|
+
totalHosts: summary.totalHosts,
|
|
15472
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15473
|
+
queriedAt,
|
|
15474
|
+
createdAt: queriedAt
|
|
15475
|
+
}).onConflictDoUpdate({
|
|
15476
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.release],
|
|
15477
|
+
set: {
|
|
15478
|
+
releaseSyncId: syncId,
|
|
15479
|
+
targetDomain,
|
|
15480
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15481
|
+
totalHosts: summary.totalHosts,
|
|
15482
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15483
|
+
queriedAt
|
|
15484
|
+
}
|
|
15485
|
+
}).run();
|
|
15486
|
+
});
|
|
15487
|
+
const finishedAt = deps.now().toISOString();
|
|
15488
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
15489
|
+
log5.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
15490
|
+
} catch (err) {
|
|
15491
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15492
|
+
const finishedAt = deps.now().toISOString();
|
|
15493
|
+
db.update(runs).set({
|
|
15494
|
+
status: RunStatuses.failed,
|
|
15495
|
+
error: errorMsg,
|
|
15496
|
+
finishedAt
|
|
15497
|
+
}).where(eq23(runs.id, runId)).run();
|
|
15498
|
+
log5.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
15499
|
+
throw err;
|
|
15500
|
+
}
|
|
15501
|
+
}
|
|
15502
|
+
function computeSummary2(rows) {
|
|
15503
|
+
if (rows.length === 0) {
|
|
15504
|
+
return { totalLinkingDomains: 0, totalHosts: 0, top10HostsShare: "0" };
|
|
15505
|
+
}
|
|
15506
|
+
const sorted = [...rows].sort((a, b) => b.numHosts - a.numHosts);
|
|
15507
|
+
const totalHosts = sorted.reduce((acc, r) => acc + r.numHosts, 0);
|
|
15508
|
+
const top10Hosts = sorted.slice(0, 10).reduce((acc, r) => acc + r.numHosts, 0);
|
|
15509
|
+
const share = totalHosts > 0 ? top10Hosts / totalHosts : 0;
|
|
15510
|
+
return {
|
|
15511
|
+
totalLinkingDomains: rows.length,
|
|
15512
|
+
totalHosts,
|
|
15513
|
+
top10HostsShare: share.toFixed(6)
|
|
15514
|
+
};
|
|
15515
|
+
}
|
|
15516
|
+
|
|
14377
15517
|
// src/provider-registry.ts
|
|
14378
15518
|
var ProviderRegistry = class {
|
|
14379
15519
|
providers = /* @__PURE__ */ new Map();
|
|
@@ -14427,8 +15567,8 @@ var ProviderRegistry = class {
|
|
|
14427
15567
|
|
|
14428
15568
|
// src/scheduler.ts
|
|
14429
15569
|
import cron from "node-cron";
|
|
14430
|
-
import { eq as
|
|
14431
|
-
var
|
|
15570
|
+
import { eq as eq24 } from "drizzle-orm";
|
|
15571
|
+
var log6 = createLogger("Scheduler");
|
|
14432
15572
|
var Scheduler = class {
|
|
14433
15573
|
db;
|
|
14434
15574
|
callbacks;
|
|
@@ -14439,16 +15579,16 @@ var Scheduler = class {
|
|
|
14439
15579
|
}
|
|
14440
15580
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
14441
15581
|
start() {
|
|
14442
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
15582
|
+
const allSchedules = this.db.select().from(schedules).where(eq24(schedules.enabled, 1)).all();
|
|
14443
15583
|
for (const schedule of allSchedules) {
|
|
14444
15584
|
const missedRunAt = schedule.nextRunAt;
|
|
14445
15585
|
this.registerCronTask(schedule);
|
|
14446
15586
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
14447
|
-
|
|
15587
|
+
log6.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
14448
15588
|
this.triggerRun(schedule.id, schedule.projectId);
|
|
14449
15589
|
}
|
|
14450
15590
|
}
|
|
14451
|
-
|
|
15591
|
+
log6.info("started", { scheduleCount: allSchedules.length });
|
|
14452
15592
|
}
|
|
14453
15593
|
/** Stop all cron tasks for graceful shutdown. */
|
|
14454
15594
|
stop() {
|
|
@@ -14464,7 +15604,7 @@ var Scheduler = class {
|
|
|
14464
15604
|
this.stopTask(projectId, existing, "Stopped");
|
|
14465
15605
|
this.tasks.delete(projectId);
|
|
14466
15606
|
}
|
|
14467
|
-
const schedule = this.db.select().from(schedules).where(
|
|
15607
|
+
const schedule = this.db.select().from(schedules).where(eq24(schedules.projectId, projectId)).get();
|
|
14468
15608
|
if (schedule && schedule.enabled === 1) {
|
|
14469
15609
|
this.registerCronTask(schedule);
|
|
14470
15610
|
}
|
|
@@ -14480,12 +15620,12 @@ var Scheduler = class {
|
|
|
14480
15620
|
stopTask(projectId, task, verb) {
|
|
14481
15621
|
task.stop();
|
|
14482
15622
|
task.destroy();
|
|
14483
|
-
|
|
15623
|
+
log6.info(`task.${verb.toLowerCase()}`, { projectId });
|
|
14484
15624
|
}
|
|
14485
15625
|
registerCronTask(schedule) {
|
|
14486
15626
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
14487
15627
|
if (!cron.validate(cronExpr)) {
|
|
14488
|
-
|
|
15628
|
+
log6.error("cron.invalid", { projectId, cronExpr });
|
|
14489
15629
|
return;
|
|
14490
15630
|
}
|
|
14491
15631
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -14497,24 +15637,24 @@ var Scheduler = class {
|
|
|
14497
15637
|
this.db.update(schedules).set({
|
|
14498
15638
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
14499
15639
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14500
|
-
}).where(
|
|
15640
|
+
}).where(eq24(schedules.id, scheduleId)).run();
|
|
14501
15641
|
const label = schedule.preset ?? cronExpr;
|
|
14502
|
-
|
|
15642
|
+
log6.info("cron.registered", { projectId, schedule: label, timezone });
|
|
14503
15643
|
}
|
|
14504
15644
|
triggerRun(scheduleId, projectId) {
|
|
14505
15645
|
try {
|
|
14506
15646
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14507
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
15647
|
+
const currentSchedule = this.db.select().from(schedules).where(eq24(schedules.id, scheduleId)).get();
|
|
14508
15648
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
14509
|
-
|
|
15649
|
+
log6.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
14510
15650
|
this.remove(projectId);
|
|
14511
15651
|
return;
|
|
14512
15652
|
}
|
|
14513
15653
|
const task = this.tasks.get(projectId);
|
|
14514
15654
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
14515
|
-
const project = this.db.select().from(projects).where(
|
|
15655
|
+
const project = this.db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
14516
15656
|
if (!project) {
|
|
14517
|
-
|
|
15657
|
+
log6.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
14518
15658
|
this.remove(projectId);
|
|
14519
15659
|
return;
|
|
14520
15660
|
}
|
|
@@ -14523,7 +15663,7 @@ var Scheduler = class {
|
|
|
14523
15663
|
if (project.defaultLocation) {
|
|
14524
15664
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
14525
15665
|
if (!loc) {
|
|
14526
|
-
|
|
15666
|
+
log6.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
14527
15667
|
return;
|
|
14528
15668
|
}
|
|
14529
15669
|
resolvedLocation = loc;
|
|
@@ -14537,11 +15677,11 @@ var Scheduler = class {
|
|
|
14537
15677
|
location: locationLabel
|
|
14538
15678
|
});
|
|
14539
15679
|
if (queueResult.conflict) {
|
|
14540
|
-
|
|
15680
|
+
log6.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
14541
15681
|
this.db.update(schedules).set({
|
|
14542
15682
|
nextRunAt,
|
|
14543
15683
|
updatedAt: now
|
|
14544
|
-
}).where(
|
|
15684
|
+
}).where(eq24(schedules.id, currentSchedule.id)).run();
|
|
14545
15685
|
return;
|
|
14546
15686
|
}
|
|
14547
15687
|
const runId = queueResult.runId;
|
|
@@ -14549,21 +15689,21 @@ var Scheduler = class {
|
|
|
14549
15689
|
lastRunAt: now,
|
|
14550
15690
|
nextRunAt,
|
|
14551
15691
|
updatedAt: now
|
|
14552
|
-
}).where(
|
|
15692
|
+
}).where(eq24(schedules.id, currentSchedule.id)).run();
|
|
14553
15693
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
14554
15694
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
14555
|
-
|
|
15695
|
+
log6.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
14556
15696
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
14557
15697
|
} catch (err) {
|
|
14558
|
-
|
|
15698
|
+
log6.error("trigger.error", { scheduleId, projectId, error: err instanceof Error ? err.message : String(err) });
|
|
14559
15699
|
}
|
|
14560
15700
|
}
|
|
14561
15701
|
};
|
|
14562
15702
|
|
|
14563
15703
|
// src/notifier.ts
|
|
14564
|
-
import { eq as
|
|
14565
|
-
import
|
|
14566
|
-
var
|
|
15704
|
+
import { eq as eq25, desc as desc10, and as and13, or as or2 } from "drizzle-orm";
|
|
15705
|
+
import crypto24 from "crypto";
|
|
15706
|
+
var log7 = createLogger("Notifier");
|
|
14567
15707
|
var Notifier = class {
|
|
14568
15708
|
db;
|
|
14569
15709
|
serverUrl;
|
|
@@ -14573,26 +15713,26 @@ var Notifier = class {
|
|
|
14573
15713
|
}
|
|
14574
15714
|
/** Called after a run completes (success, partial, or failed). */
|
|
14575
15715
|
async onRunCompleted(runId, projectId) {
|
|
14576
|
-
|
|
14577
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15716
|
+
log7.info("run.completed", { runId, projectId });
|
|
15717
|
+
const notifs = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
14578
15718
|
if (notifs.length === 0) {
|
|
14579
|
-
|
|
15719
|
+
log7.info("notifications.none-enabled", { projectId });
|
|
14580
15720
|
return;
|
|
14581
15721
|
}
|
|
14582
|
-
|
|
14583
|
-
const run = this.db.select().from(runs).where(
|
|
15722
|
+
log7.info("notifications.found", { projectId, count: notifs.length });
|
|
15723
|
+
const run = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
14584
15724
|
if (!run) {
|
|
14585
|
-
|
|
15725
|
+
log7.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
14586
15726
|
return;
|
|
14587
15727
|
}
|
|
14588
|
-
const project = this.db.select().from(projects).where(
|
|
15728
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
14589
15729
|
if (!project) {
|
|
14590
|
-
|
|
15730
|
+
log7.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
14591
15731
|
return;
|
|
14592
15732
|
}
|
|
14593
15733
|
const transitions = this.computeTransitions(runId, projectId);
|
|
14594
15734
|
const events = [];
|
|
14595
|
-
|
|
15735
|
+
log7.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
14596
15736
|
if (run.status === "completed" || run.status === "partial") {
|
|
14597
15737
|
events.push("run.completed");
|
|
14598
15738
|
}
|
|
@@ -14608,7 +15748,7 @@ var Notifier = class {
|
|
|
14608
15748
|
if (!config.url) continue;
|
|
14609
15749
|
const subscribedEvents = config.events;
|
|
14610
15750
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
14611
|
-
|
|
15751
|
+
log7.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
14612
15752
|
if (matchingEvents.length === 0) continue;
|
|
14613
15753
|
for (const event of matchingEvents) {
|
|
14614
15754
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -14632,11 +15772,11 @@ var Notifier = class {
|
|
|
14632
15772
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
14633
15773
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
14634
15774
|
if (insightEvents.length === 0) return;
|
|
14635
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15775
|
+
const notifs = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
14636
15776
|
if (notifs.length === 0) return;
|
|
14637
|
-
const run = this.db.select().from(runs).where(
|
|
15777
|
+
const run = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
14638
15778
|
if (!run) return;
|
|
14639
|
-
const project = this.db.select().from(projects).where(
|
|
15779
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
14640
15780
|
if (!project) return;
|
|
14641
15781
|
for (const notif of notifs) {
|
|
14642
15782
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -14667,11 +15807,11 @@ var Notifier = class {
|
|
|
14667
15807
|
}
|
|
14668
15808
|
computeTransitions(runId, projectId) {
|
|
14669
15809
|
const recentRuns = this.db.select().from(runs).where(
|
|
14670
|
-
|
|
14671
|
-
|
|
14672
|
-
or2(
|
|
15810
|
+
and13(
|
|
15811
|
+
eq25(runs.projectId, projectId),
|
|
15812
|
+
or2(eq25(runs.status, "completed"), eq25(runs.status, "partial"))
|
|
14673
15813
|
)
|
|
14674
|
-
).orderBy(
|
|
15814
|
+
).orderBy(desc10(runs.createdAt)).limit(2).all();
|
|
14675
15815
|
if (recentRuns.length < 2) return [];
|
|
14676
15816
|
const currentRunId = recentRuns[0].id;
|
|
14677
15817
|
const previousRunId = recentRuns[1].id;
|
|
@@ -14681,12 +15821,12 @@ var Notifier = class {
|
|
|
14681
15821
|
keyword: keywords.keyword,
|
|
14682
15822
|
provider: querySnapshots.provider,
|
|
14683
15823
|
citationState: querySnapshots.citationState
|
|
14684
|
-
}).from(querySnapshots).leftJoin(keywords,
|
|
15824
|
+
}).from(querySnapshots).leftJoin(keywords, eq25(querySnapshots.keywordId, keywords.id)).where(eq25(querySnapshots.runId, currentRunId)).all();
|
|
14685
15825
|
const previousSnapshots = this.db.select({
|
|
14686
15826
|
keywordId: querySnapshots.keywordId,
|
|
14687
15827
|
provider: querySnapshots.provider,
|
|
14688
15828
|
citationState: querySnapshots.citationState
|
|
14689
|
-
}).from(querySnapshots).where(
|
|
15829
|
+
}).from(querySnapshots).where(eq25(querySnapshots.runId, previousRunId)).all();
|
|
14690
15830
|
const prevMap = /* @__PURE__ */ new Map();
|
|
14691
15831
|
for (const s of previousSnapshots) {
|
|
14692
15832
|
prevMap.set(`${s.keywordId}:${s.provider}`, s.citationState);
|
|
@@ -14710,23 +15850,23 @@ var Notifier = class {
|
|
|
14710
15850
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
14711
15851
|
const targetCheck = await resolveWebhookTarget(url);
|
|
14712
15852
|
if (!targetCheck.ok) {
|
|
14713
|
-
|
|
15853
|
+
log7.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
14714
15854
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
14715
15855
|
return;
|
|
14716
15856
|
}
|
|
14717
|
-
|
|
15857
|
+
log7.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
14718
15858
|
const maxRetries = 3;
|
|
14719
15859
|
const delays = [1e3, 4e3, 16e3];
|
|
14720
15860
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
14721
15861
|
try {
|
|
14722
15862
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
14723
15863
|
if (response.status >= 200 && response.status < 300) {
|
|
14724
|
-
|
|
15864
|
+
log7.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
14725
15865
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
14726
15866
|
return;
|
|
14727
15867
|
}
|
|
14728
15868
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
14729
|
-
|
|
15869
|
+
log7.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
14730
15870
|
if (attempt === maxRetries - 1) {
|
|
14731
15871
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
14732
15872
|
}
|
|
@@ -14734,7 +15874,7 @@ var Notifier = class {
|
|
|
14734
15874
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
14735
15875
|
if (attempt === maxRetries - 1) {
|
|
14736
15876
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
14737
|
-
|
|
15877
|
+
log7.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
14738
15878
|
}
|
|
14739
15879
|
}
|
|
14740
15880
|
if (attempt < maxRetries - 1) {
|
|
@@ -14744,7 +15884,7 @@ var Notifier = class {
|
|
|
14744
15884
|
}
|
|
14745
15885
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
14746
15886
|
this.db.insert(auditLog).values({
|
|
14747
|
-
id:
|
|
15887
|
+
id: crypto24.randomUUID(),
|
|
14748
15888
|
projectId,
|
|
14749
15889
|
actor: "scheduler",
|
|
14750
15890
|
action: `notification.${status}`,
|
|
@@ -14757,7 +15897,7 @@ var Notifier = class {
|
|
|
14757
15897
|
};
|
|
14758
15898
|
|
|
14759
15899
|
// src/run-coordinator.ts
|
|
14760
|
-
var
|
|
15900
|
+
var log8 = createLogger("RunCoordinator");
|
|
14761
15901
|
var RunCoordinator = class {
|
|
14762
15902
|
constructor(notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
14763
15903
|
this.notifier = notifier;
|
|
@@ -14779,35 +15919,35 @@ var RunCoordinator = class {
|
|
|
14779
15919
|
try {
|
|
14780
15920
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
14781
15921
|
} catch (err) {
|
|
14782
|
-
|
|
15922
|
+
log8.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14783
15923
|
}
|
|
14784
15924
|
}
|
|
14785
15925
|
}
|
|
14786
15926
|
} catch (err) {
|
|
14787
|
-
|
|
15927
|
+
log8.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14788
15928
|
}
|
|
14789
15929
|
try {
|
|
14790
15930
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
14791
15931
|
} catch (err) {
|
|
14792
|
-
|
|
15932
|
+
log8.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14793
15933
|
}
|
|
14794
15934
|
if (this.onAeroEvent) {
|
|
14795
15935
|
try {
|
|
14796
15936
|
await this.onAeroEvent({ runId, projectId, insightCount, criticalOrHigh });
|
|
14797
15937
|
} catch (err) {
|
|
14798
|
-
|
|
15938
|
+
log8.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14799
15939
|
}
|
|
14800
15940
|
}
|
|
14801
15941
|
}
|
|
14802
15942
|
};
|
|
14803
15943
|
|
|
14804
15944
|
// src/agent/session-registry.ts
|
|
14805
|
-
import
|
|
14806
|
-
import { eq as
|
|
15945
|
+
import crypto26 from "crypto";
|
|
15946
|
+
import { eq as eq27 } from "drizzle-orm";
|
|
14807
15947
|
|
|
14808
15948
|
// src/agent/session.ts
|
|
14809
|
-
import
|
|
14810
|
-
import
|
|
15949
|
+
import fs11 from "fs";
|
|
15950
|
+
import path14 from "path";
|
|
14811
15951
|
import { Agent } from "@mariozechner/pi-agent-core";
|
|
14812
15952
|
import { registerBuiltInApiProviders } from "@mariozechner/pi-ai";
|
|
14813
15953
|
|
|
@@ -14908,26 +16048,26 @@ function buildAgentProvidersResponse(config) {
|
|
|
14908
16048
|
}
|
|
14909
16049
|
|
|
14910
16050
|
// src/agent/skill-paths.ts
|
|
14911
|
-
import
|
|
14912
|
-
import
|
|
16051
|
+
import fs9 from "fs";
|
|
16052
|
+
import path12 from "path";
|
|
14913
16053
|
import { fileURLToPath } from "url";
|
|
14914
16054
|
function resolveAeroSkillDir(pkgDir) {
|
|
14915
|
-
const here = pkgDir ??
|
|
16055
|
+
const here = pkgDir ?? path12.dirname(fileURLToPath(import.meta.url));
|
|
14916
16056
|
const candidates = [
|
|
14917
|
-
|
|
14918
|
-
|
|
14919
|
-
|
|
16057
|
+
path12.join(here, "../assets/agent-workspace/skills/aero"),
|
|
16058
|
+
path12.join(here, "../../assets/agent-workspace/skills/aero"),
|
|
16059
|
+
path12.join(here, "../../../../skills/aero")
|
|
14920
16060
|
];
|
|
14921
16061
|
for (const candidate of candidates) {
|
|
14922
|
-
if (
|
|
16062
|
+
if (fs9.existsSync(path12.join(candidate, "SKILL.md"))) return candidate;
|
|
14923
16063
|
}
|
|
14924
16064
|
throw new Error(`Aero skill not found. Searched:
|
|
14925
16065
|
${candidates.join("\n ")}`);
|
|
14926
16066
|
}
|
|
14927
16067
|
|
|
14928
16068
|
// src/agent/skill-tools.ts
|
|
14929
|
-
import
|
|
14930
|
-
import
|
|
16069
|
+
import fs10 from "fs";
|
|
16070
|
+
import path13 from "path";
|
|
14931
16071
|
import { Type } from "@sinclair/typebox";
|
|
14932
16072
|
var MAX_DOC_CHARS = 2e4;
|
|
14933
16073
|
function textResult(details) {
|
|
@@ -14948,13 +16088,13 @@ function parseDescription(body) {
|
|
|
14948
16088
|
return "(no description)";
|
|
14949
16089
|
}
|
|
14950
16090
|
function scanSkillDocs(skillDir) {
|
|
14951
|
-
const refsDir =
|
|
14952
|
-
if (!
|
|
16091
|
+
const refsDir = path13.join(skillDir ?? resolveAeroSkillDir(), "references");
|
|
16092
|
+
if (!fs10.existsSync(refsDir)) return [];
|
|
14953
16093
|
const entries = [];
|
|
14954
|
-
for (const file of
|
|
16094
|
+
for (const file of fs10.readdirSync(refsDir)) {
|
|
14955
16095
|
if (!file.endsWith(".md")) continue;
|
|
14956
|
-
const filePath =
|
|
14957
|
-
const body =
|
|
16096
|
+
const filePath = path13.join(refsDir, file);
|
|
16097
|
+
const body = fs10.readFileSync(filePath, "utf-8");
|
|
14958
16098
|
entries.push({
|
|
14959
16099
|
slug: file.replace(/\.md$/, ""),
|
|
14960
16100
|
description: parseDescription(body),
|
|
@@ -14997,8 +16137,8 @@ function buildReadSkillDocTool() {
|
|
|
14997
16137
|
availableSlugs: docs.map((d) => d.slug)
|
|
14998
16138
|
});
|
|
14999
16139
|
}
|
|
15000
|
-
const filePath =
|
|
15001
|
-
const content =
|
|
16140
|
+
const filePath = path13.join(skillDir, "references", `${match.slug}.md`);
|
|
16141
|
+
const content = fs10.readFileSync(filePath, "utf-8");
|
|
15002
16142
|
if (content.length > MAX_DOC_CHARS) {
|
|
15003
16143
|
return textResult({
|
|
15004
16144
|
slug: match.slug,
|
|
@@ -15022,8 +16162,8 @@ function buildSkillDocTools() {
|
|
|
15022
16162
|
import { Type as Type2 } from "@sinclair/typebox";
|
|
15023
16163
|
|
|
15024
16164
|
// src/agent/memory-store.ts
|
|
15025
|
-
import
|
|
15026
|
-
import { and as
|
|
16165
|
+
import crypto25 from "crypto";
|
|
16166
|
+
import { and as and14, desc as desc11, eq as eq26, like, sql as sql9 } from "drizzle-orm";
|
|
15027
16167
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
15028
16168
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
15029
16169
|
function rowToDto(row) {
|
|
@@ -15037,7 +16177,7 @@ function rowToDto(row) {
|
|
|
15037
16177
|
};
|
|
15038
16178
|
}
|
|
15039
16179
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
15040
|
-
const query = db.select().from(agentMemory).where(
|
|
16180
|
+
const query = db.select().from(agentMemory).where(eq26(agentMemory.projectId, projectId)).orderBy(desc11(agentMemory.updatedAt));
|
|
15041
16181
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
15042
16182
|
return rows.map(rowToDto);
|
|
15043
16183
|
}
|
|
@@ -15051,7 +16191,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
15051
16191
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
15052
16192
|
}
|
|
15053
16193
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15054
|
-
const id =
|
|
16194
|
+
const id = crypto25.randomUUID();
|
|
15055
16195
|
db.insert(agentMemory).values({
|
|
15056
16196
|
id,
|
|
15057
16197
|
projectId: args.projectId,
|
|
@@ -15068,12 +16208,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
15068
16208
|
updatedAt: now
|
|
15069
16209
|
}
|
|
15070
16210
|
}).run();
|
|
15071
|
-
const row = db.select().from(agentMemory).where(
|
|
16211
|
+
const row = db.select().from(agentMemory).where(and14(eq26(agentMemory.projectId, args.projectId), eq26(agentMemory.key, args.key))).get();
|
|
15072
16212
|
if (!row) throw new Error("memory upsert produced no row");
|
|
15073
16213
|
return rowToDto(row);
|
|
15074
16214
|
}
|
|
15075
16215
|
function deleteMemoryEntry(db, projectId, key) {
|
|
15076
|
-
const result = db.delete(agentMemory).where(
|
|
16216
|
+
const result = db.delete(agentMemory).where(and14(eq26(agentMemory.projectId, projectId), eq26(agentMemory.key, key))).run();
|
|
15077
16217
|
const changes = result.changes ?? 0;
|
|
15078
16218
|
return changes > 0;
|
|
15079
16219
|
}
|
|
@@ -15088,7 +16228,7 @@ function writeCompactionNote(db, args) {
|
|
|
15088
16228
|
}
|
|
15089
16229
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15090
16230
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
15091
|
-
const id =
|
|
16231
|
+
const id = crypto25.randomUUID();
|
|
15092
16232
|
let inserted;
|
|
15093
16233
|
db.transaction((tx) => {
|
|
15094
16234
|
tx.insert(agentMemory).values({
|
|
@@ -15102,16 +16242,16 @@ function writeCompactionNote(db, args) {
|
|
|
15102
16242
|
}).run();
|
|
15103
16243
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
15104
16244
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
15105
|
-
|
|
15106
|
-
|
|
16245
|
+
and14(
|
|
16246
|
+
eq26(agentMemory.projectId, args.projectId),
|
|
15107
16247
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
15108
16248
|
)
|
|
15109
|
-
).orderBy(
|
|
16249
|
+
).orderBy(desc11(agentMemory.updatedAt)).all();
|
|
15110
16250
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
15111
16251
|
if (stale.length > 0) {
|
|
15112
|
-
tx.delete(agentMemory).where(
|
|
16252
|
+
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
15113
16253
|
}
|
|
15114
|
-
const row = tx.select().from(agentMemory).where(
|
|
16254
|
+
const row = tx.select().from(agentMemory).where(and14(eq26(agentMemory.projectId, args.projectId), eq26(agentMemory.key, key))).get();
|
|
15115
16255
|
if (row) inserted = rowToDto(row);
|
|
15116
16256
|
});
|
|
15117
16257
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -15258,6 +16398,35 @@ function buildGetRunTool(ctx) {
|
|
|
15258
16398
|
}
|
|
15259
16399
|
};
|
|
15260
16400
|
}
|
|
16401
|
+
var BacklinksSchema = Type2.Object({
|
|
16402
|
+
limit: Type2.Optional(
|
|
16403
|
+
Type2.Number({
|
|
16404
|
+
description: "Max linking-domain rows to include. Default 50, max 200.",
|
|
16405
|
+
minimum: 1,
|
|
16406
|
+
maximum: 200
|
|
16407
|
+
})
|
|
16408
|
+
),
|
|
16409
|
+
release: Type2.Optional(
|
|
16410
|
+
Type2.String({
|
|
16411
|
+
description: "Common Crawl release id (e.g., cc-main-2026-jan-feb-mar). Omit for the most recent release with data."
|
|
16412
|
+
})
|
|
16413
|
+
)
|
|
16414
|
+
});
|
|
16415
|
+
function buildListBacklinksTool(ctx) {
|
|
16416
|
+
return {
|
|
16417
|
+
name: "list_backlinks",
|
|
16418
|
+
label: "List backlinks",
|
|
16419
|
+
description: "Backlink summary and top linking domains from the most recent ready Common Crawl release. Off-site authority signal that correlates with citation likelihood. Returns null summary when no release sync has completed for this workspace.",
|
|
16420
|
+
parameters: BacklinksSchema,
|
|
16421
|
+
execute: async (_toolCallId, params) => {
|
|
16422
|
+
const response = await ctx.client.backlinksDomains(ctx.projectName, {
|
|
16423
|
+
limit: params.limit ?? 50,
|
|
16424
|
+
release: params.release
|
|
16425
|
+
});
|
|
16426
|
+
return textResult2(response);
|
|
16427
|
+
}
|
|
16428
|
+
};
|
|
16429
|
+
}
|
|
15261
16430
|
var RecallSchema = Type2.Object({
|
|
15262
16431
|
limit: Type2.Optional(
|
|
15263
16432
|
Type2.Number({
|
|
@@ -15288,7 +16457,8 @@ function buildReadTools(ctx) {
|
|
|
15288
16457
|
buildListKeywordsTool(ctx),
|
|
15289
16458
|
buildListCompetitorsTool(ctx),
|
|
15290
16459
|
buildGetRunTool(ctx),
|
|
15291
|
-
buildRecallTool(ctx)
|
|
16460
|
+
buildRecallTool(ctx),
|
|
16461
|
+
buildListBacklinksTool(ctx)
|
|
15292
16462
|
];
|
|
15293
16463
|
}
|
|
15294
16464
|
var RunSweepSchema = Type2.Object({
|
|
@@ -15522,10 +16692,10 @@ function ensureBuiltinsRegistered() {
|
|
|
15522
16692
|
}
|
|
15523
16693
|
function loadAeroSystemPrompt(pkgDir) {
|
|
15524
16694
|
const skillDir = resolveAeroSkillDir(pkgDir);
|
|
15525
|
-
const skillBody =
|
|
15526
|
-
const soulPath =
|
|
15527
|
-
if (!
|
|
15528
|
-
const soulBody =
|
|
16695
|
+
const skillBody = fs11.readFileSync(path14.join(skillDir, "SKILL.md"), "utf-8");
|
|
16696
|
+
const soulPath = path14.join(skillDir, "soul.md");
|
|
16697
|
+
if (!fs11.existsSync(soulPath)) return skillBody;
|
|
16698
|
+
const soulBody = fs11.readFileSync(soulPath, "utf-8");
|
|
15529
16699
|
return `${soulBody.trimEnd()}
|
|
15530
16700
|
|
|
15531
16701
|
---
|
|
@@ -15709,7 +16879,7 @@ async function compactMessages(args) {
|
|
|
15709
16879
|
}
|
|
15710
16880
|
|
|
15711
16881
|
// src/agent/session-registry.ts
|
|
15712
|
-
var
|
|
16882
|
+
var log9 = createLogger("SessionRegistry");
|
|
15713
16883
|
var MAX_HYDRATE_NOTES = 20;
|
|
15714
16884
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
15715
16885
|
function escapeMemoryFragment(value) {
|
|
@@ -15758,7 +16928,7 @@ var SessionRegistry = class {
|
|
|
15758
16928
|
modelProvider: effectiveProvider,
|
|
15759
16929
|
modelId: effectiveModelId,
|
|
15760
16930
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15761
|
-
}).where(
|
|
16931
|
+
}).where(eq27(agentSessions.projectId, projectId)).run();
|
|
15762
16932
|
}
|
|
15763
16933
|
const agent2 = createAeroSession({
|
|
15764
16934
|
projectName,
|
|
@@ -15940,13 +17110,13 @@ ${lines.join("\n")}
|
|
|
15940
17110
|
agent.state.messages = result.messages;
|
|
15941
17111
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
15942
17112
|
this.save(projectName);
|
|
15943
|
-
|
|
17113
|
+
log9.info("compaction.completed", {
|
|
15944
17114
|
projectName,
|
|
15945
17115
|
removedCount: result.removedCount,
|
|
15946
17116
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
15947
17117
|
});
|
|
15948
17118
|
} catch (err) {
|
|
15949
|
-
|
|
17119
|
+
log9.error("compaction.failed", {
|
|
15950
17120
|
projectName,
|
|
15951
17121
|
error: err instanceof Error ? err.message : String(err)
|
|
15952
17122
|
});
|
|
@@ -15976,7 +17146,7 @@ ${lines.join("\n")}
|
|
|
15976
17146
|
modelProvider: nextProvider,
|
|
15977
17147
|
modelId: nextModelId,
|
|
15978
17148
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15979
|
-
}).where(
|
|
17149
|
+
}).where(eq27(agentSessions.projectId, projectId)).run();
|
|
15980
17150
|
}
|
|
15981
17151
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
15982
17152
|
save(projectName) {
|
|
@@ -16043,7 +17213,7 @@ ${lines.join("\n")}
|
|
|
16043
17213
|
await agent.prompt(msgs);
|
|
16044
17214
|
this.save(projectName);
|
|
16045
17215
|
} catch (err) {
|
|
16046
|
-
|
|
17216
|
+
log9.error("drain.failed", {
|
|
16047
17217
|
projectName,
|
|
16048
17218
|
error: err instanceof Error ? err.message : String(err)
|
|
16049
17219
|
});
|
|
@@ -16138,17 +17308,17 @@ ${lines.join("\n")}
|
|
|
16138
17308
|
return id;
|
|
16139
17309
|
}
|
|
16140
17310
|
tryResolveProjectId(projectName) {
|
|
16141
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
17311
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq27(projects.name, projectName)).get();
|
|
16142
17312
|
return row?.id;
|
|
16143
17313
|
}
|
|
16144
17314
|
loadRow(projectId) {
|
|
16145
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
17315
|
+
const row = this.opts.db.select().from(agentSessions).where(eq27(agentSessions.projectId, projectId)).get();
|
|
16146
17316
|
return row ?? null;
|
|
16147
17317
|
}
|
|
16148
17318
|
insertRow(params) {
|
|
16149
17319
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16150
17320
|
this.opts.db.insert(agentSessions).values({
|
|
16151
|
-
id:
|
|
17321
|
+
id: crypto26.randomUUID(),
|
|
16152
17322
|
projectId: params.projectId,
|
|
16153
17323
|
systemPrompt: params.systemPrompt,
|
|
16154
17324
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -16161,14 +17331,14 @@ ${lines.join("\n")}
|
|
|
16161
17331
|
}
|
|
16162
17332
|
updateRow(projectId, patch) {
|
|
16163
17333
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16164
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
17334
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq27(agentSessions.projectId, projectId)).run();
|
|
16165
17335
|
}
|
|
16166
17336
|
};
|
|
16167
17337
|
|
|
16168
17338
|
// src/agent/agent-routes.ts
|
|
16169
|
-
import { eq as
|
|
17339
|
+
import { eq as eq28 } from "drizzle-orm";
|
|
16170
17340
|
function resolveProject2(db, name) {
|
|
16171
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
17341
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq28(projects.name, name)).get();
|
|
16172
17342
|
if (!row) throw notFound("project", name);
|
|
16173
17343
|
return row;
|
|
16174
17344
|
}
|
|
@@ -16177,7 +17347,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16177
17347
|
"/projects/:name/agent/transcript",
|
|
16178
17348
|
async (request) => {
|
|
16179
17349
|
const project = resolveProject2(opts.db, request.params.name);
|
|
16180
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
17350
|
+
const row = opts.db.select().from(agentSessions).where(eq28(agentSessions.projectId, project.id)).get();
|
|
16181
17351
|
if (!row) {
|
|
16182
17352
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
16183
17353
|
}
|
|
@@ -16201,7 +17371,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16201
17371
|
async (request) => {
|
|
16202
17372
|
const project = resolveProject2(opts.db, request.params.name);
|
|
16203
17373
|
opts.sessionRegistry.reset(project.name);
|
|
16204
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
17374
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(agentSessions.projectId, project.id)).run();
|
|
16205
17375
|
return { status: "reset" };
|
|
16206
17376
|
}
|
|
16207
17377
|
);
|
|
@@ -16363,9 +17533,9 @@ var ApiClient = class {
|
|
|
16363
17533
|
}
|
|
16364
17534
|
return this.probePromise;
|
|
16365
17535
|
}
|
|
16366
|
-
async request(method,
|
|
17536
|
+
async request(method, path16, body) {
|
|
16367
17537
|
await this.probeBasePath();
|
|
16368
|
-
const url = `${this.baseUrl}${
|
|
17538
|
+
const url = `${this.baseUrl}${path16}`;
|
|
16369
17539
|
const serializedBody = body != null ? JSON.stringify(body) : void 0;
|
|
16370
17540
|
const headers = {
|
|
16371
17541
|
"Authorization": `Bearer ${this.apiKey}`,
|
|
@@ -16402,7 +17572,7 @@ var ApiClient = class {
|
|
|
16402
17572
|
const msg = errorObj?.message ? String(errorObj.message) : `HTTP ${res.status}: ${res.statusText}`;
|
|
16403
17573
|
const code = errorObj?.code ? String(errorObj.code) : "API_ERROR";
|
|
16404
17574
|
const exitCode = res.status >= 500 ? EXIT_SYSTEM_ERROR : EXIT_USER_ERROR;
|
|
16405
|
-
throw new CliError({ code, message: msg, exitCode });
|
|
17575
|
+
throw new CliError({ code, message: msg, exitCode, details: { httpStatus: res.status } });
|
|
16406
17576
|
}
|
|
16407
17577
|
if (res.status === 204) {
|
|
16408
17578
|
return void 0;
|
|
@@ -16453,9 +17623,9 @@ var ApiClient = class {
|
|
|
16453
17623
|
* structured-error behavior of `request()`; the caller reads `res.body`
|
|
16454
17624
|
* and releases the response when done.
|
|
16455
17625
|
*/
|
|
16456
|
-
async streamPost(
|
|
17626
|
+
async streamPost(path16, body, signal) {
|
|
16457
17627
|
await this.probeBasePath();
|
|
16458
|
-
const url = `${this.baseUrl}${
|
|
17628
|
+
const url = `${this.baseUrl}${path16}`;
|
|
16459
17629
|
const headers = {
|
|
16460
17630
|
Authorization: `Bearer ${this.apiKey}`,
|
|
16461
17631
|
"Content-Type": "application/json",
|
|
@@ -16492,7 +17662,7 @@ var ApiClient = class {
|
|
|
16492
17662
|
const msg = errorObj?.message ? String(errorObj.message) : `HTTP ${res.status}: ${res.statusText}`;
|
|
16493
17663
|
const code = errorObj?.code ? String(errorObj.code) : "API_ERROR";
|
|
16494
17664
|
const exitCode = res.status >= 500 ? EXIT_SYSTEM_ERROR : EXIT_USER_ERROR;
|
|
16495
|
-
throw new CliError({ code, message: msg, exitCode });
|
|
17665
|
+
throw new CliError({ code, message: msg, exitCode, details: { httpStatus: res.status } });
|
|
16496
17666
|
}
|
|
16497
17667
|
return res;
|
|
16498
17668
|
}
|
|
@@ -16533,6 +17703,9 @@ var ApiClient = class {
|
|
|
16533
17703
|
const query = limit != null ? `?limit=${encodeURIComponent(String(limit))}` : "";
|
|
16534
17704
|
return this.request("GET", `/projects/${encodeURIComponent(project)}/runs${query}`);
|
|
16535
17705
|
}
|
|
17706
|
+
async getLatestRun(project) {
|
|
17707
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/runs/latest`);
|
|
17708
|
+
}
|
|
16536
17709
|
async getRun(id) {
|
|
16537
17710
|
return this.request("GET", `/runs/${encodeURIComponent(id)}`);
|
|
16538
17711
|
}
|
|
@@ -16853,6 +18026,46 @@ var ApiClient = class {
|
|
|
16853
18026
|
const qs = limit ? `?limit=${limit}` : "";
|
|
16854
18027
|
return this.request("GET", `/projects/${encodeURIComponent(project)}/health/history${qs}`);
|
|
16855
18028
|
}
|
|
18029
|
+
// --- Backlinks ---------------------------------------------------------
|
|
18030
|
+
async backlinksStatus() {
|
|
18031
|
+
return this.request("GET", "/backlinks/status");
|
|
18032
|
+
}
|
|
18033
|
+
async backlinksInstall() {
|
|
18034
|
+
return this.request("POST", "/backlinks/install");
|
|
18035
|
+
}
|
|
18036
|
+
async backlinksTriggerSync(release) {
|
|
18037
|
+
return this.request("POST", "/backlinks/syncs", { release });
|
|
18038
|
+
}
|
|
18039
|
+
async backlinksLatestSync() {
|
|
18040
|
+
return this.request("GET", "/backlinks/syncs/latest");
|
|
18041
|
+
}
|
|
18042
|
+
async backlinksListSyncs() {
|
|
18043
|
+
return this.request("GET", "/backlinks/syncs");
|
|
18044
|
+
}
|
|
18045
|
+
async backlinksCachedReleases() {
|
|
18046
|
+
return this.request("GET", "/backlinks/releases");
|
|
18047
|
+
}
|
|
18048
|
+
async backlinksPruneCache(release) {
|
|
18049
|
+
return this.request("DELETE", `/backlinks/cache/${encodeURIComponent(release)}`);
|
|
18050
|
+
}
|
|
18051
|
+
async backlinksExtract(project, release) {
|
|
18052
|
+
return this.request("POST", `/projects/${encodeURIComponent(project)}/backlinks/extract`, release ? { release } : {});
|
|
18053
|
+
}
|
|
18054
|
+
async backlinksSummary(project, release) {
|
|
18055
|
+
const qs = release ? `?release=${encodeURIComponent(release)}` : "";
|
|
18056
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/summary${qs}`);
|
|
18057
|
+
}
|
|
18058
|
+
async backlinksDomains(project, opts = {}) {
|
|
18059
|
+
const qs = new URLSearchParams();
|
|
18060
|
+
if (opts.limit !== void 0) qs.set("limit", String(opts.limit));
|
|
18061
|
+
if (opts.offset !== void 0) qs.set("offset", String(opts.offset));
|
|
18062
|
+
if (opts.release) qs.set("release", opts.release);
|
|
18063
|
+
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
|
18064
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/domains${suffix}`);
|
|
18065
|
+
}
|
|
18066
|
+
async backlinksHistory(project) {
|
|
18067
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/history`);
|
|
18068
|
+
}
|
|
16856
18069
|
};
|
|
16857
18070
|
|
|
16858
18071
|
// src/snapshot-service.ts
|
|
@@ -16877,13 +18090,13 @@ function extractHostname(domain) {
|
|
|
16877
18090
|
function fetchWithPinnedAddress(target) {
|
|
16878
18091
|
return new Promise((resolve) => {
|
|
16879
18092
|
const port = target.url.port ? Number(target.url.port) : 443;
|
|
16880
|
-
const
|
|
18093
|
+
const path16 = target.url.pathname + target.url.search;
|
|
16881
18094
|
const req = https2.request(
|
|
16882
18095
|
{
|
|
16883
18096
|
hostname: target.address,
|
|
16884
18097
|
family: target.family,
|
|
16885
18098
|
port,
|
|
16886
|
-
path:
|
|
18099
|
+
path: path16,
|
|
16887
18100
|
method: "GET",
|
|
16888
18101
|
timeout: FETCH_TIMEOUT_MS,
|
|
16889
18102
|
servername: target.url.hostname,
|
|
@@ -16975,7 +18188,7 @@ function formatAuditFactorScore(factor) {
|
|
|
16975
18188
|
}
|
|
16976
18189
|
|
|
16977
18190
|
// src/snapshot-service.ts
|
|
16978
|
-
var
|
|
18191
|
+
var log10 = createLogger("Snapshot");
|
|
16979
18192
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
16980
18193
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
16981
18194
|
var ProviderExecutionGate2 = class {
|
|
@@ -17118,7 +18331,7 @@ var SnapshotService = class {
|
|
|
17118
18331
|
return mapAuditReport(report);
|
|
17119
18332
|
} catch (err) {
|
|
17120
18333
|
const message = err instanceof Error ? err.message : String(err);
|
|
17121
|
-
|
|
18334
|
+
log10.warn("audit.failed", { homepageUrl, error: message });
|
|
17122
18335
|
return {
|
|
17123
18336
|
url: homepageUrl,
|
|
17124
18337
|
finalUrl: homepageUrl,
|
|
@@ -17148,7 +18361,7 @@ var SnapshotService = class {
|
|
|
17148
18361
|
phrases: parsedPhrases
|
|
17149
18362
|
};
|
|
17150
18363
|
} catch (err) {
|
|
17151
|
-
|
|
18364
|
+
log10.warn("profile.generation-failed", {
|
|
17152
18365
|
domain: ctx.domain,
|
|
17153
18366
|
provider: ctx.analysisProvider.adapter.name,
|
|
17154
18367
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -17290,7 +18503,7 @@ var SnapshotService = class {
|
|
|
17290
18503
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
17291
18504
|
};
|
|
17292
18505
|
} catch (err) {
|
|
17293
|
-
|
|
18506
|
+
log10.warn("response.analysis-failed", {
|
|
17294
18507
|
provider: ctx.analysisProvider.adapter.name,
|
|
17295
18508
|
error: err instanceof Error ? err.message : String(err)
|
|
17296
18509
|
});
|
|
@@ -17573,9 +18786,9 @@ function clipText(value, length) {
|
|
|
17573
18786
|
}
|
|
17574
18787
|
|
|
17575
18788
|
// src/server.ts
|
|
17576
|
-
var _require2 =
|
|
18789
|
+
var _require2 = createRequire3(import.meta.url);
|
|
17577
18790
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
17578
|
-
var
|
|
18791
|
+
var log11 = createLogger("Server");
|
|
17579
18792
|
var DEFAULT_QUOTA = {
|
|
17580
18793
|
maxConcurrency: 2,
|
|
17581
18794
|
maxRequestsPerMinute: 10,
|
|
@@ -17606,7 +18819,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
17606
18819
|
};
|
|
17607
18820
|
}
|
|
17608
18821
|
function hashApiKey(key) {
|
|
17609
|
-
return
|
|
18822
|
+
return crypto27.createHash("sha256").update(key).digest("hex");
|
|
17610
18823
|
}
|
|
17611
18824
|
function parseCookies2(header) {
|
|
17612
18825
|
if (!header) return {};
|
|
@@ -17662,7 +18875,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
17662
18875
|
}
|
|
17663
18876
|
if (migratedGoogle > 0) {
|
|
17664
18877
|
saveConfigPatch({ google: config.google });
|
|
17665
|
-
|
|
18878
|
+
log11.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
17666
18879
|
}
|
|
17667
18880
|
let migratedGa4 = 0;
|
|
17668
18881
|
for (const row of rows.ga4) {
|
|
@@ -17680,7 +18893,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
17680
18893
|
}
|
|
17681
18894
|
if (migratedGa4 > 0) {
|
|
17682
18895
|
saveConfigPatch({ ga4: config.ga4 });
|
|
17683
|
-
|
|
18896
|
+
log11.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
17684
18897
|
}
|
|
17685
18898
|
}
|
|
17686
18899
|
async function createServer(opts) {
|
|
@@ -17712,11 +18925,11 @@ async function createServer(opts) {
|
|
|
17712
18925
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
17713
18926
|
dropLegacyCredentialColumns(opts.db);
|
|
17714
18927
|
} catch (err) {
|
|
17715
|
-
|
|
18928
|
+
log11.warn("credentials.migration.failed", {
|
|
17716
18929
|
error: err instanceof Error ? err.message : String(err)
|
|
17717
18930
|
});
|
|
17718
18931
|
}
|
|
17719
|
-
|
|
18932
|
+
log11.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
17720
18933
|
const p = providers[k];
|
|
17721
18934
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
17722
18935
|
}) });
|
|
@@ -17764,7 +18977,7 @@ async function createServer(opts) {
|
|
|
17764
18977
|
intelligenceService,
|
|
17765
18978
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
17766
18979
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
17767
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
18980
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq29(projects.id, projectId)).get();
|
|
17768
18981
|
if (!project) return;
|
|
17769
18982
|
sessionRegistry.queueFollowUp(project.name, {
|
|
17770
18983
|
role: "user",
|
|
@@ -17776,8 +18989,8 @@ async function createServer(opts) {
|
|
|
17776
18989
|
);
|
|
17777
18990
|
jobRunner.onRunCompleted = (runId, projectId) => runCoordinator.onRunCompleted(runId, projectId);
|
|
17778
18991
|
const snapshotService = new SnapshotService(registry);
|
|
17779
|
-
const orphanedOpenClawDir =
|
|
17780
|
-
if (
|
|
18992
|
+
const orphanedOpenClawDir = path15.join(os6.homedir(), ".openclaw-aero");
|
|
18993
|
+
if (fs12.existsSync(orphanedOpenClawDir)) {
|
|
17781
18994
|
app.log.warn(
|
|
17782
18995
|
{ path: orphanedOpenClawDir },
|
|
17783
18996
|
"OpenClaw gateway is no longer used. Remove ~/.openclaw-aero/ manually to reclaim the directory."
|
|
@@ -17858,7 +19071,7 @@ async function createServer(opts) {
|
|
|
17858
19071
|
return removed;
|
|
17859
19072
|
}
|
|
17860
19073
|
};
|
|
17861
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
19074
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto27.randomBytes(32).toString("hex");
|
|
17862
19075
|
const googleConnectionStore = {
|
|
17863
19076
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
17864
19077
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -17904,11 +19117,11 @@ async function createServer(opts) {
|
|
|
17904
19117
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
17905
19118
|
if (opts.config.apiKey) {
|
|
17906
19119
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
17907
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
19120
|
+
const existing = opts.db.select().from(apiKeys).where(eq29(apiKeys.keyHash, keyHash)).get();
|
|
17908
19121
|
if (!existing) {
|
|
17909
19122
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
17910
19123
|
opts.db.insert(apiKeys).values({
|
|
17911
|
-
id: `key_${
|
|
19124
|
+
id: `key_${crypto27.randomBytes(8).toString("hex")}`,
|
|
17912
19125
|
name: "default",
|
|
17913
19126
|
keyHash,
|
|
17914
19127
|
keyPrefix: prefix,
|
|
@@ -17932,7 +19145,7 @@ async function createServer(opts) {
|
|
|
17932
19145
|
};
|
|
17933
19146
|
const createSession = (apiKeyId) => {
|
|
17934
19147
|
pruneExpiredSessions();
|
|
17935
|
-
const sessionId =
|
|
19148
|
+
const sessionId = crypto27.randomBytes(32).toString("hex");
|
|
17936
19149
|
sessions.set(sessionId, {
|
|
17937
19150
|
apiKeyId,
|
|
17938
19151
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -17956,7 +19169,7 @@ async function createServer(opts) {
|
|
|
17956
19169
|
};
|
|
17957
19170
|
const getDefaultApiKey = () => {
|
|
17958
19171
|
if (!opts.config.apiKey) return void 0;
|
|
17959
|
-
return opts.db.select().from(apiKeys).where(
|
|
19172
|
+
return opts.db.select().from(apiKeys).where(eq29(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
17960
19173
|
};
|
|
17961
19174
|
const createPasswordSession = (reply) => {
|
|
17962
19175
|
const key = getDefaultApiKey();
|
|
@@ -18013,12 +19226,12 @@ async function createServer(opts) {
|
|
|
18013
19226
|
return reply.send({ authenticated: true });
|
|
18014
19227
|
}
|
|
18015
19228
|
if (apiKey) {
|
|
18016
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
19229
|
+
const key = opts.db.select().from(apiKeys).where(eq29(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
18017
19230
|
if (!key || key.revokedAt) {
|
|
18018
19231
|
const err2 = authInvalid();
|
|
18019
19232
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
18020
19233
|
}
|
|
18021
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
19234
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(apiKeys.id, key.id)).run();
|
|
18022
19235
|
const sessionId = createSession(key.id);
|
|
18023
19236
|
reply.header("set-cookie", serializeSessionCookie({
|
|
18024
19237
|
name: SESSION_COOKIE_NAME,
|
|
@@ -18085,6 +19298,54 @@ async function createServer(opts) {
|
|
|
18085
19298
|
app.log.error({ runId, err }, "Inspect sitemap failed");
|
|
18086
19299
|
});
|
|
18087
19300
|
},
|
|
19301
|
+
getBacklinksStatus: () => ({
|
|
19302
|
+
duckdbInstalled: isDuckdbInstalled(),
|
|
19303
|
+
duckdbVersion: readInstalledVersion() ?? void 0,
|
|
19304
|
+
duckdbSpec: DUCKDB_SPEC,
|
|
19305
|
+
pluginDir: PLUGIN_DIR
|
|
19306
|
+
}),
|
|
19307
|
+
onInstallBacklinks: async () => {
|
|
19308
|
+
const result = await installDuckdb({ onLog: (line) => app.log.info({ line }, "duckdb install") });
|
|
19309
|
+
return {
|
|
19310
|
+
installed: true,
|
|
19311
|
+
version: result.version,
|
|
19312
|
+
path: result.path,
|
|
19313
|
+
alreadyPresent: result.alreadyPresent
|
|
19314
|
+
};
|
|
19315
|
+
},
|
|
19316
|
+
onReleaseSyncRequested: (syncId, release) => {
|
|
19317
|
+
executeReleaseSync(opts.db, syncId, { release }).catch((err) => {
|
|
19318
|
+
app.log.error({ syncId, err }, "Common Crawl release sync failed");
|
|
19319
|
+
});
|
|
19320
|
+
},
|
|
19321
|
+
onBacklinkExtractRequested: (runId, projectId, release) => {
|
|
19322
|
+
executeBacklinkExtract(opts.db, runId, projectId, { release }).catch((err) => {
|
|
19323
|
+
app.log.error({ runId, err }, "Backlink extract failed");
|
|
19324
|
+
});
|
|
19325
|
+
},
|
|
19326
|
+
onBacklinksPruneCache: (release) => {
|
|
19327
|
+
try {
|
|
19328
|
+
pruneCachedRelease(release);
|
|
19329
|
+
} catch (err) {
|
|
19330
|
+
app.log.error({ release, err }, "Failed to prune cached release");
|
|
19331
|
+
}
|
|
19332
|
+
},
|
|
19333
|
+
listCachedReleases: () => {
|
|
19334
|
+
const cached = listCachedReleases();
|
|
19335
|
+
const syncByRelease = /* @__PURE__ */ new Map();
|
|
19336
|
+
for (const row of opts.db.select().from(ccReleaseSyncs).all()) {
|
|
19337
|
+
syncByRelease.set(row.release, { status: row.status, updatedAt: row.updatedAt });
|
|
19338
|
+
}
|
|
19339
|
+
return cached.map((entry) => {
|
|
19340
|
+
const sync = syncByRelease.get(entry.release);
|
|
19341
|
+
return {
|
|
19342
|
+
release: entry.release,
|
|
19343
|
+
syncStatus: sync?.status ?? null,
|
|
19344
|
+
bytes: entry.bytes,
|
|
19345
|
+
lastUsedAt: entry.lastUsedAt
|
|
19346
|
+
};
|
|
19347
|
+
});
|
|
19348
|
+
},
|
|
18088
19349
|
openApiInfo: {
|
|
18089
19350
|
title: "Canonry API",
|
|
18090
19351
|
version: PKG_VERSION,
|
|
@@ -18165,7 +19426,7 @@ async function createServer(opts) {
|
|
|
18165
19426
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
18166
19427
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
18167
19428
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
18168
|
-
id:
|
|
19429
|
+
id: crypto27.randomUUID(),
|
|
18169
19430
|
projectId,
|
|
18170
19431
|
actor: "api",
|
|
18171
19432
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -18296,10 +19557,10 @@ async function createServer(opts) {
|
|
|
18296
19557
|
return snapshotService.createReport(input);
|
|
18297
19558
|
}
|
|
18298
19559
|
});
|
|
18299
|
-
const dirname =
|
|
18300
|
-
const assetsDir =
|
|
18301
|
-
if (
|
|
18302
|
-
const indexPath =
|
|
19560
|
+
const dirname = path15.dirname(fileURLToPath2(import.meta.url));
|
|
19561
|
+
const assetsDir = path15.join(dirname, "..", "assets");
|
|
19562
|
+
if (fs12.existsSync(assetsDir)) {
|
|
19563
|
+
const indexPath = path15.join(assetsDir, "index.html");
|
|
18303
19564
|
const injectConfig = (html) => {
|
|
18304
19565
|
const clientConfig = {};
|
|
18305
19566
|
if (basePath) clientConfig.basePath = basePath;
|
|
@@ -18317,8 +19578,8 @@ async function createServer(opts) {
|
|
|
18317
19578
|
index: false
|
|
18318
19579
|
});
|
|
18319
19580
|
const serveIndex = (_request, reply) => {
|
|
18320
|
-
if (
|
|
18321
|
-
const html =
|
|
19581
|
+
if (fs12.existsSync(indexPath)) {
|
|
19582
|
+
const html = fs12.readFileSync(indexPath, "utf-8");
|
|
18322
19583
|
return reply.type("text/html").send(injectConfig(html));
|
|
18323
19584
|
}
|
|
18324
19585
|
return reply.status(404).send({ error: "Dashboard not built" });
|
|
@@ -18338,8 +19599,8 @@ async function createServer(opts) {
|
|
|
18338
19599
|
if (basePath && !url.startsWith(basePath)) {
|
|
18339
19600
|
return reply.status(404).send({ error: "Not found", path: request.url });
|
|
18340
19601
|
}
|
|
18341
|
-
if (
|
|
18342
|
-
const html =
|
|
19602
|
+
if (fs12.existsSync(indexPath)) {
|
|
19603
|
+
const html = fs12.readFileSync(indexPath, "utf-8");
|
|
18343
19604
|
return reply.type("text/html").send(injectConfig(html));
|
|
18344
19605
|
}
|
|
18345
19606
|
return reply.status(404).send({ error: "Not found" });
|
|
@@ -18422,6 +19683,7 @@ export {
|
|
|
18422
19683
|
EXIT_SYSTEM_ERROR,
|
|
18423
19684
|
CliError,
|
|
18424
19685
|
usageError,
|
|
19686
|
+
isEndpointMissing,
|
|
18425
19687
|
printCliError,
|
|
18426
19688
|
providerQuotaPolicySchema,
|
|
18427
19689
|
ProviderNames,
|