@ainyc/canonry 2.2.3 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/assets/agent-workspace/skills/canonry-setup/references/canonry-cli.md +20 -0
- package/assets/assets/index-CAewPdsZ.css +1 -0
- package/assets/assets/index-Nrl3ecFY.js +301 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-TAII35VC.js → chunk-GZF3YIHY.js} +119 -1
- package/dist/{chunk-MXUOJWNL.js → chunk-Y7GCG4GD.js} +1492 -216
- package/dist/cli.js +454 -146
- package/dist/index.js +2 -2
- package/dist/{intelligence-service-C5LAYDFM.js → intelligence-service-KM64AW7J.js} +1 -1
- package/package.json +10 -9
- package/assets/assets/index-Bmnz-wvT.js +0 -301
- package/assets/assets/index-yF1fs-OW.css +0 -1
|
@@ -4,8 +4,11 @@ import {
|
|
|
4
4
|
agentSessions,
|
|
5
5
|
apiKeys,
|
|
6
6
|
auditLog,
|
|
7
|
+
backlinkDomains,
|
|
8
|
+
backlinkSummaries,
|
|
7
9
|
bingCoverageSnapshots,
|
|
8
10
|
bingUrlInspections,
|
|
11
|
+
ccReleaseSyncs,
|
|
9
12
|
competitors,
|
|
10
13
|
createLogger,
|
|
11
14
|
dropLegacyCredentialColumns,
|
|
@@ -27,7 +30,7 @@ import {
|
|
|
27
30
|
runs,
|
|
28
31
|
schedules,
|
|
29
32
|
usageCounters
|
|
30
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-GZF3YIHY.js";
|
|
31
34
|
|
|
32
35
|
// src/config.ts
|
|
33
36
|
import fs from "fs";
|
|
@@ -342,12 +345,12 @@ function printCliError(err, format) {
|
|
|
342
345
|
}
|
|
343
346
|
|
|
344
347
|
// src/server.ts
|
|
345
|
-
import { createRequire as
|
|
346
|
-
import
|
|
347
|
-
import
|
|
348
|
-
import
|
|
348
|
+
import { createRequire as createRequire3 } from "module";
|
|
349
|
+
import crypto27 from "crypto";
|
|
350
|
+
import fs13 from "fs";
|
|
351
|
+
import path15 from "path";
|
|
349
352
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
350
|
-
import { eq as
|
|
353
|
+
import { eq as eq29 } from "drizzle-orm";
|
|
351
354
|
import Fastify from "fastify";
|
|
352
355
|
|
|
353
356
|
// ../contracts/src/config-schema.ts
|
|
@@ -445,6 +448,7 @@ var projectUpsertRequestSchema = z3.object({
|
|
|
445
448
|
providers: z3.array(providerNameSchema).optional(),
|
|
446
449
|
locations: z3.array(locationContextSchema).optional(),
|
|
447
450
|
defaultLocation: z3.string().nullable().optional(),
|
|
451
|
+
autoExtractBacklinks: z3.boolean().optional(),
|
|
448
452
|
configSource: configSourceSchema.optional()
|
|
449
453
|
});
|
|
450
454
|
var projectDtoSchema = z3.object({
|
|
@@ -459,6 +463,7 @@ var projectDtoSchema = z3.object({
|
|
|
459
463
|
labels: z3.record(z3.string(), z3.string()).default({}),
|
|
460
464
|
locations: z3.array(locationContextSchema).default([]),
|
|
461
465
|
defaultLocation: z3.string().nullable().optional(),
|
|
466
|
+
autoExtractBacklinks: z3.boolean().default(false),
|
|
462
467
|
configSource: configSourceSchema.default("cli"),
|
|
463
468
|
configRevision: z3.number().int().positive().default(1),
|
|
464
469
|
createdAt: z3.string().optional(),
|
|
@@ -532,7 +537,8 @@ var configSpecSchema = z4.object({
|
|
|
532
537
|
defaultLocation: z4.string().optional(),
|
|
533
538
|
schedule: configScheduleSchema,
|
|
534
539
|
notifications: z4.array(configNotificationSchema).optional().default([]),
|
|
535
|
-
google: configGoogleSchema
|
|
540
|
+
google: configGoogleSchema,
|
|
541
|
+
autoExtractBacklinks: z4.boolean().optional().default(false)
|
|
536
542
|
}).superRefine((spec, ctx) => {
|
|
537
543
|
const duplicateLabels = findDuplicateLocationLabels(spec.locations);
|
|
538
544
|
if (duplicateLabels.length > 0) {
|
|
@@ -616,6 +622,9 @@ function agentBusy(projectName) {
|
|
|
616
622
|
409
|
|
617
623
|
);
|
|
618
624
|
}
|
|
625
|
+
function missingDependency(message, details) {
|
|
626
|
+
return new AppError("MISSING_DEPENDENCY", message, 422, details);
|
|
627
|
+
}
|
|
619
628
|
|
|
620
629
|
// ../contracts/src/google.ts
|
|
621
630
|
import { z as z5 } from "zod";
|
|
@@ -944,7 +953,8 @@ var runKindSchema = z8.enum([
|
|
|
944
953
|
"gsc-sync",
|
|
945
954
|
"inspect-sitemap",
|
|
946
955
|
"ga-sync",
|
|
947
|
-
"bing-inspect"
|
|
956
|
+
"bing-inspect",
|
|
957
|
+
"backlink-extract"
|
|
948
958
|
]);
|
|
949
959
|
var RunKinds = runKindSchema.enum;
|
|
950
960
|
var runTriggerSchema = z8.enum(["manual", "scheduled", "config-apply"]);
|
|
@@ -1431,6 +1441,83 @@ var agentMemoryDeleteRequestSchema = z13.object({
|
|
|
1431
1441
|
key: z13.string().min(1).max(AGENT_MEMORY_KEY_MAX_LENGTH)
|
|
1432
1442
|
});
|
|
1433
1443
|
|
|
1444
|
+
// ../contracts/src/backlinks.ts
|
|
1445
|
+
import { z as z14 } from "zod";
|
|
1446
|
+
var ccReleaseSyncStatusSchema = z14.enum(["queued", "downloading", "querying", "ready", "failed"]);
|
|
1447
|
+
var CcReleaseSyncStatuses = ccReleaseSyncStatusSchema.enum;
|
|
1448
|
+
var ccReleaseSyncDtoSchema = z14.object({
|
|
1449
|
+
id: z14.string(),
|
|
1450
|
+
release: z14.string(),
|
|
1451
|
+
status: ccReleaseSyncStatusSchema,
|
|
1452
|
+
phaseDetail: z14.string().nullable().optional(),
|
|
1453
|
+
vertexPath: z14.string().nullable().optional(),
|
|
1454
|
+
edgesPath: z14.string().nullable().optional(),
|
|
1455
|
+
vertexSha256: z14.string().nullable().optional(),
|
|
1456
|
+
edgesSha256: z14.string().nullable().optional(),
|
|
1457
|
+
vertexBytes: z14.number().int().nullable().optional(),
|
|
1458
|
+
edgesBytes: z14.number().int().nullable().optional(),
|
|
1459
|
+
projectsProcessed: z14.number().int().nullable().optional(),
|
|
1460
|
+
domainsDiscovered: z14.number().int().nullable().optional(),
|
|
1461
|
+
downloadStartedAt: z14.string().nullable().optional(),
|
|
1462
|
+
downloadFinishedAt: z14.string().nullable().optional(),
|
|
1463
|
+
queryStartedAt: z14.string().nullable().optional(),
|
|
1464
|
+
queryFinishedAt: z14.string().nullable().optional(),
|
|
1465
|
+
error: z14.string().nullable().optional(),
|
|
1466
|
+
createdAt: z14.string(),
|
|
1467
|
+
updatedAt: z14.string()
|
|
1468
|
+
});
|
|
1469
|
+
var backlinkDomainDtoSchema = z14.object({
|
|
1470
|
+
linkingDomain: z14.string(),
|
|
1471
|
+
numHosts: z14.number().int()
|
|
1472
|
+
});
|
|
1473
|
+
var backlinkSummaryDtoSchema = z14.object({
|
|
1474
|
+
projectId: z14.string(),
|
|
1475
|
+
release: z14.string(),
|
|
1476
|
+
targetDomain: z14.string(),
|
|
1477
|
+
totalLinkingDomains: z14.number().int(),
|
|
1478
|
+
totalHosts: z14.number().int(),
|
|
1479
|
+
top10HostsShare: z14.string(),
|
|
1480
|
+
queriedAt: z14.string()
|
|
1481
|
+
});
|
|
1482
|
+
var backlinkListResponseSchema = z14.object({
|
|
1483
|
+
summary: backlinkSummaryDtoSchema.nullable(),
|
|
1484
|
+
total: z14.number().int(),
|
|
1485
|
+
rows: z14.array(backlinkDomainDtoSchema)
|
|
1486
|
+
});
|
|
1487
|
+
var backlinkHistoryEntrySchema = z14.object({
|
|
1488
|
+
release: z14.string(),
|
|
1489
|
+
totalLinkingDomains: z14.number().int(),
|
|
1490
|
+
totalHosts: z14.number().int(),
|
|
1491
|
+
top10HostsShare: z14.string(),
|
|
1492
|
+
queriedAt: z14.string()
|
|
1493
|
+
});
|
|
1494
|
+
var backlinksInstallStatusDtoSchema = z14.object({
|
|
1495
|
+
duckdbInstalled: z14.boolean(),
|
|
1496
|
+
duckdbVersion: z14.string().nullable().optional(),
|
|
1497
|
+
duckdbSpec: z14.string(),
|
|
1498
|
+
pluginDir: z14.string()
|
|
1499
|
+
});
|
|
1500
|
+
var backlinksInstallResultDtoSchema = z14.object({
|
|
1501
|
+
installed: z14.boolean(),
|
|
1502
|
+
version: z14.string(),
|
|
1503
|
+
path: z14.string(),
|
|
1504
|
+
alreadyPresent: z14.boolean()
|
|
1505
|
+
});
|
|
1506
|
+
var ccAvailableReleaseSchema = z14.object({
|
|
1507
|
+
release: z14.string(),
|
|
1508
|
+
vertexUrl: z14.string(),
|
|
1509
|
+
edgesUrl: z14.string(),
|
|
1510
|
+
vertexBytes: z14.number().int().nullable(),
|
|
1511
|
+
edgesBytes: z14.number().int().nullable(),
|
|
1512
|
+
lastModified: z14.string().nullable()
|
|
1513
|
+
});
|
|
1514
|
+
var ccCachedReleaseSchema = z14.object({
|
|
1515
|
+
release: z14.string(),
|
|
1516
|
+
syncStatus: ccReleaseSyncStatusSchema.nullable(),
|
|
1517
|
+
bytes: z14.number().int(),
|
|
1518
|
+
lastUsedAt: z14.string().nullable()
|
|
1519
|
+
});
|
|
1520
|
+
|
|
1434
1521
|
// ../api-routes/src/auth.ts
|
|
1435
1522
|
import crypto2 from "crypto";
|
|
1436
1523
|
import { eq } from "drizzle-orm";
|
|
@@ -1595,6 +1682,7 @@ async function projectRoutes(app, opts) {
|
|
|
1595
1682
|
defaultLocation: nextDefaultLocation
|
|
1596
1683
|
});
|
|
1597
1684
|
}
|
|
1685
|
+
const nextAutoExtractBacklinks = body.autoExtractBacklinks !== void 0 ? body.autoExtractBacklinks ? 1 : 0 : existing?.autoExtractBacklinks ?? 0;
|
|
1598
1686
|
if (existing) {
|
|
1599
1687
|
app.db.transaction((tx) => {
|
|
1600
1688
|
tx.update(projects).set({
|
|
@@ -1608,6 +1696,7 @@ async function projectRoutes(app, opts) {
|
|
|
1608
1696
|
providers: JSON.stringify(body.providers ?? []),
|
|
1609
1697
|
locations: JSON.stringify(nextLocations),
|
|
1610
1698
|
defaultLocation: nextDefaultLocation,
|
|
1699
|
+
autoExtractBacklinks: nextAutoExtractBacklinks,
|
|
1611
1700
|
configSource: body.configSource ?? "api",
|
|
1612
1701
|
configRevision: existing.configRevision + 1,
|
|
1613
1702
|
updatedAt: now
|
|
@@ -1639,6 +1728,7 @@ async function projectRoutes(app, opts) {
|
|
|
1639
1728
|
providers: JSON.stringify(body.providers ?? []),
|
|
1640
1729
|
locations: JSON.stringify(nextLocations),
|
|
1641
1730
|
defaultLocation: nextDefaultLocation,
|
|
1731
|
+
autoExtractBacklinks: nextAutoExtractBacklinks,
|
|
1642
1732
|
configSource: body.configSource ?? "api",
|
|
1643
1733
|
configRevision: 1,
|
|
1644
1734
|
createdAt: now,
|
|
@@ -1785,6 +1875,7 @@ async function projectRoutes(app, opts) {
|
|
|
1785
1875
|
providers: parseJsonColumn(project.providers, []),
|
|
1786
1876
|
locations: parseJsonColumn(project.locations, []),
|
|
1787
1877
|
...project.defaultLocation ? { defaultLocation: project.defaultLocation } : {},
|
|
1878
|
+
...project.autoExtractBacklinks === 1 ? { autoExtractBacklinks: true } : {},
|
|
1788
1879
|
notifications: notificationRows.map((row) => {
|
|
1789
1880
|
const cfg = parseJsonColumn(row.config, { url: "", events: [] });
|
|
1790
1881
|
return {
|
|
@@ -1819,6 +1910,7 @@ function formatProject(row) {
|
|
|
1819
1910
|
providers: parseJsonColumn(row.providers, []),
|
|
1820
1911
|
locations: parseJsonColumn(row.locations, []),
|
|
1821
1912
|
defaultLocation: row.defaultLocation,
|
|
1913
|
+
autoExtractBacklinks: row.autoExtractBacklinks === 1,
|
|
1822
1914
|
configSource: row.configSource,
|
|
1823
1915
|
configRevision: row.configRevision,
|
|
1824
1916
|
createdAt: row.createdAt,
|
|
@@ -2463,7 +2555,7 @@ async function deliverWebhook(target, payload, webhookSecret) {
|
|
|
2463
2555
|
const body = JSON.stringify(payload);
|
|
2464
2556
|
const isHttps = target.url.protocol === "https:";
|
|
2465
2557
|
const port = target.url.port ? Number(target.url.port) : isHttps ? 443 : 80;
|
|
2466
|
-
const
|
|
2558
|
+
const path16 = `${target.url.pathname}${target.url.search}`;
|
|
2467
2559
|
const headers = {
|
|
2468
2560
|
"Content-Length": String(Buffer.byteLength(body)),
|
|
2469
2561
|
"Content-Type": "application/json",
|
|
@@ -2479,7 +2571,7 @@ async function deliverWebhook(target, payload, webhookSecret) {
|
|
|
2479
2571
|
headers,
|
|
2480
2572
|
hostname: target.address,
|
|
2481
2573
|
method: "POST",
|
|
2482
|
-
path:
|
|
2574
|
+
path: path16,
|
|
2483
2575
|
port,
|
|
2484
2576
|
timeout: REQUEST_TIMEOUT_MS
|
|
2485
2577
|
};
|
|
@@ -2658,6 +2750,7 @@ async function applyRoutes(app, opts) {
|
|
|
2658
2750
|
providers: JSON.stringify(config.spec.providers ?? []),
|
|
2659
2751
|
locations: JSON.stringify(config.spec.locations ?? []),
|
|
2660
2752
|
defaultLocation: config.spec.defaultLocation ?? null,
|
|
2753
|
+
autoExtractBacklinks: config.spec.autoExtractBacklinks ? 1 : 0,
|
|
2661
2754
|
configSource: "config-file",
|
|
2662
2755
|
configRevision: existing.configRevision + 1,
|
|
2663
2756
|
updatedAt: now
|
|
@@ -2684,6 +2777,7 @@ async function applyRoutes(app, opts) {
|
|
|
2684
2777
|
providers: JSON.stringify(config.spec.providers ?? []),
|
|
2685
2778
|
locations: JSON.stringify(config.spec.locations ?? []),
|
|
2686
2779
|
defaultLocation: config.spec.defaultLocation ?? null,
|
|
2780
|
+
autoExtractBacklinks: config.spec.autoExtractBacklinks ? 1 : 0,
|
|
2687
2781
|
configSource: "config-file",
|
|
2688
2782
|
configRevision: 1,
|
|
2689
2783
|
createdAt: now,
|
|
@@ -2807,6 +2901,7 @@ async function applyRoutes(app, opts) {
|
|
|
2807
2901
|
providers: parseJsonColumn(project.providers, []),
|
|
2808
2902
|
locations: parseJsonColumn(project.locations, []),
|
|
2809
2903
|
defaultLocation: project.defaultLocation,
|
|
2904
|
+
autoExtractBacklinks: project.autoExtractBacklinks === 1,
|
|
2810
2905
|
configSource: project.configSource,
|
|
2811
2906
|
configRevision: project.configRevision,
|
|
2812
2907
|
createdAt: project.createdAt,
|
|
@@ -5657,6 +5752,171 @@ var routeCatalog = [
|
|
|
5657
5752
|
200: { description: "Health history returned." },
|
|
5658
5753
|
404: { description: "Project not found." }
|
|
5659
5754
|
}
|
|
5755
|
+
},
|
|
5756
|
+
{
|
|
5757
|
+
method: "get",
|
|
5758
|
+
path: "/api/v1/backlinks/status",
|
|
5759
|
+
summary: "Get the Common Crawl DuckDB plugin install status",
|
|
5760
|
+
description: "Reports whether @duckdb/node-api is installed in the local plugin dir. Returns MISSING_DEPENDENCY (422) on deployments that cannot host the plugin (e.g. the cloud API).",
|
|
5761
|
+
tags: ["backlinks"],
|
|
5762
|
+
responses: {
|
|
5763
|
+
200: { description: "Install status returned." },
|
|
5764
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5765
|
+
}
|
|
5766
|
+
},
|
|
5767
|
+
{
|
|
5768
|
+
method: "post",
|
|
5769
|
+
path: "/api/v1/backlinks/install",
|
|
5770
|
+
summary: "Install the @duckdb/node-api plugin",
|
|
5771
|
+
description: "Idempotently installs DuckDB into the canonry plugin dir. Returns MISSING_DEPENDENCY (422) when the host cannot perform the install.",
|
|
5772
|
+
tags: ["backlinks"],
|
|
5773
|
+
responses: {
|
|
5774
|
+
200: { description: "Installed (or already present)." },
|
|
5775
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5776
|
+
}
|
|
5777
|
+
},
|
|
5778
|
+
{
|
|
5779
|
+
method: "post",
|
|
5780
|
+
path: "/api/v1/backlinks/syncs",
|
|
5781
|
+
summary: "Queue a workspace-wide Common Crawl release sync",
|
|
5782
|
+
description: "Creates a `cc_release_syncs` row and fires the sync callback. Idempotent: an existing in-flight row for the same release is returned.",
|
|
5783
|
+
tags: ["backlinks"],
|
|
5784
|
+
requestBody: {
|
|
5785
|
+
required: true,
|
|
5786
|
+
content: {
|
|
5787
|
+
"application/json": {
|
|
5788
|
+
schema: {
|
|
5789
|
+
type: "object",
|
|
5790
|
+
required: ["release"],
|
|
5791
|
+
properties: {
|
|
5792
|
+
release: stringSchema
|
|
5793
|
+
}
|
|
5794
|
+
}
|
|
5795
|
+
}
|
|
5796
|
+
}
|
|
5797
|
+
},
|
|
5798
|
+
responses: {
|
|
5799
|
+
200: { description: "Existing in-flight sync returned." },
|
|
5800
|
+
201: { description: "Sync queued." },
|
|
5801
|
+
400: { description: "Invalid release id." },
|
|
5802
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5803
|
+
}
|
|
5804
|
+
},
|
|
5805
|
+
{
|
|
5806
|
+
method: "get",
|
|
5807
|
+
path: "/api/v1/backlinks/syncs",
|
|
5808
|
+
summary: "List Common Crawl release syncs",
|
|
5809
|
+
description: "Returns syncs ordered by updatedAt DESC \u2014 re-queued rows surface ahead of untouched newer rows.",
|
|
5810
|
+
tags: ["backlinks"],
|
|
5811
|
+
responses: {
|
|
5812
|
+
200: { description: "Sync history returned." }
|
|
5813
|
+
}
|
|
5814
|
+
},
|
|
5815
|
+
{
|
|
5816
|
+
method: "get",
|
|
5817
|
+
path: "/api/v1/backlinks/syncs/latest",
|
|
5818
|
+
summary: "Get the most recently-updated Common Crawl release sync",
|
|
5819
|
+
tags: ["backlinks"],
|
|
5820
|
+
responses: {
|
|
5821
|
+
200: { description: "Latest sync returned, or null when no sync exists." }
|
|
5822
|
+
}
|
|
5823
|
+
},
|
|
5824
|
+
{
|
|
5825
|
+
method: "get",
|
|
5826
|
+
path: "/api/v1/backlinks/releases",
|
|
5827
|
+
summary: "List cached Common Crawl releases on the local filesystem",
|
|
5828
|
+
tags: ["backlinks"],
|
|
5829
|
+
responses: {
|
|
5830
|
+
200: { description: "Cached release metadata returned." }
|
|
5831
|
+
}
|
|
5832
|
+
},
|
|
5833
|
+
{
|
|
5834
|
+
method: "delete",
|
|
5835
|
+
path: "/api/v1/backlinks/cache/{release}",
|
|
5836
|
+
summary: "Prune a cached Common Crawl release",
|
|
5837
|
+
tags: ["backlinks"],
|
|
5838
|
+
parameters: [
|
|
5839
|
+
{
|
|
5840
|
+
name: "release",
|
|
5841
|
+
in: "path",
|
|
5842
|
+
required: true,
|
|
5843
|
+
description: "Release id (e.g. cc-main-2026-jan-feb-mar).",
|
|
5844
|
+
schema: stringSchema
|
|
5845
|
+
}
|
|
5846
|
+
],
|
|
5847
|
+
responses: {
|
|
5848
|
+
200: { description: "Cache pruned." },
|
|
5849
|
+
400: { description: "Invalid release id." },
|
|
5850
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5851
|
+
}
|
|
5852
|
+
},
|
|
5853
|
+
{
|
|
5854
|
+
method: "post",
|
|
5855
|
+
path: "/api/v1/projects/{name}/backlinks/extract",
|
|
5856
|
+
summary: "Extract backlinks for a single project from a cached release",
|
|
5857
|
+
description: 'Creates a `runs` row with kind="backlink-extract" and fires the extract callback. Defaults to the most recent ready release when `release` is omitted.',
|
|
5858
|
+
tags: ["backlinks"],
|
|
5859
|
+
parameters: [nameParameter],
|
|
5860
|
+
requestBody: {
|
|
5861
|
+
required: false,
|
|
5862
|
+
content: {
|
|
5863
|
+
"application/json": {
|
|
5864
|
+
schema: {
|
|
5865
|
+
type: "object",
|
|
5866
|
+
properties: {
|
|
5867
|
+
release: stringSchema
|
|
5868
|
+
}
|
|
5869
|
+
}
|
|
5870
|
+
}
|
|
5871
|
+
}
|
|
5872
|
+
},
|
|
5873
|
+
responses: {
|
|
5874
|
+
201: { description: "Extract run queued." },
|
|
5875
|
+
400: { description: "Invalid release id." },
|
|
5876
|
+
404: { description: "Project not found." },
|
|
5877
|
+
422: { description: "Backlinks feature is not available on this deployment." }
|
|
5878
|
+
}
|
|
5879
|
+
},
|
|
5880
|
+
{
|
|
5881
|
+
method: "get",
|
|
5882
|
+
path: "/api/v1/projects/{name}/backlinks/summary",
|
|
5883
|
+
summary: "Get the latest backlink summary for a project",
|
|
5884
|
+
tags: ["backlinks"],
|
|
5885
|
+
parameters: [
|
|
5886
|
+
nameParameter,
|
|
5887
|
+
{ name: "release", in: "query", description: "Release id filter.", schema: stringSchema }
|
|
5888
|
+
],
|
|
5889
|
+
responses: {
|
|
5890
|
+
200: { description: "Summary returned, or null when no backlinks exist." },
|
|
5891
|
+
404: { description: "Project not found." }
|
|
5892
|
+
}
|
|
5893
|
+
},
|
|
5894
|
+
{
|
|
5895
|
+
method: "get",
|
|
5896
|
+
path: "/api/v1/projects/{name}/backlinks/domains",
|
|
5897
|
+
summary: "Paginate backlink domains for a project",
|
|
5898
|
+
tags: ["backlinks"],
|
|
5899
|
+
parameters: [
|
|
5900
|
+
nameParameter,
|
|
5901
|
+
{ name: "release", in: "query", description: "Release id filter.", schema: stringSchema },
|
|
5902
|
+
{ name: "limit", in: "query", description: "Max results (1-500).", schema: stringSchema },
|
|
5903
|
+
{ name: "offset", in: "query", description: "Pagination offset.", schema: stringSchema }
|
|
5904
|
+
],
|
|
5905
|
+
responses: {
|
|
5906
|
+
200: { description: "Domain list returned." },
|
|
5907
|
+
404: { description: "Project not found." }
|
|
5908
|
+
}
|
|
5909
|
+
},
|
|
5910
|
+
{
|
|
5911
|
+
method: "get",
|
|
5912
|
+
path: "/api/v1/projects/{name}/backlinks/history",
|
|
5913
|
+
summary: "Get per-release backlink summaries for a project",
|
|
5914
|
+
tags: ["backlinks"],
|
|
5915
|
+
parameters: [nameParameter],
|
|
5916
|
+
responses: {
|
|
5917
|
+
200: { description: "History returned oldest-first by queriedAt." },
|
|
5918
|
+
404: { description: "Project not found." }
|
|
5919
|
+
}
|
|
5660
5920
|
}
|
|
5661
5921
|
];
|
|
5662
5922
|
var canonryLocalRouteCatalog = [
|
|
@@ -5791,8 +6051,8 @@ async function openApiRoutes(app, opts = {}) {
|
|
|
5791
6051
|
return reply.type("application/json").send(buildOpenApiDocument(opts));
|
|
5792
6052
|
});
|
|
5793
6053
|
}
|
|
5794
|
-
function buildOperationId(method,
|
|
5795
|
-
const parts =
|
|
6054
|
+
function buildOperationId(method, path16) {
|
|
6055
|
+
const parts = path16.split("/").filter(Boolean).map((part) => {
|
|
5796
6056
|
if (part.startsWith("{") && part.endsWith("}")) {
|
|
5797
6057
|
return `by-${part.slice(1, -1)}`;
|
|
5798
6058
|
}
|
|
@@ -9435,10 +9695,10 @@ function buildAuthErrorMessage(res, responseText) {
|
|
|
9435
9695
|
}
|
|
9436
9696
|
return "WordPress credentials are invalid or lack permission for this action";
|
|
9437
9697
|
}
|
|
9438
|
-
async function fetchJson(connection, siteUrl,
|
|
9698
|
+
async function fetchJson(connection, siteUrl, path16, init) {
|
|
9439
9699
|
if (siteUrl.startsWith("http:")) {
|
|
9440
9700
|
}
|
|
9441
|
-
const res = await fetch(`${normalizeSiteUrl(siteUrl)}${
|
|
9701
|
+
const res = await fetch(`${normalizeSiteUrl(siteUrl)}${path16}`, {
|
|
9442
9702
|
...init,
|
|
9443
9703
|
headers: {
|
|
9444
9704
|
"Authorization": `Basic ${encodeBasicAuth(connection.username, connection.appPassword)}`,
|
|
@@ -10920,6 +11180,566 @@ async function wordpressRoutes(app, opts) {
|
|
|
10920
11180
|
});
|
|
10921
11181
|
}
|
|
10922
11182
|
|
|
11183
|
+
// ../api-routes/src/backlinks.ts
|
|
11184
|
+
import crypto18 from "crypto";
|
|
11185
|
+
import { and as and7, asc as asc2, desc as desc8, eq as eq18, sql as sql5 } from "drizzle-orm";
|
|
11186
|
+
|
|
11187
|
+
// ../integration-commoncrawl/src/constants.ts
|
|
11188
|
+
import os3 from "os";
|
|
11189
|
+
import path3 from "path";
|
|
11190
|
+
var CC_BASE_URL = "https://data.commoncrawl.org/projects/hyperlinkgraph";
|
|
11191
|
+
var PLUGIN_DIR = path3.join(os3.homedir(), ".canonry", "plugins");
|
|
11192
|
+
var PLUGIN_PKG_JSON = path3.join(PLUGIN_DIR, "package.json");
|
|
11193
|
+
var DUCKDB_SPEC = process.env.CANONRY_DUCKDB_SPEC ?? "@duckdb/node-api@1.4.4-r.3";
|
|
11194
|
+
var CC_CACHE_DIR = process.env.CANONRY_CC_CACHE_DIR ?? path3.join(os3.homedir(), ".canonry", "cache", "commoncrawl");
|
|
11195
|
+
var RELEASE_ID_REGEX = /^cc-main-(\d{4})-(jan-feb-mar|apr-may-jun|jul-aug-sep|oct-nov-dec)$/;
|
|
11196
|
+
function ccReleasePaths(release) {
|
|
11197
|
+
const base = `${CC_BASE_URL}/${release}/domain`;
|
|
11198
|
+
const vertexFilename = `${release}-domain-vertices.txt.gz`;
|
|
11199
|
+
const edgesFilename = `${release}-domain-edges.txt.gz`;
|
|
11200
|
+
return {
|
|
11201
|
+
vertexUrl: `${base}/${vertexFilename}`,
|
|
11202
|
+
edgesUrl: `${base}/${edgesFilename}`,
|
|
11203
|
+
vertexFilename,
|
|
11204
|
+
edgesFilename
|
|
11205
|
+
};
|
|
11206
|
+
}
|
|
11207
|
+
|
|
11208
|
+
// ../integration-commoncrawl/src/reverse-domain.ts
|
|
11209
|
+
function reverseDomain(domain) {
|
|
11210
|
+
return domain.split(".").reverse().join(".");
|
|
11211
|
+
}
|
|
11212
|
+
function forwardDomain(revDomain) {
|
|
11213
|
+
return revDomain.split(".").reverse().join(".");
|
|
11214
|
+
}
|
|
11215
|
+
|
|
11216
|
+
// ../integration-commoncrawl/src/release-id.ts
|
|
11217
|
+
function isValidReleaseId(id) {
|
|
11218
|
+
return RELEASE_ID_REGEX.test(id);
|
|
11219
|
+
}
|
|
11220
|
+
|
|
11221
|
+
// ../integration-commoncrawl/src/downloader.ts
|
|
11222
|
+
import { createHash } from "crypto";
|
|
11223
|
+
import { createWriteStream } from "fs";
|
|
11224
|
+
import fs3 from "fs/promises";
|
|
11225
|
+
import path4 from "path";
|
|
11226
|
+
import { pipeline } from "stream/promises";
|
|
11227
|
+
import { Readable, Transform } from "stream";
|
|
11228
|
+
async function downloadFile(opts) {
|
|
11229
|
+
const start = Date.now();
|
|
11230
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
11231
|
+
const sidecarPath = `${opts.destPath}.sha256`;
|
|
11232
|
+
try {
|
|
11233
|
+
const stat = await fs3.stat(opts.destPath);
|
|
11234
|
+
const sidecar = await readSidecar(sidecarPath);
|
|
11235
|
+
const sha2562 = sidecar ?? await hashFile(opts.destPath);
|
|
11236
|
+
if (!sidecar) await writeSidecar(sidecarPath, sha2562);
|
|
11237
|
+
return { bytes: stat.size, sha256: sha2562, cached: true, elapsedMs: Date.now() - start };
|
|
11238
|
+
} catch {
|
|
11239
|
+
}
|
|
11240
|
+
const partialPath = `${opts.destPath}.partial`;
|
|
11241
|
+
await fs3.mkdir(path4.dirname(opts.destPath), { recursive: true });
|
|
11242
|
+
await unlinkIfExists(partialPath);
|
|
11243
|
+
const res = await fetchImpl(opts.url);
|
|
11244
|
+
if (!res.ok || !res.body) {
|
|
11245
|
+
throw new Error(`HTTP ${res.status} ${res.statusText} for ${opts.url}`);
|
|
11246
|
+
}
|
|
11247
|
+
const total = parseContentLength(res.headers.get("content-length"));
|
|
11248
|
+
const hasher = createHash("sha256");
|
|
11249
|
+
let bytes = 0;
|
|
11250
|
+
const hashAndCount = new Transform({
|
|
11251
|
+
transform(chunk, _enc, cb) {
|
|
11252
|
+
hasher.update(chunk);
|
|
11253
|
+
bytes += chunk.length;
|
|
11254
|
+
opts.onProgress?.(bytes, total);
|
|
11255
|
+
cb(null, chunk);
|
|
11256
|
+
}
|
|
11257
|
+
});
|
|
11258
|
+
await pipeline(
|
|
11259
|
+
Readable.fromWeb(res.body),
|
|
11260
|
+
hashAndCount,
|
|
11261
|
+
createWriteStream(partialPath)
|
|
11262
|
+
);
|
|
11263
|
+
const sha256 = hasher.digest("hex");
|
|
11264
|
+
await fs3.rename(partialPath, opts.destPath);
|
|
11265
|
+
await writeSidecar(sidecarPath, sha256);
|
|
11266
|
+
return { bytes, sha256, cached: false, elapsedMs: Date.now() - start };
|
|
11267
|
+
}
|
|
11268
|
+
async function hashFile(filePath) {
|
|
11269
|
+
const hasher = createHash("sha256");
|
|
11270
|
+
const handle = await fs3.open(filePath, "r");
|
|
11271
|
+
try {
|
|
11272
|
+
const stream = handle.createReadStream();
|
|
11273
|
+
for await (const chunk of stream) hasher.update(chunk);
|
|
11274
|
+
} finally {
|
|
11275
|
+
await handle.close();
|
|
11276
|
+
}
|
|
11277
|
+
return hasher.digest("hex");
|
|
11278
|
+
}
|
|
11279
|
+
async function readSidecar(sidecarPath) {
|
|
11280
|
+
try {
|
|
11281
|
+
const raw = await fs3.readFile(sidecarPath, "utf8");
|
|
11282
|
+
const trimmed = raw.trim();
|
|
11283
|
+
return /^[0-9a-f]{64}$/i.test(trimmed) ? trimmed.toLowerCase() : null;
|
|
11284
|
+
} catch {
|
|
11285
|
+
return null;
|
|
11286
|
+
}
|
|
11287
|
+
}
|
|
11288
|
+
async function writeSidecar(sidecarPath, sha256) {
|
|
11289
|
+
await fs3.writeFile(sidecarPath, `${sha256}
|
|
11290
|
+
`);
|
|
11291
|
+
}
|
|
11292
|
+
async function unlinkIfExists(p) {
|
|
11293
|
+
try {
|
|
11294
|
+
await fs3.unlink(p);
|
|
11295
|
+
} catch {
|
|
11296
|
+
}
|
|
11297
|
+
}
|
|
11298
|
+
function parseContentLength(value) {
|
|
11299
|
+
if (!value) return null;
|
|
11300
|
+
const n = Number.parseInt(value, 10);
|
|
11301
|
+
return Number.isFinite(n) ? n : null;
|
|
11302
|
+
}
|
|
11303
|
+
|
|
11304
|
+
// ../integration-commoncrawl/src/plugin-resolver.ts
|
|
11305
|
+
import fs4 from "fs";
|
|
11306
|
+
import { createRequire as createRequire2 } from "module";
|
|
11307
|
+
import path5 from "path";
|
|
11308
|
+
function pluginDirFor(pkgJson) {
|
|
11309
|
+
return path5.dirname(pkgJson);
|
|
11310
|
+
}
|
|
11311
|
+
function duckdbPkgJsonFor(pluginDir) {
|
|
11312
|
+
return path5.join(pluginDir, "node_modules", "@duckdb", "node-api", "package.json");
|
|
11313
|
+
}
|
|
11314
|
+
function loadDuckdb(opts = {}) {
|
|
11315
|
+
const pkgJson = opts.pluginPkgJson ?? PLUGIN_PKG_JSON;
|
|
11316
|
+
const pluginDir = pluginDirFor(pkgJson);
|
|
11317
|
+
const duckdbPkg = duckdbPkgJsonFor(pluginDir);
|
|
11318
|
+
if (!fs4.existsSync(duckdbPkg)) {
|
|
11319
|
+
throw missingDependency(
|
|
11320
|
+
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature.",
|
|
11321
|
+
{ pluginDir }
|
|
11322
|
+
);
|
|
11323
|
+
}
|
|
11324
|
+
try {
|
|
11325
|
+
const pluginRequire = createRequire2(duckdbPkg);
|
|
11326
|
+
return pluginRequire("@duckdb/node-api");
|
|
11327
|
+
} catch {
|
|
11328
|
+
throw missingDependency(
|
|
11329
|
+
"@duckdb/node-api is installed but failed to load. Re-run `canonry backlinks install`.",
|
|
11330
|
+
{ pluginDir }
|
|
11331
|
+
);
|
|
11332
|
+
}
|
|
11333
|
+
}
|
|
11334
|
+
function isDuckdbInstalled(opts = {}) {
|
|
11335
|
+
const pkgJson = opts.pluginPkgJson ?? PLUGIN_PKG_JSON;
|
|
11336
|
+
return fs4.existsSync(duckdbPkgJsonFor(pluginDirFor(pkgJson)));
|
|
11337
|
+
}
|
|
11338
|
+
function readInstalledVersion(opts = {}) {
|
|
11339
|
+
const pluginDir = opts.pluginPkgJson ? pluginDirFor(opts.pluginPkgJson) : PLUGIN_DIR;
|
|
11340
|
+
try {
|
|
11341
|
+
const raw = fs4.readFileSync(duckdbPkgJsonFor(pluginDir), "utf8");
|
|
11342
|
+
const pkg = JSON.parse(raw);
|
|
11343
|
+
return pkg.version ?? null;
|
|
11344
|
+
} catch {
|
|
11345
|
+
return null;
|
|
11346
|
+
}
|
|
11347
|
+
}
|
|
11348
|
+
|
|
11349
|
+
// ../integration-commoncrawl/src/plugin-installer.ts
|
|
11350
|
+
import { spawn } from "child_process";
|
|
11351
|
+
import fs5 from "fs/promises";
|
|
11352
|
+
import path6 from "path";
|
|
11353
|
+
async function installDuckdb(opts = {}) {
|
|
11354
|
+
const pluginDir = opts.pluginDir ?? PLUGIN_DIR;
|
|
11355
|
+
const pluginPkgJson = path6.join(pluginDir, "package.json");
|
|
11356
|
+
const spec = opts.spec ?? DUCKDB_SPEC;
|
|
11357
|
+
const pkgManager = opts.packageManager ?? "npm";
|
|
11358
|
+
await ensurePluginDir(pluginDir, pluginPkgJson);
|
|
11359
|
+
if (isDuckdbInstalled({ pluginPkgJson })) {
|
|
11360
|
+
const version2 = readInstalledVersion({ pluginPkgJson }) ?? "unknown";
|
|
11361
|
+
return { alreadyPresent: true, version: version2, path: pluginDir };
|
|
11362
|
+
}
|
|
11363
|
+
await runInstall(pkgManager, spec, pluginDir, opts.onLog);
|
|
11364
|
+
if (!isDuckdbInstalled({ pluginPkgJson })) {
|
|
11365
|
+
throw new Error(`${pkgManager} install completed but @duckdb/node-api still cannot be resolved from ${pluginDir}`);
|
|
11366
|
+
}
|
|
11367
|
+
const version = readInstalledVersion({ pluginPkgJson }) ?? "unknown";
|
|
11368
|
+
return { alreadyPresent: false, version, path: pluginDir };
|
|
11369
|
+
}
|
|
11370
|
+
async function ensurePluginDir(pluginDir = PLUGIN_DIR, pluginPkgJson = PLUGIN_PKG_JSON) {
|
|
11371
|
+
await fs5.mkdir(pluginDir, { recursive: true });
|
|
11372
|
+
try {
|
|
11373
|
+
await fs5.access(pluginPkgJson);
|
|
11374
|
+
} catch {
|
|
11375
|
+
const contents = JSON.stringify({ name: "canonry-plugins", private: true, dependencies: {} }, null, 2);
|
|
11376
|
+
await fs5.writeFile(pluginPkgJson, `${contents}
|
|
11377
|
+
`);
|
|
11378
|
+
}
|
|
11379
|
+
}
|
|
11380
|
+
async function runInstall(pkgManager, spec, pluginDir, onLog) {
|
|
11381
|
+
const args = pkgManager === "pnpm" ? ["add", spec, "--dir", pluginDir] : ["install", spec, "--prefix", pluginDir];
|
|
11382
|
+
await new Promise((resolve, reject) => {
|
|
11383
|
+
const child = spawn(pkgManager, args, {
|
|
11384
|
+
stdio: onLog ? ["ignore", "pipe", "pipe"] : "inherit"
|
|
11385
|
+
});
|
|
11386
|
+
if (onLog) {
|
|
11387
|
+
child.stdout?.setEncoding("utf8");
|
|
11388
|
+
child.stderr?.setEncoding("utf8");
|
|
11389
|
+
child.stdout?.on("data", (chunk) => {
|
|
11390
|
+
for (const line of chunk.split(/\r?\n/)) {
|
|
11391
|
+
if (line.length > 0) onLog(line);
|
|
11392
|
+
}
|
|
11393
|
+
});
|
|
11394
|
+
child.stderr?.on("data", (chunk) => {
|
|
11395
|
+
for (const line of chunk.split(/\r?\n/)) {
|
|
11396
|
+
if (line.length > 0) onLog(line);
|
|
11397
|
+
}
|
|
11398
|
+
});
|
|
11399
|
+
}
|
|
11400
|
+
child.on("error", reject);
|
|
11401
|
+
child.on("exit", (code) => {
|
|
11402
|
+
if (code === 0) resolve();
|
|
11403
|
+
else reject(new Error(`${pkgManager} install exited with code ${code}`));
|
|
11404
|
+
});
|
|
11405
|
+
});
|
|
11406
|
+
}
|
|
11407
|
+
|
|
11408
|
+
// ../integration-commoncrawl/src/duckdb-query.ts
|
|
11409
|
+
async function queryBacklinks(opts) {
|
|
11410
|
+
if (opts.targets.length === 0) return [];
|
|
11411
|
+
const duckdb = opts.duckdb;
|
|
11412
|
+
const reversed = opts.targets.map(reverseDomain);
|
|
11413
|
+
const targetList = reversed.map(quote).join(", ");
|
|
11414
|
+
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
11415
|
+
const sql10 = `
|
|
11416
|
+
WITH vertices AS (
|
|
11417
|
+
SELECT * FROM read_csv(
|
|
11418
|
+
${quote(opts.vertexPath)},
|
|
11419
|
+
delim=' ', header=false,
|
|
11420
|
+
columns={'id':'BIGINT','rev_domain':'VARCHAR','num_hosts':'BIGINT'}
|
|
11421
|
+
)
|
|
11422
|
+
),
|
|
11423
|
+
targets AS (
|
|
11424
|
+
SELECT v.id AS target_id, v.rev_domain AS target_rev_domain
|
|
11425
|
+
FROM vertices v
|
|
11426
|
+
WHERE v.rev_domain IN (${targetList})
|
|
11427
|
+
),
|
|
11428
|
+
inbound AS (
|
|
11429
|
+
SELECT e.from_id, e.to_id
|
|
11430
|
+
FROM read_csv(
|
|
11431
|
+
${quote(opts.edgesPath)},
|
|
11432
|
+
delim=' ', header=false,
|
|
11433
|
+
columns={'from_id':'BIGINT','to_id':'BIGINT'}
|
|
11434
|
+
) e
|
|
11435
|
+
WHERE e.to_id IN (SELECT target_id FROM targets)
|
|
11436
|
+
)
|
|
11437
|
+
SELECT
|
|
11438
|
+
t.target_rev_domain,
|
|
11439
|
+
v.rev_domain AS linking_rev_domain,
|
|
11440
|
+
v.num_hosts
|
|
11441
|
+
FROM inbound i
|
|
11442
|
+
JOIN targets t ON t.target_id = i.to_id
|
|
11443
|
+
JOIN vertices v ON v.id = i.from_id
|
|
11444
|
+
${limitClause}
|
|
11445
|
+
ORDER BY t.target_rev_domain, v.num_hosts DESC
|
|
11446
|
+
`;
|
|
11447
|
+
const instance = await duckdb.DuckDBInstance.create(":memory:");
|
|
11448
|
+
const conn = await instance.connect();
|
|
11449
|
+
let rows;
|
|
11450
|
+
try {
|
|
11451
|
+
const reader = await conn.runAndReadAll(sql10);
|
|
11452
|
+
rows = reader.getRowObjects();
|
|
11453
|
+
} finally {
|
|
11454
|
+
conn.disconnectSync?.();
|
|
11455
|
+
conn.closeSync?.();
|
|
11456
|
+
instance.closeSync?.();
|
|
11457
|
+
}
|
|
11458
|
+
return rows.map((r) => ({
|
|
11459
|
+
targetDomain: forwardDomain(String(r["target_rev_domain"])),
|
|
11460
|
+
linkingDomain: forwardDomain(String(r["linking_rev_domain"])),
|
|
11461
|
+
numHosts: Number(r["num_hosts"])
|
|
11462
|
+
}));
|
|
11463
|
+
}
|
|
11464
|
+
function quote(s) {
|
|
11465
|
+
return `'${s.replace(/'/g, "''")}'`;
|
|
11466
|
+
}
|
|
11467
|
+
|
|
11468
|
+
// ../integration-commoncrawl/src/cache.ts
|
|
11469
|
+
import fs6 from "fs";
|
|
11470
|
+
import path7 from "path";
|
|
11471
|
+
function cacheRoot(opts = {}) {
|
|
11472
|
+
return opts.cacheDir ?? CC_CACHE_DIR;
|
|
11473
|
+
}
|
|
11474
|
+
function directoryBytesAndLastUsed(dir) {
|
|
11475
|
+
let bytes = 0;
|
|
11476
|
+
let latestMtimeMs = 0;
|
|
11477
|
+
const walk = (p) => {
|
|
11478
|
+
let stat;
|
|
11479
|
+
try {
|
|
11480
|
+
stat = fs6.statSync(p);
|
|
11481
|
+
} catch {
|
|
11482
|
+
return;
|
|
11483
|
+
}
|
|
11484
|
+
if (stat.isDirectory()) {
|
|
11485
|
+
let entries;
|
|
11486
|
+
try {
|
|
11487
|
+
entries = fs6.readdirSync(p);
|
|
11488
|
+
} catch {
|
|
11489
|
+
return;
|
|
11490
|
+
}
|
|
11491
|
+
for (const e of entries) walk(path7.join(p, e));
|
|
11492
|
+
} else if (stat.isFile()) {
|
|
11493
|
+
bytes += stat.size;
|
|
11494
|
+
const mtime = Math.max(stat.mtimeMs, stat.atimeMs);
|
|
11495
|
+
if (mtime > latestMtimeMs) latestMtimeMs = mtime;
|
|
11496
|
+
}
|
|
11497
|
+
};
|
|
11498
|
+
walk(dir);
|
|
11499
|
+
return {
|
|
11500
|
+
bytes,
|
|
11501
|
+
lastUsedAt: latestMtimeMs > 0 ? new Date(latestMtimeMs).toISOString() : null
|
|
11502
|
+
};
|
|
11503
|
+
}
|
|
11504
|
+
function listCachedReleases(opts = {}) {
|
|
11505
|
+
const root = cacheRoot(opts);
|
|
11506
|
+
if (!fs6.existsSync(root)) return [];
|
|
11507
|
+
const entries = fs6.readdirSync(root, { withFileTypes: true });
|
|
11508
|
+
const result = [];
|
|
11509
|
+
for (const entry of entries) {
|
|
11510
|
+
if (!entry.isDirectory()) continue;
|
|
11511
|
+
if (!RELEASE_ID_REGEX.test(entry.name)) continue;
|
|
11512
|
+
const dir = path7.join(root, entry.name);
|
|
11513
|
+
const stats = directoryBytesAndLastUsed(dir);
|
|
11514
|
+
result.push({ release: entry.name, bytes: stats.bytes, lastUsedAt: stats.lastUsedAt });
|
|
11515
|
+
}
|
|
11516
|
+
result.sort((a, b) => (b.lastUsedAt ?? "").localeCompare(a.lastUsedAt ?? ""));
|
|
11517
|
+
return result;
|
|
11518
|
+
}
|
|
11519
|
+
function pruneCachedRelease(release, opts = {}) {
|
|
11520
|
+
if (!RELEASE_ID_REGEX.test(release)) {
|
|
11521
|
+
throw new Error(`Invalid release id: ${release}`);
|
|
11522
|
+
}
|
|
11523
|
+
const dir = path7.join(cacheRoot(opts), release);
|
|
11524
|
+
fs6.rmSync(dir, { recursive: true, force: true });
|
|
11525
|
+
}
|
|
11526
|
+
|
|
11527
|
+
// ../api-routes/src/backlinks.ts
|
|
11528
|
+
var BACKLINKS_UNSUPPORTED_MESSAGE = "Backlinks sync and install are only available from a local canonry install. Run `canonry backlinks install` locally to use this feature.";
|
|
11529
|
+
var NON_TERMINAL_SYNC_STATUSES = /* @__PURE__ */ new Set([
|
|
11530
|
+
CcReleaseSyncStatuses.queued,
|
|
11531
|
+
CcReleaseSyncStatuses.downloading,
|
|
11532
|
+
CcReleaseSyncStatuses.querying
|
|
11533
|
+
]);
|
|
11534
|
+
function mapSyncRow(row) {
|
|
11535
|
+
return {
|
|
11536
|
+
id: row.id,
|
|
11537
|
+
release: row.release,
|
|
11538
|
+
status: row.status,
|
|
11539
|
+
phaseDetail: row.phaseDetail ?? null,
|
|
11540
|
+
vertexPath: row.vertexPath ?? null,
|
|
11541
|
+
edgesPath: row.edgesPath ?? null,
|
|
11542
|
+
vertexSha256: row.vertexSha256 ?? null,
|
|
11543
|
+
edgesSha256: row.edgesSha256 ?? null,
|
|
11544
|
+
vertexBytes: row.vertexBytes ?? null,
|
|
11545
|
+
edgesBytes: row.edgesBytes ?? null,
|
|
11546
|
+
projectsProcessed: row.projectsProcessed ?? null,
|
|
11547
|
+
domainsDiscovered: row.domainsDiscovered ?? null,
|
|
11548
|
+
downloadStartedAt: row.downloadStartedAt ?? null,
|
|
11549
|
+
downloadFinishedAt: row.downloadFinishedAt ?? null,
|
|
11550
|
+
queryStartedAt: row.queryStartedAt ?? null,
|
|
11551
|
+
queryFinishedAt: row.queryFinishedAt ?? null,
|
|
11552
|
+
error: row.error ?? null,
|
|
11553
|
+
createdAt: row.createdAt,
|
|
11554
|
+
updatedAt: row.updatedAt
|
|
11555
|
+
};
|
|
11556
|
+
}
|
|
11557
|
+
function mapSummaryRow(row) {
|
|
11558
|
+
return {
|
|
11559
|
+
projectId: row.projectId,
|
|
11560
|
+
release: row.release,
|
|
11561
|
+
targetDomain: row.targetDomain,
|
|
11562
|
+
totalLinkingDomains: row.totalLinkingDomains,
|
|
11563
|
+
totalHosts: row.totalHosts,
|
|
11564
|
+
top10HostsShare: row.top10HostsShare,
|
|
11565
|
+
queriedAt: row.queriedAt
|
|
11566
|
+
};
|
|
11567
|
+
}
|
|
11568
|
+
function mapRunRow(row) {
|
|
11569
|
+
return {
|
|
11570
|
+
id: row.id,
|
|
11571
|
+
projectId: row.projectId,
|
|
11572
|
+
kind: row.kind,
|
|
11573
|
+
status: row.status,
|
|
11574
|
+
trigger: row.trigger,
|
|
11575
|
+
location: row.location ?? null,
|
|
11576
|
+
startedAt: row.startedAt ?? null,
|
|
11577
|
+
finishedAt: row.finishedAt ?? null,
|
|
11578
|
+
error: row.error ?? null,
|
|
11579
|
+
createdAt: row.createdAt
|
|
11580
|
+
};
|
|
11581
|
+
}
|
|
11582
|
+
function latestSummaryForProject(db, projectId, release) {
|
|
11583
|
+
const condition = release ? and7(eq18(backlinkSummaries.projectId, projectId), eq18(backlinkSummaries.release, release)) : eq18(backlinkSummaries.projectId, projectId);
|
|
11584
|
+
return db.select().from(backlinkSummaries).where(condition).orderBy(desc8(backlinkSummaries.queriedAt)).limit(1).get();
|
|
11585
|
+
}
|
|
11586
|
+
async function backlinksRoutes(app, opts) {
|
|
11587
|
+
app.get("/backlinks/status", async (_request, reply) => {
|
|
11588
|
+
if (!opts.getBacklinksStatus) {
|
|
11589
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11590
|
+
}
|
|
11591
|
+
return reply.send(opts.getBacklinksStatus());
|
|
11592
|
+
});
|
|
11593
|
+
app.post("/backlinks/install", async (_request, reply) => {
|
|
11594
|
+
if (!opts.onInstallBacklinks) {
|
|
11595
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11596
|
+
}
|
|
11597
|
+
const result = await opts.onInstallBacklinks();
|
|
11598
|
+
return reply.status(200).send(result);
|
|
11599
|
+
});
|
|
11600
|
+
app.post("/backlinks/syncs", async (request, reply) => {
|
|
11601
|
+
const release = request.body?.release;
|
|
11602
|
+
if (!release || !isValidReleaseId(release)) {
|
|
11603
|
+
throw validationError("Invalid release id. Expected form: cc-main-YYYY-{jan-feb-mar,apr-may-jun,jul-aug-sep,oct-nov-dec}");
|
|
11604
|
+
}
|
|
11605
|
+
if (!opts.getBacklinksStatus || !opts.onReleaseSyncRequested) {
|
|
11606
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11607
|
+
}
|
|
11608
|
+
if (!opts.getBacklinksStatus().duckdbInstalled) {
|
|
11609
|
+
throw missingDependency(
|
|
11610
|
+
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature."
|
|
11611
|
+
);
|
|
11612
|
+
}
|
|
11613
|
+
const existing = app.db.select().from(ccReleaseSyncs).where(eq18(ccReleaseSyncs.release, release)).get();
|
|
11614
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11615
|
+
if (existing) {
|
|
11616
|
+
if (NON_TERMINAL_SYNC_STATUSES.has(existing.status)) {
|
|
11617
|
+
return reply.status(200).send(mapSyncRow(existing));
|
|
11618
|
+
}
|
|
11619
|
+
app.db.update(ccReleaseSyncs).set({
|
|
11620
|
+
status: CcReleaseSyncStatuses.queued,
|
|
11621
|
+
phaseDetail: null,
|
|
11622
|
+
error: null,
|
|
11623
|
+
updatedAt: now
|
|
11624
|
+
}).where(eq18(ccReleaseSyncs.id, existing.id)).run();
|
|
11625
|
+
opts.onReleaseSyncRequested(existing.id, release);
|
|
11626
|
+
const refreshed = app.db.select().from(ccReleaseSyncs).where(eq18(ccReleaseSyncs.id, existing.id)).get();
|
|
11627
|
+
return reply.status(200).send(mapSyncRow(refreshed));
|
|
11628
|
+
}
|
|
11629
|
+
const id = crypto18.randomUUID();
|
|
11630
|
+
app.db.insert(ccReleaseSyncs).values({
|
|
11631
|
+
id,
|
|
11632
|
+
release,
|
|
11633
|
+
status: CcReleaseSyncStatuses.queued,
|
|
11634
|
+
createdAt: now,
|
|
11635
|
+
updatedAt: now
|
|
11636
|
+
}).run();
|
|
11637
|
+
opts.onReleaseSyncRequested(id, release);
|
|
11638
|
+
const inserted = app.db.select().from(ccReleaseSyncs).where(eq18(ccReleaseSyncs.id, id)).get();
|
|
11639
|
+
return reply.status(201).send(mapSyncRow(inserted));
|
|
11640
|
+
});
|
|
11641
|
+
app.get("/backlinks/syncs/latest", async (_request, reply) => {
|
|
11642
|
+
const row = app.db.select().from(ccReleaseSyncs).orderBy(desc8(ccReleaseSyncs.updatedAt)).limit(1).get();
|
|
11643
|
+
return reply.send(row ? mapSyncRow(row) : null);
|
|
11644
|
+
});
|
|
11645
|
+
app.get("/backlinks/syncs", async (_request, reply) => {
|
|
11646
|
+
const rows = app.db.select().from(ccReleaseSyncs).orderBy(desc8(ccReleaseSyncs.updatedAt)).all();
|
|
11647
|
+
return reply.send(rows.map(mapSyncRow));
|
|
11648
|
+
});
|
|
11649
|
+
app.get("/backlinks/releases", async (_request, reply) => {
|
|
11650
|
+
const releases = opts.listCachedReleases?.() ?? [];
|
|
11651
|
+
return reply.send(releases);
|
|
11652
|
+
});
|
|
11653
|
+
app.delete("/backlinks/cache/:release", async (request, reply) => {
|
|
11654
|
+
const release = request.params.release;
|
|
11655
|
+
if (!isValidReleaseId(release)) {
|
|
11656
|
+
throw validationError("Invalid release id");
|
|
11657
|
+
}
|
|
11658
|
+
if (!opts.onBacklinksPruneCache) {
|
|
11659
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11660
|
+
}
|
|
11661
|
+
opts.onBacklinksPruneCache(release);
|
|
11662
|
+
return reply.send({ ok: true });
|
|
11663
|
+
});
|
|
11664
|
+
app.post("/projects/:name/backlinks/extract", async (request, reply) => {
|
|
11665
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11666
|
+
if (!opts.getBacklinksStatus || !opts.onBacklinkExtractRequested) {
|
|
11667
|
+
throw missingDependency(BACKLINKS_UNSUPPORTED_MESSAGE);
|
|
11668
|
+
}
|
|
11669
|
+
if (!opts.getBacklinksStatus().duckdbInstalled) {
|
|
11670
|
+
throw missingDependency(
|
|
11671
|
+
"@duckdb/node-api is not installed. Run `canonry backlinks install` to enable the backlinks feature."
|
|
11672
|
+
);
|
|
11673
|
+
}
|
|
11674
|
+
const release = request.body?.release;
|
|
11675
|
+
if (release !== void 0 && !isValidReleaseId(release)) {
|
|
11676
|
+
throw validationError("Invalid release id");
|
|
11677
|
+
}
|
|
11678
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11679
|
+
const runId = crypto18.randomUUID();
|
|
11680
|
+
app.db.insert(runs).values({
|
|
11681
|
+
id: runId,
|
|
11682
|
+
projectId: project.id,
|
|
11683
|
+
kind: RunKinds["backlink-extract"],
|
|
11684
|
+
status: RunStatuses.queued,
|
|
11685
|
+
trigger: RunTriggers.manual,
|
|
11686
|
+
createdAt: now
|
|
11687
|
+
}).run();
|
|
11688
|
+
opts.onBacklinkExtractRequested(runId, project.id, release);
|
|
11689
|
+
const run = app.db.select().from(runs).where(eq18(runs.id, runId)).get();
|
|
11690
|
+
return reply.status(201).send(mapRunRow(run));
|
|
11691
|
+
});
|
|
11692
|
+
app.get(
|
|
11693
|
+
"/projects/:name/backlinks/summary",
|
|
11694
|
+
async (request, reply) => {
|
|
11695
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11696
|
+
const row = latestSummaryForProject(app.db, project.id, request.query.release);
|
|
11697
|
+
return reply.send(row ? mapSummaryRow(row) : null);
|
|
11698
|
+
}
|
|
11699
|
+
);
|
|
11700
|
+
app.get("/projects/:name/backlinks/domains", async (request, reply) => {
|
|
11701
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11702
|
+
const summaryRow = latestSummaryForProject(app.db, project.id, request.query.release);
|
|
11703
|
+
const targetRelease = request.query.release ?? summaryRow?.release;
|
|
11704
|
+
if (!targetRelease) {
|
|
11705
|
+
const response2 = { summary: null, total: 0, rows: [] };
|
|
11706
|
+
return reply.send(response2);
|
|
11707
|
+
}
|
|
11708
|
+
const limit = Math.min(Math.max(parseInt(request.query.limit ?? "50", 10) || 50, 1), 500);
|
|
11709
|
+
const offset = Math.max(parseInt(request.query.offset ?? "0", 10) || 0, 0);
|
|
11710
|
+
const domainCondition = and7(
|
|
11711
|
+
eq18(backlinkDomains.projectId, project.id),
|
|
11712
|
+
eq18(backlinkDomains.release, targetRelease)
|
|
11713
|
+
);
|
|
11714
|
+
const totalRow = app.db.select({ count: sql5`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
11715
|
+
const rows = app.db.select({
|
|
11716
|
+
linkingDomain: backlinkDomains.linkingDomain,
|
|
11717
|
+
numHosts: backlinkDomains.numHosts
|
|
11718
|
+
}).from(backlinkDomains).where(domainCondition).orderBy(desc8(backlinkDomains.numHosts)).limit(limit).offset(offset).all();
|
|
11719
|
+
const response = {
|
|
11720
|
+
summary: summaryRow ? mapSummaryRow(summaryRow) : null,
|
|
11721
|
+
total: Number(totalRow?.count ?? 0),
|
|
11722
|
+
rows
|
|
11723
|
+
};
|
|
11724
|
+
return reply.send(response);
|
|
11725
|
+
});
|
|
11726
|
+
app.get(
|
|
11727
|
+
"/projects/:name/backlinks/history",
|
|
11728
|
+
async (request, reply) => {
|
|
11729
|
+
const project = resolveProject(app.db, request.params.name);
|
|
11730
|
+
const rows = app.db.select().from(backlinkSummaries).where(eq18(backlinkSummaries.projectId, project.id)).orderBy(asc2(backlinkSummaries.queriedAt)).all();
|
|
11731
|
+
const response = rows.map((r) => ({
|
|
11732
|
+
release: r.release,
|
|
11733
|
+
totalLinkingDomains: r.totalLinkingDomains,
|
|
11734
|
+
totalHosts: r.totalHosts,
|
|
11735
|
+
top10HostsShare: r.top10HostsShare,
|
|
11736
|
+
queriedAt: r.queriedAt
|
|
11737
|
+
}));
|
|
11738
|
+
return reply.send(response);
|
|
11739
|
+
}
|
|
11740
|
+
);
|
|
11741
|
+
}
|
|
11742
|
+
|
|
10923
11743
|
// ../api-routes/src/index.ts
|
|
10924
11744
|
async function apiRoutes(app, opts) {
|
|
10925
11745
|
app.decorate("db", opts.db);
|
|
@@ -11028,6 +11848,14 @@ async function apiRoutes(app, opts) {
|
|
|
11028
11848
|
googleConnectionStore: opts.googleConnectionStore,
|
|
11029
11849
|
getGoogleAuthConfig: opts.getGoogleAuthConfig
|
|
11030
11850
|
});
|
|
11851
|
+
await api.register(backlinksRoutes, {
|
|
11852
|
+
getBacklinksStatus: opts.getBacklinksStatus,
|
|
11853
|
+
onInstallBacklinks: opts.onInstallBacklinks,
|
|
11854
|
+
onReleaseSyncRequested: opts.onReleaseSyncRequested,
|
|
11855
|
+
onBacklinkExtractRequested: opts.onBacklinkExtractRequested,
|
|
11856
|
+
onBacklinksPruneCache: opts.onBacklinksPruneCache,
|
|
11857
|
+
listCachedReleases: opts.listCachedReleases
|
|
11858
|
+
});
|
|
11031
11859
|
if (opts.registerAuthenticatedRoutes) {
|
|
11032
11860
|
await opts.registerAuthenticatedRoutes(api);
|
|
11033
11861
|
}
|
|
@@ -11035,7 +11863,7 @@ async function apiRoutes(app, opts) {
|
|
|
11035
11863
|
}
|
|
11036
11864
|
|
|
11037
11865
|
// src/server.ts
|
|
11038
|
-
import
|
|
11866
|
+
import os6 from "os";
|
|
11039
11867
|
|
|
11040
11868
|
// ../provider-gemini/src/normalize.ts
|
|
11041
11869
|
import { GoogleGenAI } from "@google/genai";
|
|
@@ -12423,8 +13251,8 @@ var localAdapter = {
|
|
|
12423
13251
|
};
|
|
12424
13252
|
|
|
12425
13253
|
// ../provider-cdp/src/adapter.ts
|
|
12426
|
-
import
|
|
12427
|
-
import
|
|
13254
|
+
import path9 from "path";
|
|
13255
|
+
import os4 from "os";
|
|
12428
13256
|
|
|
12429
13257
|
// ../provider-cdp/src/connection.ts
|
|
12430
13258
|
import CDP from "chrome-remote-interface";
|
|
@@ -12788,12 +13616,12 @@ function sleep2(ms) {
|
|
|
12788
13616
|
}
|
|
12789
13617
|
|
|
12790
13618
|
// ../provider-cdp/src/screenshot.ts
|
|
12791
|
-
import
|
|
12792
|
-
import
|
|
13619
|
+
import fs7 from "fs";
|
|
13620
|
+
import path8 from "path";
|
|
12793
13621
|
async function captureElementScreenshot(client, selector, outputPath) {
|
|
12794
|
-
const dir =
|
|
12795
|
-
if (!
|
|
12796
|
-
|
|
13622
|
+
const dir = path8.dirname(outputPath);
|
|
13623
|
+
if (!fs7.existsSync(dir)) {
|
|
13624
|
+
fs7.mkdirSync(dir, { recursive: true });
|
|
12797
13625
|
}
|
|
12798
13626
|
let clip;
|
|
12799
13627
|
try {
|
|
@@ -12827,7 +13655,7 @@ async function captureElementScreenshot(client, selector, outputPath) {
|
|
|
12827
13655
|
}
|
|
12828
13656
|
const { data } = await client.Page.captureScreenshot(screenshotParams);
|
|
12829
13657
|
const buffer = Buffer.from(data, "base64");
|
|
12830
|
-
|
|
13658
|
+
fs7.writeFileSync(outputPath, buffer);
|
|
12831
13659
|
return outputPath;
|
|
12832
13660
|
}
|
|
12833
13661
|
|
|
@@ -12888,7 +13716,7 @@ function getConnection(config) {
|
|
|
12888
13716
|
return conn;
|
|
12889
13717
|
}
|
|
12890
13718
|
function getScreenshotDir2() {
|
|
12891
|
-
return
|
|
13719
|
+
return path9.join(os4.homedir(), ".canonry", "screenshots");
|
|
12892
13720
|
}
|
|
12893
13721
|
var cdpChatgptAdapter = {
|
|
12894
13722
|
name: "cdp:chatgpt",
|
|
@@ -12952,7 +13780,7 @@ var cdpChatgptAdapter = {
|
|
|
12952
13780
|
const answerText = await target.extractAnswer(client);
|
|
12953
13781
|
const groundingSources = await target.extractCitations(client);
|
|
12954
13782
|
const screenshotId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
12955
|
-
const screenshotPath =
|
|
13783
|
+
const screenshotPath = path9.join(getScreenshotDir2(), `${screenshotId}.png`);
|
|
12956
13784
|
let capturedScreenshotPath;
|
|
12957
13785
|
try {
|
|
12958
13786
|
capturedScreenshotPath = await captureElementScreenshot(
|
|
@@ -13488,11 +14316,11 @@ function removeWordpressConnection(config, projectName) {
|
|
|
13488
14316
|
}
|
|
13489
14317
|
|
|
13490
14318
|
// src/job-runner.ts
|
|
13491
|
-
import
|
|
13492
|
-
import
|
|
13493
|
-
import
|
|
13494
|
-
import
|
|
13495
|
-
import { and as
|
|
14319
|
+
import crypto19 from "crypto";
|
|
14320
|
+
import fs8 from "fs";
|
|
14321
|
+
import path10 from "path";
|
|
14322
|
+
import os5 from "os";
|
|
14323
|
+
import { and as and8, eq as eq19, inArray as inArray3, sql as sql6 } from "drizzle-orm";
|
|
13496
14324
|
|
|
13497
14325
|
// src/citation-utils.ts
|
|
13498
14326
|
function domainMatches(domain, canonicalDomain) {
|
|
@@ -13728,7 +14556,7 @@ var JobRunner = class {
|
|
|
13728
14556
|
if (stale.length === 0) return;
|
|
13729
14557
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13730
14558
|
for (const run of stale) {
|
|
13731
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
14559
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq19(runs.id, run.id)).run();
|
|
13732
14560
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
13733
14561
|
}
|
|
13734
14562
|
}
|
|
@@ -13756,10 +14584,10 @@ var JobRunner = class {
|
|
|
13756
14584
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
13757
14585
|
}
|
|
13758
14586
|
if (existingRun.status === "queued") {
|
|
13759
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
14587
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and8(eq19(runs.id, runId), eq19(runs.status, "queued"))).run();
|
|
13760
14588
|
}
|
|
13761
14589
|
this.throwIfRunCancelled(runId);
|
|
13762
|
-
const project = this.db.select().from(projects).where(
|
|
14590
|
+
const project = this.db.select().from(projects).where(eq19(projects.id, projectId)).get();
|
|
13763
14591
|
if (!project) {
|
|
13764
14592
|
throw new Error(`Project ${projectId} not found`);
|
|
13765
14593
|
}
|
|
@@ -13779,8 +14607,8 @@ var JobRunner = class {
|
|
|
13779
14607
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
13780
14608
|
}
|
|
13781
14609
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
13782
|
-
projectKeywords = this.db.select().from(keywords).where(
|
|
13783
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
14610
|
+
projectKeywords = this.db.select().from(keywords).where(eq19(keywords.projectId, projectId)).all();
|
|
14611
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq19(competitors.projectId, projectId)).all();
|
|
13784
14612
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
13785
14613
|
const allDomains = effectiveDomains({
|
|
13786
14614
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -13796,7 +14624,7 @@ var JobRunner = class {
|
|
|
13796
14624
|
const todayPeriod = getCurrentUsageDay();
|
|
13797
14625
|
for (const p of activeProviders) {
|
|
13798
14626
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
13799
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
14627
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq19(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
13800
14628
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
13801
14629
|
if (providerUsage + queriesPerProvider > limit) {
|
|
13802
14630
|
throw new Error(
|
|
@@ -13855,12 +14683,12 @@ var JobRunner = class {
|
|
|
13855
14683
|
competitorDomains
|
|
13856
14684
|
);
|
|
13857
14685
|
let screenshotRelPath = null;
|
|
13858
|
-
if (raw.screenshotPath &&
|
|
13859
|
-
const snapshotId =
|
|
13860
|
-
const screenshotDir =
|
|
13861
|
-
if (!
|
|
13862
|
-
const destPath =
|
|
13863
|
-
|
|
14686
|
+
if (raw.screenshotPath && fs8.existsSync(raw.screenshotPath)) {
|
|
14687
|
+
const snapshotId = crypto19.randomUUID();
|
|
14688
|
+
const screenshotDir = path10.join(os5.homedir(), ".canonry", "screenshots", runId);
|
|
14689
|
+
if (!fs8.existsSync(screenshotDir)) fs8.mkdirSync(screenshotDir, { recursive: true });
|
|
14690
|
+
const destPath = path10.join(screenshotDir, `${snapshotId}.png`);
|
|
14691
|
+
fs8.renameSync(raw.screenshotPath, destPath);
|
|
13864
14692
|
screenshotRelPath = `${runId}/${snapshotId}.png`;
|
|
13865
14693
|
this.db.insert(querySnapshots).values({
|
|
13866
14694
|
id: snapshotId,
|
|
@@ -13886,7 +14714,7 @@ var JobRunner = class {
|
|
|
13886
14714
|
}).run();
|
|
13887
14715
|
} else {
|
|
13888
14716
|
this.db.insert(querySnapshots).values({
|
|
13889
|
-
id:
|
|
14717
|
+
id: crypto19.randomUUID(),
|
|
13890
14718
|
runId,
|
|
13891
14719
|
keywordId: kw.id,
|
|
13892
14720
|
provider: providerName,
|
|
@@ -13937,12 +14765,12 @@ var JobRunner = class {
|
|
|
13937
14765
|
const someFailed = providerErrors.size > 0;
|
|
13938
14766
|
if (allFailed) {
|
|
13939
14767
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
13940
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
14768
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq19(runs.id, runId)).run();
|
|
13941
14769
|
} else if (someFailed) {
|
|
13942
14770
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
13943
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
14771
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq19(runs.id, runId)).run();
|
|
13944
14772
|
} else {
|
|
13945
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14773
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(runs.id, runId)).run();
|
|
13946
14774
|
}
|
|
13947
14775
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
13948
14776
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -13977,7 +14805,7 @@ var JobRunner = class {
|
|
|
13977
14805
|
status: "failed",
|
|
13978
14806
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13979
14807
|
error: errorMessage
|
|
13980
|
-
}).where(
|
|
14808
|
+
}).where(eq19(runs.id, runId)).run();
|
|
13981
14809
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
13982
14810
|
trackEvent("run.completed", {
|
|
13983
14811
|
status: "failed",
|
|
@@ -13998,7 +14826,7 @@ var JobRunner = class {
|
|
|
13998
14826
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
13999
14827
|
const period = now.slice(0, 10);
|
|
14000
14828
|
this.db.insert(usageCounters).values({
|
|
14001
|
-
id:
|
|
14829
|
+
id: crypto19.randomUUID(),
|
|
14002
14830
|
scope,
|
|
14003
14831
|
period,
|
|
14004
14832
|
metric,
|
|
@@ -14006,7 +14834,7 @@ var JobRunner = class {
|
|
|
14006
14834
|
updatedAt: now
|
|
14007
14835
|
}).onConflictDoUpdate({
|
|
14008
14836
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
14009
|
-
set: { count:
|
|
14837
|
+
set: { count: sql6`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
14010
14838
|
}).run();
|
|
14011
14839
|
}
|
|
14012
14840
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -14020,7 +14848,7 @@ var JobRunner = class {
|
|
|
14020
14848
|
status: runs.status,
|
|
14021
14849
|
finishedAt: runs.finishedAt,
|
|
14022
14850
|
error: runs.error
|
|
14023
|
-
}).from(runs).where(
|
|
14851
|
+
}).from(runs).where(eq19(runs.id, runId)).get();
|
|
14024
14852
|
}
|
|
14025
14853
|
isRunCancelled(runId) {
|
|
14026
14854
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -14036,7 +14864,7 @@ var JobRunner = class {
|
|
|
14036
14864
|
this.db.update(runs).set({
|
|
14037
14865
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14038
14866
|
error: currentRun.error ?? "Cancelled by user"
|
|
14039
|
-
}).where(
|
|
14867
|
+
}).where(eq19(runs.id, runId)).run();
|
|
14040
14868
|
}
|
|
14041
14869
|
trackEvent("run.completed", {
|
|
14042
14870
|
status: "cancelled",
|
|
@@ -14058,8 +14886,8 @@ function getCurrentUsageDay() {
|
|
|
14058
14886
|
}
|
|
14059
14887
|
|
|
14060
14888
|
// src/gsc-sync.ts
|
|
14061
|
-
import
|
|
14062
|
-
import { eq as
|
|
14889
|
+
import crypto20 from "crypto";
|
|
14890
|
+
import { eq as eq20, and as and9, sql as sql7 } from "drizzle-orm";
|
|
14063
14891
|
var log2 = createLogger("GscSync");
|
|
14064
14892
|
function formatDate2(d) {
|
|
14065
14893
|
return d.toISOString().split("T")[0];
|
|
@@ -14071,13 +14899,13 @@ function daysAgo(n) {
|
|
|
14071
14899
|
}
|
|
14072
14900
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
14073
14901
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14074
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
14902
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq20(runs.id, runId)).run();
|
|
14075
14903
|
try {
|
|
14076
14904
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
14077
14905
|
if (!googleClientId || !googleClientSecret) {
|
|
14078
14906
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
14079
14907
|
}
|
|
14080
|
-
const project = db.select().from(projects).where(
|
|
14908
|
+
const project = db.select().from(projects).where(eq20(projects.id, projectId)).get();
|
|
14081
14909
|
if (!project) {
|
|
14082
14910
|
throw new Error(`Project not found: ${projectId}`);
|
|
14083
14911
|
}
|
|
@@ -14111,10 +14939,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14111
14939
|
});
|
|
14112
14940
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
14113
14941
|
db.delete(gscSearchData).where(
|
|
14114
|
-
|
|
14115
|
-
|
|
14116
|
-
|
|
14117
|
-
|
|
14942
|
+
and9(
|
|
14943
|
+
eq20(gscSearchData.projectId, projectId),
|
|
14944
|
+
sql7`${gscSearchData.date} >= ${startDate}`,
|
|
14945
|
+
sql7`${gscSearchData.date} <= ${endDate}`
|
|
14118
14946
|
)
|
|
14119
14947
|
).run();
|
|
14120
14948
|
const batchSize = 500;
|
|
@@ -14124,7 +14952,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14124
14952
|
for (const row of batch) {
|
|
14125
14953
|
const [query, page, country, device, date] = row.keys;
|
|
14126
14954
|
db.insert(gscSearchData).values({
|
|
14127
|
-
id:
|
|
14955
|
+
id: crypto20.randomUUID(),
|
|
14128
14956
|
projectId,
|
|
14129
14957
|
syncRunId: runId,
|
|
14130
14958
|
date: date ?? "",
|
|
@@ -14158,7 +14986,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14158
14986
|
const rich = ir.richResultsResult;
|
|
14159
14987
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14160
14988
|
db.insert(gscUrlInspections).values({
|
|
14161
|
-
id:
|
|
14989
|
+
id: crypto20.randomUUID(),
|
|
14162
14990
|
projectId,
|
|
14163
14991
|
syncRunId: runId,
|
|
14164
14992
|
url: pageUrl,
|
|
@@ -14179,7 +15007,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14179
15007
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
14180
15008
|
}
|
|
14181
15009
|
}
|
|
14182
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
15010
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq20(gscUrlInspections.projectId, projectId)).all();
|
|
14183
15011
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
14184
15012
|
for (const row of allInspections) {
|
|
14185
15013
|
const existing = latestByUrl.get(row.url);
|
|
@@ -14200,9 +15028,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14200
15028
|
}
|
|
14201
15029
|
}
|
|
14202
15030
|
const snapshotDate = formatDate2(/* @__PURE__ */ new Date());
|
|
14203
|
-
db.delete(gscCoverageSnapshots).where(
|
|
15031
|
+
db.delete(gscCoverageSnapshots).where(and9(eq20(gscCoverageSnapshots.projectId, projectId), eq20(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
14204
15032
|
db.insert(gscCoverageSnapshots).values({
|
|
14205
|
-
id:
|
|
15033
|
+
id: crypto20.randomUUID(),
|
|
14206
15034
|
projectId,
|
|
14207
15035
|
syncRunId: runId,
|
|
14208
15036
|
date: snapshotDate,
|
|
@@ -14211,19 +15039,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
14211
15039
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
14212
15040
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14213
15041
|
}).run();
|
|
14214
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15042
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(runs.id, runId)).run();
|
|
14215
15043
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
14216
15044
|
} catch (err) {
|
|
14217
15045
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
14218
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15046
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq20(runs.id, runId)).run();
|
|
14219
15047
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
14220
15048
|
throw err;
|
|
14221
15049
|
}
|
|
14222
15050
|
}
|
|
14223
15051
|
|
|
14224
15052
|
// src/gsc-inspect-sitemap.ts
|
|
14225
|
-
import
|
|
14226
|
-
import { eq as
|
|
15053
|
+
import crypto21 from "crypto";
|
|
15054
|
+
import { eq as eq21, and as and10 } from "drizzle-orm";
|
|
14227
15055
|
|
|
14228
15056
|
// src/sitemap-parser.ts
|
|
14229
15057
|
var LOC_REGEX = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
|
|
@@ -14292,13 +15120,13 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
14292
15120
|
var log3 = createLogger("InspectSitemap");
|
|
14293
15121
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
14294
15122
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14295
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
15123
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq21(runs.id, runId)).run();
|
|
14296
15124
|
try {
|
|
14297
15125
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
14298
15126
|
if (!googleClientId || !googleClientSecret) {
|
|
14299
15127
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
14300
15128
|
}
|
|
14301
|
-
const project = db.select().from(projects).where(
|
|
15129
|
+
const project = db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
14302
15130
|
if (!project) {
|
|
14303
15131
|
throw new Error(`Project not found: ${projectId}`);
|
|
14304
15132
|
}
|
|
@@ -14339,7 +15167,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14339
15167
|
const rich = ir.richResultsResult;
|
|
14340
15168
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
14341
15169
|
db.insert(gscUrlInspections).values({
|
|
14342
|
-
id:
|
|
15170
|
+
id: crypto21.randomUUID(),
|
|
14343
15171
|
projectId,
|
|
14344
15172
|
syncRunId: runId,
|
|
14345
15173
|
url: pageUrl,
|
|
@@ -14366,7 +15194,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14366
15194
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
14367
15195
|
}
|
|
14368
15196
|
}
|
|
14369
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
15197
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq21(gscUrlInspections.projectId, projectId)).all();
|
|
14370
15198
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
14371
15199
|
for (const row of allInspections) {
|
|
14372
15200
|
const existing = latestByUrl.get(row.url);
|
|
@@ -14387,9 +15215,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14387
15215
|
}
|
|
14388
15216
|
}
|
|
14389
15217
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
14390
|
-
db.delete(gscCoverageSnapshots).where(
|
|
15218
|
+
db.delete(gscCoverageSnapshots).where(and10(eq21(gscCoverageSnapshots.projectId, projectId), eq21(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
14391
15219
|
db.insert(gscCoverageSnapshots).values({
|
|
14392
|
-
id:
|
|
15220
|
+
id: crypto21.randomUUID(),
|
|
14393
15221
|
projectId,
|
|
14394
15222
|
syncRunId: runId,
|
|
14395
15223
|
date: snapshotDate,
|
|
@@ -14399,16 +15227,325 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
14399
15227
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14400
15228
|
}).run();
|
|
14401
15229
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
14402
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15230
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
14403
15231
|
log3.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
14404
15232
|
} catch (err) {
|
|
14405
15233
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
14406
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
15234
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq21(runs.id, runId)).run();
|
|
14407
15235
|
log3.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
14408
15236
|
throw err;
|
|
14409
15237
|
}
|
|
14410
15238
|
}
|
|
14411
15239
|
|
|
15240
|
+
// src/commoncrawl-sync.ts
|
|
15241
|
+
import crypto22 from "crypto";
|
|
15242
|
+
import path11 from "path";
|
|
15243
|
+
import { and as and11, eq as eq22, sql as sql8 } from "drizzle-orm";
|
|
15244
|
+
var log4 = createLogger("CommonCrawlSync");
|
|
15245
|
+
var INSERT_CHUNK_SIZE = 1e4;
|
|
15246
|
+
function defaultDeps() {
|
|
15247
|
+
return {
|
|
15248
|
+
downloadFile,
|
|
15249
|
+
queryBacklinks,
|
|
15250
|
+
loadDuckdb,
|
|
15251
|
+
now: () => /* @__PURE__ */ new Date(),
|
|
15252
|
+
cacheDir: CC_CACHE_DIR
|
|
15253
|
+
};
|
|
15254
|
+
}
|
|
15255
|
+
async function executeReleaseSync(db, syncId, opts) {
|
|
15256
|
+
const deps = { ...defaultDeps(), ...opts.deps };
|
|
15257
|
+
const release = opts.release;
|
|
15258
|
+
try {
|
|
15259
|
+
if (!isValidReleaseId(release)) {
|
|
15260
|
+
throw new Error(`Invalid release id: ${release}`);
|
|
15261
|
+
}
|
|
15262
|
+
const downloadStartedAt = deps.now().toISOString();
|
|
15263
|
+
db.update(ccReleaseSyncs).set({
|
|
15264
|
+
status: CcReleaseSyncStatuses.downloading,
|
|
15265
|
+
downloadStartedAt,
|
|
15266
|
+
phaseDetail: "downloading vertices + edges",
|
|
15267
|
+
updatedAt: downloadStartedAt,
|
|
15268
|
+
error: null
|
|
15269
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15270
|
+
const paths = ccReleasePaths(release);
|
|
15271
|
+
const releaseCacheDir = path11.join(deps.cacheDir, release);
|
|
15272
|
+
const vertexPath = path11.join(releaseCacheDir, paths.vertexFilename);
|
|
15273
|
+
const edgesPath = path11.join(releaseCacheDir, paths.edgesFilename);
|
|
15274
|
+
const [vertex, edges] = await Promise.all([
|
|
15275
|
+
deps.downloadFile({ url: paths.vertexUrl, destPath: vertexPath }),
|
|
15276
|
+
deps.downloadFile({ url: paths.edgesUrl, destPath: edgesPath })
|
|
15277
|
+
]);
|
|
15278
|
+
const downloadFinishedAt = deps.now().toISOString();
|
|
15279
|
+
const queryStartedAt = downloadFinishedAt;
|
|
15280
|
+
db.update(ccReleaseSyncs).set({
|
|
15281
|
+
status: CcReleaseSyncStatuses.querying,
|
|
15282
|
+
downloadFinishedAt,
|
|
15283
|
+
queryStartedAt,
|
|
15284
|
+
phaseDetail: "querying backlinks",
|
|
15285
|
+
vertexPath,
|
|
15286
|
+
edgesPath,
|
|
15287
|
+
vertexBytes: vertex.bytes,
|
|
15288
|
+
edgesBytes: edges.bytes,
|
|
15289
|
+
vertexSha256: vertex.sha256,
|
|
15290
|
+
edgesSha256: edges.sha256,
|
|
15291
|
+
updatedAt: downloadFinishedAt
|
|
15292
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15293
|
+
const allProjects = db.select().from(projects).all();
|
|
15294
|
+
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
15295
|
+
let rows = [];
|
|
15296
|
+
if (targets.length > 0) {
|
|
15297
|
+
const duckdb = deps.loadDuckdb();
|
|
15298
|
+
rows = await deps.queryBacklinks({ vertexPath, edgesPath, targets, duckdb });
|
|
15299
|
+
}
|
|
15300
|
+
const projectsByDomain = /* @__PURE__ */ new Map();
|
|
15301
|
+
for (const p of allProjects) {
|
|
15302
|
+
const ids = projectsByDomain.get(p.canonicalDomain) ?? [];
|
|
15303
|
+
ids.push(p.id);
|
|
15304
|
+
projectsByDomain.set(p.canonicalDomain, ids);
|
|
15305
|
+
}
|
|
15306
|
+
const queriedAt = deps.now().toISOString();
|
|
15307
|
+
db.transaction((tx) => {
|
|
15308
|
+
tx.delete(backlinkDomains).where(eq22(backlinkDomains.releaseSyncId, syncId)).run();
|
|
15309
|
+
tx.delete(backlinkSummaries).where(eq22(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
15310
|
+
const expanded = [];
|
|
15311
|
+
for (const r of rows) {
|
|
15312
|
+
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
15313
|
+
if (!projectIds) continue;
|
|
15314
|
+
for (const projectId of projectIds) {
|
|
15315
|
+
expanded.push({
|
|
15316
|
+
id: crypto22.randomUUID(),
|
|
15317
|
+
projectId,
|
|
15318
|
+
releaseSyncId: syncId,
|
|
15319
|
+
release,
|
|
15320
|
+
targetDomain: r.targetDomain,
|
|
15321
|
+
linkingDomain: r.linkingDomain,
|
|
15322
|
+
numHosts: r.numHosts,
|
|
15323
|
+
createdAt: queriedAt
|
|
15324
|
+
});
|
|
15325
|
+
}
|
|
15326
|
+
}
|
|
15327
|
+
for (let i = 0; i < expanded.length; i += INSERT_CHUNK_SIZE) {
|
|
15328
|
+
const chunk = expanded.slice(i, i + INSERT_CHUNK_SIZE);
|
|
15329
|
+
if (chunk.length > 0) tx.insert(backlinkDomains).values(chunk).run();
|
|
15330
|
+
}
|
|
15331
|
+
const rowsByProject = groupByProject(rows, projectsByDomain);
|
|
15332
|
+
for (const p of allProjects) {
|
|
15333
|
+
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
15334
|
+
const summary = computeSummary(projectRows);
|
|
15335
|
+
tx.insert(backlinkSummaries).values({
|
|
15336
|
+
id: crypto22.randomUUID(),
|
|
15337
|
+
projectId: p.id,
|
|
15338
|
+
releaseSyncId: syncId,
|
|
15339
|
+
release,
|
|
15340
|
+
targetDomain: p.canonicalDomain,
|
|
15341
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15342
|
+
totalHosts: summary.totalHosts,
|
|
15343
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15344
|
+
queriedAt,
|
|
15345
|
+
createdAt: queriedAt
|
|
15346
|
+
}).onConflictDoUpdate({
|
|
15347
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.release],
|
|
15348
|
+
set: {
|
|
15349
|
+
releaseSyncId: syncId,
|
|
15350
|
+
targetDomain: p.canonicalDomain,
|
|
15351
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15352
|
+
totalHosts: summary.totalHosts,
|
|
15353
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15354
|
+
queriedAt
|
|
15355
|
+
}
|
|
15356
|
+
}).run();
|
|
15357
|
+
}
|
|
15358
|
+
});
|
|
15359
|
+
const finishedAt = deps.now().toISOString();
|
|
15360
|
+
db.update(ccReleaseSyncs).set({
|
|
15361
|
+
status: CcReleaseSyncStatuses.ready,
|
|
15362
|
+
queryFinishedAt: finishedAt,
|
|
15363
|
+
phaseDetail: null,
|
|
15364
|
+
projectsProcessed: allProjects.length,
|
|
15365
|
+
domainsDiscovered: rows.length,
|
|
15366
|
+
updatedAt: finishedAt,
|
|
15367
|
+
error: null
|
|
15368
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15369
|
+
log4.info("sync.completed", {
|
|
15370
|
+
syncId,
|
|
15371
|
+
release,
|
|
15372
|
+
projectsProcessed: allProjects.length,
|
|
15373
|
+
domainsDiscovered: rows.length
|
|
15374
|
+
});
|
|
15375
|
+
if (deps.enqueueAutoExtract) {
|
|
15376
|
+
const autoExtractProjects = allProjects.filter((p) => p.autoExtractBacklinks === 1);
|
|
15377
|
+
for (const p of autoExtractProjects) {
|
|
15378
|
+
try {
|
|
15379
|
+
deps.enqueueAutoExtract({ projectId: p.id, release });
|
|
15380
|
+
} catch (err) {
|
|
15381
|
+
log4.error("auto-extract.enqueue-failed", {
|
|
15382
|
+
syncId,
|
|
15383
|
+
release,
|
|
15384
|
+
projectId: p.id,
|
|
15385
|
+
error: err instanceof Error ? err.message : String(err)
|
|
15386
|
+
});
|
|
15387
|
+
}
|
|
15388
|
+
}
|
|
15389
|
+
}
|
|
15390
|
+
} catch (err) {
|
|
15391
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15392
|
+
const finishedAt = deps.now().toISOString();
|
|
15393
|
+
db.update(ccReleaseSyncs).set({
|
|
15394
|
+
status: CcReleaseSyncStatuses.failed,
|
|
15395
|
+
error: errorMsg,
|
|
15396
|
+
phaseDetail: null,
|
|
15397
|
+
updatedAt: finishedAt
|
|
15398
|
+
}).where(eq22(ccReleaseSyncs.id, syncId)).run();
|
|
15399
|
+
log4.error("sync.failed", { syncId, release, error: errorMsg });
|
|
15400
|
+
throw err;
|
|
15401
|
+
}
|
|
15402
|
+
}
|
|
15403
|
+
function groupByProject(rows, projectsByDomain) {
|
|
15404
|
+
const out = /* @__PURE__ */ new Map();
|
|
15405
|
+
for (const row of rows) {
|
|
15406
|
+
const projectIds = projectsByDomain.get(row.targetDomain);
|
|
15407
|
+
if (!projectIds) continue;
|
|
15408
|
+
for (const projectId of projectIds) {
|
|
15409
|
+
const bucket = out.get(projectId) ?? [];
|
|
15410
|
+
bucket.push(row);
|
|
15411
|
+
out.set(projectId, bucket);
|
|
15412
|
+
}
|
|
15413
|
+
}
|
|
15414
|
+
return out;
|
|
15415
|
+
}
|
|
15416
|
+
function computeSummary(rows) {
|
|
15417
|
+
if (rows.length === 0) {
|
|
15418
|
+
return { totalLinkingDomains: 0, totalHosts: 0, top10HostsShare: "0" };
|
|
15419
|
+
}
|
|
15420
|
+
const sorted = [...rows].sort((a, b) => b.numHosts - a.numHosts);
|
|
15421
|
+
const totalHosts = sorted.reduce((acc, r) => acc + r.numHosts, 0);
|
|
15422
|
+
const top10Hosts = sorted.slice(0, 10).reduce((acc, r) => acc + r.numHosts, 0);
|
|
15423
|
+
const share = totalHosts > 0 ? top10Hosts / totalHosts : 0;
|
|
15424
|
+
return {
|
|
15425
|
+
totalLinkingDomains: rows.length,
|
|
15426
|
+
totalHosts,
|
|
15427
|
+
top10HostsShare: share.toFixed(6)
|
|
15428
|
+
};
|
|
15429
|
+
}
|
|
15430
|
+
|
|
15431
|
+
// src/backlink-extract.ts
|
|
15432
|
+
import crypto23 from "crypto";
|
|
15433
|
+
import fs9 from "fs";
|
|
15434
|
+
import { and as and12, desc as desc9, eq as eq23 } from "drizzle-orm";
|
|
15435
|
+
var log5 = createLogger("BacklinkExtract");
|
|
15436
|
+
function defaultDeps2() {
|
|
15437
|
+
return {
|
|
15438
|
+
queryBacklinks,
|
|
15439
|
+
loadDuckdb,
|
|
15440
|
+
now: () => /* @__PURE__ */ new Date()
|
|
15441
|
+
};
|
|
15442
|
+
}
|
|
15443
|
+
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
15444
|
+
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
15445
|
+
const startedAt = deps.now().toISOString();
|
|
15446
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq23(runs.id, runId)).run();
|
|
15447
|
+
try {
|
|
15448
|
+
const project = db.select().from(projects).where(eq23(projects.id, projectId)).get();
|
|
15449
|
+
if (!project) {
|
|
15450
|
+
throw new Error(`Project not found: ${projectId}`);
|
|
15451
|
+
}
|
|
15452
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq23(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq23(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc9(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
15453
|
+
if (!sync) {
|
|
15454
|
+
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
15455
|
+
}
|
|
15456
|
+
if (sync.status !== CcReleaseSyncStatuses.ready) {
|
|
15457
|
+
throw new Error(`Release ${sync.release} is not ready (status=${sync.status})`);
|
|
15458
|
+
}
|
|
15459
|
+
if (!sync.vertexPath || !sync.edgesPath) {
|
|
15460
|
+
throw new Error(`Release ${sync.release} is missing cached file paths`);
|
|
15461
|
+
}
|
|
15462
|
+
if (!fs9.existsSync(sync.vertexPath) || !fs9.existsSync(sync.edgesPath)) {
|
|
15463
|
+
throw new Error(
|
|
15464
|
+
`Cache for release ${sync.release} is missing from disk (expected at ${sync.vertexPath}). The sync record exists in the database, but the ~16 GB dump was deleted or never present on this machine. Re-sync this release from the Backlinks admin page to restore the cache.`
|
|
15465
|
+
);
|
|
15466
|
+
}
|
|
15467
|
+
const duckdb = deps.loadDuckdb();
|
|
15468
|
+
const rows = await deps.queryBacklinks({
|
|
15469
|
+
vertexPath: sync.vertexPath,
|
|
15470
|
+
edgesPath: sync.edgesPath,
|
|
15471
|
+
targets: [project.canonicalDomain],
|
|
15472
|
+
duckdb
|
|
15473
|
+
});
|
|
15474
|
+
const queriedAt = deps.now().toISOString();
|
|
15475
|
+
const syncId = sync.id;
|
|
15476
|
+
const release = sync.release;
|
|
15477
|
+
const targetDomain = project.canonicalDomain;
|
|
15478
|
+
db.transaction((tx) => {
|
|
15479
|
+
tx.delete(backlinkDomains).where(
|
|
15480
|
+
and12(eq23(backlinkDomains.projectId, projectId), eq23(backlinkDomains.release, release))
|
|
15481
|
+
).run();
|
|
15482
|
+
if (rows.length > 0) {
|
|
15483
|
+
const values = rows.map((r) => ({
|
|
15484
|
+
id: crypto23.randomUUID(),
|
|
15485
|
+
projectId,
|
|
15486
|
+
releaseSyncId: syncId,
|
|
15487
|
+
release,
|
|
15488
|
+
targetDomain,
|
|
15489
|
+
linkingDomain: r.linkingDomain,
|
|
15490
|
+
numHosts: r.numHosts,
|
|
15491
|
+
createdAt: queriedAt
|
|
15492
|
+
}));
|
|
15493
|
+
tx.insert(backlinkDomains).values(values).run();
|
|
15494
|
+
}
|
|
15495
|
+
const summary = computeSummary2(rows);
|
|
15496
|
+
tx.insert(backlinkSummaries).values({
|
|
15497
|
+
id: crypto23.randomUUID(),
|
|
15498
|
+
projectId,
|
|
15499
|
+
releaseSyncId: syncId,
|
|
15500
|
+
release,
|
|
15501
|
+
targetDomain,
|
|
15502
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15503
|
+
totalHosts: summary.totalHosts,
|
|
15504
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15505
|
+
queriedAt,
|
|
15506
|
+
createdAt: queriedAt
|
|
15507
|
+
}).onConflictDoUpdate({
|
|
15508
|
+
target: [backlinkSummaries.projectId, backlinkSummaries.release],
|
|
15509
|
+
set: {
|
|
15510
|
+
releaseSyncId: syncId,
|
|
15511
|
+
targetDomain,
|
|
15512
|
+
totalLinkingDomains: summary.totalLinkingDomains,
|
|
15513
|
+
totalHosts: summary.totalHosts,
|
|
15514
|
+
top10HostsShare: summary.top10HostsShare,
|
|
15515
|
+
queriedAt
|
|
15516
|
+
}
|
|
15517
|
+
}).run();
|
|
15518
|
+
});
|
|
15519
|
+
const finishedAt = deps.now().toISOString();
|
|
15520
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
15521
|
+
log5.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
15522
|
+
} catch (err) {
|
|
15523
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
15524
|
+
const finishedAt = deps.now().toISOString();
|
|
15525
|
+
db.update(runs).set({
|
|
15526
|
+
status: RunStatuses.failed,
|
|
15527
|
+
error: errorMsg,
|
|
15528
|
+
finishedAt
|
|
15529
|
+
}).where(eq23(runs.id, runId)).run();
|
|
15530
|
+
log5.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
15531
|
+
throw err;
|
|
15532
|
+
}
|
|
15533
|
+
}
|
|
15534
|
+
function computeSummary2(rows) {
|
|
15535
|
+
if (rows.length === 0) {
|
|
15536
|
+
return { totalLinkingDomains: 0, totalHosts: 0, top10HostsShare: "0" };
|
|
15537
|
+
}
|
|
15538
|
+
const sorted = [...rows].sort((a, b) => b.numHosts - a.numHosts);
|
|
15539
|
+
const totalHosts = sorted.reduce((acc, r) => acc + r.numHosts, 0);
|
|
15540
|
+
const top10Hosts = sorted.slice(0, 10).reduce((acc, r) => acc + r.numHosts, 0);
|
|
15541
|
+
const share = totalHosts > 0 ? top10Hosts / totalHosts : 0;
|
|
15542
|
+
return {
|
|
15543
|
+
totalLinkingDomains: rows.length,
|
|
15544
|
+
totalHosts,
|
|
15545
|
+
top10HostsShare: share.toFixed(6)
|
|
15546
|
+
};
|
|
15547
|
+
}
|
|
15548
|
+
|
|
14412
15549
|
// src/provider-registry.ts
|
|
14413
15550
|
var ProviderRegistry = class {
|
|
14414
15551
|
providers = /* @__PURE__ */ new Map();
|
|
@@ -14462,8 +15599,8 @@ var ProviderRegistry = class {
|
|
|
14462
15599
|
|
|
14463
15600
|
// src/scheduler.ts
|
|
14464
15601
|
import cron from "node-cron";
|
|
14465
|
-
import { eq as
|
|
14466
|
-
var
|
|
15602
|
+
import { eq as eq24 } from "drizzle-orm";
|
|
15603
|
+
var log6 = createLogger("Scheduler");
|
|
14467
15604
|
var Scheduler = class {
|
|
14468
15605
|
db;
|
|
14469
15606
|
callbacks;
|
|
@@ -14474,16 +15611,16 @@ var Scheduler = class {
|
|
|
14474
15611
|
}
|
|
14475
15612
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
14476
15613
|
start() {
|
|
14477
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
15614
|
+
const allSchedules = this.db.select().from(schedules).where(eq24(schedules.enabled, 1)).all();
|
|
14478
15615
|
for (const schedule of allSchedules) {
|
|
14479
15616
|
const missedRunAt = schedule.nextRunAt;
|
|
14480
15617
|
this.registerCronTask(schedule);
|
|
14481
15618
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
14482
|
-
|
|
15619
|
+
log6.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
14483
15620
|
this.triggerRun(schedule.id, schedule.projectId);
|
|
14484
15621
|
}
|
|
14485
15622
|
}
|
|
14486
|
-
|
|
15623
|
+
log6.info("started", { scheduleCount: allSchedules.length });
|
|
14487
15624
|
}
|
|
14488
15625
|
/** Stop all cron tasks for graceful shutdown. */
|
|
14489
15626
|
stop() {
|
|
@@ -14499,7 +15636,7 @@ var Scheduler = class {
|
|
|
14499
15636
|
this.stopTask(projectId, existing, "Stopped");
|
|
14500
15637
|
this.tasks.delete(projectId);
|
|
14501
15638
|
}
|
|
14502
|
-
const schedule = this.db.select().from(schedules).where(
|
|
15639
|
+
const schedule = this.db.select().from(schedules).where(eq24(schedules.projectId, projectId)).get();
|
|
14503
15640
|
if (schedule && schedule.enabled === 1) {
|
|
14504
15641
|
this.registerCronTask(schedule);
|
|
14505
15642
|
}
|
|
@@ -14515,12 +15652,12 @@ var Scheduler = class {
|
|
|
14515
15652
|
stopTask(projectId, task, verb) {
|
|
14516
15653
|
task.stop();
|
|
14517
15654
|
task.destroy();
|
|
14518
|
-
|
|
15655
|
+
log6.info(`task.${verb.toLowerCase()}`, { projectId });
|
|
14519
15656
|
}
|
|
14520
15657
|
registerCronTask(schedule) {
|
|
14521
15658
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
14522
15659
|
if (!cron.validate(cronExpr)) {
|
|
14523
|
-
|
|
15660
|
+
log6.error("cron.invalid", { projectId, cronExpr });
|
|
14524
15661
|
return;
|
|
14525
15662
|
}
|
|
14526
15663
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -14532,24 +15669,24 @@ var Scheduler = class {
|
|
|
14532
15669
|
this.db.update(schedules).set({
|
|
14533
15670
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
14534
15671
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14535
|
-
}).where(
|
|
15672
|
+
}).where(eq24(schedules.id, scheduleId)).run();
|
|
14536
15673
|
const label = schedule.preset ?? cronExpr;
|
|
14537
|
-
|
|
15674
|
+
log6.info("cron.registered", { projectId, schedule: label, timezone });
|
|
14538
15675
|
}
|
|
14539
15676
|
triggerRun(scheduleId, projectId) {
|
|
14540
15677
|
try {
|
|
14541
15678
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14542
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
15679
|
+
const currentSchedule = this.db.select().from(schedules).where(eq24(schedules.id, scheduleId)).get();
|
|
14543
15680
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
14544
|
-
|
|
15681
|
+
log6.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
14545
15682
|
this.remove(projectId);
|
|
14546
15683
|
return;
|
|
14547
15684
|
}
|
|
14548
15685
|
const task = this.tasks.get(projectId);
|
|
14549
15686
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
14550
|
-
const project = this.db.select().from(projects).where(
|
|
15687
|
+
const project = this.db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
14551
15688
|
if (!project) {
|
|
14552
|
-
|
|
15689
|
+
log6.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
14553
15690
|
this.remove(projectId);
|
|
14554
15691
|
return;
|
|
14555
15692
|
}
|
|
@@ -14558,7 +15695,7 @@ var Scheduler = class {
|
|
|
14558
15695
|
if (project.defaultLocation) {
|
|
14559
15696
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
14560
15697
|
if (!loc) {
|
|
14561
|
-
|
|
15698
|
+
log6.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
14562
15699
|
return;
|
|
14563
15700
|
}
|
|
14564
15701
|
resolvedLocation = loc;
|
|
@@ -14572,11 +15709,11 @@ var Scheduler = class {
|
|
|
14572
15709
|
location: locationLabel
|
|
14573
15710
|
});
|
|
14574
15711
|
if (queueResult.conflict) {
|
|
14575
|
-
|
|
15712
|
+
log6.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
14576
15713
|
this.db.update(schedules).set({
|
|
14577
15714
|
nextRunAt,
|
|
14578
15715
|
updatedAt: now
|
|
14579
|
-
}).where(
|
|
15716
|
+
}).where(eq24(schedules.id, currentSchedule.id)).run();
|
|
14580
15717
|
return;
|
|
14581
15718
|
}
|
|
14582
15719
|
const runId = queueResult.runId;
|
|
@@ -14584,21 +15721,21 @@ var Scheduler = class {
|
|
|
14584
15721
|
lastRunAt: now,
|
|
14585
15722
|
nextRunAt,
|
|
14586
15723
|
updatedAt: now
|
|
14587
|
-
}).where(
|
|
15724
|
+
}).where(eq24(schedules.id, currentSchedule.id)).run();
|
|
14588
15725
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
14589
15726
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
14590
|
-
|
|
15727
|
+
log6.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
14591
15728
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
14592
15729
|
} catch (err) {
|
|
14593
|
-
|
|
15730
|
+
log6.error("trigger.error", { scheduleId, projectId, error: err instanceof Error ? err.message : String(err) });
|
|
14594
15731
|
}
|
|
14595
15732
|
}
|
|
14596
15733
|
};
|
|
14597
15734
|
|
|
14598
15735
|
// src/notifier.ts
|
|
14599
|
-
import { eq as
|
|
14600
|
-
import
|
|
14601
|
-
var
|
|
15736
|
+
import { eq as eq25, desc as desc10, and as and13, or as or2 } from "drizzle-orm";
|
|
15737
|
+
import crypto24 from "crypto";
|
|
15738
|
+
var log7 = createLogger("Notifier");
|
|
14602
15739
|
var Notifier = class {
|
|
14603
15740
|
db;
|
|
14604
15741
|
serverUrl;
|
|
@@ -14608,26 +15745,26 @@ var Notifier = class {
|
|
|
14608
15745
|
}
|
|
14609
15746
|
/** Called after a run completes (success, partial, or failed). */
|
|
14610
15747
|
async onRunCompleted(runId, projectId) {
|
|
14611
|
-
|
|
14612
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15748
|
+
log7.info("run.completed", { runId, projectId });
|
|
15749
|
+
const notifs = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
14613
15750
|
if (notifs.length === 0) {
|
|
14614
|
-
|
|
15751
|
+
log7.info("notifications.none-enabled", { projectId });
|
|
14615
15752
|
return;
|
|
14616
15753
|
}
|
|
14617
|
-
|
|
14618
|
-
const run = this.db.select().from(runs).where(
|
|
15754
|
+
log7.info("notifications.found", { projectId, count: notifs.length });
|
|
15755
|
+
const run = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
14619
15756
|
if (!run) {
|
|
14620
|
-
|
|
15757
|
+
log7.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
14621
15758
|
return;
|
|
14622
15759
|
}
|
|
14623
|
-
const project = this.db.select().from(projects).where(
|
|
15760
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
14624
15761
|
if (!project) {
|
|
14625
|
-
|
|
15762
|
+
log7.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
14626
15763
|
return;
|
|
14627
15764
|
}
|
|
14628
15765
|
const transitions = this.computeTransitions(runId, projectId);
|
|
14629
15766
|
const events = [];
|
|
14630
|
-
|
|
15767
|
+
log7.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
14631
15768
|
if (run.status === "completed" || run.status === "partial") {
|
|
14632
15769
|
events.push("run.completed");
|
|
14633
15770
|
}
|
|
@@ -14643,7 +15780,7 @@ var Notifier = class {
|
|
|
14643
15780
|
if (!config.url) continue;
|
|
14644
15781
|
const subscribedEvents = config.events;
|
|
14645
15782
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
14646
|
-
|
|
15783
|
+
log7.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
14647
15784
|
if (matchingEvents.length === 0) continue;
|
|
14648
15785
|
for (const event of matchingEvents) {
|
|
14649
15786
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -14667,11 +15804,11 @@ var Notifier = class {
|
|
|
14667
15804
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
14668
15805
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
14669
15806
|
if (insightEvents.length === 0) return;
|
|
14670
|
-
const notifs = this.db.select().from(notifications).where(
|
|
15807
|
+
const notifs = this.db.select().from(notifications).where(eq25(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
14671
15808
|
if (notifs.length === 0) return;
|
|
14672
|
-
const run = this.db.select().from(runs).where(
|
|
15809
|
+
const run = this.db.select().from(runs).where(eq25(runs.id, runId)).get();
|
|
14673
15810
|
if (!run) return;
|
|
14674
|
-
const project = this.db.select().from(projects).where(
|
|
15811
|
+
const project = this.db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
14675
15812
|
if (!project) return;
|
|
14676
15813
|
for (const notif of notifs) {
|
|
14677
15814
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -14702,11 +15839,11 @@ var Notifier = class {
|
|
|
14702
15839
|
}
|
|
14703
15840
|
computeTransitions(runId, projectId) {
|
|
14704
15841
|
const recentRuns = this.db.select().from(runs).where(
|
|
14705
|
-
|
|
14706
|
-
|
|
14707
|
-
or2(
|
|
15842
|
+
and13(
|
|
15843
|
+
eq25(runs.projectId, projectId),
|
|
15844
|
+
or2(eq25(runs.status, "completed"), eq25(runs.status, "partial"))
|
|
14708
15845
|
)
|
|
14709
|
-
).orderBy(
|
|
15846
|
+
).orderBy(desc10(runs.createdAt)).limit(2).all();
|
|
14710
15847
|
if (recentRuns.length < 2) return [];
|
|
14711
15848
|
const currentRunId = recentRuns[0].id;
|
|
14712
15849
|
const previousRunId = recentRuns[1].id;
|
|
@@ -14716,12 +15853,12 @@ var Notifier = class {
|
|
|
14716
15853
|
keyword: keywords.keyword,
|
|
14717
15854
|
provider: querySnapshots.provider,
|
|
14718
15855
|
citationState: querySnapshots.citationState
|
|
14719
|
-
}).from(querySnapshots).leftJoin(keywords,
|
|
15856
|
+
}).from(querySnapshots).leftJoin(keywords, eq25(querySnapshots.keywordId, keywords.id)).where(eq25(querySnapshots.runId, currentRunId)).all();
|
|
14720
15857
|
const previousSnapshots = this.db.select({
|
|
14721
15858
|
keywordId: querySnapshots.keywordId,
|
|
14722
15859
|
provider: querySnapshots.provider,
|
|
14723
15860
|
citationState: querySnapshots.citationState
|
|
14724
|
-
}).from(querySnapshots).where(
|
|
15861
|
+
}).from(querySnapshots).where(eq25(querySnapshots.runId, previousRunId)).all();
|
|
14725
15862
|
const prevMap = /* @__PURE__ */ new Map();
|
|
14726
15863
|
for (const s of previousSnapshots) {
|
|
14727
15864
|
prevMap.set(`${s.keywordId}:${s.provider}`, s.citationState);
|
|
@@ -14745,23 +15882,23 @@ var Notifier = class {
|
|
|
14745
15882
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
14746
15883
|
const targetCheck = await resolveWebhookTarget(url);
|
|
14747
15884
|
if (!targetCheck.ok) {
|
|
14748
|
-
|
|
15885
|
+
log7.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
14749
15886
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
14750
15887
|
return;
|
|
14751
15888
|
}
|
|
14752
|
-
|
|
15889
|
+
log7.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
14753
15890
|
const maxRetries = 3;
|
|
14754
15891
|
const delays = [1e3, 4e3, 16e3];
|
|
14755
15892
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
14756
15893
|
try {
|
|
14757
15894
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
14758
15895
|
if (response.status >= 200 && response.status < 300) {
|
|
14759
|
-
|
|
15896
|
+
log7.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
14760
15897
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
14761
15898
|
return;
|
|
14762
15899
|
}
|
|
14763
15900
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
14764
|
-
|
|
15901
|
+
log7.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
14765
15902
|
if (attempt === maxRetries - 1) {
|
|
14766
15903
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
14767
15904
|
}
|
|
@@ -14769,7 +15906,7 @@ var Notifier = class {
|
|
|
14769
15906
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
14770
15907
|
if (attempt === maxRetries - 1) {
|
|
14771
15908
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
14772
|
-
|
|
15909
|
+
log7.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
14773
15910
|
}
|
|
14774
15911
|
}
|
|
14775
15912
|
if (attempt < maxRetries - 1) {
|
|
@@ -14779,7 +15916,7 @@ var Notifier = class {
|
|
|
14779
15916
|
}
|
|
14780
15917
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
14781
15918
|
this.db.insert(auditLog).values({
|
|
14782
|
-
id:
|
|
15919
|
+
id: crypto24.randomUUID(),
|
|
14783
15920
|
projectId,
|
|
14784
15921
|
actor: "scheduler",
|
|
14785
15922
|
action: `notification.${status}`,
|
|
@@ -14792,7 +15929,7 @@ var Notifier = class {
|
|
|
14792
15929
|
};
|
|
14793
15930
|
|
|
14794
15931
|
// src/run-coordinator.ts
|
|
14795
|
-
var
|
|
15932
|
+
var log8 = createLogger("RunCoordinator");
|
|
14796
15933
|
var RunCoordinator = class {
|
|
14797
15934
|
constructor(notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
14798
15935
|
this.notifier = notifier;
|
|
@@ -14814,35 +15951,35 @@ var RunCoordinator = class {
|
|
|
14814
15951
|
try {
|
|
14815
15952
|
await this.onInsightsGenerated(runId, projectId, result);
|
|
14816
15953
|
} catch (err) {
|
|
14817
|
-
|
|
15954
|
+
log8.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14818
15955
|
}
|
|
14819
15956
|
}
|
|
14820
15957
|
}
|
|
14821
15958
|
} catch (err) {
|
|
14822
|
-
|
|
15959
|
+
log8.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14823
15960
|
}
|
|
14824
15961
|
try {
|
|
14825
15962
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
14826
15963
|
} catch (err) {
|
|
14827
|
-
|
|
15964
|
+
log8.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14828
15965
|
}
|
|
14829
15966
|
if (this.onAeroEvent) {
|
|
14830
15967
|
try {
|
|
14831
15968
|
await this.onAeroEvent({ runId, projectId, insightCount, criticalOrHigh });
|
|
14832
15969
|
} catch (err) {
|
|
14833
|
-
|
|
15970
|
+
log8.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14834
15971
|
}
|
|
14835
15972
|
}
|
|
14836
15973
|
}
|
|
14837
15974
|
};
|
|
14838
15975
|
|
|
14839
15976
|
// src/agent/session-registry.ts
|
|
14840
|
-
import
|
|
14841
|
-
import { eq as
|
|
15977
|
+
import crypto26 from "crypto";
|
|
15978
|
+
import { eq as eq27 } from "drizzle-orm";
|
|
14842
15979
|
|
|
14843
15980
|
// src/agent/session.ts
|
|
14844
|
-
import
|
|
14845
|
-
import
|
|
15981
|
+
import fs12 from "fs";
|
|
15982
|
+
import path14 from "path";
|
|
14846
15983
|
import { Agent } from "@mariozechner/pi-agent-core";
|
|
14847
15984
|
import { registerBuiltInApiProviders } from "@mariozechner/pi-ai";
|
|
14848
15985
|
|
|
@@ -14943,26 +16080,26 @@ function buildAgentProvidersResponse(config) {
|
|
|
14943
16080
|
}
|
|
14944
16081
|
|
|
14945
16082
|
// src/agent/skill-paths.ts
|
|
14946
|
-
import
|
|
14947
|
-
import
|
|
16083
|
+
import fs10 from "fs";
|
|
16084
|
+
import path12 from "path";
|
|
14948
16085
|
import { fileURLToPath } from "url";
|
|
14949
16086
|
function resolveAeroSkillDir(pkgDir) {
|
|
14950
|
-
const here = pkgDir ??
|
|
16087
|
+
const here = pkgDir ?? path12.dirname(fileURLToPath(import.meta.url));
|
|
14951
16088
|
const candidates = [
|
|
14952
|
-
|
|
14953
|
-
|
|
14954
|
-
|
|
16089
|
+
path12.join(here, "../assets/agent-workspace/skills/aero"),
|
|
16090
|
+
path12.join(here, "../../assets/agent-workspace/skills/aero"),
|
|
16091
|
+
path12.join(here, "../../../../skills/aero")
|
|
14955
16092
|
];
|
|
14956
16093
|
for (const candidate of candidates) {
|
|
14957
|
-
if (
|
|
16094
|
+
if (fs10.existsSync(path12.join(candidate, "SKILL.md"))) return candidate;
|
|
14958
16095
|
}
|
|
14959
16096
|
throw new Error(`Aero skill not found. Searched:
|
|
14960
16097
|
${candidates.join("\n ")}`);
|
|
14961
16098
|
}
|
|
14962
16099
|
|
|
14963
16100
|
// src/agent/skill-tools.ts
|
|
14964
|
-
import
|
|
14965
|
-
import
|
|
16101
|
+
import fs11 from "fs";
|
|
16102
|
+
import path13 from "path";
|
|
14966
16103
|
import { Type } from "@sinclair/typebox";
|
|
14967
16104
|
var MAX_DOC_CHARS = 2e4;
|
|
14968
16105
|
function textResult(details) {
|
|
@@ -14983,13 +16120,13 @@ function parseDescription(body) {
|
|
|
14983
16120
|
return "(no description)";
|
|
14984
16121
|
}
|
|
14985
16122
|
function scanSkillDocs(skillDir) {
|
|
14986
|
-
const refsDir =
|
|
14987
|
-
if (!
|
|
16123
|
+
const refsDir = path13.join(skillDir ?? resolveAeroSkillDir(), "references");
|
|
16124
|
+
if (!fs11.existsSync(refsDir)) return [];
|
|
14988
16125
|
const entries = [];
|
|
14989
|
-
for (const file of
|
|
16126
|
+
for (const file of fs11.readdirSync(refsDir)) {
|
|
14990
16127
|
if (!file.endsWith(".md")) continue;
|
|
14991
|
-
const filePath =
|
|
14992
|
-
const body =
|
|
16128
|
+
const filePath = path13.join(refsDir, file);
|
|
16129
|
+
const body = fs11.readFileSync(filePath, "utf-8");
|
|
14993
16130
|
entries.push({
|
|
14994
16131
|
slug: file.replace(/\.md$/, ""),
|
|
14995
16132
|
description: parseDescription(body),
|
|
@@ -15032,8 +16169,8 @@ function buildReadSkillDocTool() {
|
|
|
15032
16169
|
availableSlugs: docs.map((d) => d.slug)
|
|
15033
16170
|
});
|
|
15034
16171
|
}
|
|
15035
|
-
const filePath =
|
|
15036
|
-
const content =
|
|
16172
|
+
const filePath = path13.join(skillDir, "references", `${match.slug}.md`);
|
|
16173
|
+
const content = fs11.readFileSync(filePath, "utf-8");
|
|
15037
16174
|
if (content.length > MAX_DOC_CHARS) {
|
|
15038
16175
|
return textResult({
|
|
15039
16176
|
slug: match.slug,
|
|
@@ -15057,8 +16194,8 @@ function buildSkillDocTools() {
|
|
|
15057
16194
|
import { Type as Type2 } from "@sinclair/typebox";
|
|
15058
16195
|
|
|
15059
16196
|
// src/agent/memory-store.ts
|
|
15060
|
-
import
|
|
15061
|
-
import { and as
|
|
16197
|
+
import crypto25 from "crypto";
|
|
16198
|
+
import { and as and14, desc as desc11, eq as eq26, like, sql as sql9 } from "drizzle-orm";
|
|
15062
16199
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
15063
16200
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
15064
16201
|
function rowToDto(row) {
|
|
@@ -15072,7 +16209,7 @@ function rowToDto(row) {
|
|
|
15072
16209
|
};
|
|
15073
16210
|
}
|
|
15074
16211
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
15075
|
-
const query = db.select().from(agentMemory).where(
|
|
16212
|
+
const query = db.select().from(agentMemory).where(eq26(agentMemory.projectId, projectId)).orderBy(desc11(agentMemory.updatedAt));
|
|
15076
16213
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
15077
16214
|
return rows.map(rowToDto);
|
|
15078
16215
|
}
|
|
@@ -15086,7 +16223,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
15086
16223
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
15087
16224
|
}
|
|
15088
16225
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15089
|
-
const id =
|
|
16226
|
+
const id = crypto25.randomUUID();
|
|
15090
16227
|
db.insert(agentMemory).values({
|
|
15091
16228
|
id,
|
|
15092
16229
|
projectId: args.projectId,
|
|
@@ -15103,12 +16240,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
15103
16240
|
updatedAt: now
|
|
15104
16241
|
}
|
|
15105
16242
|
}).run();
|
|
15106
|
-
const row = db.select().from(agentMemory).where(
|
|
16243
|
+
const row = db.select().from(agentMemory).where(and14(eq26(agentMemory.projectId, args.projectId), eq26(agentMemory.key, args.key))).get();
|
|
15107
16244
|
if (!row) throw new Error("memory upsert produced no row");
|
|
15108
16245
|
return rowToDto(row);
|
|
15109
16246
|
}
|
|
15110
16247
|
function deleteMemoryEntry(db, projectId, key) {
|
|
15111
|
-
const result = db.delete(agentMemory).where(
|
|
16248
|
+
const result = db.delete(agentMemory).where(and14(eq26(agentMemory.projectId, projectId), eq26(agentMemory.key, key))).run();
|
|
15112
16249
|
const changes = result.changes ?? 0;
|
|
15113
16250
|
return changes > 0;
|
|
15114
16251
|
}
|
|
@@ -15123,7 +16260,7 @@ function writeCompactionNote(db, args) {
|
|
|
15123
16260
|
}
|
|
15124
16261
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
15125
16262
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
15126
|
-
const id =
|
|
16263
|
+
const id = crypto25.randomUUID();
|
|
15127
16264
|
let inserted;
|
|
15128
16265
|
db.transaction((tx) => {
|
|
15129
16266
|
tx.insert(agentMemory).values({
|
|
@@ -15137,16 +16274,16 @@ function writeCompactionNote(db, args) {
|
|
|
15137
16274
|
}).run();
|
|
15138
16275
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
15139
16276
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
15140
|
-
|
|
15141
|
-
|
|
16277
|
+
and14(
|
|
16278
|
+
eq26(agentMemory.projectId, args.projectId),
|
|
15142
16279
|
like(agentMemory.key, `${sessionPrefix}%`)
|
|
15143
16280
|
)
|
|
15144
|
-
).orderBy(
|
|
16281
|
+
).orderBy(desc11(agentMemory.updatedAt)).all();
|
|
15145
16282
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
15146
16283
|
if (stale.length > 0) {
|
|
15147
|
-
tx.delete(agentMemory).where(
|
|
16284
|
+
tx.delete(agentMemory).where(sql9`${agentMemory.id} IN (${sql9.join(stale.map((s) => sql9`${s}`), sql9`, `)})`).run();
|
|
15148
16285
|
}
|
|
15149
|
-
const row = tx.select().from(agentMemory).where(
|
|
16286
|
+
const row = tx.select().from(agentMemory).where(and14(eq26(agentMemory.projectId, args.projectId), eq26(agentMemory.key, key))).get();
|
|
15150
16287
|
if (row) inserted = rowToDto(row);
|
|
15151
16288
|
});
|
|
15152
16289
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -15293,6 +16430,35 @@ function buildGetRunTool(ctx) {
|
|
|
15293
16430
|
}
|
|
15294
16431
|
};
|
|
15295
16432
|
}
|
|
16433
|
+
var BacklinksSchema = Type2.Object({
|
|
16434
|
+
limit: Type2.Optional(
|
|
16435
|
+
Type2.Number({
|
|
16436
|
+
description: "Max linking-domain rows to include. Default 50, max 200.",
|
|
16437
|
+
minimum: 1,
|
|
16438
|
+
maximum: 200
|
|
16439
|
+
})
|
|
16440
|
+
),
|
|
16441
|
+
release: Type2.Optional(
|
|
16442
|
+
Type2.String({
|
|
16443
|
+
description: "Common Crawl release id (e.g., cc-main-2026-jan-feb-mar). Omit for the most recent release with data."
|
|
16444
|
+
})
|
|
16445
|
+
)
|
|
16446
|
+
});
|
|
16447
|
+
function buildListBacklinksTool(ctx) {
|
|
16448
|
+
return {
|
|
16449
|
+
name: "list_backlinks",
|
|
16450
|
+
label: "List backlinks",
|
|
16451
|
+
description: "Backlink summary and top linking domains from the most recent ready Common Crawl release. Off-site authority signal that correlates with citation likelihood. Returns null summary when no release sync has completed for this workspace.",
|
|
16452
|
+
parameters: BacklinksSchema,
|
|
16453
|
+
execute: async (_toolCallId, params) => {
|
|
16454
|
+
const response = await ctx.client.backlinksDomains(ctx.projectName, {
|
|
16455
|
+
limit: params.limit ?? 50,
|
|
16456
|
+
release: params.release
|
|
16457
|
+
});
|
|
16458
|
+
return textResult2(response);
|
|
16459
|
+
}
|
|
16460
|
+
};
|
|
16461
|
+
}
|
|
15296
16462
|
var RecallSchema = Type2.Object({
|
|
15297
16463
|
limit: Type2.Optional(
|
|
15298
16464
|
Type2.Number({
|
|
@@ -15323,7 +16489,8 @@ function buildReadTools(ctx) {
|
|
|
15323
16489
|
buildListKeywordsTool(ctx),
|
|
15324
16490
|
buildListCompetitorsTool(ctx),
|
|
15325
16491
|
buildGetRunTool(ctx),
|
|
15326
|
-
buildRecallTool(ctx)
|
|
16492
|
+
buildRecallTool(ctx),
|
|
16493
|
+
buildListBacklinksTool(ctx)
|
|
15327
16494
|
];
|
|
15328
16495
|
}
|
|
15329
16496
|
var RunSweepSchema = Type2.Object({
|
|
@@ -15557,10 +16724,10 @@ function ensureBuiltinsRegistered() {
|
|
|
15557
16724
|
}
|
|
15558
16725
|
function loadAeroSystemPrompt(pkgDir) {
|
|
15559
16726
|
const skillDir = resolveAeroSkillDir(pkgDir);
|
|
15560
|
-
const skillBody =
|
|
15561
|
-
const soulPath =
|
|
15562
|
-
if (!
|
|
15563
|
-
const soulBody =
|
|
16727
|
+
const skillBody = fs12.readFileSync(path14.join(skillDir, "SKILL.md"), "utf-8");
|
|
16728
|
+
const soulPath = path14.join(skillDir, "soul.md");
|
|
16729
|
+
if (!fs12.existsSync(soulPath)) return skillBody;
|
|
16730
|
+
const soulBody = fs12.readFileSync(soulPath, "utf-8");
|
|
15564
16731
|
return `${soulBody.trimEnd()}
|
|
15565
16732
|
|
|
15566
16733
|
---
|
|
@@ -15744,7 +16911,7 @@ async function compactMessages(args) {
|
|
|
15744
16911
|
}
|
|
15745
16912
|
|
|
15746
16913
|
// src/agent/session-registry.ts
|
|
15747
|
-
var
|
|
16914
|
+
var log9 = createLogger("SessionRegistry");
|
|
15748
16915
|
var MAX_HYDRATE_NOTES = 20;
|
|
15749
16916
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
15750
16917
|
function escapeMemoryFragment(value) {
|
|
@@ -15793,7 +16960,7 @@ var SessionRegistry = class {
|
|
|
15793
16960
|
modelProvider: effectiveProvider,
|
|
15794
16961
|
modelId: effectiveModelId,
|
|
15795
16962
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
15796
|
-
}).where(
|
|
16963
|
+
}).where(eq27(agentSessions.projectId, projectId)).run();
|
|
15797
16964
|
}
|
|
15798
16965
|
const agent2 = createAeroSession({
|
|
15799
16966
|
projectName,
|
|
@@ -15975,13 +17142,13 @@ ${lines.join("\n")}
|
|
|
15975
17142
|
agent.state.messages = result.messages;
|
|
15976
17143
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
15977
17144
|
this.save(projectName);
|
|
15978
|
-
|
|
17145
|
+
log9.info("compaction.completed", {
|
|
15979
17146
|
projectName,
|
|
15980
17147
|
removedCount: result.removedCount,
|
|
15981
17148
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
15982
17149
|
});
|
|
15983
17150
|
} catch (err) {
|
|
15984
|
-
|
|
17151
|
+
log9.error("compaction.failed", {
|
|
15985
17152
|
projectName,
|
|
15986
17153
|
error: err instanceof Error ? err.message : String(err)
|
|
15987
17154
|
});
|
|
@@ -16011,7 +17178,7 @@ ${lines.join("\n")}
|
|
|
16011
17178
|
modelProvider: nextProvider,
|
|
16012
17179
|
modelId: nextModelId,
|
|
16013
17180
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
16014
|
-
}).where(
|
|
17181
|
+
}).where(eq27(agentSessions.projectId, projectId)).run();
|
|
16015
17182
|
}
|
|
16016
17183
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
16017
17184
|
save(projectName) {
|
|
@@ -16078,7 +17245,7 @@ ${lines.join("\n")}
|
|
|
16078
17245
|
await agent.prompt(msgs);
|
|
16079
17246
|
this.save(projectName);
|
|
16080
17247
|
} catch (err) {
|
|
16081
|
-
|
|
17248
|
+
log9.error("drain.failed", {
|
|
16082
17249
|
projectName,
|
|
16083
17250
|
error: err instanceof Error ? err.message : String(err)
|
|
16084
17251
|
});
|
|
@@ -16173,17 +17340,17 @@ ${lines.join("\n")}
|
|
|
16173
17340
|
return id;
|
|
16174
17341
|
}
|
|
16175
17342
|
tryResolveProjectId(projectName) {
|
|
16176
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
17343
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq27(projects.name, projectName)).get();
|
|
16177
17344
|
return row?.id;
|
|
16178
17345
|
}
|
|
16179
17346
|
loadRow(projectId) {
|
|
16180
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
17347
|
+
const row = this.opts.db.select().from(agentSessions).where(eq27(agentSessions.projectId, projectId)).get();
|
|
16181
17348
|
return row ?? null;
|
|
16182
17349
|
}
|
|
16183
17350
|
insertRow(params) {
|
|
16184
17351
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16185
17352
|
this.opts.db.insert(agentSessions).values({
|
|
16186
|
-
id:
|
|
17353
|
+
id: crypto26.randomUUID(),
|
|
16187
17354
|
projectId: params.projectId,
|
|
16188
17355
|
systemPrompt: params.systemPrompt,
|
|
16189
17356
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -16196,14 +17363,14 @@ ${lines.join("\n")}
|
|
|
16196
17363
|
}
|
|
16197
17364
|
updateRow(projectId, patch) {
|
|
16198
17365
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16199
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
17366
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq27(agentSessions.projectId, projectId)).run();
|
|
16200
17367
|
}
|
|
16201
17368
|
};
|
|
16202
17369
|
|
|
16203
17370
|
// src/agent/agent-routes.ts
|
|
16204
|
-
import { eq as
|
|
17371
|
+
import { eq as eq28 } from "drizzle-orm";
|
|
16205
17372
|
function resolveProject2(db, name) {
|
|
16206
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
17373
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq28(projects.name, name)).get();
|
|
16207
17374
|
if (!row) throw notFound("project", name);
|
|
16208
17375
|
return row;
|
|
16209
17376
|
}
|
|
@@ -16212,7 +17379,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16212
17379
|
"/projects/:name/agent/transcript",
|
|
16213
17380
|
async (request) => {
|
|
16214
17381
|
const project = resolveProject2(opts.db, request.params.name);
|
|
16215
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
17382
|
+
const row = opts.db.select().from(agentSessions).where(eq28(agentSessions.projectId, project.id)).get();
|
|
16216
17383
|
if (!row) {
|
|
16217
17384
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
16218
17385
|
}
|
|
@@ -16236,7 +17403,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
16236
17403
|
async (request) => {
|
|
16237
17404
|
const project = resolveProject2(opts.db, request.params.name);
|
|
16238
17405
|
opts.sessionRegistry.reset(project.name);
|
|
16239
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
17406
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(agentSessions.projectId, project.id)).run();
|
|
16240
17407
|
return { status: "reset" };
|
|
16241
17408
|
}
|
|
16242
17409
|
);
|
|
@@ -16398,9 +17565,9 @@ var ApiClient = class {
|
|
|
16398
17565
|
}
|
|
16399
17566
|
return this.probePromise;
|
|
16400
17567
|
}
|
|
16401
|
-
async request(method,
|
|
17568
|
+
async request(method, path16, body) {
|
|
16402
17569
|
await this.probeBasePath();
|
|
16403
|
-
const url = `${this.baseUrl}${
|
|
17570
|
+
const url = `${this.baseUrl}${path16}`;
|
|
16404
17571
|
const serializedBody = body != null ? JSON.stringify(body) : void 0;
|
|
16405
17572
|
const headers = {
|
|
16406
17573
|
"Authorization": `Bearer ${this.apiKey}`,
|
|
@@ -16488,9 +17655,9 @@ var ApiClient = class {
|
|
|
16488
17655
|
* structured-error behavior of `request()`; the caller reads `res.body`
|
|
16489
17656
|
* and releases the response when done.
|
|
16490
17657
|
*/
|
|
16491
|
-
async streamPost(
|
|
17658
|
+
async streamPost(path16, body, signal) {
|
|
16492
17659
|
await this.probeBasePath();
|
|
16493
|
-
const url = `${this.baseUrl}${
|
|
17660
|
+
const url = `${this.baseUrl}${path16}`;
|
|
16494
17661
|
const headers = {
|
|
16495
17662
|
Authorization: `Bearer ${this.apiKey}`,
|
|
16496
17663
|
"Content-Type": "application/json",
|
|
@@ -16891,6 +18058,46 @@ var ApiClient = class {
|
|
|
16891
18058
|
const qs = limit ? `?limit=${limit}` : "";
|
|
16892
18059
|
return this.request("GET", `/projects/${encodeURIComponent(project)}/health/history${qs}`);
|
|
16893
18060
|
}
|
|
18061
|
+
// --- Backlinks ---------------------------------------------------------
|
|
18062
|
+
async backlinksStatus() {
|
|
18063
|
+
return this.request("GET", "/backlinks/status");
|
|
18064
|
+
}
|
|
18065
|
+
async backlinksInstall() {
|
|
18066
|
+
return this.request("POST", "/backlinks/install");
|
|
18067
|
+
}
|
|
18068
|
+
async backlinksTriggerSync(release) {
|
|
18069
|
+
return this.request("POST", "/backlinks/syncs", { release });
|
|
18070
|
+
}
|
|
18071
|
+
async backlinksLatestSync() {
|
|
18072
|
+
return this.request("GET", "/backlinks/syncs/latest");
|
|
18073
|
+
}
|
|
18074
|
+
async backlinksListSyncs() {
|
|
18075
|
+
return this.request("GET", "/backlinks/syncs");
|
|
18076
|
+
}
|
|
18077
|
+
async backlinksCachedReleases() {
|
|
18078
|
+
return this.request("GET", "/backlinks/releases");
|
|
18079
|
+
}
|
|
18080
|
+
async backlinksPruneCache(release) {
|
|
18081
|
+
return this.request("DELETE", `/backlinks/cache/${encodeURIComponent(release)}`);
|
|
18082
|
+
}
|
|
18083
|
+
async backlinksExtract(project, release) {
|
|
18084
|
+
return this.request("POST", `/projects/${encodeURIComponent(project)}/backlinks/extract`, release ? { release } : {});
|
|
18085
|
+
}
|
|
18086
|
+
async backlinksSummary(project, release) {
|
|
18087
|
+
const qs = release ? `?release=${encodeURIComponent(release)}` : "";
|
|
18088
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/summary${qs}`);
|
|
18089
|
+
}
|
|
18090
|
+
async backlinksDomains(project, opts = {}) {
|
|
18091
|
+
const qs = new URLSearchParams();
|
|
18092
|
+
if (opts.limit !== void 0) qs.set("limit", String(opts.limit));
|
|
18093
|
+
if (opts.offset !== void 0) qs.set("offset", String(opts.offset));
|
|
18094
|
+
if (opts.release) qs.set("release", opts.release);
|
|
18095
|
+
const suffix = qs.toString() ? `?${qs.toString()}` : "";
|
|
18096
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/domains${suffix}`);
|
|
18097
|
+
}
|
|
18098
|
+
async backlinksHistory(project) {
|
|
18099
|
+
return this.request("GET", `/projects/${encodeURIComponent(project)}/backlinks/history`);
|
|
18100
|
+
}
|
|
16894
18101
|
};
|
|
16895
18102
|
|
|
16896
18103
|
// src/snapshot-service.ts
|
|
@@ -16915,13 +18122,13 @@ function extractHostname(domain) {
|
|
|
16915
18122
|
function fetchWithPinnedAddress(target) {
|
|
16916
18123
|
return new Promise((resolve) => {
|
|
16917
18124
|
const port = target.url.port ? Number(target.url.port) : 443;
|
|
16918
|
-
const
|
|
18125
|
+
const path16 = target.url.pathname + target.url.search;
|
|
16919
18126
|
const req = https2.request(
|
|
16920
18127
|
{
|
|
16921
18128
|
hostname: target.address,
|
|
16922
18129
|
family: target.family,
|
|
16923
18130
|
port,
|
|
16924
|
-
path:
|
|
18131
|
+
path: path16,
|
|
16925
18132
|
method: "GET",
|
|
16926
18133
|
timeout: FETCH_TIMEOUT_MS,
|
|
16927
18134
|
servername: target.url.hostname,
|
|
@@ -17013,7 +18220,7 @@ function formatAuditFactorScore(factor) {
|
|
|
17013
18220
|
}
|
|
17014
18221
|
|
|
17015
18222
|
// src/snapshot-service.ts
|
|
17016
|
-
var
|
|
18223
|
+
var log10 = createLogger("Snapshot");
|
|
17017
18224
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
17018
18225
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
17019
18226
|
var ProviderExecutionGate2 = class {
|
|
@@ -17156,7 +18363,7 @@ var SnapshotService = class {
|
|
|
17156
18363
|
return mapAuditReport(report);
|
|
17157
18364
|
} catch (err) {
|
|
17158
18365
|
const message = err instanceof Error ? err.message : String(err);
|
|
17159
|
-
|
|
18366
|
+
log10.warn("audit.failed", { homepageUrl, error: message });
|
|
17160
18367
|
return {
|
|
17161
18368
|
url: homepageUrl,
|
|
17162
18369
|
finalUrl: homepageUrl,
|
|
@@ -17186,7 +18393,7 @@ var SnapshotService = class {
|
|
|
17186
18393
|
phrases: parsedPhrases
|
|
17187
18394
|
};
|
|
17188
18395
|
} catch (err) {
|
|
17189
|
-
|
|
18396
|
+
log10.warn("profile.generation-failed", {
|
|
17190
18397
|
domain: ctx.domain,
|
|
17191
18398
|
provider: ctx.analysisProvider.adapter.name,
|
|
17192
18399
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -17328,7 +18535,7 @@ var SnapshotService = class {
|
|
|
17328
18535
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
17329
18536
|
};
|
|
17330
18537
|
} catch (err) {
|
|
17331
|
-
|
|
18538
|
+
log10.warn("response.analysis-failed", {
|
|
17332
18539
|
provider: ctx.analysisProvider.adapter.name,
|
|
17333
18540
|
error: err instanceof Error ? err.message : String(err)
|
|
17334
18541
|
});
|
|
@@ -17611,9 +18818,9 @@ function clipText(value, length) {
|
|
|
17611
18818
|
}
|
|
17612
18819
|
|
|
17613
18820
|
// src/server.ts
|
|
17614
|
-
var _require2 =
|
|
18821
|
+
var _require2 = createRequire3(import.meta.url);
|
|
17615
18822
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
17616
|
-
var
|
|
18823
|
+
var log11 = createLogger("Server");
|
|
17617
18824
|
var DEFAULT_QUOTA = {
|
|
17618
18825
|
maxConcurrency: 2,
|
|
17619
18826
|
maxRequestsPerMinute: 10,
|
|
@@ -17644,7 +18851,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
17644
18851
|
};
|
|
17645
18852
|
}
|
|
17646
18853
|
function hashApiKey(key) {
|
|
17647
|
-
return
|
|
18854
|
+
return crypto27.createHash("sha256").update(key).digest("hex");
|
|
17648
18855
|
}
|
|
17649
18856
|
function parseCookies2(header) {
|
|
17650
18857
|
if (!header) return {};
|
|
@@ -17700,7 +18907,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
17700
18907
|
}
|
|
17701
18908
|
if (migratedGoogle > 0) {
|
|
17702
18909
|
saveConfigPatch({ google: config.google });
|
|
17703
|
-
|
|
18910
|
+
log11.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
17704
18911
|
}
|
|
17705
18912
|
let migratedGa4 = 0;
|
|
17706
18913
|
for (const row of rows.ga4) {
|
|
@@ -17718,7 +18925,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
17718
18925
|
}
|
|
17719
18926
|
if (migratedGa4 > 0) {
|
|
17720
18927
|
saveConfigPatch({ ga4: config.ga4 });
|
|
17721
|
-
|
|
18928
|
+
log11.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
17722
18929
|
}
|
|
17723
18930
|
}
|
|
17724
18931
|
async function createServer(opts) {
|
|
@@ -17750,11 +18957,11 @@ async function createServer(opts) {
|
|
|
17750
18957
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
17751
18958
|
dropLegacyCredentialColumns(opts.db);
|
|
17752
18959
|
} catch (err) {
|
|
17753
|
-
|
|
18960
|
+
log11.warn("credentials.migration.failed", {
|
|
17754
18961
|
error: err instanceof Error ? err.message : String(err)
|
|
17755
18962
|
});
|
|
17756
18963
|
}
|
|
17757
|
-
|
|
18964
|
+
log11.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
17758
18965
|
const p = providers[k];
|
|
17759
18966
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
17760
18967
|
}) });
|
|
@@ -17802,7 +19009,7 @@ async function createServer(opts) {
|
|
|
17802
19009
|
intelligenceService,
|
|
17803
19010
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
17804
19011
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
17805
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
19012
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq29(projects.id, projectId)).get();
|
|
17806
19013
|
if (!project) return;
|
|
17807
19014
|
sessionRegistry.queueFollowUp(project.name, {
|
|
17808
19015
|
role: "user",
|
|
@@ -17814,8 +19021,8 @@ async function createServer(opts) {
|
|
|
17814
19021
|
);
|
|
17815
19022
|
jobRunner.onRunCompleted = (runId, projectId) => runCoordinator.onRunCompleted(runId, projectId);
|
|
17816
19023
|
const snapshotService = new SnapshotService(registry);
|
|
17817
|
-
const orphanedOpenClawDir =
|
|
17818
|
-
if (
|
|
19024
|
+
const orphanedOpenClawDir = path15.join(os6.homedir(), ".openclaw-aero");
|
|
19025
|
+
if (fs13.existsSync(orphanedOpenClawDir)) {
|
|
17819
19026
|
app.log.warn(
|
|
17820
19027
|
{ path: orphanedOpenClawDir },
|
|
17821
19028
|
"OpenClaw gateway is no longer used. Remove ~/.openclaw-aero/ manually to reclaim the directory."
|
|
@@ -17896,7 +19103,7 @@ async function createServer(opts) {
|
|
|
17896
19103
|
return removed;
|
|
17897
19104
|
}
|
|
17898
19105
|
};
|
|
17899
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
19106
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto27.randomBytes(32).toString("hex");
|
|
17900
19107
|
const googleConnectionStore = {
|
|
17901
19108
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
17902
19109
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -17942,11 +19149,11 @@ async function createServer(opts) {
|
|
|
17942
19149
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
17943
19150
|
if (opts.config.apiKey) {
|
|
17944
19151
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
17945
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
19152
|
+
const existing = opts.db.select().from(apiKeys).where(eq29(apiKeys.keyHash, keyHash)).get();
|
|
17946
19153
|
if (!existing) {
|
|
17947
19154
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
17948
19155
|
opts.db.insert(apiKeys).values({
|
|
17949
|
-
id: `key_${
|
|
19156
|
+
id: `key_${crypto27.randomBytes(8).toString("hex")}`,
|
|
17950
19157
|
name: "default",
|
|
17951
19158
|
keyHash,
|
|
17952
19159
|
keyPrefix: prefix,
|
|
@@ -17970,7 +19177,7 @@ async function createServer(opts) {
|
|
|
17970
19177
|
};
|
|
17971
19178
|
const createSession = (apiKeyId) => {
|
|
17972
19179
|
pruneExpiredSessions();
|
|
17973
|
-
const sessionId =
|
|
19180
|
+
const sessionId = crypto27.randomBytes(32).toString("hex");
|
|
17974
19181
|
sessions.set(sessionId, {
|
|
17975
19182
|
apiKeyId,
|
|
17976
19183
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -17994,7 +19201,7 @@ async function createServer(opts) {
|
|
|
17994
19201
|
};
|
|
17995
19202
|
const getDefaultApiKey = () => {
|
|
17996
19203
|
if (!opts.config.apiKey) return void 0;
|
|
17997
|
-
return opts.db.select().from(apiKeys).where(
|
|
19204
|
+
return opts.db.select().from(apiKeys).where(eq29(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
17998
19205
|
};
|
|
17999
19206
|
const createPasswordSession = (reply) => {
|
|
18000
19207
|
const key = getDefaultApiKey();
|
|
@@ -18051,12 +19258,12 @@ async function createServer(opts) {
|
|
|
18051
19258
|
return reply.send({ authenticated: true });
|
|
18052
19259
|
}
|
|
18053
19260
|
if (apiKey) {
|
|
18054
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
19261
|
+
const key = opts.db.select().from(apiKeys).where(eq29(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
18055
19262
|
if (!key || key.revokedAt) {
|
|
18056
19263
|
const err2 = authInvalid();
|
|
18057
19264
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
18058
19265
|
}
|
|
18059
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
19266
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(apiKeys.id, key.id)).run();
|
|
18060
19267
|
const sessionId = createSession(key.id);
|
|
18061
19268
|
reply.header("set-cookie", serializeSessionCookie({
|
|
18062
19269
|
name: SESSION_COOKIE_NAME,
|
|
@@ -18123,6 +19330,73 @@ async function createServer(opts) {
|
|
|
18123
19330
|
app.log.error({ runId, err }, "Inspect sitemap failed");
|
|
18124
19331
|
});
|
|
18125
19332
|
},
|
|
19333
|
+
getBacklinksStatus: () => ({
|
|
19334
|
+
duckdbInstalled: isDuckdbInstalled(),
|
|
19335
|
+
duckdbVersion: readInstalledVersion() ?? void 0,
|
|
19336
|
+
duckdbSpec: DUCKDB_SPEC,
|
|
19337
|
+
pluginDir: PLUGIN_DIR
|
|
19338
|
+
}),
|
|
19339
|
+
onInstallBacklinks: async () => {
|
|
19340
|
+
const result = await installDuckdb({ onLog: (line) => app.log.info({ line }, "duckdb install") });
|
|
19341
|
+
return {
|
|
19342
|
+
installed: true,
|
|
19343
|
+
version: result.version,
|
|
19344
|
+
path: result.path,
|
|
19345
|
+
alreadyPresent: result.alreadyPresent
|
|
19346
|
+
};
|
|
19347
|
+
},
|
|
19348
|
+
onReleaseSyncRequested: (syncId, release) => {
|
|
19349
|
+
executeReleaseSync(opts.db, syncId, {
|
|
19350
|
+
release,
|
|
19351
|
+
deps: {
|
|
19352
|
+
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
19353
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19354
|
+
const runId = crypto27.randomUUID();
|
|
19355
|
+
opts.db.insert(runs).values({
|
|
19356
|
+
id: runId,
|
|
19357
|
+
projectId,
|
|
19358
|
+
kind: RunKinds["backlink-extract"],
|
|
19359
|
+
status: RunStatuses.queued,
|
|
19360
|
+
trigger: RunTriggers.scheduled,
|
|
19361
|
+
createdAt: now
|
|
19362
|
+
}).run();
|
|
19363
|
+
executeBacklinkExtract(opts.db, runId, projectId, { release: r }).catch((err) => {
|
|
19364
|
+
app.log.error({ runId, projectId, err }, "Auto backlink extract failed");
|
|
19365
|
+
});
|
|
19366
|
+
}
|
|
19367
|
+
}
|
|
19368
|
+
}).catch((err) => {
|
|
19369
|
+
app.log.error({ syncId, err }, "Common Crawl release sync failed");
|
|
19370
|
+
});
|
|
19371
|
+
},
|
|
19372
|
+
onBacklinkExtractRequested: (runId, projectId, release) => {
|
|
19373
|
+
executeBacklinkExtract(opts.db, runId, projectId, { release }).catch((err) => {
|
|
19374
|
+
app.log.error({ runId, err }, "Backlink extract failed");
|
|
19375
|
+
});
|
|
19376
|
+
},
|
|
19377
|
+
onBacklinksPruneCache: (release) => {
|
|
19378
|
+
try {
|
|
19379
|
+
pruneCachedRelease(release);
|
|
19380
|
+
} catch (err) {
|
|
19381
|
+
app.log.error({ release, err }, "Failed to prune cached release");
|
|
19382
|
+
}
|
|
19383
|
+
},
|
|
19384
|
+
listCachedReleases: () => {
|
|
19385
|
+
const cached = listCachedReleases();
|
|
19386
|
+
const syncByRelease = /* @__PURE__ */ new Map();
|
|
19387
|
+
for (const row of opts.db.select().from(ccReleaseSyncs).all()) {
|
|
19388
|
+
syncByRelease.set(row.release, { status: row.status, updatedAt: row.updatedAt });
|
|
19389
|
+
}
|
|
19390
|
+
return cached.map((entry) => {
|
|
19391
|
+
const sync = syncByRelease.get(entry.release);
|
|
19392
|
+
return {
|
|
19393
|
+
release: entry.release,
|
|
19394
|
+
syncStatus: sync?.status ?? null,
|
|
19395
|
+
bytes: entry.bytes,
|
|
19396
|
+
lastUsedAt: entry.lastUsedAt
|
|
19397
|
+
};
|
|
19398
|
+
});
|
|
19399
|
+
},
|
|
18126
19400
|
openApiInfo: {
|
|
18127
19401
|
title: "Canonry API",
|
|
18128
19402
|
version: PKG_VERSION,
|
|
@@ -18203,7 +19477,7 @@ async function createServer(opts) {
|
|
|
18203
19477
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
18204
19478
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
18205
19479
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
18206
|
-
id:
|
|
19480
|
+
id: crypto27.randomUUID(),
|
|
18207
19481
|
projectId,
|
|
18208
19482
|
actor: "api",
|
|
18209
19483
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -18334,10 +19608,10 @@ async function createServer(opts) {
|
|
|
18334
19608
|
return snapshotService.createReport(input);
|
|
18335
19609
|
}
|
|
18336
19610
|
});
|
|
18337
|
-
const dirname =
|
|
18338
|
-
const assetsDir =
|
|
18339
|
-
if (
|
|
18340
|
-
const indexPath =
|
|
19611
|
+
const dirname = path15.dirname(fileURLToPath2(import.meta.url));
|
|
19612
|
+
const assetsDir = path15.join(dirname, "..", "assets");
|
|
19613
|
+
if (fs13.existsSync(assetsDir)) {
|
|
19614
|
+
const indexPath = path15.join(assetsDir, "index.html");
|
|
18341
19615
|
const injectConfig = (html) => {
|
|
18342
19616
|
const clientConfig = {};
|
|
18343
19617
|
if (basePath) clientConfig.basePath = basePath;
|
|
@@ -18355,8 +19629,8 @@ async function createServer(opts) {
|
|
|
18355
19629
|
index: false
|
|
18356
19630
|
});
|
|
18357
19631
|
const serveIndex = (_request, reply) => {
|
|
18358
|
-
if (
|
|
18359
|
-
const html =
|
|
19632
|
+
if (fs13.existsSync(indexPath)) {
|
|
19633
|
+
const html = fs13.readFileSync(indexPath, "utf-8");
|
|
18360
19634
|
return reply.type("text/html").send(injectConfig(html));
|
|
18361
19635
|
}
|
|
18362
19636
|
return reply.status(404).send({ error: "Dashboard not built" });
|
|
@@ -18376,8 +19650,8 @@ async function createServer(opts) {
|
|
|
18376
19650
|
if (basePath && !url.startsWith(basePath)) {
|
|
18377
19651
|
return reply.status(404).send({ error: "Not found", path: request.url });
|
|
18378
19652
|
}
|
|
18379
|
-
if (
|
|
18380
|
-
const html =
|
|
19653
|
+
if (fs13.existsSync(indexPath)) {
|
|
19654
|
+
const html = fs13.readFileSync(indexPath, "utf-8");
|
|
18381
19655
|
return reply.type("text/html").send(injectConfig(html));
|
|
18382
19656
|
}
|
|
18383
19657
|
return reply.status(404).send({ error: "Not found" });
|
|
@@ -18467,8 +19741,10 @@ export {
|
|
|
18467
19741
|
resolveProviderInput,
|
|
18468
19742
|
notificationEventSchema,
|
|
18469
19743
|
effectiveDomains,
|
|
19744
|
+
RunStatuses,
|
|
18470
19745
|
RunKinds,
|
|
18471
19746
|
determineAnswerMentioned,
|
|
19747
|
+
CcReleaseSyncStatuses,
|
|
18472
19748
|
reparseStoredResult2 as reparseStoredResult,
|
|
18473
19749
|
reparseStoredResult3 as reparseStoredResult2,
|
|
18474
19750
|
reparseStoredResult as reparseStoredResult3,
|