@ainyc/canonry 4.24.0 → 4.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agent-workspace/skills/aero/references/aeo-discovery.md +89 -0
- package/assets/agent-workspace/skills/canonry-setup/references/canonry-cli.md +14 -0
- package/assets/assets/index-C4scWriC.js +302 -0
- package/assets/index.html +1 -1
- package/dist/{chunk-E5PZ23OS.js → chunk-6J6WQOGH.js} +905 -195
- package/dist/{chunk-6EJ54OX7.js → chunk-A7HQ6X43.js} +91 -1
- package/dist/{chunk-EUGCQSFC.js → chunk-CRQMGNPH.js} +133 -1
- package/dist/{chunk-OYYFXKRK.js → chunk-IS65IYNZ.js} +115 -2
- package/dist/cli.js +424 -123
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-NVN2PAR7.js → intelligence-service-XLUYTE57.js} +2 -2
- package/dist/mcp.js +9 -3
- package/package.json +8 -8
- package/assets/assets/index-BFfB9cRq.js +0 -302
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
loadConfig,
|
|
6
6
|
loadConfigRaw,
|
|
7
7
|
saveConfigPatch
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-A7HQ6X43.js";
|
|
9
9
|
import {
|
|
10
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
11
11
|
IntelligenceService,
|
|
@@ -40,6 +40,8 @@ import {
|
|
|
40
40
|
competitors,
|
|
41
41
|
crawlerEventsHourly,
|
|
42
42
|
createLogger,
|
|
43
|
+
discoveryProbes,
|
|
44
|
+
discoverySessions,
|
|
43
45
|
dropLegacyCredentialColumns,
|
|
44
46
|
extractLegacyCredentials,
|
|
45
47
|
gaAiReferrals,
|
|
@@ -66,7 +68,7 @@ import {
|
|
|
66
68
|
schedules,
|
|
67
69
|
trafficSources,
|
|
68
70
|
usageCounters
|
|
69
|
-
} from "./chunk-
|
|
71
|
+
} from "./chunk-IS65IYNZ.js";
|
|
70
72
|
import {
|
|
71
73
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
72
74
|
AGENT_PROVIDER_IDS,
|
|
@@ -77,6 +79,8 @@ import {
|
|
|
77
79
|
CheckScopes,
|
|
78
80
|
CheckStatuses,
|
|
79
81
|
CitationStates,
|
|
82
|
+
DiscoveryBuckets,
|
|
83
|
+
DiscoverySessionStatuses,
|
|
80
84
|
MemorySources,
|
|
81
85
|
RunKinds,
|
|
82
86
|
RunStatuses,
|
|
@@ -101,7 +105,9 @@ import {
|
|
|
101
105
|
buildRunErrorFromMessages,
|
|
102
106
|
categorizeSource,
|
|
103
107
|
categoryLabel,
|
|
108
|
+
citationStateSchema,
|
|
104
109
|
citationStateToCited,
|
|
110
|
+
clusterByCosine,
|
|
105
111
|
competitorBatchRequestSchema,
|
|
106
112
|
contentActionLabel,
|
|
107
113
|
dedupeReportActions,
|
|
@@ -110,6 +116,8 @@ import {
|
|
|
110
116
|
deltaPercent,
|
|
111
117
|
deltaTone,
|
|
112
118
|
determineAnswerMentioned,
|
|
119
|
+
discoveryBucketSchema,
|
|
120
|
+
discoveryRunRequestSchema,
|
|
113
121
|
effectiveDomains,
|
|
114
122
|
emptyCitationVisibility,
|
|
115
123
|
extractAnswerMentions,
|
|
@@ -134,6 +142,7 @@ import {
|
|
|
134
142
|
notImplemented,
|
|
135
143
|
parseRunError,
|
|
136
144
|
parseWindow,
|
|
145
|
+
pickClusterRepresentative,
|
|
137
146
|
projectConfigSchema,
|
|
138
147
|
projectUpsertRequestSchema,
|
|
139
148
|
providerError,
|
|
@@ -160,7 +169,7 @@ import {
|
|
|
160
169
|
visibilityStateFromAnswerMentioned,
|
|
161
170
|
windowCutoff,
|
|
162
171
|
wordpressEnvSchema
|
|
163
|
-
} from "./chunk-
|
|
172
|
+
} from "./chunk-CRQMGNPH.js";
|
|
164
173
|
|
|
165
174
|
// src/telemetry.ts
|
|
166
175
|
import crypto from "crypto";
|
|
@@ -315,11 +324,11 @@ function trackEvent(event, properties, options) {
|
|
|
315
324
|
|
|
316
325
|
// src/server.ts
|
|
317
326
|
import { createRequire as createRequire3 } from "module";
|
|
318
|
-
import
|
|
327
|
+
import crypto34 from "crypto";
|
|
319
328
|
import fs12 from "fs";
|
|
320
329
|
import path14 from "path";
|
|
321
330
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
322
|
-
import { eq as
|
|
331
|
+
import { eq as eq40 } from "drizzle-orm";
|
|
323
332
|
import Fastify from "fastify";
|
|
324
333
|
|
|
325
334
|
// ../api-routes/src/auth.ts
|
|
@@ -741,6 +750,7 @@ async function queryRoutes(app, opts) {
|
|
|
741
750
|
id: crypto5.randomUUID(),
|
|
742
751
|
projectId: project.id,
|
|
743
752
|
query: q,
|
|
753
|
+
provenance: "cli",
|
|
744
754
|
createdAt: now
|
|
745
755
|
}).run();
|
|
746
756
|
}
|
|
@@ -797,6 +807,7 @@ async function queryRoutes(app, opts) {
|
|
|
797
807
|
id: crypto5.randomUUID(),
|
|
798
808
|
projectId: project.id,
|
|
799
809
|
query: q,
|
|
810
|
+
provenance: "cli",
|
|
800
811
|
createdAt: now
|
|
801
812
|
}).run();
|
|
802
813
|
added.push(q);
|
|
@@ -874,6 +885,7 @@ async function queryRoutes(app, opts) {
|
|
|
874
885
|
id: crypto5.randomUUID(),
|
|
875
886
|
projectId: project.id,
|
|
876
887
|
query: keyword,
|
|
888
|
+
provenance: "cli",
|
|
877
889
|
createdAt: now
|
|
878
890
|
}).run();
|
|
879
891
|
}
|
|
@@ -930,6 +942,7 @@ async function queryRoutes(app, opts) {
|
|
|
930
942
|
id: crypto5.randomUUID(),
|
|
931
943
|
projectId: project.id,
|
|
932
944
|
query: keyword,
|
|
945
|
+
provenance: "cli",
|
|
933
946
|
createdAt: now
|
|
934
947
|
}).run();
|
|
935
948
|
added.push(keyword);
|
|
@@ -1032,6 +1045,7 @@ async function competitorRoutes(app) {
|
|
|
1032
1045
|
id: crypto6.randomUUID(),
|
|
1033
1046
|
projectId: project.id,
|
|
1034
1047
|
domain,
|
|
1048
|
+
provenance: "cli",
|
|
1035
1049
|
createdAt: now
|
|
1036
1050
|
}).run();
|
|
1037
1051
|
}
|
|
@@ -1061,6 +1075,7 @@ async function competitorRoutes(app) {
|
|
|
1061
1075
|
id: crypto6.randomUUID(),
|
|
1062
1076
|
projectId: project.id,
|
|
1063
1077
|
domain,
|
|
1078
|
+
provenance: "cli",
|
|
1064
1079
|
createdAt: now
|
|
1065
1080
|
}).onConflictDoNothing({
|
|
1066
1081
|
target: [competitors.projectId, competitors.domain]
|
|
@@ -1821,6 +1836,7 @@ async function applyRoutes(app, opts) {
|
|
|
1821
1836
|
id: crypto10.randomUUID(),
|
|
1822
1837
|
projectId,
|
|
1823
1838
|
query: q,
|
|
1839
|
+
provenance: "cli",
|
|
1824
1840
|
createdAt: now
|
|
1825
1841
|
}).run();
|
|
1826
1842
|
}
|
|
@@ -1838,6 +1854,7 @@ async function applyRoutes(app, opts) {
|
|
|
1838
1854
|
id: crypto10.randomUUID(),
|
|
1839
1855
|
projectId,
|
|
1840
1856
|
domain,
|
|
1857
|
+
provenance: "cli",
|
|
1841
1858
|
createdAt: now
|
|
1842
1859
|
}).run();
|
|
1843
1860
|
}
|
|
@@ -6789,6 +6806,15 @@ import { eq as eq15, and as and6, desc as desc7, sql as sql4, like, or as or3, i
|
|
|
6789
6806
|
var TOP_INSIGHT_LIMIT = 5;
|
|
6790
6807
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
6791
6808
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
6809
|
+
var INTEGRATION_SYNC_KINDS = /* @__PURE__ */ new Set([
|
|
6810
|
+
RunKinds["gsc-sync"],
|
|
6811
|
+
RunKinds["inspect-sitemap"],
|
|
6812
|
+
RunKinds["ga-sync"],
|
|
6813
|
+
RunKinds["bing-inspect"],
|
|
6814
|
+
RunKinds["bing-inspect-sitemap"],
|
|
6815
|
+
RunKinds["backlink-extract"],
|
|
6816
|
+
RunKinds["traffic-sync"]
|
|
6817
|
+
]);
|
|
6792
6818
|
async function compositeRoutes(app) {
|
|
6793
6819
|
app.get("/projects/:name/overview", async (request, reply) => {
|
|
6794
6820
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -7176,7 +7202,7 @@ function buildAttentionItems(insightRows, allRuns) {
|
|
|
7176
7202
|
}
|
|
7177
7203
|
const sortedRuns = [...allRuns].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
7178
7204
|
const latestVisRun = sortedRuns.find((r) => r.kind === RunKinds["answer-visibility"]);
|
|
7179
|
-
const latestSyncRun = sortedRuns.find((r) => r.kind
|
|
7205
|
+
const latestSyncRun = sortedRuns.find((r) => INTEGRATION_SYNC_KINDS.has(r.kind));
|
|
7180
7206
|
if (latestVisRun && latestSyncRun) {
|
|
7181
7207
|
const visibilityAge = new Date(latestSyncRun.createdAt).getTime() - new Date(latestVisRun.createdAt).getTime();
|
|
7182
7208
|
const ONE_DAY = 24 * 60 * 60 * 1e3;
|
|
@@ -10227,6 +10253,79 @@ var routeCatalog = [
|
|
|
10227
10253
|
400: { description: "Invalid query parameters." },
|
|
10228
10254
|
404: { description: "Project not found." }
|
|
10229
10255
|
}
|
|
10256
|
+
},
|
|
10257
|
+
{
|
|
10258
|
+
method: "post",
|
|
10259
|
+
path: "/api/v1/projects/{name}/discover/run",
|
|
10260
|
+
summary: "Start a tracked-basket discovery session",
|
|
10261
|
+
description: 'Kicks off a discovery session for the project. The pipeline: ICP description \u2192 Gemini grounded seed prompt \u2192 embed + cluster (cosine \u2265 0.85 by default) \u2192 pick canonical representatives \u2192 probe each canonical via Gemini grounding \u2192 classify into cited / aspirational / wasted-surface \u2192 aggregate competitor map. Returns immediately with `{ runId, sessionId, status: "running" }`; the actual work runs in the background. Poll `GET /projects/{name}/discover/sessions/{id}` until `status` is `completed` or `failed`.',
|
|
10262
|
+
tags: ["discovery"],
|
|
10263
|
+
parameters: [nameParameter],
|
|
10264
|
+
requestBody: {
|
|
10265
|
+
required: false,
|
|
10266
|
+
content: {
|
|
10267
|
+
"application/json": {
|
|
10268
|
+
schema: {
|
|
10269
|
+
type: "object",
|
|
10270
|
+
properties: {
|
|
10271
|
+
icpDescription: { type: "string", description: "Free-text ICP. Required if the project does not have spec.icpDescription stored." },
|
|
10272
|
+
dedupThreshold: { type: "number", description: "Cosine similarity threshold for clustering. Defaults to 0.85." },
|
|
10273
|
+
maxProbes: { type: "integer", description: "Max canonical queries to probe in this session. Default 100, hard cap 500." }
|
|
10274
|
+
}
|
|
10275
|
+
}
|
|
10276
|
+
}
|
|
10277
|
+
}
|
|
10278
|
+
},
|
|
10279
|
+
responses: {
|
|
10280
|
+
201: { description: "Discovery session enqueued; returns { runId, sessionId, status }." },
|
|
10281
|
+
400: { description: "Missing or invalid ICP / parameters." },
|
|
10282
|
+
404: { description: "Project not found." }
|
|
10283
|
+
}
|
|
10284
|
+
},
|
|
10285
|
+
{
|
|
10286
|
+
method: "get",
|
|
10287
|
+
path: "/api/v1/projects/{name}/discover/sessions",
|
|
10288
|
+
summary: "List discovery sessions for a project",
|
|
10289
|
+
description: "Returns sessions newest-first. Each row carries seed counts, bucket counts, the competitor map, and timing fields. Drill into `GET /projects/{name}/discover/sessions/{id}` for per-query probe rows.",
|
|
10290
|
+
tags: ["discovery"],
|
|
10291
|
+
parameters: [
|
|
10292
|
+
nameParameter,
|
|
10293
|
+
{ name: "limit", in: "query", description: "Max sessions returned. Default 50.", schema: stringSchema }
|
|
10294
|
+
],
|
|
10295
|
+
responses: {
|
|
10296
|
+
200: { description: "Sessions returned." },
|
|
10297
|
+
404: { description: "Project not found." }
|
|
10298
|
+
}
|
|
10299
|
+
},
|
|
10300
|
+
{
|
|
10301
|
+
method: "get",
|
|
10302
|
+
path: "/api/v1/projects/{name}/discover/sessions/{id}",
|
|
10303
|
+
summary: "Get a discovery session with its probe list",
|
|
10304
|
+
description: 'Returns one discovery session plus the full list of per-canonical probes (query, bucket, cited domains, citation state). Use this to answer "what did discovery find for project X?" in a single call.',
|
|
10305
|
+
tags: ["discovery"],
|
|
10306
|
+
parameters: [
|
|
10307
|
+
nameParameter,
|
|
10308
|
+
{ name: "id", in: "path", required: true, description: "Discovery session ID.", schema: stringSchema }
|
|
10309
|
+
],
|
|
10310
|
+
responses: {
|
|
10311
|
+
200: { description: "Session detail returned." },
|
|
10312
|
+
404: { description: "Project or session not found." }
|
|
10313
|
+
}
|
|
10314
|
+
},
|
|
10315
|
+
{
|
|
10316
|
+
method: "get",
|
|
10317
|
+
path: "/api/v1/projects/{name}/discover/sessions/{id}/promote",
|
|
10318
|
+
summary: "Preview a discovery promotion plan (read-only)",
|
|
10319
|
+
description: "Returns the payload `canonry discover promote` (PR 2) will persist: queries grouped by bucket, plus suggested new competitor domains. v1 is preview-only; the actual merge ships in PR 2.",
|
|
10320
|
+
tags: ["discovery"],
|
|
10321
|
+
parameters: [
|
|
10322
|
+
nameParameter,
|
|
10323
|
+
{ name: "id", in: "path", required: true, description: "Discovery session ID.", schema: stringSchema }
|
|
10324
|
+
],
|
|
10325
|
+
responses: {
|
|
10326
|
+
200: { description: "Promote preview returned." },
|
|
10327
|
+
404: { description: "Project or session not found." }
|
|
10328
|
+
}
|
|
10230
10329
|
}
|
|
10231
10330
|
];
|
|
10232
10331
|
var canonryLocalRouteCatalog = [
|
|
@@ -19288,6 +19387,298 @@ async function doctorRoutes(app, opts) {
|
|
|
19288
19387
|
});
|
|
19289
19388
|
}
|
|
19290
19389
|
|
|
19390
|
+
// ../api-routes/src/discovery/routes.ts
|
|
19391
|
+
import crypto21 from "crypto";
|
|
19392
|
+
import { eq as eq25, desc as desc13 } from "drizzle-orm";
|
|
19393
|
+
async function discoveryRoutes(app, opts) {
|
|
19394
|
+
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
19395
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19396
|
+
const parsed = discoveryRunRequestSchema.safeParse(request.body ?? {});
|
|
19397
|
+
if (!parsed.success) {
|
|
19398
|
+
throw validationError("Invalid discovery run request", {
|
|
19399
|
+
issues: parsed.error.issues.map((issue) => ({
|
|
19400
|
+
path: issue.path.join("."),
|
|
19401
|
+
message: issue.message
|
|
19402
|
+
}))
|
|
19403
|
+
});
|
|
19404
|
+
}
|
|
19405
|
+
const icpDescription = parsed.data.icpDescription?.trim() || (project.icpDescription ?? "").trim();
|
|
19406
|
+
if (!icpDescription) {
|
|
19407
|
+
throw validationError(
|
|
19408
|
+
"icpDescription is required. Pass it in the request body or store it on the project (spec.icpDescription)."
|
|
19409
|
+
);
|
|
19410
|
+
}
|
|
19411
|
+
if (!opts.onDiscoveryRunRequested) {
|
|
19412
|
+
throw validationError("Discovery is not available on this deployment.", {
|
|
19413
|
+
reason: "no-discovery-handler"
|
|
19414
|
+
});
|
|
19415
|
+
}
|
|
19416
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19417
|
+
const sessionId = crypto21.randomUUID();
|
|
19418
|
+
const runId = crypto21.randomUUID();
|
|
19419
|
+
app.db.transaction((tx) => {
|
|
19420
|
+
tx.insert(discoverySessions).values({
|
|
19421
|
+
id: sessionId,
|
|
19422
|
+
projectId: project.id,
|
|
19423
|
+
runId,
|
|
19424
|
+
status: DiscoverySessionStatuses.queued,
|
|
19425
|
+
icpDescription,
|
|
19426
|
+
dedupThreshold: parsed.data.dedupThreshold,
|
|
19427
|
+
competitorMap: "[]",
|
|
19428
|
+
createdAt: now
|
|
19429
|
+
}).run();
|
|
19430
|
+
tx.insert(runs).values({
|
|
19431
|
+
id: runId,
|
|
19432
|
+
projectId: project.id,
|
|
19433
|
+
kind: RunKinds["aeo-discover-probe"],
|
|
19434
|
+
status: RunStatuses.queued,
|
|
19435
|
+
trigger: RunTriggers.manual,
|
|
19436
|
+
createdAt: now
|
|
19437
|
+
}).run();
|
|
19438
|
+
writeAuditLog(tx, {
|
|
19439
|
+
projectId: project.id,
|
|
19440
|
+
actor: "api",
|
|
19441
|
+
action: "discovery.created",
|
|
19442
|
+
entityType: "discovery_session",
|
|
19443
|
+
entityId: sessionId
|
|
19444
|
+
});
|
|
19445
|
+
});
|
|
19446
|
+
opts.onDiscoveryRunRequested({
|
|
19447
|
+
runId,
|
|
19448
|
+
sessionId,
|
|
19449
|
+
projectId: project.id,
|
|
19450
|
+
icpDescription,
|
|
19451
|
+
dedupThreshold: parsed.data.dedupThreshold,
|
|
19452
|
+
maxProbes: parsed.data.maxProbes
|
|
19453
|
+
});
|
|
19454
|
+
return reply.status(201).send({ runId, sessionId, status: "running" });
|
|
19455
|
+
});
|
|
19456
|
+
app.get(
|
|
19457
|
+
"/projects/:name/discover/sessions",
|
|
19458
|
+
async (request, reply) => {
|
|
19459
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19460
|
+
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
19461
|
+
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
19462
|
+
const rows = app.db.select().from(discoverySessions).where(eq25(discoverySessions.projectId, project.id)).orderBy(desc13(discoverySessions.createdAt)).limit(limit).all();
|
|
19463
|
+
return reply.send(rows.map(serializeSession));
|
|
19464
|
+
}
|
|
19465
|
+
);
|
|
19466
|
+
app.get(
|
|
19467
|
+
"/projects/:name/discover/sessions/:id",
|
|
19468
|
+
async (request, reply) => {
|
|
19469
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19470
|
+
const session = app.db.select().from(discoverySessions).where(eq25(discoverySessions.id, request.params.id)).get();
|
|
19471
|
+
if (!session || session.projectId !== project.id) {
|
|
19472
|
+
throw notFound("Discovery session", request.params.id);
|
|
19473
|
+
}
|
|
19474
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq25(discoveryProbes.sessionId, session.id)).all();
|
|
19475
|
+
const detail = {
|
|
19476
|
+
...serializeSession(session),
|
|
19477
|
+
probes: probeRows.map(serializeProbe)
|
|
19478
|
+
};
|
|
19479
|
+
return reply.send(detail);
|
|
19480
|
+
}
|
|
19481
|
+
);
|
|
19482
|
+
app.get(
|
|
19483
|
+
"/projects/:name/discover/sessions/:id/promote",
|
|
19484
|
+
async (request, reply) => {
|
|
19485
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19486
|
+
const session = app.db.select().from(discoverySessions).where(eq25(discoverySessions.id, request.params.id)).get();
|
|
19487
|
+
if (!session || session.projectId !== project.id) {
|
|
19488
|
+
throw notFound("Discovery session", request.params.id);
|
|
19489
|
+
}
|
|
19490
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq25(discoveryProbes.sessionId, session.id)).all();
|
|
19491
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq25(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
19492
|
+
const seenCompetitors = new Set(existingCompetitors);
|
|
19493
|
+
const cited = /* @__PURE__ */ new Set();
|
|
19494
|
+
const aspirational = /* @__PURE__ */ new Set();
|
|
19495
|
+
const wasted = /* @__PURE__ */ new Set();
|
|
19496
|
+
for (const probe of probeRows) {
|
|
19497
|
+
const bucket = probe.bucket;
|
|
19498
|
+
if (!bucket) continue;
|
|
19499
|
+
if (bucket === DiscoveryBuckets.cited) cited.add(probe.query);
|
|
19500
|
+
else if (bucket === DiscoveryBuckets.aspirational) aspirational.add(probe.query);
|
|
19501
|
+
else if (bucket === DiscoveryBuckets["wasted-surface"]) wasted.add(probe.query);
|
|
19502
|
+
}
|
|
19503
|
+
const competitorMap = parseJsonColumn(session.competitorMap, []);
|
|
19504
|
+
const newCompetitors = competitorMap.filter((entry) => !seenCompetitors.has(entry.domain.toLowerCase())).slice(0, 20);
|
|
19505
|
+
return reply.send({
|
|
19506
|
+
sessionId: session.id,
|
|
19507
|
+
projectId: project.id,
|
|
19508
|
+
queriesByBucket: {
|
|
19509
|
+
cited: Array.from(cited).sort(),
|
|
19510
|
+
aspirational: Array.from(aspirational).sort(),
|
|
19511
|
+
"wasted-surface": Array.from(wasted).sort()
|
|
19512
|
+
},
|
|
19513
|
+
suggestedCompetitors: newCompetitors,
|
|
19514
|
+
status: session.status
|
|
19515
|
+
});
|
|
19516
|
+
}
|
|
19517
|
+
);
|
|
19518
|
+
}
|
|
19519
|
+
function serializeSession(row) {
|
|
19520
|
+
return {
|
|
19521
|
+
id: row.id,
|
|
19522
|
+
projectId: row.projectId,
|
|
19523
|
+
status: row.status,
|
|
19524
|
+
icpDescription: row.icpDescription ?? null,
|
|
19525
|
+
seedProvider: row.seedProvider ?? null,
|
|
19526
|
+
seedCountRaw: row.seedCountRaw ?? null,
|
|
19527
|
+
seedCount: row.seedCount ?? null,
|
|
19528
|
+
dedupThreshold: row.dedupThreshold ?? null,
|
|
19529
|
+
probeCount: row.probeCount ?? null,
|
|
19530
|
+
citedCount: row.citedCount ?? null,
|
|
19531
|
+
aspirationalCount: row.aspirationalCount ?? null,
|
|
19532
|
+
wastedCount: row.wastedCount ?? null,
|
|
19533
|
+
competitorMap: parseJsonColumn(row.competitorMap, []),
|
|
19534
|
+
error: row.error ?? null,
|
|
19535
|
+
startedAt: row.startedAt ?? null,
|
|
19536
|
+
finishedAt: row.finishedAt ?? null,
|
|
19537
|
+
createdAt: row.createdAt
|
|
19538
|
+
};
|
|
19539
|
+
}
|
|
19540
|
+
function serializeProbe(row) {
|
|
19541
|
+
const bucketParsed = row.bucket ? discoveryBucketSchema.safeParse(row.bucket) : null;
|
|
19542
|
+
const stateParsed = citationStateSchema.safeParse(row.citationState);
|
|
19543
|
+
return {
|
|
19544
|
+
id: row.id,
|
|
19545
|
+
sessionId: row.sessionId,
|
|
19546
|
+
projectId: row.projectId,
|
|
19547
|
+
query: row.query,
|
|
19548
|
+
bucket: bucketParsed?.success ? bucketParsed.data : null,
|
|
19549
|
+
citationState: stateParsed.success ? stateParsed.data : "not-cited",
|
|
19550
|
+
citedDomains: parseJsonColumn(row.citedDomains, []),
|
|
19551
|
+
createdAt: row.createdAt
|
|
19552
|
+
};
|
|
19553
|
+
}
|
|
19554
|
+
|
|
19555
|
+
// ../api-routes/src/discovery/orchestrate.ts
|
|
19556
|
+
import crypto22 from "crypto";
|
|
19557
|
+
import { eq as eq26 } from "drizzle-orm";
|
|
19558
|
+
var DEFAULT_DEDUP_THRESHOLD = 0.85;
|
|
19559
|
+
var DEFAULT_MAX_PROBES = 100;
|
|
19560
|
+
var ABSOLUTE_MAX_PROBES = 500;
|
|
19561
|
+
function classifyProbeBucket(input) {
|
|
19562
|
+
const cited = new Set(input.citedDomains.map((d) => d.toLowerCase()));
|
|
19563
|
+
const canonicalHit = input.project.canonicalDomains.some((d) => cited.has(d.toLowerCase()));
|
|
19564
|
+
if (canonicalHit) return DiscoveryBuckets.cited;
|
|
19565
|
+
const competitorHit = input.project.competitorDomains.some((d) => cited.has(d.toLowerCase()));
|
|
19566
|
+
if (competitorHit) return DiscoveryBuckets["wasted-surface"];
|
|
19567
|
+
return DiscoveryBuckets.aspirational;
|
|
19568
|
+
}
|
|
19569
|
+
function buildCompetitorMap(probes, project) {
|
|
19570
|
+
const canonical = new Set(project.canonicalDomains.map((d) => d.toLowerCase()));
|
|
19571
|
+
const counts = /* @__PURE__ */ new Map();
|
|
19572
|
+
for (const probe of probes) {
|
|
19573
|
+
const seenInProbe = /* @__PURE__ */ new Set();
|
|
19574
|
+
for (const raw of probe.citedDomains) {
|
|
19575
|
+
const domain = raw.toLowerCase();
|
|
19576
|
+
if (canonical.has(domain)) continue;
|
|
19577
|
+
if (seenInProbe.has(domain)) continue;
|
|
19578
|
+
seenInProbe.add(domain);
|
|
19579
|
+
counts.set(domain, (counts.get(domain) ?? 0) + 1);
|
|
19580
|
+
}
|
|
19581
|
+
}
|
|
19582
|
+
return Array.from(counts.entries()).map(([domain, hits]) => ({ domain, hits })).sort((a, b) => b.hits - a.hits || a.domain.localeCompare(b.domain));
|
|
19583
|
+
}
|
|
19584
|
+
async function pickCanonicals(candidates, deps, dedupThreshold) {
|
|
19585
|
+
if (candidates.length === 0) return [];
|
|
19586
|
+
if (candidates.length === 1) return candidates;
|
|
19587
|
+
const vectors = await deps.embed(candidates);
|
|
19588
|
+
const clusters = clusterByCosine(candidates, vectors, dedupThreshold);
|
|
19589
|
+
return clusters.map(pickClusterRepresentative);
|
|
19590
|
+
}
|
|
19591
|
+
async function executeDiscovery(opts) {
|
|
19592
|
+
const dedupThreshold = opts.dedupThreshold ?? DEFAULT_DEDUP_THRESHOLD;
|
|
19593
|
+
const requestedMax = opts.maxProbes ?? DEFAULT_MAX_PROBES;
|
|
19594
|
+
const maxProbes = Math.min(Math.max(1, requestedMax), ABSOLUTE_MAX_PROBES);
|
|
19595
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19596
|
+
opts.db.update(discoverySessions).set({
|
|
19597
|
+
status: DiscoverySessionStatuses.seeding,
|
|
19598
|
+
dedupThreshold,
|
|
19599
|
+
startedAt
|
|
19600
|
+
}).where(eq26(discoverySessions.id, opts.sessionId)).run();
|
|
19601
|
+
const seedResult = await opts.deps.seed({
|
|
19602
|
+
project: opts.project,
|
|
19603
|
+
icpDescription: opts.icpDescription
|
|
19604
|
+
});
|
|
19605
|
+
const rawCandidates = dedupeStrings(seedResult.candidates);
|
|
19606
|
+
const seedCountRaw = rawCandidates.length;
|
|
19607
|
+
const canonicals = await pickCanonicals(
|
|
19608
|
+
rawCandidates,
|
|
19609
|
+
{ embed: opts.deps.embed },
|
|
19610
|
+
dedupThreshold
|
|
19611
|
+
);
|
|
19612
|
+
const probedCanonicals = canonicals.slice(0, maxProbes);
|
|
19613
|
+
const seedCount = probedCanonicals.length;
|
|
19614
|
+
opts.db.update(discoverySessions).set({
|
|
19615
|
+
status: DiscoverySessionStatuses.probing,
|
|
19616
|
+
seedProvider: seedResult.provider,
|
|
19617
|
+
seedCountRaw,
|
|
19618
|
+
seedCount
|
|
19619
|
+
}).where(eq26(discoverySessions.id, opts.sessionId)).run();
|
|
19620
|
+
const probeRows = [];
|
|
19621
|
+
const buckets = { cited: 0, aspirational: 0, "wasted-surface": 0 };
|
|
19622
|
+
for (const query of probedCanonicals) {
|
|
19623
|
+
const probe = await opts.deps.probe({ project: opts.project, query });
|
|
19624
|
+
const bucket = classifyProbeBucket({
|
|
19625
|
+
citationState: probe.citationState,
|
|
19626
|
+
citedDomains: probe.citedDomains,
|
|
19627
|
+
project: opts.project
|
|
19628
|
+
});
|
|
19629
|
+
probeRows.push({ citedDomains: probe.citedDomains, bucket });
|
|
19630
|
+
buckets[bucket]++;
|
|
19631
|
+
opts.db.insert(discoveryProbes).values({
|
|
19632
|
+
id: crypto22.randomUUID(),
|
|
19633
|
+
sessionId: opts.sessionId,
|
|
19634
|
+
projectId: opts.project.id,
|
|
19635
|
+
query,
|
|
19636
|
+
bucket,
|
|
19637
|
+
citationState: probe.citationState,
|
|
19638
|
+
citedDomains: JSON.stringify(probe.citedDomains),
|
|
19639
|
+
rawResponse: JSON.stringify(probe.rawResponse),
|
|
19640
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19641
|
+
}).run();
|
|
19642
|
+
}
|
|
19643
|
+
const competitorMap = buildCompetitorMap(probeRows, opts.project);
|
|
19644
|
+
opts.db.update(discoverySessions).set({
|
|
19645
|
+
status: DiscoverySessionStatuses.completed,
|
|
19646
|
+
probeCount: probedCanonicals.length,
|
|
19647
|
+
citedCount: buckets.cited,
|
|
19648
|
+
aspirationalCount: buckets.aspirational,
|
|
19649
|
+
wastedCount: buckets["wasted-surface"],
|
|
19650
|
+
competitorMap: JSON.stringify(competitorMap),
|
|
19651
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19652
|
+
}).where(eq26(discoverySessions.id, opts.sessionId)).run();
|
|
19653
|
+
return {
|
|
19654
|
+
buckets,
|
|
19655
|
+
competitorMap,
|
|
19656
|
+
seedCountRaw,
|
|
19657
|
+
seedCount,
|
|
19658
|
+
seedProvider: seedResult.provider
|
|
19659
|
+
};
|
|
19660
|
+
}
|
|
19661
|
+
function markSessionFailed(db, sessionId, error) {
|
|
19662
|
+
db.update(discoverySessions).set({
|
|
19663
|
+
status: DiscoverySessionStatuses.failed,
|
|
19664
|
+
error,
|
|
19665
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19666
|
+
}).where(eq26(discoverySessions.id, sessionId)).run();
|
|
19667
|
+
}
|
|
19668
|
+
function dedupeStrings(input) {
|
|
19669
|
+
const seen = /* @__PURE__ */ new Set();
|
|
19670
|
+
const out = [];
|
|
19671
|
+
for (const raw of input) {
|
|
19672
|
+
const trimmed = raw.trim();
|
|
19673
|
+
if (!trimmed) continue;
|
|
19674
|
+
const key = trimmed.toLowerCase();
|
|
19675
|
+
if (seen.has(key)) continue;
|
|
19676
|
+
seen.add(key);
|
|
19677
|
+
out.push(trimmed);
|
|
19678
|
+
}
|
|
19679
|
+
return out;
|
|
19680
|
+
}
|
|
19681
|
+
|
|
19291
19682
|
// ../api-routes/src/index.ts
|
|
19292
19683
|
async function apiRoutes(app, opts) {
|
|
19293
19684
|
app.decorate("db", opts.db);
|
|
@@ -19418,6 +19809,9 @@ async function apiRoutes(app, opts) {
|
|
|
19418
19809
|
listCachedReleases: opts.listCachedReleases,
|
|
19419
19810
|
discoverLatestRelease: opts.discoverLatestRelease
|
|
19420
19811
|
});
|
|
19812
|
+
await api.register(discoveryRoutes, {
|
|
19813
|
+
onDiscoveryRunRequested: opts.onDiscoveryRunRequested
|
|
19814
|
+
});
|
|
19421
19815
|
await api.register(doctorRoutes, {
|
|
19422
19816
|
googleConnectionStore: opts.googleConnectionStore,
|
|
19423
19817
|
bingConnectionStore: opts.bingConnectionStore,
|
|
@@ -19834,6 +20228,54 @@ function responseToRecord(response) {
|
|
|
19834
20228
|
}
|
|
19835
20229
|
}
|
|
19836
20230
|
|
|
20231
|
+
// ../provider-gemini/src/embeddings.ts
|
|
20232
|
+
import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
|
|
20233
|
+
var DEFAULT_EMBED_MODEL = "gemini-embedding-001";
|
|
20234
|
+
var DEFAULT_OUTPUT_DIMENSIONALITY = 768;
|
|
20235
|
+
var CLUSTERING_TASK_TYPE = "CLUSTERING";
|
|
20236
|
+
async function embedQueries(queries2, options) {
|
|
20237
|
+
if (queries2.length === 0) return [];
|
|
20238
|
+
if (!options.apiKey && !options.client) {
|
|
20239
|
+
throw new Error("embedQueries: missing apiKey");
|
|
20240
|
+
}
|
|
20241
|
+
const client = options.client ?? createGeminiEmbedClient(options.apiKey);
|
|
20242
|
+
return client.embedBatch(queries2, {
|
|
20243
|
+
model: options.model ?? DEFAULT_EMBED_MODEL,
|
|
20244
|
+
taskType: CLUSTERING_TASK_TYPE,
|
|
20245
|
+
outputDimensionality: options.outputDimensionality ?? DEFAULT_OUTPUT_DIMENSIONALITY
|
|
20246
|
+
});
|
|
20247
|
+
}
|
|
20248
|
+
function extractEmbeddingVectors(response, expectedLength) {
|
|
20249
|
+
const embeddings = response?.embeddings ?? [];
|
|
20250
|
+
if (embeddings.length !== expectedLength) {
|
|
20251
|
+
throw new Error(
|
|
20252
|
+
`embedQueries: expected ${expectedLength} embeddings, got ${embeddings.length}`
|
|
20253
|
+
);
|
|
20254
|
+
}
|
|
20255
|
+
return embeddings.map((e, i) => {
|
|
20256
|
+
if (!e.values || e.values.length === 0) {
|
|
20257
|
+
throw new Error(`embedQueries: missing values for query at index ${i}`);
|
|
20258
|
+
}
|
|
20259
|
+
return e.values;
|
|
20260
|
+
});
|
|
20261
|
+
}
|
|
20262
|
+
function createGeminiEmbedClient(apiKey) {
|
|
20263
|
+
const genai = new GoogleGenAI2({ apiKey });
|
|
20264
|
+
return {
|
|
20265
|
+
async embedBatch(queries2, opts) {
|
|
20266
|
+
const response = await genai.models.embedContent({
|
|
20267
|
+
model: opts.model,
|
|
20268
|
+
contents: queries2,
|
|
20269
|
+
config: {
|
|
20270
|
+
taskType: opts.taskType,
|
|
20271
|
+
outputDimensionality: opts.outputDimensionality
|
|
20272
|
+
}
|
|
20273
|
+
});
|
|
20274
|
+
return extractEmbeddingVectors(response, queries2.length);
|
|
20275
|
+
}
|
|
20276
|
+
};
|
|
20277
|
+
}
|
|
20278
|
+
|
|
19837
20279
|
// ../provider-gemini/src/adapter.ts
|
|
19838
20280
|
function toGeminiConfig(config) {
|
|
19839
20281
|
return {
|
|
@@ -22038,14 +22480,14 @@ function removeWordpressConnection(config, projectName) {
|
|
|
22038
22480
|
}
|
|
22039
22481
|
|
|
22040
22482
|
// src/job-runner.ts
|
|
22041
|
-
import
|
|
22483
|
+
import crypto24 from "crypto";
|
|
22042
22484
|
import fs7 from "fs";
|
|
22043
22485
|
import path9 from "path";
|
|
22044
22486
|
import os5 from "os";
|
|
22045
|
-
import { and as and16, eq as
|
|
22487
|
+
import { and as and16, eq as eq27, inArray as inArray7, sql as sql10 } from "drizzle-orm";
|
|
22046
22488
|
|
|
22047
22489
|
// src/run-telemetry.ts
|
|
22048
|
-
import
|
|
22490
|
+
import crypto23 from "crypto";
|
|
22049
22491
|
function extractRegistrableHost(input) {
|
|
22050
22492
|
if (!input) return null;
|
|
22051
22493
|
const trimmed = input.trim();
|
|
@@ -22065,7 +22507,7 @@ function extractRegistrableHost(input) {
|
|
|
22065
22507
|
function hashDomain(input) {
|
|
22066
22508
|
const host = extractRegistrableHost(input);
|
|
22067
22509
|
if (!host) return null;
|
|
22068
|
-
return
|
|
22510
|
+
return crypto23.createHash("sha256").update(host).digest("hex");
|
|
22069
22511
|
}
|
|
22070
22512
|
function buildRunCompletedProps(input) {
|
|
22071
22513
|
const totalMs = input.phases?.total_ms ?? Date.now() - input.startTime;
|
|
@@ -22387,7 +22829,7 @@ var JobRunner = class {
|
|
|
22387
22829
|
if (stale.length === 0) return;
|
|
22388
22830
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22389
22831
|
for (const run of stale) {
|
|
22390
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
22832
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq27(runs.id, run.id)).run();
|
|
22391
22833
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
22392
22834
|
}
|
|
22393
22835
|
}
|
|
@@ -22421,10 +22863,10 @@ var JobRunner = class {
|
|
|
22421
22863
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
22422
22864
|
}
|
|
22423
22865
|
if (existingRun.status === "queued") {
|
|
22424
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(and16(
|
|
22866
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and16(eq27(runs.id, runId), eq27(runs.status, "queued"))).run();
|
|
22425
22867
|
}
|
|
22426
22868
|
this.throwIfRunCancelled(runId);
|
|
22427
|
-
const project = this.db.select().from(projects).where(
|
|
22869
|
+
const project = this.db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
22428
22870
|
if (!project) {
|
|
22429
22871
|
throw new Error(`Project ${projectId} not found`);
|
|
22430
22872
|
}
|
|
@@ -22445,8 +22887,8 @@ var JobRunner = class {
|
|
|
22445
22887
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
22446
22888
|
}
|
|
22447
22889
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
22448
|
-
projectQueries = this.db.select().from(queries).where(
|
|
22449
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
22890
|
+
projectQueries = this.db.select().from(queries).where(eq27(queries.projectId, projectId)).all();
|
|
22891
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq27(competitors.projectId, projectId)).all();
|
|
22450
22892
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
22451
22893
|
const allDomains = effectiveDomains({
|
|
22452
22894
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -22464,7 +22906,7 @@ var JobRunner = class {
|
|
|
22464
22906
|
const todayPeriod = getCurrentUsageDay();
|
|
22465
22907
|
for (const p of activeProviders) {
|
|
22466
22908
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
22467
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
22909
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq27(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
22468
22910
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
22469
22911
|
if (providerUsage + queriesPerProvider > limit) {
|
|
22470
22912
|
throw new Error(
|
|
@@ -22524,7 +22966,7 @@ var JobRunner = class {
|
|
|
22524
22966
|
);
|
|
22525
22967
|
let screenshotRelPath = null;
|
|
22526
22968
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
22527
|
-
const snapshotId =
|
|
22969
|
+
const snapshotId = crypto24.randomUUID();
|
|
22528
22970
|
const screenshotDir = path9.join(os5.homedir(), ".canonry", "screenshots", runId);
|
|
22529
22971
|
if (!fs7.existsSync(screenshotDir)) fs7.mkdirSync(screenshotDir, { recursive: true });
|
|
22530
22972
|
const destPath = path9.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -22554,7 +22996,7 @@ var JobRunner = class {
|
|
|
22554
22996
|
}).run();
|
|
22555
22997
|
} else {
|
|
22556
22998
|
this.db.insert(querySnapshots).values({
|
|
22557
|
-
id:
|
|
22999
|
+
id: crypto24.randomUUID(),
|
|
22558
23000
|
runId,
|
|
22559
23001
|
queryId: q.id,
|
|
22560
23002
|
provider: providerName,
|
|
@@ -22607,12 +23049,12 @@ var JobRunner = class {
|
|
|
22607
23049
|
const someFailed = providerErrors.size > 0;
|
|
22608
23050
|
if (allFailed) {
|
|
22609
23051
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
22610
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
23052
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq27(runs.id, runId)).run();
|
|
22611
23053
|
} else if (someFailed) {
|
|
22612
23054
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
22613
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
23055
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq27(runs.id, runId)).run();
|
|
22614
23056
|
} else {
|
|
22615
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23057
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
22616
23058
|
}
|
|
22617
23059
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
22618
23060
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -22658,7 +23100,7 @@ var JobRunner = class {
|
|
|
22658
23100
|
status: "failed",
|
|
22659
23101
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22660
23102
|
error: errorMessage
|
|
22661
|
-
}).where(
|
|
23103
|
+
}).where(eq27(runs.id, runId)).run();
|
|
22662
23104
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
22663
23105
|
const abortReason = classifyRunAbortReason(errorMessage);
|
|
22664
23106
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
@@ -22703,7 +23145,7 @@ var JobRunner = class {
|
|
|
22703
23145
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22704
23146
|
const period = now.slice(0, 10);
|
|
22705
23147
|
this.db.insert(usageCounters).values({
|
|
22706
|
-
id:
|
|
23148
|
+
id: crypto24.randomUUID(),
|
|
22707
23149
|
scope,
|
|
22708
23150
|
period,
|
|
22709
23151
|
metric,
|
|
@@ -22726,7 +23168,7 @@ var JobRunner = class {
|
|
|
22726
23168
|
finishedAt: runs.finishedAt,
|
|
22727
23169
|
error: runs.error,
|
|
22728
23170
|
trigger: runs.trigger
|
|
22729
|
-
}).from(runs).where(
|
|
23171
|
+
}).from(runs).where(eq27(runs.id, runId)).get();
|
|
22730
23172
|
}
|
|
22731
23173
|
isRunCancelled(runId) {
|
|
22732
23174
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -22742,7 +23184,7 @@ var JobRunner = class {
|
|
|
22742
23184
|
this.db.update(runs).set({
|
|
22743
23185
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22744
23186
|
error: currentRun.error ?? "Cancelled by user"
|
|
22745
|
-
}).where(
|
|
23187
|
+
}).where(eq27(runs.id, runId)).run();
|
|
22746
23188
|
}
|
|
22747
23189
|
trackEvent(
|
|
22748
23190
|
"run.completed",
|
|
@@ -22779,8 +23221,8 @@ function buildPhases(input) {
|
|
|
22779
23221
|
}
|
|
22780
23222
|
|
|
22781
23223
|
// src/gsc-sync.ts
|
|
22782
|
-
import
|
|
22783
|
-
import { eq as
|
|
23224
|
+
import crypto25 from "crypto";
|
|
23225
|
+
import { eq as eq28, and as and17, sql as sql11 } from "drizzle-orm";
|
|
22784
23226
|
var log2 = createLogger("GscSync");
|
|
22785
23227
|
function formatDate3(d) {
|
|
22786
23228
|
return d.toISOString().split("T")[0];
|
|
@@ -22792,13 +23234,13 @@ function daysAgo(n) {
|
|
|
22792
23234
|
}
|
|
22793
23235
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
22794
23236
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22795
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
23237
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq28(runs.id, runId)).run();
|
|
22796
23238
|
try {
|
|
22797
23239
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
22798
23240
|
if (!googleClientId || !googleClientSecret) {
|
|
22799
23241
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
22800
23242
|
}
|
|
22801
|
-
const project = db.select().from(projects).where(
|
|
23243
|
+
const project = db.select().from(projects).where(eq28(projects.id, projectId)).get();
|
|
22802
23244
|
if (!project) {
|
|
22803
23245
|
throw new Error(`Project not found: ${projectId}`);
|
|
22804
23246
|
}
|
|
@@ -22833,7 +23275,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22833
23275
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
22834
23276
|
db.delete(gscSearchData).where(
|
|
22835
23277
|
and17(
|
|
22836
|
-
|
|
23278
|
+
eq28(gscSearchData.projectId, projectId),
|
|
22837
23279
|
sql11`${gscSearchData.date} >= ${startDate}`,
|
|
22838
23280
|
sql11`${gscSearchData.date} <= ${endDate}`
|
|
22839
23281
|
)
|
|
@@ -22845,7 +23287,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22845
23287
|
for (const row of batch) {
|
|
22846
23288
|
const [query, page, country, device, date] = row.keys;
|
|
22847
23289
|
db.insert(gscSearchData).values({
|
|
22848
|
-
id:
|
|
23290
|
+
id: crypto25.randomUUID(),
|
|
22849
23291
|
projectId,
|
|
22850
23292
|
syncRunId: runId,
|
|
22851
23293
|
date: date ?? "",
|
|
@@ -22879,7 +23321,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22879
23321
|
const rich = ir.richResultsResult;
|
|
22880
23322
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
22881
23323
|
db.insert(gscUrlInspections).values({
|
|
22882
|
-
id:
|
|
23324
|
+
id: crypto25.randomUUID(),
|
|
22883
23325
|
projectId,
|
|
22884
23326
|
syncRunId: runId,
|
|
22885
23327
|
url: pageUrl,
|
|
@@ -22900,7 +23342,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22900
23342
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
22901
23343
|
}
|
|
22902
23344
|
}
|
|
22903
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
23345
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq28(gscUrlInspections.projectId, projectId)).all();
|
|
22904
23346
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
22905
23347
|
for (const row of allInspections) {
|
|
22906
23348
|
const existing = latestByUrl.get(row.url);
|
|
@@ -22921,9 +23363,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22921
23363
|
}
|
|
22922
23364
|
}
|
|
22923
23365
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
22924
|
-
db.delete(gscCoverageSnapshots).where(and17(
|
|
23366
|
+
db.delete(gscCoverageSnapshots).where(and17(eq28(gscCoverageSnapshots.projectId, projectId), eq28(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
22925
23367
|
db.insert(gscCoverageSnapshots).values({
|
|
22926
|
-
id:
|
|
23368
|
+
id: crypto25.randomUUID(),
|
|
22927
23369
|
projectId,
|
|
22928
23370
|
syncRunId: runId,
|
|
22929
23371
|
date: snapshotDate,
|
|
@@ -22932,19 +23374,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22932
23374
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
22933
23375
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22934
23376
|
}).run();
|
|
22935
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23377
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
22936
23378
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
22937
23379
|
} catch (err) {
|
|
22938
23380
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
22939
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23381
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
22940
23382
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
22941
23383
|
throw err;
|
|
22942
23384
|
}
|
|
22943
23385
|
}
|
|
22944
23386
|
|
|
22945
23387
|
// src/gsc-inspect-sitemap.ts
|
|
22946
|
-
import
|
|
22947
|
-
import { eq as
|
|
23388
|
+
import crypto26 from "crypto";
|
|
23389
|
+
import { eq as eq29, and as and18 } from "drizzle-orm";
|
|
22948
23390
|
|
|
22949
23391
|
// src/sitemap-parser.ts
|
|
22950
23392
|
var log3 = createLogger("SitemapParser");
|
|
@@ -23065,13 +23507,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
23065
23507
|
var log4 = createLogger("InspectSitemap");
|
|
23066
23508
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
23067
23509
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23068
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
23510
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq29(runs.id, runId)).run();
|
|
23069
23511
|
try {
|
|
23070
23512
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
23071
23513
|
if (!googleClientId || !googleClientSecret) {
|
|
23072
23514
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
23073
23515
|
}
|
|
23074
|
-
const project = db.select().from(projects).where(
|
|
23516
|
+
const project = db.select().from(projects).where(eq29(projects.id, projectId)).get();
|
|
23075
23517
|
if (!project) {
|
|
23076
23518
|
throw new Error(`Project not found: ${projectId}`);
|
|
23077
23519
|
}
|
|
@@ -23112,7 +23554,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23112
23554
|
const rich = ir.richResultsResult;
|
|
23113
23555
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23114
23556
|
db.insert(gscUrlInspections).values({
|
|
23115
|
-
id:
|
|
23557
|
+
id: crypto26.randomUUID(),
|
|
23116
23558
|
projectId,
|
|
23117
23559
|
syncRunId: runId,
|
|
23118
23560
|
url: pageUrl,
|
|
@@ -23139,7 +23581,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23139
23581
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
23140
23582
|
}
|
|
23141
23583
|
}
|
|
23142
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
23584
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq29(gscUrlInspections.projectId, projectId)).all();
|
|
23143
23585
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
23144
23586
|
for (const row of allInspections) {
|
|
23145
23587
|
const existing = latestByUrl.get(row.url);
|
|
@@ -23160,9 +23602,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23160
23602
|
}
|
|
23161
23603
|
}
|
|
23162
23604
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
23163
|
-
db.delete(gscCoverageSnapshots).where(and18(
|
|
23605
|
+
db.delete(gscCoverageSnapshots).where(and18(eq29(gscCoverageSnapshots.projectId, projectId), eq29(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
23164
23606
|
db.insert(gscCoverageSnapshots).values({
|
|
23165
|
-
id:
|
|
23607
|
+
id: crypto26.randomUUID(),
|
|
23166
23608
|
projectId,
|
|
23167
23609
|
syncRunId: runId,
|
|
23168
23610
|
date: snapshotDate,
|
|
@@ -23172,19 +23614,19 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23172
23614
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23173
23615
|
}).run();
|
|
23174
23616
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
23175
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23617
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(runs.id, runId)).run();
|
|
23176
23618
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
23177
23619
|
} catch (err) {
|
|
23178
23620
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
23179
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23621
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(runs.id, runId)).run();
|
|
23180
23622
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
23181
23623
|
throw err;
|
|
23182
23624
|
}
|
|
23183
23625
|
}
|
|
23184
23626
|
|
|
23185
23627
|
// src/bing-inspect-sitemap.ts
|
|
23186
|
-
import
|
|
23187
|
-
import { eq as
|
|
23628
|
+
import crypto27 from "crypto";
|
|
23629
|
+
import { eq as eq30, desc as desc14 } from "drizzle-orm";
|
|
23188
23630
|
var log5 = createLogger("BingInspectSitemap");
|
|
23189
23631
|
function parseBingDate2(value) {
|
|
23190
23632
|
if (!value) return null;
|
|
@@ -23202,9 +23644,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
23202
23644
|
}
|
|
23203
23645
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
23204
23646
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23205
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
23647
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq30(runs.id, runId)).run();
|
|
23206
23648
|
try {
|
|
23207
|
-
const project = db.select().from(projects).where(
|
|
23649
|
+
const project = db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
23208
23650
|
if (!project) {
|
|
23209
23651
|
throw new Error(`Project not found: ${projectId}`);
|
|
23210
23652
|
}
|
|
@@ -23222,7 +23664,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23222
23664
|
if (sitemapUrls.length === 0) {
|
|
23223
23665
|
throw new Error("No URLs found in sitemap");
|
|
23224
23666
|
}
|
|
23225
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
23667
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq30(bingUrlInspections.projectId, projectId)).all();
|
|
23226
23668
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
23227
23669
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
23228
23670
|
log5.info("sitemap.diff", {
|
|
@@ -23271,7 +23713,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23271
23713
|
derivedInIndex = false;
|
|
23272
23714
|
}
|
|
23273
23715
|
db.insert(bingUrlInspections).values({
|
|
23274
|
-
id:
|
|
23716
|
+
id: crypto27.randomUUID(),
|
|
23275
23717
|
projectId,
|
|
23276
23718
|
url: pageUrl,
|
|
23277
23719
|
httpCode,
|
|
@@ -23305,7 +23747,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23305
23747
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
23306
23748
|
}
|
|
23307
23749
|
}
|
|
23308
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
23750
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq30(bingUrlInspections.projectId, projectId)).orderBy(desc14(bingUrlInspections.inspectedAt)).all();
|
|
23309
23751
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
23310
23752
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
23311
23753
|
for (const row of allInspections) {
|
|
@@ -23329,7 +23771,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23329
23771
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
23330
23772
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
23331
23773
|
db.insert(bingCoverageSnapshots).values({
|
|
23332
|
-
id:
|
|
23774
|
+
id: crypto27.randomUUID(),
|
|
23333
23775
|
projectId,
|
|
23334
23776
|
syncRunId: runId,
|
|
23335
23777
|
date: snapshotDate,
|
|
@@ -23348,7 +23790,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23348
23790
|
}
|
|
23349
23791
|
}).run();
|
|
23350
23792
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
23351
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23793
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq30(runs.id, runId)).run();
|
|
23352
23794
|
log5.info("inspect.completed", {
|
|
23353
23795
|
runId,
|
|
23354
23796
|
projectId,
|
|
@@ -23362,16 +23804,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23362
23804
|
});
|
|
23363
23805
|
} catch (err) {
|
|
23364
23806
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
23365
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23807
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq30(runs.id, runId)).run();
|
|
23366
23808
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
23367
23809
|
throw err;
|
|
23368
23810
|
}
|
|
23369
23811
|
}
|
|
23370
23812
|
|
|
23371
23813
|
// src/commoncrawl-sync.ts
|
|
23372
|
-
import
|
|
23814
|
+
import crypto28 from "crypto";
|
|
23373
23815
|
import path10 from "path";
|
|
23374
|
-
import { and as and19, eq as
|
|
23816
|
+
import { and as and19, eq as eq31, sql as sql12 } from "drizzle-orm";
|
|
23375
23817
|
var log6 = createLogger("CommonCrawlSync");
|
|
23376
23818
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
23377
23819
|
function defaultDeps() {
|
|
@@ -23397,7 +23839,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23397
23839
|
phaseDetail: "downloading vertices + edges",
|
|
23398
23840
|
updatedAt: downloadStartedAt,
|
|
23399
23841
|
error: null
|
|
23400
|
-
}).where(
|
|
23842
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23401
23843
|
const paths = ccReleasePaths(release);
|
|
23402
23844
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
23403
23845
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -23420,7 +23862,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23420
23862
|
vertexSha256: vertex.sha256,
|
|
23421
23863
|
edgesSha256: edges.sha256,
|
|
23422
23864
|
updatedAt: downloadFinishedAt
|
|
23423
|
-
}).where(
|
|
23865
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23424
23866
|
const allProjects = db.select().from(projects).all();
|
|
23425
23867
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
23426
23868
|
let rows = [];
|
|
@@ -23436,15 +23878,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23436
23878
|
}
|
|
23437
23879
|
const queriedAt = deps.now().toISOString();
|
|
23438
23880
|
db.transaction((tx) => {
|
|
23439
|
-
tx.delete(backlinkDomains).where(
|
|
23440
|
-
tx.delete(backlinkSummaries).where(
|
|
23881
|
+
tx.delete(backlinkDomains).where(eq31(backlinkDomains.releaseSyncId, syncId)).run();
|
|
23882
|
+
tx.delete(backlinkSummaries).where(eq31(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
23441
23883
|
const expanded = [];
|
|
23442
23884
|
for (const r of rows) {
|
|
23443
23885
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
23444
23886
|
if (!projectIds) continue;
|
|
23445
23887
|
for (const projectId of projectIds) {
|
|
23446
23888
|
expanded.push({
|
|
23447
|
-
id:
|
|
23889
|
+
id: crypto28.randomUUID(),
|
|
23448
23890
|
projectId,
|
|
23449
23891
|
releaseSyncId: syncId,
|
|
23450
23892
|
release,
|
|
@@ -23464,7 +23906,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23464
23906
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
23465
23907
|
const summary = computeSummary(projectRows);
|
|
23466
23908
|
tx.insert(backlinkSummaries).values({
|
|
23467
|
-
id:
|
|
23909
|
+
id: crypto28.randomUUID(),
|
|
23468
23910
|
projectId: p.id,
|
|
23469
23911
|
releaseSyncId: syncId,
|
|
23470
23912
|
release,
|
|
@@ -23496,7 +23938,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23496
23938
|
domainsDiscovered: rows.length,
|
|
23497
23939
|
updatedAt: finishedAt,
|
|
23498
23940
|
error: null
|
|
23499
|
-
}).where(
|
|
23941
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23500
23942
|
log6.info("sync.completed", {
|
|
23501
23943
|
syncId,
|
|
23502
23944
|
release,
|
|
@@ -23526,7 +23968,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23526
23968
|
error: errorMsg,
|
|
23527
23969
|
phaseDetail: null,
|
|
23528
23970
|
updatedAt: finishedAt
|
|
23529
|
-
}).where(
|
|
23971
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23530
23972
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
23531
23973
|
throw err;
|
|
23532
23974
|
}
|
|
@@ -23560,9 +24002,9 @@ function computeSummary(rows) {
|
|
|
23560
24002
|
}
|
|
23561
24003
|
|
|
23562
24004
|
// src/backlink-extract.ts
|
|
23563
|
-
import
|
|
24005
|
+
import crypto29 from "crypto";
|
|
23564
24006
|
import fs8 from "fs";
|
|
23565
|
-
import { and as and20, desc as
|
|
24007
|
+
import { and as and20, desc as desc15, eq as eq32 } from "drizzle-orm";
|
|
23566
24008
|
var log7 = createLogger("BacklinkExtract");
|
|
23567
24009
|
function defaultDeps2() {
|
|
23568
24010
|
return {
|
|
@@ -23574,13 +24016,13 @@ function defaultDeps2() {
|
|
|
23574
24016
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
23575
24017
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
23576
24018
|
const startedAt = deps.now().toISOString();
|
|
23577
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
24019
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq32(runs.id, runId)).run();
|
|
23578
24020
|
try {
|
|
23579
|
-
const project = db.select().from(projects).where(
|
|
24021
|
+
const project = db.select().from(projects).where(eq32(projects.id, projectId)).get();
|
|
23580
24022
|
if (!project) {
|
|
23581
24023
|
throw new Error(`Project not found: ${projectId}`);
|
|
23582
24024
|
}
|
|
23583
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
24025
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq32(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq32(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc15(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
23584
24026
|
if (!sync) {
|
|
23585
24027
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
23586
24028
|
}
|
|
@@ -23608,11 +24050,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23608
24050
|
const targetDomain = project.canonicalDomain;
|
|
23609
24051
|
db.transaction((tx) => {
|
|
23610
24052
|
tx.delete(backlinkDomains).where(
|
|
23611
|
-
and20(
|
|
24053
|
+
and20(eq32(backlinkDomains.projectId, projectId), eq32(backlinkDomains.release, release))
|
|
23612
24054
|
).run();
|
|
23613
24055
|
if (rows.length > 0) {
|
|
23614
24056
|
const values = rows.map((r) => ({
|
|
23615
|
-
id:
|
|
24057
|
+
id: crypto29.randomUUID(),
|
|
23616
24058
|
projectId,
|
|
23617
24059
|
releaseSyncId: syncId,
|
|
23618
24060
|
release,
|
|
@@ -23625,7 +24067,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23625
24067
|
}
|
|
23626
24068
|
const summary = computeSummary2(rows);
|
|
23627
24069
|
tx.insert(backlinkSummaries).values({
|
|
23628
|
-
id:
|
|
24070
|
+
id: crypto29.randomUUID(),
|
|
23629
24071
|
projectId,
|
|
23630
24072
|
releaseSyncId: syncId,
|
|
23631
24073
|
release,
|
|
@@ -23648,7 +24090,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23648
24090
|
}).run();
|
|
23649
24091
|
});
|
|
23650
24092
|
const finishedAt = deps.now().toISOString();
|
|
23651
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
24093
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq32(runs.id, runId)).run();
|
|
23652
24094
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
23653
24095
|
} catch (err) {
|
|
23654
24096
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -23657,7 +24099,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23657
24099
|
status: RunStatuses.failed,
|
|
23658
24100
|
error: errorMsg,
|
|
23659
24101
|
finishedAt
|
|
23660
|
-
}).where(
|
|
24102
|
+
}).where(eq32(runs.id, runId)).run();
|
|
23661
24103
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
23662
24104
|
throw err;
|
|
23663
24105
|
}
|
|
@@ -23677,6 +24119,205 @@ function computeSummary2(rows) {
|
|
|
23677
24119
|
};
|
|
23678
24120
|
}
|
|
23679
24121
|
|
|
24122
|
+
// src/discovery-run.ts
|
|
24123
|
+
import crypto30 from "crypto";
|
|
24124
|
+
import { eq as eq33 } from "drizzle-orm";
|
|
24125
|
+
var log8 = createLogger("DiscoveryRun");
|
|
24126
|
+
var DEFAULT_SEED_COUNT = 30;
|
|
24127
|
+
async function executeDiscoveryRun(opts) {
|
|
24128
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
24129
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq33(runs.id, opts.runId)).run();
|
|
24130
|
+
try {
|
|
24131
|
+
const projectRow = opts.db.select().from(projects).where(eq33(projects.id, opts.projectId)).get();
|
|
24132
|
+
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
24133
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq33(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
24134
|
+
const canonicalDomains = effectiveDomains({
|
|
24135
|
+
canonicalDomain: projectRow.canonicalDomain,
|
|
24136
|
+
ownedDomains: parseJsonColumn(projectRow.ownedDomains, [])
|
|
24137
|
+
});
|
|
24138
|
+
const project = {
|
|
24139
|
+
id: projectRow.id,
|
|
24140
|
+
name: projectRow.name,
|
|
24141
|
+
canonicalDomains,
|
|
24142
|
+
competitorDomains: projectCompetitors
|
|
24143
|
+
};
|
|
24144
|
+
const deps = opts.deps ?? buildDefaultDeps(opts.registry);
|
|
24145
|
+
const result = await executeDiscovery({
|
|
24146
|
+
db: opts.db,
|
|
24147
|
+
runId: opts.runId,
|
|
24148
|
+
sessionId: opts.sessionId,
|
|
24149
|
+
project,
|
|
24150
|
+
icpDescription: opts.icpDescription,
|
|
24151
|
+
dedupThreshold: opts.dedupThreshold,
|
|
24152
|
+
maxProbes: opts.maxProbes,
|
|
24153
|
+
deps
|
|
24154
|
+
});
|
|
24155
|
+
writeDiscoveryInsight(opts.db, {
|
|
24156
|
+
projectId: opts.projectId,
|
|
24157
|
+
runId: opts.runId,
|
|
24158
|
+
sessionId: opts.sessionId,
|
|
24159
|
+
seedProvider: result.seedProvider,
|
|
24160
|
+
result
|
|
24161
|
+
});
|
|
24162
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq33(runs.id, opts.runId)).run();
|
|
24163
|
+
log8.info("discovery.completed", {
|
|
24164
|
+
runId: opts.runId,
|
|
24165
|
+
sessionId: opts.sessionId,
|
|
24166
|
+
buckets: result.buckets,
|
|
24167
|
+
competitorCount: result.competitorMap.length
|
|
24168
|
+
});
|
|
24169
|
+
} catch (err) {
|
|
24170
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
24171
|
+
log8.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
24172
|
+
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
24173
|
+
opts.db.update(runs).set({
|
|
24174
|
+
status: RunStatuses.failed,
|
|
24175
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24176
|
+
error: errorMsg
|
|
24177
|
+
}).where(eq33(runs.id, opts.runId)).run();
|
|
24178
|
+
}
|
|
24179
|
+
}
|
|
24180
|
+
function buildDefaultDeps(registry) {
|
|
24181
|
+
const gemini = registry.get("gemini");
|
|
24182
|
+
if (!gemini) {
|
|
24183
|
+
throw new Error("Gemini provider is not configured. Add a Gemini API key (or Vertex project) before running discovery.");
|
|
24184
|
+
}
|
|
24185
|
+
const cfg = gemini.config;
|
|
24186
|
+
if (!cfg.apiKey && !cfg.vertexProject) {
|
|
24187
|
+
throw new Error("Gemini provider is missing both apiKey and vertexProject \u2014 cannot run discovery.");
|
|
24188
|
+
}
|
|
24189
|
+
const adapter = gemini.adapter;
|
|
24190
|
+
return {
|
|
24191
|
+
async seed(input) {
|
|
24192
|
+
const prompt = buildSeedPrompt(input);
|
|
24193
|
+
const raw = await adapter.executeTrackedQuery(
|
|
24194
|
+
{
|
|
24195
|
+
query: prompt,
|
|
24196
|
+
canonicalDomains: input.project.canonicalDomains,
|
|
24197
|
+
competitorDomains: input.project.competitorDomains
|
|
24198
|
+
},
|
|
24199
|
+
cfg
|
|
24200
|
+
);
|
|
24201
|
+
const normalized = adapter.normalizeResult(raw);
|
|
24202
|
+
const fromAnswer = parseQueryLines(normalized.answerText, DEFAULT_SEED_COUNT * 2);
|
|
24203
|
+
const fromGrounding = normalized.searchQueries ?? [];
|
|
24204
|
+
return {
|
|
24205
|
+
candidates: [...fromAnswer, ...fromGrounding],
|
|
24206
|
+
provider: "gemini"
|
|
24207
|
+
};
|
|
24208
|
+
},
|
|
24209
|
+
async embed(queries2) {
|
|
24210
|
+
if (cfg.apiKey) {
|
|
24211
|
+
return embedQueries(queries2, { apiKey: cfg.apiKey });
|
|
24212
|
+
}
|
|
24213
|
+
throw new Error("Discovery currently requires a Gemini API key. Vertex-mode embeddings are not yet implemented.");
|
|
24214
|
+
},
|
|
24215
|
+
async probe(input) {
|
|
24216
|
+
const raw = await adapter.executeTrackedQuery(
|
|
24217
|
+
{
|
|
24218
|
+
query: input.query,
|
|
24219
|
+
canonicalDomains: input.project.canonicalDomains,
|
|
24220
|
+
competitorDomains: input.project.competitorDomains
|
|
24221
|
+
},
|
|
24222
|
+
cfg
|
|
24223
|
+
);
|
|
24224
|
+
const normalized = adapter.normalizeResult(raw);
|
|
24225
|
+
const canonical = new Set(input.project.canonicalDomains.map((d) => d.toLowerCase()));
|
|
24226
|
+
const isCited = normalized.citedDomains.some((d) => canonical.has(d.toLowerCase()));
|
|
24227
|
+
return {
|
|
24228
|
+
citationState: isCited ? "cited" : "not-cited",
|
|
24229
|
+
citedDomains: normalized.citedDomains,
|
|
24230
|
+
rawResponse: raw.rawResponse
|
|
24231
|
+
};
|
|
24232
|
+
}
|
|
24233
|
+
};
|
|
24234
|
+
}
|
|
24235
|
+
function buildSeedPrompt(input) {
|
|
24236
|
+
return [
|
|
24237
|
+
"You are an AEO (Answer Engine Optimization) analyst expanding a tracked-query basket for a customer.",
|
|
24238
|
+
"",
|
|
24239
|
+
`Customer: ${input.project.name} (domains: ${input.project.canonicalDomains.join(", ")})`,
|
|
24240
|
+
`ICP: ${input.icpDescription}`,
|
|
24241
|
+
"",
|
|
24242
|
+
"Brainstorm a wide set of queries a member of this ICP would type into an AI answer engine (Gemini, ChatGPT, Perplexity) when they are about to make a decision in this space. Aim for 30+ candidates covering:",
|
|
24243
|
+
' - Comparison queries ("best X for Y")',
|
|
24244
|
+
" - Specific feature / capability queries",
|
|
24245
|
+
" - Pricing / vendor-shortlist queries",
|
|
24246
|
+
" - Workflow / how-to queries",
|
|
24247
|
+
" - Adjacent jobs-to-be-done queries",
|
|
24248
|
+
"",
|
|
24249
|
+
"Return ONE query per line. Plain text only \u2014 no numbering, bullets, quotes, or commentary."
|
|
24250
|
+
].join("\n");
|
|
24251
|
+
}
|
|
24252
|
+
function parseQueryLines(text, max) {
|
|
24253
|
+
const lines = text.split("\n");
|
|
24254
|
+
const out = [];
|
|
24255
|
+
const seen = /* @__PURE__ */ new Set();
|
|
24256
|
+
for (const raw of lines) {
|
|
24257
|
+
let line = raw.trim();
|
|
24258
|
+
if (!line) continue;
|
|
24259
|
+
line = line.replace(/^\s*(?:\d+[.)]\s*|[-*•]\s*)/, "").replace(/^["']|["']$/g, "").trim();
|
|
24260
|
+
if (!line) continue;
|
|
24261
|
+
if (/^(here are|sure|certainly|of course|i['']ve|these are|below are)/i.test(line)) continue;
|
|
24262
|
+
const key = line.toLowerCase();
|
|
24263
|
+
if (seen.has(key)) continue;
|
|
24264
|
+
seen.add(key);
|
|
24265
|
+
out.push(line);
|
|
24266
|
+
if (out.length >= max) break;
|
|
24267
|
+
}
|
|
24268
|
+
return out;
|
|
24269
|
+
}
|
|
24270
|
+
function writeDiscoveryInsight(db, input) {
|
|
24271
|
+
const { buckets, competitorMap } = input.result;
|
|
24272
|
+
const totalProbes = buckets.cited + buckets.aspirational + buckets["wasted-surface"];
|
|
24273
|
+
if (totalProbes === 0) return;
|
|
24274
|
+
const wastedRatio = buckets["wasted-surface"] / totalProbes;
|
|
24275
|
+
const citedRatio = buckets.cited / totalProbes;
|
|
24276
|
+
const severity = wastedRatio >= 0.4 || buckets["wasted-surface"] > buckets.cited && wastedRatio >= 0.2 ? "high" : citedRatio >= 0.6 ? "low" : "medium";
|
|
24277
|
+
const topCompetitors = competitorMap.slice(0, 5);
|
|
24278
|
+
const title = buildDiscoveryInsightTitle({
|
|
24279
|
+
cited: buckets.cited,
|
|
24280
|
+
wasted: buckets["wasted-surface"],
|
|
24281
|
+
aspirational: buckets.aspirational,
|
|
24282
|
+
totalProbes
|
|
24283
|
+
});
|
|
24284
|
+
db.insert(insights).values({
|
|
24285
|
+
id: crypto30.randomUUID(),
|
|
24286
|
+
projectId: input.projectId,
|
|
24287
|
+
runId: input.runId,
|
|
24288
|
+
type: "discovery.basket-divergence",
|
|
24289
|
+
severity,
|
|
24290
|
+
title,
|
|
24291
|
+
// query/provider fields don't fit the visibility-snapshot model for a
|
|
24292
|
+
// session-level insight. Use the session marker so the
|
|
24293
|
+
// (query, provider) index stays distinct across sessions; PR 5 will
|
|
24294
|
+
// formalize a session-scoped insight subtype.
|
|
24295
|
+
query: `discovery:${input.sessionId}`,
|
|
24296
|
+
provider: input.seedProvider,
|
|
24297
|
+
recommendation: JSON.stringify({
|
|
24298
|
+
action: "review-discovered-basket",
|
|
24299
|
+
summary: `Run \`canonry discover show ${input.sessionId} --format json\` to inspect the per-query breakdown. PR 2 will add \`canonry discover promote\` to merge the basket into the project.`,
|
|
24300
|
+
bucketCounts: buckets,
|
|
24301
|
+
topCompetitors
|
|
24302
|
+
}),
|
|
24303
|
+
cause: JSON.stringify({
|
|
24304
|
+
sessionId: input.sessionId,
|
|
24305
|
+
totalProbes,
|
|
24306
|
+
seedProvider: input.seedProvider
|
|
24307
|
+
}),
|
|
24308
|
+
dismissed: false,
|
|
24309
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24310
|
+
}).run();
|
|
24311
|
+
}
|
|
24312
|
+
function buildDiscoveryInsightTitle(input) {
|
|
24313
|
+
const parts = [];
|
|
24314
|
+
parts.push(`Discovery probed ${input.totalProbes} representative queries`);
|
|
24315
|
+
if (input.wasted > 0) parts.push(`${input.wasted} where competitors are cited but you are not`);
|
|
24316
|
+
if (input.cited > 0) parts.push(`${input.cited} where you are cited`);
|
|
24317
|
+
if (input.aspirational > 0) parts.push(`${input.aspirational} aspirational greenfield queries`);
|
|
24318
|
+
return parts.join(" \u2022 ");
|
|
24319
|
+
}
|
|
24320
|
+
|
|
23680
24321
|
// src/provider-registry.ts
|
|
23681
24322
|
var ProviderRegistry = class {
|
|
23682
24323
|
providers = /* @__PURE__ */ new Map();
|
|
@@ -23730,8 +24371,8 @@ var ProviderRegistry = class {
|
|
|
23730
24371
|
|
|
23731
24372
|
// src/scheduler.ts
|
|
23732
24373
|
import cron from "node-cron";
|
|
23733
|
-
import { and as and21, eq as
|
|
23734
|
-
var
|
|
24374
|
+
import { and as and21, eq as eq34 } from "drizzle-orm";
|
|
24375
|
+
var log9 = createLogger("Scheduler");
|
|
23735
24376
|
function taskKey(projectId, kind) {
|
|
23736
24377
|
return `${projectId}::${kind}`;
|
|
23737
24378
|
}
|
|
@@ -23745,16 +24386,16 @@ var Scheduler = class {
|
|
|
23745
24386
|
}
|
|
23746
24387
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
23747
24388
|
start() {
|
|
23748
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
24389
|
+
const allSchedules = this.db.select().from(schedules).where(eq34(schedules.enabled, 1)).all();
|
|
23749
24390
|
for (const schedule of allSchedules) {
|
|
23750
24391
|
const missedRunAt = schedule.nextRunAt;
|
|
23751
24392
|
this.registerCronTask(schedule);
|
|
23752
24393
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
23753
|
-
|
|
24394
|
+
log9.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
23754
24395
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
23755
24396
|
}
|
|
23756
24397
|
}
|
|
23757
|
-
|
|
24398
|
+
log9.info("started", { scheduleCount: allSchedules.length });
|
|
23758
24399
|
}
|
|
23759
24400
|
/** Stop all cron tasks for graceful shutdown. */
|
|
23760
24401
|
stop() {
|
|
@@ -23775,7 +24416,7 @@ var Scheduler = class {
|
|
|
23775
24416
|
this.stopTask(key, existing, "Stopped");
|
|
23776
24417
|
this.tasks.delete(key);
|
|
23777
24418
|
}
|
|
23778
|
-
const schedule = this.db.select().from(schedules).where(and21(
|
|
24419
|
+
const schedule = this.db.select().from(schedules).where(and21(eq34(schedules.projectId, projectId), eq34(schedules.kind, kind))).get();
|
|
23779
24420
|
if (schedule && schedule.enabled === 1) {
|
|
23780
24421
|
this.registerCronTask(schedule);
|
|
23781
24422
|
}
|
|
@@ -23798,13 +24439,13 @@ var Scheduler = class {
|
|
|
23798
24439
|
stopTask(key, task, verb) {
|
|
23799
24440
|
task.stop();
|
|
23800
24441
|
task.destroy();
|
|
23801
|
-
|
|
24442
|
+
log9.info(`task.${verb.toLowerCase()}`, { key });
|
|
23802
24443
|
}
|
|
23803
24444
|
registerCronTask(schedule) {
|
|
23804
24445
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
23805
24446
|
const kind = schedule.kind;
|
|
23806
24447
|
if (!cron.validate(cronExpr)) {
|
|
23807
|
-
|
|
24448
|
+
log9.error("cron.invalid", { projectId, kind, cronExpr });
|
|
23808
24449
|
return;
|
|
23809
24450
|
}
|
|
23810
24451
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -23816,43 +24457,43 @@ var Scheduler = class {
|
|
|
23816
24457
|
this.db.update(schedules).set({
|
|
23817
24458
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
23818
24459
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23819
|
-
}).where(
|
|
24460
|
+
}).where(eq34(schedules.id, scheduleId)).run();
|
|
23820
24461
|
const label = schedule.preset ?? cronExpr;
|
|
23821
|
-
|
|
24462
|
+
log9.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
23822
24463
|
}
|
|
23823
24464
|
triggerRun(scheduleId, projectId, kind) {
|
|
23824
24465
|
try {
|
|
23825
24466
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23826
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
24467
|
+
const currentSchedule = this.db.select().from(schedules).where(eq34(schedules.id, scheduleId)).get();
|
|
23827
24468
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
23828
|
-
|
|
24469
|
+
log9.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
23829
24470
|
this.remove(projectId, kind);
|
|
23830
24471
|
return;
|
|
23831
24472
|
}
|
|
23832
24473
|
const task = this.tasks.get(taskKey(projectId, kind));
|
|
23833
24474
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
23834
|
-
const project = this.db.select().from(projects).where(
|
|
24475
|
+
const project = this.db.select().from(projects).where(eq34(projects.id, projectId)).get();
|
|
23835
24476
|
if (!project) {
|
|
23836
|
-
|
|
24477
|
+
log9.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
23837
24478
|
this.remove(projectId, kind);
|
|
23838
24479
|
return;
|
|
23839
24480
|
}
|
|
23840
24481
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
23841
24482
|
const sourceId = currentSchedule.sourceId;
|
|
23842
24483
|
if (!sourceId) {
|
|
23843
|
-
|
|
24484
|
+
log9.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
23844
24485
|
return;
|
|
23845
24486
|
}
|
|
23846
24487
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
23847
|
-
|
|
24488
|
+
log9.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
23848
24489
|
return;
|
|
23849
24490
|
}
|
|
23850
24491
|
this.db.update(schedules).set({
|
|
23851
24492
|
lastRunAt: now,
|
|
23852
24493
|
nextRunAt,
|
|
23853
24494
|
updatedAt: now
|
|
23854
|
-
}).where(
|
|
23855
|
-
|
|
24495
|
+
}).where(eq34(schedules.id, currentSchedule.id)).run();
|
|
24496
|
+
log9.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
23856
24497
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
23857
24498
|
return;
|
|
23858
24499
|
}
|
|
@@ -23861,7 +24502,7 @@ var Scheduler = class {
|
|
|
23861
24502
|
if (project.defaultLocation) {
|
|
23862
24503
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
23863
24504
|
if (!loc) {
|
|
23864
|
-
|
|
24505
|
+
log9.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
23865
24506
|
return;
|
|
23866
24507
|
}
|
|
23867
24508
|
resolvedLocation = loc;
|
|
@@ -23875,11 +24516,11 @@ var Scheduler = class {
|
|
|
23875
24516
|
location: locationLabel
|
|
23876
24517
|
});
|
|
23877
24518
|
if (queueResult.conflict) {
|
|
23878
|
-
|
|
24519
|
+
log9.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
23879
24520
|
this.db.update(schedules).set({
|
|
23880
24521
|
nextRunAt,
|
|
23881
24522
|
updatedAt: now
|
|
23882
|
-
}).where(
|
|
24523
|
+
}).where(eq34(schedules.id, currentSchedule.id)).run();
|
|
23883
24524
|
return;
|
|
23884
24525
|
}
|
|
23885
24526
|
const runId = queueResult.runId;
|
|
@@ -23887,21 +24528,21 @@ var Scheduler = class {
|
|
|
23887
24528
|
lastRunAt: now,
|
|
23888
24529
|
nextRunAt,
|
|
23889
24530
|
updatedAt: now
|
|
23890
|
-
}).where(
|
|
24531
|
+
}).where(eq34(schedules.id, currentSchedule.id)).run();
|
|
23891
24532
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
23892
24533
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
23893
|
-
|
|
24534
|
+
log9.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
23894
24535
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
23895
24536
|
} catch (err) {
|
|
23896
|
-
|
|
24537
|
+
log9.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
23897
24538
|
}
|
|
23898
24539
|
}
|
|
23899
24540
|
};
|
|
23900
24541
|
|
|
23901
24542
|
// src/notifier.ts
|
|
23902
|
-
import { eq as
|
|
23903
|
-
import
|
|
23904
|
-
var
|
|
24543
|
+
import { eq as eq35, desc as desc16, and as and22, or as or4 } from "drizzle-orm";
|
|
24544
|
+
import crypto31 from "crypto";
|
|
24545
|
+
var log10 = createLogger("Notifier");
|
|
23905
24546
|
var Notifier = class {
|
|
23906
24547
|
db;
|
|
23907
24548
|
serverUrl;
|
|
@@ -23911,26 +24552,26 @@ var Notifier = class {
|
|
|
23911
24552
|
}
|
|
23912
24553
|
/** Called after a run completes (success, partial, or failed). */
|
|
23913
24554
|
async onRunCompleted(runId, projectId) {
|
|
23914
|
-
|
|
23915
|
-
const notifs = this.db.select().from(notifications).where(
|
|
24555
|
+
log10.info("run.completed", { runId, projectId });
|
|
24556
|
+
const notifs = this.db.select().from(notifications).where(eq35(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
23916
24557
|
if (notifs.length === 0) {
|
|
23917
|
-
|
|
24558
|
+
log10.info("notifications.none-enabled", { projectId });
|
|
23918
24559
|
return;
|
|
23919
24560
|
}
|
|
23920
|
-
|
|
23921
|
-
const run = this.db.select().from(runs).where(
|
|
24561
|
+
log10.info("notifications.found", { projectId, count: notifs.length });
|
|
24562
|
+
const run = this.db.select().from(runs).where(eq35(runs.id, runId)).get();
|
|
23922
24563
|
if (!run) {
|
|
23923
|
-
|
|
24564
|
+
log10.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
23924
24565
|
return;
|
|
23925
24566
|
}
|
|
23926
|
-
const project = this.db.select().from(projects).where(
|
|
24567
|
+
const project = this.db.select().from(projects).where(eq35(projects.id, projectId)).get();
|
|
23927
24568
|
if (!project) {
|
|
23928
|
-
|
|
24569
|
+
log10.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
23929
24570
|
return;
|
|
23930
24571
|
}
|
|
23931
24572
|
const transitions = this.computeTransitions(runId, projectId);
|
|
23932
24573
|
const events = [];
|
|
23933
|
-
|
|
24574
|
+
log10.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
23934
24575
|
if (run.status === "completed" || run.status === "partial") {
|
|
23935
24576
|
events.push("run.completed");
|
|
23936
24577
|
}
|
|
@@ -23946,7 +24587,7 @@ var Notifier = class {
|
|
|
23946
24587
|
if (!config.url) continue;
|
|
23947
24588
|
const subscribedEvents = config.events;
|
|
23948
24589
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
23949
|
-
|
|
24590
|
+
log10.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
23950
24591
|
if (matchingEvents.length === 0) continue;
|
|
23951
24592
|
for (const event of matchingEvents) {
|
|
23952
24593
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -23970,11 +24611,11 @@ var Notifier = class {
|
|
|
23970
24611
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
23971
24612
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
23972
24613
|
if (insightEvents.length === 0) return;
|
|
23973
|
-
const notifs = this.db.select().from(notifications).where(
|
|
24614
|
+
const notifs = this.db.select().from(notifications).where(eq35(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
23974
24615
|
if (notifs.length === 0) return;
|
|
23975
|
-
const run = this.db.select().from(runs).where(
|
|
24616
|
+
const run = this.db.select().from(runs).where(eq35(runs.id, runId)).get();
|
|
23976
24617
|
if (!run) return;
|
|
23977
|
-
const project = this.db.select().from(projects).where(
|
|
24618
|
+
const project = this.db.select().from(projects).where(eq35(projects.id, projectId)).get();
|
|
23978
24619
|
if (!project) return;
|
|
23979
24620
|
for (const notif of notifs) {
|
|
23980
24621
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -24006,10 +24647,10 @@ var Notifier = class {
|
|
|
24006
24647
|
computeTransitions(runId, projectId) {
|
|
24007
24648
|
const recentRuns = this.db.select().from(runs).where(
|
|
24008
24649
|
and22(
|
|
24009
|
-
|
|
24010
|
-
or4(
|
|
24650
|
+
eq35(runs.projectId, projectId),
|
|
24651
|
+
or4(eq35(runs.status, "completed"), eq35(runs.status, "partial"))
|
|
24011
24652
|
)
|
|
24012
|
-
).orderBy(
|
|
24653
|
+
).orderBy(desc16(runs.createdAt)).limit(2).all();
|
|
24013
24654
|
if (recentRuns.length < 2) return [];
|
|
24014
24655
|
const currentRunId = recentRuns[0].id;
|
|
24015
24656
|
const previousRunId = recentRuns[1].id;
|
|
@@ -24019,12 +24660,12 @@ var Notifier = class {
|
|
|
24019
24660
|
query: queries.query,
|
|
24020
24661
|
provider: querySnapshots.provider,
|
|
24021
24662
|
citationState: querySnapshots.citationState
|
|
24022
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
24663
|
+
}).from(querySnapshots).leftJoin(queries, eq35(querySnapshots.queryId, queries.id)).where(eq35(querySnapshots.runId, currentRunId)).all();
|
|
24023
24664
|
const previousSnapshots = this.db.select({
|
|
24024
24665
|
queryId: querySnapshots.queryId,
|
|
24025
24666
|
provider: querySnapshots.provider,
|
|
24026
24667
|
citationState: querySnapshots.citationState
|
|
24027
|
-
}).from(querySnapshots).where(
|
|
24668
|
+
}).from(querySnapshots).where(eq35(querySnapshots.runId, previousRunId)).all();
|
|
24028
24669
|
const prevMap = /* @__PURE__ */ new Map();
|
|
24029
24670
|
for (const s of previousSnapshots) {
|
|
24030
24671
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -24048,23 +24689,23 @@ var Notifier = class {
|
|
|
24048
24689
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
24049
24690
|
const targetCheck = await resolveWebhookTarget(url);
|
|
24050
24691
|
if (!targetCheck.ok) {
|
|
24051
|
-
|
|
24692
|
+
log10.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
24052
24693
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
24053
24694
|
return;
|
|
24054
24695
|
}
|
|
24055
|
-
|
|
24696
|
+
log10.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
24056
24697
|
const maxRetries = 3;
|
|
24057
24698
|
const delays = [1e3, 4e3, 16e3];
|
|
24058
24699
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
24059
24700
|
try {
|
|
24060
24701
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
24061
24702
|
if (response.status >= 200 && response.status < 300) {
|
|
24062
|
-
|
|
24703
|
+
log10.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
24063
24704
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
24064
24705
|
return;
|
|
24065
24706
|
}
|
|
24066
24707
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
24067
|
-
|
|
24708
|
+
log10.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
24068
24709
|
if (attempt === maxRetries - 1) {
|
|
24069
24710
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
24070
24711
|
}
|
|
@@ -24072,7 +24713,7 @@ var Notifier = class {
|
|
|
24072
24713
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
24073
24714
|
if (attempt === maxRetries - 1) {
|
|
24074
24715
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
24075
|
-
|
|
24716
|
+
log10.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
24076
24717
|
}
|
|
24077
24718
|
}
|
|
24078
24719
|
if (attempt < maxRetries - 1) {
|
|
@@ -24082,7 +24723,7 @@ var Notifier = class {
|
|
|
24082
24723
|
}
|
|
24083
24724
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
24084
24725
|
this.db.insert(auditLog).values({
|
|
24085
|
-
id:
|
|
24726
|
+
id: crypto31.randomUUID(),
|
|
24086
24727
|
projectId,
|
|
24087
24728
|
actor: "scheduler",
|
|
24088
24729
|
action: `notification.${status}`,
|
|
@@ -24095,53 +24736,96 @@ var Notifier = class {
|
|
|
24095
24736
|
};
|
|
24096
24737
|
|
|
24097
24738
|
// src/run-coordinator.ts
|
|
24098
|
-
|
|
24739
|
+
import { eq as eq36 } from "drizzle-orm";
|
|
24740
|
+
var log11 = createLogger("RunCoordinator");
|
|
24099
24741
|
var RunCoordinator = class {
|
|
24100
|
-
constructor(notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
24742
|
+
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
24743
|
+
this.db = db;
|
|
24101
24744
|
this.notifier = notifier;
|
|
24102
24745
|
this.intelligenceService = intelligenceService;
|
|
24103
24746
|
this.onInsightsGenerated = onInsightsGenerated;
|
|
24104
24747
|
this.onAeroEvent = onAeroEvent;
|
|
24105
24748
|
}
|
|
24106
24749
|
async onRunCompleted(runId, projectId) {
|
|
24750
|
+
const runRow = this.db.select().from(runs).where(eq36(runs.id, runId)).get();
|
|
24751
|
+
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
24107
24752
|
let insightCount = 0;
|
|
24108
24753
|
let criticalOrHigh = 0;
|
|
24109
|
-
|
|
24110
|
-
|
|
24111
|
-
|
|
24112
|
-
|
|
24113
|
-
|
|
24114
|
-
|
|
24115
|
-
|
|
24116
|
-
|
|
24117
|
-
|
|
24118
|
-
|
|
24119
|
-
|
|
24120
|
-
|
|
24754
|
+
if (kind === RunKinds["answer-visibility"]) {
|
|
24755
|
+
try {
|
|
24756
|
+
const result = this.intelligenceService.analyzeAndPersist(runId, projectId);
|
|
24757
|
+
if (result) {
|
|
24758
|
+
insightCount = result.insights.length;
|
|
24759
|
+
criticalOrHigh = result.insights.filter(
|
|
24760
|
+
(i) => i.severity === "critical" || i.severity === "high"
|
|
24761
|
+
).length;
|
|
24762
|
+
if (this.onInsightsGenerated && criticalOrHigh > 0) {
|
|
24763
|
+
try {
|
|
24764
|
+
await this.onInsightsGenerated(runId, projectId, result);
|
|
24765
|
+
} catch (err) {
|
|
24766
|
+
log11.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24767
|
+
}
|
|
24121
24768
|
}
|
|
24122
24769
|
}
|
|
24770
|
+
} catch (err) {
|
|
24771
|
+
log11.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24123
24772
|
}
|
|
24124
|
-
} catch (err) {
|
|
24125
|
-
log10.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24126
24773
|
}
|
|
24127
24774
|
try {
|
|
24128
24775
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
24129
24776
|
} catch (err) {
|
|
24130
|
-
|
|
24777
|
+
log11.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24131
24778
|
}
|
|
24132
24779
|
if (this.onAeroEvent) {
|
|
24133
24780
|
try {
|
|
24134
|
-
|
|
24781
|
+
const ctx = kind === RunKinds["aeo-discover-probe"] ? this.buildDiscoveryAeroContext(runId, projectId, runRow?.status === "failed" ? "failed" : "completed", runRow?.error ?? null) : {
|
|
24782
|
+
kind,
|
|
24783
|
+
runId,
|
|
24784
|
+
projectId,
|
|
24785
|
+
insightCount,
|
|
24786
|
+
criticalOrHigh
|
|
24787
|
+
};
|
|
24788
|
+
await this.onAeroEvent(ctx);
|
|
24135
24789
|
} catch (err) {
|
|
24136
|
-
|
|
24790
|
+
log11.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24137
24791
|
}
|
|
24138
24792
|
}
|
|
24139
24793
|
}
|
|
24794
|
+
/**
|
|
24795
|
+
* Pull the discovery session that owns this run and project a payload Aero
|
|
24796
|
+
* can act on: bucket counts, top competitors, the seed provider, and the
|
|
24797
|
+
* session ID it can pass to `canonry_discover_session_get` for the per-query
|
|
24798
|
+
* breakdown. Looked up by `runId` (the POST handler populates
|
|
24799
|
+
* `discovery_sessions.runId` in the same transaction that creates the run)
|
|
24800
|
+
* so two concurrent discovery sessions on the same project don't get
|
|
24801
|
+
* cross-wired. Falls back to a zero payload when the session row is missing
|
|
24802
|
+
* so the Aero queue is never starved of a follow-up.
|
|
24803
|
+
*/
|
|
24804
|
+
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
24805
|
+
const session = this.db.select().from(discoverySessions).where(eq36(discoverySessions.runId, runId)).get();
|
|
24806
|
+
const competitorMap = session ? parseJsonColumn(session.competitorMap, []) : [];
|
|
24807
|
+
return {
|
|
24808
|
+
kind: RunKinds["aeo-discover-probe"],
|
|
24809
|
+
runId,
|
|
24810
|
+
projectId,
|
|
24811
|
+
sessionId: session?.id ?? "",
|
|
24812
|
+
seedProvider: session?.seedProvider ?? null,
|
|
24813
|
+
buckets: {
|
|
24814
|
+
cited: session?.citedCount ?? 0,
|
|
24815
|
+
aspirational: session?.aspirationalCount ?? 0,
|
|
24816
|
+
"wasted-surface": session?.wastedCount ?? 0
|
|
24817
|
+
},
|
|
24818
|
+
probeCount: session?.probeCount ?? 0,
|
|
24819
|
+
topCompetitors: competitorMap.slice(0, 5),
|
|
24820
|
+
status,
|
|
24821
|
+
error
|
|
24822
|
+
};
|
|
24823
|
+
}
|
|
24140
24824
|
};
|
|
24141
24825
|
|
|
24142
24826
|
// src/agent/session-registry.ts
|
|
24143
|
-
import
|
|
24144
|
-
import { eq as
|
|
24827
|
+
import crypto33 from "crypto";
|
|
24828
|
+
import { eq as eq38 } from "drizzle-orm";
|
|
24145
24829
|
|
|
24146
24830
|
// src/agent/session.ts
|
|
24147
24831
|
import fs11 from "fs";
|
|
@@ -24490,8 +25174,8 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
24490
25174
|
}
|
|
24491
25175
|
|
|
24492
25176
|
// src/agent/memory-store.ts
|
|
24493
|
-
import
|
|
24494
|
-
import { and as and23, desc as
|
|
25177
|
+
import crypto32 from "crypto";
|
|
25178
|
+
import { and as and23, desc as desc17, eq as eq37, like as like2, sql as sql13 } from "drizzle-orm";
|
|
24495
25179
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
24496
25180
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
24497
25181
|
function rowToDto2(row) {
|
|
@@ -24505,7 +25189,7 @@ function rowToDto2(row) {
|
|
|
24505
25189
|
};
|
|
24506
25190
|
}
|
|
24507
25191
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
24508
|
-
const query = db.select().from(agentMemory).where(
|
|
25192
|
+
const query = db.select().from(agentMemory).where(eq37(agentMemory.projectId, projectId)).orderBy(desc17(agentMemory.updatedAt));
|
|
24509
25193
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
24510
25194
|
return rows.map(rowToDto2);
|
|
24511
25195
|
}
|
|
@@ -24519,7 +25203,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
24519
25203
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
24520
25204
|
}
|
|
24521
25205
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
24522
|
-
const id =
|
|
25206
|
+
const id = crypto32.randomUUID();
|
|
24523
25207
|
db.insert(agentMemory).values({
|
|
24524
25208
|
id,
|
|
24525
25209
|
projectId: args.projectId,
|
|
@@ -24536,12 +25220,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
24536
25220
|
updatedAt: now
|
|
24537
25221
|
}
|
|
24538
25222
|
}).run();
|
|
24539
|
-
const row = db.select().from(agentMemory).where(and23(
|
|
25223
|
+
const row = db.select().from(agentMemory).where(and23(eq37(agentMemory.projectId, args.projectId), eq37(agentMemory.key, args.key))).get();
|
|
24540
25224
|
if (!row) throw new Error("memory upsert produced no row");
|
|
24541
25225
|
return rowToDto2(row);
|
|
24542
25226
|
}
|
|
24543
25227
|
function deleteMemoryEntry(db, projectId, key) {
|
|
24544
|
-
const result = db.delete(agentMemory).where(and23(
|
|
25228
|
+
const result = db.delete(agentMemory).where(and23(eq37(agentMemory.projectId, projectId), eq37(agentMemory.key, key))).run();
|
|
24545
25229
|
const changes = result.changes ?? 0;
|
|
24546
25230
|
return changes > 0;
|
|
24547
25231
|
}
|
|
@@ -24556,7 +25240,7 @@ function writeCompactionNote(db, args) {
|
|
|
24556
25240
|
}
|
|
24557
25241
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
24558
25242
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
24559
|
-
const id =
|
|
25243
|
+
const id = crypto32.randomUUID();
|
|
24560
25244
|
let inserted;
|
|
24561
25245
|
db.transaction((tx) => {
|
|
24562
25246
|
tx.insert(agentMemory).values({
|
|
@@ -24571,15 +25255,15 @@ function writeCompactionNote(db, args) {
|
|
|
24571
25255
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
24572
25256
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
24573
25257
|
and23(
|
|
24574
|
-
|
|
25258
|
+
eq37(agentMemory.projectId, args.projectId),
|
|
24575
25259
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
24576
25260
|
)
|
|
24577
|
-
).orderBy(
|
|
25261
|
+
).orderBy(desc17(agentMemory.updatedAt)).all();
|
|
24578
25262
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
24579
25263
|
if (stale.length > 0) {
|
|
24580
25264
|
tx.delete(agentMemory).where(sql13`${agentMemory.id} IN (${sql13.join(stale.map((s) => sql13`${s}`), sql13`, `)})`).run();
|
|
24581
25265
|
}
|
|
24582
|
-
const row = tx.select().from(agentMemory).where(and23(
|
|
25266
|
+
const row = tx.select().from(agentMemory).where(and23(eq37(agentMemory.projectId, args.projectId), eq37(agentMemory.key, key))).get();
|
|
24583
25267
|
if (row) inserted = rowToDto2(row);
|
|
24584
25268
|
});
|
|
24585
25269
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -24712,7 +25396,7 @@ async function compactMessages(args) {
|
|
|
24712
25396
|
}
|
|
24713
25397
|
|
|
24714
25398
|
// src/agent/session-registry.ts
|
|
24715
|
-
var
|
|
25399
|
+
var log12 = createLogger("SessionRegistry");
|
|
24716
25400
|
var MAX_HYDRATE_NOTES = 20;
|
|
24717
25401
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
24718
25402
|
function escapeMemoryFragment(value) {
|
|
@@ -24761,7 +25445,7 @@ var SessionRegistry = class {
|
|
|
24761
25445
|
modelProvider: effectiveProvider,
|
|
24762
25446
|
modelId: effectiveModelId,
|
|
24763
25447
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24764
|
-
}).where(
|
|
25448
|
+
}).where(eq38(agentSessions.projectId, projectId)).run();
|
|
24765
25449
|
}
|
|
24766
25450
|
const agent2 = createAeroSession({
|
|
24767
25451
|
projectName,
|
|
@@ -24939,13 +25623,13 @@ ${lines.join("\n")}
|
|
|
24939
25623
|
agent.state.messages = result.messages;
|
|
24940
25624
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
24941
25625
|
this.save(projectName);
|
|
24942
|
-
|
|
25626
|
+
log12.info("compaction.completed", {
|
|
24943
25627
|
projectName,
|
|
24944
25628
|
removedCount: result.removedCount,
|
|
24945
25629
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
24946
25630
|
});
|
|
24947
25631
|
} catch (err) {
|
|
24948
|
-
|
|
25632
|
+
log12.error("compaction.failed", {
|
|
24949
25633
|
projectName,
|
|
24950
25634
|
error: err instanceof Error ? err.message : String(err)
|
|
24951
25635
|
});
|
|
@@ -24975,7 +25659,7 @@ ${lines.join("\n")}
|
|
|
24975
25659
|
modelProvider: nextProvider,
|
|
24976
25660
|
modelId: nextModelId,
|
|
24977
25661
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24978
|
-
}).where(
|
|
25662
|
+
}).where(eq38(agentSessions.projectId, projectId)).run();
|
|
24979
25663
|
}
|
|
24980
25664
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
24981
25665
|
save(projectName) {
|
|
@@ -25042,7 +25726,7 @@ ${lines.join("\n")}
|
|
|
25042
25726
|
await agent.prompt(msgs);
|
|
25043
25727
|
this.save(projectName);
|
|
25044
25728
|
} catch (err) {
|
|
25045
|
-
|
|
25729
|
+
log12.error("drain.failed", {
|
|
25046
25730
|
projectName,
|
|
25047
25731
|
error: err instanceof Error ? err.message : String(err)
|
|
25048
25732
|
});
|
|
@@ -25137,17 +25821,17 @@ ${lines.join("\n")}
|
|
|
25137
25821
|
return id;
|
|
25138
25822
|
}
|
|
25139
25823
|
tryResolveProjectId(projectName) {
|
|
25140
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
25824
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq38(projects.name, projectName)).get();
|
|
25141
25825
|
return row?.id;
|
|
25142
25826
|
}
|
|
25143
25827
|
loadRow(projectId) {
|
|
25144
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
25828
|
+
const row = this.opts.db.select().from(agentSessions).where(eq38(agentSessions.projectId, projectId)).get();
|
|
25145
25829
|
return row ?? null;
|
|
25146
25830
|
}
|
|
25147
25831
|
insertRow(params) {
|
|
25148
25832
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
25149
25833
|
this.opts.db.insert(agentSessions).values({
|
|
25150
|
-
id:
|
|
25834
|
+
id: crypto33.randomUUID(),
|
|
25151
25835
|
projectId: params.projectId,
|
|
25152
25836
|
systemPrompt: params.systemPrompt,
|
|
25153
25837
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -25160,14 +25844,14 @@ ${lines.join("\n")}
|
|
|
25160
25844
|
}
|
|
25161
25845
|
updateRow(projectId, patch) {
|
|
25162
25846
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
25163
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
25847
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq38(agentSessions.projectId, projectId)).run();
|
|
25164
25848
|
}
|
|
25165
25849
|
};
|
|
25166
25850
|
|
|
25167
25851
|
// src/agent/agent-routes.ts
|
|
25168
|
-
import { eq as
|
|
25852
|
+
import { eq as eq39 } from "drizzle-orm";
|
|
25169
25853
|
function resolveProject2(db, name) {
|
|
25170
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
25854
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq39(projects.name, name)).get();
|
|
25171
25855
|
if (!row) throw notFound("project", name);
|
|
25172
25856
|
return row;
|
|
25173
25857
|
}
|
|
@@ -25176,7 +25860,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
25176
25860
|
"/projects/:name/agent/transcript",
|
|
25177
25861
|
async (request) => {
|
|
25178
25862
|
const project = resolveProject2(opts.db, request.params.name);
|
|
25179
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
25863
|
+
const row = opts.db.select().from(agentSessions).where(eq39(agentSessions.projectId, project.id)).get();
|
|
25180
25864
|
if (!row) {
|
|
25181
25865
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
25182
25866
|
}
|
|
@@ -25200,7 +25884,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
25200
25884
|
async (request) => {
|
|
25201
25885
|
const project = resolveProject2(opts.db, request.params.name);
|
|
25202
25886
|
opts.sessionRegistry.reset(project.name);
|
|
25203
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
25887
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq39(agentSessions.projectId, project.id)).run();
|
|
25204
25888
|
return { status: "reset" };
|
|
25205
25889
|
}
|
|
25206
25890
|
);
|
|
@@ -25433,7 +26117,7 @@ function formatAuditFactorScore(factor) {
|
|
|
25433
26117
|
}
|
|
25434
26118
|
|
|
25435
26119
|
// src/snapshot-service.ts
|
|
25436
|
-
var
|
|
26120
|
+
var log13 = createLogger("Snapshot");
|
|
25437
26121
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
25438
26122
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
25439
26123
|
var ProviderExecutionGate2 = class {
|
|
@@ -25576,7 +26260,7 @@ var SnapshotService = class {
|
|
|
25576
26260
|
return mapAuditReport(report);
|
|
25577
26261
|
} catch (err) {
|
|
25578
26262
|
const message = err instanceof Error ? err.message : String(err);
|
|
25579
|
-
|
|
26263
|
+
log13.warn("audit.failed", { homepageUrl, error: message });
|
|
25580
26264
|
return {
|
|
25581
26265
|
url: homepageUrl,
|
|
25582
26266
|
finalUrl: homepageUrl,
|
|
@@ -25606,7 +26290,7 @@ var SnapshotService = class {
|
|
|
25606
26290
|
queries: parsedQueries
|
|
25607
26291
|
};
|
|
25608
26292
|
} catch (err) {
|
|
25609
|
-
|
|
26293
|
+
log13.warn("profile.generation-failed", {
|
|
25610
26294
|
domain: ctx.domain,
|
|
25611
26295
|
provider: ctx.analysisProvider.adapter.name,
|
|
25612
26296
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -25748,7 +26432,7 @@ var SnapshotService = class {
|
|
|
25748
26432
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
25749
26433
|
};
|
|
25750
26434
|
} catch (err) {
|
|
25751
|
-
|
|
26435
|
+
log13.warn("response.analysis-failed", {
|
|
25752
26436
|
provider: ctx.analysisProvider.adapter.name,
|
|
25753
26437
|
error: err instanceof Error ? err.message : String(err)
|
|
25754
26438
|
});
|
|
@@ -26033,7 +26717,7 @@ function clipText(value, length) {
|
|
|
26033
26717
|
// src/server.ts
|
|
26034
26718
|
var _require2 = createRequire3(import.meta.url);
|
|
26035
26719
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
26036
|
-
var
|
|
26720
|
+
var log14 = createLogger("Server");
|
|
26037
26721
|
var DEFAULT_QUOTA = {
|
|
26038
26722
|
maxConcurrency: 2,
|
|
26039
26723
|
maxRequestsPerMinute: 10,
|
|
@@ -26064,7 +26748,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
26064
26748
|
};
|
|
26065
26749
|
}
|
|
26066
26750
|
function hashApiKey(key) {
|
|
26067
|
-
return
|
|
26751
|
+
return crypto34.createHash("sha256").update(key).digest("hex");
|
|
26068
26752
|
}
|
|
26069
26753
|
function parseCookies2(header) {
|
|
26070
26754
|
if (!header) return {};
|
|
@@ -26120,7 +26804,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
26120
26804
|
}
|
|
26121
26805
|
if (migratedGoogle > 0) {
|
|
26122
26806
|
saveConfigPatch({ google: config.google });
|
|
26123
|
-
|
|
26807
|
+
log14.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
26124
26808
|
}
|
|
26125
26809
|
let migratedGa4 = 0;
|
|
26126
26810
|
for (const row of rows.ga4) {
|
|
@@ -26138,7 +26822,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
26138
26822
|
}
|
|
26139
26823
|
if (migratedGa4 > 0) {
|
|
26140
26824
|
saveConfigPatch({ ga4: config.ga4 });
|
|
26141
|
-
|
|
26825
|
+
log14.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
26142
26826
|
}
|
|
26143
26827
|
}
|
|
26144
26828
|
async function createServer(opts) {
|
|
@@ -26170,11 +26854,11 @@ async function createServer(opts) {
|
|
|
26170
26854
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
26171
26855
|
dropLegacyCredentialColumns(opts.db);
|
|
26172
26856
|
} catch (err) {
|
|
26173
|
-
|
|
26857
|
+
log14.warn("credentials.migration.failed", {
|
|
26174
26858
|
error: err instanceof Error ? err.message : String(err)
|
|
26175
26859
|
});
|
|
26176
26860
|
}
|
|
26177
|
-
|
|
26861
|
+
log14.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
26178
26862
|
const p = providers[k];
|
|
26179
26863
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
26180
26864
|
}) });
|
|
@@ -26218,15 +26902,27 @@ async function createServer(opts) {
|
|
|
26218
26902
|
config: opts.config
|
|
26219
26903
|
});
|
|
26220
26904
|
const runCoordinator = new RunCoordinator(
|
|
26905
|
+
opts.db,
|
|
26221
26906
|
notifier,
|
|
26222
26907
|
intelligenceService,
|
|
26223
26908
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
26224
|
-
async (
|
|
26225
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
26909
|
+
async (ctx) => {
|
|
26910
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq40(projects.id, ctx.projectId)).get();
|
|
26226
26911
|
if (!project) return;
|
|
26912
|
+
let content;
|
|
26913
|
+
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
26914
|
+
if (ctx.status === "failed") {
|
|
26915
|
+
content = `[system] Discovery run ${ctx.runId} failed for project ${project.name}: ${ctx.error ?? "unknown error"}. Surface a one-line diagnosis and a suggested next step.`;
|
|
26916
|
+
} else {
|
|
26917
|
+
const top = ctx.topCompetitors.map((c) => `${c.domain}(${c.hits})`).join(", ") || "none";
|
|
26918
|
+
content = `[system] Discovery run ${ctx.runId} completed for project ${project.name} (session ${ctx.sessionId}). Buckets \u2014 cited:${ctx.buckets.cited}, wasted-surface:${ctx.buckets["wasted-surface"]}, aspirational:${ctx.buckets.aspirational} (${ctx.probeCount} probes; seed provider: ${ctx.seedProvider ?? "unknown"}). Top recurring competitor domains: ${top}. Use canonry_discover_session_get to pull per-query buckets and call out anything worth promoting to the tracked basket. Keep it tight.`;
|
|
26919
|
+
}
|
|
26920
|
+
} else {
|
|
26921
|
+
content = `[system] Run ${ctx.runId} completed for project ${project.name}. ${ctx.insightCount} insights generated (${ctx.criticalOrHigh} critical/high). Use canonry_run_get to inspect the run and canonry_insights_list to review new findings. Surface anything notable briefly \u2014 skip chit-chat.`;
|
|
26922
|
+
}
|
|
26227
26923
|
sessionRegistry.queueFollowUp(project.name, {
|
|
26228
26924
|
role: "user",
|
|
26229
|
-
content
|
|
26925
|
+
content,
|
|
26230
26926
|
timestamp: Date.now()
|
|
26231
26927
|
});
|
|
26232
26928
|
void sessionRegistry.drainNow(project.name);
|
|
@@ -26351,7 +27047,7 @@ async function createServer(opts) {
|
|
|
26351
27047
|
return removed;
|
|
26352
27048
|
}
|
|
26353
27049
|
};
|
|
26354
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
27050
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto34.randomBytes(32).toString("hex");
|
|
26355
27051
|
const googleConnectionStore = {
|
|
26356
27052
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
26357
27053
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -26397,11 +27093,11 @@ async function createServer(opts) {
|
|
|
26397
27093
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
26398
27094
|
if (opts.config.apiKey) {
|
|
26399
27095
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
26400
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
27096
|
+
const existing = opts.db.select().from(apiKeys).where(eq40(apiKeys.keyHash, keyHash)).get();
|
|
26401
27097
|
if (!existing) {
|
|
26402
27098
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
26403
27099
|
opts.db.insert(apiKeys).values({
|
|
26404
|
-
id: `key_${
|
|
27100
|
+
id: `key_${crypto34.randomBytes(8).toString("hex")}`,
|
|
26405
27101
|
name: "default",
|
|
26406
27102
|
keyHash,
|
|
26407
27103
|
keyPrefix: prefix,
|
|
@@ -26425,7 +27121,7 @@ async function createServer(opts) {
|
|
|
26425
27121
|
};
|
|
26426
27122
|
const createSession = (apiKeyId) => {
|
|
26427
27123
|
pruneExpiredSessions();
|
|
26428
|
-
const sessionId =
|
|
27124
|
+
const sessionId = crypto34.randomBytes(32).toString("hex");
|
|
26429
27125
|
sessions.set(sessionId, {
|
|
26430
27126
|
apiKeyId,
|
|
26431
27127
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -26449,7 +27145,7 @@ async function createServer(opts) {
|
|
|
26449
27145
|
};
|
|
26450
27146
|
const getDefaultApiKey = () => {
|
|
26451
27147
|
if (!opts.config.apiKey) return void 0;
|
|
26452
|
-
return opts.db.select().from(apiKeys).where(
|
|
27148
|
+
return opts.db.select().from(apiKeys).where(eq40(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
26453
27149
|
};
|
|
26454
27150
|
const createPasswordSession = (reply) => {
|
|
26455
27151
|
const key = getDefaultApiKey();
|
|
@@ -26506,12 +27202,12 @@ async function createServer(opts) {
|
|
|
26506
27202
|
return reply.send({ authenticated: true });
|
|
26507
27203
|
}
|
|
26508
27204
|
if (apiKey) {
|
|
26509
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
27205
|
+
const key = opts.db.select().from(apiKeys).where(eq40(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
26510
27206
|
if (!key || key.revokedAt) {
|
|
26511
27207
|
const err2 = authInvalid();
|
|
26512
27208
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
26513
27209
|
}
|
|
26514
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
27210
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq40(apiKeys.id, key.id)).run();
|
|
26515
27211
|
const sessionId = createSession(key.id);
|
|
26516
27212
|
reply.header("set-cookie", serializeSessionCookie({
|
|
26517
27213
|
name: SESSION_COOKIE_NAME,
|
|
@@ -26621,7 +27317,7 @@ async function createServer(opts) {
|
|
|
26621
27317
|
deps: {
|
|
26622
27318
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
26623
27319
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
26624
|
-
const runId =
|
|
27320
|
+
const runId = crypto34.randomUUID();
|
|
26625
27321
|
opts.db.insert(runs).values({
|
|
26626
27322
|
id: runId,
|
|
26627
27323
|
projectId,
|
|
@@ -26644,6 +27340,20 @@ async function createServer(opts) {
|
|
|
26644
27340
|
app.log.error({ runId, err }, "Backlink extract failed");
|
|
26645
27341
|
});
|
|
26646
27342
|
},
|
|
27343
|
+
onDiscoveryRunRequested: (input) => {
|
|
27344
|
+
executeDiscoveryRun({
|
|
27345
|
+
db: opts.db,
|
|
27346
|
+
registry,
|
|
27347
|
+
runId: input.runId,
|
|
27348
|
+
sessionId: input.sessionId,
|
|
27349
|
+
projectId: input.projectId,
|
|
27350
|
+
icpDescription: input.icpDescription,
|
|
27351
|
+
dedupThreshold: input.dedupThreshold,
|
|
27352
|
+
maxProbes: input.maxProbes
|
|
27353
|
+
}).then(() => runCoordinator.onRunCompleted(input.runId, input.projectId)).catch((err) => {
|
|
27354
|
+
app.log.error({ runId: input.runId, err }, "Discovery run failed");
|
|
27355
|
+
});
|
|
27356
|
+
},
|
|
26647
27357
|
onBacklinksPruneCache: (release) => {
|
|
26648
27358
|
try {
|
|
26649
27359
|
pruneCachedRelease(release);
|
|
@@ -26769,7 +27479,7 @@ async function createServer(opts) {
|
|
|
26769
27479
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
26770
27480
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
26771
27481
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
26772
|
-
id:
|
|
27482
|
+
id: crypto34.randomUUID(),
|
|
26773
27483
|
projectId,
|
|
26774
27484
|
actor: "api",
|
|
26775
27485
|
action: existing ? "provider.updated" : "provider.created",
|