@ainyc/canonry 4.24.1 → 4.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/aero/references/aeo-discovery.md +89 -0
- package/assets/agent-workspace/skills/canonry-setup/references/canonry-cli.md +15 -0
- package/assets/assets/{index-BzD9HUxc.js → index-X1r0qycv.js} +84 -84
- package/assets/index.html +1 -1
- package/dist/{chunk-6EJ54OX7.js → chunk-2FAEQ56I.js} +92 -2
- package/dist/{chunk-E5PZ23OS.js → chunk-H4RE4WLW.js} +1130 -305
- package/dist/{chunk-EUGCQSFC.js → chunk-HVW665A4.js} +135 -1
- package/dist/{chunk-OYYFXKRK.js → chunk-PN24DAGC.js} +127 -2
- package/dist/cli.js +440 -125
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-NVN2PAR7.js → intelligence-service-VUBODIGG.js} +2 -2
- package/dist/mcp.js +9 -3
- package/package.json +10 -10
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
loadConfig,
|
|
6
6
|
loadConfigRaw,
|
|
7
7
|
saveConfigPatch
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-2FAEQ56I.js";
|
|
9
9
|
import {
|
|
10
10
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
11
11
|
IntelligenceService,
|
|
@@ -40,6 +40,8 @@ import {
|
|
|
40
40
|
competitors,
|
|
41
41
|
crawlerEventsHourly,
|
|
42
42
|
createLogger,
|
|
43
|
+
discoveryProbes,
|
|
44
|
+
discoverySessions,
|
|
43
45
|
dropLegacyCredentialColumns,
|
|
44
46
|
extractLegacyCredentials,
|
|
45
47
|
gaAiReferrals,
|
|
@@ -66,7 +68,7 @@ import {
|
|
|
66
68
|
schedules,
|
|
67
69
|
trafficSources,
|
|
68
70
|
usageCounters
|
|
69
|
-
} from "./chunk-
|
|
71
|
+
} from "./chunk-PN24DAGC.js";
|
|
70
72
|
import {
|
|
71
73
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
72
74
|
AGENT_PROVIDER_IDS,
|
|
@@ -77,6 +79,8 @@ import {
|
|
|
77
79
|
CheckScopes,
|
|
78
80
|
CheckStatuses,
|
|
79
81
|
CitationStates,
|
|
82
|
+
DiscoveryBuckets,
|
|
83
|
+
DiscoverySessionStatuses,
|
|
80
84
|
MemorySources,
|
|
81
85
|
RunKinds,
|
|
82
86
|
RunStatuses,
|
|
@@ -101,7 +105,9 @@ import {
|
|
|
101
105
|
buildRunErrorFromMessages,
|
|
102
106
|
categorizeSource,
|
|
103
107
|
categoryLabel,
|
|
108
|
+
citationStateSchema,
|
|
104
109
|
citationStateToCited,
|
|
110
|
+
clusterByCosine,
|
|
105
111
|
competitorBatchRequestSchema,
|
|
106
112
|
contentActionLabel,
|
|
107
113
|
dedupeReportActions,
|
|
@@ -110,6 +116,8 @@ import {
|
|
|
110
116
|
deltaPercent,
|
|
111
117
|
deltaTone,
|
|
112
118
|
determineAnswerMentioned,
|
|
119
|
+
discoveryBucketSchema,
|
|
120
|
+
discoveryRunRequestSchema,
|
|
113
121
|
effectiveDomains,
|
|
114
122
|
emptyCitationVisibility,
|
|
115
123
|
extractAnswerMentions,
|
|
@@ -134,6 +142,7 @@ import {
|
|
|
134
142
|
notImplemented,
|
|
135
143
|
parseRunError,
|
|
136
144
|
parseWindow,
|
|
145
|
+
pickClusterRepresentative,
|
|
137
146
|
projectConfigSchema,
|
|
138
147
|
projectUpsertRequestSchema,
|
|
139
148
|
providerError,
|
|
@@ -160,7 +169,7 @@ import {
|
|
|
160
169
|
visibilityStateFromAnswerMentioned,
|
|
161
170
|
windowCutoff,
|
|
162
171
|
wordpressEnvSchema
|
|
163
|
-
} from "./chunk-
|
|
172
|
+
} from "./chunk-HVW665A4.js";
|
|
164
173
|
|
|
165
174
|
// src/telemetry.ts
|
|
166
175
|
import crypto from "crypto";
|
|
@@ -315,11 +324,11 @@ function trackEvent(event, properties, options) {
|
|
|
315
324
|
|
|
316
325
|
// src/server.ts
|
|
317
326
|
import { createRequire as createRequire3 } from "module";
|
|
318
|
-
import
|
|
327
|
+
import crypto34 from "crypto";
|
|
319
328
|
import fs12 from "fs";
|
|
320
329
|
import path14 from "path";
|
|
321
330
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
322
|
-
import { eq as
|
|
331
|
+
import { eq as eq40 } from "drizzle-orm";
|
|
323
332
|
import Fastify from "fastify";
|
|
324
333
|
|
|
325
334
|
// ../api-routes/src/auth.ts
|
|
@@ -741,6 +750,7 @@ async function queryRoutes(app, opts) {
|
|
|
741
750
|
id: crypto5.randomUUID(),
|
|
742
751
|
projectId: project.id,
|
|
743
752
|
query: q,
|
|
753
|
+
provenance: "cli",
|
|
744
754
|
createdAt: now
|
|
745
755
|
}).run();
|
|
746
756
|
}
|
|
@@ -797,6 +807,7 @@ async function queryRoutes(app, opts) {
|
|
|
797
807
|
id: crypto5.randomUUID(),
|
|
798
808
|
projectId: project.id,
|
|
799
809
|
query: q,
|
|
810
|
+
provenance: "cli",
|
|
800
811
|
createdAt: now
|
|
801
812
|
}).run();
|
|
802
813
|
added.push(q);
|
|
@@ -874,6 +885,7 @@ async function queryRoutes(app, opts) {
|
|
|
874
885
|
id: crypto5.randomUUID(),
|
|
875
886
|
projectId: project.id,
|
|
876
887
|
query: keyword,
|
|
888
|
+
provenance: "cli",
|
|
877
889
|
createdAt: now
|
|
878
890
|
}).run();
|
|
879
891
|
}
|
|
@@ -930,6 +942,7 @@ async function queryRoutes(app, opts) {
|
|
|
930
942
|
id: crypto5.randomUUID(),
|
|
931
943
|
projectId: project.id,
|
|
932
944
|
query: keyword,
|
|
945
|
+
provenance: "cli",
|
|
933
946
|
createdAt: now
|
|
934
947
|
}).run();
|
|
935
948
|
added.push(keyword);
|
|
@@ -1032,6 +1045,7 @@ async function competitorRoutes(app) {
|
|
|
1032
1045
|
id: crypto6.randomUUID(),
|
|
1033
1046
|
projectId: project.id,
|
|
1034
1047
|
domain,
|
|
1048
|
+
provenance: "cli",
|
|
1035
1049
|
createdAt: now
|
|
1036
1050
|
}).run();
|
|
1037
1051
|
}
|
|
@@ -1061,6 +1075,7 @@ async function competitorRoutes(app) {
|
|
|
1061
1075
|
id: crypto6.randomUUID(),
|
|
1062
1076
|
projectId: project.id,
|
|
1063
1077
|
domain,
|
|
1078
|
+
provenance: "cli",
|
|
1064
1079
|
createdAt: now
|
|
1065
1080
|
}).onConflictDoNothing({
|
|
1066
1081
|
target: [competitors.projectId, competitors.domain]
|
|
@@ -1140,6 +1155,7 @@ function queueRunIfProjectIdle(db, params) {
|
|
|
1140
1155
|
status: "queued",
|
|
1141
1156
|
trigger,
|
|
1142
1157
|
location: params.location ?? null,
|
|
1158
|
+
queries: params.queries ?? null,
|
|
1143
1159
|
createdAt
|
|
1144
1160
|
}).run();
|
|
1145
1161
|
return { conflict: false, runId };
|
|
@@ -1170,6 +1186,20 @@ async function runRoutes(app, opts) {
|
|
|
1170
1186
|
rawProviders.splice(0, rawProviders.length, ...normalized);
|
|
1171
1187
|
}
|
|
1172
1188
|
const providers = rawProviders?.length ? rawProviders : void 0;
|
|
1189
|
+
let scopedQueries = null;
|
|
1190
|
+
if (body.queries?.length) {
|
|
1191
|
+
const trackedRows = app.db.select({ query: queries.query }).from(queries).where(eq7(queries.projectId, project.id)).all();
|
|
1192
|
+
const tracked = new Set(trackedRows.map((r) => r.query));
|
|
1193
|
+
const missing = body.queries.filter((q) => !tracked.has(q));
|
|
1194
|
+
if (missing.length) {
|
|
1195
|
+
throw validationError(`Queries not tracked on project "${project.name}": ${missing.join(", ")}`, {
|
|
1196
|
+
missing,
|
|
1197
|
+
tracked: [...tracked]
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1200
|
+
scopedQueries = body.queries;
|
|
1201
|
+
}
|
|
1202
|
+
const queriesColumn = scopedQueries ? JSON.stringify(scopedQueries) : null;
|
|
1173
1203
|
let resolvedLocation;
|
|
1174
1204
|
const projectLocations = parseJsonColumn(project.locations, []);
|
|
1175
1205
|
if (body.noLocation) {
|
|
@@ -1202,6 +1232,7 @@ async function runRoutes(app, opts) {
|
|
|
1202
1232
|
status: "queued",
|
|
1203
1233
|
trigger,
|
|
1204
1234
|
location: loc.label,
|
|
1235
|
+
queries: queriesColumn,
|
|
1205
1236
|
createdAt: now
|
|
1206
1237
|
}).run();
|
|
1207
1238
|
newRuns.push({ runId: runId2, loc });
|
|
@@ -1229,7 +1260,8 @@ async function runRoutes(app, opts) {
|
|
|
1229
1260
|
kind,
|
|
1230
1261
|
projectId: project.id,
|
|
1231
1262
|
trigger,
|
|
1232
|
-
location: locationLabel
|
|
1263
|
+
location: locationLabel,
|
|
1264
|
+
queries: queriesColumn
|
|
1233
1265
|
});
|
|
1234
1266
|
if (queueResult.conflict) throw runInProgress(project.name);
|
|
1235
1267
|
const runId = queueResult.runId;
|
|
@@ -1375,6 +1407,7 @@ function formatRun(row) {
|
|
|
1375
1407
|
status: row.status,
|
|
1376
1408
|
trigger: row.trigger,
|
|
1377
1409
|
location: row.location,
|
|
1410
|
+
queries: parseJsonColumn(row.queries, null),
|
|
1378
1411
|
startedAt: row.startedAt,
|
|
1379
1412
|
finishedAt: row.finishedAt,
|
|
1380
1413
|
error: parseRunError(row.error),
|
|
@@ -1821,6 +1854,7 @@ async function applyRoutes(app, opts) {
|
|
|
1821
1854
|
id: crypto10.randomUUID(),
|
|
1822
1855
|
projectId,
|
|
1823
1856
|
query: q,
|
|
1857
|
+
provenance: "cli",
|
|
1824
1858
|
createdAt: now
|
|
1825
1859
|
}).run();
|
|
1826
1860
|
}
|
|
@@ -1838,6 +1872,7 @@ async function applyRoutes(app, opts) {
|
|
|
1838
1872
|
id: crypto10.randomUUID(),
|
|
1839
1873
|
projectId,
|
|
1840
1874
|
domain,
|
|
1875
|
+
provenance: "cli",
|
|
1841
1876
|
createdAt: now
|
|
1842
1877
|
}).run();
|
|
1843
1878
|
}
|
|
@@ -6789,6 +6824,15 @@ import { eq as eq15, and as and6, desc as desc7, sql as sql4, like, or as or3, i
|
|
|
6789
6824
|
var TOP_INSIGHT_LIMIT = 5;
|
|
6790
6825
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
6791
6826
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
6827
|
+
var INTEGRATION_SYNC_KINDS = /* @__PURE__ */ new Set([
|
|
6828
|
+
RunKinds["gsc-sync"],
|
|
6829
|
+
RunKinds["inspect-sitemap"],
|
|
6830
|
+
RunKinds["ga-sync"],
|
|
6831
|
+
RunKinds["bing-inspect"],
|
|
6832
|
+
RunKinds["bing-inspect-sitemap"],
|
|
6833
|
+
RunKinds["backlink-extract"],
|
|
6834
|
+
RunKinds["traffic-sync"]
|
|
6835
|
+
]);
|
|
6792
6836
|
async function compositeRoutes(app) {
|
|
6793
6837
|
app.get("/projects/:name/overview", async (request, reply) => {
|
|
6794
6838
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -7176,7 +7220,7 @@ function buildAttentionItems(insightRows, allRuns) {
|
|
|
7176
7220
|
}
|
|
7177
7221
|
const sortedRuns = [...allRuns].sort((a, b) => b.createdAt.localeCompare(a.createdAt));
|
|
7178
7222
|
const latestVisRun = sortedRuns.find((r) => r.kind === RunKinds["answer-visibility"]);
|
|
7179
|
-
const latestSyncRun = sortedRuns.find((r) => r.kind
|
|
7223
|
+
const latestSyncRun = sortedRuns.find((r) => INTEGRATION_SYNC_KINDS.has(r.kind));
|
|
7180
7224
|
if (latestVisRun && latestSyncRun) {
|
|
7181
7225
|
const visibilityAge = new Date(latestSyncRun.createdAt).getTime() - new Date(latestVisRun.createdAt).getTime();
|
|
7182
7226
|
const ONE_DAY = 24 * 60 * 60 * 1e3;
|
|
@@ -7977,6 +8021,7 @@ var routeCatalog = [
|
|
|
7977
8021
|
kind: stringSchema,
|
|
7978
8022
|
trigger: stringSchema,
|
|
7979
8023
|
providers: stringArraySchema,
|
|
8024
|
+
queries: stringArraySchema,
|
|
7980
8025
|
location: stringSchema,
|
|
7981
8026
|
allLocations: booleanSchema,
|
|
7982
8027
|
noLocation: booleanSchema
|
|
@@ -10145,8 +10190,8 @@ var routeCatalog = [
|
|
|
10145
10190
|
{
|
|
10146
10191
|
method: "post",
|
|
10147
10192
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/backfill",
|
|
10148
|
-
summary: "Reclassify historical
|
|
10149
|
-
description: 'Async one-shot backfill: pulls the last `days` of
|
|
10193
|
+
summary: "Reclassify historical traffic-source logs",
|
|
10194
|
+
description: 'Async one-shot backfill: pulls the last `days` of events (clamped server-side to the upstream retention ceiling \u2014 30d for Cloud Logging `_Default`; the WordPress plugin honours the same window via `since`/`until` query params), classifies them with the current rules, and replaces the hourly rollup buckets + sample slice in the window inside one transaction. Returns immediately with `{ runId, status: "running" }`; poll `GET /runs/{id}` for completion. lastSyncedAt only advances forward, so a backfill never undoes incremental sync progress that ran ahead of it. Supported source types: `cloud-run`, `wordpress`.',
|
|
10150
10195
|
tags: ["traffic"],
|
|
10151
10196
|
parameters: [
|
|
10152
10197
|
nameParameter,
|
|
@@ -10227,6 +10272,79 @@ var routeCatalog = [
|
|
|
10227
10272
|
400: { description: "Invalid query parameters." },
|
|
10228
10273
|
404: { description: "Project not found." }
|
|
10229
10274
|
}
|
|
10275
|
+
},
|
|
10276
|
+
{
|
|
10277
|
+
method: "post",
|
|
10278
|
+
path: "/api/v1/projects/{name}/discover/run",
|
|
10279
|
+
summary: "Start a tracked-basket discovery session",
|
|
10280
|
+
description: 'Kicks off a discovery session for the project. The pipeline: ICP description \u2192 Gemini grounded seed prompt \u2192 embed + cluster (cosine \u2265 0.85 by default) \u2192 pick canonical representatives \u2192 probe each canonical via Gemini grounding \u2192 classify into cited / aspirational / wasted-surface \u2192 aggregate competitor map. Returns immediately with `{ runId, sessionId, status: "running" }`; the actual work runs in the background. Poll `GET /projects/{name}/discover/sessions/{id}` until `status` is `completed` or `failed`.',
|
|
10281
|
+
tags: ["discovery"],
|
|
10282
|
+
parameters: [nameParameter],
|
|
10283
|
+
requestBody: {
|
|
10284
|
+
required: false,
|
|
10285
|
+
content: {
|
|
10286
|
+
"application/json": {
|
|
10287
|
+
schema: {
|
|
10288
|
+
type: "object",
|
|
10289
|
+
properties: {
|
|
10290
|
+
icpDescription: { type: "string", description: "Free-text ICP. Required if the project does not have spec.icpDescription stored." },
|
|
10291
|
+
dedupThreshold: { type: "number", description: "Cosine similarity threshold for clustering. Defaults to 0.85." },
|
|
10292
|
+
maxProbes: { type: "integer", description: "Max canonical queries to probe in this session. Default 100, hard cap 500." }
|
|
10293
|
+
}
|
|
10294
|
+
}
|
|
10295
|
+
}
|
|
10296
|
+
}
|
|
10297
|
+
},
|
|
10298
|
+
responses: {
|
|
10299
|
+
201: { description: "Discovery session enqueued; returns { runId, sessionId, status }." },
|
|
10300
|
+
400: { description: "Missing or invalid ICP / parameters." },
|
|
10301
|
+
404: { description: "Project not found." }
|
|
10302
|
+
}
|
|
10303
|
+
},
|
|
10304
|
+
{
|
|
10305
|
+
method: "get",
|
|
10306
|
+
path: "/api/v1/projects/{name}/discover/sessions",
|
|
10307
|
+
summary: "List discovery sessions for a project",
|
|
10308
|
+
description: "Returns sessions newest-first. Each row carries seed counts, bucket counts, the competitor map, and timing fields. Drill into `GET /projects/{name}/discover/sessions/{id}` for per-query probe rows.",
|
|
10309
|
+
tags: ["discovery"],
|
|
10310
|
+
parameters: [
|
|
10311
|
+
nameParameter,
|
|
10312
|
+
{ name: "limit", in: "query", description: "Max sessions returned. Default 50.", schema: stringSchema }
|
|
10313
|
+
],
|
|
10314
|
+
responses: {
|
|
10315
|
+
200: { description: "Sessions returned." },
|
|
10316
|
+
404: { description: "Project not found." }
|
|
10317
|
+
}
|
|
10318
|
+
},
|
|
10319
|
+
{
|
|
10320
|
+
method: "get",
|
|
10321
|
+
path: "/api/v1/projects/{name}/discover/sessions/{id}",
|
|
10322
|
+
summary: "Get a discovery session with its probe list",
|
|
10323
|
+
description: 'Returns one discovery session plus the full list of per-canonical probes (query, bucket, cited domains, citation state). Use this to answer "what did discovery find for project X?" in a single call.',
|
|
10324
|
+
tags: ["discovery"],
|
|
10325
|
+
parameters: [
|
|
10326
|
+
nameParameter,
|
|
10327
|
+
{ name: "id", in: "path", required: true, description: "Discovery session ID.", schema: stringSchema }
|
|
10328
|
+
],
|
|
10329
|
+
responses: {
|
|
10330
|
+
200: { description: "Session detail returned." },
|
|
10331
|
+
404: { description: "Project or session not found." }
|
|
10332
|
+
}
|
|
10333
|
+
},
|
|
10334
|
+
{
|
|
10335
|
+
method: "get",
|
|
10336
|
+
path: "/api/v1/projects/{name}/discover/sessions/{id}/promote",
|
|
10337
|
+
summary: "Preview a discovery promotion plan (read-only)",
|
|
10338
|
+
description: "Returns the payload `canonry discover promote` (PR 2) will persist: queries grouped by bucket, plus suggested new competitor domains. v1 is preview-only; the actual merge ships in PR 2.",
|
|
10339
|
+
tags: ["discovery"],
|
|
10340
|
+
parameters: [
|
|
10341
|
+
nameParameter,
|
|
10342
|
+
{ name: "id", in: "path", required: true, description: "Discovery session ID.", schema: stringSchema }
|
|
10343
|
+
],
|
|
10344
|
+
responses: {
|
|
10345
|
+
200: { description: "Promote preview returned." },
|
|
10346
|
+
404: { description: "Project or session not found." }
|
|
10347
|
+
}
|
|
10230
10348
|
}
|
|
10231
10349
|
];
|
|
10232
10350
|
var canonryLocalRouteCatalog = [
|
|
@@ -17312,6 +17430,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
17312
17430
|
let cursor = options.cursor;
|
|
17313
17431
|
let rawEntryCount = 0;
|
|
17314
17432
|
let skippedEntryCount = 0;
|
|
17433
|
+
let hasMore = false;
|
|
17315
17434
|
const events = [];
|
|
17316
17435
|
for (let page = 0; page < maxPages; page += 1) {
|
|
17317
17436
|
const url = new URL(endpoint);
|
|
@@ -17319,6 +17438,12 @@ async function listWordpressTrafficEvents(options) {
|
|
|
17319
17438
|
if (cursor !== void 0 && cursor !== "") {
|
|
17320
17439
|
url.searchParams.set("cursor", cursor);
|
|
17321
17440
|
}
|
|
17441
|
+
if (options.since !== void 0 && options.since !== "") {
|
|
17442
|
+
url.searchParams.set("since", options.since);
|
|
17443
|
+
}
|
|
17444
|
+
if (options.until !== void 0 && options.until !== "") {
|
|
17445
|
+
url.searchParams.set("until", options.until);
|
|
17446
|
+
}
|
|
17322
17447
|
const response = await fetch(url, {
|
|
17323
17448
|
method: "GET",
|
|
17324
17449
|
headers: {
|
|
@@ -17347,6 +17472,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
17347
17472
|
}
|
|
17348
17473
|
}
|
|
17349
17474
|
cursor = body.next_cursor ?? void 0;
|
|
17475
|
+
hasMore = Boolean(body.has_more) && Boolean(cursor);
|
|
17350
17476
|
if (!body.has_more || !cursor) break;
|
|
17351
17477
|
}
|
|
17352
17478
|
return {
|
|
@@ -17354,6 +17480,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
17354
17480
|
rawEntryCount,
|
|
17355
17481
|
skippedEntryCount,
|
|
17356
17482
|
nextCursor: cursor,
|
|
17483
|
+
hasMore,
|
|
17357
17484
|
endpoint
|
|
17358
17485
|
};
|
|
17359
17486
|
}
|
|
@@ -17363,6 +17490,8 @@ var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
|
17363
17490
|
var DEFAULT_PAGE_SIZE3 = 1e3;
|
|
17364
17491
|
var DEFAULT_MAX_PAGES3 = 5;
|
|
17365
17492
|
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
17493
|
+
var DEFAULT_WP_PAGE_SIZE = 500;
|
|
17494
|
+
var DEFAULT_WP_MAX_PAGES = 20;
|
|
17366
17495
|
var MAX_TRACKED_EVENT_IDS = 1e3;
|
|
17367
17496
|
var DEFAULT_BACKFILL_DAYS = 30;
|
|
17368
17497
|
var MAX_BACKFILL_DAYS = 30;
|
|
@@ -17404,14 +17533,10 @@ async function runBackfillTask(options) {
|
|
|
17404
17533
|
runId,
|
|
17405
17534
|
project,
|
|
17406
17535
|
sourceRow,
|
|
17407
|
-
gcpProjectId,
|
|
17408
|
-
serviceName,
|
|
17409
|
-
location,
|
|
17410
|
-
credential,
|
|
17411
17536
|
windowStart,
|
|
17412
17537
|
windowEnd,
|
|
17413
|
-
|
|
17414
|
-
|
|
17538
|
+
pullForBackfill,
|
|
17539
|
+
pullErrorPrefix
|
|
17415
17540
|
} = options;
|
|
17416
17541
|
const markFailed = (msg) => {
|
|
17417
17542
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -17423,33 +17548,11 @@ async function runBackfillTask(options) {
|
|
|
17423
17548
|
} catch {
|
|
17424
17549
|
}
|
|
17425
17550
|
};
|
|
17426
|
-
let
|
|
17427
|
-
try {
|
|
17428
|
-
accessToken = await resolveAccessToken2(credential);
|
|
17429
|
-
} catch (e) {
|
|
17430
|
-
markFailed(`Failed to resolve Cloud Run access token: ${e instanceof Error ? e.message : String(e)}`);
|
|
17431
|
-
return;
|
|
17432
|
-
}
|
|
17433
|
-
const allEvents = [];
|
|
17551
|
+
let allEvents;
|
|
17434
17552
|
try {
|
|
17435
|
-
|
|
17436
|
-
gcpProjectId,
|
|
17437
|
-
serviceName,
|
|
17438
|
-
location,
|
|
17439
|
-
startTime: windowStart.toISOString(),
|
|
17440
|
-
endTime: windowEnd.toISOString(),
|
|
17441
|
-
pageSize: DEFAULT_PAGE_SIZE3,
|
|
17442
|
-
maxPages: BACKFILL_MAX_PAGES,
|
|
17443
|
-
// Backfill is intentionally `firstSync: false`. We don't want desc
|
|
17444
|
-
// ordering — the in-memory rollup builder handles any order, and the
|
|
17445
|
-
// ring-buffer reseed at the end takes the most-recent IDs from the
|
|
17446
|
-
// dedupedEvents anyway.
|
|
17447
|
-
firstSync: false,
|
|
17448
|
-
orderBy: "timestamp asc"
|
|
17449
|
-
});
|
|
17450
|
-
allEvents.push(...page.events);
|
|
17553
|
+
allEvents = await pullForBackfill();
|
|
17451
17554
|
} catch (e) {
|
|
17452
|
-
markFailed(
|
|
17555
|
+
markFailed(`${pullErrorPrefix}: ${e instanceof Error ? e.message : String(e)}`);
|
|
17453
17556
|
return;
|
|
17454
17557
|
}
|
|
17455
17558
|
if (allEvents.length === 0) {
|
|
@@ -17739,33 +17842,12 @@ async function trafficRoutes(app, opts) {
|
|
|
17739
17842
|
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
17740
17843
|
throw notFound("Traffic source", request.params.id);
|
|
17741
17844
|
}
|
|
17742
|
-
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"]) {
|
|
17845
|
+
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"] && sourceRow.sourceType !== TrafficSourceTypes.wordpress) {
|
|
17743
17846
|
throw validationError(
|
|
17744
|
-
`Sync for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run
|
|
17847
|
+
`Sync for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run and wordpress are supported in v1.`
|
|
17745
17848
|
);
|
|
17746
17849
|
}
|
|
17747
|
-
const credentialStore = opts.cloudRunCredentialStore;
|
|
17748
|
-
if (!credentialStore) {
|
|
17749
|
-
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
17750
|
-
}
|
|
17751
|
-
const credential = credentialStore.getConnection(project.name);
|
|
17752
|
-
if (!credential) {
|
|
17753
|
-
throw validationError(
|
|
17754
|
-
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
17755
|
-
);
|
|
17756
|
-
}
|
|
17757
|
-
const config = parseSourceConfig(sourceRow);
|
|
17758
|
-
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
17759
|
-
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
17760
|
-
const location = config.location ?? credential.location ?? void 0;
|
|
17761
|
-
const requestedMinutes = request.body?.sinceMinutes;
|
|
17762
|
-
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
17763
17850
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
17764
|
-
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
17765
|
-
const lastSyncedMs = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
17766
|
-
const windowStart = new Date(
|
|
17767
|
-
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
17768
|
-
);
|
|
17769
17851
|
const startedAt = windowEnd.toISOString();
|
|
17770
17852
|
const syncStartedAtMs = windowEnd.getTime();
|
|
17771
17853
|
const runId = crypto20.randomUUID();
|
|
@@ -17799,32 +17881,100 @@ async function trafficRoutes(app, opts) {
|
|
|
17799
17881
|
} catch {
|
|
17800
17882
|
}
|
|
17801
17883
|
};
|
|
17802
|
-
let
|
|
17803
|
-
|
|
17804
|
-
|
|
17805
|
-
|
|
17806
|
-
|
|
17807
|
-
|
|
17808
|
-
|
|
17809
|
-
|
|
17810
|
-
|
|
17811
|
-
|
|
17812
|
-
|
|
17813
|
-
|
|
17814
|
-
|
|
17815
|
-
|
|
17816
|
-
|
|
17817
|
-
|
|
17818
|
-
|
|
17819
|
-
|
|
17820
|
-
|
|
17821
|
-
|
|
17822
|
-
|
|
17823
|
-
|
|
17824
|
-
|
|
17825
|
-
const
|
|
17826
|
-
|
|
17827
|
-
|
|
17884
|
+
let windowStart;
|
|
17885
|
+
let allEvents;
|
|
17886
|
+
let nextCursor;
|
|
17887
|
+
let auditAction;
|
|
17888
|
+
if (sourceRow.sourceType === TrafficSourceTypes["cloud-run"]) {
|
|
17889
|
+
auditAction = "traffic.cloud-run.synced";
|
|
17890
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
17891
|
+
if (!credentialStore) {
|
|
17892
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
17893
|
+
}
|
|
17894
|
+
const credential = credentialStore.getConnection(project.name);
|
|
17895
|
+
if (!credential) {
|
|
17896
|
+
throw validationError(
|
|
17897
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
17898
|
+
);
|
|
17899
|
+
}
|
|
17900
|
+
const config = parseSourceConfig(sourceRow);
|
|
17901
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
17902
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
17903
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
17904
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
17905
|
+
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
17906
|
+
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
17907
|
+
const lastSyncedMs = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
17908
|
+
windowStart = new Date(
|
|
17909
|
+
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
17910
|
+
);
|
|
17911
|
+
let accessToken;
|
|
17912
|
+
try {
|
|
17913
|
+
accessToken = await resolveAccessToken2(credential);
|
|
17914
|
+
} catch (e) {
|
|
17915
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
17916
|
+
markFailed(msg, "PROVIDER_AUTH");
|
|
17917
|
+
throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
17918
|
+
}
|
|
17919
|
+
const isFirstSync = !sourceRow.lastSyncedAt;
|
|
17920
|
+
try {
|
|
17921
|
+
const page = await pullEvents(accessToken, {
|
|
17922
|
+
gcpProjectId,
|
|
17923
|
+
serviceName,
|
|
17924
|
+
location,
|
|
17925
|
+
startTime: windowStart.toISOString(),
|
|
17926
|
+
endTime: windowEnd.toISOString(),
|
|
17927
|
+
pageSize,
|
|
17928
|
+
maxPages,
|
|
17929
|
+
firstSync: isFirstSync
|
|
17930
|
+
});
|
|
17931
|
+
allEvents = page.events;
|
|
17932
|
+
} catch (e) {
|
|
17933
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
17934
|
+
markFailed(msg, "PROVIDER_PULL");
|
|
17935
|
+
throw providerError(`Cloud Run pull failed: ${msg}`);
|
|
17936
|
+
}
|
|
17937
|
+
} else {
|
|
17938
|
+
auditAction = "traffic.wordpress.synced";
|
|
17939
|
+
const credentialStore = opts.wordpressTrafficCredentialStore;
|
|
17940
|
+
if (!credentialStore) {
|
|
17941
|
+
throw validationError("WordPress traffic credential storage is not configured for this deployment");
|
|
17942
|
+
}
|
|
17943
|
+
const credential = credentialStore.getConnection(project.name);
|
|
17944
|
+
if (!credential) {
|
|
17945
|
+
app.db.delete(runs).where(eq23(runs.id, runId)).run();
|
|
17946
|
+
throw validationError(
|
|
17947
|
+
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
17948
|
+
);
|
|
17949
|
+
}
|
|
17950
|
+
windowStart = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt) : windowEnd;
|
|
17951
|
+
const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
|
|
17952
|
+
const wpMaxPages = opts.defaultWordpressMaxPages ?? DEFAULT_WP_MAX_PAGES;
|
|
17953
|
+
const collected = [];
|
|
17954
|
+
let cursor = sourceRow.lastCursor ?? void 0;
|
|
17955
|
+
try {
|
|
17956
|
+
for (let page = 0; page < wpMaxPages; page += 1) {
|
|
17957
|
+
const pageResult = await pullWordpressEvents({
|
|
17958
|
+
baseUrl: credential.baseUrl,
|
|
17959
|
+
username: credential.username,
|
|
17960
|
+
applicationPassword: credential.applicationPassword,
|
|
17961
|
+
cursor,
|
|
17962
|
+
pageSize: wpPageSize,
|
|
17963
|
+
maxPages: 1
|
|
17964
|
+
});
|
|
17965
|
+
collected.push(...pageResult.events);
|
|
17966
|
+
const previousCursor = cursor;
|
|
17967
|
+
cursor = pageResult.nextCursor;
|
|
17968
|
+
if (!pageResult.hasMore) break;
|
|
17969
|
+
if (!cursor || cursor === previousCursor) break;
|
|
17970
|
+
}
|
|
17971
|
+
allEvents = collected;
|
|
17972
|
+
nextCursor = cursor;
|
|
17973
|
+
} catch (e) {
|
|
17974
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
17975
|
+
markFailed(msg, "PROVIDER_PULL");
|
|
17976
|
+
throw providerError(`WordPress pull failed: ${msg}`);
|
|
17977
|
+
}
|
|
17828
17978
|
}
|
|
17829
17979
|
let crawlerBucketRows = 0;
|
|
17830
17980
|
let aiReferralBucketRows = 0;
|
|
@@ -17951,7 +18101,7 @@ async function trafficRoutes(app, opts) {
|
|
|
17951
18101
|
}).run();
|
|
17952
18102
|
sampleRows += 1;
|
|
17953
18103
|
}
|
|
17954
|
-
|
|
18104
|
+
const sourceUpdate = {
|
|
17955
18105
|
status: TrafficSourceStatuses.connected,
|
|
17956
18106
|
// Advance to windowEnd, not finishedAt — events arriving at the
|
|
17957
18107
|
// source between windowEnd and finishedAt aren't in this pull's
|
|
@@ -17961,13 +18111,17 @@ async function trafficRoutes(app, opts) {
|
|
|
17961
18111
|
lastError: null,
|
|
17962
18112
|
lastEventIds: JSON.stringify(nextEventIds),
|
|
17963
18113
|
updatedAt: finishedAt
|
|
17964
|
-
}
|
|
18114
|
+
};
|
|
18115
|
+
if (sourceRow.sourceType === TrafficSourceTypes.wordpress) {
|
|
18116
|
+
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
18117
|
+
}
|
|
18118
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
17965
18119
|
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
17966
18120
|
});
|
|
17967
18121
|
writeAuditLog(app.db, {
|
|
17968
18122
|
projectId: project.id,
|
|
17969
18123
|
actor: "api",
|
|
17970
|
-
action:
|
|
18124
|
+
action: auditAction,
|
|
17971
18125
|
entityType: "traffic_source",
|
|
17972
18126
|
entityId: sourceRow.id
|
|
17973
18127
|
});
|
|
@@ -18005,19 +18159,9 @@ async function trafficRoutes(app, opts) {
|
|
|
18005
18159
|
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
18006
18160
|
throw notFound("Traffic source", request.params.id);
|
|
18007
18161
|
}
|
|
18008
|
-
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"]) {
|
|
18162
|
+
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"] && sourceRow.sourceType !== TrafficSourceTypes.wordpress) {
|
|
18009
18163
|
throw validationError(
|
|
18010
|
-
`Backfill for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run
|
|
18011
|
-
);
|
|
18012
|
-
}
|
|
18013
|
-
const credentialStore = opts.cloudRunCredentialStore;
|
|
18014
|
-
if (!credentialStore) {
|
|
18015
|
-
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
18016
|
-
}
|
|
18017
|
-
const credential = credentialStore.getConnection(project.name);
|
|
18018
|
-
if (!credential) {
|
|
18019
|
-
throw validationError(
|
|
18020
|
-
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
18164
|
+
`Backfill for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run and wordpress are supported in v1.`
|
|
18021
18165
|
);
|
|
18022
18166
|
}
|
|
18023
18167
|
const requestedDays = request.body?.days ?? DEFAULT_BACKFILL_DAYS;
|
|
@@ -18025,13 +18169,86 @@ async function trafficRoutes(app, opts) {
|
|
|
18025
18169
|
throw validationError('"days" must be a positive integer');
|
|
18026
18170
|
}
|
|
18027
18171
|
const appliedDays = Math.min(requestedDays, MAX_BACKFILL_DAYS);
|
|
18028
|
-
const config = parseSourceConfig(sourceRow);
|
|
18029
|
-
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
18030
|
-
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
18031
|
-
const location = config.location ?? credential.location ?? void 0;
|
|
18032
18172
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
18033
18173
|
const windowStart = new Date(windowEnd.getTime() - appliedDays * 864e5);
|
|
18034
18174
|
windowStart.setUTCMinutes(0, 0, 0);
|
|
18175
|
+
let pullForBackfill;
|
|
18176
|
+
let pullErrorPrefix;
|
|
18177
|
+
if (sourceRow.sourceType === TrafficSourceTypes["cloud-run"]) {
|
|
18178
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
18179
|
+
if (!credentialStore) {
|
|
18180
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
18181
|
+
}
|
|
18182
|
+
const credential = credentialStore.getConnection(project.name);
|
|
18183
|
+
if (!credential) {
|
|
18184
|
+
throw validationError(
|
|
18185
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
18186
|
+
);
|
|
18187
|
+
}
|
|
18188
|
+
const config = parseSourceConfig(sourceRow);
|
|
18189
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
18190
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
18191
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
18192
|
+
pullErrorPrefix = "Cloud Run pull failed";
|
|
18193
|
+
pullForBackfill = async () => {
|
|
18194
|
+
const accessToken = await resolveAccessToken2(credential);
|
|
18195
|
+
const page = await pullEvents(accessToken, {
|
|
18196
|
+
gcpProjectId,
|
|
18197
|
+
serviceName,
|
|
18198
|
+
location,
|
|
18199
|
+
startTime: windowStart.toISOString(),
|
|
18200
|
+
endTime: windowEnd.toISOString(),
|
|
18201
|
+
pageSize: DEFAULT_PAGE_SIZE3,
|
|
18202
|
+
maxPages: BACKFILL_MAX_PAGES,
|
|
18203
|
+
// Backfill is intentionally `firstSync: false`. We don't want desc
|
|
18204
|
+
// ordering — the in-memory rollup builder handles any order, and the
|
|
18205
|
+
// ring-buffer reseed at the end takes the most-recent IDs from the
|
|
18206
|
+
// dedupedEvents anyway.
|
|
18207
|
+
firstSync: false,
|
|
18208
|
+
orderBy: "timestamp asc"
|
|
18209
|
+
});
|
|
18210
|
+
return page.events;
|
|
18211
|
+
};
|
|
18212
|
+
} else {
|
|
18213
|
+
const credentialStore = opts.wordpressTrafficCredentialStore;
|
|
18214
|
+
if (!credentialStore) {
|
|
18215
|
+
throw validationError("WordPress traffic credential storage is not configured for this deployment");
|
|
18216
|
+
}
|
|
18217
|
+
const credential = credentialStore.getConnection(project.name);
|
|
18218
|
+
if (!credential) {
|
|
18219
|
+
throw validationError(
|
|
18220
|
+
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
18221
|
+
);
|
|
18222
|
+
}
|
|
18223
|
+
const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
|
|
18224
|
+
pullErrorPrefix = "WordPress pull failed";
|
|
18225
|
+
pullForBackfill = async () => {
|
|
18226
|
+
const collected = [];
|
|
18227
|
+
const windowStartIso = windowStart.toISOString();
|
|
18228
|
+
const windowEndIso = windowEnd.toISOString();
|
|
18229
|
+
let cursor = void 0;
|
|
18230
|
+
for (let page = 0; page < BACKFILL_MAX_PAGES; page += 1) {
|
|
18231
|
+
const pageResult = await pullWordpressEvents({
|
|
18232
|
+
baseUrl: credential.baseUrl,
|
|
18233
|
+
username: credential.username,
|
|
18234
|
+
applicationPassword: credential.applicationPassword,
|
|
18235
|
+
cursor,
|
|
18236
|
+
pageSize: wpPageSize,
|
|
18237
|
+
// Each call fetches a single page; the for-loop drives
|
|
18238
|
+
// continuation. Matches the WP sync path's pattern.
|
|
18239
|
+
maxPages: 1,
|
|
18240
|
+
since: windowStartIso,
|
|
18241
|
+
until: windowEndIso
|
|
18242
|
+
});
|
|
18243
|
+
collected.push(...pageResult.events);
|
|
18244
|
+
const previousCursor = cursor;
|
|
18245
|
+
cursor = pageResult.nextCursor;
|
|
18246
|
+
if (!pageResult.hasMore) break;
|
|
18247
|
+
if (!cursor || cursor === previousCursor) break;
|
|
18248
|
+
}
|
|
18249
|
+
return collected;
|
|
18250
|
+
};
|
|
18251
|
+
}
|
|
18035
18252
|
const startedAt = windowEnd.toISOString();
|
|
18036
18253
|
const runId = crypto20.randomUUID();
|
|
18037
18254
|
app.db.insert(runs).values({
|
|
@@ -18049,15 +18266,10 @@ async function trafficRoutes(app, opts) {
|
|
|
18049
18266
|
runId,
|
|
18050
18267
|
project,
|
|
18051
18268
|
sourceRow,
|
|
18052
|
-
gcpProjectId,
|
|
18053
|
-
serviceName,
|
|
18054
|
-
location,
|
|
18055
|
-
credential,
|
|
18056
18269
|
windowStart,
|
|
18057
18270
|
windowEnd,
|
|
18058
|
-
|
|
18059
|
-
|
|
18060
|
-
resolveAccessToken: resolveAccessToken2
|
|
18271
|
+
pullForBackfill,
|
|
18272
|
+
pullErrorPrefix
|
|
18061
18273
|
}).catch(() => {
|
|
18062
18274
|
});
|
|
18063
18275
|
const response = {
|
|
@@ -19288,6 +19500,298 @@ async function doctorRoutes(app, opts) {
|
|
|
19288
19500
|
});
|
|
19289
19501
|
}
|
|
19290
19502
|
|
|
19503
|
+
// ../api-routes/src/discovery/routes.ts
|
|
19504
|
+
import crypto21 from "crypto";
|
|
19505
|
+
import { eq as eq25, desc as desc13 } from "drizzle-orm";
|
|
19506
|
+
async function discoveryRoutes(app, opts) {
|
|
19507
|
+
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
19508
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19509
|
+
const parsed = discoveryRunRequestSchema.safeParse(request.body ?? {});
|
|
19510
|
+
if (!parsed.success) {
|
|
19511
|
+
throw validationError("Invalid discovery run request", {
|
|
19512
|
+
issues: parsed.error.issues.map((issue) => ({
|
|
19513
|
+
path: issue.path.join("."),
|
|
19514
|
+
message: issue.message
|
|
19515
|
+
}))
|
|
19516
|
+
});
|
|
19517
|
+
}
|
|
19518
|
+
const icpDescription = parsed.data.icpDescription?.trim() || (project.icpDescription ?? "").trim();
|
|
19519
|
+
if (!icpDescription) {
|
|
19520
|
+
throw validationError(
|
|
19521
|
+
"icpDescription is required. Pass it in the request body or store it on the project (spec.icpDescription)."
|
|
19522
|
+
);
|
|
19523
|
+
}
|
|
19524
|
+
if (!opts.onDiscoveryRunRequested) {
|
|
19525
|
+
throw validationError("Discovery is not available on this deployment.", {
|
|
19526
|
+
reason: "no-discovery-handler"
|
|
19527
|
+
});
|
|
19528
|
+
}
|
|
19529
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19530
|
+
const sessionId = crypto21.randomUUID();
|
|
19531
|
+
const runId = crypto21.randomUUID();
|
|
19532
|
+
app.db.transaction((tx) => {
|
|
19533
|
+
tx.insert(discoverySessions).values({
|
|
19534
|
+
id: sessionId,
|
|
19535
|
+
projectId: project.id,
|
|
19536
|
+
runId,
|
|
19537
|
+
status: DiscoverySessionStatuses.queued,
|
|
19538
|
+
icpDescription,
|
|
19539
|
+
dedupThreshold: parsed.data.dedupThreshold,
|
|
19540
|
+
competitorMap: "[]",
|
|
19541
|
+
createdAt: now
|
|
19542
|
+
}).run();
|
|
19543
|
+
tx.insert(runs).values({
|
|
19544
|
+
id: runId,
|
|
19545
|
+
projectId: project.id,
|
|
19546
|
+
kind: RunKinds["aeo-discover-probe"],
|
|
19547
|
+
status: RunStatuses.queued,
|
|
19548
|
+
trigger: RunTriggers.manual,
|
|
19549
|
+
createdAt: now
|
|
19550
|
+
}).run();
|
|
19551
|
+
writeAuditLog(tx, {
|
|
19552
|
+
projectId: project.id,
|
|
19553
|
+
actor: "api",
|
|
19554
|
+
action: "discovery.created",
|
|
19555
|
+
entityType: "discovery_session",
|
|
19556
|
+
entityId: sessionId
|
|
19557
|
+
});
|
|
19558
|
+
});
|
|
19559
|
+
opts.onDiscoveryRunRequested({
|
|
19560
|
+
runId,
|
|
19561
|
+
sessionId,
|
|
19562
|
+
projectId: project.id,
|
|
19563
|
+
icpDescription,
|
|
19564
|
+
dedupThreshold: parsed.data.dedupThreshold,
|
|
19565
|
+
maxProbes: parsed.data.maxProbes
|
|
19566
|
+
});
|
|
19567
|
+
return reply.status(201).send({ runId, sessionId, status: "running" });
|
|
19568
|
+
});
|
|
19569
|
+
app.get(
|
|
19570
|
+
"/projects/:name/discover/sessions",
|
|
19571
|
+
async (request, reply) => {
|
|
19572
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19573
|
+
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
19574
|
+
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
19575
|
+
const rows = app.db.select().from(discoverySessions).where(eq25(discoverySessions.projectId, project.id)).orderBy(desc13(discoverySessions.createdAt)).limit(limit).all();
|
|
19576
|
+
return reply.send(rows.map(serializeSession));
|
|
19577
|
+
}
|
|
19578
|
+
);
|
|
19579
|
+
app.get(
|
|
19580
|
+
"/projects/:name/discover/sessions/:id",
|
|
19581
|
+
async (request, reply) => {
|
|
19582
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19583
|
+
const session = app.db.select().from(discoverySessions).where(eq25(discoverySessions.id, request.params.id)).get();
|
|
19584
|
+
if (!session || session.projectId !== project.id) {
|
|
19585
|
+
throw notFound("Discovery session", request.params.id);
|
|
19586
|
+
}
|
|
19587
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq25(discoveryProbes.sessionId, session.id)).all();
|
|
19588
|
+
const detail = {
|
|
19589
|
+
...serializeSession(session),
|
|
19590
|
+
probes: probeRows.map(serializeProbe)
|
|
19591
|
+
};
|
|
19592
|
+
return reply.send(detail);
|
|
19593
|
+
}
|
|
19594
|
+
);
|
|
19595
|
+
app.get(
|
|
19596
|
+
"/projects/:name/discover/sessions/:id/promote",
|
|
19597
|
+
async (request, reply) => {
|
|
19598
|
+
const project = resolveProject(app.db, request.params.name);
|
|
19599
|
+
const session = app.db.select().from(discoverySessions).where(eq25(discoverySessions.id, request.params.id)).get();
|
|
19600
|
+
if (!session || session.projectId !== project.id) {
|
|
19601
|
+
throw notFound("Discovery session", request.params.id);
|
|
19602
|
+
}
|
|
19603
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq25(discoveryProbes.sessionId, session.id)).all();
|
|
19604
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq25(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
19605
|
+
const seenCompetitors = new Set(existingCompetitors);
|
|
19606
|
+
const cited = /* @__PURE__ */ new Set();
|
|
19607
|
+
const aspirational = /* @__PURE__ */ new Set();
|
|
19608
|
+
const wasted = /* @__PURE__ */ new Set();
|
|
19609
|
+
for (const probe of probeRows) {
|
|
19610
|
+
const bucket = probe.bucket;
|
|
19611
|
+
if (!bucket) continue;
|
|
19612
|
+
if (bucket === DiscoveryBuckets.cited) cited.add(probe.query);
|
|
19613
|
+
else if (bucket === DiscoveryBuckets.aspirational) aspirational.add(probe.query);
|
|
19614
|
+
else if (bucket === DiscoveryBuckets["wasted-surface"]) wasted.add(probe.query);
|
|
19615
|
+
}
|
|
19616
|
+
const competitorMap = parseJsonColumn(session.competitorMap, []);
|
|
19617
|
+
const newCompetitors = competitorMap.filter((entry) => !seenCompetitors.has(entry.domain.toLowerCase())).slice(0, 20);
|
|
19618
|
+
return reply.send({
|
|
19619
|
+
sessionId: session.id,
|
|
19620
|
+
projectId: project.id,
|
|
19621
|
+
queriesByBucket: {
|
|
19622
|
+
cited: Array.from(cited).sort(),
|
|
19623
|
+
aspirational: Array.from(aspirational).sort(),
|
|
19624
|
+
"wasted-surface": Array.from(wasted).sort()
|
|
19625
|
+
},
|
|
19626
|
+
suggestedCompetitors: newCompetitors,
|
|
19627
|
+
status: session.status
|
|
19628
|
+
});
|
|
19629
|
+
}
|
|
19630
|
+
);
|
|
19631
|
+
}
|
|
19632
|
+
function serializeSession(row) {
|
|
19633
|
+
return {
|
|
19634
|
+
id: row.id,
|
|
19635
|
+
projectId: row.projectId,
|
|
19636
|
+
status: row.status,
|
|
19637
|
+
icpDescription: row.icpDescription ?? null,
|
|
19638
|
+
seedProvider: row.seedProvider ?? null,
|
|
19639
|
+
seedCountRaw: row.seedCountRaw ?? null,
|
|
19640
|
+
seedCount: row.seedCount ?? null,
|
|
19641
|
+
dedupThreshold: row.dedupThreshold ?? null,
|
|
19642
|
+
probeCount: row.probeCount ?? null,
|
|
19643
|
+
citedCount: row.citedCount ?? null,
|
|
19644
|
+
aspirationalCount: row.aspirationalCount ?? null,
|
|
19645
|
+
wastedCount: row.wastedCount ?? null,
|
|
19646
|
+
competitorMap: parseJsonColumn(row.competitorMap, []),
|
|
19647
|
+
error: row.error ?? null,
|
|
19648
|
+
startedAt: row.startedAt ?? null,
|
|
19649
|
+
finishedAt: row.finishedAt ?? null,
|
|
19650
|
+
createdAt: row.createdAt
|
|
19651
|
+
};
|
|
19652
|
+
}
|
|
19653
|
+
function serializeProbe(row) {
|
|
19654
|
+
const bucketParsed = row.bucket ? discoveryBucketSchema.safeParse(row.bucket) : null;
|
|
19655
|
+
const stateParsed = citationStateSchema.safeParse(row.citationState);
|
|
19656
|
+
return {
|
|
19657
|
+
id: row.id,
|
|
19658
|
+
sessionId: row.sessionId,
|
|
19659
|
+
projectId: row.projectId,
|
|
19660
|
+
query: row.query,
|
|
19661
|
+
bucket: bucketParsed?.success ? bucketParsed.data : null,
|
|
19662
|
+
citationState: stateParsed.success ? stateParsed.data : "not-cited",
|
|
19663
|
+
citedDomains: parseJsonColumn(row.citedDomains, []),
|
|
19664
|
+
createdAt: row.createdAt
|
|
19665
|
+
};
|
|
19666
|
+
}
|
|
19667
|
+
|
|
19668
|
+
// ../api-routes/src/discovery/orchestrate.ts
|
|
19669
|
+
import crypto22 from "crypto";
|
|
19670
|
+
import { eq as eq26 } from "drizzle-orm";
|
|
19671
|
+
var DEFAULT_DEDUP_THRESHOLD = 0.85;
|
|
19672
|
+
var DEFAULT_MAX_PROBES = 100;
|
|
19673
|
+
var ABSOLUTE_MAX_PROBES = 500;
|
|
19674
|
+
function classifyProbeBucket(input) {
|
|
19675
|
+
const cited = new Set(input.citedDomains.map((d) => d.toLowerCase()));
|
|
19676
|
+
const canonicalHit = input.project.canonicalDomains.some((d) => cited.has(d.toLowerCase()));
|
|
19677
|
+
if (canonicalHit) return DiscoveryBuckets.cited;
|
|
19678
|
+
const competitorHit = input.project.competitorDomains.some((d) => cited.has(d.toLowerCase()));
|
|
19679
|
+
if (competitorHit) return DiscoveryBuckets["wasted-surface"];
|
|
19680
|
+
return DiscoveryBuckets.aspirational;
|
|
19681
|
+
}
|
|
19682
|
+
function buildCompetitorMap(probes, project) {
|
|
19683
|
+
const canonical = new Set(project.canonicalDomains.map((d) => d.toLowerCase()));
|
|
19684
|
+
const counts = /* @__PURE__ */ new Map();
|
|
19685
|
+
for (const probe of probes) {
|
|
19686
|
+
const seenInProbe = /* @__PURE__ */ new Set();
|
|
19687
|
+
for (const raw of probe.citedDomains) {
|
|
19688
|
+
const domain = raw.toLowerCase();
|
|
19689
|
+
if (canonical.has(domain)) continue;
|
|
19690
|
+
if (seenInProbe.has(domain)) continue;
|
|
19691
|
+
seenInProbe.add(domain);
|
|
19692
|
+
counts.set(domain, (counts.get(domain) ?? 0) + 1);
|
|
19693
|
+
}
|
|
19694
|
+
}
|
|
19695
|
+
return Array.from(counts.entries()).map(([domain, hits]) => ({ domain, hits })).sort((a, b) => b.hits - a.hits || a.domain.localeCompare(b.domain));
|
|
19696
|
+
}
|
|
19697
|
+
async function pickCanonicals(candidates, deps, dedupThreshold) {
|
|
19698
|
+
if (candidates.length === 0) return [];
|
|
19699
|
+
if (candidates.length === 1) return candidates;
|
|
19700
|
+
const vectors = await deps.embed(candidates);
|
|
19701
|
+
const clusters = clusterByCosine(candidates, vectors, dedupThreshold);
|
|
19702
|
+
return clusters.map(pickClusterRepresentative);
|
|
19703
|
+
}
|
|
19704
|
+
async function executeDiscovery(opts) {
|
|
19705
|
+
const dedupThreshold = opts.dedupThreshold ?? DEFAULT_DEDUP_THRESHOLD;
|
|
19706
|
+
const requestedMax = opts.maxProbes ?? DEFAULT_MAX_PROBES;
|
|
19707
|
+
const maxProbes = Math.min(Math.max(1, requestedMax), ABSOLUTE_MAX_PROBES);
|
|
19708
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19709
|
+
opts.db.update(discoverySessions).set({
|
|
19710
|
+
status: DiscoverySessionStatuses.seeding,
|
|
19711
|
+
dedupThreshold,
|
|
19712
|
+
startedAt
|
|
19713
|
+
}).where(eq26(discoverySessions.id, opts.sessionId)).run();
|
|
19714
|
+
const seedResult = await opts.deps.seed({
|
|
19715
|
+
project: opts.project,
|
|
19716
|
+
icpDescription: opts.icpDescription
|
|
19717
|
+
});
|
|
19718
|
+
const rawCandidates = dedupeStrings(seedResult.candidates);
|
|
19719
|
+
const seedCountRaw = rawCandidates.length;
|
|
19720
|
+
const canonicals = await pickCanonicals(
|
|
19721
|
+
rawCandidates,
|
|
19722
|
+
{ embed: opts.deps.embed },
|
|
19723
|
+
dedupThreshold
|
|
19724
|
+
);
|
|
19725
|
+
const probedCanonicals = canonicals.slice(0, maxProbes);
|
|
19726
|
+
const seedCount = probedCanonicals.length;
|
|
19727
|
+
opts.db.update(discoverySessions).set({
|
|
19728
|
+
status: DiscoverySessionStatuses.probing,
|
|
19729
|
+
seedProvider: seedResult.provider,
|
|
19730
|
+
seedCountRaw,
|
|
19731
|
+
seedCount
|
|
19732
|
+
}).where(eq26(discoverySessions.id, opts.sessionId)).run();
|
|
19733
|
+
const probeRows = [];
|
|
19734
|
+
const buckets = { cited: 0, aspirational: 0, "wasted-surface": 0 };
|
|
19735
|
+
for (const query of probedCanonicals) {
|
|
19736
|
+
const probe = await opts.deps.probe({ project: opts.project, query });
|
|
19737
|
+
const bucket = classifyProbeBucket({
|
|
19738
|
+
citationState: probe.citationState,
|
|
19739
|
+
citedDomains: probe.citedDomains,
|
|
19740
|
+
project: opts.project
|
|
19741
|
+
});
|
|
19742
|
+
probeRows.push({ citedDomains: probe.citedDomains, bucket });
|
|
19743
|
+
buckets[bucket]++;
|
|
19744
|
+
opts.db.insert(discoveryProbes).values({
|
|
19745
|
+
id: crypto22.randomUUID(),
|
|
19746
|
+
sessionId: opts.sessionId,
|
|
19747
|
+
projectId: opts.project.id,
|
|
19748
|
+
query,
|
|
19749
|
+
bucket,
|
|
19750
|
+
citationState: probe.citationState,
|
|
19751
|
+
citedDomains: JSON.stringify(probe.citedDomains),
|
|
19752
|
+
rawResponse: JSON.stringify(probe.rawResponse),
|
|
19753
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19754
|
+
}).run();
|
|
19755
|
+
}
|
|
19756
|
+
const competitorMap = buildCompetitorMap(probeRows, opts.project);
|
|
19757
|
+
opts.db.update(discoverySessions).set({
|
|
19758
|
+
status: DiscoverySessionStatuses.completed,
|
|
19759
|
+
probeCount: probedCanonicals.length,
|
|
19760
|
+
citedCount: buckets.cited,
|
|
19761
|
+
aspirationalCount: buckets.aspirational,
|
|
19762
|
+
wastedCount: buckets["wasted-surface"],
|
|
19763
|
+
competitorMap: JSON.stringify(competitorMap),
|
|
19764
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19765
|
+
}).where(eq26(discoverySessions.id, opts.sessionId)).run();
|
|
19766
|
+
return {
|
|
19767
|
+
buckets,
|
|
19768
|
+
competitorMap,
|
|
19769
|
+
seedCountRaw,
|
|
19770
|
+
seedCount,
|
|
19771
|
+
seedProvider: seedResult.provider
|
|
19772
|
+
};
|
|
19773
|
+
}
|
|
19774
|
+
function markSessionFailed(db, sessionId, error) {
|
|
19775
|
+
db.update(discoverySessions).set({
|
|
19776
|
+
status: DiscoverySessionStatuses.failed,
|
|
19777
|
+
error,
|
|
19778
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19779
|
+
}).where(eq26(discoverySessions.id, sessionId)).run();
|
|
19780
|
+
}
|
|
19781
|
+
function dedupeStrings(input) {
|
|
19782
|
+
const seen = /* @__PURE__ */ new Set();
|
|
19783
|
+
const out = [];
|
|
19784
|
+
for (const raw of input) {
|
|
19785
|
+
const trimmed = raw.trim();
|
|
19786
|
+
if (!trimmed) continue;
|
|
19787
|
+
const key = trimmed.toLowerCase();
|
|
19788
|
+
if (seen.has(key)) continue;
|
|
19789
|
+
seen.add(key);
|
|
19790
|
+
out.push(trimmed);
|
|
19791
|
+
}
|
|
19792
|
+
return out;
|
|
19793
|
+
}
|
|
19794
|
+
|
|
19291
19795
|
// ../api-routes/src/index.ts
|
|
19292
19796
|
async function apiRoutes(app, opts) {
|
|
19293
19797
|
app.decorate("db", opts.db);
|
|
@@ -19418,6 +19922,9 @@ async function apiRoutes(app, opts) {
|
|
|
19418
19922
|
listCachedReleases: opts.listCachedReleases,
|
|
19419
19923
|
discoverLatestRelease: opts.discoverLatestRelease
|
|
19420
19924
|
});
|
|
19925
|
+
await api.register(discoveryRoutes, {
|
|
19926
|
+
onDiscoveryRunRequested: opts.onDiscoveryRunRequested
|
|
19927
|
+
});
|
|
19421
19928
|
await api.register(doctorRoutes, {
|
|
19422
19929
|
googleConnectionStore: opts.googleConnectionStore,
|
|
19423
19930
|
bingConnectionStore: opts.bingConnectionStore,
|
|
@@ -19834,6 +20341,54 @@ function responseToRecord(response) {
|
|
|
19834
20341
|
}
|
|
19835
20342
|
}
|
|
19836
20343
|
|
|
20344
|
+
// ../provider-gemini/src/embeddings.ts
|
|
20345
|
+
import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
|
|
20346
|
+
var DEFAULT_EMBED_MODEL = "gemini-embedding-001";
|
|
20347
|
+
var DEFAULT_OUTPUT_DIMENSIONALITY = 768;
|
|
20348
|
+
var CLUSTERING_TASK_TYPE = "CLUSTERING";
|
|
20349
|
+
async function embedQueries(queries2, options) {
|
|
20350
|
+
if (queries2.length === 0) return [];
|
|
20351
|
+
if (!options.apiKey && !options.client) {
|
|
20352
|
+
throw new Error("embedQueries: missing apiKey");
|
|
20353
|
+
}
|
|
20354
|
+
const client = options.client ?? createGeminiEmbedClient(options.apiKey);
|
|
20355
|
+
return client.embedBatch(queries2, {
|
|
20356
|
+
model: options.model ?? DEFAULT_EMBED_MODEL,
|
|
20357
|
+
taskType: CLUSTERING_TASK_TYPE,
|
|
20358
|
+
outputDimensionality: options.outputDimensionality ?? DEFAULT_OUTPUT_DIMENSIONALITY
|
|
20359
|
+
});
|
|
20360
|
+
}
|
|
20361
|
+
function extractEmbeddingVectors(response, expectedLength) {
|
|
20362
|
+
const embeddings = response?.embeddings ?? [];
|
|
20363
|
+
if (embeddings.length !== expectedLength) {
|
|
20364
|
+
throw new Error(
|
|
20365
|
+
`embedQueries: expected ${expectedLength} embeddings, got ${embeddings.length}`
|
|
20366
|
+
);
|
|
20367
|
+
}
|
|
20368
|
+
return embeddings.map((e, i) => {
|
|
20369
|
+
if (!e.values || e.values.length === 0) {
|
|
20370
|
+
throw new Error(`embedQueries: missing values for query at index ${i}`);
|
|
20371
|
+
}
|
|
20372
|
+
return e.values;
|
|
20373
|
+
});
|
|
20374
|
+
}
|
|
20375
|
+
function createGeminiEmbedClient(apiKey) {
|
|
20376
|
+
const genai = new GoogleGenAI2({ apiKey });
|
|
20377
|
+
return {
|
|
20378
|
+
async embedBatch(queries2, opts) {
|
|
20379
|
+
const response = await genai.models.embedContent({
|
|
20380
|
+
model: opts.model,
|
|
20381
|
+
contents: queries2,
|
|
20382
|
+
config: {
|
|
20383
|
+
taskType: opts.taskType,
|
|
20384
|
+
outputDimensionality: opts.outputDimensionality
|
|
20385
|
+
}
|
|
20386
|
+
});
|
|
20387
|
+
return extractEmbeddingVectors(response, queries2.length);
|
|
20388
|
+
}
|
|
20389
|
+
};
|
|
20390
|
+
}
|
|
20391
|
+
|
|
19837
20392
|
// ../provider-gemini/src/adapter.ts
|
|
19838
20393
|
function toGeminiConfig(config) {
|
|
19839
20394
|
return {
|
|
@@ -22038,14 +22593,14 @@ function removeWordpressConnection(config, projectName) {
|
|
|
22038
22593
|
}
|
|
22039
22594
|
|
|
22040
22595
|
// src/job-runner.ts
|
|
22041
|
-
import
|
|
22596
|
+
import crypto24 from "crypto";
|
|
22042
22597
|
import fs7 from "fs";
|
|
22043
22598
|
import path9 from "path";
|
|
22044
22599
|
import os5 from "os";
|
|
22045
|
-
import { and as and16, eq as
|
|
22600
|
+
import { and as and16, eq as eq27, inArray as inArray7, sql as sql10 } from "drizzle-orm";
|
|
22046
22601
|
|
|
22047
22602
|
// src/run-telemetry.ts
|
|
22048
|
-
import
|
|
22603
|
+
import crypto23 from "crypto";
|
|
22049
22604
|
function extractRegistrableHost(input) {
|
|
22050
22605
|
if (!input) return null;
|
|
22051
22606
|
const trimmed = input.trim();
|
|
@@ -22065,7 +22620,7 @@ function extractRegistrableHost(input) {
|
|
|
22065
22620
|
function hashDomain(input) {
|
|
22066
22621
|
const host = extractRegistrableHost(input);
|
|
22067
22622
|
if (!host) return null;
|
|
22068
|
-
return
|
|
22623
|
+
return crypto23.createHash("sha256").update(host).digest("hex");
|
|
22069
22624
|
}
|
|
22070
22625
|
function buildRunCompletedProps(input) {
|
|
22071
22626
|
const totalMs = input.phases?.total_ms ?? Date.now() - input.startTime;
|
|
@@ -22387,7 +22942,7 @@ var JobRunner = class {
|
|
|
22387
22942
|
if (stale.length === 0) return;
|
|
22388
22943
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22389
22944
|
for (const run of stale) {
|
|
22390
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
22945
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq27(runs.id, run.id)).run();
|
|
22391
22946
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
22392
22947
|
}
|
|
22393
22948
|
}
|
|
@@ -22421,10 +22976,10 @@ var JobRunner = class {
|
|
|
22421
22976
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
22422
22977
|
}
|
|
22423
22978
|
if (existingRun.status === "queued") {
|
|
22424
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(and16(
|
|
22979
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and16(eq27(runs.id, runId), eq27(runs.status, "queued"))).run();
|
|
22425
22980
|
}
|
|
22426
22981
|
this.throwIfRunCancelled(runId);
|
|
22427
|
-
const project = this.db.select().from(projects).where(
|
|
22982
|
+
const project = this.db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
22428
22983
|
if (!project) {
|
|
22429
22984
|
throw new Error(`Project ${projectId} not found`);
|
|
22430
22985
|
}
|
|
@@ -22445,8 +23000,9 @@ var JobRunner = class {
|
|
|
22445
23000
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
22446
23001
|
}
|
|
22447
23002
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
22448
|
-
|
|
22449
|
-
|
|
23003
|
+
const scopedQueryNames = parseJsonColumn(existingRun.queries, null);
|
|
23004
|
+
projectQueries = scopedQueryNames ? this.db.select().from(queries).where(and16(eq27(queries.projectId, projectId), inArray7(queries.query, scopedQueryNames))).all() : this.db.select().from(queries).where(eq27(queries.projectId, projectId)).all();
|
|
23005
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq27(competitors.projectId, projectId)).all();
|
|
22450
23006
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
22451
23007
|
const allDomains = effectiveDomains({
|
|
22452
23008
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -22464,7 +23020,7 @@ var JobRunner = class {
|
|
|
22464
23020
|
const todayPeriod = getCurrentUsageDay();
|
|
22465
23021
|
for (const p of activeProviders) {
|
|
22466
23022
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
22467
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
23023
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq27(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
22468
23024
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
22469
23025
|
if (providerUsage + queriesPerProvider > limit) {
|
|
22470
23026
|
throw new Error(
|
|
@@ -22524,7 +23080,7 @@ var JobRunner = class {
|
|
|
22524
23080
|
);
|
|
22525
23081
|
let screenshotRelPath = null;
|
|
22526
23082
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
22527
|
-
const snapshotId =
|
|
23083
|
+
const snapshotId = crypto24.randomUUID();
|
|
22528
23084
|
const screenshotDir = path9.join(os5.homedir(), ".canonry", "screenshots", runId);
|
|
22529
23085
|
if (!fs7.existsSync(screenshotDir)) fs7.mkdirSync(screenshotDir, { recursive: true });
|
|
22530
23086
|
const destPath = path9.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -22554,7 +23110,7 @@ var JobRunner = class {
|
|
|
22554
23110
|
}).run();
|
|
22555
23111
|
} else {
|
|
22556
23112
|
this.db.insert(querySnapshots).values({
|
|
22557
|
-
id:
|
|
23113
|
+
id: crypto24.randomUUID(),
|
|
22558
23114
|
runId,
|
|
22559
23115
|
queryId: q.id,
|
|
22560
23116
|
provider: providerName,
|
|
@@ -22607,12 +23163,12 @@ var JobRunner = class {
|
|
|
22607
23163
|
const someFailed = providerErrors.size > 0;
|
|
22608
23164
|
if (allFailed) {
|
|
22609
23165
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
22610
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
23166
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq27(runs.id, runId)).run();
|
|
22611
23167
|
} else if (someFailed) {
|
|
22612
23168
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
22613
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
23169
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq27(runs.id, runId)).run();
|
|
22614
23170
|
} else {
|
|
22615
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23171
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
22616
23172
|
}
|
|
22617
23173
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
22618
23174
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -22658,7 +23214,7 @@ var JobRunner = class {
|
|
|
22658
23214
|
status: "failed",
|
|
22659
23215
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22660
23216
|
error: errorMessage
|
|
22661
|
-
}).where(
|
|
23217
|
+
}).where(eq27(runs.id, runId)).run();
|
|
22662
23218
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
22663
23219
|
const abortReason = classifyRunAbortReason(errorMessage);
|
|
22664
23220
|
const phases = buildPhases({ startTime, providerCallStart, providerCallEnd });
|
|
@@ -22703,7 +23259,7 @@ var JobRunner = class {
|
|
|
22703
23259
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22704
23260
|
const period = now.slice(0, 10);
|
|
22705
23261
|
this.db.insert(usageCounters).values({
|
|
22706
|
-
id:
|
|
23262
|
+
id: crypto24.randomUUID(),
|
|
22707
23263
|
scope,
|
|
22708
23264
|
period,
|
|
22709
23265
|
metric,
|
|
@@ -22725,8 +23281,9 @@ var JobRunner = class {
|
|
|
22725
23281
|
status: runs.status,
|
|
22726
23282
|
finishedAt: runs.finishedAt,
|
|
22727
23283
|
error: runs.error,
|
|
22728
|
-
trigger: runs.trigger
|
|
22729
|
-
|
|
23284
|
+
trigger: runs.trigger,
|
|
23285
|
+
queries: runs.queries
|
|
23286
|
+
}).from(runs).where(eq27(runs.id, runId)).get();
|
|
22730
23287
|
}
|
|
22731
23288
|
isRunCancelled(runId) {
|
|
22732
23289
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -22742,7 +23299,7 @@ var JobRunner = class {
|
|
|
22742
23299
|
this.db.update(runs).set({
|
|
22743
23300
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22744
23301
|
error: currentRun.error ?? "Cancelled by user"
|
|
22745
|
-
}).where(
|
|
23302
|
+
}).where(eq27(runs.id, runId)).run();
|
|
22746
23303
|
}
|
|
22747
23304
|
trackEvent(
|
|
22748
23305
|
"run.completed",
|
|
@@ -22779,8 +23336,8 @@ function buildPhases(input) {
|
|
|
22779
23336
|
}
|
|
22780
23337
|
|
|
22781
23338
|
// src/gsc-sync.ts
|
|
22782
|
-
import
|
|
22783
|
-
import { eq as
|
|
23339
|
+
import crypto25 from "crypto";
|
|
23340
|
+
import { eq as eq28, and as and17, sql as sql11 } from "drizzle-orm";
|
|
22784
23341
|
var log2 = createLogger("GscSync");
|
|
22785
23342
|
function formatDate3(d) {
|
|
22786
23343
|
return d.toISOString().split("T")[0];
|
|
@@ -22792,13 +23349,13 @@ function daysAgo(n) {
|
|
|
22792
23349
|
}
|
|
22793
23350
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
22794
23351
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
22795
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
23352
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq28(runs.id, runId)).run();
|
|
22796
23353
|
try {
|
|
22797
23354
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
22798
23355
|
if (!googleClientId || !googleClientSecret) {
|
|
22799
23356
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
22800
23357
|
}
|
|
22801
|
-
const project = db.select().from(projects).where(
|
|
23358
|
+
const project = db.select().from(projects).where(eq28(projects.id, projectId)).get();
|
|
22802
23359
|
if (!project) {
|
|
22803
23360
|
throw new Error(`Project not found: ${projectId}`);
|
|
22804
23361
|
}
|
|
@@ -22833,7 +23390,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22833
23390
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
22834
23391
|
db.delete(gscSearchData).where(
|
|
22835
23392
|
and17(
|
|
22836
|
-
|
|
23393
|
+
eq28(gscSearchData.projectId, projectId),
|
|
22837
23394
|
sql11`${gscSearchData.date} >= ${startDate}`,
|
|
22838
23395
|
sql11`${gscSearchData.date} <= ${endDate}`
|
|
22839
23396
|
)
|
|
@@ -22845,7 +23402,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22845
23402
|
for (const row of batch) {
|
|
22846
23403
|
const [query, page, country, device, date] = row.keys;
|
|
22847
23404
|
db.insert(gscSearchData).values({
|
|
22848
|
-
id:
|
|
23405
|
+
id: crypto25.randomUUID(),
|
|
22849
23406
|
projectId,
|
|
22850
23407
|
syncRunId: runId,
|
|
22851
23408
|
date: date ?? "",
|
|
@@ -22879,7 +23436,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22879
23436
|
const rich = ir.richResultsResult;
|
|
22880
23437
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
22881
23438
|
db.insert(gscUrlInspections).values({
|
|
22882
|
-
id:
|
|
23439
|
+
id: crypto25.randomUUID(),
|
|
22883
23440
|
projectId,
|
|
22884
23441
|
syncRunId: runId,
|
|
22885
23442
|
url: pageUrl,
|
|
@@ -22900,7 +23457,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22900
23457
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
22901
23458
|
}
|
|
22902
23459
|
}
|
|
22903
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
23460
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq28(gscUrlInspections.projectId, projectId)).all();
|
|
22904
23461
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
22905
23462
|
for (const row of allInspections) {
|
|
22906
23463
|
const existing = latestByUrl.get(row.url);
|
|
@@ -22921,9 +23478,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22921
23478
|
}
|
|
22922
23479
|
}
|
|
22923
23480
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
22924
|
-
db.delete(gscCoverageSnapshots).where(and17(
|
|
23481
|
+
db.delete(gscCoverageSnapshots).where(and17(eq28(gscCoverageSnapshots.projectId, projectId), eq28(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
22925
23482
|
db.insert(gscCoverageSnapshots).values({
|
|
22926
|
-
id:
|
|
23483
|
+
id: crypto25.randomUUID(),
|
|
22927
23484
|
projectId,
|
|
22928
23485
|
syncRunId: runId,
|
|
22929
23486
|
date: snapshotDate,
|
|
@@ -22932,19 +23489,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
22932
23489
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
22933
23490
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22934
23491
|
}).run();
|
|
22935
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23492
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
22936
23493
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
22937
23494
|
} catch (err) {
|
|
22938
23495
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
22939
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23496
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq28(runs.id, runId)).run();
|
|
22940
23497
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
22941
23498
|
throw err;
|
|
22942
23499
|
}
|
|
22943
23500
|
}
|
|
22944
23501
|
|
|
22945
23502
|
// src/gsc-inspect-sitemap.ts
|
|
22946
|
-
import
|
|
22947
|
-
import { eq as
|
|
23503
|
+
import crypto26 from "crypto";
|
|
23504
|
+
import { eq as eq29, and as and18 } from "drizzle-orm";
|
|
22948
23505
|
|
|
22949
23506
|
// src/sitemap-parser.ts
|
|
22950
23507
|
var log3 = createLogger("SitemapParser");
|
|
@@ -23065,13 +23622,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
23065
23622
|
var log4 = createLogger("InspectSitemap");
|
|
23066
23623
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
23067
23624
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23068
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
23625
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq29(runs.id, runId)).run();
|
|
23069
23626
|
try {
|
|
23070
23627
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
23071
23628
|
if (!googleClientId || !googleClientSecret) {
|
|
23072
23629
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
23073
23630
|
}
|
|
23074
|
-
const project = db.select().from(projects).where(
|
|
23631
|
+
const project = db.select().from(projects).where(eq29(projects.id, projectId)).get();
|
|
23075
23632
|
if (!project) {
|
|
23076
23633
|
throw new Error(`Project not found: ${projectId}`);
|
|
23077
23634
|
}
|
|
@@ -23112,7 +23669,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23112
23669
|
const rich = ir.richResultsResult;
|
|
23113
23670
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23114
23671
|
db.insert(gscUrlInspections).values({
|
|
23115
|
-
id:
|
|
23672
|
+
id: crypto26.randomUUID(),
|
|
23116
23673
|
projectId,
|
|
23117
23674
|
syncRunId: runId,
|
|
23118
23675
|
url: pageUrl,
|
|
@@ -23139,7 +23696,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23139
23696
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
23140
23697
|
}
|
|
23141
23698
|
}
|
|
23142
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
23699
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq29(gscUrlInspections.projectId, projectId)).all();
|
|
23143
23700
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
23144
23701
|
for (const row of allInspections) {
|
|
23145
23702
|
const existing = latestByUrl.get(row.url);
|
|
@@ -23160,9 +23717,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23160
23717
|
}
|
|
23161
23718
|
}
|
|
23162
23719
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
23163
|
-
db.delete(gscCoverageSnapshots).where(and18(
|
|
23720
|
+
db.delete(gscCoverageSnapshots).where(and18(eq29(gscCoverageSnapshots.projectId, projectId), eq29(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
23164
23721
|
db.insert(gscCoverageSnapshots).values({
|
|
23165
|
-
id:
|
|
23722
|
+
id: crypto26.randomUUID(),
|
|
23166
23723
|
projectId,
|
|
23167
23724
|
syncRunId: runId,
|
|
23168
23725
|
date: snapshotDate,
|
|
@@ -23172,19 +23729,19 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
23172
23729
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23173
23730
|
}).run();
|
|
23174
23731
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
23175
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23732
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(runs.id, runId)).run();
|
|
23176
23733
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
23177
23734
|
} catch (err) {
|
|
23178
23735
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
23179
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23736
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq29(runs.id, runId)).run();
|
|
23180
23737
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
23181
23738
|
throw err;
|
|
23182
23739
|
}
|
|
23183
23740
|
}
|
|
23184
23741
|
|
|
23185
23742
|
// src/bing-inspect-sitemap.ts
|
|
23186
|
-
import
|
|
23187
|
-
import { eq as
|
|
23743
|
+
import crypto27 from "crypto";
|
|
23744
|
+
import { eq as eq30, desc as desc14 } from "drizzle-orm";
|
|
23188
23745
|
var log5 = createLogger("BingInspectSitemap");
|
|
23189
23746
|
function parseBingDate2(value) {
|
|
23190
23747
|
if (!value) return null;
|
|
@@ -23202,9 +23759,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
23202
23759
|
}
|
|
23203
23760
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
23204
23761
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23205
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
23762
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq30(runs.id, runId)).run();
|
|
23206
23763
|
try {
|
|
23207
|
-
const project = db.select().from(projects).where(
|
|
23764
|
+
const project = db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
23208
23765
|
if (!project) {
|
|
23209
23766
|
throw new Error(`Project not found: ${projectId}`);
|
|
23210
23767
|
}
|
|
@@ -23222,7 +23779,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23222
23779
|
if (sitemapUrls.length === 0) {
|
|
23223
23780
|
throw new Error("No URLs found in sitemap");
|
|
23224
23781
|
}
|
|
23225
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
23782
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq30(bingUrlInspections.projectId, projectId)).all();
|
|
23226
23783
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
23227
23784
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
23228
23785
|
log5.info("sitemap.diff", {
|
|
@@ -23271,7 +23828,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23271
23828
|
derivedInIndex = false;
|
|
23272
23829
|
}
|
|
23273
23830
|
db.insert(bingUrlInspections).values({
|
|
23274
|
-
id:
|
|
23831
|
+
id: crypto27.randomUUID(),
|
|
23275
23832
|
projectId,
|
|
23276
23833
|
url: pageUrl,
|
|
23277
23834
|
httpCode,
|
|
@@ -23305,7 +23862,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23305
23862
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
23306
23863
|
}
|
|
23307
23864
|
}
|
|
23308
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
23865
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq30(bingUrlInspections.projectId, projectId)).orderBy(desc14(bingUrlInspections.inspectedAt)).all();
|
|
23309
23866
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
23310
23867
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
23311
23868
|
for (const row of allInspections) {
|
|
@@ -23329,7 +23886,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23329
23886
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
23330
23887
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
23331
23888
|
db.insert(bingCoverageSnapshots).values({
|
|
23332
|
-
id:
|
|
23889
|
+
id: crypto27.randomUUID(),
|
|
23333
23890
|
projectId,
|
|
23334
23891
|
syncRunId: runId,
|
|
23335
23892
|
date: snapshotDate,
|
|
@@ -23348,7 +23905,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23348
23905
|
}
|
|
23349
23906
|
}).run();
|
|
23350
23907
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
23351
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23908
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq30(runs.id, runId)).run();
|
|
23352
23909
|
log5.info("inspect.completed", {
|
|
23353
23910
|
runId,
|
|
23354
23911
|
projectId,
|
|
@@ -23362,16 +23919,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
23362
23919
|
});
|
|
23363
23920
|
} catch (err) {
|
|
23364
23921
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
23365
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23922
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq30(runs.id, runId)).run();
|
|
23366
23923
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
23367
23924
|
throw err;
|
|
23368
23925
|
}
|
|
23369
23926
|
}
|
|
23370
23927
|
|
|
23371
23928
|
// src/commoncrawl-sync.ts
|
|
23372
|
-
import
|
|
23929
|
+
import crypto28 from "crypto";
|
|
23373
23930
|
import path10 from "path";
|
|
23374
|
-
import { and as and19, eq as
|
|
23931
|
+
import { and as and19, eq as eq31, sql as sql12 } from "drizzle-orm";
|
|
23375
23932
|
var log6 = createLogger("CommonCrawlSync");
|
|
23376
23933
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
23377
23934
|
function defaultDeps() {
|
|
@@ -23397,7 +23954,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23397
23954
|
phaseDetail: "downloading vertices + edges",
|
|
23398
23955
|
updatedAt: downloadStartedAt,
|
|
23399
23956
|
error: null
|
|
23400
|
-
}).where(
|
|
23957
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23401
23958
|
const paths = ccReleasePaths(release);
|
|
23402
23959
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
23403
23960
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -23420,7 +23977,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23420
23977
|
vertexSha256: vertex.sha256,
|
|
23421
23978
|
edgesSha256: edges.sha256,
|
|
23422
23979
|
updatedAt: downloadFinishedAt
|
|
23423
|
-
}).where(
|
|
23980
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23424
23981
|
const allProjects = db.select().from(projects).all();
|
|
23425
23982
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
23426
23983
|
let rows = [];
|
|
@@ -23436,15 +23993,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23436
23993
|
}
|
|
23437
23994
|
const queriedAt = deps.now().toISOString();
|
|
23438
23995
|
db.transaction((tx) => {
|
|
23439
|
-
tx.delete(backlinkDomains).where(
|
|
23440
|
-
tx.delete(backlinkSummaries).where(
|
|
23996
|
+
tx.delete(backlinkDomains).where(eq31(backlinkDomains.releaseSyncId, syncId)).run();
|
|
23997
|
+
tx.delete(backlinkSummaries).where(eq31(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
23441
23998
|
const expanded = [];
|
|
23442
23999
|
for (const r of rows) {
|
|
23443
24000
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
23444
24001
|
if (!projectIds) continue;
|
|
23445
24002
|
for (const projectId of projectIds) {
|
|
23446
24003
|
expanded.push({
|
|
23447
|
-
id:
|
|
24004
|
+
id: crypto28.randomUUID(),
|
|
23448
24005
|
projectId,
|
|
23449
24006
|
releaseSyncId: syncId,
|
|
23450
24007
|
release,
|
|
@@ -23464,7 +24021,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23464
24021
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
23465
24022
|
const summary = computeSummary(projectRows);
|
|
23466
24023
|
tx.insert(backlinkSummaries).values({
|
|
23467
|
-
id:
|
|
24024
|
+
id: crypto28.randomUUID(),
|
|
23468
24025
|
projectId: p.id,
|
|
23469
24026
|
releaseSyncId: syncId,
|
|
23470
24027
|
release,
|
|
@@ -23496,7 +24053,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23496
24053
|
domainsDiscovered: rows.length,
|
|
23497
24054
|
updatedAt: finishedAt,
|
|
23498
24055
|
error: null
|
|
23499
|
-
}).where(
|
|
24056
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23500
24057
|
log6.info("sync.completed", {
|
|
23501
24058
|
syncId,
|
|
23502
24059
|
release,
|
|
@@ -23526,7 +24083,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
23526
24083
|
error: errorMsg,
|
|
23527
24084
|
phaseDetail: null,
|
|
23528
24085
|
updatedAt: finishedAt
|
|
23529
|
-
}).where(
|
|
24086
|
+
}).where(eq31(ccReleaseSyncs.id, syncId)).run();
|
|
23530
24087
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
23531
24088
|
throw err;
|
|
23532
24089
|
}
|
|
@@ -23560,9 +24117,9 @@ function computeSummary(rows) {
|
|
|
23560
24117
|
}
|
|
23561
24118
|
|
|
23562
24119
|
// src/backlink-extract.ts
|
|
23563
|
-
import
|
|
24120
|
+
import crypto29 from "crypto";
|
|
23564
24121
|
import fs8 from "fs";
|
|
23565
|
-
import { and as and20, desc as
|
|
24122
|
+
import { and as and20, desc as desc15, eq as eq32 } from "drizzle-orm";
|
|
23566
24123
|
var log7 = createLogger("BacklinkExtract");
|
|
23567
24124
|
function defaultDeps2() {
|
|
23568
24125
|
return {
|
|
@@ -23574,13 +24131,13 @@ function defaultDeps2() {
|
|
|
23574
24131
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
23575
24132
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
23576
24133
|
const startedAt = deps.now().toISOString();
|
|
23577
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
24134
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq32(runs.id, runId)).run();
|
|
23578
24135
|
try {
|
|
23579
|
-
const project = db.select().from(projects).where(
|
|
24136
|
+
const project = db.select().from(projects).where(eq32(projects.id, projectId)).get();
|
|
23580
24137
|
if (!project) {
|
|
23581
24138
|
throw new Error(`Project not found: ${projectId}`);
|
|
23582
24139
|
}
|
|
23583
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
24140
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq32(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq32(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc15(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
23584
24141
|
if (!sync) {
|
|
23585
24142
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
23586
24143
|
}
|
|
@@ -23608,11 +24165,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23608
24165
|
const targetDomain = project.canonicalDomain;
|
|
23609
24166
|
db.transaction((tx) => {
|
|
23610
24167
|
tx.delete(backlinkDomains).where(
|
|
23611
|
-
and20(
|
|
24168
|
+
and20(eq32(backlinkDomains.projectId, projectId), eq32(backlinkDomains.release, release))
|
|
23612
24169
|
).run();
|
|
23613
24170
|
if (rows.length > 0) {
|
|
23614
24171
|
const values = rows.map((r) => ({
|
|
23615
|
-
id:
|
|
24172
|
+
id: crypto29.randomUUID(),
|
|
23616
24173
|
projectId,
|
|
23617
24174
|
releaseSyncId: syncId,
|
|
23618
24175
|
release,
|
|
@@ -23625,7 +24182,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23625
24182
|
}
|
|
23626
24183
|
const summary = computeSummary2(rows);
|
|
23627
24184
|
tx.insert(backlinkSummaries).values({
|
|
23628
|
-
id:
|
|
24185
|
+
id: crypto29.randomUUID(),
|
|
23629
24186
|
projectId,
|
|
23630
24187
|
releaseSyncId: syncId,
|
|
23631
24188
|
release,
|
|
@@ -23648,7 +24205,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23648
24205
|
}).run();
|
|
23649
24206
|
});
|
|
23650
24207
|
const finishedAt = deps.now().toISOString();
|
|
23651
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
24208
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq32(runs.id, runId)).run();
|
|
23652
24209
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
23653
24210
|
} catch (err) {
|
|
23654
24211
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -23657,7 +24214,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
23657
24214
|
status: RunStatuses.failed,
|
|
23658
24215
|
error: errorMsg,
|
|
23659
24216
|
finishedAt
|
|
23660
|
-
}).where(
|
|
24217
|
+
}).where(eq32(runs.id, runId)).run();
|
|
23661
24218
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
23662
24219
|
throw err;
|
|
23663
24220
|
}
|
|
@@ -23677,6 +24234,205 @@ function computeSummary2(rows) {
|
|
|
23677
24234
|
};
|
|
23678
24235
|
}
|
|
23679
24236
|
|
|
24237
|
+
// src/discovery-run.ts
|
|
24238
|
+
import crypto30 from "crypto";
|
|
24239
|
+
import { eq as eq33 } from "drizzle-orm";
|
|
24240
|
+
var log8 = createLogger("DiscoveryRun");
|
|
24241
|
+
var DEFAULT_SEED_COUNT = 30;
|
|
24242
|
+
async function executeDiscoveryRun(opts) {
|
|
24243
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
24244
|
+
opts.db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq33(runs.id, opts.runId)).run();
|
|
24245
|
+
try {
|
|
24246
|
+
const projectRow = opts.db.select().from(projects).where(eq33(projects.id, opts.projectId)).get();
|
|
24247
|
+
if (!projectRow) throw new Error(`Project ${opts.projectId} not found`);
|
|
24248
|
+
const projectCompetitors = opts.db.select({ domain: competitors.domain }).from(competitors).where(eq33(competitors.projectId, opts.projectId)).all().map((r) => r.domain.toLowerCase());
|
|
24249
|
+
const canonicalDomains = effectiveDomains({
|
|
24250
|
+
canonicalDomain: projectRow.canonicalDomain,
|
|
24251
|
+
ownedDomains: parseJsonColumn(projectRow.ownedDomains, [])
|
|
24252
|
+
});
|
|
24253
|
+
const project = {
|
|
24254
|
+
id: projectRow.id,
|
|
24255
|
+
name: projectRow.name,
|
|
24256
|
+
canonicalDomains,
|
|
24257
|
+
competitorDomains: projectCompetitors
|
|
24258
|
+
};
|
|
24259
|
+
const deps = opts.deps ?? buildDefaultDeps(opts.registry);
|
|
24260
|
+
const result = await executeDiscovery({
|
|
24261
|
+
db: opts.db,
|
|
24262
|
+
runId: opts.runId,
|
|
24263
|
+
sessionId: opts.sessionId,
|
|
24264
|
+
project,
|
|
24265
|
+
icpDescription: opts.icpDescription,
|
|
24266
|
+
dedupThreshold: opts.dedupThreshold,
|
|
24267
|
+
maxProbes: opts.maxProbes,
|
|
24268
|
+
deps
|
|
24269
|
+
});
|
|
24270
|
+
writeDiscoveryInsight(opts.db, {
|
|
24271
|
+
projectId: opts.projectId,
|
|
24272
|
+
runId: opts.runId,
|
|
24273
|
+
sessionId: opts.sessionId,
|
|
24274
|
+
seedProvider: result.seedProvider,
|
|
24275
|
+
result
|
|
24276
|
+
});
|
|
24277
|
+
opts.db.update(runs).set({ status: RunStatuses.completed, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq33(runs.id, opts.runId)).run();
|
|
24278
|
+
log8.info("discovery.completed", {
|
|
24279
|
+
runId: opts.runId,
|
|
24280
|
+
sessionId: opts.sessionId,
|
|
24281
|
+
buckets: result.buckets,
|
|
24282
|
+
competitorCount: result.competitorMap.length
|
|
24283
|
+
});
|
|
24284
|
+
} catch (err) {
|
|
24285
|
+
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
24286
|
+
log8.error("discovery.failed", { runId: opts.runId, sessionId: opts.sessionId, error: errorMsg });
|
|
24287
|
+
markSessionFailed(opts.db, opts.sessionId, errorMsg);
|
|
24288
|
+
opts.db.update(runs).set({
|
|
24289
|
+
status: RunStatuses.failed,
|
|
24290
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24291
|
+
error: errorMsg
|
|
24292
|
+
}).where(eq33(runs.id, opts.runId)).run();
|
|
24293
|
+
}
|
|
24294
|
+
}
|
|
24295
|
+
function buildDefaultDeps(registry) {
|
|
24296
|
+
const gemini = registry.get("gemini");
|
|
24297
|
+
if (!gemini) {
|
|
24298
|
+
throw new Error("Gemini provider is not configured. Add a Gemini API key (or Vertex project) before running discovery.");
|
|
24299
|
+
}
|
|
24300
|
+
const cfg = gemini.config;
|
|
24301
|
+
if (!cfg.apiKey && !cfg.vertexProject) {
|
|
24302
|
+
throw new Error("Gemini provider is missing both apiKey and vertexProject \u2014 cannot run discovery.");
|
|
24303
|
+
}
|
|
24304
|
+
const adapter = gemini.adapter;
|
|
24305
|
+
return {
|
|
24306
|
+
async seed(input) {
|
|
24307
|
+
const prompt = buildSeedPrompt(input);
|
|
24308
|
+
const raw = await adapter.executeTrackedQuery(
|
|
24309
|
+
{
|
|
24310
|
+
query: prompt,
|
|
24311
|
+
canonicalDomains: input.project.canonicalDomains,
|
|
24312
|
+
competitorDomains: input.project.competitorDomains
|
|
24313
|
+
},
|
|
24314
|
+
cfg
|
|
24315
|
+
);
|
|
24316
|
+
const normalized = adapter.normalizeResult(raw);
|
|
24317
|
+
const fromAnswer = parseQueryLines(normalized.answerText, DEFAULT_SEED_COUNT * 2);
|
|
24318
|
+
const fromGrounding = normalized.searchQueries ?? [];
|
|
24319
|
+
return {
|
|
24320
|
+
candidates: [...fromAnswer, ...fromGrounding],
|
|
24321
|
+
provider: "gemini"
|
|
24322
|
+
};
|
|
24323
|
+
},
|
|
24324
|
+
async embed(queries2) {
|
|
24325
|
+
if (cfg.apiKey) {
|
|
24326
|
+
return embedQueries(queries2, { apiKey: cfg.apiKey });
|
|
24327
|
+
}
|
|
24328
|
+
throw new Error("Discovery currently requires a Gemini API key. Vertex-mode embeddings are not yet implemented.");
|
|
24329
|
+
},
|
|
24330
|
+
async probe(input) {
|
|
24331
|
+
const raw = await adapter.executeTrackedQuery(
|
|
24332
|
+
{
|
|
24333
|
+
query: input.query,
|
|
24334
|
+
canonicalDomains: input.project.canonicalDomains,
|
|
24335
|
+
competitorDomains: input.project.competitorDomains
|
|
24336
|
+
},
|
|
24337
|
+
cfg
|
|
24338
|
+
);
|
|
24339
|
+
const normalized = adapter.normalizeResult(raw);
|
|
24340
|
+
const canonical = new Set(input.project.canonicalDomains.map((d) => d.toLowerCase()));
|
|
24341
|
+
const isCited = normalized.citedDomains.some((d) => canonical.has(d.toLowerCase()));
|
|
24342
|
+
return {
|
|
24343
|
+
citationState: isCited ? "cited" : "not-cited",
|
|
24344
|
+
citedDomains: normalized.citedDomains,
|
|
24345
|
+
rawResponse: raw.rawResponse
|
|
24346
|
+
};
|
|
24347
|
+
}
|
|
24348
|
+
};
|
|
24349
|
+
}
|
|
24350
|
+
function buildSeedPrompt(input) {
|
|
24351
|
+
return [
|
|
24352
|
+
"You are an AEO (Answer Engine Optimization) analyst expanding a tracked-query basket for a customer.",
|
|
24353
|
+
"",
|
|
24354
|
+
`Customer: ${input.project.name} (domains: ${input.project.canonicalDomains.join(", ")})`,
|
|
24355
|
+
`ICP: ${input.icpDescription}`,
|
|
24356
|
+
"",
|
|
24357
|
+
"Brainstorm a wide set of queries a member of this ICP would type into an AI answer engine (Gemini, ChatGPT, Perplexity) when they are about to make a decision in this space. Aim for 30+ candidates covering:",
|
|
24358
|
+
' - Comparison queries ("best X for Y")',
|
|
24359
|
+
" - Specific feature / capability queries",
|
|
24360
|
+
" - Pricing / vendor-shortlist queries",
|
|
24361
|
+
" - Workflow / how-to queries",
|
|
24362
|
+
" - Adjacent jobs-to-be-done queries",
|
|
24363
|
+
"",
|
|
24364
|
+
"Return ONE query per line. Plain text only \u2014 no numbering, bullets, quotes, or commentary."
|
|
24365
|
+
].join("\n");
|
|
24366
|
+
}
|
|
24367
|
+
function parseQueryLines(text, max) {
|
|
24368
|
+
const lines = text.split("\n");
|
|
24369
|
+
const out = [];
|
|
24370
|
+
const seen = /* @__PURE__ */ new Set();
|
|
24371
|
+
for (const raw of lines) {
|
|
24372
|
+
let line = raw.trim();
|
|
24373
|
+
if (!line) continue;
|
|
24374
|
+
line = line.replace(/^\s*(?:\d+[.)]\s*|[-*•]\s*)/, "").replace(/^["']|["']$/g, "").trim();
|
|
24375
|
+
if (!line) continue;
|
|
24376
|
+
if (/^(here are|sure|certainly|of course|i['']ve|these are|below are)/i.test(line)) continue;
|
|
24377
|
+
const key = line.toLowerCase();
|
|
24378
|
+
if (seen.has(key)) continue;
|
|
24379
|
+
seen.add(key);
|
|
24380
|
+
out.push(line);
|
|
24381
|
+
if (out.length >= max) break;
|
|
24382
|
+
}
|
|
24383
|
+
return out;
|
|
24384
|
+
}
|
|
24385
|
+
function writeDiscoveryInsight(db, input) {
|
|
24386
|
+
const { buckets, competitorMap } = input.result;
|
|
24387
|
+
const totalProbes = buckets.cited + buckets.aspirational + buckets["wasted-surface"];
|
|
24388
|
+
if (totalProbes === 0) return;
|
|
24389
|
+
const wastedRatio = buckets["wasted-surface"] / totalProbes;
|
|
24390
|
+
const citedRatio = buckets.cited / totalProbes;
|
|
24391
|
+
const severity = wastedRatio >= 0.4 || buckets["wasted-surface"] > buckets.cited && wastedRatio >= 0.2 ? "high" : citedRatio >= 0.6 ? "low" : "medium";
|
|
24392
|
+
const topCompetitors = competitorMap.slice(0, 5);
|
|
24393
|
+
const title = buildDiscoveryInsightTitle({
|
|
24394
|
+
cited: buckets.cited,
|
|
24395
|
+
wasted: buckets["wasted-surface"],
|
|
24396
|
+
aspirational: buckets.aspirational,
|
|
24397
|
+
totalProbes
|
|
24398
|
+
});
|
|
24399
|
+
db.insert(insights).values({
|
|
24400
|
+
id: crypto30.randomUUID(),
|
|
24401
|
+
projectId: input.projectId,
|
|
24402
|
+
runId: input.runId,
|
|
24403
|
+
type: "discovery.basket-divergence",
|
|
24404
|
+
severity,
|
|
24405
|
+
title,
|
|
24406
|
+
// query/provider fields don't fit the visibility-snapshot model for a
|
|
24407
|
+
// session-level insight. Use the session marker so the
|
|
24408
|
+
// (query, provider) index stays distinct across sessions; PR 5 will
|
|
24409
|
+
// formalize a session-scoped insight subtype.
|
|
24410
|
+
query: `discovery:${input.sessionId}`,
|
|
24411
|
+
provider: input.seedProvider,
|
|
24412
|
+
recommendation: JSON.stringify({
|
|
24413
|
+
action: "review-discovered-basket",
|
|
24414
|
+
summary: `Run \`canonry discover show ${input.sessionId} --format json\` to inspect the per-query breakdown. PR 2 will add \`canonry discover promote\` to merge the basket into the project.`,
|
|
24415
|
+
bucketCounts: buckets,
|
|
24416
|
+
topCompetitors
|
|
24417
|
+
}),
|
|
24418
|
+
cause: JSON.stringify({
|
|
24419
|
+
sessionId: input.sessionId,
|
|
24420
|
+
totalProbes,
|
|
24421
|
+
seedProvider: input.seedProvider
|
|
24422
|
+
}),
|
|
24423
|
+
dismissed: false,
|
|
24424
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24425
|
+
}).run();
|
|
24426
|
+
}
|
|
24427
|
+
function buildDiscoveryInsightTitle(input) {
|
|
24428
|
+
const parts = [];
|
|
24429
|
+
parts.push(`Discovery probed ${input.totalProbes} representative queries`);
|
|
24430
|
+
if (input.wasted > 0) parts.push(`${input.wasted} where competitors are cited but you are not`);
|
|
24431
|
+
if (input.cited > 0) parts.push(`${input.cited} where you are cited`);
|
|
24432
|
+
if (input.aspirational > 0) parts.push(`${input.aspirational} aspirational greenfield queries`);
|
|
24433
|
+
return parts.join(" \u2022 ");
|
|
24434
|
+
}
|
|
24435
|
+
|
|
23680
24436
|
// src/provider-registry.ts
|
|
23681
24437
|
var ProviderRegistry = class {
|
|
23682
24438
|
providers = /* @__PURE__ */ new Map();
|
|
@@ -23730,8 +24486,8 @@ var ProviderRegistry = class {
|
|
|
23730
24486
|
|
|
23731
24487
|
// src/scheduler.ts
|
|
23732
24488
|
import cron from "node-cron";
|
|
23733
|
-
import { and as and21, eq as
|
|
23734
|
-
var
|
|
24489
|
+
import { and as and21, eq as eq34 } from "drizzle-orm";
|
|
24490
|
+
var log9 = createLogger("Scheduler");
|
|
23735
24491
|
function taskKey(projectId, kind) {
|
|
23736
24492
|
return `${projectId}::${kind}`;
|
|
23737
24493
|
}
|
|
@@ -23745,16 +24501,16 @@ var Scheduler = class {
|
|
|
23745
24501
|
}
|
|
23746
24502
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
23747
24503
|
start() {
|
|
23748
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
24504
|
+
const allSchedules = this.db.select().from(schedules).where(eq34(schedules.enabled, 1)).all();
|
|
23749
24505
|
for (const schedule of allSchedules) {
|
|
23750
24506
|
const missedRunAt = schedule.nextRunAt;
|
|
23751
24507
|
this.registerCronTask(schedule);
|
|
23752
24508
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
23753
|
-
|
|
24509
|
+
log9.info("run.catch-up", { projectId: schedule.projectId, kind: schedule.kind, missedRunAt });
|
|
23754
24510
|
this.triggerRun(schedule.id, schedule.projectId, schedule.kind);
|
|
23755
24511
|
}
|
|
23756
24512
|
}
|
|
23757
|
-
|
|
24513
|
+
log9.info("started", { scheduleCount: allSchedules.length });
|
|
23758
24514
|
}
|
|
23759
24515
|
/** Stop all cron tasks for graceful shutdown. */
|
|
23760
24516
|
stop() {
|
|
@@ -23775,7 +24531,7 @@ var Scheduler = class {
|
|
|
23775
24531
|
this.stopTask(key, existing, "Stopped");
|
|
23776
24532
|
this.tasks.delete(key);
|
|
23777
24533
|
}
|
|
23778
|
-
const schedule = this.db.select().from(schedules).where(and21(
|
|
24534
|
+
const schedule = this.db.select().from(schedules).where(and21(eq34(schedules.projectId, projectId), eq34(schedules.kind, kind))).get();
|
|
23779
24535
|
if (schedule && schedule.enabled === 1) {
|
|
23780
24536
|
this.registerCronTask(schedule);
|
|
23781
24537
|
}
|
|
@@ -23798,13 +24554,13 @@ var Scheduler = class {
|
|
|
23798
24554
|
stopTask(key, task, verb) {
|
|
23799
24555
|
task.stop();
|
|
23800
24556
|
task.destroy();
|
|
23801
|
-
|
|
24557
|
+
log9.info(`task.${verb.toLowerCase()}`, { key });
|
|
23802
24558
|
}
|
|
23803
24559
|
registerCronTask(schedule) {
|
|
23804
24560
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
23805
24561
|
const kind = schedule.kind;
|
|
23806
24562
|
if (!cron.validate(cronExpr)) {
|
|
23807
|
-
|
|
24563
|
+
log9.error("cron.invalid", { projectId, kind, cronExpr });
|
|
23808
24564
|
return;
|
|
23809
24565
|
}
|
|
23810
24566
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -23816,43 +24572,43 @@ var Scheduler = class {
|
|
|
23816
24572
|
this.db.update(schedules).set({
|
|
23817
24573
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
23818
24574
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
23819
|
-
}).where(
|
|
24575
|
+
}).where(eq34(schedules.id, scheduleId)).run();
|
|
23820
24576
|
const label = schedule.preset ?? cronExpr;
|
|
23821
|
-
|
|
24577
|
+
log9.info("cron.registered", { projectId, kind, schedule: label, timezone });
|
|
23822
24578
|
}
|
|
23823
24579
|
triggerRun(scheduleId, projectId, kind) {
|
|
23824
24580
|
try {
|
|
23825
24581
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23826
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
24582
|
+
const currentSchedule = this.db.select().from(schedules).where(eq34(schedules.id, scheduleId)).get();
|
|
23827
24583
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
23828
|
-
|
|
24584
|
+
log9.warn("schedule.stale", { scheduleId, projectId, kind, msg: "schedule no longer exists or is disabled" });
|
|
23829
24585
|
this.remove(projectId, kind);
|
|
23830
24586
|
return;
|
|
23831
24587
|
}
|
|
23832
24588
|
const task = this.tasks.get(taskKey(projectId, kind));
|
|
23833
24589
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
23834
|
-
const project = this.db.select().from(projects).where(
|
|
24590
|
+
const project = this.db.select().from(projects).where(eq34(projects.id, projectId)).get();
|
|
23835
24591
|
if (!project) {
|
|
23836
|
-
|
|
24592
|
+
log9.error("project.not-found", { projectId, kind, msg: "skipping scheduled run" });
|
|
23837
24593
|
this.remove(projectId, kind);
|
|
23838
24594
|
return;
|
|
23839
24595
|
}
|
|
23840
24596
|
if (kind === SchedulableRunKinds["traffic-sync"]) {
|
|
23841
24597
|
const sourceId = currentSchedule.sourceId;
|
|
23842
24598
|
if (!sourceId) {
|
|
23843
|
-
|
|
24599
|
+
log9.warn("traffic-sync.missing-source", { scheduleId, projectId });
|
|
23844
24600
|
return;
|
|
23845
24601
|
}
|
|
23846
24602
|
if (!this.callbacks.onTrafficSyncRequested) {
|
|
23847
|
-
|
|
24603
|
+
log9.warn("traffic-sync.no-callback", { scheduleId, projectId, msg: "host did not register onTrafficSyncRequested" });
|
|
23848
24604
|
return;
|
|
23849
24605
|
}
|
|
23850
24606
|
this.db.update(schedules).set({
|
|
23851
24607
|
lastRunAt: now,
|
|
23852
24608
|
nextRunAt,
|
|
23853
24609
|
updatedAt: now
|
|
23854
|
-
}).where(
|
|
23855
|
-
|
|
24610
|
+
}).where(eq34(schedules.id, currentSchedule.id)).run();
|
|
24611
|
+
log9.info("traffic-sync.triggered", { projectName: project.name, sourceId });
|
|
23856
24612
|
this.callbacks.onTrafficSyncRequested(project.name, sourceId);
|
|
23857
24613
|
return;
|
|
23858
24614
|
}
|
|
@@ -23861,7 +24617,7 @@ var Scheduler = class {
|
|
|
23861
24617
|
if (project.defaultLocation) {
|
|
23862
24618
|
const loc = projectLocations.find((l) => l.label === project.defaultLocation);
|
|
23863
24619
|
if (!loc) {
|
|
23864
|
-
|
|
24620
|
+
log9.warn("default-location.stale", { scheduleId, projectId, label: project.defaultLocation });
|
|
23865
24621
|
return;
|
|
23866
24622
|
}
|
|
23867
24623
|
resolvedLocation = loc;
|
|
@@ -23875,11 +24631,11 @@ var Scheduler = class {
|
|
|
23875
24631
|
location: locationLabel
|
|
23876
24632
|
});
|
|
23877
24633
|
if (queueResult.conflict) {
|
|
23878
|
-
|
|
24634
|
+
log9.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
23879
24635
|
this.db.update(schedules).set({
|
|
23880
24636
|
nextRunAt,
|
|
23881
24637
|
updatedAt: now
|
|
23882
|
-
}).where(
|
|
24638
|
+
}).where(eq34(schedules.id, currentSchedule.id)).run();
|
|
23883
24639
|
return;
|
|
23884
24640
|
}
|
|
23885
24641
|
const runId = queueResult.runId;
|
|
@@ -23887,21 +24643,21 @@ var Scheduler = class {
|
|
|
23887
24643
|
lastRunAt: now,
|
|
23888
24644
|
nextRunAt,
|
|
23889
24645
|
updatedAt: now
|
|
23890
|
-
}).where(
|
|
24646
|
+
}).where(eq34(schedules.id, currentSchedule.id)).run();
|
|
23891
24647
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
23892
24648
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
23893
|
-
|
|
24649
|
+
log9.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
23894
24650
|
this.callbacks.onRunCreated(runId, projectId, providers, resolvedLocation);
|
|
23895
24651
|
} catch (err) {
|
|
23896
|
-
|
|
24652
|
+
log9.error("trigger.error", { scheduleId, projectId, kind, error: err instanceof Error ? err.message : String(err) });
|
|
23897
24653
|
}
|
|
23898
24654
|
}
|
|
23899
24655
|
};
|
|
23900
24656
|
|
|
23901
24657
|
// src/notifier.ts
|
|
23902
|
-
import { eq as
|
|
23903
|
-
import
|
|
23904
|
-
var
|
|
24658
|
+
import { eq as eq35, desc as desc16, and as and22, or as or4 } from "drizzle-orm";
|
|
24659
|
+
import crypto31 from "crypto";
|
|
24660
|
+
var log10 = createLogger("Notifier");
|
|
23905
24661
|
var Notifier = class {
|
|
23906
24662
|
db;
|
|
23907
24663
|
serverUrl;
|
|
@@ -23911,26 +24667,26 @@ var Notifier = class {
|
|
|
23911
24667
|
}
|
|
23912
24668
|
/** Called after a run completes (success, partial, or failed). */
|
|
23913
24669
|
async onRunCompleted(runId, projectId) {
|
|
23914
|
-
|
|
23915
|
-
const notifs = this.db.select().from(notifications).where(
|
|
24670
|
+
log10.info("run.completed", { runId, projectId });
|
|
24671
|
+
const notifs = this.db.select().from(notifications).where(eq35(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
23916
24672
|
if (notifs.length === 0) {
|
|
23917
|
-
|
|
24673
|
+
log10.info("notifications.none-enabled", { projectId });
|
|
23918
24674
|
return;
|
|
23919
24675
|
}
|
|
23920
|
-
|
|
23921
|
-
const run = this.db.select().from(runs).where(
|
|
24676
|
+
log10.info("notifications.found", { projectId, count: notifs.length });
|
|
24677
|
+
const run = this.db.select().from(runs).where(eq35(runs.id, runId)).get();
|
|
23922
24678
|
if (!run) {
|
|
23923
|
-
|
|
24679
|
+
log10.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
23924
24680
|
return;
|
|
23925
24681
|
}
|
|
23926
|
-
const project = this.db.select().from(projects).where(
|
|
24682
|
+
const project = this.db.select().from(projects).where(eq35(projects.id, projectId)).get();
|
|
23927
24683
|
if (!project) {
|
|
23928
|
-
|
|
24684
|
+
log10.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
23929
24685
|
return;
|
|
23930
24686
|
}
|
|
23931
24687
|
const transitions = this.computeTransitions(runId, projectId);
|
|
23932
24688
|
const events = [];
|
|
23933
|
-
|
|
24689
|
+
log10.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
23934
24690
|
if (run.status === "completed" || run.status === "partial") {
|
|
23935
24691
|
events.push("run.completed");
|
|
23936
24692
|
}
|
|
@@ -23946,7 +24702,7 @@ var Notifier = class {
|
|
|
23946
24702
|
if (!config.url) continue;
|
|
23947
24703
|
const subscribedEvents = config.events;
|
|
23948
24704
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
23949
|
-
|
|
24705
|
+
log10.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
23950
24706
|
if (matchingEvents.length === 0) continue;
|
|
23951
24707
|
for (const event of matchingEvents) {
|
|
23952
24708
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -23970,11 +24726,11 @@ var Notifier = class {
|
|
|
23970
24726
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
23971
24727
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
23972
24728
|
if (insightEvents.length === 0) return;
|
|
23973
|
-
const notifs = this.db.select().from(notifications).where(
|
|
24729
|
+
const notifs = this.db.select().from(notifications).where(eq35(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
23974
24730
|
if (notifs.length === 0) return;
|
|
23975
|
-
const run = this.db.select().from(runs).where(
|
|
24731
|
+
const run = this.db.select().from(runs).where(eq35(runs.id, runId)).get();
|
|
23976
24732
|
if (!run) return;
|
|
23977
|
-
const project = this.db.select().from(projects).where(
|
|
24733
|
+
const project = this.db.select().from(projects).where(eq35(projects.id, projectId)).get();
|
|
23978
24734
|
if (!project) return;
|
|
23979
24735
|
for (const notif of notifs) {
|
|
23980
24736
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -24006,10 +24762,10 @@ var Notifier = class {
|
|
|
24006
24762
|
computeTransitions(runId, projectId) {
|
|
24007
24763
|
const recentRuns = this.db.select().from(runs).where(
|
|
24008
24764
|
and22(
|
|
24009
|
-
|
|
24010
|
-
or4(
|
|
24765
|
+
eq35(runs.projectId, projectId),
|
|
24766
|
+
or4(eq35(runs.status, "completed"), eq35(runs.status, "partial"))
|
|
24011
24767
|
)
|
|
24012
|
-
).orderBy(
|
|
24768
|
+
).orderBy(desc16(runs.createdAt)).limit(2).all();
|
|
24013
24769
|
if (recentRuns.length < 2) return [];
|
|
24014
24770
|
const currentRunId = recentRuns[0].id;
|
|
24015
24771
|
const previousRunId = recentRuns[1].id;
|
|
@@ -24019,12 +24775,12 @@ var Notifier = class {
|
|
|
24019
24775
|
query: queries.query,
|
|
24020
24776
|
provider: querySnapshots.provider,
|
|
24021
24777
|
citationState: querySnapshots.citationState
|
|
24022
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
24778
|
+
}).from(querySnapshots).leftJoin(queries, eq35(querySnapshots.queryId, queries.id)).where(eq35(querySnapshots.runId, currentRunId)).all();
|
|
24023
24779
|
const previousSnapshots = this.db.select({
|
|
24024
24780
|
queryId: querySnapshots.queryId,
|
|
24025
24781
|
provider: querySnapshots.provider,
|
|
24026
24782
|
citationState: querySnapshots.citationState
|
|
24027
|
-
}).from(querySnapshots).where(
|
|
24783
|
+
}).from(querySnapshots).where(eq35(querySnapshots.runId, previousRunId)).all();
|
|
24028
24784
|
const prevMap = /* @__PURE__ */ new Map();
|
|
24029
24785
|
for (const s of previousSnapshots) {
|
|
24030
24786
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -24048,23 +24804,23 @@ var Notifier = class {
|
|
|
24048
24804
|
const targetLabel = redactNotificationUrl(url).urlDisplay;
|
|
24049
24805
|
const targetCheck = await resolveWebhookTarget(url);
|
|
24050
24806
|
if (!targetCheck.ok) {
|
|
24051
|
-
|
|
24807
|
+
log10.error("webhook.ssrf-blocked", { url: targetLabel, reason: targetCheck.message });
|
|
24052
24808
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
24053
24809
|
return;
|
|
24054
24810
|
}
|
|
24055
|
-
|
|
24811
|
+
log10.info("webhook.send", { event: payload.event, url: targetLabel });
|
|
24056
24812
|
const maxRetries = 3;
|
|
24057
24813
|
const delays = [1e3, 4e3, 16e3];
|
|
24058
24814
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
24059
24815
|
try {
|
|
24060
24816
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
24061
24817
|
if (response.status >= 200 && response.status < 300) {
|
|
24062
|
-
|
|
24818
|
+
log10.info("webhook.delivered", { event: payload.event, url: targetLabel, httpStatus: response.status });
|
|
24063
24819
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
24064
24820
|
return;
|
|
24065
24821
|
}
|
|
24066
24822
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
24067
|
-
|
|
24823
|
+
log10.warn("webhook.attempt-failed", { event: payload.event, url: targetLabel, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
24068
24824
|
if (attempt === maxRetries - 1) {
|
|
24069
24825
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
24070
24826
|
}
|
|
@@ -24072,7 +24828,7 @@ var Notifier = class {
|
|
|
24072
24828
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
24073
24829
|
if (attempt === maxRetries - 1) {
|
|
24074
24830
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
24075
|
-
|
|
24831
|
+
log10.error("webhook.exhausted", { event: payload.event, url: targetLabel, maxRetries, error: errorDetail });
|
|
24076
24832
|
}
|
|
24077
24833
|
}
|
|
24078
24834
|
if (attempt < maxRetries - 1) {
|
|
@@ -24082,7 +24838,7 @@ var Notifier = class {
|
|
|
24082
24838
|
}
|
|
24083
24839
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
24084
24840
|
this.db.insert(auditLog).values({
|
|
24085
|
-
id:
|
|
24841
|
+
id: crypto31.randomUUID(),
|
|
24086
24842
|
projectId,
|
|
24087
24843
|
actor: "scheduler",
|
|
24088
24844
|
action: `notification.${status}`,
|
|
@@ -24095,53 +24851,96 @@ var Notifier = class {
|
|
|
24095
24851
|
};
|
|
24096
24852
|
|
|
24097
24853
|
// src/run-coordinator.ts
|
|
24098
|
-
|
|
24854
|
+
import { eq as eq36 } from "drizzle-orm";
|
|
24855
|
+
var log11 = createLogger("RunCoordinator");
|
|
24099
24856
|
var RunCoordinator = class {
|
|
24100
|
-
constructor(notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
24857
|
+
constructor(db, notifier, intelligenceService, onInsightsGenerated, onAeroEvent) {
|
|
24858
|
+
this.db = db;
|
|
24101
24859
|
this.notifier = notifier;
|
|
24102
24860
|
this.intelligenceService = intelligenceService;
|
|
24103
24861
|
this.onInsightsGenerated = onInsightsGenerated;
|
|
24104
24862
|
this.onAeroEvent = onAeroEvent;
|
|
24105
24863
|
}
|
|
24106
24864
|
async onRunCompleted(runId, projectId) {
|
|
24865
|
+
const runRow = this.db.select().from(runs).where(eq36(runs.id, runId)).get();
|
|
24866
|
+
const kind = runRow?.kind ?? RunKinds["answer-visibility"];
|
|
24107
24867
|
let insightCount = 0;
|
|
24108
24868
|
let criticalOrHigh = 0;
|
|
24109
|
-
|
|
24110
|
-
|
|
24111
|
-
|
|
24112
|
-
|
|
24113
|
-
|
|
24114
|
-
|
|
24115
|
-
|
|
24116
|
-
|
|
24117
|
-
|
|
24118
|
-
|
|
24119
|
-
|
|
24120
|
-
|
|
24869
|
+
if (kind === RunKinds["answer-visibility"]) {
|
|
24870
|
+
try {
|
|
24871
|
+
const result = this.intelligenceService.analyzeAndPersist(runId, projectId);
|
|
24872
|
+
if (result) {
|
|
24873
|
+
insightCount = result.insights.length;
|
|
24874
|
+
criticalOrHigh = result.insights.filter(
|
|
24875
|
+
(i) => i.severity === "critical" || i.severity === "high"
|
|
24876
|
+
).length;
|
|
24877
|
+
if (this.onInsightsGenerated && criticalOrHigh > 0) {
|
|
24878
|
+
try {
|
|
24879
|
+
await this.onInsightsGenerated(runId, projectId, result);
|
|
24880
|
+
} catch (err) {
|
|
24881
|
+
log11.error("insight-webhook.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24882
|
+
}
|
|
24121
24883
|
}
|
|
24122
24884
|
}
|
|
24885
|
+
} catch (err) {
|
|
24886
|
+
log11.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24123
24887
|
}
|
|
24124
|
-
} catch (err) {
|
|
24125
|
-
log10.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24126
24888
|
}
|
|
24127
24889
|
try {
|
|
24128
24890
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
24129
24891
|
} catch (err) {
|
|
24130
|
-
|
|
24892
|
+
log11.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24131
24893
|
}
|
|
24132
24894
|
if (this.onAeroEvent) {
|
|
24133
24895
|
try {
|
|
24134
|
-
|
|
24896
|
+
const ctx = kind === RunKinds["aeo-discover-probe"] ? this.buildDiscoveryAeroContext(runId, projectId, runRow?.status === "failed" ? "failed" : "completed", runRow?.error ?? null) : {
|
|
24897
|
+
kind,
|
|
24898
|
+
runId,
|
|
24899
|
+
projectId,
|
|
24900
|
+
insightCount,
|
|
24901
|
+
criticalOrHigh
|
|
24902
|
+
};
|
|
24903
|
+
await this.onAeroEvent(ctx);
|
|
24135
24904
|
} catch (err) {
|
|
24136
|
-
|
|
24905
|
+
log11.error("aero.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
24137
24906
|
}
|
|
24138
24907
|
}
|
|
24139
24908
|
}
|
|
24909
|
+
/**
|
|
24910
|
+
* Pull the discovery session that owns this run and project a payload Aero
|
|
24911
|
+
* can act on: bucket counts, top competitors, the seed provider, and the
|
|
24912
|
+
* session ID it can pass to `canonry_discover_session_get` for the per-query
|
|
24913
|
+
* breakdown. Looked up by `runId` (the POST handler populates
|
|
24914
|
+
* `discovery_sessions.runId` in the same transaction that creates the run)
|
|
24915
|
+
* so two concurrent discovery sessions on the same project don't get
|
|
24916
|
+
* cross-wired. Falls back to a zero payload when the session row is missing
|
|
24917
|
+
* so the Aero queue is never starved of a follow-up.
|
|
24918
|
+
*/
|
|
24919
|
+
buildDiscoveryAeroContext(runId, projectId, status, error) {
|
|
24920
|
+
const session = this.db.select().from(discoverySessions).where(eq36(discoverySessions.runId, runId)).get();
|
|
24921
|
+
const competitorMap = session ? parseJsonColumn(session.competitorMap, []) : [];
|
|
24922
|
+
return {
|
|
24923
|
+
kind: RunKinds["aeo-discover-probe"],
|
|
24924
|
+
runId,
|
|
24925
|
+
projectId,
|
|
24926
|
+
sessionId: session?.id ?? "",
|
|
24927
|
+
seedProvider: session?.seedProvider ?? null,
|
|
24928
|
+
buckets: {
|
|
24929
|
+
cited: session?.citedCount ?? 0,
|
|
24930
|
+
aspirational: session?.aspirationalCount ?? 0,
|
|
24931
|
+
"wasted-surface": session?.wastedCount ?? 0
|
|
24932
|
+
},
|
|
24933
|
+
probeCount: session?.probeCount ?? 0,
|
|
24934
|
+
topCompetitors: competitorMap.slice(0, 5),
|
|
24935
|
+
status,
|
|
24936
|
+
error
|
|
24937
|
+
};
|
|
24938
|
+
}
|
|
24140
24939
|
};
|
|
24141
24940
|
|
|
24142
24941
|
// src/agent/session-registry.ts
|
|
24143
|
-
import
|
|
24144
|
-
import { eq as
|
|
24942
|
+
import crypto33 from "crypto";
|
|
24943
|
+
import { eq as eq38 } from "drizzle-orm";
|
|
24145
24944
|
|
|
24146
24945
|
// src/agent/session.ts
|
|
24147
24946
|
import fs11 from "fs";
|
|
@@ -24490,8 +25289,8 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
24490
25289
|
}
|
|
24491
25290
|
|
|
24492
25291
|
// src/agent/memory-store.ts
|
|
24493
|
-
import
|
|
24494
|
-
import { and as and23, desc as
|
|
25292
|
+
import crypto32 from "crypto";
|
|
25293
|
+
import { and as and23, desc as desc17, eq as eq37, like as like2, sql as sql13 } from "drizzle-orm";
|
|
24495
25294
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
24496
25295
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
24497
25296
|
function rowToDto2(row) {
|
|
@@ -24505,7 +25304,7 @@ function rowToDto2(row) {
|
|
|
24505
25304
|
};
|
|
24506
25305
|
}
|
|
24507
25306
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
24508
|
-
const query = db.select().from(agentMemory).where(
|
|
25307
|
+
const query = db.select().from(agentMemory).where(eq37(agentMemory.projectId, projectId)).orderBy(desc17(agentMemory.updatedAt));
|
|
24509
25308
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
24510
25309
|
return rows.map(rowToDto2);
|
|
24511
25310
|
}
|
|
@@ -24519,7 +25318,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
24519
25318
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
24520
25319
|
}
|
|
24521
25320
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
24522
|
-
const id =
|
|
25321
|
+
const id = crypto32.randomUUID();
|
|
24523
25322
|
db.insert(agentMemory).values({
|
|
24524
25323
|
id,
|
|
24525
25324
|
projectId: args.projectId,
|
|
@@ -24536,12 +25335,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
24536
25335
|
updatedAt: now
|
|
24537
25336
|
}
|
|
24538
25337
|
}).run();
|
|
24539
|
-
const row = db.select().from(agentMemory).where(and23(
|
|
25338
|
+
const row = db.select().from(agentMemory).where(and23(eq37(agentMemory.projectId, args.projectId), eq37(agentMemory.key, args.key))).get();
|
|
24540
25339
|
if (!row) throw new Error("memory upsert produced no row");
|
|
24541
25340
|
return rowToDto2(row);
|
|
24542
25341
|
}
|
|
24543
25342
|
function deleteMemoryEntry(db, projectId, key) {
|
|
24544
|
-
const result = db.delete(agentMemory).where(and23(
|
|
25343
|
+
const result = db.delete(agentMemory).where(and23(eq37(agentMemory.projectId, projectId), eq37(agentMemory.key, key))).run();
|
|
24545
25344
|
const changes = result.changes ?? 0;
|
|
24546
25345
|
return changes > 0;
|
|
24547
25346
|
}
|
|
@@ -24556,7 +25355,7 @@ function writeCompactionNote(db, args) {
|
|
|
24556
25355
|
}
|
|
24557
25356
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
24558
25357
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
24559
|
-
const id =
|
|
25358
|
+
const id = crypto32.randomUUID();
|
|
24560
25359
|
let inserted;
|
|
24561
25360
|
db.transaction((tx) => {
|
|
24562
25361
|
tx.insert(agentMemory).values({
|
|
@@ -24571,15 +25370,15 @@ function writeCompactionNote(db, args) {
|
|
|
24571
25370
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
24572
25371
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
24573
25372
|
and23(
|
|
24574
|
-
|
|
25373
|
+
eq37(agentMemory.projectId, args.projectId),
|
|
24575
25374
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
24576
25375
|
)
|
|
24577
|
-
).orderBy(
|
|
25376
|
+
).orderBy(desc17(agentMemory.updatedAt)).all();
|
|
24578
25377
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
24579
25378
|
if (stale.length > 0) {
|
|
24580
25379
|
tx.delete(agentMemory).where(sql13`${agentMemory.id} IN (${sql13.join(stale.map((s) => sql13`${s}`), sql13`, `)})`).run();
|
|
24581
25380
|
}
|
|
24582
|
-
const row = tx.select().from(agentMemory).where(and23(
|
|
25381
|
+
const row = tx.select().from(agentMemory).where(and23(eq37(agentMemory.projectId, args.projectId), eq37(agentMemory.key, key))).get();
|
|
24583
25382
|
if (row) inserted = rowToDto2(row);
|
|
24584
25383
|
});
|
|
24585
25384
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
@@ -24712,7 +25511,7 @@ async function compactMessages(args) {
|
|
|
24712
25511
|
}
|
|
24713
25512
|
|
|
24714
25513
|
// src/agent/session-registry.ts
|
|
24715
|
-
var
|
|
25514
|
+
var log12 = createLogger("SessionRegistry");
|
|
24716
25515
|
var MAX_HYDRATE_NOTES = 20;
|
|
24717
25516
|
var MAX_HYDRATE_BYTES = 32 * 1024;
|
|
24718
25517
|
function escapeMemoryFragment(value) {
|
|
@@ -24761,7 +25560,7 @@ var SessionRegistry = class {
|
|
|
24761
25560
|
modelProvider: effectiveProvider,
|
|
24762
25561
|
modelId: effectiveModelId,
|
|
24763
25562
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24764
|
-
}).where(
|
|
25563
|
+
}).where(eq38(agentSessions.projectId, projectId)).run();
|
|
24765
25564
|
}
|
|
24766
25565
|
const agent2 = createAeroSession({
|
|
24767
25566
|
projectName,
|
|
@@ -24939,13 +25738,13 @@ ${lines.join("\n")}
|
|
|
24939
25738
|
agent.state.messages = result.messages;
|
|
24940
25739
|
agent.state.systemPrompt = this.buildHydratedSystemPrompt(projectId, row.systemPrompt);
|
|
24941
25740
|
this.save(projectName);
|
|
24942
|
-
|
|
25741
|
+
log12.info("compaction.completed", {
|
|
24943
25742
|
projectName,
|
|
24944
25743
|
removedCount: result.removedCount,
|
|
24945
25744
|
summaryBytes: Buffer.byteLength(result.summary, "utf8")
|
|
24946
25745
|
});
|
|
24947
25746
|
} catch (err) {
|
|
24948
|
-
|
|
25747
|
+
log12.error("compaction.failed", {
|
|
24949
25748
|
projectName,
|
|
24950
25749
|
error: err instanceof Error ? err.message : String(err)
|
|
24951
25750
|
});
|
|
@@ -24975,7 +25774,7 @@ ${lines.join("\n")}
|
|
|
24975
25774
|
modelProvider: nextProvider,
|
|
24976
25775
|
modelId: nextModelId,
|
|
24977
25776
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24978
|
-
}).where(
|
|
25777
|
+
}).where(eq38(agentSessions.projectId, projectId)).run();
|
|
24979
25778
|
}
|
|
24980
25779
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
24981
25780
|
save(projectName) {
|
|
@@ -25042,7 +25841,7 @@ ${lines.join("\n")}
|
|
|
25042
25841
|
await agent.prompt(msgs);
|
|
25043
25842
|
this.save(projectName);
|
|
25044
25843
|
} catch (err) {
|
|
25045
|
-
|
|
25844
|
+
log12.error("drain.failed", {
|
|
25046
25845
|
projectName,
|
|
25047
25846
|
error: err instanceof Error ? err.message : String(err)
|
|
25048
25847
|
});
|
|
@@ -25137,17 +25936,17 @@ ${lines.join("\n")}
|
|
|
25137
25936
|
return id;
|
|
25138
25937
|
}
|
|
25139
25938
|
tryResolveProjectId(projectName) {
|
|
25140
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
25939
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq38(projects.name, projectName)).get();
|
|
25141
25940
|
return row?.id;
|
|
25142
25941
|
}
|
|
25143
25942
|
loadRow(projectId) {
|
|
25144
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
25943
|
+
const row = this.opts.db.select().from(agentSessions).where(eq38(agentSessions.projectId, projectId)).get();
|
|
25145
25944
|
return row ?? null;
|
|
25146
25945
|
}
|
|
25147
25946
|
insertRow(params) {
|
|
25148
25947
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
25149
25948
|
this.opts.db.insert(agentSessions).values({
|
|
25150
|
-
id:
|
|
25949
|
+
id: crypto33.randomUUID(),
|
|
25151
25950
|
projectId: params.projectId,
|
|
25152
25951
|
systemPrompt: params.systemPrompt,
|
|
25153
25952
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -25160,14 +25959,14 @@ ${lines.join("\n")}
|
|
|
25160
25959
|
}
|
|
25161
25960
|
updateRow(projectId, patch) {
|
|
25162
25961
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
25163
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
25962
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq38(agentSessions.projectId, projectId)).run();
|
|
25164
25963
|
}
|
|
25165
25964
|
};
|
|
25166
25965
|
|
|
25167
25966
|
// src/agent/agent-routes.ts
|
|
25168
|
-
import { eq as
|
|
25967
|
+
import { eq as eq39 } from "drizzle-orm";
|
|
25169
25968
|
function resolveProject2(db, name) {
|
|
25170
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
25969
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq39(projects.name, name)).get();
|
|
25171
25970
|
if (!row) throw notFound("project", name);
|
|
25172
25971
|
return row;
|
|
25173
25972
|
}
|
|
@@ -25176,7 +25975,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
25176
25975
|
"/projects/:name/agent/transcript",
|
|
25177
25976
|
async (request) => {
|
|
25178
25977
|
const project = resolveProject2(opts.db, request.params.name);
|
|
25179
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
25978
|
+
const row = opts.db.select().from(agentSessions).where(eq39(agentSessions.projectId, project.id)).get();
|
|
25180
25979
|
if (!row) {
|
|
25181
25980
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
25182
25981
|
}
|
|
@@ -25200,7 +25999,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
25200
25999
|
async (request) => {
|
|
25201
26000
|
const project = resolveProject2(opts.db, request.params.name);
|
|
25202
26001
|
opts.sessionRegistry.reset(project.name);
|
|
25203
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
26002
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq39(agentSessions.projectId, project.id)).run();
|
|
25204
26003
|
return { status: "reset" };
|
|
25205
26004
|
}
|
|
25206
26005
|
);
|
|
@@ -25433,7 +26232,7 @@ function formatAuditFactorScore(factor) {
|
|
|
25433
26232
|
}
|
|
25434
26233
|
|
|
25435
26234
|
// src/snapshot-service.ts
|
|
25436
|
-
var
|
|
26235
|
+
var log13 = createLogger("Snapshot");
|
|
25437
26236
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
25438
26237
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
25439
26238
|
var ProviderExecutionGate2 = class {
|
|
@@ -25576,7 +26375,7 @@ var SnapshotService = class {
|
|
|
25576
26375
|
return mapAuditReport(report);
|
|
25577
26376
|
} catch (err) {
|
|
25578
26377
|
const message = err instanceof Error ? err.message : String(err);
|
|
25579
|
-
|
|
26378
|
+
log13.warn("audit.failed", { homepageUrl, error: message });
|
|
25580
26379
|
return {
|
|
25581
26380
|
url: homepageUrl,
|
|
25582
26381
|
finalUrl: homepageUrl,
|
|
@@ -25606,7 +26405,7 @@ var SnapshotService = class {
|
|
|
25606
26405
|
queries: parsedQueries
|
|
25607
26406
|
};
|
|
25608
26407
|
} catch (err) {
|
|
25609
|
-
|
|
26408
|
+
log13.warn("profile.generation-failed", {
|
|
25610
26409
|
domain: ctx.domain,
|
|
25611
26410
|
provider: ctx.analysisProvider.adapter.name,
|
|
25612
26411
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -25748,7 +26547,7 @@ var SnapshotService = class {
|
|
|
25748
26547
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
25749
26548
|
};
|
|
25750
26549
|
} catch (err) {
|
|
25751
|
-
|
|
26550
|
+
log13.warn("response.analysis-failed", {
|
|
25752
26551
|
provider: ctx.analysisProvider.adapter.name,
|
|
25753
26552
|
error: err instanceof Error ? err.message : String(err)
|
|
25754
26553
|
});
|
|
@@ -26033,7 +26832,7 @@ function clipText(value, length) {
|
|
|
26033
26832
|
// src/server.ts
|
|
26034
26833
|
var _require2 = createRequire3(import.meta.url);
|
|
26035
26834
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
26036
|
-
var
|
|
26835
|
+
var log14 = createLogger("Server");
|
|
26037
26836
|
var DEFAULT_QUOTA = {
|
|
26038
26837
|
maxConcurrency: 2,
|
|
26039
26838
|
maxRequestsPerMinute: 10,
|
|
@@ -26064,7 +26863,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
26064
26863
|
};
|
|
26065
26864
|
}
|
|
26066
26865
|
function hashApiKey(key) {
|
|
26067
|
-
return
|
|
26866
|
+
return crypto34.createHash("sha256").update(key).digest("hex");
|
|
26068
26867
|
}
|
|
26069
26868
|
function parseCookies2(header) {
|
|
26070
26869
|
if (!header) return {};
|
|
@@ -26120,7 +26919,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
26120
26919
|
}
|
|
26121
26920
|
if (migratedGoogle > 0) {
|
|
26122
26921
|
saveConfigPatch({ google: config.google });
|
|
26123
|
-
|
|
26922
|
+
log14.info("credentials.migrated", { type: "google", count: migratedGoogle });
|
|
26124
26923
|
}
|
|
26125
26924
|
let migratedGa4 = 0;
|
|
26126
26925
|
for (const row of rows.ga4) {
|
|
@@ -26138,7 +26937,7 @@ function applyLegacyCredentials(rows, config) {
|
|
|
26138
26937
|
}
|
|
26139
26938
|
if (migratedGa4 > 0) {
|
|
26140
26939
|
saveConfigPatch({ ga4: config.ga4 });
|
|
26141
|
-
|
|
26940
|
+
log14.info("credentials.migrated", { type: "ga4", count: migratedGa4 });
|
|
26142
26941
|
}
|
|
26143
26942
|
}
|
|
26144
26943
|
async function createServer(opts) {
|
|
@@ -26170,11 +26969,11 @@ async function createServer(opts) {
|
|
|
26170
26969
|
applyLegacyCredentials(legacyRows, opts.config);
|
|
26171
26970
|
dropLegacyCredentialColumns(opts.db);
|
|
26172
26971
|
} catch (err) {
|
|
26173
|
-
|
|
26972
|
+
log14.warn("credentials.migration.failed", {
|
|
26174
26973
|
error: err instanceof Error ? err.message : String(err)
|
|
26175
26974
|
});
|
|
26176
26975
|
}
|
|
26177
|
-
|
|
26976
|
+
log14.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
26178
26977
|
const p = providers[k];
|
|
26179
26978
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
26180
26979
|
}) });
|
|
@@ -26218,15 +27017,27 @@ async function createServer(opts) {
|
|
|
26218
27017
|
config: opts.config
|
|
26219
27018
|
});
|
|
26220
27019
|
const runCoordinator = new RunCoordinator(
|
|
27020
|
+
opts.db,
|
|
26221
27021
|
notifier,
|
|
26222
27022
|
intelligenceService,
|
|
26223
27023
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
26224
|
-
async (
|
|
26225
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
27024
|
+
async (ctx) => {
|
|
27025
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq40(projects.id, ctx.projectId)).get();
|
|
26226
27026
|
if (!project) return;
|
|
27027
|
+
let content;
|
|
27028
|
+
if (ctx.kind === RunKinds["aeo-discover-probe"]) {
|
|
27029
|
+
if (ctx.status === "failed") {
|
|
27030
|
+
content = `[system] Discovery run ${ctx.runId} failed for project ${project.name}: ${ctx.error ?? "unknown error"}. Surface a one-line diagnosis and a suggested next step.`;
|
|
27031
|
+
} else {
|
|
27032
|
+
const top = ctx.topCompetitors.map((c) => `${c.domain}(${c.hits})`).join(", ") || "none";
|
|
27033
|
+
content = `[system] Discovery run ${ctx.runId} completed for project ${project.name} (session ${ctx.sessionId}). Buckets \u2014 cited:${ctx.buckets.cited}, wasted-surface:${ctx.buckets["wasted-surface"]}, aspirational:${ctx.buckets.aspirational} (${ctx.probeCount} probes; seed provider: ${ctx.seedProvider ?? "unknown"}). Top recurring competitor domains: ${top}. Use canonry_discover_session_get to pull per-query buckets and call out anything worth promoting to the tracked basket. Keep it tight.`;
|
|
27034
|
+
}
|
|
27035
|
+
} else {
|
|
27036
|
+
content = `[system] Run ${ctx.runId} completed for project ${project.name}. ${ctx.insightCount} insights generated (${ctx.criticalOrHigh} critical/high). Use canonry_run_get to inspect the run and canonry_insights_list to review new findings. Surface anything notable briefly \u2014 skip chit-chat.`;
|
|
27037
|
+
}
|
|
26227
27038
|
sessionRegistry.queueFollowUp(project.name, {
|
|
26228
27039
|
role: "user",
|
|
26229
|
-
content
|
|
27040
|
+
content,
|
|
26230
27041
|
timestamp: Date.now()
|
|
26231
27042
|
});
|
|
26232
27043
|
void sessionRegistry.drainNow(project.name);
|
|
@@ -26351,7 +27162,7 @@ async function createServer(opts) {
|
|
|
26351
27162
|
return removed;
|
|
26352
27163
|
}
|
|
26353
27164
|
};
|
|
26354
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
27165
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto34.randomBytes(32).toString("hex");
|
|
26355
27166
|
const googleConnectionStore = {
|
|
26356
27167
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
26357
27168
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -26397,11 +27208,11 @@ async function createServer(opts) {
|
|
|
26397
27208
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
26398
27209
|
if (opts.config.apiKey) {
|
|
26399
27210
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
26400
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
27211
|
+
const existing = opts.db.select().from(apiKeys).where(eq40(apiKeys.keyHash, keyHash)).get();
|
|
26401
27212
|
if (!existing) {
|
|
26402
27213
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
26403
27214
|
opts.db.insert(apiKeys).values({
|
|
26404
|
-
id: `key_${
|
|
27215
|
+
id: `key_${crypto34.randomBytes(8).toString("hex")}`,
|
|
26405
27216
|
name: "default",
|
|
26406
27217
|
keyHash,
|
|
26407
27218
|
keyPrefix: prefix,
|
|
@@ -26425,7 +27236,7 @@ async function createServer(opts) {
|
|
|
26425
27236
|
};
|
|
26426
27237
|
const createSession = (apiKeyId) => {
|
|
26427
27238
|
pruneExpiredSessions();
|
|
26428
|
-
const sessionId =
|
|
27239
|
+
const sessionId = crypto34.randomBytes(32).toString("hex");
|
|
26429
27240
|
sessions.set(sessionId, {
|
|
26430
27241
|
apiKeyId,
|
|
26431
27242
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -26449,7 +27260,7 @@ async function createServer(opts) {
|
|
|
26449
27260
|
};
|
|
26450
27261
|
const getDefaultApiKey = () => {
|
|
26451
27262
|
if (!opts.config.apiKey) return void 0;
|
|
26452
|
-
return opts.db.select().from(apiKeys).where(
|
|
27263
|
+
return opts.db.select().from(apiKeys).where(eq40(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
26453
27264
|
};
|
|
26454
27265
|
const createPasswordSession = (reply) => {
|
|
26455
27266
|
const key = getDefaultApiKey();
|
|
@@ -26506,12 +27317,12 @@ async function createServer(opts) {
|
|
|
26506
27317
|
return reply.send({ authenticated: true });
|
|
26507
27318
|
}
|
|
26508
27319
|
if (apiKey) {
|
|
26509
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
27320
|
+
const key = opts.db.select().from(apiKeys).where(eq40(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
26510
27321
|
if (!key || key.revokedAt) {
|
|
26511
27322
|
const err2 = authInvalid();
|
|
26512
27323
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
26513
27324
|
}
|
|
26514
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
27325
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq40(apiKeys.id, key.id)).run();
|
|
26515
27326
|
const sessionId = createSession(key.id);
|
|
26516
27327
|
reply.header("set-cookie", serializeSessionCookie({
|
|
26517
27328
|
name: SESSION_COOKIE_NAME,
|
|
@@ -26621,7 +27432,7 @@ async function createServer(opts) {
|
|
|
26621
27432
|
deps: {
|
|
26622
27433
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
26623
27434
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
26624
|
-
const runId =
|
|
27435
|
+
const runId = crypto34.randomUUID();
|
|
26625
27436
|
opts.db.insert(runs).values({
|
|
26626
27437
|
id: runId,
|
|
26627
27438
|
projectId,
|
|
@@ -26644,6 +27455,20 @@ async function createServer(opts) {
|
|
|
26644
27455
|
app.log.error({ runId, err }, "Backlink extract failed");
|
|
26645
27456
|
});
|
|
26646
27457
|
},
|
|
27458
|
+
onDiscoveryRunRequested: (input) => {
|
|
27459
|
+
executeDiscoveryRun({
|
|
27460
|
+
db: opts.db,
|
|
27461
|
+
registry,
|
|
27462
|
+
runId: input.runId,
|
|
27463
|
+
sessionId: input.sessionId,
|
|
27464
|
+
projectId: input.projectId,
|
|
27465
|
+
icpDescription: input.icpDescription,
|
|
27466
|
+
dedupThreshold: input.dedupThreshold,
|
|
27467
|
+
maxProbes: input.maxProbes
|
|
27468
|
+
}).then(() => runCoordinator.onRunCompleted(input.runId, input.projectId)).catch((err) => {
|
|
27469
|
+
app.log.error({ runId: input.runId, err }, "Discovery run failed");
|
|
27470
|
+
});
|
|
27471
|
+
},
|
|
26647
27472
|
onBacklinksPruneCache: (release) => {
|
|
26648
27473
|
try {
|
|
26649
27474
|
pruneCachedRelease(release);
|
|
@@ -26769,7 +27594,7 @@ async function createServer(opts) {
|
|
|
26769
27594
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
26770
27595
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
26771
27596
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
26772
|
-
id:
|
|
27597
|
+
id: crypto34.randomUUID(),
|
|
26773
27598
|
projectId,
|
|
26774
27599
|
actor: "api",
|
|
26775
27600
|
action: existing ? "provider.updated" : "provider.created",
|