@ainyc/canonry 1.39.4 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{index-DWD1vfsF.js → index-Du0I3brY.js} +39 -39
- package/assets/index.html +1 -1
- package/dist/chunk-FOWWBLXD.js +1218 -0
- package/dist/{chunk-H6FO4LHC.js → chunk-FXHVGU5S.js} +626 -1593
- package/dist/cli.js +321 -83
- package/dist/index.js +2 -1
- package/dist/intelligence-service-PDZOIB7L.js +6 -0
- package/package.json +7 -7
|
@@ -1,8 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import {
|
|
2
|
+
IntelligenceService,
|
|
3
|
+
apiKeys,
|
|
4
|
+
auditLog,
|
|
5
|
+
bingUrlInspections,
|
|
6
|
+
competitors,
|
|
7
|
+
createLogger,
|
|
8
|
+
gaAiReferrals,
|
|
9
|
+
gaTrafficSnapshots,
|
|
10
|
+
gaTrafficSummaries,
|
|
11
|
+
gscCoverageSnapshots,
|
|
12
|
+
gscSearchData,
|
|
13
|
+
gscUrlInspections,
|
|
14
|
+
healthSnapshots,
|
|
15
|
+
insights,
|
|
16
|
+
keywords,
|
|
17
|
+
notifications,
|
|
18
|
+
parseJsonColumn,
|
|
19
|
+
projects,
|
|
20
|
+
querySnapshots,
|
|
21
|
+
runs,
|
|
22
|
+
schedules,
|
|
23
|
+
usageCounters
|
|
24
|
+
} from "./chunk-FOWWBLXD.js";
|
|
6
25
|
|
|
7
26
|
// src/config.ts
|
|
8
27
|
import fs from "fs";
|
|
@@ -243,11 +262,11 @@ function trackEvent(event, properties) {
|
|
|
243
262
|
|
|
244
263
|
// src/server.ts
|
|
245
264
|
import { createRequire as createRequire2 } from "module";
|
|
246
|
-
import
|
|
265
|
+
import crypto22 from "crypto";
|
|
247
266
|
import fs5 from "fs";
|
|
248
267
|
import path6 from "path";
|
|
249
268
|
import { fileURLToPath } from "url";
|
|
250
|
-
import { eq as
|
|
269
|
+
import { eq as eq23 } from "drizzle-orm";
|
|
251
270
|
import Fastify from "fastify";
|
|
252
271
|
|
|
253
272
|
// ../contracts/src/config-schema.ts
|
|
@@ -260,6 +279,14 @@ var providerQuotaPolicySchema = z.object({
|
|
|
260
279
|
maxRequestsPerMinute: z.number().int().positive(),
|
|
261
280
|
maxRequestsPerDay: z.number().int().positive()
|
|
262
281
|
});
|
|
282
|
+
var ProviderNames = {
|
|
283
|
+
gemini: "gemini",
|
|
284
|
+
openai: "openai",
|
|
285
|
+
claude: "claude",
|
|
286
|
+
perplexity: "perplexity",
|
|
287
|
+
local: "local",
|
|
288
|
+
cdpChatgpt: "cdp:chatgpt"
|
|
289
|
+
};
|
|
263
290
|
var providerNameSchema = z.string().min(1);
|
|
264
291
|
var apiProviderNameSchema = z.string().min(1);
|
|
265
292
|
function isBrowserProvider(name) {
|
|
@@ -1206,783 +1233,6 @@ function escapeRegExp(value) {
|
|
|
1206
1233
|
// ../api-routes/src/auth.ts
|
|
1207
1234
|
import crypto2 from "crypto";
|
|
1208
1235
|
import { eq } from "drizzle-orm";
|
|
1209
|
-
|
|
1210
|
-
// ../db/src/client.ts
|
|
1211
|
-
import { mkdirSync } from "fs";
|
|
1212
|
-
import { dirname } from "path";
|
|
1213
|
-
import Database from "better-sqlite3";
|
|
1214
|
-
import { drizzle } from "drizzle-orm/better-sqlite3";
|
|
1215
|
-
|
|
1216
|
-
// ../db/src/schema.ts
|
|
1217
|
-
var schema_exports = {};
|
|
1218
|
-
__export(schema_exports, {
|
|
1219
|
-
apiKeys: () => apiKeys,
|
|
1220
|
-
auditLog: () => auditLog,
|
|
1221
|
-
bingConnections: () => bingConnections,
|
|
1222
|
-
bingKeywordStats: () => bingKeywordStats,
|
|
1223
|
-
bingUrlInspections: () => bingUrlInspections,
|
|
1224
|
-
competitors: () => competitors,
|
|
1225
|
-
gaAiReferrals: () => gaAiReferrals,
|
|
1226
|
-
gaConnections: () => gaConnections,
|
|
1227
|
-
gaTrafficSnapshots: () => gaTrafficSnapshots,
|
|
1228
|
-
gaTrafficSummaries: () => gaTrafficSummaries,
|
|
1229
|
-
googleConnections: () => googleConnections,
|
|
1230
|
-
gscCoverageSnapshots: () => gscCoverageSnapshots,
|
|
1231
|
-
gscSearchData: () => gscSearchData,
|
|
1232
|
-
gscUrlInspections: () => gscUrlInspections,
|
|
1233
|
-
healthSnapshots: () => healthSnapshots,
|
|
1234
|
-
insights: () => insights,
|
|
1235
|
-
keywords: () => keywords,
|
|
1236
|
-
notifications: () => notifications,
|
|
1237
|
-
projects: () => projects,
|
|
1238
|
-
querySnapshots: () => querySnapshots,
|
|
1239
|
-
runs: () => runs,
|
|
1240
|
-
schedules: () => schedules,
|
|
1241
|
-
usageCounters: () => usageCounters
|
|
1242
|
-
});
|
|
1243
|
-
import { index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
|
|
1244
|
-
var projects = sqliteTable("projects", {
|
|
1245
|
-
id: text("id").primaryKey(),
|
|
1246
|
-
name: text("name").notNull().unique(),
|
|
1247
|
-
displayName: text("display_name").notNull(),
|
|
1248
|
-
canonicalDomain: text("canonical_domain").notNull(),
|
|
1249
|
-
ownedDomains: text("owned_domains").notNull().default("[]"),
|
|
1250
|
-
country: text("country").notNull(),
|
|
1251
|
-
language: text("language").notNull(),
|
|
1252
|
-
tags: text("tags").notNull().default("[]"),
|
|
1253
|
-
labels: text("labels").notNull().default("{}"),
|
|
1254
|
-
providers: text("providers").notNull().default("[]"),
|
|
1255
|
-
locations: text("locations").notNull().default("[]"),
|
|
1256
|
-
defaultLocation: text("default_location"),
|
|
1257
|
-
configSource: text("config_source").notNull().default("cli"),
|
|
1258
|
-
configRevision: integer("config_revision").notNull().default(1),
|
|
1259
|
-
createdAt: text("created_at").notNull(),
|
|
1260
|
-
updatedAt: text("updated_at").notNull()
|
|
1261
|
-
});
|
|
1262
|
-
var keywords = sqliteTable("keywords", {
|
|
1263
|
-
id: text("id").primaryKey(),
|
|
1264
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1265
|
-
keyword: text("keyword").notNull(),
|
|
1266
|
-
createdAt: text("created_at").notNull()
|
|
1267
|
-
}, (table) => [
|
|
1268
|
-
index("idx_keywords_project").on(table.projectId),
|
|
1269
|
-
uniqueIndex("idx_keywords_project_keyword").on(table.projectId, table.keyword)
|
|
1270
|
-
]);
|
|
1271
|
-
var competitors = sqliteTable("competitors", {
|
|
1272
|
-
id: text("id").primaryKey(),
|
|
1273
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1274
|
-
domain: text("domain").notNull(),
|
|
1275
|
-
createdAt: text("created_at").notNull()
|
|
1276
|
-
}, (table) => [
|
|
1277
|
-
index("idx_competitors_project").on(table.projectId),
|
|
1278
|
-
uniqueIndex("idx_competitors_project_domain").on(table.projectId, table.domain)
|
|
1279
|
-
]);
|
|
1280
|
-
var runs = sqliteTable("runs", {
|
|
1281
|
-
id: text("id").primaryKey(),
|
|
1282
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1283
|
-
kind: text("kind").notNull().default("answer-visibility"),
|
|
1284
|
-
status: text("status").notNull().default("queued"),
|
|
1285
|
-
trigger: text("trigger").notNull().default("manual"),
|
|
1286
|
-
location: text("location"),
|
|
1287
|
-
startedAt: text("started_at"),
|
|
1288
|
-
finishedAt: text("finished_at"),
|
|
1289
|
-
error: text("error"),
|
|
1290
|
-
createdAt: text("created_at").notNull()
|
|
1291
|
-
}, (table) => [
|
|
1292
|
-
index("idx_runs_project").on(table.projectId),
|
|
1293
|
-
index("idx_runs_status").on(table.status)
|
|
1294
|
-
]);
|
|
1295
|
-
var querySnapshots = sqliteTable("query_snapshots", {
|
|
1296
|
-
id: text("id").primaryKey(),
|
|
1297
|
-
runId: text("run_id").notNull().references(() => runs.id, { onDelete: "cascade" }),
|
|
1298
|
-
keywordId: text("keyword_id").notNull().references(() => keywords.id, { onDelete: "cascade" }),
|
|
1299
|
-
provider: text("provider").notNull().default("gemini"),
|
|
1300
|
-
model: text("model"),
|
|
1301
|
-
citationState: text("citation_state").notNull(),
|
|
1302
|
-
answerMentioned: integer("answer_mentioned", { mode: "boolean" }),
|
|
1303
|
-
answerText: text("answer_text"),
|
|
1304
|
-
citedDomains: text("cited_domains").notNull().default("[]"),
|
|
1305
|
-
competitorOverlap: text("competitor_overlap").notNull().default("[]"),
|
|
1306
|
-
recommendedCompetitors: text("recommended_competitors").notNull().default("[]"),
|
|
1307
|
-
location: text("location"),
|
|
1308
|
-
screenshotPath: text("screenshot_path"),
|
|
1309
|
-
rawResponse: text("raw_response"),
|
|
1310
|
-
createdAt: text("created_at").notNull()
|
|
1311
|
-
}, (table) => [
|
|
1312
|
-
index("idx_snapshots_run").on(table.runId),
|
|
1313
|
-
index("idx_snapshots_keyword").on(table.keywordId)
|
|
1314
|
-
]);
|
|
1315
|
-
var auditLog = sqliteTable("audit_log", {
|
|
1316
|
-
id: text("id").primaryKey(),
|
|
1317
|
-
projectId: text("project_id").references(() => projects.id, { onDelete: "cascade" }),
|
|
1318
|
-
actor: text("actor").notNull(),
|
|
1319
|
-
action: text("action").notNull(),
|
|
1320
|
-
entityType: text("entity_type").notNull(),
|
|
1321
|
-
entityId: text("entity_id"),
|
|
1322
|
-
diff: text("diff"),
|
|
1323
|
-
createdAt: text("created_at").notNull()
|
|
1324
|
-
}, (table) => [
|
|
1325
|
-
index("idx_audit_log_project").on(table.projectId),
|
|
1326
|
-
index("idx_audit_log_created").on(table.createdAt)
|
|
1327
|
-
]);
|
|
1328
|
-
var apiKeys = sqliteTable("api_keys", {
|
|
1329
|
-
id: text("id").primaryKey(),
|
|
1330
|
-
name: text("name").notNull(),
|
|
1331
|
-
keyHash: text("key_hash").notNull().unique(),
|
|
1332
|
-
keyPrefix: text("key_prefix").notNull(),
|
|
1333
|
-
scopes: text("scopes").notNull().default('["*"]'),
|
|
1334
|
-
createdAt: text("created_at").notNull(),
|
|
1335
|
-
lastUsedAt: text("last_used_at"),
|
|
1336
|
-
revokedAt: text("revoked_at")
|
|
1337
|
-
}, (table) => [
|
|
1338
|
-
index("idx_api_keys_prefix").on(table.keyPrefix)
|
|
1339
|
-
]);
|
|
1340
|
-
var schedules = sqliteTable("schedules", {
|
|
1341
|
-
id: text("id").primaryKey(),
|
|
1342
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1343
|
-
cronExpr: text("cron_expr").notNull(),
|
|
1344
|
-
preset: text("preset"),
|
|
1345
|
-
timezone: text("timezone").notNull().default("UTC"),
|
|
1346
|
-
enabled: integer("enabled").notNull().default(1),
|
|
1347
|
-
providers: text("providers").notNull().default("[]"),
|
|
1348
|
-
lastRunAt: text("last_run_at"),
|
|
1349
|
-
nextRunAt: text("next_run_at"),
|
|
1350
|
-
createdAt: text("created_at").notNull(),
|
|
1351
|
-
updatedAt: text("updated_at").notNull()
|
|
1352
|
-
}, (table) => [
|
|
1353
|
-
uniqueIndex("idx_schedules_project").on(table.projectId)
|
|
1354
|
-
]);
|
|
1355
|
-
var notifications = sqliteTable("notifications", {
|
|
1356
|
-
id: text("id").primaryKey(),
|
|
1357
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1358
|
-
channel: text("channel").notNull(),
|
|
1359
|
-
config: text("config").notNull(),
|
|
1360
|
-
webhookSecret: text("webhook_secret"),
|
|
1361
|
-
enabled: integer("enabled").notNull().default(1),
|
|
1362
|
-
createdAt: text("created_at").notNull(),
|
|
1363
|
-
updatedAt: text("updated_at").notNull()
|
|
1364
|
-
}, (table) => [
|
|
1365
|
-
index("idx_notifications_project").on(table.projectId)
|
|
1366
|
-
]);
|
|
1367
|
-
var googleConnections = sqliteTable("google_connections", {
|
|
1368
|
-
id: text("id").primaryKey(),
|
|
1369
|
-
domain: text("domain").notNull(),
|
|
1370
|
-
connectionType: text("connection_type").notNull(),
|
|
1371
|
-
propertyId: text("property_id"),
|
|
1372
|
-
sitemapUrl: text("sitemap_url"),
|
|
1373
|
-
// WARNING: Authentication material should be stored in config.yaml per CLAUDE.md
|
|
1374
|
-
accessToken: text("access_token"),
|
|
1375
|
-
refreshToken: text("refresh_token"),
|
|
1376
|
-
tokenExpiresAt: text("token_expires_at"),
|
|
1377
|
-
scopes: text("scopes").notNull().default("[]"),
|
|
1378
|
-
createdAt: text("created_at").notNull(),
|
|
1379
|
-
updatedAt: text("updated_at").notNull()
|
|
1380
|
-
}, (table) => [
|
|
1381
|
-
uniqueIndex("idx_google_conn_domain_type").on(table.domain, table.connectionType)
|
|
1382
|
-
]);
|
|
1383
|
-
var gscSearchData = sqliteTable("gsc_search_data", {
|
|
1384
|
-
id: text("id").primaryKey(),
|
|
1385
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1386
|
-
syncRunId: text("sync_run_id").notNull().references(() => runs.id, { onDelete: "cascade" }),
|
|
1387
|
-
date: text("date").notNull(),
|
|
1388
|
-
query: text("query").notNull(),
|
|
1389
|
-
page: text("page").notNull(),
|
|
1390
|
-
country: text("country"),
|
|
1391
|
-
device: text("device"),
|
|
1392
|
-
clicks: integer("clicks").notNull().default(0),
|
|
1393
|
-
impressions: integer("impressions").notNull().default(0),
|
|
1394
|
-
ctr: text("ctr").notNull().default("0"),
|
|
1395
|
-
position: text("position").notNull().default("0"),
|
|
1396
|
-
createdAt: text("created_at").notNull()
|
|
1397
|
-
}, (table) => [
|
|
1398
|
-
index("idx_gsc_search_project_date").on(table.projectId, table.date),
|
|
1399
|
-
index("idx_gsc_search_query").on(table.query),
|
|
1400
|
-
index("idx_gsc_search_run").on(table.syncRunId)
|
|
1401
|
-
]);
|
|
1402
|
-
var gscUrlInspections = sqliteTable("gsc_url_inspections", {
|
|
1403
|
-
id: text("id").primaryKey(),
|
|
1404
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1405
|
-
syncRunId: text("sync_run_id").references(() => runs.id, { onDelete: "cascade" }),
|
|
1406
|
-
url: text("url").notNull(),
|
|
1407
|
-
indexingState: text("indexing_state"),
|
|
1408
|
-
verdict: text("verdict"),
|
|
1409
|
-
coverageState: text("coverage_state"),
|
|
1410
|
-
pageFetchState: text("page_fetch_state"),
|
|
1411
|
-
robotsTxtState: text("robots_txt_state"),
|
|
1412
|
-
crawlTime: text("crawl_time"),
|
|
1413
|
-
lastCrawlResult: text("last_crawl_result"),
|
|
1414
|
-
isMobileFriendly: integer("is_mobile_friendly"),
|
|
1415
|
-
richResults: text("rich_results").notNull().default("[]"),
|
|
1416
|
-
referringUrls: text("referring_urls").notNull().default("[]"),
|
|
1417
|
-
inspectedAt: text("inspected_at").notNull(),
|
|
1418
|
-
createdAt: text("created_at").notNull()
|
|
1419
|
-
}, (table) => [
|
|
1420
|
-
index("idx_gsc_inspect_project_url").on(table.projectId, table.url),
|
|
1421
|
-
index("idx_gsc_inspect_run").on(table.syncRunId),
|
|
1422
|
-
index("idx_gsc_inspect_url_time").on(table.url, table.inspectedAt)
|
|
1423
|
-
]);
|
|
1424
|
-
var gscCoverageSnapshots = sqliteTable("gsc_coverage_snapshots", {
|
|
1425
|
-
id: text("id").primaryKey(),
|
|
1426
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1427
|
-
syncRunId: text("sync_run_id").references(() => runs.id, { onDelete: "cascade" }),
|
|
1428
|
-
date: text("date").notNull(),
|
|
1429
|
-
indexed: integer("indexed").notNull().default(0),
|
|
1430
|
-
notIndexed: integer("not_indexed").notNull().default(0),
|
|
1431
|
-
reasonBreakdown: text("reason_breakdown").notNull().default("{}"),
|
|
1432
|
-
createdAt: text("created_at").notNull()
|
|
1433
|
-
}, (table) => [
|
|
1434
|
-
index("idx_gsc_coverage_snap_project_date").on(table.projectId, table.date),
|
|
1435
|
-
index("idx_gsc_coverage_snap_run").on(table.syncRunId)
|
|
1436
|
-
]);
|
|
1437
|
-
var bingConnections = sqliteTable("bing_connections", {
|
|
1438
|
-
id: text("id").primaryKey(),
|
|
1439
|
-
domain: text("domain").notNull(),
|
|
1440
|
-
siteUrl: text("site_url"),
|
|
1441
|
-
createdAt: text("created_at").notNull(),
|
|
1442
|
-
updatedAt: text("updated_at").notNull()
|
|
1443
|
-
}, (table) => [
|
|
1444
|
-
uniqueIndex("idx_bing_conn_domain").on(table.domain)
|
|
1445
|
-
]);
|
|
1446
|
-
var bingUrlInspections = sqliteTable("bing_url_inspections", {
|
|
1447
|
-
id: text("id").primaryKey(),
|
|
1448
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1449
|
-
url: text("url").notNull(),
|
|
1450
|
-
httpCode: integer("http_code"),
|
|
1451
|
-
inIndex: integer("in_index"),
|
|
1452
|
-
lastCrawledDate: text("last_crawled_date"),
|
|
1453
|
-
inIndexDate: text("in_index_date"),
|
|
1454
|
-
inspectedAt: text("inspected_at").notNull(),
|
|
1455
|
-
createdAt: text("created_at").notNull(),
|
|
1456
|
-
documentSize: integer("document_size"),
|
|
1457
|
-
anchorCount: integer("anchor_count"),
|
|
1458
|
-
discoveryDate: text("discovery_date")
|
|
1459
|
-
}, (table) => [
|
|
1460
|
-
index("idx_bing_inspect_project_url").on(table.projectId, table.url),
|
|
1461
|
-
index("idx_bing_inspect_url_time").on(table.url, table.inspectedAt)
|
|
1462
|
-
]);
|
|
1463
|
-
var bingKeywordStats = sqliteTable("bing_keyword_stats", {
|
|
1464
|
-
id: text("id").primaryKey(),
|
|
1465
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1466
|
-
query: text("query").notNull(),
|
|
1467
|
-
impressions: integer("impressions").notNull().default(0),
|
|
1468
|
-
clicks: integer("clicks").notNull().default(0),
|
|
1469
|
-
ctr: text("ctr").notNull().default("0"),
|
|
1470
|
-
averagePosition: text("average_position").notNull().default("0"),
|
|
1471
|
-
syncedAt: text("synced_at").notNull(),
|
|
1472
|
-
createdAt: text("created_at").notNull()
|
|
1473
|
-
}, (table) => [
|
|
1474
|
-
index("idx_bing_keyword_project").on(table.projectId),
|
|
1475
|
-
index("idx_bing_keyword_query").on(table.query)
|
|
1476
|
-
]);
|
|
1477
|
-
var gaConnections = sqliteTable("ga_connections", {
|
|
1478
|
-
id: text("id").primaryKey(),
|
|
1479
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1480
|
-
propertyId: text("property_id").notNull(),
|
|
1481
|
-
clientEmail: text("client_email").notNull(),
|
|
1482
|
-
// WARNING: Authentication material should be stored in config.yaml per CLAUDE.md
|
|
1483
|
-
privateKey: text("private_key").notNull(),
|
|
1484
|
-
createdAt: text("created_at").notNull(),
|
|
1485
|
-
updatedAt: text("updated_at").notNull()
|
|
1486
|
-
}, (table) => [
|
|
1487
|
-
uniqueIndex("idx_ga_conn_project").on(table.projectId)
|
|
1488
|
-
]);
|
|
1489
|
-
var gaTrafficSnapshots = sqliteTable("ga_traffic_snapshots", {
|
|
1490
|
-
id: text("id").primaryKey(),
|
|
1491
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1492
|
-
date: text("date").notNull(),
|
|
1493
|
-
landingPage: text("landing_page").notNull(),
|
|
1494
|
-
sessions: integer("sessions").notNull().default(0),
|
|
1495
|
-
organicSessions: integer("organic_sessions").notNull().default(0),
|
|
1496
|
-
users: integer("users").notNull().default(0),
|
|
1497
|
-
syncedAt: text("synced_at").notNull()
|
|
1498
|
-
}, (table) => [
|
|
1499
|
-
index("idx_ga_traffic_project_date").on(table.projectId, table.date),
|
|
1500
|
-
index("idx_ga_traffic_page").on(table.landingPage)
|
|
1501
|
-
]);
|
|
1502
|
-
var gaAiReferrals = sqliteTable("ga_ai_referrals", {
|
|
1503
|
-
id: text("id").primaryKey(),
|
|
1504
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1505
|
-
date: text("date").notNull(),
|
|
1506
|
-
source: text("source").notNull(),
|
|
1507
|
-
medium: text("medium").notNull(),
|
|
1508
|
-
/** Which GA4 dimension produced this row: 'session' | 'first_user' | 'manual_utm' */
|
|
1509
|
-
sourceDimension: text("source_dimension").notNull().default("session"),
|
|
1510
|
-
sessions: integer("sessions").notNull().default(0),
|
|
1511
|
-
users: integer("users").notNull().default(0),
|
|
1512
|
-
syncedAt: text("synced_at").notNull()
|
|
1513
|
-
}, (table) => [
|
|
1514
|
-
index("idx_ga_ai_ref_project_date").on(table.projectId, table.date),
|
|
1515
|
-
index("idx_ga_ai_ref_source").on(table.source),
|
|
1516
|
-
uniqueIndex("idx_ga_ai_ref_unique_v2").on(table.projectId, table.date, table.source, table.medium, table.sourceDimension)
|
|
1517
|
-
]);
|
|
1518
|
-
var gaTrafficSummaries = sqliteTable("ga_traffic_summaries", {
|
|
1519
|
-
id: text("id").primaryKey(),
|
|
1520
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1521
|
-
periodStart: text("period_start").notNull(),
|
|
1522
|
-
periodEnd: text("period_end").notNull(),
|
|
1523
|
-
totalSessions: integer("total_sessions").notNull().default(0),
|
|
1524
|
-
totalOrganicSessions: integer("total_organic_sessions").notNull().default(0),
|
|
1525
|
-
totalUsers: integer("total_users").notNull().default(0),
|
|
1526
|
-
syncedAt: text("synced_at").notNull()
|
|
1527
|
-
}, (table) => [
|
|
1528
|
-
index("idx_ga_summary_project").on(table.projectId)
|
|
1529
|
-
]);
|
|
1530
|
-
var usageCounters = sqliteTable("usage_counters", {
|
|
1531
|
-
id: text("id").primaryKey(),
|
|
1532
|
-
scope: text("scope").notNull(),
|
|
1533
|
-
period: text("period").notNull(),
|
|
1534
|
-
metric: text("metric").notNull(),
|
|
1535
|
-
count: integer("count").notNull().default(0),
|
|
1536
|
-
updatedAt: text("updated_at").notNull()
|
|
1537
|
-
}, (table) => [
|
|
1538
|
-
uniqueIndex("idx_usage_scope_period_metric").on(table.scope, table.period, table.metric),
|
|
1539
|
-
index("idx_usage_scope_period").on(table.scope, table.period)
|
|
1540
|
-
]);
|
|
1541
|
-
var insights = sqliteTable("insights", {
|
|
1542
|
-
id: text("id").primaryKey(),
|
|
1543
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1544
|
-
runId: text("run_id").references(() => runs.id, { onDelete: "cascade" }),
|
|
1545
|
-
type: text("type").notNull(),
|
|
1546
|
-
severity: text("severity").notNull(),
|
|
1547
|
-
title: text("title").notNull(),
|
|
1548
|
-
keyword: text("keyword").notNull(),
|
|
1549
|
-
provider: text("provider").notNull(),
|
|
1550
|
-
recommendation: text("recommendation"),
|
|
1551
|
-
cause: text("cause"),
|
|
1552
|
-
dismissed: integer("dismissed", { mode: "boolean" }).notNull().default(false),
|
|
1553
|
-
createdAt: text("created_at").notNull()
|
|
1554
|
-
}, (table) => [
|
|
1555
|
-
index("idx_insights_project").on(table.projectId),
|
|
1556
|
-
index("idx_insights_run").on(table.runId),
|
|
1557
|
-
index("idx_insights_created").on(table.createdAt),
|
|
1558
|
-
index("idx_insights_keyword_provider").on(table.keyword, table.provider)
|
|
1559
|
-
]);
|
|
1560
|
-
var healthSnapshots = sqliteTable("health_snapshots", {
|
|
1561
|
-
id: text("id").primaryKey(),
|
|
1562
|
-
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1563
|
-
runId: text("run_id").references(() => runs.id, { onDelete: "cascade" }),
|
|
1564
|
-
overallCitedRate: text("overall_cited_rate").notNull(),
|
|
1565
|
-
totalPairs: integer("total_pairs").notNull(),
|
|
1566
|
-
citedPairs: integer("cited_pairs").notNull(),
|
|
1567
|
-
providerBreakdown: text("provider_breakdown").notNull().default("{}"),
|
|
1568
|
-
createdAt: text("created_at").notNull()
|
|
1569
|
-
}, (table) => [
|
|
1570
|
-
index("idx_health_snapshots_project").on(table.projectId),
|
|
1571
|
-
index("idx_health_snapshots_run").on(table.runId),
|
|
1572
|
-
index("idx_health_snapshots_created").on(table.createdAt)
|
|
1573
|
-
]);
|
|
1574
|
-
|
|
1575
|
-
// ../db/src/client.ts
|
|
1576
|
-
function createClient(databasePath) {
|
|
1577
|
-
mkdirSync(dirname(databasePath), { recursive: true });
|
|
1578
|
-
const sqlite = new Database(databasePath);
|
|
1579
|
-
sqlite.pragma("journal_mode = WAL");
|
|
1580
|
-
sqlite.pragma("foreign_keys = ON");
|
|
1581
|
-
sqlite.pragma("busy_timeout = 5000");
|
|
1582
|
-
return drizzle(sqlite, { schema: schema_exports });
|
|
1583
|
-
}
|
|
1584
|
-
|
|
1585
|
-
// ../db/src/json.ts
|
|
1586
|
-
function parseJsonColumn(value, fallback) {
|
|
1587
|
-
if (value == null || value === "") return fallback;
|
|
1588
|
-
try {
|
|
1589
|
-
return JSON.parse(value);
|
|
1590
|
-
} catch {
|
|
1591
|
-
return fallback;
|
|
1592
|
-
}
|
|
1593
|
-
}
|
|
1594
|
-
|
|
1595
|
-
// ../db/src/migrate.ts
|
|
1596
|
-
import { sql } from "drizzle-orm";
|
|
1597
|
-
var MIGRATION_SQL = `
|
|
1598
|
-
CREATE TABLE IF NOT EXISTS projects (
|
|
1599
|
-
id TEXT PRIMARY KEY,
|
|
1600
|
-
name TEXT NOT NULL UNIQUE,
|
|
1601
|
-
display_name TEXT NOT NULL,
|
|
1602
|
-
canonical_domain TEXT NOT NULL,
|
|
1603
|
-
owned_domains TEXT NOT NULL DEFAULT '[]',
|
|
1604
|
-
country TEXT NOT NULL,
|
|
1605
|
-
language TEXT NOT NULL,
|
|
1606
|
-
tags TEXT NOT NULL DEFAULT '[]',
|
|
1607
|
-
labels TEXT NOT NULL DEFAULT '{}',
|
|
1608
|
-
providers TEXT NOT NULL DEFAULT '[]',
|
|
1609
|
-
config_source TEXT NOT NULL DEFAULT 'cli',
|
|
1610
|
-
config_revision INTEGER NOT NULL DEFAULT 1,
|
|
1611
|
-
created_at TEXT NOT NULL,
|
|
1612
|
-
updated_at TEXT NOT NULL
|
|
1613
|
-
);
|
|
1614
|
-
|
|
1615
|
-
CREATE TABLE IF NOT EXISTS keywords (
|
|
1616
|
-
id TEXT PRIMARY KEY,
|
|
1617
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1618
|
-
keyword TEXT NOT NULL,
|
|
1619
|
-
created_at TEXT NOT NULL,
|
|
1620
|
-
UNIQUE(project_id, keyword)
|
|
1621
|
-
);
|
|
1622
|
-
|
|
1623
|
-
CREATE TABLE IF NOT EXISTS competitors (
|
|
1624
|
-
id TEXT PRIMARY KEY,
|
|
1625
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1626
|
-
domain TEXT NOT NULL,
|
|
1627
|
-
created_at TEXT NOT NULL,
|
|
1628
|
-
UNIQUE(project_id, domain)
|
|
1629
|
-
);
|
|
1630
|
-
|
|
1631
|
-
CREATE TABLE IF NOT EXISTS runs (
|
|
1632
|
-
id TEXT PRIMARY KEY,
|
|
1633
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1634
|
-
kind TEXT NOT NULL DEFAULT 'answer-visibility',
|
|
1635
|
-
status TEXT NOT NULL DEFAULT 'queued',
|
|
1636
|
-
trigger TEXT NOT NULL DEFAULT 'manual',
|
|
1637
|
-
started_at TEXT,
|
|
1638
|
-
finished_at TEXT,
|
|
1639
|
-
error TEXT,
|
|
1640
|
-
created_at TEXT NOT NULL
|
|
1641
|
-
);
|
|
1642
|
-
|
|
1643
|
-
CREATE TABLE IF NOT EXISTS query_snapshots (
|
|
1644
|
-
id TEXT PRIMARY KEY,
|
|
1645
|
-
run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
|
|
1646
|
-
keyword_id TEXT NOT NULL REFERENCES keywords(id) ON DELETE CASCADE,
|
|
1647
|
-
provider TEXT NOT NULL DEFAULT 'gemini',
|
|
1648
|
-
citation_state TEXT NOT NULL,
|
|
1649
|
-
answer_text TEXT,
|
|
1650
|
-
cited_domains TEXT NOT NULL DEFAULT '[]',
|
|
1651
|
-
competitor_overlap TEXT NOT NULL DEFAULT '[]',
|
|
1652
|
-
raw_response TEXT,
|
|
1653
|
-
created_at TEXT NOT NULL
|
|
1654
|
-
);
|
|
1655
|
-
|
|
1656
|
-
CREATE TABLE IF NOT EXISTS audit_log (
|
|
1657
|
-
id TEXT PRIMARY KEY,
|
|
1658
|
-
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
1659
|
-
actor TEXT NOT NULL,
|
|
1660
|
-
action TEXT NOT NULL,
|
|
1661
|
-
entity_type TEXT NOT NULL,
|
|
1662
|
-
entity_id TEXT,
|
|
1663
|
-
diff TEXT,
|
|
1664
|
-
created_at TEXT NOT NULL
|
|
1665
|
-
);
|
|
1666
|
-
|
|
1667
|
-
CREATE TABLE IF NOT EXISTS api_keys (
|
|
1668
|
-
id TEXT PRIMARY KEY,
|
|
1669
|
-
name TEXT NOT NULL,
|
|
1670
|
-
key_hash TEXT NOT NULL UNIQUE,
|
|
1671
|
-
key_prefix TEXT NOT NULL,
|
|
1672
|
-
scopes TEXT NOT NULL DEFAULT '["*"]',
|
|
1673
|
-
created_at TEXT NOT NULL,
|
|
1674
|
-
last_used_at TEXT,
|
|
1675
|
-
revoked_at TEXT
|
|
1676
|
-
);
|
|
1677
|
-
|
|
1678
|
-
CREATE TABLE IF NOT EXISTS usage_counters (
|
|
1679
|
-
id TEXT PRIMARY KEY,
|
|
1680
|
-
scope TEXT NOT NULL,
|
|
1681
|
-
period TEXT NOT NULL,
|
|
1682
|
-
metric TEXT NOT NULL,
|
|
1683
|
-
count INTEGER NOT NULL DEFAULT 0,
|
|
1684
|
-
updated_at TEXT NOT NULL,
|
|
1685
|
-
UNIQUE(scope, period, metric)
|
|
1686
|
-
);
|
|
1687
|
-
|
|
1688
|
-
CREATE INDEX IF NOT EXISTS idx_keywords_project ON keywords(project_id);
|
|
1689
|
-
CREATE INDEX IF NOT EXISTS idx_competitors_project ON competitors(project_id);
|
|
1690
|
-
CREATE INDEX IF NOT EXISTS idx_runs_project ON runs(project_id);
|
|
1691
|
-
CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status);
|
|
1692
|
-
CREATE INDEX IF NOT EXISTS idx_snapshots_run ON query_snapshots(run_id);
|
|
1693
|
-
CREATE INDEX IF NOT EXISTS idx_snapshots_keyword ON query_snapshots(keyword_id);
|
|
1694
|
-
CREATE INDEX IF NOT EXISTS idx_audit_log_project ON audit_log(project_id);
|
|
1695
|
-
CREATE INDEX IF NOT EXISTS idx_audit_log_created ON audit_log(created_at);
|
|
1696
|
-
CREATE TABLE IF NOT EXISTS schedules (
|
|
1697
|
-
id TEXT PRIMARY KEY,
|
|
1698
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1699
|
-
cron_expr TEXT NOT NULL,
|
|
1700
|
-
preset TEXT,
|
|
1701
|
-
timezone TEXT NOT NULL DEFAULT 'UTC',
|
|
1702
|
-
enabled INTEGER NOT NULL DEFAULT 1,
|
|
1703
|
-
providers TEXT NOT NULL DEFAULT '[]',
|
|
1704
|
-
last_run_at TEXT,
|
|
1705
|
-
next_run_at TEXT,
|
|
1706
|
-
created_at TEXT NOT NULL,
|
|
1707
|
-
updated_at TEXT NOT NULL,
|
|
1708
|
-
UNIQUE(project_id)
|
|
1709
|
-
);
|
|
1710
|
-
|
|
1711
|
-
CREATE TABLE IF NOT EXISTS notifications (
|
|
1712
|
-
id TEXT PRIMARY KEY,
|
|
1713
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1714
|
-
channel TEXT NOT NULL,
|
|
1715
|
-
config TEXT NOT NULL,
|
|
1716
|
-
enabled INTEGER NOT NULL DEFAULT 1,
|
|
1717
|
-
created_at TEXT NOT NULL,
|
|
1718
|
-
updated_at TEXT NOT NULL
|
|
1719
|
-
);
|
|
1720
|
-
|
|
1721
|
-
CREATE INDEX IF NOT EXISTS idx_api_keys_prefix ON api_keys(key_prefix);
|
|
1722
|
-
CREATE INDEX IF NOT EXISTS idx_usage_scope_period ON usage_counters(scope, period);
|
|
1723
|
-
CREATE UNIQUE INDEX IF NOT EXISTS idx_schedules_project ON schedules(project_id);
|
|
1724
|
-
CREATE INDEX IF NOT EXISTS idx_notifications_project ON notifications(project_id);
|
|
1725
|
-
`;
|
|
1726
|
-
var MIGRATIONS = [
|
|
1727
|
-
// v2: Add providers column to projects for multi-provider support
|
|
1728
|
-
`ALTER TABLE projects ADD COLUMN providers TEXT NOT NULL DEFAULT '[]'`,
|
|
1729
|
-
// v3: Add webhook_secret column to notifications for HMAC signing
|
|
1730
|
-
`ALTER TABLE notifications ADD COLUMN webhook_secret TEXT`,
|
|
1731
|
-
// v4: Add owned_domains column to projects for multi-domain citation matching
|
|
1732
|
-
`ALTER TABLE projects ADD COLUMN owned_domains TEXT NOT NULL DEFAULT '[]'`,
|
|
1733
|
-
// v5: Add model column to query_snapshots for per-model scoring
|
|
1734
|
-
`ALTER TABLE query_snapshots ADD COLUMN model TEXT`,
|
|
1735
|
-
// v5b: Backfill model from rawResponse JSON for existing snapshots
|
|
1736
|
-
`UPDATE query_snapshots SET model = json_extract(raw_response, '$.model') WHERE model IS NULL AND raw_response IS NOT NULL AND json_extract(raw_response, '$.model') IS NOT NULL`,
|
|
1737
|
-
// v6: Google Search Console integration — google_connections table (domain-scoped)
|
|
1738
|
-
// WARNING: access_token, refresh_token are authentication material; consider storing in config.yaml per CLAUDE.md
|
|
1739
|
-
`CREATE TABLE IF NOT EXISTS google_connections (
|
|
1740
|
-
id TEXT PRIMARY KEY,
|
|
1741
|
-
domain TEXT NOT NULL,
|
|
1742
|
-
connection_type TEXT NOT NULL,
|
|
1743
|
-
property_id TEXT,
|
|
1744
|
-
access_token TEXT,
|
|
1745
|
-
refresh_token TEXT,
|
|
1746
|
-
token_expires_at TEXT,
|
|
1747
|
-
scopes TEXT NOT NULL DEFAULT '[]',
|
|
1748
|
-
created_at TEXT NOT NULL,
|
|
1749
|
-
updated_at TEXT NOT NULL
|
|
1750
|
-
)`,
|
|
1751
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_google_conn_domain_type ON google_connections(domain, connection_type)`,
|
|
1752
|
-
// v6: Google Search Console integration — gsc_search_data table
|
|
1753
|
-
`CREATE TABLE IF NOT EXISTS gsc_search_data (
|
|
1754
|
-
id TEXT PRIMARY KEY,
|
|
1755
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1756
|
-
sync_run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
|
|
1757
|
-
date TEXT NOT NULL,
|
|
1758
|
-
query TEXT NOT NULL,
|
|
1759
|
-
page TEXT NOT NULL,
|
|
1760
|
-
country TEXT,
|
|
1761
|
-
device TEXT,
|
|
1762
|
-
clicks INTEGER NOT NULL DEFAULT 0,
|
|
1763
|
-
impressions INTEGER NOT NULL DEFAULT 0,
|
|
1764
|
-
ctr TEXT NOT NULL DEFAULT '0',
|
|
1765
|
-
position TEXT NOT NULL DEFAULT '0',
|
|
1766
|
-
created_at TEXT NOT NULL
|
|
1767
|
-
)`,
|
|
1768
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_search_project_date ON gsc_search_data(project_id, date)`,
|
|
1769
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_search_query ON gsc_search_data(query)`,
|
|
1770
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_search_run ON gsc_search_data(sync_run_id)`,
|
|
1771
|
-
// v6: Google Search Console integration — gsc_url_inspections table
|
|
1772
|
-
`CREATE TABLE IF NOT EXISTS gsc_url_inspections (
|
|
1773
|
-
id TEXT PRIMARY KEY,
|
|
1774
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1775
|
-
sync_run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
|
1776
|
-
url TEXT NOT NULL,
|
|
1777
|
-
indexing_state TEXT,
|
|
1778
|
-
verdict TEXT,
|
|
1779
|
-
coverage_state TEXT,
|
|
1780
|
-
page_fetch_state TEXT,
|
|
1781
|
-
robots_txt_state TEXT,
|
|
1782
|
-
crawl_time TEXT,
|
|
1783
|
-
last_crawl_result TEXT,
|
|
1784
|
-
is_mobile_friendly INTEGER,
|
|
1785
|
-
rich_results TEXT NOT NULL DEFAULT '[]',
|
|
1786
|
-
referring_urls TEXT NOT NULL DEFAULT '[]',
|
|
1787
|
-
inspected_at TEXT NOT NULL,
|
|
1788
|
-
created_at TEXT NOT NULL
|
|
1789
|
-
)`,
|
|
1790
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_inspect_project_url ON gsc_url_inspections(project_id, url)`,
|
|
1791
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_inspect_run ON gsc_url_inspections(sync_run_id)`,
|
|
1792
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_inspect_url_time ON gsc_url_inspections(url, inspected_at)`,
|
|
1793
|
-
// v7: GSC coverage snapshots for historical tracking
|
|
1794
|
-
`CREATE TABLE IF NOT EXISTS gsc_coverage_snapshots (
|
|
1795
|
-
id TEXT PRIMARY KEY,
|
|
1796
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1797
|
-
sync_run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
|
1798
|
-
date TEXT NOT NULL,
|
|
1799
|
-
indexed INTEGER NOT NULL DEFAULT 0,
|
|
1800
|
-
not_indexed INTEGER NOT NULL DEFAULT 0,
|
|
1801
|
-
reason_breakdown TEXT NOT NULL DEFAULT '{}',
|
|
1802
|
-
created_at TEXT NOT NULL
|
|
1803
|
-
)`,
|
|
1804
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_coverage_snap_project_date ON gsc_coverage_snapshots(project_id, date)`,
|
|
1805
|
-
`CREATE INDEX IF NOT EXISTS idx_gsc_coverage_snap_run ON gsc_coverage_snapshots(sync_run_id)`,
|
|
1806
|
-
// v8: Location-aware sweeps — project locations + snapshot location tag
|
|
1807
|
-
`ALTER TABLE projects ADD COLUMN locations TEXT NOT NULL DEFAULT '[]'`,
|
|
1808
|
-
`ALTER TABLE projects ADD COLUMN default_location TEXT`,
|
|
1809
|
-
`ALTER TABLE query_snapshots ADD COLUMN location TEXT`,
|
|
1810
|
-
// v9: Add location column to runs for per-location run tracking
|
|
1811
|
-
`ALTER TABLE runs ADD COLUMN location TEXT`,
|
|
1812
|
-
// v10: Add sitemapUrl to google_connections for persistent sitemap storage
|
|
1813
|
-
`ALTER TABLE google_connections ADD COLUMN sitemap_url TEXT`,
|
|
1814
|
-
// v11: CDP browser provider — screenshot path for captured evidence
|
|
1815
|
-
`ALTER TABLE query_snapshots ADD COLUMN screenshot_path TEXT`,
|
|
1816
|
-
// v12: Bing Webmaster Tools — bing_connections table
|
|
1817
|
-
`CREATE TABLE IF NOT EXISTS bing_connections (
|
|
1818
|
-
id TEXT PRIMARY KEY,
|
|
1819
|
-
domain TEXT NOT NULL,
|
|
1820
|
-
site_url TEXT,
|
|
1821
|
-
created_at TEXT NOT NULL,
|
|
1822
|
-
updated_at TEXT NOT NULL
|
|
1823
|
-
)`,
|
|
1824
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_bing_conn_domain ON bing_connections(domain)`,
|
|
1825
|
-
// v12: Bing Webmaster Tools — bing_url_inspections table
|
|
1826
|
-
`CREATE TABLE IF NOT EXISTS bing_url_inspections (
|
|
1827
|
-
id TEXT PRIMARY KEY,
|
|
1828
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1829
|
-
url TEXT NOT NULL,
|
|
1830
|
-
http_code INTEGER,
|
|
1831
|
-
in_index INTEGER,
|
|
1832
|
-
last_crawled_date TEXT,
|
|
1833
|
-
in_index_date TEXT,
|
|
1834
|
-
inspected_at TEXT NOT NULL,
|
|
1835
|
-
created_at TEXT NOT NULL
|
|
1836
|
-
)`,
|
|
1837
|
-
`CREATE INDEX IF NOT EXISTS idx_bing_inspect_project_url ON bing_url_inspections(project_id, url)`,
|
|
1838
|
-
`CREATE INDEX IF NOT EXISTS idx_bing_inspect_url_time ON bing_url_inspections(url, inspected_at)`,
|
|
1839
|
-
// v12: Bing Webmaster Tools — bing_keyword_stats table
|
|
1840
|
-
`CREATE TABLE IF NOT EXISTS bing_keyword_stats (
|
|
1841
|
-
id TEXT PRIMARY KEY,
|
|
1842
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1843
|
-
query TEXT NOT NULL,
|
|
1844
|
-
impressions INTEGER NOT NULL DEFAULT 0,
|
|
1845
|
-
clicks INTEGER NOT NULL DEFAULT 0,
|
|
1846
|
-
ctr TEXT NOT NULL DEFAULT '0',
|
|
1847
|
-
average_position TEXT NOT NULL DEFAULT '0',
|
|
1848
|
-
synced_at TEXT NOT NULL,
|
|
1849
|
-
created_at TEXT NOT NULL
|
|
1850
|
-
)`,
|
|
1851
|
-
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_project ON bing_keyword_stats(project_id)`,
|
|
1852
|
-
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_query ON bing_keyword_stats(query)`,
|
|
1853
|
-
// v13: Google Analytics 4 — ga_connections table (service account auth)
|
|
1854
|
-
// WARNING: private_key is authentication material; consider storing in config.yaml per CLAUDE.md
|
|
1855
|
-
`CREATE TABLE IF NOT EXISTS ga_connections (
|
|
1856
|
-
id TEXT PRIMARY KEY,
|
|
1857
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1858
|
-
property_id TEXT NOT NULL,
|
|
1859
|
-
client_email TEXT NOT NULL,
|
|
1860
|
-
private_key TEXT NOT NULL,
|
|
1861
|
-
created_at TEXT NOT NULL,
|
|
1862
|
-
updated_at TEXT NOT NULL
|
|
1863
|
-
)`,
|
|
1864
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_conn_project ON ga_connections(project_id)`,
|
|
1865
|
-
// v13: Google Analytics 4 — ga_traffic_snapshots table
|
|
1866
|
-
`CREATE TABLE IF NOT EXISTS ga_traffic_snapshots (
|
|
1867
|
-
id TEXT PRIMARY KEY,
|
|
1868
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1869
|
-
date TEXT NOT NULL,
|
|
1870
|
-
landing_page TEXT NOT NULL,
|
|
1871
|
-
sessions INTEGER NOT NULL DEFAULT 0,
|
|
1872
|
-
organic_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1873
|
-
users INTEGER NOT NULL DEFAULT 0,
|
|
1874
|
-
synced_at TEXT NOT NULL
|
|
1875
|
-
)`,
|
|
1876
|
-
`CREATE INDEX IF NOT EXISTS idx_ga_traffic_project_date ON ga_traffic_snapshots(project_id, date)`,
|
|
1877
|
-
`CREATE INDEX IF NOT EXISTS idx_ga_traffic_page ON ga_traffic_snapshots(landing_page)`,
|
|
1878
|
-
// v14: GA4 aggregate summaries — stores true unique user count per sync period
|
|
1879
|
-
`CREATE TABLE IF NOT EXISTS ga_traffic_summaries (
|
|
1880
|
-
id TEXT PRIMARY KEY,
|
|
1881
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1882
|
-
period_start TEXT NOT NULL,
|
|
1883
|
-
period_end TEXT NOT NULL,
|
|
1884
|
-
total_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1885
|
-
total_organic_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1886
|
-
total_users INTEGER NOT NULL DEFAULT 0,
|
|
1887
|
-
synced_at TEXT NOT NULL
|
|
1888
|
-
)`,
|
|
1889
|
-
`CREATE INDEX IF NOT EXISTS idx_ga_summary_project ON ga_traffic_summaries(project_id)`,
|
|
1890
|
-
// v15: Bing URL inspections — document_size, anchor_count, discovery_date columns
|
|
1891
|
-
`ALTER TABLE bing_url_inspections ADD COLUMN document_size INTEGER`,
|
|
1892
|
-
`ALTER TABLE bing_url_inspections ADD COLUMN anchor_count INTEGER`,
|
|
1893
|
-
`ALTER TABLE bing_url_inspections ADD COLUMN discovery_date TEXT`,
|
|
1894
|
-
// v16: Recommended competitor names extracted from run answers
|
|
1895
|
-
`ALTER TABLE query_snapshots ADD COLUMN recommended_competitors TEXT NOT NULL DEFAULT '[]'`,
|
|
1896
|
-
// v17: GA4 AI referral tracking — ga_ai_referrals table
|
|
1897
|
-
`CREATE TABLE IF NOT EXISTS ga_ai_referrals (
|
|
1898
|
-
id TEXT PRIMARY KEY,
|
|
1899
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1900
|
-
date TEXT NOT NULL,
|
|
1901
|
-
source TEXT NOT NULL,
|
|
1902
|
-
medium TEXT NOT NULL,
|
|
1903
|
-
sessions INTEGER NOT NULL DEFAULT 0,
|
|
1904
|
-
users INTEGER NOT NULL DEFAULT 0,
|
|
1905
|
-
synced_at TEXT NOT NULL
|
|
1906
|
-
)`,
|
|
1907
|
-
`CREATE INDEX IF NOT EXISTS idx_ga_ai_ref_project_date ON ga_ai_referrals(project_id, date)`,
|
|
1908
|
-
`CREATE INDEX IF NOT EXISTS idx_ga_ai_ref_source ON ga_ai_referrals(source)`,
|
|
1909
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_ai_ref_unique ON ga_ai_referrals(project_id, date, source, medium)`,
|
|
1910
|
-
// v18: Answer-level visibility derived from answer text
|
|
1911
|
-
`ALTER TABLE query_snapshots ADD COLUMN answer_mentioned INTEGER`,
|
|
1912
|
-
// v19: Add named unique indexes and missing columns from early tables
|
|
1913
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_keywords_project_keyword ON keywords(project_id, keyword)`,
|
|
1914
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_competitors_project_domain ON competitors(project_id, domain)`,
|
|
1915
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_schedules_project ON schedules(project_id)`,
|
|
1916
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_usage_scope_period_metric ON usage_counters(scope, period, metric)`,
|
|
1917
|
-
`ALTER TABLE projects ADD COLUMN config_source TEXT NOT NULL DEFAULT 'cli'`,
|
|
1918
|
-
`ALTER TABLE projects ADD COLUMN config_revision INTEGER NOT NULL DEFAULT 1`,
|
|
1919
|
-
// v20: Track which GA4 dimension produced each AI referral row
|
|
1920
|
-
// Values: 'session' (sessionSource), 'first_user' (firstUserSource), 'manual_utm' (manualSource/utm_source)
|
|
1921
|
-
`ALTER TABLE ga_ai_referrals ADD COLUMN source_dimension TEXT NOT NULL DEFAULT 'session'`,
|
|
1922
|
-
// Replace old unique index with one that includes source_dimension
|
|
1923
|
-
`DROP INDEX IF EXISTS idx_ga_ai_ref_unique`,
|
|
1924
|
-
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_ai_ref_unique_v2 ON ga_ai_referrals(project_id, date, source, medium, source_dimension)`,
|
|
1925
|
-
// v21: Add missing indexes for query_snapshots filtering
|
|
1926
|
-
`CREATE INDEX IF NOT EXISTS idx_snapshots_citation_state ON query_snapshots(citation_state)`,
|
|
1927
|
-
`CREATE INDEX IF NOT EXISTS idx_snapshots_provider_model ON query_snapshots(provider, model)`,
|
|
1928
|
-
`CREATE INDEX IF NOT EXISTS idx_snapshots_location ON query_snapshots(location)`,
|
|
1929
|
-
// v22: Intelligence — insights table for regression/gain/opportunity tracking
|
|
1930
|
-
`CREATE TABLE IF NOT EXISTS insights (
|
|
1931
|
-
id TEXT PRIMARY KEY,
|
|
1932
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1933
|
-
type TEXT NOT NULL,
|
|
1934
|
-
severity TEXT NOT NULL,
|
|
1935
|
-
title TEXT NOT NULL,
|
|
1936
|
-
keyword TEXT NOT NULL,
|
|
1937
|
-
provider TEXT NOT NULL,
|
|
1938
|
-
recommendation TEXT,
|
|
1939
|
-
cause TEXT,
|
|
1940
|
-
dismissed INTEGER NOT NULL DEFAULT 0,
|
|
1941
|
-
created_at TEXT NOT NULL
|
|
1942
|
-
)`,
|
|
1943
|
-
`CREATE INDEX IF NOT EXISTS idx_insights_project ON insights(project_id)`,
|
|
1944
|
-
`CREATE INDEX IF NOT EXISTS idx_insights_created ON insights(created_at)`,
|
|
1945
|
-
`CREATE INDEX IF NOT EXISTS idx_insights_keyword_provider ON insights(keyword, provider)`,
|
|
1946
|
-
// v23: Intelligence — health_snapshots table for citation health over time
|
|
1947
|
-
`CREATE TABLE IF NOT EXISTS health_snapshots (
|
|
1948
|
-
id TEXT PRIMARY KEY,
|
|
1949
|
-
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1950
|
-
overall_cited_rate TEXT NOT NULL,
|
|
1951
|
-
total_pairs INTEGER NOT NULL,
|
|
1952
|
-
cited_pairs INTEGER NOT NULL,
|
|
1953
|
-
provider_breakdown TEXT NOT NULL DEFAULT '{}',
|
|
1954
|
-
created_at TEXT NOT NULL
|
|
1955
|
-
)`,
|
|
1956
|
-
`CREATE INDEX IF NOT EXISTS idx_health_snapshots_project ON health_snapshots(project_id)`,
|
|
1957
|
-
`CREATE INDEX IF NOT EXISTS idx_health_snapshots_created ON health_snapshots(created_at)`,
|
|
1958
|
-
// v24: Intelligence — add run_id to insights and health_snapshots for per-run correlation and idempotency
|
|
1959
|
-
`ALTER TABLE insights ADD COLUMN run_id TEXT REFERENCES runs(id) ON DELETE CASCADE`,
|
|
1960
|
-
`CREATE INDEX IF NOT EXISTS idx_insights_run ON insights(run_id)`,
|
|
1961
|
-
`ALTER TABLE health_snapshots ADD COLUMN run_id TEXT REFERENCES runs(id) ON DELETE CASCADE`,
|
|
1962
|
-
`CREATE INDEX IF NOT EXISTS idx_health_snapshots_run ON health_snapshots(run_id)`
|
|
1963
|
-
];
|
|
1964
|
-
function isDuplicateColumnError(err) {
|
|
1965
|
-
if (!(err instanceof Error)) return false;
|
|
1966
|
-
if (err.message.includes("duplicate column name")) return true;
|
|
1967
|
-
if (err.cause instanceof Error && err.cause.message.includes("duplicate column name")) return true;
|
|
1968
|
-
return false;
|
|
1969
|
-
}
|
|
1970
|
-
function migrate(db) {
|
|
1971
|
-
const statements = MIGRATION_SQL.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
1972
|
-
for (const statement of statements) {
|
|
1973
|
-
db.run(sql.raw(statement));
|
|
1974
|
-
}
|
|
1975
|
-
for (const migration of MIGRATIONS) {
|
|
1976
|
-
try {
|
|
1977
|
-
db.run(sql.raw(migration));
|
|
1978
|
-
} catch (err) {
|
|
1979
|
-
if (isDuplicateColumnError(err)) continue;
|
|
1980
|
-
throw err;
|
|
1981
|
-
}
|
|
1982
|
-
}
|
|
1983
|
-
}
|
|
1984
|
-
|
|
1985
|
-
// ../api-routes/src/auth.ts
|
|
1986
1236
|
function hashKey(key) {
|
|
1987
1237
|
return crypto2.createHash("sha256").update(key).digest("hex");
|
|
1988
1238
|
}
|
|
@@ -2055,7 +1305,7 @@ import { eq as eq3 } from "drizzle-orm";
|
|
|
2055
1305
|
|
|
2056
1306
|
// ../api-routes/src/helpers.ts
|
|
2057
1307
|
import crypto3 from "crypto";
|
|
2058
|
-
import { eq as eq2, sql
|
|
1308
|
+
import { eq as eq2, sql } from "drizzle-orm";
|
|
2059
1309
|
function resolveProject(db, name) {
|
|
2060
1310
|
const project = db.select().from(projects).where(eq2(projects.name, name)).get();
|
|
2061
1311
|
if (!project) {
|
|
@@ -6572,7 +5822,7 @@ function formatNotification(row) {
|
|
|
6572
5822
|
|
|
6573
5823
|
// ../api-routes/src/google.ts
|
|
6574
5824
|
import crypto14 from "crypto";
|
|
6575
|
-
import { eq as eq14, and as and3, desc as desc5, sql as
|
|
5825
|
+
import { eq as eq14, and as and3, desc as desc5, sql as sql2 } from "drizzle-orm";
|
|
6576
5826
|
|
|
6577
5827
|
// ../integration-google/src/constants.ts
|
|
6578
5828
|
var GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
|
|
@@ -7536,10 +6786,10 @@ async function googleRoutes(app, opts) {
|
|
|
7536
6786
|
const project = resolveProject(app.db, request.params.name);
|
|
7537
6787
|
const { startDate, endDate, query, page, limit } = request.query;
|
|
7538
6788
|
const conditions = [eq14(gscSearchData.projectId, project.id)];
|
|
7539
|
-
if (startDate) conditions.push(
|
|
7540
|
-
if (endDate) conditions.push(
|
|
7541
|
-
if (query) conditions.push(
|
|
7542
|
-
if (page) conditions.push(
|
|
6789
|
+
if (startDate) conditions.push(sql2`${gscSearchData.date} >= ${startDate}`);
|
|
6790
|
+
if (endDate) conditions.push(sql2`${gscSearchData.date} <= ${endDate}`);
|
|
6791
|
+
if (query) conditions.push(sql2`${gscSearchData.query} LIKE ${"%" + query + "%"}`);
|
|
6792
|
+
if (page) conditions.push(sql2`${gscSearchData.page} LIKE ${"%" + page + "%"}`);
|
|
7543
6793
|
const rows = app.db.select().from(gscSearchData).where(and3(...conditions)).orderBy(desc5(gscSearchData.date)).limit(parseInt(limit ?? "500", 10)).all();
|
|
7544
6794
|
return rows.map((r) => ({
|
|
7545
6795
|
date: r.date,
|
|
@@ -8081,12 +7331,12 @@ async function bingFetch(apiKey, endpoint, opts) {
|
|
|
8081
7331
|
bingClientLog("error", "http.error", { endpoint, method, httpStatus: res.status, responseBody: body });
|
|
8082
7332
|
throw new BingApiError(`Bing API error (${res.status}): ${body}`, res.status);
|
|
8083
7333
|
}
|
|
8084
|
-
const
|
|
8085
|
-
if (!
|
|
7334
|
+
const text = await res.text();
|
|
7335
|
+
if (!text || text.trim() === "") {
|
|
8086
7336
|
return void 0;
|
|
8087
7337
|
}
|
|
8088
7338
|
try {
|
|
8089
|
-
const parsed = JSON.parse(
|
|
7339
|
+
const parsed = JSON.parse(text);
|
|
8090
7340
|
if (parsed && typeof parsed === "object" && "d" in parsed) {
|
|
8091
7341
|
return parsed.d;
|
|
8092
7342
|
}
|
|
@@ -8702,7 +7952,7 @@ async function cdpRoutes(app, opts) {
|
|
|
8702
7952
|
|
|
8703
7953
|
// ../api-routes/src/ga.ts
|
|
8704
7954
|
import crypto16 from "crypto";
|
|
8705
|
-
import { eq as eq17, desc as desc7, and as and6, sql as
|
|
7955
|
+
import { eq as eq17, desc as desc7, and as and6, sql as sql3 } from "drizzle-orm";
|
|
8706
7956
|
function gaLog(level, action, ctx) {
|
|
8707
7957
|
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
8708
7958
|
const stream = level === "error" ? process.stderr : process.stdout;
|
|
@@ -8917,8 +8167,8 @@ async function ga4Routes(app, opts) {
|
|
|
8917
8167
|
tx.delete(gaTrafficSnapshots).where(
|
|
8918
8168
|
and6(
|
|
8919
8169
|
eq17(gaTrafficSnapshots.projectId, project.id),
|
|
8920
|
-
|
|
8921
|
-
|
|
8170
|
+
sql3`${gaTrafficSnapshots.date} >= ${summary.periodStart}`,
|
|
8171
|
+
sql3`${gaTrafficSnapshots.date} <= ${summary.periodEnd}`
|
|
8922
8172
|
)
|
|
8923
8173
|
).run();
|
|
8924
8174
|
if (rows.length > 0) {
|
|
@@ -8938,8 +8188,8 @@ async function ga4Routes(app, opts) {
|
|
|
8938
8188
|
tx.delete(gaAiReferrals).where(
|
|
8939
8189
|
and6(
|
|
8940
8190
|
eq17(gaAiReferrals.projectId, project.id),
|
|
8941
|
-
|
|
8942
|
-
|
|
8191
|
+
sql3`${gaAiReferrals.date} >= ${summary.periodStart}`,
|
|
8192
|
+
sql3`${gaAiReferrals.date} <= ${summary.periodEnd}`
|
|
8943
8193
|
)
|
|
8944
8194
|
).run();
|
|
8945
8195
|
if (aiReferrals.length > 0) {
|
|
@@ -8995,22 +8245,22 @@ async function ga4Routes(app, opts) {
|
|
|
8995
8245
|
}).from(gaTrafficSummaries).where(eq17(gaTrafficSummaries.projectId, project.id)).get();
|
|
8996
8246
|
const rows = app.db.select({
|
|
8997
8247
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
8998
|
-
sessions:
|
|
8999
|
-
organicSessions:
|
|
9000
|
-
users:
|
|
9001
|
-
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(
|
|
8248
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
8249
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
8250
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
8251
|
+
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql3`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
9002
8252
|
const aiReferrals = app.db.select({
|
|
9003
8253
|
source: gaAiReferrals.source,
|
|
9004
8254
|
medium: gaAiReferrals.medium,
|
|
9005
8255
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
9006
|
-
sessions:
|
|
9007
|
-
users:
|
|
9008
|
-
}).from(gaAiReferrals).where(eq17(gaAiReferrals.projectId, project.id)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).orderBy(
|
|
8256
|
+
sessions: sql3`SUM(${gaAiReferrals.sessions})`,
|
|
8257
|
+
users: sql3`SUM(${gaAiReferrals.users})`
|
|
8258
|
+
}).from(gaAiReferrals).where(eq17(gaAiReferrals.projectId, project.id)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).orderBy(sql3`SUM(${gaAiReferrals.sessions}) DESC`).all();
|
|
9009
8259
|
const aiDeduped = app.db.select({
|
|
9010
|
-
sessions:
|
|
9011
|
-
users:
|
|
8260
|
+
sessions: sql3`SUM(max_sessions)`,
|
|
8261
|
+
users: sql3`SUM(max_users)`
|
|
9012
8262
|
}).from(
|
|
9013
|
-
|
|
8263
|
+
sql3`(
|
|
9014
8264
|
SELECT date, source, medium,
|
|
9015
8265
|
MAX(sessions) AS max_sessions,
|
|
9016
8266
|
MAX(users) AS max_users
|
|
@@ -9060,9 +8310,9 @@ async function ga4Routes(app, opts) {
|
|
|
9060
8310
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
9061
8311
|
const rows = app.db.select({
|
|
9062
8312
|
date: gaTrafficSnapshots.date,
|
|
9063
|
-
sessions:
|
|
9064
|
-
organicSessions:
|
|
9065
|
-
users:
|
|
8313
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
8314
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
8315
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
9066
8316
|
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
9067
8317
|
return rows.map((r) => ({
|
|
9068
8318
|
date: r.date,
|
|
@@ -9076,10 +8326,10 @@ async function ga4Routes(app, opts) {
|
|
|
9076
8326
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
9077
8327
|
const trafficPages = app.db.select({
|
|
9078
8328
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
9079
|
-
sessions:
|
|
9080
|
-
organicSessions:
|
|
9081
|
-
users:
|
|
9082
|
-
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(
|
|
8329
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
8330
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
8331
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
8332
|
+
}).from(gaTrafficSnapshots).where(eq17(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql3`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
9083
8333
|
return {
|
|
9084
8334
|
pages: trafficPages.map((r) => ({
|
|
9085
8335
|
landingPage: r.landingPage,
|
|
@@ -9306,15 +8556,15 @@ async function fetchJson(connection, siteUrl, path7, init) {
|
|
|
9306
8556
|
signal: AbortSignal.timeout(WP_REQUEST_TIMEOUT_MS)
|
|
9307
8557
|
});
|
|
9308
8558
|
if (res.status === 401 || res.status === 403) {
|
|
9309
|
-
const
|
|
9310
|
-
throw new WordpressApiError("AUTH_INVALID", buildAuthErrorMessage(res,
|
|
8559
|
+
const text = await res.text().catch(() => "");
|
|
8560
|
+
throw new WordpressApiError("AUTH_INVALID", buildAuthErrorMessage(res, text), res.status);
|
|
9311
8561
|
}
|
|
9312
8562
|
if (res.status === 404) {
|
|
9313
8563
|
throw new WordpressApiError("NOT_FOUND", "WordPress endpoint not found", 404);
|
|
9314
8564
|
}
|
|
9315
8565
|
if (!res.ok) {
|
|
9316
|
-
const
|
|
9317
|
-
throw new WordpressApiError("UPSTREAM_ERROR", `WordPress API error (${res.status}): ${
|
|
8566
|
+
const text = await res.text().catch(() => "");
|
|
8567
|
+
throw new WordpressApiError("UPSTREAM_ERROR", `WordPress API error (${res.status}): ${text || res.statusText}`, res.status);
|
|
9318
8568
|
}
|
|
9319
8569
|
return {
|
|
9320
8570
|
body: await res.json(),
|
|
@@ -9415,9 +8665,9 @@ function computeWordCount(content) {
|
|
|
9415
8665
|
return clean.split(/\s+/).filter(Boolean).length;
|
|
9416
8666
|
}
|
|
9417
8667
|
function buildSnippet(content) {
|
|
9418
|
-
const
|
|
9419
|
-
if (
|
|
9420
|
-
return `${
|
|
8668
|
+
const text = stripHtml(content);
|
|
8669
|
+
if (text.length <= 160) return text;
|
|
8670
|
+
return `${text.slice(0, 157)}...`;
|
|
9421
8671
|
}
|
|
9422
8672
|
function contentHash(content) {
|
|
9423
8673
|
return crypto17.createHash("sha256").update(content).digest("hex");
|
|
@@ -9434,9 +8684,9 @@ async function mapWithConcurrency(items, limit, iteratee) {
|
|
|
9434
8684
|
let nextIndex = 0;
|
|
9435
8685
|
async function worker() {
|
|
9436
8686
|
while (nextIndex < items.length) {
|
|
9437
|
-
const
|
|
8687
|
+
const index = nextIndex;
|
|
9438
8688
|
nextIndex += 1;
|
|
9439
|
-
results[
|
|
8689
|
+
results[index] = await iteratee(items[index], index);
|
|
9440
8690
|
}
|
|
9441
8691
|
}
|
|
9442
8692
|
const workerCount = Math.max(1, Math.min(limit, items.length));
|
|
@@ -10913,7 +10163,7 @@ function isVertexConfig(config) {
|
|
|
10913
10163
|
function resolveModel(config) {
|
|
10914
10164
|
return config.model || DEFAULT_MODEL;
|
|
10915
10165
|
}
|
|
10916
|
-
function
|
|
10166
|
+
function createClient(config) {
|
|
10917
10167
|
if (isVertexConfig(config)) {
|
|
10918
10168
|
return new GoogleGenAI({
|
|
10919
10169
|
vertexai: true,
|
|
@@ -10953,19 +10203,19 @@ async function healthcheck(config) {
|
|
|
10953
10203
|
if (!validation.ok) return validation;
|
|
10954
10204
|
try {
|
|
10955
10205
|
const model = resolveModel(config);
|
|
10956
|
-
const client =
|
|
10206
|
+
const client = createClient(config);
|
|
10957
10207
|
const result = await withRetry(
|
|
10958
10208
|
() => client.models.generateContent({
|
|
10959
10209
|
model,
|
|
10960
10210
|
contents: 'Say "ok"'
|
|
10961
10211
|
})
|
|
10962
10212
|
);
|
|
10963
|
-
const
|
|
10213
|
+
const text = result.text ?? "";
|
|
10964
10214
|
const backend = isVertexConfig(config) ? "vertex ai" : "api key";
|
|
10965
10215
|
return {
|
|
10966
|
-
ok:
|
|
10216
|
+
ok: text.length > 0,
|
|
10967
10217
|
provider: "gemini",
|
|
10968
|
-
message:
|
|
10218
|
+
message: text.length > 0 ? `gemini ${backend} verified` : `empty response from gemini ${backend}`,
|
|
10969
10219
|
model
|
|
10970
10220
|
};
|
|
10971
10221
|
} catch (err) {
|
|
@@ -10980,7 +10230,7 @@ async function healthcheck(config) {
|
|
|
10980
10230
|
async function executeTrackedQuery(input) {
|
|
10981
10231
|
const model = resolveModel(input.config);
|
|
10982
10232
|
const prompt = buildPrompt(input.keyword, input.location);
|
|
10983
|
-
const client =
|
|
10233
|
+
const client = createClient(input.config);
|
|
10984
10234
|
try {
|
|
10985
10235
|
const result = await withRetry(
|
|
10986
10236
|
() => client.models.generateContent({
|
|
@@ -10991,14 +10241,14 @@ async function executeTrackedQuery(input) {
|
|
|
10991
10241
|
}
|
|
10992
10242
|
})
|
|
10993
10243
|
);
|
|
10994
|
-
const
|
|
10995
|
-
const
|
|
10244
|
+
const rawResponse = responseToRecord(result);
|
|
10245
|
+
const parsed = reparseStoredResult(rawResponse);
|
|
10996
10246
|
return {
|
|
10997
10247
|
provider: "gemini",
|
|
10998
|
-
rawResponse
|
|
10248
|
+
rawResponse,
|
|
10999
10249
|
model,
|
|
11000
|
-
groundingSources,
|
|
11001
|
-
searchQueries
|
|
10250
|
+
groundingSources: parsed.groundingSources,
|
|
10251
|
+
searchQueries: parsed.searchQueries
|
|
11002
10252
|
};
|
|
11003
10253
|
} catch (err) {
|
|
11004
10254
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -11006,14 +10256,31 @@ async function executeTrackedQuery(input) {
|
|
|
11006
10256
|
}
|
|
11007
10257
|
}
|
|
11008
10258
|
function normalizeResult(raw) {
|
|
11009
|
-
const
|
|
11010
|
-
const
|
|
10259
|
+
const parsed = reparseStoredResult(raw.rawResponse);
|
|
10260
|
+
const useParsed = hasParsedResponseContent(raw.rawResponse);
|
|
10261
|
+
const groundingSources = useParsed ? parsed.groundingSources : raw.groundingSources;
|
|
10262
|
+
const searchQueries = useParsed ? parsed.searchQueries : raw.searchQueries;
|
|
10263
|
+
const citedDomains = extractCitedDomainsFromSources(groundingSources);
|
|
11011
10264
|
return {
|
|
11012
10265
|
provider: "gemini",
|
|
11013
|
-
answerText,
|
|
10266
|
+
answerText: parsed.answerText,
|
|
11014
10267
|
citedDomains,
|
|
11015
|
-
groundingSources
|
|
11016
|
-
searchQueries
|
|
10268
|
+
groundingSources,
|
|
10269
|
+
searchQueries
|
|
10270
|
+
};
|
|
10271
|
+
}
|
|
10272
|
+
function hasParsedResponseContent(rawResponse) {
|
|
10273
|
+
return Array.isArray(rawResponse.candidates) && rawResponse.candidates.length > 0;
|
|
10274
|
+
}
|
|
10275
|
+
function reparseStoredResult(rawResponse) {
|
|
10276
|
+
const groundingSources = extractGroundingMetadataFromRaw(rawResponse);
|
|
10277
|
+
const searchQueries = extractSearchQueriesFromRaw(rawResponse);
|
|
10278
|
+
return {
|
|
10279
|
+
provider: "gemini",
|
|
10280
|
+
answerText: extractAnswerText(rawResponse),
|
|
10281
|
+
citedDomains: extractCitedDomainsFromSources(groundingSources),
|
|
10282
|
+
groundingSources,
|
|
10283
|
+
searchQueries
|
|
11017
10284
|
};
|
|
11018
10285
|
}
|
|
11019
10286
|
function buildPrompt(keyword, location) {
|
|
@@ -11033,33 +10300,51 @@ function extractAnswerText(rawResponse) {
|
|
|
11033
10300
|
return "";
|
|
11034
10301
|
}
|
|
11035
10302
|
}
|
|
11036
|
-
function
|
|
10303
|
+
function extractGroundingMetadataFromRaw(rawResponse) {
|
|
11037
10304
|
try {
|
|
11038
|
-
const
|
|
10305
|
+
const candidates = rawResponse.candidates;
|
|
10306
|
+
const candidate = candidates?.[0];
|
|
11039
10307
|
if (!candidate) return [];
|
|
11040
10308
|
const metadata = candidate.groundingMetadata;
|
|
11041
10309
|
if (!metadata) return [];
|
|
11042
10310
|
const chunks = metadata.groundingChunks;
|
|
11043
10311
|
if (!chunks) return [];
|
|
11044
|
-
|
|
11045
|
-
|
|
11046
|
-
|
|
11047
|
-
|
|
10312
|
+
const indices = /* @__PURE__ */ new Set();
|
|
10313
|
+
for (const support of metadata.groundingSupports ?? []) {
|
|
10314
|
+
for (const index of support.groundingChunkIndices ?? []) {
|
|
10315
|
+
if (Number.isInteger(index) && index >= 0 && index < chunks.length) {
|
|
10316
|
+
indices.add(index);
|
|
10317
|
+
}
|
|
10318
|
+
}
|
|
10319
|
+
}
|
|
10320
|
+
const selectedChunks = indices.size > 0 ? [...indices].map((index) => chunks[index]).filter(Boolean) : chunks;
|
|
10321
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10322
|
+
const sources = [];
|
|
10323
|
+
for (const chunk of selectedChunks) {
|
|
10324
|
+
if (!chunk.web?.uri || seen.has(chunk.web.uri)) continue;
|
|
10325
|
+
seen.add(chunk.web.uri);
|
|
10326
|
+
sources.push({
|
|
10327
|
+
uri: chunk.web.uri,
|
|
10328
|
+
title: chunk.web.title ?? ""
|
|
10329
|
+
});
|
|
10330
|
+
}
|
|
10331
|
+
return sources;
|
|
11048
10332
|
} catch {
|
|
11049
10333
|
return [];
|
|
11050
10334
|
}
|
|
11051
10335
|
}
|
|
11052
|
-
function
|
|
10336
|
+
function extractSearchQueriesFromRaw(rawResponse) {
|
|
11053
10337
|
try {
|
|
11054
|
-
const
|
|
10338
|
+
const candidates = rawResponse.candidates;
|
|
10339
|
+
const candidate = candidates?.[0];
|
|
11055
10340
|
return candidate?.groundingMetadata?.webSearchQueries ?? [];
|
|
11056
10341
|
} catch {
|
|
11057
10342
|
return [];
|
|
11058
10343
|
}
|
|
11059
10344
|
}
|
|
11060
|
-
function
|
|
10345
|
+
function extractCitedDomainsFromSources(groundingSources) {
|
|
11061
10346
|
const domains = /* @__PURE__ */ new Set();
|
|
11062
|
-
for (const source of
|
|
10347
|
+
for (const source of groundingSources) {
|
|
11063
10348
|
const domain = extractDomainFromUri(source.uri);
|
|
11064
10349
|
if (domain) {
|
|
11065
10350
|
domains.add(domain);
|
|
@@ -11104,7 +10389,7 @@ function extractDomainFromUri(uri) {
|
|
|
11104
10389
|
}
|
|
11105
10390
|
async function generateText(prompt, config) {
|
|
11106
10391
|
const model = resolveModel(config);
|
|
11107
|
-
const client =
|
|
10392
|
+
const client = createClient(config);
|
|
11108
10393
|
const result = await withRetry(
|
|
11109
10394
|
() => client.models.generateContent({
|
|
11110
10395
|
model,
|
|
@@ -11120,7 +10405,11 @@ function responseToRecord(response) {
|
|
|
11120
10405
|
finishReason: c.finishReason,
|
|
11121
10406
|
groundingMetadata: c.groundingMetadata ? {
|
|
11122
10407
|
webSearchQueries: c.groundingMetadata.webSearchQueries,
|
|
11123
|
-
groundingChunks: c.groundingMetadata.groundingChunks
|
|
10408
|
+
groundingChunks: c.groundingMetadata.groundingChunks,
|
|
10409
|
+
// Preserve support-to-chunk mappings so stored snapshots can be reparsed using
|
|
10410
|
+
// Google's documented citation model.
|
|
10411
|
+
// Docs: https://ai.google.dev/gemini-api/docs/google-search
|
|
10412
|
+
groundingSupports: c.groundingMetadata.groundingSupports
|
|
11124
10413
|
} : void 0
|
|
11125
10414
|
}));
|
|
11126
10415
|
return {
|
|
@@ -11263,11 +10552,11 @@ async function healthcheck2(config) {
|
|
|
11263
10552
|
input: 'Say "ok"'
|
|
11264
10553
|
})
|
|
11265
10554
|
);
|
|
11266
|
-
const
|
|
10555
|
+
const text = extractResponseText(response);
|
|
11267
10556
|
return {
|
|
11268
|
-
ok:
|
|
10557
|
+
ok: text.length > 0,
|
|
11269
10558
|
provider: "openai",
|
|
11270
|
-
message:
|
|
10559
|
+
message: text.length > 0 ? "openai api key verified" : "empty response from openai",
|
|
11271
10560
|
model: config.model ?? DEFAULT_MODEL2
|
|
11272
10561
|
};
|
|
11273
10562
|
} catch (err) {
|
|
@@ -11301,14 +10590,14 @@ async function executeTrackedQuery2(input) {
|
|
|
11301
10590
|
input: buildPrompt2(input.keyword)
|
|
11302
10591
|
})
|
|
11303
10592
|
);
|
|
11304
|
-
const
|
|
11305
|
-
const
|
|
10593
|
+
const rawResponse = responseToRecord2(response);
|
|
10594
|
+
const parsed = reparseStoredResult2(rawResponse);
|
|
11306
10595
|
return {
|
|
11307
10596
|
provider: "openai",
|
|
11308
|
-
rawResponse
|
|
10597
|
+
rawResponse,
|
|
11309
10598
|
model,
|
|
11310
|
-
groundingSources,
|
|
11311
|
-
searchQueries
|
|
10599
|
+
groundingSources: parsed.groundingSources,
|
|
10600
|
+
searchQueries: parsed.searchQueries
|
|
11312
10601
|
};
|
|
11313
10602
|
} catch (err) {
|
|
11314
10603
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -11316,14 +10605,31 @@ async function executeTrackedQuery2(input) {
|
|
|
11316
10605
|
}
|
|
11317
10606
|
}
|
|
11318
10607
|
function normalizeResult2(raw) {
|
|
11319
|
-
const
|
|
11320
|
-
const
|
|
10608
|
+
const parsed = reparseStoredResult2(raw.rawResponse);
|
|
10609
|
+
const useParsed = hasParsedResponseContent2(raw.rawResponse);
|
|
10610
|
+
const groundingSources = useParsed ? parsed.groundingSources : raw.groundingSources;
|
|
10611
|
+
const searchQueries = useParsed ? parsed.searchQueries : raw.searchQueries;
|
|
10612
|
+
const citedDomains = extractCitedDomainsFromSources2(groundingSources);
|
|
11321
10613
|
return {
|
|
11322
10614
|
provider: "openai",
|
|
11323
|
-
answerText,
|
|
10615
|
+
answerText: parsed.answerText,
|
|
11324
10616
|
citedDomains,
|
|
11325
|
-
groundingSources
|
|
11326
|
-
searchQueries
|
|
10617
|
+
groundingSources,
|
|
10618
|
+
searchQueries
|
|
10619
|
+
};
|
|
10620
|
+
}
|
|
10621
|
+
function hasParsedResponseContent2(rawResponse) {
|
|
10622
|
+
return Array.isArray(rawResponse.output) && rawResponse.output.length > 0;
|
|
10623
|
+
}
|
|
10624
|
+
function reparseStoredResult2(rawResponse) {
|
|
10625
|
+
const groundingSources = extractGroundingSourcesFromRaw(rawResponse);
|
|
10626
|
+
const searchQueries = extractSearchQueriesFromRaw2(rawResponse);
|
|
10627
|
+
return {
|
|
10628
|
+
provider: "openai",
|
|
10629
|
+
answerText: extractAnswerTextFromRaw(rawResponse),
|
|
10630
|
+
citedDomains: extractCitedDomainsFromSources2(groundingSources),
|
|
10631
|
+
groundingSources,
|
|
10632
|
+
searchQueries
|
|
11327
10633
|
};
|
|
11328
10634
|
}
|
|
11329
10635
|
function buildPrompt2(keyword) {
|
|
@@ -11346,16 +10652,18 @@ function extractResponseText(response) {
|
|
|
11346
10652
|
return "";
|
|
11347
10653
|
}
|
|
11348
10654
|
}
|
|
11349
|
-
function
|
|
10655
|
+
function extractGroundingSourcesFromRaw(rawResponse) {
|
|
11350
10656
|
const sources = [];
|
|
11351
10657
|
const seen = /* @__PURE__ */ new Set();
|
|
11352
10658
|
try {
|
|
11353
|
-
|
|
10659
|
+
const output = rawResponse.output;
|
|
10660
|
+
if (!output) return [];
|
|
10661
|
+
for (const item of output) {
|
|
11354
10662
|
if (item.type === "message") {
|
|
11355
|
-
for (const content of item.content) {
|
|
10663
|
+
for (const content of item.content ?? []) {
|
|
11356
10664
|
if (content.type === "output_text" && content.annotations) {
|
|
11357
10665
|
for (const annotation of content.annotations) {
|
|
11358
|
-
if (annotation.type === "url_citation" && !seen.has(annotation.url)) {
|
|
10666
|
+
if (annotation.type === "url_citation" && typeof annotation.url === "string" && !seen.has(annotation.url)) {
|
|
11359
10667
|
seen.add(annotation.url);
|
|
11360
10668
|
sources.push({
|
|
11361
10669
|
uri: annotation.url,
|
|
@@ -11371,20 +10679,28 @@ function extractGroundingSources(response) {
|
|
|
11371
10679
|
}
|
|
11372
10680
|
return sources;
|
|
11373
10681
|
}
|
|
11374
|
-
function
|
|
11375
|
-
const queries =
|
|
10682
|
+
function extractSearchQueriesFromRaw2(rawResponse) {
|
|
10683
|
+
const queries = /* @__PURE__ */ new Set();
|
|
11376
10684
|
try {
|
|
11377
|
-
|
|
11378
|
-
|
|
11379
|
-
|
|
11380
|
-
|
|
11381
|
-
|
|
10685
|
+
const output = rawResponse.output;
|
|
10686
|
+
if (!output) return [];
|
|
10687
|
+
for (const item of output) {
|
|
10688
|
+
if (item.type !== "web_search_call" || !item.action) continue;
|
|
10689
|
+
const action = item.action;
|
|
10690
|
+
if (typeof action.query === "string" && action.query.length > 0) {
|
|
10691
|
+
queries.add(action.query);
|
|
10692
|
+
}
|
|
10693
|
+
if (Array.isArray(action.queries)) {
|
|
10694
|
+
for (const query of action.queries) {
|
|
10695
|
+
if (typeof query === "string" && query.length > 0) {
|
|
10696
|
+
queries.add(query);
|
|
10697
|
+
}
|
|
11382
10698
|
}
|
|
11383
10699
|
}
|
|
11384
10700
|
}
|
|
11385
10701
|
} catch {
|
|
11386
10702
|
}
|
|
11387
|
-
return queries;
|
|
10703
|
+
return [...queries];
|
|
11388
10704
|
}
|
|
11389
10705
|
function extractAnswerTextFromRaw(rawResponse) {
|
|
11390
10706
|
try {
|
|
@@ -11405,9 +10721,9 @@ function extractAnswerTextFromRaw(rawResponse) {
|
|
|
11405
10721
|
return "";
|
|
11406
10722
|
}
|
|
11407
10723
|
}
|
|
11408
|
-
function
|
|
10724
|
+
function extractCitedDomainsFromSources2(groundingSources) {
|
|
11409
10725
|
const domains = /* @__PURE__ */ new Set();
|
|
11410
|
-
for (const source of
|
|
10726
|
+
for (const source of groundingSources) {
|
|
11411
10727
|
const domain = extractDomainFromUri2(source.uri);
|
|
11412
10728
|
if (domain) domains.add(domain);
|
|
11413
10729
|
}
|
|
@@ -11583,11 +10899,11 @@ async function healthcheck3(config) {
|
|
|
11583
10899
|
messages: [{ role: "user", content: 'Say "ok"' }]
|
|
11584
10900
|
})
|
|
11585
10901
|
);
|
|
11586
|
-
const
|
|
10902
|
+
const text = extractTextFromResponse(response);
|
|
11587
10903
|
return {
|
|
11588
|
-
ok:
|
|
10904
|
+
ok: text.length > 0,
|
|
11589
10905
|
provider: "claude",
|
|
11590
|
-
message:
|
|
10906
|
+
message: text.length > 0 ? "claude api key verified" : "empty response from claude",
|
|
11591
10907
|
model
|
|
11592
10908
|
};
|
|
11593
10909
|
} catch (err) {
|
|
@@ -11625,14 +10941,17 @@ async function executeTrackedQuery3(input) {
|
|
|
11625
10941
|
messages: [{ role: "user", content: input.keyword }]
|
|
11626
10942
|
})
|
|
11627
10943
|
);
|
|
11628
|
-
const
|
|
11629
|
-
const
|
|
10944
|
+
const rawResponse = responseToRecord3(response);
|
|
10945
|
+
const parsed = reparseStoredResult3(rawResponse);
|
|
10946
|
+
if (parsed.providerError) {
|
|
10947
|
+
throw new Error(parsed.providerError);
|
|
10948
|
+
}
|
|
11630
10949
|
return {
|
|
11631
10950
|
provider: "claude",
|
|
11632
|
-
rawResponse
|
|
10951
|
+
rawResponse,
|
|
11633
10952
|
model,
|
|
11634
|
-
groundingSources,
|
|
11635
|
-
searchQueries
|
|
10953
|
+
groundingSources: parsed.groundingSources,
|
|
10954
|
+
searchQueries: parsed.searchQueries
|
|
11636
10955
|
};
|
|
11637
10956
|
} catch (err) {
|
|
11638
10957
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -11640,14 +10959,33 @@ async function executeTrackedQuery3(input) {
|
|
|
11640
10959
|
}
|
|
11641
10960
|
}
|
|
11642
10961
|
function normalizeResult3(raw) {
|
|
11643
|
-
const
|
|
11644
|
-
const
|
|
10962
|
+
const parsed = reparseStoredResult3(raw.rawResponse);
|
|
10963
|
+
const useParsed = hasParsedResponseContent3(raw.rawResponse);
|
|
10964
|
+
const groundingSources = useParsed ? parsed.groundingSources : raw.groundingSources;
|
|
10965
|
+
const searchQueries = useParsed ? parsed.searchQueries : raw.searchQueries;
|
|
10966
|
+
const citedDomains = extractCitedDomainsFromSources3(groundingSources);
|
|
11645
10967
|
return {
|
|
11646
10968
|
provider: "claude",
|
|
11647
|
-
answerText,
|
|
10969
|
+
answerText: parsed.answerText,
|
|
11648
10970
|
citedDomains,
|
|
11649
|
-
groundingSources
|
|
11650
|
-
searchQueries
|
|
10971
|
+
groundingSources,
|
|
10972
|
+
searchQueries
|
|
10973
|
+
};
|
|
10974
|
+
}
|
|
10975
|
+
function hasParsedResponseContent3(rawResponse) {
|
|
10976
|
+
return Array.isArray(rawResponse.content) && rawResponse.content.length > 0;
|
|
10977
|
+
}
|
|
10978
|
+
function reparseStoredResult3(rawResponse) {
|
|
10979
|
+
const groundingSources = extractGroundingSourcesFromRaw2(rawResponse);
|
|
10980
|
+
const searchQueries = extractSearchQueriesFromRaw3(rawResponse);
|
|
10981
|
+
const providerErrors = extractWebSearchToolErrors(rawResponse);
|
|
10982
|
+
return {
|
|
10983
|
+
provider: "claude",
|
|
10984
|
+
answerText: extractAnswerTextFromRaw2(rawResponse),
|
|
10985
|
+
citedDomains: extractCitedDomainsFromSources3(groundingSources),
|
|
10986
|
+
groundingSources,
|
|
10987
|
+
searchQueries,
|
|
10988
|
+
...providerErrors.length > 0 ? { providerError: `web_search tool error: ${providerErrors.join(", ")}` } : {}
|
|
11651
10989
|
};
|
|
11652
10990
|
}
|
|
11653
10991
|
function extractTextFromResponse(response) {
|
|
@@ -11663,16 +11001,20 @@ function extractTextFromResponse(response) {
|
|
|
11663
11001
|
return "";
|
|
11664
11002
|
}
|
|
11665
11003
|
}
|
|
11666
|
-
function
|
|
11004
|
+
function extractGroundingSourcesFromRaw2(rawResponse) {
|
|
11667
11005
|
const sources = [];
|
|
11006
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11668
11007
|
try {
|
|
11669
|
-
|
|
11670
|
-
|
|
11671
|
-
|
|
11672
|
-
|
|
11008
|
+
const content = rawResponse.content;
|
|
11009
|
+
if (!content) return [];
|
|
11010
|
+
for (const block of content) {
|
|
11011
|
+
if (block.type === "text" && Array.isArray(block.citations)) {
|
|
11012
|
+
for (const citation of block.citations) {
|
|
11013
|
+
if (citation.type === "web_search_result_location" && typeof citation.url === "string" && !seen.has(citation.url)) {
|
|
11014
|
+
seen.add(citation.url);
|
|
11673
11015
|
sources.push({
|
|
11674
|
-
uri:
|
|
11675
|
-
title:
|
|
11016
|
+
uri: citation.url,
|
|
11017
|
+
title: citation.title ?? ""
|
|
11676
11018
|
});
|
|
11677
11019
|
}
|
|
11678
11020
|
}
|
|
@@ -11682,20 +11024,28 @@ function extractGroundingSources2(response) {
|
|
|
11682
11024
|
}
|
|
11683
11025
|
return sources;
|
|
11684
11026
|
}
|
|
11685
|
-
function
|
|
11686
|
-
const queries =
|
|
11027
|
+
function extractSearchQueriesFromRaw3(rawResponse) {
|
|
11028
|
+
const queries = /* @__PURE__ */ new Set();
|
|
11687
11029
|
try {
|
|
11688
|
-
|
|
11030
|
+
const content = rawResponse.content;
|
|
11031
|
+
if (!content) return [];
|
|
11032
|
+
for (const block of content) {
|
|
11689
11033
|
if (block.type === "server_tool_use" && block.name === "web_search") {
|
|
11690
|
-
|
|
11691
|
-
|
|
11692
|
-
|
|
11034
|
+
if (typeof block.input?.query === "string" && block.input.query.length > 0) {
|
|
11035
|
+
queries.add(block.input.query);
|
|
11036
|
+
}
|
|
11037
|
+
if (Array.isArray(block.input?.queries)) {
|
|
11038
|
+
for (const query of block.input.queries) {
|
|
11039
|
+
if (typeof query === "string" && query.length > 0) {
|
|
11040
|
+
queries.add(query);
|
|
11041
|
+
}
|
|
11042
|
+
}
|
|
11693
11043
|
}
|
|
11694
11044
|
}
|
|
11695
11045
|
}
|
|
11696
11046
|
} catch {
|
|
11697
11047
|
}
|
|
11698
|
-
return queries;
|
|
11048
|
+
return [...queries];
|
|
11699
11049
|
}
|
|
11700
11050
|
function extractAnswerTextFromRaw2(rawResponse) {
|
|
11701
11051
|
try {
|
|
@@ -11712,9 +11062,26 @@ function extractAnswerTextFromRaw2(rawResponse) {
|
|
|
11712
11062
|
return "";
|
|
11713
11063
|
}
|
|
11714
11064
|
}
|
|
11715
|
-
function
|
|
11065
|
+
function extractWebSearchToolErrors(rawResponse) {
|
|
11066
|
+
const errors = /* @__PURE__ */ new Set();
|
|
11067
|
+
try {
|
|
11068
|
+
const content = rawResponse.content;
|
|
11069
|
+
if (!content) return [];
|
|
11070
|
+
for (const block of content) {
|
|
11071
|
+
if (block.type !== "web_search_tool_result") continue;
|
|
11072
|
+
if (block.content === null || typeof block.content !== "object" || Array.isArray(block.content)) continue;
|
|
11073
|
+
const errorCode = block.content.error_code;
|
|
11074
|
+
if (typeof errorCode === "string" && errorCode.length > 0) {
|
|
11075
|
+
errors.add(errorCode);
|
|
11076
|
+
}
|
|
11077
|
+
}
|
|
11078
|
+
} catch {
|
|
11079
|
+
}
|
|
11080
|
+
return [...errors];
|
|
11081
|
+
}
|
|
11082
|
+
function extractCitedDomainsFromSources3(groundingSources) {
|
|
11716
11083
|
const domains = /* @__PURE__ */ new Set();
|
|
11717
|
-
for (const source of
|
|
11084
|
+
for (const source of groundingSources) {
|
|
11718
11085
|
const domain = extractDomainFromUri3(source.uri);
|
|
11719
11086
|
if (domain) domains.add(domain);
|
|
11720
11087
|
}
|
|
@@ -11967,15 +11334,15 @@ async function generateText4(prompt, config) {
|
|
|
11967
11334
|
);
|
|
11968
11335
|
return response.choices[0]?.message?.content ?? "";
|
|
11969
11336
|
}
|
|
11970
|
-
function extractDomainMentions(
|
|
11337
|
+
function extractDomainMentions(text) {
|
|
11971
11338
|
const domains = /* @__PURE__ */ new Set();
|
|
11972
11339
|
const urlPattern = /https?:\/\/([a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z0-9][-a-zA-Z0-9]*)+)/g;
|
|
11973
11340
|
let match;
|
|
11974
|
-
while ((match = urlPattern.exec(
|
|
11341
|
+
while ((match = urlPattern.exec(text)) !== null) {
|
|
11975
11342
|
domains.add(match[1].replace(/^www\./, "").toLowerCase());
|
|
11976
11343
|
}
|
|
11977
11344
|
const domainPattern = /(?:^|[\s(["'])((?:[a-zA-Z0-9][-a-zA-Z0-9]*\.)+(?:com|org|net|io|co|dev|ai|app|edu|gov|biz|info|tech|health|dental|legal|law|med|uk|us|ca|au|de|fr|es|it|nl|se|no|dk|fi|jp|cn|kr|br|mx|ru|in|sg|nz|za)(?:\.[a-zA-Z]{2})?)(?:[\s).,;/"']|$)/g;
|
|
11978
|
-
while ((match = domainPattern.exec(
|
|
11345
|
+
while ((match = domainPattern.exec(text)) !== null) {
|
|
11979
11346
|
domains.add(match[1].replace(/^www\./, "").toLowerCase());
|
|
11980
11347
|
}
|
|
11981
11348
|
return [...domains];
|
|
@@ -12341,11 +11708,11 @@ var chatgptTarget = {
|
|
|
12341
11708
|
})()`,
|
|
12342
11709
|
returnByValue: true
|
|
12343
11710
|
});
|
|
12344
|
-
const
|
|
12345
|
-
if (!
|
|
11711
|
+
const text = String(result.value ?? "");
|
|
11712
|
+
if (!text) {
|
|
12346
11713
|
throw new CDPProviderError("CDP_TARGET_SELECTOR_FAILED", "Could not extract ChatGPT answer text");
|
|
12347
11714
|
}
|
|
12348
|
-
return
|
|
11715
|
+
return text;
|
|
12349
11716
|
},
|
|
12350
11717
|
async extractCitations(client) {
|
|
12351
11718
|
const { result } = await client.Runtime.evaluate({
|
|
@@ -12468,7 +11835,7 @@ async function captureElementScreenshot(client, selector, outputPath) {
|
|
|
12468
11835
|
}
|
|
12469
11836
|
|
|
12470
11837
|
// ../provider-cdp/src/normalize.ts
|
|
12471
|
-
function
|
|
11838
|
+
function extractCitedDomains(groundingSources) {
|
|
12472
11839
|
const domains = /* @__PURE__ */ new Set();
|
|
12473
11840
|
for (const source of groundingSources) {
|
|
12474
11841
|
try {
|
|
@@ -12497,7 +11864,7 @@ function normalizeResult5(raw) {
|
|
|
12497
11864
|
return {
|
|
12498
11865
|
provider: raw.provider,
|
|
12499
11866
|
answerText,
|
|
12500
|
-
citedDomains:
|
|
11867
|
+
citedDomains: extractCitedDomains(raw.groundingSources),
|
|
12501
11868
|
groundingSources: raw.groundingSources,
|
|
12502
11869
|
searchQueries: raw.searchQueries
|
|
12503
11870
|
};
|
|
@@ -12670,11 +12037,11 @@ async function healthcheck5(config) {
|
|
|
12670
12037
|
messages: [{ role: "user", content: 'Say "ok"' }]
|
|
12671
12038
|
})
|
|
12672
12039
|
);
|
|
12673
|
-
const
|
|
12040
|
+
const text = response.choices[0]?.message?.content ?? "";
|
|
12674
12041
|
return {
|
|
12675
|
-
ok:
|
|
12042
|
+
ok: text.length > 0,
|
|
12676
12043
|
provider: "perplexity",
|
|
12677
|
-
message:
|
|
12044
|
+
message: text.length > 0 ? "perplexity api key verified" : "empty response from perplexity",
|
|
12678
12045
|
model: config.model ?? DEFAULT_MODEL5
|
|
12679
12046
|
};
|
|
12680
12047
|
} catch (err) {
|
|
@@ -12698,17 +12065,13 @@ async function executeTrackedQuery5(input) {
|
|
|
12698
12065
|
})
|
|
12699
12066
|
);
|
|
12700
12067
|
const rawResponse = responseToRecord5(response);
|
|
12701
|
-
const
|
|
12702
|
-
const groundingSources = citations.map((url) => ({
|
|
12703
|
-
uri: url,
|
|
12704
|
-
title: ""
|
|
12705
|
-
}));
|
|
12068
|
+
const parsed = reparseStoredResult4(rawResponse);
|
|
12706
12069
|
return {
|
|
12707
12070
|
provider: "perplexity",
|
|
12708
12071
|
rawResponse,
|
|
12709
12072
|
model,
|
|
12710
|
-
groundingSources,
|
|
12711
|
-
searchQueries:
|
|
12073
|
+
groundingSources: parsed.groundingSources,
|
|
12074
|
+
searchQueries: parsed.searchQueries
|
|
12712
12075
|
};
|
|
12713
12076
|
} catch (err) {
|
|
12714
12077
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -12716,14 +12079,38 @@ async function executeTrackedQuery5(input) {
|
|
|
12716
12079
|
}
|
|
12717
12080
|
}
|
|
12718
12081
|
function normalizeResult6(raw) {
|
|
12719
|
-
const
|
|
12720
|
-
const
|
|
12082
|
+
const parsed = reparseStoredResult4(raw.rawResponse);
|
|
12083
|
+
const useParsed = hasParsedResponseContent4(raw.rawResponse);
|
|
12084
|
+
const groundingSources = useParsed ? parsed.groundingSources : raw.groundingSources;
|
|
12085
|
+
const searchQueries = useParsed ? parsed.searchQueries : raw.searchQueries;
|
|
12086
|
+
const citedDomains = extractCitedDomains2(groundingSources);
|
|
12721
12087
|
return {
|
|
12722
12088
|
provider: "perplexity",
|
|
12723
|
-
answerText,
|
|
12089
|
+
answerText: parsed.answerText,
|
|
12724
12090
|
citedDomains,
|
|
12725
|
-
groundingSources
|
|
12726
|
-
searchQueries
|
|
12091
|
+
groundingSources,
|
|
12092
|
+
searchQueries
|
|
12093
|
+
};
|
|
12094
|
+
}
|
|
12095
|
+
function hasParsedResponseContent4(rawResponse) {
|
|
12096
|
+
if (Array.isArray(rawResponse.choices) && rawResponse.choices.length > 0) return true;
|
|
12097
|
+
if (Array.isArray(rawResponse.search_results) && rawResponse.search_results.length > 0) return true;
|
|
12098
|
+
if (Array.isArray(rawResponse.citations) && rawResponse.citations.length > 0) return true;
|
|
12099
|
+
const nestedResponse = extractNestedApiResponse(rawResponse);
|
|
12100
|
+
if (!nestedResponse) return false;
|
|
12101
|
+
return Array.isArray(nestedResponse.choices) && nestedResponse.choices.length > 0 || Array.isArray(nestedResponse.search_results) && nestedResponse.search_results.length > 0 || Array.isArray(nestedResponse.citations) && nestedResponse.citations.length > 0;
|
|
12102
|
+
}
|
|
12103
|
+
function reparseStoredResult4(rawResponse) {
|
|
12104
|
+
const groundingSources = extractGroundingSources(rawResponse);
|
|
12105
|
+
return {
|
|
12106
|
+
provider: "perplexity",
|
|
12107
|
+
answerText: extractAnswerText3(rawResponse),
|
|
12108
|
+
citedDomains: extractCitedDomains2(groundingSources),
|
|
12109
|
+
groundingSources,
|
|
12110
|
+
// Perplexity documents `search_results` and `citations` on the response structure but
|
|
12111
|
+
// does not document returned search-query telemetry, so Canonry does not synthesize it.
|
|
12112
|
+
// Docs: https://docs.perplexity.ai/docs/sonar/openai-compatibility
|
|
12113
|
+
searchQueries: []
|
|
12727
12114
|
};
|
|
12728
12115
|
}
|
|
12729
12116
|
function buildPrompt4(keyword, location) {
|
|
@@ -12736,25 +12123,80 @@ function extractCitations(rawResponse) {
|
|
|
12736
12123
|
if (Array.isArray(rawResponse.citations)) {
|
|
12737
12124
|
return rawResponse.citations.filter((c) => typeof c === "string");
|
|
12738
12125
|
}
|
|
12739
|
-
const
|
|
12740
|
-
if (
|
|
12741
|
-
const nested =
|
|
12126
|
+
const nestedResponse = extractNestedApiResponse(rawResponse);
|
|
12127
|
+
if (nestedResponse) {
|
|
12128
|
+
const nested = nestedResponse.citations;
|
|
12742
12129
|
if (Array.isArray(nested)) {
|
|
12743
12130
|
return nested.filter((c) => typeof c === "string");
|
|
12744
12131
|
}
|
|
12745
12132
|
}
|
|
12746
12133
|
return [];
|
|
12747
12134
|
}
|
|
12135
|
+
function extractGroundingSources(rawResponse) {
|
|
12136
|
+
const searchResults = extractSearchResults(rawResponse);
|
|
12137
|
+
if (searchResults.length > 0) {
|
|
12138
|
+
const seen = /* @__PURE__ */ new Set();
|
|
12139
|
+
const sources = [];
|
|
12140
|
+
for (const result of searchResults) {
|
|
12141
|
+
if (seen.has(result.uri)) continue;
|
|
12142
|
+
seen.add(result.uri);
|
|
12143
|
+
sources.push(result);
|
|
12144
|
+
}
|
|
12145
|
+
return sources;
|
|
12146
|
+
}
|
|
12147
|
+
return extractCitations(rawResponse).map((url) => ({
|
|
12148
|
+
uri: url,
|
|
12149
|
+
title: ""
|
|
12150
|
+
}));
|
|
12151
|
+
}
|
|
12152
|
+
function extractSearchResults(rawResponse) {
|
|
12153
|
+
const direct = parseSearchResultsArray(rawResponse.search_results);
|
|
12154
|
+
if (direct.length > 0) return direct;
|
|
12155
|
+
const nestedResponse = extractNestedApiResponse(rawResponse);
|
|
12156
|
+
if (nestedResponse) {
|
|
12157
|
+
return parseSearchResultsArray(nestedResponse.search_results);
|
|
12158
|
+
}
|
|
12159
|
+
return [];
|
|
12160
|
+
}
|
|
12161
|
+
function parseSearchResultsArray(value) {
|
|
12162
|
+
if (!Array.isArray(value)) return [];
|
|
12163
|
+
return value.flatMap((result) => {
|
|
12164
|
+
if (result === null || typeof result !== "object" || Array.isArray(result)) {
|
|
12165
|
+
return [];
|
|
12166
|
+
}
|
|
12167
|
+
const url = result.url;
|
|
12168
|
+
if (typeof url !== "string" || url.length === 0) {
|
|
12169
|
+
return [];
|
|
12170
|
+
}
|
|
12171
|
+
const title = result.title;
|
|
12172
|
+
return [{
|
|
12173
|
+
uri: url,
|
|
12174
|
+
title: typeof title === "string" ? title : ""
|
|
12175
|
+
}];
|
|
12176
|
+
});
|
|
12177
|
+
}
|
|
12748
12178
|
function extractAnswerText3(rawResponse) {
|
|
12749
12179
|
try {
|
|
12750
|
-
const
|
|
12751
|
-
if (
|
|
12752
|
-
|
|
12180
|
+
const directChoices = rawResponse.choices;
|
|
12181
|
+
if (directChoices?.length) {
|
|
12182
|
+
return directChoices[0].message?.content ?? "";
|
|
12183
|
+
}
|
|
12184
|
+
const nestedResponse = extractNestedApiResponse(rawResponse);
|
|
12185
|
+
const nestedChoices = nestedResponse?.choices;
|
|
12186
|
+
if (!nestedChoices?.length) return "";
|
|
12187
|
+
return nestedChoices[0].message?.content ?? "";
|
|
12753
12188
|
} catch {
|
|
12754
12189
|
return "";
|
|
12755
12190
|
}
|
|
12756
12191
|
}
|
|
12757
|
-
function
|
|
12192
|
+
function extractNestedApiResponse(rawResponse) {
|
|
12193
|
+
const apiResponse = rawResponse.apiResponse;
|
|
12194
|
+
if (apiResponse !== null && typeof apiResponse === "object" && !Array.isArray(apiResponse)) {
|
|
12195
|
+
return apiResponse;
|
|
12196
|
+
}
|
|
12197
|
+
return null;
|
|
12198
|
+
}
|
|
12199
|
+
function extractCitedDomains2(groundingSources) {
|
|
12758
12200
|
const domains = /* @__PURE__ */ new Set();
|
|
12759
12201
|
for (const source of groundingSources) {
|
|
12760
12202
|
const domain = extractDomainFromUri4(source.uri);
|
|
@@ -12901,7 +12343,7 @@ function getGoogleConnection(config, domain, connectionType) {
|
|
|
12901
12343
|
}
|
|
12902
12344
|
function upsertGoogleConnection(config, connection) {
|
|
12903
12345
|
const connections = ensureConnections(config);
|
|
12904
|
-
const
|
|
12346
|
+
const index = connections.findIndex((entry) => entry.domain === connection.domain && entry.connectionType === connection.connectionType);
|
|
12905
12347
|
const normalized = {
|
|
12906
12348
|
...connection,
|
|
12907
12349
|
propertyId: connection.propertyId ?? null,
|
|
@@ -12910,11 +12352,11 @@ function upsertGoogleConnection(config, connection) {
|
|
|
12910
12352
|
tokenExpiresAt: connection.tokenExpiresAt ?? null,
|
|
12911
12353
|
scopes: connection.scopes ?? []
|
|
12912
12354
|
};
|
|
12913
|
-
if (
|
|
12355
|
+
if (index === -1) {
|
|
12914
12356
|
connections.push(normalized);
|
|
12915
12357
|
return normalized;
|
|
12916
12358
|
}
|
|
12917
|
-
connections[
|
|
12359
|
+
connections[index] = normalized;
|
|
12918
12360
|
return normalized;
|
|
12919
12361
|
}
|
|
12920
12362
|
function patchGoogleConnection(config, domain, connectionType, patch) {
|
|
@@ -12954,12 +12396,12 @@ function getGa4Connection(config, projectName) {
|
|
|
12954
12396
|
}
|
|
12955
12397
|
function upsertGa4Connection(config, connection) {
|
|
12956
12398
|
const connections = ensureConnections2(config);
|
|
12957
|
-
const
|
|
12958
|
-
if (
|
|
12399
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
12400
|
+
if (index === -1) {
|
|
12959
12401
|
connections.push(connection);
|
|
12960
12402
|
return connection;
|
|
12961
12403
|
}
|
|
12962
|
-
connections[
|
|
12404
|
+
connections[index] = connection;
|
|
12963
12405
|
return connection;
|
|
12964
12406
|
}
|
|
12965
12407
|
function removeGa4Connection(config, projectName) {
|
|
@@ -12995,12 +12437,12 @@ function getWordpressConnection(config, projectName) {
|
|
|
12995
12437
|
function upsertWordpressConnection(config, connection) {
|
|
12996
12438
|
const connections = ensureConnections3(config);
|
|
12997
12439
|
const normalized = normalizeConnection(connection);
|
|
12998
|
-
const
|
|
12999
|
-
if (
|
|
12440
|
+
const index = connections.findIndex((entry) => entry.projectName === connection.projectName);
|
|
12441
|
+
if (index === -1) {
|
|
13000
12442
|
connections.push(normalized);
|
|
13001
12443
|
return normalized;
|
|
13002
12444
|
}
|
|
13003
|
-
connections[
|
|
12445
|
+
connections[index] = normalized;
|
|
13004
12446
|
return normalized;
|
|
13005
12447
|
}
|
|
13006
12448
|
function patchWordpressConnection(config, projectName, patch) {
|
|
@@ -13031,85 +12473,199 @@ import crypto18 from "crypto";
|
|
|
13031
12473
|
import fs4 from "fs";
|
|
13032
12474
|
import path5 from "path";
|
|
13033
12475
|
import os4 from "os";
|
|
13034
|
-
import { and as and7, eq as eq18, inArray as inArray3, sql as
|
|
12476
|
+
import { and as and7, eq as eq18, inArray as inArray3, sql as sql4 } from "drizzle-orm";
|
|
13035
12477
|
|
|
13036
|
-
// src/
|
|
13037
|
-
|
|
13038
|
-
|
|
13039
|
-
const
|
|
13040
|
-
|
|
13041
|
-
const levelTag = level === "error" ? "\x1B[31mERR\x1B[0m" : level === "warn" ? "\x1B[33mWRN\x1B[0m" : "\x1B[36mINF\x1B[0m";
|
|
13042
|
-
const ctxParts = Object.entries(ctx).filter(([, v]) => v !== void 0 && v !== null).map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`).join(" ");
|
|
13043
|
-
const msgPart = msg ? ` ${msg}` : "";
|
|
13044
|
-
const ctxPart = ctxParts ? ` ${ctxParts}` : "";
|
|
13045
|
-
return `${time} ${levelTag} [${module}] ${action}${msgPart}${ctxPart}`;
|
|
13046
|
-
}
|
|
13047
|
-
function emit(entry) {
|
|
13048
|
-
const stream = entry.level === "error" ? process.stderr : process.stdout;
|
|
13049
|
-
if (IS_TTY) {
|
|
13050
|
-
stream.write(formatTTY(entry) + "\n");
|
|
13051
|
-
} else {
|
|
13052
|
-
stream.write(JSON.stringify(entry) + "\n");
|
|
13053
|
-
}
|
|
12478
|
+
// src/citation-utils.ts
|
|
12479
|
+
function domainMatches(domain, canonicalDomain) {
|
|
12480
|
+
const normalized = normalizeProjectDomain(canonicalDomain);
|
|
12481
|
+
const d = normalizeProjectDomain(domain);
|
|
12482
|
+
return d === normalized || d.endsWith(`.${normalized}`);
|
|
13054
12483
|
}
|
|
13055
|
-
function
|
|
13056
|
-
|
|
13057
|
-
const
|
|
13058
|
-
|
|
13059
|
-
|
|
13060
|
-
|
|
13061
|
-
|
|
13062
|
-
|
|
13063
|
-
|
|
13064
|
-
|
|
12484
|
+
function determineCitationState(normalized, domains) {
|
|
12485
|
+
for (const canonicalDomain of domains) {
|
|
12486
|
+
const bareDomain = normalizeProjectDomain(canonicalDomain);
|
|
12487
|
+
if (normalized.citedDomains.some((d) => domainMatches(d, bareDomain))) {
|
|
12488
|
+
return "cited";
|
|
12489
|
+
}
|
|
12490
|
+
const lowerDomain = bareDomain.toLowerCase();
|
|
12491
|
+
for (const source of normalized.groundingSources) {
|
|
12492
|
+
try {
|
|
12493
|
+
const uri = source.uri.toLowerCase();
|
|
12494
|
+
if (lowerDomain.includes(".") && uri.includes(lowerDomain)) {
|
|
12495
|
+
return "cited";
|
|
12496
|
+
}
|
|
12497
|
+
} catch {
|
|
12498
|
+
}
|
|
12499
|
+
if (source.title) {
|
|
12500
|
+
const titleLower = source.title.toLowerCase().replace(/^www\./, "");
|
|
12501
|
+
if (titleLower === lowerDomain || titleLower.endsWith(`.${lowerDomain}`)) {
|
|
12502
|
+
return "cited";
|
|
12503
|
+
}
|
|
12504
|
+
}
|
|
12505
|
+
}
|
|
13065
12506
|
}
|
|
13066
|
-
return
|
|
13067
|
-
info: (action, ctx) => log10("info", action, ctx),
|
|
13068
|
-
warn: (action, ctx) => log10("warn", action, ctx),
|
|
13069
|
-
error: (action, ctx) => log10("error", action, ctx)
|
|
13070
|
-
};
|
|
12507
|
+
return "not-cited";
|
|
13071
12508
|
}
|
|
13072
|
-
|
|
13073
|
-
|
|
13074
|
-
|
|
13075
|
-
|
|
13076
|
-
|
|
13077
|
-
|
|
13078
|
-
|
|
13079
|
-
}
|
|
13080
|
-
};
|
|
13081
|
-
var ProviderExecutionGate = class {
|
|
13082
|
-
constructor(maxConcurrency, maxPerMinute) {
|
|
13083
|
-
this.maxConcurrency = maxConcurrency;
|
|
13084
|
-
this.maxPerMinute = maxPerMinute;
|
|
13085
|
-
}
|
|
13086
|
-
window = [];
|
|
13087
|
-
waiters = [];
|
|
13088
|
-
rateLimitChain = Promise.resolve();
|
|
13089
|
-
inFlight = 0;
|
|
13090
|
-
async run(task) {
|
|
13091
|
-
await this.acquire();
|
|
13092
|
-
try {
|
|
13093
|
-
await this.waitForRateLimit();
|
|
13094
|
-
return await task();
|
|
13095
|
-
} finally {
|
|
13096
|
-
this.release();
|
|
12509
|
+
function computeCompetitorOverlap(normalized, competitorDomains) {
|
|
12510
|
+
const overlapSet = /* @__PURE__ */ new Set();
|
|
12511
|
+
for (const d of normalized.citedDomains) {
|
|
12512
|
+
for (const cd of competitorDomains) {
|
|
12513
|
+
if (domainMatches(d, cd)) {
|
|
12514
|
+
overlapSet.add(cd);
|
|
12515
|
+
}
|
|
13097
12516
|
}
|
|
13098
12517
|
}
|
|
13099
|
-
|
|
13100
|
-
|
|
13101
|
-
|
|
13102
|
-
|
|
12518
|
+
for (const source of normalized.groundingSources) {
|
|
12519
|
+
const uri = source.uri.toLowerCase();
|
|
12520
|
+
for (const cd of competitorDomains) {
|
|
12521
|
+
if (uri.includes(cd.toLowerCase())) {
|
|
12522
|
+
overlapSet.add(cd);
|
|
12523
|
+
}
|
|
13103
12524
|
}
|
|
13104
|
-
|
|
13105
|
-
|
|
13106
|
-
|
|
13107
|
-
|
|
13108
|
-
|
|
13109
|
-
|
|
13110
|
-
|
|
13111
|
-
|
|
13112
|
-
|
|
12525
|
+
}
|
|
12526
|
+
if (normalized.answerText) {
|
|
12527
|
+
const lowerAnswer = normalized.answerText.toLowerCase();
|
|
12528
|
+
for (const cd of competitorDomains) {
|
|
12529
|
+
if (lowerAnswer.includes(cd.toLowerCase())) {
|
|
12530
|
+
overlapSet.add(cd);
|
|
12531
|
+
}
|
|
12532
|
+
const brand = cd.split(".")[0];
|
|
12533
|
+
if (brand && brand.length >= 4 && new RegExp(`\\b${brand}\\b`, "i").test(lowerAnswer)) {
|
|
12534
|
+
overlapSet.add(cd);
|
|
12535
|
+
}
|
|
12536
|
+
}
|
|
12537
|
+
}
|
|
12538
|
+
return [...overlapSet];
|
|
12539
|
+
}
|
|
12540
|
+
function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, competitorDomains) {
|
|
12541
|
+
if (!answerText || answerText.length < 20) return [];
|
|
12542
|
+
const ownBrandKeys = new Set(
|
|
12543
|
+
ownDomains.flatMap((domain) => collectBrandKeysFromDomain(domain))
|
|
12544
|
+
);
|
|
12545
|
+
const knownCompetitorKeys = new Set(
|
|
12546
|
+
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandKeysFromDomain(domain)).filter((key) => !ownBrandKeys.has(key))
|
|
12547
|
+
);
|
|
12548
|
+
if (knownCompetitorKeys.size === 0) return [];
|
|
12549
|
+
const candidatePatterns = [
|
|
12550
|
+
/^\s*(?:[-*]|\d+\.)\s+(?:\*\*)?([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)(?:\*\*)?\s*[:\u2014\u2013–-]/gm,
|
|
12551
|
+
/\*\*([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)\*\*/g,
|
|
12552
|
+
/^#{1,4}\s+(?:\d+\.\s+)?(?:\*\*)?([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)(?:\*\*)?$/gm,
|
|
12553
|
+
/\[([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)\]\(https?:\/\/[^\s)]+\)/g
|
|
12554
|
+
];
|
|
12555
|
+
const genericKeys = /* @__PURE__ */ new Set([
|
|
12556
|
+
"additional",
|
|
12557
|
+
"best",
|
|
12558
|
+
"benefits",
|
|
12559
|
+
"bottomline",
|
|
12560
|
+
"comparison",
|
|
12561
|
+
"conclusion",
|
|
12562
|
+
"directorylisting",
|
|
12563
|
+
"example",
|
|
12564
|
+
"expertise",
|
|
12565
|
+
"features",
|
|
12566
|
+
"finalthoughts",
|
|
12567
|
+
"howitworks",
|
|
12568
|
+
"important",
|
|
12569
|
+
"keybenefits",
|
|
12570
|
+
"keyfeatures",
|
|
12571
|
+
"major",
|
|
12572
|
+
"note",
|
|
12573
|
+
"notable",
|
|
12574
|
+
"option",
|
|
12575
|
+
"other",
|
|
12576
|
+
"overview",
|
|
12577
|
+
"pricing",
|
|
12578
|
+
"pros",
|
|
12579
|
+
"reviews",
|
|
12580
|
+
"step",
|
|
12581
|
+
"summary",
|
|
12582
|
+
"top",
|
|
12583
|
+
"verdict",
|
|
12584
|
+
"whattolookfor",
|
|
12585
|
+
"whyitmatters",
|
|
12586
|
+
"whyitstandsout",
|
|
12587
|
+
"whywechoseit"
|
|
12588
|
+
]);
|
|
12589
|
+
const seen = /* @__PURE__ */ new Map();
|
|
12590
|
+
for (const pattern of candidatePatterns) {
|
|
12591
|
+
let match;
|
|
12592
|
+
while ((match = pattern.exec(answerText)) !== null) {
|
|
12593
|
+
const candidate = cleanCandidateName(match[1] ?? "");
|
|
12594
|
+
const candidateKey = brandKeyFromText(candidate);
|
|
12595
|
+
if (!candidateKey) continue;
|
|
12596
|
+
if (genericKeys.has(candidateKey)) continue;
|
|
12597
|
+
if (candidate.split(/\s+/).length > 6) continue;
|
|
12598
|
+
if (matchesBrandKey(candidateKey, ownBrandKeys)) continue;
|
|
12599
|
+
if (!matchesBrandKey(candidateKey, knownCompetitorKeys)) continue;
|
|
12600
|
+
if (!seen.has(candidateKey)) seen.set(candidateKey, candidate);
|
|
12601
|
+
}
|
|
12602
|
+
}
|
|
12603
|
+
return [...seen.values()].slice(0, 10);
|
|
12604
|
+
}
|
|
12605
|
+
function cleanCandidateName(candidate) {
|
|
12606
|
+
return candidate.replace(/^[\s"'`]+|[\s"'`.,:;!?]+$/g, "").replace(/\s+/g, " ").trim();
|
|
12607
|
+
}
|
|
12608
|
+
function collectBrandKeysFromDomain(domain) {
|
|
12609
|
+
const hostname = normalizeProjectDomain(domain).split("/")[0] ?? "";
|
|
12610
|
+
const labels = hostname.split(".").filter(Boolean);
|
|
12611
|
+
const keys = /* @__PURE__ */ new Set();
|
|
12612
|
+
const hostnameKey = hostname.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
12613
|
+
if (hostnameKey.length >= 4) keys.add(hostnameKey);
|
|
12614
|
+
for (const label of labels) {
|
|
12615
|
+
const key = label.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
12616
|
+
if (key.length >= 4) keys.add(key);
|
|
12617
|
+
}
|
|
12618
|
+
return [...keys];
|
|
12619
|
+
}
|
|
12620
|
+
function matchesBrandKey(candidateKey, brandKeys) {
|
|
12621
|
+
for (const brandKey of brandKeys) {
|
|
12622
|
+
if (candidateKey === brandKey) return true;
|
|
12623
|
+
if (candidateKey.startsWith(brandKey) || candidateKey.endsWith(brandKey)) return true;
|
|
12624
|
+
if (brandKey.startsWith(candidateKey) || brandKey.endsWith(candidateKey)) return true;
|
|
12625
|
+
}
|
|
12626
|
+
return false;
|
|
12627
|
+
}
|
|
12628
|
+
|
|
12629
|
+
// src/job-runner.ts
|
|
12630
|
+
var log = createLogger("JobRunner");
|
|
12631
|
+
var RunCancelledError = class extends Error {
|
|
12632
|
+
constructor(runId) {
|
|
12633
|
+
super(`Run ${runId} was cancelled`);
|
|
12634
|
+
this.name = "RunCancelledError";
|
|
12635
|
+
}
|
|
12636
|
+
};
|
|
12637
|
+
var ProviderExecutionGate = class {
|
|
12638
|
+
constructor(maxConcurrency, maxPerMinute) {
|
|
12639
|
+
this.maxConcurrency = maxConcurrency;
|
|
12640
|
+
this.maxPerMinute = maxPerMinute;
|
|
12641
|
+
}
|
|
12642
|
+
window = [];
|
|
12643
|
+
waiters = [];
|
|
12644
|
+
rateLimitChain = Promise.resolve();
|
|
12645
|
+
inFlight = 0;
|
|
12646
|
+
async run(task) {
|
|
12647
|
+
await this.acquire();
|
|
12648
|
+
try {
|
|
12649
|
+
await this.waitForRateLimit();
|
|
12650
|
+
return await task();
|
|
12651
|
+
} finally {
|
|
12652
|
+
this.release();
|
|
12653
|
+
}
|
|
12654
|
+
}
|
|
12655
|
+
async acquire() {
|
|
12656
|
+
if (this.inFlight < Math.max(1, this.maxConcurrency)) {
|
|
12657
|
+
this.inFlight++;
|
|
12658
|
+
return;
|
|
12659
|
+
}
|
|
12660
|
+
await new Promise((resolve) => {
|
|
12661
|
+
this.waiters.push(resolve);
|
|
12662
|
+
});
|
|
12663
|
+
this.inFlight++;
|
|
12664
|
+
}
|
|
12665
|
+
release() {
|
|
12666
|
+
this.inFlight = Math.max(0, this.inFlight - 1);
|
|
12667
|
+
const next = this.waiters.shift();
|
|
12668
|
+
next?.();
|
|
13113
12669
|
}
|
|
13114
12670
|
async waitForRateLimit() {
|
|
13115
12671
|
let releaseChain;
|
|
@@ -13431,7 +12987,7 @@ var JobRunner = class {
|
|
|
13431
12987
|
updatedAt: now
|
|
13432
12988
|
}).onConflictDoUpdate({
|
|
13433
12989
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
13434
|
-
set: { count:
|
|
12990
|
+
set: { count: sql4`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
13435
12991
|
}).run();
|
|
13436
12992
|
}
|
|
13437
12993
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -13481,159 +13037,10 @@ var JobRunner = class {
|
|
|
13481
13037
|
function getCurrentUsageDay() {
|
|
13482
13038
|
return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
13483
13039
|
}
|
|
13484
|
-
function domainMatches(domain, canonicalDomain) {
|
|
13485
|
-
const normalized = normalizeProjectDomain(canonicalDomain);
|
|
13486
|
-
const d = normalizeProjectDomain(domain);
|
|
13487
|
-
return d === normalized || d.endsWith(`.${normalized}`);
|
|
13488
|
-
}
|
|
13489
|
-
function determineCitationState(normalized, domains) {
|
|
13490
|
-
for (const canonicalDomain of domains) {
|
|
13491
|
-
const bareDomain = normalizeProjectDomain(canonicalDomain);
|
|
13492
|
-
if (normalized.citedDomains.some((d) => domainMatches(d, bareDomain))) {
|
|
13493
|
-
return "cited";
|
|
13494
|
-
}
|
|
13495
|
-
const lowerDomain = bareDomain.toLowerCase();
|
|
13496
|
-
for (const source of normalized.groundingSources) {
|
|
13497
|
-
try {
|
|
13498
|
-
const uri = source.uri.toLowerCase();
|
|
13499
|
-
if (lowerDomain.includes(".") && uri.includes(lowerDomain)) {
|
|
13500
|
-
return "cited";
|
|
13501
|
-
}
|
|
13502
|
-
} catch {
|
|
13503
|
-
}
|
|
13504
|
-
if (source.title) {
|
|
13505
|
-
const titleLower = source.title.toLowerCase().replace(/^www\./, "");
|
|
13506
|
-
if (titleLower === lowerDomain || titleLower.endsWith(`.${lowerDomain}`)) {
|
|
13507
|
-
return "cited";
|
|
13508
|
-
}
|
|
13509
|
-
}
|
|
13510
|
-
}
|
|
13511
|
-
}
|
|
13512
|
-
return "not-cited";
|
|
13513
|
-
}
|
|
13514
|
-
function computeCompetitorOverlap(normalized, competitorDomains) {
|
|
13515
|
-
const overlapSet = /* @__PURE__ */ new Set();
|
|
13516
|
-
for (const d of normalized.citedDomains) {
|
|
13517
|
-
for (const cd of competitorDomains) {
|
|
13518
|
-
if (domainMatches(d, cd)) {
|
|
13519
|
-
overlapSet.add(cd);
|
|
13520
|
-
}
|
|
13521
|
-
}
|
|
13522
|
-
}
|
|
13523
|
-
for (const source of normalized.groundingSources) {
|
|
13524
|
-
const uri = source.uri.toLowerCase();
|
|
13525
|
-
for (const cd of competitorDomains) {
|
|
13526
|
-
if (uri.includes(cd.toLowerCase())) {
|
|
13527
|
-
overlapSet.add(cd);
|
|
13528
|
-
}
|
|
13529
|
-
}
|
|
13530
|
-
}
|
|
13531
|
-
if (normalized.answerText) {
|
|
13532
|
-
const lowerAnswer = normalized.answerText.toLowerCase();
|
|
13533
|
-
for (const cd of competitorDomains) {
|
|
13534
|
-
if (lowerAnswer.includes(cd.toLowerCase())) {
|
|
13535
|
-
overlapSet.add(cd);
|
|
13536
|
-
}
|
|
13537
|
-
const brand = cd.split(".")[0];
|
|
13538
|
-
if (brand && brand.length >= 4 && new RegExp(`\\b${brand}\\b`, "i").test(lowerAnswer)) {
|
|
13539
|
-
overlapSet.add(cd);
|
|
13540
|
-
}
|
|
13541
|
-
}
|
|
13542
|
-
}
|
|
13543
|
-
return [...overlapSet];
|
|
13544
|
-
}
|
|
13545
|
-
function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, competitorDomains) {
|
|
13546
|
-
if (!answerText || answerText.length < 20) return [];
|
|
13547
|
-
const ownBrandKeys = new Set(
|
|
13548
|
-
ownDomains.flatMap((domain) => collectBrandKeysFromDomain(domain))
|
|
13549
|
-
);
|
|
13550
|
-
const knownCompetitorKeys = new Set(
|
|
13551
|
-
[...citedDomains, ...competitorDomains].flatMap((domain) => collectBrandKeysFromDomain(domain)).filter((key) => !ownBrandKeys.has(key))
|
|
13552
|
-
);
|
|
13553
|
-
if (knownCompetitorKeys.size === 0) return [];
|
|
13554
|
-
const candidatePatterns = [
|
|
13555
|
-
/^\s*(?:[-*]|\d+\.)\s+(?:\*\*)?([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)(?:\*\*)?\s*[:\u2014\u2013–-]/gm,
|
|
13556
|
-
/\*\*([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)\*\*/g,
|
|
13557
|
-
/^#{1,4}\s+(?:\d+\.\s+)?(?:\*\*)?([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)(?:\*\*)?$/gm,
|
|
13558
|
-
/\[([A-Z0-9][A-Za-z0-9][\w\s.&',/()-]{1,50}?)\]\(https?:\/\/[^\s)]+\)/g
|
|
13559
|
-
];
|
|
13560
|
-
const genericKeys = /* @__PURE__ */ new Set([
|
|
13561
|
-
"additional",
|
|
13562
|
-
"best",
|
|
13563
|
-
"benefits",
|
|
13564
|
-
"bottomline",
|
|
13565
|
-
"comparison",
|
|
13566
|
-
"conclusion",
|
|
13567
|
-
"directorylisting",
|
|
13568
|
-
"example",
|
|
13569
|
-
"expertise",
|
|
13570
|
-
"features",
|
|
13571
|
-
"finalthoughts",
|
|
13572
|
-
"howitworks",
|
|
13573
|
-
"important",
|
|
13574
|
-
"keybenefits",
|
|
13575
|
-
"keyfeatures",
|
|
13576
|
-
"major",
|
|
13577
|
-
"note",
|
|
13578
|
-
"notable",
|
|
13579
|
-
"option",
|
|
13580
|
-
"other",
|
|
13581
|
-
"overview",
|
|
13582
|
-
"pricing",
|
|
13583
|
-
"pros",
|
|
13584
|
-
"reviews",
|
|
13585
|
-
"step",
|
|
13586
|
-
"summary",
|
|
13587
|
-
"top",
|
|
13588
|
-
"verdict",
|
|
13589
|
-
"whattolookfor",
|
|
13590
|
-
"whyitmatters",
|
|
13591
|
-
"whyitstandsout",
|
|
13592
|
-
"whywechoseit"
|
|
13593
|
-
]);
|
|
13594
|
-
const seen = /* @__PURE__ */ new Map();
|
|
13595
|
-
for (const pattern of candidatePatterns) {
|
|
13596
|
-
let match;
|
|
13597
|
-
while ((match = pattern.exec(answerText)) !== null) {
|
|
13598
|
-
const candidate = cleanCandidateName(match[1] ?? "");
|
|
13599
|
-
const candidateKey = brandKeyFromText(candidate);
|
|
13600
|
-
if (!candidateKey) continue;
|
|
13601
|
-
if (genericKeys.has(candidateKey)) continue;
|
|
13602
|
-
if (candidate.split(/\s+/).length > 6) continue;
|
|
13603
|
-
if (matchesBrandKey(candidateKey, ownBrandKeys)) continue;
|
|
13604
|
-
if (!matchesBrandKey(candidateKey, knownCompetitorKeys)) continue;
|
|
13605
|
-
if (!seen.has(candidateKey)) seen.set(candidateKey, candidate);
|
|
13606
|
-
}
|
|
13607
|
-
}
|
|
13608
|
-
return [...seen.values()].slice(0, 10);
|
|
13609
|
-
}
|
|
13610
|
-
function cleanCandidateName(candidate) {
|
|
13611
|
-
return candidate.replace(/^[\s"'`]+|[\s"'`.,:;!?]+$/g, "").replace(/\s+/g, " ").trim();
|
|
13612
|
-
}
|
|
13613
|
-
function collectBrandKeysFromDomain(domain) {
|
|
13614
|
-
const hostname = normalizeProjectDomain(domain).split("/")[0] ?? "";
|
|
13615
|
-
const labels = hostname.split(".").filter(Boolean);
|
|
13616
|
-
const keys = /* @__PURE__ */ new Set();
|
|
13617
|
-
const hostnameKey = hostname.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
13618
|
-
if (hostnameKey.length >= 4) keys.add(hostnameKey);
|
|
13619
|
-
for (const label of labels) {
|
|
13620
|
-
const key = label.replace(/[^a-z0-9]/gi, "").toLowerCase();
|
|
13621
|
-
if (key.length >= 4) keys.add(key);
|
|
13622
|
-
}
|
|
13623
|
-
return [...keys];
|
|
13624
|
-
}
|
|
13625
|
-
function matchesBrandKey(candidateKey, brandKeys) {
|
|
13626
|
-
for (const brandKey of brandKeys) {
|
|
13627
|
-
if (candidateKey === brandKey) return true;
|
|
13628
|
-
if (candidateKey.startsWith(brandKey) || candidateKey.endsWith(brandKey)) return true;
|
|
13629
|
-
if (brandKey.startsWith(candidateKey) || brandKey.endsWith(candidateKey)) return true;
|
|
13630
|
-
}
|
|
13631
|
-
return false;
|
|
13632
|
-
}
|
|
13633
13040
|
|
|
13634
13041
|
// src/gsc-sync.ts
|
|
13635
13042
|
import crypto19 from "crypto";
|
|
13636
|
-
import { eq as eq19, and as and8, sql as
|
|
13043
|
+
import { eq as eq19, and as and8, sql as sql5 } from "drizzle-orm";
|
|
13637
13044
|
var log2 = createLogger("GscSync");
|
|
13638
13045
|
function formatDate2(d) {
|
|
13639
13046
|
return d.toISOString().split("T")[0];
|
|
@@ -13687,8 +13094,8 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
13687
13094
|
db.delete(gscSearchData).where(
|
|
13688
13095
|
and8(
|
|
13689
13096
|
eq19(gscSearchData.projectId, projectId),
|
|
13690
|
-
|
|
13691
|
-
|
|
13097
|
+
sql5`${gscSearchData.date} >= ${startDate}`,
|
|
13098
|
+
sql5`${gscSearchData.date} <= ${endDate}`
|
|
13692
13099
|
)
|
|
13693
13100
|
).run();
|
|
13694
13101
|
const batchSize = 500;
|
|
@@ -14213,7 +13620,8 @@ var Notifier = class {
|
|
|
14213
13620
|
if (lostTransitions.length > 0) events.push("citation.lost");
|
|
14214
13621
|
if (gainedTransitions.length > 0) events.push("citation.gained");
|
|
14215
13622
|
for (const notif of notifs) {
|
|
14216
|
-
const config =
|
|
13623
|
+
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
13624
|
+
if (!config.url) continue;
|
|
14217
13625
|
const subscribedEvents = config.events;
|
|
14218
13626
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
14219
13627
|
log5.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
@@ -14323,380 +13731,8 @@ var Notifier = class {
|
|
|
14323
13731
|
}
|
|
14324
13732
|
};
|
|
14325
13733
|
|
|
14326
|
-
// src/intelligence-service.ts
|
|
14327
|
-
import { eq as eq23, desc as desc9, asc as asc2, and as and11, or as or3 } from "drizzle-orm";
|
|
14328
|
-
|
|
14329
|
-
// ../intelligence/src/regressions.ts
|
|
14330
|
-
function detectRegressions(currentRun, previousRun) {
|
|
14331
|
-
const regressions = [];
|
|
14332
|
-
const previousCited = /* @__PURE__ */ new Map();
|
|
14333
|
-
for (const snap of previousRun.snapshots) {
|
|
14334
|
-
if (snap.cited) {
|
|
14335
|
-
previousCited.set(`${snap.keyword}:${snap.provider}`, {
|
|
14336
|
-
citationUrl: snap.citationUrl,
|
|
14337
|
-
position: snap.position
|
|
14338
|
-
});
|
|
14339
|
-
}
|
|
14340
|
-
}
|
|
14341
|
-
for (const snap of currentRun.snapshots) {
|
|
14342
|
-
const key = `${snap.keyword}:${snap.provider}`;
|
|
14343
|
-
if (!snap.cited && previousCited.has(key)) {
|
|
14344
|
-
const prev = previousCited.get(key);
|
|
14345
|
-
regressions.push({
|
|
14346
|
-
keyword: snap.keyword,
|
|
14347
|
-
provider: snap.provider,
|
|
14348
|
-
previousCitationUrl: prev.citationUrl,
|
|
14349
|
-
previousPosition: prev.position,
|
|
14350
|
-
currentRunId: currentRun.runId,
|
|
14351
|
-
previousRunId: previousRun.runId
|
|
14352
|
-
});
|
|
14353
|
-
}
|
|
14354
|
-
}
|
|
14355
|
-
return regressions;
|
|
14356
|
-
}
|
|
14357
|
-
|
|
14358
|
-
// ../intelligence/src/gains.ts
|
|
14359
|
-
function detectGains(currentRun, previousRun) {
|
|
14360
|
-
const gains = [];
|
|
14361
|
-
const previousCited = /* @__PURE__ */ new Set();
|
|
14362
|
-
for (const snap of previousRun.snapshots) {
|
|
14363
|
-
if (snap.cited) {
|
|
14364
|
-
previousCited.add(`${snap.keyword}:${snap.provider}`);
|
|
14365
|
-
}
|
|
14366
|
-
}
|
|
14367
|
-
for (const snap of currentRun.snapshots) {
|
|
14368
|
-
const key = `${snap.keyword}:${snap.provider}`;
|
|
14369
|
-
if (snap.cited && !previousCited.has(key)) {
|
|
14370
|
-
gains.push({
|
|
14371
|
-
keyword: snap.keyword,
|
|
14372
|
-
provider: snap.provider,
|
|
14373
|
-
citationUrl: snap.citationUrl,
|
|
14374
|
-
position: snap.position,
|
|
14375
|
-
snippet: snap.snippet,
|
|
14376
|
-
runId: currentRun.runId
|
|
14377
|
-
});
|
|
14378
|
-
}
|
|
14379
|
-
}
|
|
14380
|
-
return gains;
|
|
14381
|
-
}
|
|
14382
|
-
|
|
14383
|
-
// ../intelligence/src/health.ts
|
|
14384
|
-
function computeHealth(run) {
|
|
14385
|
-
const providerStats = /* @__PURE__ */ new Map();
|
|
14386
|
-
let totalPairs = 0;
|
|
14387
|
-
let citedPairs = 0;
|
|
14388
|
-
for (const snap of run.snapshots) {
|
|
14389
|
-
totalPairs++;
|
|
14390
|
-
if (snap.cited) citedPairs++;
|
|
14391
|
-
const stats = providerStats.get(snap.provider) ?? { cited: 0, total: 0 };
|
|
14392
|
-
stats.total++;
|
|
14393
|
-
if (snap.cited) stats.cited++;
|
|
14394
|
-
providerStats.set(snap.provider, stats);
|
|
14395
|
-
}
|
|
14396
|
-
const providerBreakdown = {};
|
|
14397
|
-
for (const [provider, stats] of providerStats) {
|
|
14398
|
-
providerBreakdown[provider] = {
|
|
14399
|
-
citedRate: stats.total > 0 ? stats.cited / stats.total : 0,
|
|
14400
|
-
cited: stats.cited,
|
|
14401
|
-
total: stats.total
|
|
14402
|
-
};
|
|
14403
|
-
}
|
|
14404
|
-
return {
|
|
14405
|
-
overallCitedRate: totalPairs > 0 ? citedPairs / totalPairs : 0,
|
|
14406
|
-
totalPairs,
|
|
14407
|
-
citedPairs,
|
|
14408
|
-
providerBreakdown
|
|
14409
|
-
};
|
|
14410
|
-
}
|
|
14411
|
-
function computeHealthTrend(runs2) {
|
|
14412
|
-
if (runs2.length === 0) {
|
|
14413
|
-
return { current: 0, previous: 0, delta: 0 };
|
|
14414
|
-
}
|
|
14415
|
-
const current = computeHealth(runs2[runs2.length - 1]).overallCitedRate;
|
|
14416
|
-
if (runs2.length === 1) {
|
|
14417
|
-
return { current, previous: 0, delta: current };
|
|
14418
|
-
}
|
|
14419
|
-
const previous = computeHealth(runs2[runs2.length - 2]).overallCitedRate;
|
|
14420
|
-
return {
|
|
14421
|
-
current,
|
|
14422
|
-
previous,
|
|
14423
|
-
delta: current - previous
|
|
14424
|
-
};
|
|
14425
|
-
}
|
|
14426
|
-
|
|
14427
|
-
// ../intelligence/src/causes.ts
|
|
14428
|
-
function analyzeCause(regression, currentSnapshots) {
|
|
14429
|
-
const currentSnap = currentSnapshots.find(
|
|
14430
|
-
(s) => s.keyword === regression.keyword && s.provider === regression.provider && !s.cited && s.competitorDomain
|
|
14431
|
-
);
|
|
14432
|
-
if (currentSnap) {
|
|
14433
|
-
return {
|
|
14434
|
-
cause: "competitor_gain",
|
|
14435
|
-
competitorDomain: currentSnap.competitorDomain,
|
|
14436
|
-
details: `Competitor ${currentSnap.competitorDomain} now cited for "${regression.keyword}" on ${regression.provider}`
|
|
14437
|
-
};
|
|
14438
|
-
}
|
|
14439
|
-
return {
|
|
14440
|
-
cause: "unknown",
|
|
14441
|
-
details: `No specific cause identified for loss of "${regression.keyword}" on ${regression.provider}`
|
|
14442
|
-
};
|
|
14443
|
-
}
|
|
14444
|
-
|
|
14445
|
-
// ../intelligence/src/insights.ts
|
|
14446
|
-
import { randomUUID } from "crypto";
|
|
14447
|
-
function generateInsights(regressions, gains, health, causes) {
|
|
14448
|
-
const insights2 = [];
|
|
14449
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14450
|
-
for (const reg of regressions) {
|
|
14451
|
-
const key = `${reg.keyword}:${reg.provider}`;
|
|
14452
|
-
const cause = causes.get(key);
|
|
14453
|
-
insights2.push({
|
|
14454
|
-
id: `ins_${randomUUID().slice(0, 8)}`,
|
|
14455
|
-
type: "regression",
|
|
14456
|
-
severity: "high",
|
|
14457
|
-
title: `Lost ${reg.provider} citation for "${reg.keyword}"`,
|
|
14458
|
-
keyword: reg.keyword,
|
|
14459
|
-
provider: reg.provider,
|
|
14460
|
-
recommendation: {
|
|
14461
|
-
action: "audit",
|
|
14462
|
-
target: reg.previousCitationUrl,
|
|
14463
|
-
reason: `Page was previously cited at position ${reg.previousPosition ?? "unknown"}. Run aeo-audit to check for content or schema issues.`
|
|
14464
|
-
},
|
|
14465
|
-
cause,
|
|
14466
|
-
createdAt: now
|
|
14467
|
-
});
|
|
14468
|
-
}
|
|
14469
|
-
for (const gain of gains) {
|
|
14470
|
-
insights2.push({
|
|
14471
|
-
id: `ins_${randomUUID().slice(0, 8)}`,
|
|
14472
|
-
type: "gain",
|
|
14473
|
-
severity: "low",
|
|
14474
|
-
title: `New ${gain.provider} citation for "${gain.keyword}"`,
|
|
14475
|
-
keyword: gain.keyword,
|
|
14476
|
-
provider: gain.provider,
|
|
14477
|
-
recommendation: {
|
|
14478
|
-
action: "monitor",
|
|
14479
|
-
target: gain.citationUrl,
|
|
14480
|
-
reason: `New citation appeared at position ${gain.position ?? "unknown"}. Monitor to confirm it persists.`
|
|
14481
|
-
},
|
|
14482
|
-
createdAt: now
|
|
14483
|
-
});
|
|
14484
|
-
}
|
|
14485
|
-
return insights2;
|
|
14486
|
-
}
|
|
14487
|
-
|
|
14488
|
-
// ../intelligence/src/analyzer.ts
|
|
14489
|
-
function analyzeRuns(currentRun, previousRun, allRuns) {
|
|
14490
|
-
const regressions = detectRegressions(currentRun, previousRun);
|
|
14491
|
-
const gains = detectGains(currentRun, previousRun);
|
|
14492
|
-
const health = computeHealth(currentRun);
|
|
14493
|
-
const trend = allRuns ? computeHealthTrend(allRuns) : void 0;
|
|
14494
|
-
const causes = /* @__PURE__ */ new Map();
|
|
14495
|
-
for (const reg of regressions) {
|
|
14496
|
-
const cause = analyzeCause(reg, currentRun.snapshots);
|
|
14497
|
-
causes.set(`${reg.keyword}:${reg.provider}`, cause);
|
|
14498
|
-
}
|
|
14499
|
-
const insights2 = generateInsights(regressions, gains, health, causes);
|
|
14500
|
-
return {
|
|
14501
|
-
regressions,
|
|
14502
|
-
gains,
|
|
14503
|
-
health,
|
|
14504
|
-
trend,
|
|
14505
|
-
insights: insights2
|
|
14506
|
-
};
|
|
14507
|
-
}
|
|
14508
|
-
|
|
14509
|
-
// src/intelligence-service.ts
|
|
14510
|
-
import crypto22 from "crypto";
|
|
14511
|
-
var log6 = createLogger("IntelligenceService");
|
|
14512
|
-
var IntelligenceService = class {
|
|
14513
|
-
constructor(db) {
|
|
14514
|
-
this.db = db;
|
|
14515
|
-
}
|
|
14516
|
-
/**
|
|
14517
|
-
* Analyze a completed run and persist insights + health snapshot.
|
|
14518
|
-
* Idempotent: deletes prior results for the same runId before inserting.
|
|
14519
|
-
* Returns the analysis result for the coordinator to inspect (e.g. for webhook dispatch).
|
|
14520
|
-
*/
|
|
14521
|
-
analyzeAndPersist(runId, projectId) {
|
|
14522
|
-
const recentRuns = this.db.select().from(runs).where(
|
|
14523
|
-
and11(
|
|
14524
|
-
eq23(runs.projectId, projectId),
|
|
14525
|
-
or3(eq23(runs.status, "completed"), eq23(runs.status, "partial"))
|
|
14526
|
-
)
|
|
14527
|
-
).orderBy(desc9(runs.createdAt)).limit(2).all();
|
|
14528
|
-
if (recentRuns.length === 0) {
|
|
14529
|
-
log6.info("intelligence.skip", { runId, reason: "no completed runs" });
|
|
14530
|
-
return null;
|
|
14531
|
-
}
|
|
14532
|
-
const currentRunRecord = recentRuns.find((r) => r.id === runId);
|
|
14533
|
-
if (!currentRunRecord) {
|
|
14534
|
-
log6.info("intelligence.skip", { runId, reason: "run not in recent completed list" });
|
|
14535
|
-
return null;
|
|
14536
|
-
}
|
|
14537
|
-
const currentRun = this.buildRunData(runId, projectId, currentRunRecord.finishedAt ?? currentRunRecord.createdAt);
|
|
14538
|
-
if (currentRun.snapshots.length === 0) {
|
|
14539
|
-
log6.info("intelligence.skip", { runId, reason: "no snapshots" });
|
|
14540
|
-
return null;
|
|
14541
|
-
}
|
|
14542
|
-
const previousRunRecord = recentRuns.find((r) => r.id !== runId);
|
|
14543
|
-
const previousRun = previousRunRecord ? this.buildRunData(previousRunRecord.id, projectId, previousRunRecord.finishedAt ?? previousRunRecord.createdAt) : null;
|
|
14544
|
-
if (!previousRun) {
|
|
14545
|
-
const result2 = analyzeRuns(currentRun, currentRun);
|
|
14546
|
-
log6.info("intelligence.analyzed", {
|
|
14547
|
-
runId,
|
|
14548
|
-
regressions: 0,
|
|
14549
|
-
gains: 0,
|
|
14550
|
-
citedRate: result2.health.overallCitedRate,
|
|
14551
|
-
insights: 0
|
|
14552
|
-
});
|
|
14553
|
-
this.persistResult({ ...result2, insights: [], regressions: [], gains: [] }, runId, projectId);
|
|
14554
|
-
return result2;
|
|
14555
|
-
}
|
|
14556
|
-
const result = analyzeRuns(currentRun, previousRun);
|
|
14557
|
-
log6.info("intelligence.analyzed", {
|
|
14558
|
-
runId,
|
|
14559
|
-
regressions: result.regressions.length,
|
|
14560
|
-
gains: result.gains.length,
|
|
14561
|
-
citedRate: result.health.overallCitedRate,
|
|
14562
|
-
insights: result.insights.length
|
|
14563
|
-
});
|
|
14564
|
-
this.persistResult(result, runId, projectId);
|
|
14565
|
-
return result;
|
|
14566
|
-
}
|
|
14567
|
-
/**
|
|
14568
|
-
* Analyze a single run given an explicit previous run (or null for first run).
|
|
14569
|
-
* Used by backfill where we control the run ordering.
|
|
14570
|
-
*/
|
|
14571
|
-
analyzeRunWithPrevious(runRecord, previousRunRecord) {
|
|
14572
|
-
const currentRun = this.buildRunData(runRecord.id, runRecord.projectId, runRecord.finishedAt ?? runRecord.createdAt);
|
|
14573
|
-
if (currentRun.snapshots.length === 0) {
|
|
14574
|
-
return null;
|
|
14575
|
-
}
|
|
14576
|
-
const previousRun = previousRunRecord ? this.buildRunData(previousRunRecord.id, previousRunRecord.projectId, previousRunRecord.finishedAt ?? previousRunRecord.createdAt) : null;
|
|
14577
|
-
if (!previousRun) {
|
|
14578
|
-
const result2 = analyzeRuns(currentRun, currentRun);
|
|
14579
|
-
this.persistResult({ ...result2, insights: [], regressions: [], gains: [] }, runRecord.id, runRecord.projectId);
|
|
14580
|
-
return result2;
|
|
14581
|
-
}
|
|
14582
|
-
const result = analyzeRuns(currentRun, previousRun);
|
|
14583
|
-
this.persistResult(result, runRecord.id, runRecord.projectId);
|
|
14584
|
-
return result;
|
|
14585
|
-
}
|
|
14586
|
-
/**
|
|
14587
|
-
* Backfill intelligence for all completed/partial runs of a project.
|
|
14588
|
-
* Processes runs in chronological order so each run compares against its predecessor.
|
|
14589
|
-
*/
|
|
14590
|
-
backfill(projectName, opts, onProgress) {
|
|
14591
|
-
const project = this.db.select().from(projects).where(eq23(projects.name, projectName)).get();
|
|
14592
|
-
if (!project) {
|
|
14593
|
-
throw new Error(`Project "${projectName}" not found`);
|
|
14594
|
-
}
|
|
14595
|
-
const allRuns = this.db.select().from(runs).where(
|
|
14596
|
-
and11(
|
|
14597
|
-
eq23(runs.projectId, project.id),
|
|
14598
|
-
or3(eq23(runs.status, "completed"), eq23(runs.status, "partial"))
|
|
14599
|
-
)
|
|
14600
|
-
).orderBy(asc2(runs.finishedAt)).all();
|
|
14601
|
-
let startIdx = 0;
|
|
14602
|
-
let endIdx = allRuns.length;
|
|
14603
|
-
if (opts?.fromRunId) {
|
|
14604
|
-
const idx = allRuns.findIndex((r) => r.id === opts.fromRunId);
|
|
14605
|
-
if (idx === -1) throw new Error(`Run "${opts.fromRunId}" not found in project`);
|
|
14606
|
-
startIdx = idx;
|
|
14607
|
-
}
|
|
14608
|
-
if (opts?.toRunId) {
|
|
14609
|
-
const idx = allRuns.findIndex((r) => r.id === opts.toRunId);
|
|
14610
|
-
if (idx === -1) throw new Error(`Run "${opts.toRunId}" not found in project`);
|
|
14611
|
-
endIdx = idx + 1;
|
|
14612
|
-
}
|
|
14613
|
-
const targetRuns = allRuns.slice(startIdx, endIdx);
|
|
14614
|
-
let processed = 0;
|
|
14615
|
-
let skipped = 0;
|
|
14616
|
-
let totalInsights = 0;
|
|
14617
|
-
for (let i = 0; i < targetRuns.length; i++) {
|
|
14618
|
-
const run = targetRuns[i];
|
|
14619
|
-
const globalIdx = allRuns.indexOf(run);
|
|
14620
|
-
const previousRun = globalIdx > 0 ? allRuns[globalIdx - 1] : null;
|
|
14621
|
-
const result = this.analyzeRunWithPrevious(run, previousRun);
|
|
14622
|
-
if (result) {
|
|
14623
|
-
processed++;
|
|
14624
|
-
totalInsights += result.insights.length;
|
|
14625
|
-
onProgress?.({ runId: run.id, index: i + 1, total: targetRuns.length, insights: result.insights.length });
|
|
14626
|
-
} else {
|
|
14627
|
-
skipped++;
|
|
14628
|
-
onProgress?.({ runId: run.id, index: i + 1, total: targetRuns.length, insights: 0 });
|
|
14629
|
-
}
|
|
14630
|
-
}
|
|
14631
|
-
return { processed, skipped, totalInsights };
|
|
14632
|
-
}
|
|
14633
|
-
persistResult(result, runId, projectId) {
|
|
14634
|
-
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
14635
|
-
const existingInsights = this.db.select({ keyword: insights.keyword, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq23(insights.runId, runId)).all();
|
|
14636
|
-
for (const row of existingInsights) {
|
|
14637
|
-
if (row.dismissed) {
|
|
14638
|
-
previouslyDismissed.add(`${row.keyword}:${row.provider}:${row.type}`);
|
|
14639
|
-
}
|
|
14640
|
-
}
|
|
14641
|
-
this.db.transaction((tx) => {
|
|
14642
|
-
tx.delete(insights).where(eq23(insights.runId, runId)).run();
|
|
14643
|
-
tx.delete(healthSnapshots).where(eq23(healthSnapshots.runId, runId)).run();
|
|
14644
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
14645
|
-
for (const insight of result.insights) {
|
|
14646
|
-
const wasDismissed = previouslyDismissed.has(`${insight.keyword}:${insight.provider}:${insight.type}`);
|
|
14647
|
-
tx.insert(insights).values({
|
|
14648
|
-
id: insight.id,
|
|
14649
|
-
projectId,
|
|
14650
|
-
runId,
|
|
14651
|
-
type: insight.type,
|
|
14652
|
-
severity: insight.severity,
|
|
14653
|
-
title: insight.title,
|
|
14654
|
-
keyword: insight.keyword,
|
|
14655
|
-
provider: insight.provider,
|
|
14656
|
-
recommendation: insight.recommendation ? JSON.stringify(insight.recommendation) : null,
|
|
14657
|
-
cause: insight.cause ? JSON.stringify(insight.cause) : null,
|
|
14658
|
-
dismissed: wasDismissed,
|
|
14659
|
-
createdAt: insight.createdAt
|
|
14660
|
-
}).run();
|
|
14661
|
-
}
|
|
14662
|
-
tx.insert(healthSnapshots).values({
|
|
14663
|
-
id: crypto22.randomUUID(),
|
|
14664
|
-
projectId,
|
|
14665
|
-
runId,
|
|
14666
|
-
overallCitedRate: String(result.health.overallCitedRate),
|
|
14667
|
-
totalPairs: result.health.totalPairs,
|
|
14668
|
-
citedPairs: result.health.citedPairs,
|
|
14669
|
-
providerBreakdown: JSON.stringify(result.health.providerBreakdown),
|
|
14670
|
-
createdAt: now
|
|
14671
|
-
}).run();
|
|
14672
|
-
});
|
|
14673
|
-
log6.info("intelligence.persisted", { runId, insights: result.insights.length });
|
|
14674
|
-
}
|
|
14675
|
-
buildRunData(runId, projectId, completedAt) {
|
|
14676
|
-
const rows = this.db.select({
|
|
14677
|
-
keyword: keywords.keyword,
|
|
14678
|
-
provider: querySnapshots.provider,
|
|
14679
|
-
citationState: querySnapshots.citationState,
|
|
14680
|
-
citedDomains: querySnapshots.citedDomains,
|
|
14681
|
-
competitorOverlap: querySnapshots.competitorOverlap
|
|
14682
|
-
}).from(querySnapshots).leftJoin(keywords, eq23(querySnapshots.keywordId, keywords.id)).where(eq23(querySnapshots.runId, runId)).all();
|
|
14683
|
-
const snapshots = rows.map((r) => {
|
|
14684
|
-
const domains = parseJsonColumn(r.citedDomains, []);
|
|
14685
|
-
const competitors2 = parseJsonColumn(r.competitorOverlap, []);
|
|
14686
|
-
return {
|
|
14687
|
-
keyword: r.keyword ?? "",
|
|
14688
|
-
provider: r.provider,
|
|
14689
|
-
cited: r.citationState === "cited",
|
|
14690
|
-
citationUrl: domains[0] ?? void 0,
|
|
14691
|
-
competitorDomain: competitors2[0] ?? void 0
|
|
14692
|
-
};
|
|
14693
|
-
});
|
|
14694
|
-
return { runId, projectId, completedAt, snapshots };
|
|
14695
|
-
}
|
|
14696
|
-
};
|
|
14697
|
-
|
|
14698
13734
|
// src/run-coordinator.ts
|
|
14699
|
-
var
|
|
13735
|
+
var log6 = createLogger("RunCoordinator");
|
|
14700
13736
|
var RunCoordinator = class {
|
|
14701
13737
|
constructor(notifier, intelligenceService) {
|
|
14702
13738
|
this.notifier = notifier;
|
|
@@ -14706,12 +13742,12 @@ var RunCoordinator = class {
|
|
|
14706
13742
|
try {
|
|
14707
13743
|
this.intelligenceService.analyzeAndPersist(runId, projectId);
|
|
14708
13744
|
} catch (err) {
|
|
14709
|
-
|
|
13745
|
+
log6.error("intelligence.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14710
13746
|
}
|
|
14711
13747
|
try {
|
|
14712
13748
|
await this.notifier.onRunCompleted(runId, projectId);
|
|
14713
13749
|
} catch (err) {
|
|
14714
|
-
|
|
13750
|
+
log6.error("notifier.failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
14715
13751
|
}
|
|
14716
13752
|
}
|
|
14717
13753
|
};
|
|
@@ -14814,20 +13850,20 @@ async function fetchSiteText(domain) {
|
|
|
14814
13850
|
}
|
|
14815
13851
|
function stripHtml2(html) {
|
|
14816
13852
|
if (!html) return "";
|
|
14817
|
-
let
|
|
14818
|
-
|
|
14819
|
-
|
|
14820
|
-
|
|
14821
|
-
|
|
14822
|
-
|
|
14823
|
-
|
|
14824
|
-
|
|
14825
|
-
|
|
14826
|
-
|
|
14827
|
-
if (
|
|
14828
|
-
|
|
14829
|
-
}
|
|
14830
|
-
return
|
|
13853
|
+
let text = html.replace(/<script[\s\S]*?<\/script>/gi, " ");
|
|
13854
|
+
text = text.replace(/<style[\s\S]*?<\/style>/gi, " ");
|
|
13855
|
+
text = text.replace(/<[^>]+>/g, " ");
|
|
13856
|
+
text = text.replace(/&/g, "&");
|
|
13857
|
+
text = text.replace(/</g, "<");
|
|
13858
|
+
text = text.replace(/>/g, ">");
|
|
13859
|
+
text = text.replace(/"/g, '"');
|
|
13860
|
+
text = text.replace(/'/g, "'");
|
|
13861
|
+
text = text.replace(/ /g, " ");
|
|
13862
|
+
text = text.replace(/\s+/g, " ").trim();
|
|
13863
|
+
if (text.length > MAX_TEXT_LENGTH) {
|
|
13864
|
+
text = text.slice(0, MAX_TEXT_LENGTH);
|
|
13865
|
+
}
|
|
13866
|
+
return text;
|
|
14831
13867
|
}
|
|
14832
13868
|
|
|
14833
13869
|
// src/snapshot-format.ts
|
|
@@ -14836,7 +13872,7 @@ function formatAuditFactorScore(factor) {
|
|
|
14836
13872
|
}
|
|
14837
13873
|
|
|
14838
13874
|
// src/snapshot-service.ts
|
|
14839
|
-
var
|
|
13875
|
+
var log7 = createLogger("Snapshot");
|
|
14840
13876
|
var ANALYSIS_PROVIDER_PRIORITY = ["openai", "claude", "gemini", "perplexity", "local"];
|
|
14841
13877
|
var SNAPSHOT_QUERY_COUNT = 6;
|
|
14842
13878
|
var ProviderExecutionGate2 = class {
|
|
@@ -14979,7 +14015,7 @@ var SnapshotService = class {
|
|
|
14979
14015
|
return mapAuditReport(report);
|
|
14980
14016
|
} catch (err) {
|
|
14981
14017
|
const message = err instanceof Error ? err.message : String(err);
|
|
14982
|
-
|
|
14018
|
+
log7.warn("audit.failed", { homepageUrl, error: message });
|
|
14983
14019
|
return {
|
|
14984
14020
|
url: homepageUrl,
|
|
14985
14021
|
finalUrl: homepageUrl,
|
|
@@ -15009,7 +14045,7 @@ var SnapshotService = class {
|
|
|
15009
14045
|
phrases: parsedPhrases
|
|
15010
14046
|
};
|
|
15011
14047
|
} catch (err) {
|
|
15012
|
-
|
|
14048
|
+
log7.warn("profile.generation-failed", {
|
|
15013
14049
|
domain: ctx.domain,
|
|
15014
14050
|
provider: ctx.analysisProvider.adapter.name,
|
|
15015
14051
|
error: err instanceof Error ? err.message : String(err)
|
|
@@ -15151,7 +14187,7 @@ var SnapshotService = class {
|
|
|
15151
14187
|
recommendedActions: uniqueStrings(parsed.recommendedActions ?? []).slice(0, 4)
|
|
15152
14188
|
};
|
|
15153
14189
|
} catch (err) {
|
|
15154
|
-
|
|
14190
|
+
log7.warn("response.analysis-failed", {
|
|
15155
14191
|
provider: ctx.analysisProvider.adapter.name,
|
|
15156
14192
|
error: err instanceof Error ? err.message : String(err)
|
|
15157
14193
|
});
|
|
@@ -15436,7 +14472,7 @@ function clipText(value, length) {
|
|
|
15436
14472
|
// src/server.ts
|
|
15437
14473
|
var _require2 = createRequire2(import.meta.url);
|
|
15438
14474
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
15439
|
-
var
|
|
14475
|
+
var log8 = createLogger("Server");
|
|
15440
14476
|
var DEFAULT_QUOTA = {
|
|
15441
14477
|
maxConcurrency: 2,
|
|
15442
14478
|
maxRequestsPerMinute: 10,
|
|
@@ -15467,7 +14503,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
15467
14503
|
};
|
|
15468
14504
|
}
|
|
15469
14505
|
function hashApiKey(key) {
|
|
15470
|
-
return
|
|
14506
|
+
return crypto22.createHash("sha256").update(key).digest("hex");
|
|
15471
14507
|
}
|
|
15472
14508
|
function parseCookies2(header) {
|
|
15473
14509
|
if (!header) return {};
|
|
@@ -15526,7 +14562,7 @@ async function createServer(opts) {
|
|
|
15526
14562
|
quota: opts.config.geminiQuota
|
|
15527
14563
|
};
|
|
15528
14564
|
}
|
|
15529
|
-
|
|
14565
|
+
log8.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
15530
14566
|
const p = providers[k];
|
|
15531
14567
|
return p?.apiKey || p?.baseUrl || p?.vertexProject;
|
|
15532
14568
|
}) });
|
|
@@ -15641,7 +14677,7 @@ async function createServer(opts) {
|
|
|
15641
14677
|
return removed;
|
|
15642
14678
|
}
|
|
15643
14679
|
};
|
|
15644
|
-
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ??
|
|
14680
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto22.randomBytes(32).toString("hex");
|
|
15645
14681
|
const googleConnectionStore = {
|
|
15646
14682
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
15647
14683
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -15687,11 +14723,11 @@ async function createServer(opts) {
|
|
|
15687
14723
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
15688
14724
|
if (opts.config.apiKey) {
|
|
15689
14725
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
15690
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
14726
|
+
const existing = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, keyHash)).get();
|
|
15691
14727
|
if (!existing) {
|
|
15692
14728
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
15693
14729
|
opts.db.insert(apiKeys).values({
|
|
15694
|
-
id: `key_${
|
|
14730
|
+
id: `key_${crypto22.randomBytes(8).toString("hex")}`,
|
|
15695
14731
|
name: "default",
|
|
15696
14732
|
keyHash,
|
|
15697
14733
|
keyPrefix: prefix,
|
|
@@ -15715,7 +14751,7 @@ async function createServer(opts) {
|
|
|
15715
14751
|
};
|
|
15716
14752
|
const createSession = (apiKeyId) => {
|
|
15717
14753
|
pruneExpiredSessions();
|
|
15718
|
-
const sessionId =
|
|
14754
|
+
const sessionId = crypto22.randomBytes(32).toString("hex");
|
|
15719
14755
|
sessions.set(sessionId, {
|
|
15720
14756
|
apiKeyId,
|
|
15721
14757
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -15739,7 +14775,7 @@ async function createServer(opts) {
|
|
|
15739
14775
|
};
|
|
15740
14776
|
const getDefaultApiKey = () => {
|
|
15741
14777
|
if (!opts.config.apiKey) return void 0;
|
|
15742
|
-
return opts.db.select().from(apiKeys).where(
|
|
14778
|
+
return opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
15743
14779
|
};
|
|
15744
14780
|
const createPasswordSession = (reply) => {
|
|
15745
14781
|
const key = getDefaultApiKey();
|
|
@@ -15796,12 +14832,12 @@ async function createServer(opts) {
|
|
|
15796
14832
|
return reply.send({ authenticated: true });
|
|
15797
14833
|
}
|
|
15798
14834
|
if (apiKey) {
|
|
15799
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
14835
|
+
const key = opts.db.select().from(apiKeys).where(eq23(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
15800
14836
|
if (!key || key.revokedAt) {
|
|
15801
14837
|
const err2 = authInvalid();
|
|
15802
14838
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
15803
14839
|
}
|
|
15804
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
14840
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(apiKeys.id, key.id)).run();
|
|
15805
14841
|
const sessionId = createSession(key.id);
|
|
15806
14842
|
reply.header("set-cookie", serializeSessionCookie({
|
|
15807
14843
|
name: SESSION_COOKIE_NAME,
|
|
@@ -15936,17 +14972,13 @@ async function createServer(opts) {
|
|
|
15936
14972
|
after: afterConfig
|
|
15937
14973
|
});
|
|
15938
14974
|
const affectedProjectIds = opts.db.select({ id: projects.id, providers: projects.providers }).from(projects).all().filter((project) => {
|
|
15939
|
-
|
|
15940
|
-
|
|
15941
|
-
return configuredProviders.length === 0 || configuredProviders.includes(name);
|
|
15942
|
-
} catch {
|
|
15943
|
-
return false;
|
|
15944
|
-
}
|
|
14975
|
+
const configuredProviders = parseJsonColumn(project.providers, []);
|
|
14976
|
+
return configuredProviders.length === 0 || configuredProviders.includes(name);
|
|
15945
14977
|
}).map((project) => project.id);
|
|
15946
14978
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
15947
14979
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
15948
14980
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
15949
|
-
id:
|
|
14981
|
+
id: crypto22.randomUUID(),
|
|
15950
14982
|
projectId,
|
|
15951
14983
|
actor: "api",
|
|
15952
14984
|
action: existing ? "provider.updated" : "provider.created",
|
|
@@ -16077,8 +15109,8 @@ async function createServer(opts) {
|
|
|
16077
15109
|
return snapshotService.createReport(input);
|
|
16078
15110
|
}
|
|
16079
15111
|
});
|
|
16080
|
-
const
|
|
16081
|
-
const assetsDir = path6.join(
|
|
15112
|
+
const dirname = path6.dirname(fileURLToPath(import.meta.url));
|
|
15113
|
+
const assetsDir = path6.join(dirname, "..", "assets");
|
|
16082
15114
|
if (fs5.existsSync(assetsDir)) {
|
|
16083
15115
|
const indexPath = path6.join(assetsDir, "index.html");
|
|
16084
15116
|
const injectConfig = (html) => {
|
|
@@ -16200,19 +15232,20 @@ export {
|
|
|
16200
15232
|
isFirstRun,
|
|
16201
15233
|
showFirstRunNotice,
|
|
16202
15234
|
trackEvent,
|
|
16203
|
-
projects,
|
|
16204
|
-
runs,
|
|
16205
|
-
querySnapshots,
|
|
16206
|
-
apiKeys,
|
|
16207
|
-
createClient,
|
|
16208
|
-
parseJsonColumn,
|
|
16209
|
-
migrate,
|
|
16210
15235
|
providerQuotaPolicySchema,
|
|
15236
|
+
ProviderNames,
|
|
16211
15237
|
resolveProviderInput,
|
|
16212
15238
|
notificationEventSchema,
|
|
16213
15239
|
effectiveDomains,
|
|
15240
|
+
RunKinds,
|
|
16214
15241
|
determineAnswerMentioned,
|
|
16215
|
-
|
|
15242
|
+
reparseStoredResult2 as reparseStoredResult,
|
|
15243
|
+
reparseStoredResult3 as reparseStoredResult2,
|
|
15244
|
+
reparseStoredResult as reparseStoredResult3,
|
|
15245
|
+
reparseStoredResult4,
|
|
15246
|
+
determineCitationState,
|
|
15247
|
+
computeCompetitorOverlap,
|
|
15248
|
+
extractRecommendedCompetitors,
|
|
16216
15249
|
setGoogleAuthConfig,
|
|
16217
15250
|
formatAuditFactorScore,
|
|
16218
15251
|
createServer
|