@canonry/canonry 4.129.0 → 4.130.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -60
- package/assets/assets/{AuditHistoryPanel-DwJpVWme.js → AuditHistoryPanel-Ydzb2Y_Y.js} +1 -1
- package/assets/assets/BacklinksPage-DYodEusK.js +1 -0
- package/assets/assets/{ChartPrimitives-6oFKrJFk.js → ChartPrimitives-D67hsnF7.js} +1 -1
- package/assets/assets/{HistoryPage-DwOAoN1Z.js → HistoryPage-_h9ITL1w.js} +1 -1
- package/assets/assets/ProjectPage-fCJKCcHE.js +8 -0
- package/assets/assets/{RunRow-lt9IS99U.js → RunRow-DzUNyTE-.js} +1 -1
- package/assets/assets/{RunsPage-DitYcIdx.js → RunsPage-FTvAjptk.js} +1 -1
- package/assets/assets/{SettingsPage-CAOrh65Q.js → SettingsPage-CvgpYEAt.js} +1 -1
- package/assets/assets/{TrafficPage-BqzNvx75.js → TrafficPage-qaMW9k6o.js} +1 -1
- package/assets/assets/TrafficSourceDetailPage-CRCso7_A.js +1 -0
- package/assets/assets/{arrow-left-AmG4gtUR.js → arrow-left-C--lHdP2.js} +1 -1
- package/assets/assets/extract-error-message-DVqYbKpB.js +1 -0
- package/assets/assets/index-CdXn0JEf.js +210 -0
- package/assets/assets/{index-jymsMBpZ.css → index-JhX-wfQO.css} +1 -1
- package/assets/assets/trash-2-CyKRU3tR.js +1 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-EF67Y6CI.js → chunk-5XU6WEMK.js} +728 -292
- package/dist/{chunk-BLSHHRRU.js → chunk-NVRJZU6V.js} +2758 -2613
- package/dist/{chunk-ZHWWSU7H.js → chunk-QR5U2K3O.js} +508 -355
- package/dist/{chunk-IHRW6WWV.js → chunk-WIALMCXM.js} +127 -6
- package/dist/cli.js +446 -184
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-NTIKN7YG.js → intelligence-service-53GUE6EI.js} +2 -2
- package/dist/mcp.js +5 -5
- package/package.json +6 -6
- package/assets/assets/BacklinksPage-K8FQUBxI.js +0 -1
- package/assets/assets/ProjectPage-CdgFUk7h.js +0 -7
- package/assets/assets/TrafficSourceDetailPage-DXbygZUA.js +0 -1
- package/assets/assets/extract-error-message-DlVtjoPz.js +0 -1
- package/assets/assets/index-BiCDkOH4.js +0 -210
- package/assets/assets/trash-2-gFHWd1A1.js +0 -1
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
ContentActions,
|
|
30
30
|
DEFAULT_DISCOVERY_PROMOTE_BUCKETS,
|
|
31
31
|
DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES,
|
|
32
|
+
DEFAULT_MEASUREMENT_CONFIG,
|
|
32
33
|
DISCOVERY_DEFAULT_DEDUP_THRESHOLD,
|
|
33
34
|
DISCOVERY_DEFAULT_PROBE_CONCURRENCY,
|
|
34
35
|
DISCOVERY_PROBE_CONCURRENCY_CAP,
|
|
@@ -38,6 +39,8 @@ import {
|
|
|
38
39
|
DiscoveryCompetitorTypes,
|
|
39
40
|
DiscoverySessionStatuses,
|
|
40
41
|
MODEL_ATTRIBUTION_EVENT_LIMIT,
|
|
42
|
+
ResearchQueryStatuses,
|
|
43
|
+
ResearchRunStatuses,
|
|
41
44
|
RunKinds,
|
|
42
45
|
RunStatuses,
|
|
43
46
|
RunTriggers,
|
|
@@ -47,6 +50,7 @@ import {
|
|
|
47
50
|
TrafficEventConfidences,
|
|
48
51
|
TrafficEventKinds,
|
|
49
52
|
TrafficEvidenceKinds,
|
|
53
|
+
TrafficSeriesGranularities,
|
|
50
54
|
TrafficSourceAuthModes,
|
|
51
55
|
TrafficSourceStatuses,
|
|
52
56
|
TrafficSourceTypes,
|
|
@@ -226,6 +230,7 @@ import {
|
|
|
226
230
|
hostOf,
|
|
227
231
|
indexingRequestResponseDtoSchema,
|
|
228
232
|
internalError,
|
|
233
|
+
isBrowserProvider,
|
|
229
234
|
isReadOnlyKey,
|
|
230
235
|
keywordDtoSchema,
|
|
231
236
|
keywordGenerateRequestSchema,
|
|
@@ -270,6 +275,9 @@ import {
|
|
|
270
275
|
reportConfidenceLabel,
|
|
271
276
|
reportHorizonLabel,
|
|
272
277
|
reportSeverityLabel,
|
|
278
|
+
researchRunCreateSchema,
|
|
279
|
+
researchRunDetailSchema,
|
|
280
|
+
researchRunListSchema,
|
|
273
281
|
resolveConfigSpecQueries,
|
|
274
282
|
resolveLocations,
|
|
275
283
|
resolveSnapshotRequestQueries,
|
|
@@ -336,10 +344,10 @@ import {
|
|
|
336
344
|
wordpressSchemaDeployResultDtoSchema,
|
|
337
345
|
wordpressSchemaStatusResultDtoSchema,
|
|
338
346
|
wordpressStatusDtoSchema
|
|
339
|
-
} from "./chunk-
|
|
347
|
+
} from "./chunk-NVRJZU6V.js";
|
|
340
348
|
|
|
341
349
|
// src/intelligence-service.ts
|
|
342
|
-
import { eq as
|
|
350
|
+
import { eq as eq42, desc as desc19, asc as asc9, and as and33, ne as ne6, or as or8, inArray as inArray15, gte as gte10, lte as lte6 } from "drizzle-orm";
|
|
343
351
|
|
|
344
352
|
// ../db/src/client.ts
|
|
345
353
|
import { mkdirSync } from "fs";
|
|
@@ -378,9 +386,12 @@ __export(schema_exports, {
|
|
|
378
386
|
discoveryProbes: () => discoveryProbes,
|
|
379
387
|
discoverySessions: () => discoverySessions,
|
|
380
388
|
domainClassifications: () => domainClassifications,
|
|
389
|
+
gaAcquisitionDaily: () => gaAcquisitionDaily,
|
|
381
390
|
gaAiReferrals: () => gaAiReferrals,
|
|
382
391
|
gaConnections: () => gaConnections,
|
|
383
392
|
gaDailyTotals: () => gaDailyTotals,
|
|
393
|
+
gaLeadEventsDaily: () => gaLeadEventsDaily,
|
|
394
|
+
gaMeasurementSyncStates: () => gaMeasurementSyncStates,
|
|
384
395
|
gaSocialReferrals: () => gaSocialReferrals,
|
|
385
396
|
gaTrafficSnapshots: () => gaTrafficSnapshots,
|
|
386
397
|
gaTrafficSummaries: () => gaTrafficSummaries,
|
|
@@ -410,6 +421,8 @@ __export(schema_exports, {
|
|
|
410
421
|
rawEventSamples: () => rawEventSamples,
|
|
411
422
|
recommendationBriefs: () => recommendationBriefs,
|
|
412
423
|
recommendationExplanations: () => recommendationExplanations,
|
|
424
|
+
researchRunQueries: () => researchRunQueries,
|
|
425
|
+
researchRuns: () => researchRuns,
|
|
413
426
|
runs: () => runs,
|
|
414
427
|
schedules: () => schedules,
|
|
415
428
|
siteAuditPages: () => siteAuditPages,
|
|
@@ -417,7 +430,8 @@ __export(schema_exports, {
|
|
|
417
430
|
trafficSources: () => trafficSources,
|
|
418
431
|
usageCounters: () => usageCounters
|
|
419
432
|
});
|
|
420
|
-
import {
|
|
433
|
+
import { sql } from "drizzle-orm";
|
|
434
|
+
import { check, index, integer, primaryKey, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core";
|
|
421
435
|
var projects = sqliteTable("projects", {
|
|
422
436
|
id: text("id").primaryKey(),
|
|
423
437
|
name: text("name").notNull().unique(),
|
|
@@ -431,6 +445,11 @@ var projects = sqliteTable("projects", {
|
|
|
431
445
|
labels: text("labels", { mode: "json" }).$type().notNull().default({}),
|
|
432
446
|
providers: text("providers", { mode: "json" }).$type().notNull().default([]),
|
|
433
447
|
providerModels: text("provider_models", { mode: "json" }).$type().notNull().default({}),
|
|
448
|
+
measurement: text("measurement_config", { mode: "json" }).$type().notNull().default({
|
|
449
|
+
marketingHosts: [],
|
|
450
|
+
brandTerms: [],
|
|
451
|
+
leadEventNames: ["generate_lead"]
|
|
452
|
+
}),
|
|
434
453
|
locations: text("locations", { mode: "json" }).$type().notNull().default([]),
|
|
435
454
|
defaultLocation: text("default_location"),
|
|
436
455
|
autoExtractBacklinks: integer("auto_extract_backlinks", { mode: "boolean" }).notNull().default(false),
|
|
@@ -862,6 +881,97 @@ var gaDailyTotals = sqliteTable("ga_daily_totals", {
|
|
|
862
881
|
index("idx_ga_daily_totals_project").on(table.projectId),
|
|
863
882
|
index("idx_ga_daily_totals_run").on(table.syncRunId)
|
|
864
883
|
]);
|
|
884
|
+
var gaAcquisitionDaily = sqliteTable("ga_acquisition_daily", {
|
|
885
|
+
id: text("id").primaryKey(),
|
|
886
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
887
|
+
date: text("date").notNull(),
|
|
888
|
+
channelGroup: text("channel_group").notNull(),
|
|
889
|
+
source: text("source").notNull(),
|
|
890
|
+
medium: text("medium").notNull(),
|
|
891
|
+
hostName: text("host_name").notNull(),
|
|
892
|
+
landingPage: text("landing_page").notNull(),
|
|
893
|
+
landingPageNormalized: text("landing_page_normalized"),
|
|
894
|
+
sessions: integer("sessions").notNull().default(0),
|
|
895
|
+
syncedAt: text("synced_at").notNull(),
|
|
896
|
+
syncRunId: text("sync_run_id").references(() => runs.id, { onDelete: "cascade" }),
|
|
897
|
+
createdAt: text("created_at").notNull()
|
|
898
|
+
}, (table) => [
|
|
899
|
+
uniqueIndex("idx_ga_acquisition_daily_grain").on(
|
|
900
|
+
table.projectId,
|
|
901
|
+
table.date,
|
|
902
|
+
table.channelGroup,
|
|
903
|
+
table.source,
|
|
904
|
+
table.medium,
|
|
905
|
+
table.hostName,
|
|
906
|
+
table.landingPage
|
|
907
|
+
),
|
|
908
|
+
index("idx_ga_acquisition_daily_project_date").on(table.projectId, table.date),
|
|
909
|
+
index("idx_ga_acquisition_daily_project_channel").on(table.projectId, table.date, table.channelGroup),
|
|
910
|
+
index("idx_ga_acquisition_daily_project_page").on(table.projectId, table.date, table.landingPageNormalized),
|
|
911
|
+
check("chk_ga_acquisition_daily_sessions", sql`${table.sessions} >= 0`)
|
|
912
|
+
]);
|
|
913
|
+
var gaLeadEventsDaily = sqliteTable("ga_lead_events_daily", {
|
|
914
|
+
id: text("id").primaryKey(),
|
|
915
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
916
|
+
date: text("date").notNull(),
|
|
917
|
+
eventName: text("event_name").notNull(),
|
|
918
|
+
channelGroup: text("channel_group").notNull(),
|
|
919
|
+
source: text("source").notNull(),
|
|
920
|
+
medium: text("medium").notNull(),
|
|
921
|
+
hostName: text("host_name").notNull(),
|
|
922
|
+
landingPage: text("landing_page").notNull(),
|
|
923
|
+
landingPageNormalized: text("landing_page_normalized"),
|
|
924
|
+
attributionScope: text("attribution_scope").$type().notNull(),
|
|
925
|
+
eventCount: integer("event_count").notNull().default(0),
|
|
926
|
+
syncedAt: text("synced_at").notNull(),
|
|
927
|
+
syncRunId: text("sync_run_id").references(() => runs.id, { onDelete: "cascade" }),
|
|
928
|
+
createdAt: text("created_at").notNull()
|
|
929
|
+
}, (table) => [
|
|
930
|
+
uniqueIndex("idx_ga_lead_events_daily_grain").on(
|
|
931
|
+
table.projectId,
|
|
932
|
+
table.date,
|
|
933
|
+
table.eventName,
|
|
934
|
+
table.channelGroup,
|
|
935
|
+
table.source,
|
|
936
|
+
table.medium,
|
|
937
|
+
table.hostName,
|
|
938
|
+
table.landingPage,
|
|
939
|
+
table.attributionScope
|
|
940
|
+
),
|
|
941
|
+
index("idx_ga_lead_events_daily_project_date").on(table.projectId, table.date),
|
|
942
|
+
index("idx_ga_lead_events_daily_project_channel").on(table.projectId, table.date, table.channelGroup),
|
|
943
|
+
index("idx_ga_lead_events_daily_project_event").on(table.projectId, table.date, table.eventName),
|
|
944
|
+
index("idx_ga_lead_events_daily_project_page").on(table.projectId, table.date, table.landingPageNormalized),
|
|
945
|
+
check("chk_ga_lead_events_daily_count", sql`${table.eventCount} >= 0`),
|
|
946
|
+
check(
|
|
947
|
+
"chk_ga_lead_events_daily_scope",
|
|
948
|
+
sql`${table.attributionScope} IN ('landing-page', 'channel')`
|
|
949
|
+
)
|
|
950
|
+
]);
|
|
951
|
+
var gaMeasurementSyncStates = sqliteTable("ga_measurement_sync_state", {
|
|
952
|
+
projectId: text("project_id").primaryKey().references(() => projects.id, { onDelete: "cascade" }),
|
|
953
|
+
acquisitionStatus: text("acquisition_status").$type().notNull().default("never-synced"),
|
|
954
|
+
acquisitionError: text("acquisition_error"),
|
|
955
|
+
acquisitionSyncedAt: text("acquisition_synced_at"),
|
|
956
|
+
leadStatus: text("lead_status").$type().notNull().default("never-synced"),
|
|
957
|
+
leadError: text("lead_error"),
|
|
958
|
+
leadSyncedAt: text("lead_synced_at"),
|
|
959
|
+
leadAttributionScope: text("lead_attribution_scope").$type(),
|
|
960
|
+
updatedAt: text("updated_at").notNull()
|
|
961
|
+
}, (table) => [
|
|
962
|
+
check(
|
|
963
|
+
"chk_ga_measurement_sync_acquisition_status",
|
|
964
|
+
sql`${table.acquisitionStatus} IN ('never-synced', 'ready', 'error')`
|
|
965
|
+
),
|
|
966
|
+
check(
|
|
967
|
+
"chk_ga_measurement_sync_lead_status",
|
|
968
|
+
sql`${table.leadStatus} IN ('never-synced', 'ready', 'error')`
|
|
969
|
+
),
|
|
970
|
+
check(
|
|
971
|
+
"chk_ga_measurement_sync_lead_scope",
|
|
972
|
+
sql`${table.leadAttributionScope} IS NULL OR ${table.leadAttributionScope} IN ('landing-page', 'channel')`
|
|
973
|
+
)
|
|
974
|
+
]);
|
|
865
975
|
var gaAiReferrals = sqliteTable("ga_ai_referrals", {
|
|
866
976
|
id: text("id").primaryKey(),
|
|
867
977
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
@@ -1309,6 +1419,45 @@ var discoverySessions = sqliteTable("discovery_sessions", {
|
|
|
1309
1419
|
index("idx_discovery_sessions_project_created").on(table.projectId, table.createdAt),
|
|
1310
1420
|
index("idx_discovery_sessions_run").on(table.runId)
|
|
1311
1421
|
]);
|
|
1422
|
+
var researchRuns = sqliteTable("research_runs", {
|
|
1423
|
+
id: text("id").primaryKey(),
|
|
1424
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1425
|
+
status: text("status").notNull().default("queued"),
|
|
1426
|
+
provider: text("provider").notNull(),
|
|
1427
|
+
requestedModel: text("requested_model"),
|
|
1428
|
+
resolvedModel: text("resolved_model").notNull(),
|
|
1429
|
+
location: text("location", { mode: "json" }).$type(),
|
|
1430
|
+
totalQueries: integer("total_queries").notNull(),
|
|
1431
|
+
completedQueries: integer("completed_queries").notNull().default(0),
|
|
1432
|
+
failedQueries: integer("failed_queries").notNull().default(0),
|
|
1433
|
+
idempotencyKey: text("idempotency_key"),
|
|
1434
|
+
requestHash: text("request_hash"),
|
|
1435
|
+
error: text("error"),
|
|
1436
|
+
startedAt: text("started_at"),
|
|
1437
|
+
finishedAt: text("finished_at"),
|
|
1438
|
+
createdAt: text("created_at").notNull()
|
|
1439
|
+
}, (table) => [index("idx_research_runs_project_created").on(table.projectId, table.createdAt), index("idx_research_runs_status").on(table.status), uniqueIndex("idx_research_runs_project_idempotency").on(table.projectId, table.idempotencyKey)]);
|
|
1440
|
+
var researchRunQueries = sqliteTable("research_run_queries", {
|
|
1441
|
+
id: text("id").primaryKey(),
|
|
1442
|
+
researchRunId: text("research_run_id").notNull().references(() => researchRuns.id, { onDelete: "cascade" }),
|
|
1443
|
+
position: integer("position").notNull(),
|
|
1444
|
+
queryText: text("query_text").notNull(),
|
|
1445
|
+
status: text("status").notNull().default("queued"),
|
|
1446
|
+
requestedModel: text("requested_model"),
|
|
1447
|
+
resolvedModel: text("resolved_model").notNull(),
|
|
1448
|
+
servedModel: text("served_model"),
|
|
1449
|
+
answerText: text("answer_text"),
|
|
1450
|
+
groundingSources: text("grounding_sources", { mode: "json" }).$type().notNull().default([]),
|
|
1451
|
+
citedDomains: text("cited_domains", { mode: "json" }).$type().notNull().default([]),
|
|
1452
|
+
searchQueries: text("search_queries", { mode: "json" }).$type().notNull().default([]),
|
|
1453
|
+
answerMentioned: integer("answer_mentioned", { mode: "boolean" }),
|
|
1454
|
+
citationState: text("citation_state"),
|
|
1455
|
+
rawResponse: text("raw_response", { mode: "json" }).$type(),
|
|
1456
|
+
error: text("error"),
|
|
1457
|
+
startedAt: text("started_at"),
|
|
1458
|
+
finishedAt: text("finished_at"),
|
|
1459
|
+
createdAt: text("created_at").notNull()
|
|
1460
|
+
}, (table) => [index("idx_research_run_queries_run").on(table.researchRunId), uniqueIndex("idx_research_run_queries_run_position").on(table.researchRunId, table.position)]);
|
|
1312
1461
|
var discoveryProbes = sqliteTable("discovery_probes", {
|
|
1313
1462
|
id: text("id").primaryKey(),
|
|
1314
1463
|
sessionId: text("session_id").notNull().references(() => discoverySessions.id, { onDelete: "cascade" }),
|
|
@@ -1715,7 +1864,7 @@ function parseJsonColumn(value, fallback) {
|
|
|
1715
1864
|
}
|
|
1716
1865
|
|
|
1717
1866
|
// ../db/src/migrate.ts
|
|
1718
|
-
import { sql } from "drizzle-orm";
|
|
1867
|
+
import { sql as sql2 } from "drizzle-orm";
|
|
1719
1868
|
var MIGRATION_SQL = `
|
|
1720
1869
|
CREATE TABLE IF NOT EXISTS projects (
|
|
1721
1870
|
id TEXT PRIMARY KEY,
|
|
@@ -1858,14 +2007,14 @@ CREATE TABLE IF NOT EXISTS _migrations (
|
|
|
1858
2007
|
);
|
|
1859
2008
|
`;
|
|
1860
2009
|
function relinkOrphanedSnapshotQueryIds(tx) {
|
|
1861
|
-
const orphans = tx.all(
|
|
2010
|
+
const orphans = tx.all(sql2`
|
|
1862
2011
|
SELECT qs.id AS snapId, qs.query_text AS text, r.project_id AS projectId
|
|
1863
2012
|
FROM query_snapshots qs
|
|
1864
2013
|
JOIN runs r ON r.id = qs.run_id
|
|
1865
2014
|
WHERE qs.query_id IS NULL AND qs.query_text IS NOT NULL
|
|
1866
2015
|
`);
|
|
1867
2016
|
if (orphans.length === 0) return;
|
|
1868
|
-
const queryRows = tx.all(
|
|
2017
|
+
const queryRows = tx.all(sql2`SELECT id, project_id AS projectId, query FROM queries`);
|
|
1869
2018
|
const byProject = /* @__PURE__ */ new Map();
|
|
1870
2019
|
for (const row of queryRows) {
|
|
1871
2020
|
let perProject = byProject.get(row.projectId);
|
|
@@ -1879,12 +2028,12 @@ function relinkOrphanedSnapshotQueryIds(tx) {
|
|
|
1879
2028
|
for (const orphan of orphans) {
|
|
1880
2029
|
const queryId = byProject.get(orphan.projectId)?.get(normalizeQueryText(orphan.text));
|
|
1881
2030
|
if (queryId) {
|
|
1882
|
-
tx.run(
|
|
2031
|
+
tx.run(sql2`UPDATE query_snapshots SET query_id = ${queryId} WHERE id = ${orphan.snapId}`);
|
|
1883
2032
|
}
|
|
1884
2033
|
}
|
|
1885
2034
|
}
|
|
1886
2035
|
function backfillQuerySnapshotServedModel(tx) {
|
|
1887
|
-
tx.run(
|
|
2036
|
+
tx.run(sql2.raw(`
|
|
1888
2037
|
UPDATE query_snapshots
|
|
1889
2038
|
SET served_model = json_extract(raw_response, '$.apiResponse.model')
|
|
1890
2039
|
WHERE served_model IS NULL
|
|
@@ -2669,10 +2818,10 @@ var MIGRATION_VERSIONS = [
|
|
|
2669
2818
|
run: (tx) => {
|
|
2670
2819
|
if (!tableExists(tx, "ga_ai_referrals")) return;
|
|
2671
2820
|
if (!columnExists(tx, "ga_ai_referrals", "channel_group")) {
|
|
2672
|
-
tx.run(
|
|
2821
|
+
tx.run(sql2.raw(`ALTER TABLE ga_ai_referrals ADD COLUMN channel_group TEXT NOT NULL DEFAULT '(not set)'`));
|
|
2673
2822
|
}
|
|
2674
|
-
tx.run(
|
|
2675
|
-
tx.run(
|
|
2823
|
+
tx.run(sql2.raw(`DROP INDEX IF EXISTS idx_ga_ai_ref_unique_v3`));
|
|
2824
|
+
tx.run(sql2.raw(`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_ai_ref_unique_v4
|
|
2676
2825
|
ON ga_ai_referrals(project_id, date, source, medium, source_dimension, channel_group, landing_page)`));
|
|
2677
2826
|
}
|
|
2678
2827
|
},
|
|
@@ -3125,11 +3274,11 @@ var MIGRATION_VERSIONS = [
|
|
|
3125
3274
|
statements: [],
|
|
3126
3275
|
run: (db) => {
|
|
3127
3276
|
if (tableExists(db, "google_connections") && !columnExists(db, "google_connections", "created_by_project_id")) {
|
|
3128
|
-
db.run(
|
|
3277
|
+
db.run(sql2.raw(
|
|
3129
3278
|
`ALTER TABLE google_connections ADD COLUMN created_by_project_id TEXT REFERENCES projects(id) ON DELETE SET NULL`
|
|
3130
3279
|
));
|
|
3131
|
-
db.run(
|
|
3132
|
-
db.run(
|
|
3280
|
+
db.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_google_conn_project ON google_connections(created_by_project_id)`));
|
|
3281
|
+
db.run(sql2.raw(
|
|
3133
3282
|
`UPDATE google_connections
|
|
3134
3283
|
SET created_by_project_id = (
|
|
3135
3284
|
SELECT p.id FROM projects p
|
|
@@ -3141,11 +3290,11 @@ var MIGRATION_VERSIONS = [
|
|
|
3141
3290
|
));
|
|
3142
3291
|
}
|
|
3143
3292
|
if (tableExists(db, "bing_connections") && !columnExists(db, "bing_connections", "created_by_project_id")) {
|
|
3144
|
-
db.run(
|
|
3293
|
+
db.run(sql2.raw(
|
|
3145
3294
|
`ALTER TABLE bing_connections ADD COLUMN created_by_project_id TEXT REFERENCES projects(id) ON DELETE SET NULL`
|
|
3146
3295
|
));
|
|
3147
|
-
db.run(
|
|
3148
|
-
db.run(
|
|
3296
|
+
db.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_bing_conn_project ON bing_connections(created_by_project_id)`));
|
|
3297
|
+
db.run(sql2.raw(
|
|
3149
3298
|
`UPDATE bing_connections
|
|
3150
3299
|
SET created_by_project_id = (
|
|
3151
3300
|
SELECT p.id FROM projects p
|
|
@@ -3515,10 +3664,10 @@ var MIGRATION_VERSIONS = [
|
|
|
3515
3664
|
run: (db) => {
|
|
3516
3665
|
if (!tableExists(db, "health_snapshots")) return;
|
|
3517
3666
|
if (!columnExists(db, "health_snapshots", "overall_mention_rate")) {
|
|
3518
|
-
db.run(
|
|
3667
|
+
db.run(sql2.raw(`ALTER TABLE health_snapshots ADD COLUMN overall_mention_rate TEXT`));
|
|
3519
3668
|
}
|
|
3520
3669
|
if (!columnExists(db, "health_snapshots", "mentioned_pairs")) {
|
|
3521
|
-
db.run(
|
|
3670
|
+
db.run(sql2.raw(`ALTER TABLE health_snapshots ADD COLUMN mentioned_pairs INTEGER`));
|
|
3522
3671
|
}
|
|
3523
3672
|
}
|
|
3524
3673
|
},
|
|
@@ -3539,7 +3688,7 @@ var MIGRATION_VERSIONS = [
|
|
|
3539
3688
|
];
|
|
3540
3689
|
for (const col of cols) {
|
|
3541
3690
|
if (!columnExists(db, "gbp_locations", col)) {
|
|
3542
|
-
db.run(
|
|
3691
|
+
db.run(sql2.raw(`ALTER TABLE gbp_locations ADD COLUMN ${col} TEXT`));
|
|
3543
3692
|
}
|
|
3544
3693
|
}
|
|
3545
3694
|
}
|
|
@@ -3735,9 +3884,9 @@ var MIGRATION_VERSIONS = [
|
|
|
3735
3884
|
run: (tx) => {
|
|
3736
3885
|
if (!tableExists(tx, "ga_ai_referrals")) return;
|
|
3737
3886
|
if (!columnExists(tx, "ga_ai_referrals", "traffic_class")) {
|
|
3738
|
-
tx.run(
|
|
3887
|
+
tx.run(sql2.raw(`ALTER TABLE ga_ai_referrals ADD COLUMN traffic_class TEXT NOT NULL DEFAULT 'organic'`));
|
|
3739
3888
|
}
|
|
3740
|
-
tx.run(
|
|
3889
|
+
tx.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_ga_ai_ref_traffic_class ON ga_ai_referrals(project_id, date, traffic_class)`));
|
|
3741
3890
|
}
|
|
3742
3891
|
},
|
|
3743
3892
|
{
|
|
@@ -4021,14 +4170,98 @@ var MIGRATION_VERSIONS = [
|
|
|
4021
4170
|
`CREATE INDEX IF NOT EXISTS idx_gsc_query_daily_totals_query ON gsc_query_daily_totals(query)`,
|
|
4022
4171
|
`CREATE INDEX IF NOT EXISTS idx_gsc_query_daily_totals_run ON gsc_query_daily_totals(sync_run_id)`
|
|
4023
4172
|
]
|
|
4173
|
+
},
|
|
4174
|
+
{
|
|
4175
|
+
version: 108,
|
|
4176
|
+
name: "research-query-runs",
|
|
4177
|
+
statements: [
|
|
4178
|
+
`CREATE TABLE IF NOT EXISTS research_runs (id TEXT PRIMARY KEY, project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE, status TEXT NOT NULL DEFAULT 'queued', provider TEXT NOT NULL, requested_model TEXT, resolved_model TEXT NOT NULL, location TEXT, total_queries INTEGER NOT NULL, completed_queries INTEGER NOT NULL DEFAULT 0, failed_queries INTEGER NOT NULL DEFAULT 0, idempotency_key TEXT, request_hash TEXT, error TEXT, started_at TEXT, finished_at TEXT, created_at TEXT NOT NULL)`,
|
|
4179
|
+
`CREATE INDEX IF NOT EXISTS idx_research_runs_project_created ON research_runs(project_id, created_at)`,
|
|
4180
|
+
`CREATE INDEX IF NOT EXISTS idx_research_runs_status ON research_runs(status)`,
|
|
4181
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_research_runs_project_idempotency ON research_runs(project_id, idempotency_key)`,
|
|
4182
|
+
`CREATE TABLE IF NOT EXISTS research_run_queries (id TEXT PRIMARY KEY, research_run_id TEXT NOT NULL REFERENCES research_runs(id) ON DELETE CASCADE, position INTEGER NOT NULL, query_text TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued', requested_model TEXT, resolved_model TEXT NOT NULL, served_model TEXT, answer_text TEXT, grounding_sources TEXT NOT NULL DEFAULT '[]', cited_domains TEXT NOT NULL DEFAULT '[]', search_queries TEXT NOT NULL DEFAULT '[]', answer_mentioned INTEGER, citation_state TEXT, raw_response TEXT, error TEXT, started_at TEXT, finished_at TEXT, created_at TEXT NOT NULL)`,
|
|
4183
|
+
`CREATE INDEX IF NOT EXISTS idx_research_run_queries_run ON research_run_queries(research_run_id)`,
|
|
4184
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_research_run_queries_run_position ON research_run_queries(research_run_id, position)`
|
|
4185
|
+
]
|
|
4186
|
+
},
|
|
4187
|
+
{
|
|
4188
|
+
version: 109,
|
|
4189
|
+
name: "ga-measurement-foundation",
|
|
4190
|
+
statements: [
|
|
4191
|
+
`ALTER TABLE projects ADD COLUMN measurement_config TEXT NOT NULL DEFAULT '{"marketingHosts":[],"brandTerms":[],"leadEventNames":["generate_lead"]}'`,
|
|
4192
|
+
`CREATE TABLE IF NOT EXISTS ga_acquisition_daily (
|
|
4193
|
+
id TEXT PRIMARY KEY,
|
|
4194
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
4195
|
+
date TEXT NOT NULL,
|
|
4196
|
+
channel_group TEXT NOT NULL,
|
|
4197
|
+
source TEXT NOT NULL,
|
|
4198
|
+
medium TEXT NOT NULL,
|
|
4199
|
+
host_name TEXT NOT NULL,
|
|
4200
|
+
landing_page TEXT NOT NULL,
|
|
4201
|
+
landing_page_normalized TEXT,
|
|
4202
|
+
sessions INTEGER NOT NULL DEFAULT 0 CHECK (sessions >= 0),
|
|
4203
|
+
synced_at TEXT NOT NULL,
|
|
4204
|
+
sync_run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
|
4205
|
+
created_at TEXT NOT NULL
|
|
4206
|
+
)`,
|
|
4207
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_acquisition_daily_grain
|
|
4208
|
+
ON ga_acquisition_daily(project_id, date, channel_group, source, medium, host_name, landing_page)`,
|
|
4209
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_acquisition_daily_project_date
|
|
4210
|
+
ON ga_acquisition_daily(project_id, date)`,
|
|
4211
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_acquisition_daily_project_channel
|
|
4212
|
+
ON ga_acquisition_daily(project_id, date, channel_group)`,
|
|
4213
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_acquisition_daily_project_page
|
|
4214
|
+
ON ga_acquisition_daily(project_id, date, landing_page_normalized)`,
|
|
4215
|
+
`CREATE TABLE IF NOT EXISTS ga_lead_events_daily (
|
|
4216
|
+
id TEXT PRIMARY KEY,
|
|
4217
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
4218
|
+
date TEXT NOT NULL,
|
|
4219
|
+
event_name TEXT NOT NULL,
|
|
4220
|
+
channel_group TEXT NOT NULL,
|
|
4221
|
+
source TEXT NOT NULL,
|
|
4222
|
+
medium TEXT NOT NULL,
|
|
4223
|
+
host_name TEXT NOT NULL,
|
|
4224
|
+
landing_page TEXT NOT NULL,
|
|
4225
|
+
landing_page_normalized TEXT,
|
|
4226
|
+
attribution_scope TEXT NOT NULL CHECK (attribution_scope IN ('landing-page', 'channel')),
|
|
4227
|
+
event_count INTEGER NOT NULL DEFAULT 0 CHECK (event_count >= 0),
|
|
4228
|
+
synced_at TEXT NOT NULL,
|
|
4229
|
+
sync_run_id TEXT REFERENCES runs(id) ON DELETE CASCADE,
|
|
4230
|
+
created_at TEXT NOT NULL
|
|
4231
|
+
)`,
|
|
4232
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_lead_events_daily_grain
|
|
4233
|
+
ON ga_lead_events_daily(project_id, date, event_name, channel_group, source, medium, host_name, landing_page, attribution_scope)`,
|
|
4234
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_lead_events_daily_project_date
|
|
4235
|
+
ON ga_lead_events_daily(project_id, date)`,
|
|
4236
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_lead_events_daily_project_channel
|
|
4237
|
+
ON ga_lead_events_daily(project_id, date, channel_group)`,
|
|
4238
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_lead_events_daily_project_event
|
|
4239
|
+
ON ga_lead_events_daily(project_id, date, event_name)`,
|
|
4240
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_lead_events_daily_project_page
|
|
4241
|
+
ON ga_lead_events_daily(project_id, date, landing_page_normalized)`,
|
|
4242
|
+
`CREATE TABLE IF NOT EXISTS ga_measurement_sync_state (
|
|
4243
|
+
project_id TEXT PRIMARY KEY REFERENCES projects(id) ON DELETE CASCADE,
|
|
4244
|
+
acquisition_status TEXT NOT NULL DEFAULT 'never-synced'
|
|
4245
|
+
CHECK (acquisition_status IN ('never-synced', 'ready', 'error')),
|
|
4246
|
+
acquisition_error TEXT,
|
|
4247
|
+
acquisition_synced_at TEXT,
|
|
4248
|
+
lead_status TEXT NOT NULL DEFAULT 'never-synced'
|
|
4249
|
+
CHECK (lead_status IN ('never-synced', 'ready', 'error')),
|
|
4250
|
+
lead_error TEXT,
|
|
4251
|
+
lead_synced_at TEXT,
|
|
4252
|
+
lead_attribution_scope TEXT
|
|
4253
|
+
CHECK (lead_attribution_scope IS NULL OR lead_attribution_scope IN ('landing-page', 'channel')),
|
|
4254
|
+
updated_at TEXT NOT NULL
|
|
4255
|
+
)`
|
|
4256
|
+
]
|
|
4024
4257
|
}
|
|
4025
4258
|
];
|
|
4026
4259
|
function rebuildBacklinkTableWithSource(tx, table) {
|
|
4027
4260
|
if (!tableExists(tx, table)) return;
|
|
4028
4261
|
if (columnExists(tx, table, "source")) return;
|
|
4029
4262
|
if (table === "backlink_domains") {
|
|
4030
|
-
tx.run(
|
|
4031
|
-
tx.run(
|
|
4263
|
+
tx.run(sql2.raw(`DROP TABLE IF EXISTS backlink_domains_v78`));
|
|
4264
|
+
tx.run(sql2.raw(`CREATE TABLE backlink_domains_v78 (
|
|
4032
4265
|
id TEXT PRIMARY KEY,
|
|
4033
4266
|
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
4034
4267
|
release_sync_id TEXT REFERENCES cc_release_syncs(id) ON DELETE CASCADE,
|
|
@@ -4039,24 +4272,24 @@ function rebuildBacklinkTableWithSource(tx, table) {
|
|
|
4039
4272
|
num_hosts INTEGER NOT NULL,
|
|
4040
4273
|
created_at TEXT NOT NULL
|
|
4041
4274
|
)`));
|
|
4042
|
-
tx.run(
|
|
4275
|
+
tx.run(sql2.raw(`INSERT INTO backlink_domains_v78
|
|
4043
4276
|
(id, project_id, release_sync_id, source, release, target_domain, linking_domain, num_hosts, created_at)
|
|
4044
4277
|
SELECT bd.id, bd.project_id,
|
|
4045
4278
|
CASE WHEN bd.release_sync_id IN (SELECT id FROM cc_release_syncs) THEN bd.release_sync_id ELSE NULL END,
|
|
4046
4279
|
'commoncrawl', bd.release, bd.target_domain, bd.linking_domain, bd.num_hosts, bd.created_at
|
|
4047
4280
|
FROM backlink_domains bd
|
|
4048
4281
|
WHERE bd.project_id IN (SELECT id FROM projects)`));
|
|
4049
|
-
tx.run(
|
|
4050
|
-
tx.run(
|
|
4051
|
-
tx.run(
|
|
4052
|
-
tx.run(
|
|
4053
|
-
tx.run(
|
|
4054
|
-
tx.run(
|
|
4055
|
-
tx.run(
|
|
4282
|
+
tx.run(sql2.raw(`DROP TABLE backlink_domains`));
|
|
4283
|
+
tx.run(sql2.raw(`ALTER TABLE backlink_domains_v78 RENAME TO backlink_domains`));
|
|
4284
|
+
tx.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_backlink_domains_project ON backlink_domains(project_id)`));
|
|
4285
|
+
tx.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_backlink_domains_release_sync ON backlink_domains(release_sync_id)`));
|
|
4286
|
+
tx.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_backlink_domains_project_release ON backlink_domains(project_id, release)`));
|
|
4287
|
+
tx.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_backlink_domains_hosts ON backlink_domains(num_hosts)`));
|
|
4288
|
+
tx.run(sql2.raw(`CREATE UNIQUE INDEX IF NOT EXISTS idx_backlink_domains_unique ON backlink_domains(project_id, source, release, linking_domain)`));
|
|
4056
4289
|
return;
|
|
4057
4290
|
}
|
|
4058
|
-
tx.run(
|
|
4059
|
-
tx.run(
|
|
4291
|
+
tx.run(sql2.raw(`DROP TABLE IF EXISTS backlink_summaries_v78`));
|
|
4292
|
+
tx.run(sql2.raw(`CREATE TABLE backlink_summaries_v78 (
|
|
4060
4293
|
id TEXT PRIMARY KEY,
|
|
4061
4294
|
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
4062
4295
|
release_sync_id TEXT REFERENCES cc_release_syncs(id) ON DELETE CASCADE,
|
|
@@ -4069,17 +4302,17 @@ function rebuildBacklinkTableWithSource(tx, table) {
|
|
|
4069
4302
|
queried_at TEXT NOT NULL,
|
|
4070
4303
|
created_at TEXT NOT NULL
|
|
4071
4304
|
)`));
|
|
4072
|
-
tx.run(
|
|
4305
|
+
tx.run(sql2.raw(`INSERT INTO backlink_summaries_v78
|
|
4073
4306
|
(id, project_id, release_sync_id, source, release, target_domain, total_linking_domains, total_hosts, top_10_hosts_share, queried_at, created_at)
|
|
4074
4307
|
SELECT bs.id, bs.project_id,
|
|
4075
4308
|
CASE WHEN bs.release_sync_id IN (SELECT id FROM cc_release_syncs) THEN bs.release_sync_id ELSE NULL END,
|
|
4076
4309
|
'commoncrawl', bs.release, bs.target_domain, bs.total_linking_domains, bs.total_hosts, bs.top_10_hosts_share, bs.queried_at, bs.created_at
|
|
4077
4310
|
FROM backlink_summaries bs
|
|
4078
4311
|
WHERE bs.project_id IN (SELECT id FROM projects)`));
|
|
4079
|
-
tx.run(
|
|
4080
|
-
tx.run(
|
|
4081
|
-
tx.run(
|
|
4082
|
-
tx.run(
|
|
4312
|
+
tx.run(sql2.raw(`DROP TABLE backlink_summaries`));
|
|
4313
|
+
tx.run(sql2.raw(`ALTER TABLE backlink_summaries_v78 RENAME TO backlink_summaries`));
|
|
4314
|
+
tx.run(sql2.raw(`CREATE UNIQUE INDEX IF NOT EXISTS idx_backlink_summaries_project_release ON backlink_summaries(project_id, source, release)`));
|
|
4315
|
+
tx.run(sql2.raw(`CREATE INDEX IF NOT EXISTS idx_backlink_summaries_project ON backlink_summaries(project_id)`));
|
|
4083
4316
|
}
|
|
4084
4317
|
function addBacklinkSourceDiscriminator(tx) {
|
|
4085
4318
|
rebuildBacklinkTableWithSource(tx, "backlink_domains");
|
|
@@ -4092,25 +4325,25 @@ function isDuplicateColumnError(err) {
|
|
|
4092
4325
|
return false;
|
|
4093
4326
|
}
|
|
4094
4327
|
function columnExists(db, table, column) {
|
|
4095
|
-
const rows = db.all(
|
|
4328
|
+
const rows = db.all(sql2.raw(
|
|
4096
4329
|
`SELECT COUNT(*) as c FROM pragma_table_info('${table}') WHERE name = '${column}'`
|
|
4097
4330
|
));
|
|
4098
4331
|
return (rows[0]?.c ?? 0) > 0;
|
|
4099
4332
|
}
|
|
4100
4333
|
function tableExists(db, table) {
|
|
4101
|
-
const rows = db.all(
|
|
4334
|
+
const rows = db.all(sql2.raw(
|
|
4102
4335
|
`SELECT COUNT(*) as c FROM sqlite_master WHERE type = 'table' AND name = '${table}'`
|
|
4103
4336
|
));
|
|
4104
4337
|
return (rows[0]?.c ?? 0) > 0;
|
|
4105
4338
|
}
|
|
4106
4339
|
function tableIsEmpty(db, table) {
|
|
4107
|
-
const rows = db.all(
|
|
4340
|
+
const rows = db.all(sql2.raw(`SELECT COUNT(*) as c FROM ${table}`));
|
|
4108
4341
|
return (rows[0]?.c ?? 0) === 0;
|
|
4109
4342
|
}
|
|
4110
4343
|
function backfillGaAiReferralTrafficClass(tx) {
|
|
4111
4344
|
if (!tableExists(tx, "ga_ai_referrals")) return 0;
|
|
4112
4345
|
if (!columnExists(tx, "ga_ai_referrals", "traffic_class")) return 0;
|
|
4113
|
-
const rows = tx.all(
|
|
4346
|
+
const rows = tx.all(sql2.raw(
|
|
4114
4347
|
`SELECT id, source, medium, channel_group, landing_page, traffic_class FROM ga_ai_referrals`
|
|
4115
4348
|
));
|
|
4116
4349
|
let updated = 0;
|
|
@@ -4122,7 +4355,7 @@ function backfillGaAiReferralTrafficClass(tx) {
|
|
|
4122
4355
|
landingPage: row.landing_page
|
|
4123
4356
|
});
|
|
4124
4357
|
if (next === row.traffic_class) continue;
|
|
4125
|
-
tx.run(
|
|
4358
|
+
tx.run(sql2`UPDATE ga_ai_referrals SET traffic_class = ${next} WHERE id = ${row.id}`);
|
|
4126
4359
|
updated += 1;
|
|
4127
4360
|
}
|
|
4128
4361
|
return updated;
|
|
@@ -4136,28 +4369,28 @@ function normalizeLegacyQuerySchema(db) {
|
|
|
4136
4369
|
if (!tableIsEmpty(db, "queries")) {
|
|
4137
4370
|
throw new Error("Cannot migrate keywords to queries because both tables contain data");
|
|
4138
4371
|
}
|
|
4139
|
-
db.run(
|
|
4372
|
+
db.run(sql2.raw(`DROP TABLE queries`));
|
|
4140
4373
|
}
|
|
4141
|
-
db.run(
|
|
4142
|
-
db.run(
|
|
4143
|
-
db.run(
|
|
4144
|
-
db.run(
|
|
4374
|
+
db.run(sql2.raw(`DROP INDEX IF EXISTS idx_keywords_project`));
|
|
4375
|
+
db.run(sql2.raw(`DROP INDEX IF EXISTS idx_keywords_project_keyword`));
|
|
4376
|
+
db.run(sql2.raw(`DROP INDEX IF EXISTS idx_snapshots_keyword`));
|
|
4377
|
+
db.run(sql2.raw(`DROP INDEX IF EXISTS idx_insights_keyword_provider`));
|
|
4145
4378
|
if (tableExists(db, "keywords")) {
|
|
4146
|
-
db.run(
|
|
4379
|
+
db.run(sql2.raw(`ALTER TABLE keywords RENAME TO queries`));
|
|
4147
4380
|
}
|
|
4148
4381
|
if (columnExists(db, "queries", "keyword")) {
|
|
4149
|
-
db.run(
|
|
4382
|
+
db.run(sql2.raw(`ALTER TABLE queries RENAME COLUMN keyword TO query`));
|
|
4150
4383
|
}
|
|
4151
4384
|
if (columnExists(db, "query_snapshots", "keyword_id")) {
|
|
4152
|
-
db.run(
|
|
4385
|
+
db.run(sql2.raw(`ALTER TABLE query_snapshots RENAME COLUMN keyword_id TO query_id`));
|
|
4153
4386
|
}
|
|
4154
4387
|
if (columnExists(db, "insights", "keyword")) {
|
|
4155
|
-
db.run(
|
|
4388
|
+
db.run(sql2.raw(`ALTER TABLE insights RENAME COLUMN keyword TO query`));
|
|
4156
4389
|
}
|
|
4157
4390
|
}
|
|
4158
4391
|
function dropColumnIfExists(db, table, column) {
|
|
4159
4392
|
try {
|
|
4160
|
-
db.run(
|
|
4393
|
+
db.run(sql2.raw(`ALTER TABLE ${table} DROP COLUMN ${column}`));
|
|
4161
4394
|
} catch (err) {
|
|
4162
4395
|
if (!(err instanceof Error)) throw err;
|
|
4163
4396
|
const msg = err.message;
|
|
@@ -4172,7 +4405,7 @@ function dropColumnIfExists(db, table, column) {
|
|
|
4172
4405
|
function extractLegacyCredentials(db) {
|
|
4173
4406
|
const out = { google: [], ga4: [] };
|
|
4174
4407
|
if (columnExists(db, "google_connections", "access_token")) {
|
|
4175
|
-
const rows = db.all(
|
|
4408
|
+
const rows = db.all(sql2.raw(
|
|
4176
4409
|
`SELECT domain, connection_type, property_id, sitemap_url, access_token, refresh_token, token_expires_at, scopes, created_at, updated_at
|
|
4177
4410
|
FROM google_connections
|
|
4178
4411
|
WHERE refresh_token IS NOT NULL AND refresh_token != ''`
|
|
@@ -4193,7 +4426,7 @@ function extractLegacyCredentials(db) {
|
|
|
4193
4426
|
}
|
|
4194
4427
|
}
|
|
4195
4428
|
if (columnExists(db, "ga_connections", "private_key")) {
|
|
4196
|
-
const rows = db.all(
|
|
4429
|
+
const rows = db.all(sql2.raw(
|
|
4197
4430
|
`SELECT p.name AS project_name, ga.property_id, ga.client_email, ga.private_key, ga.created_at, ga.updated_at
|
|
4198
4431
|
FROM ga_connections ga
|
|
4199
4432
|
INNER JOIN projects p ON p.id = ga.project_id
|
|
@@ -4227,11 +4460,11 @@ function dropLegacyCredentialColumns(db) {
|
|
|
4227
4460
|
}
|
|
4228
4461
|
}
|
|
4229
4462
|
function getAppliedVersion(db) {
|
|
4230
|
-
const rows = db.all(
|
|
4463
|
+
const rows = db.all(sql2`SELECT MAX(version) as max_version FROM _migrations`);
|
|
4231
4464
|
return rows[0]?.max_version ?? 0;
|
|
4232
4465
|
}
|
|
4233
4466
|
function recordMigration(db, version, name) {
|
|
4234
|
-
db.run(
|
|
4467
|
+
db.run(sql2`INSERT OR IGNORE INTO _migrations (version, name) VALUES (${version}, ${name})`);
|
|
4235
4468
|
}
|
|
4236
4469
|
function migrate(db, versions = MIGRATION_VERSIONS) {
|
|
4237
4470
|
db.transaction((tx) => {
|
|
@@ -4239,7 +4472,7 @@ function migrate(db, versions = MIGRATION_VERSIONS) {
|
|
|
4239
4472
|
});
|
|
4240
4473
|
const statements = MIGRATION_SQL.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
4241
4474
|
for (const statement of statements) {
|
|
4242
|
-
db.run(
|
|
4475
|
+
db.run(sql2.raw(statement));
|
|
4243
4476
|
}
|
|
4244
4477
|
const appliedVersion = getAppliedVersion(db);
|
|
4245
4478
|
for (const mv of versions) {
|
|
@@ -4247,7 +4480,7 @@ function migrate(db, versions = MIGRATION_VERSIONS) {
|
|
|
4247
4480
|
db.transaction((tx) => {
|
|
4248
4481
|
for (const statement of mv.statements) {
|
|
4249
4482
|
try {
|
|
4250
|
-
tx.run(
|
|
4483
|
+
tx.run(sql2.raw(statement));
|
|
4251
4484
|
} catch (err) {
|
|
4252
4485
|
if (isDuplicateColumnError(err)) continue;
|
|
4253
4486
|
throw err;
|
|
@@ -6553,11 +6786,11 @@ async function authPlugin(app, opts = {}) {
|
|
|
6553
6786
|
|
|
6554
6787
|
// ../api-routes/src/projects.ts
|
|
6555
6788
|
import crypto4 from "crypto";
|
|
6556
|
-
import { eq as eq3, sql as
|
|
6789
|
+
import { eq as eq3, sql as sql4 } from "drizzle-orm";
|
|
6557
6790
|
|
|
6558
6791
|
// ../api-routes/src/helpers.ts
|
|
6559
6792
|
import crypto3 from "crypto";
|
|
6560
|
-
import { eq as eq2, ne, sql as
|
|
6793
|
+
import { eq as eq2, ne, sql as sql3 } from "drizzle-orm";
|
|
6561
6794
|
function notProbeRun() {
|
|
6562
6795
|
return ne(runs.trigger, RunTriggers.probe);
|
|
6563
6796
|
}
|
|
@@ -6835,6 +7068,7 @@ async function projectRoutes(app, opts) {
|
|
|
6835
7068
|
});
|
|
6836
7069
|
}
|
|
6837
7070
|
const nextAutoExtractBacklinks = body.autoExtractBacklinks !== void 0 ? body.autoExtractBacklinks : existing?.autoExtractBacklinks ?? false;
|
|
7071
|
+
const nextMeasurement = body.measurement ?? existing?.measurement ?? DEFAULT_MEASUREMENT_CONFIG;
|
|
6838
7072
|
const nextAliases = normalizeProjectAliases(body.displayName, body.aliases ?? []);
|
|
6839
7073
|
if (existing) {
|
|
6840
7074
|
const prevAliases = existing.aliases;
|
|
@@ -6851,6 +7085,7 @@ async function projectRoutes(app, opts) {
|
|
|
6851
7085
|
labels: body.labels ?? {},
|
|
6852
7086
|
providers: body.providers ?? [],
|
|
6853
7087
|
providerModels,
|
|
7088
|
+
measurement: nextMeasurement,
|
|
6854
7089
|
locations: nextLocations,
|
|
6855
7090
|
defaultLocation: nextDefaultLocation,
|
|
6856
7091
|
autoExtractBacklinks: nextAutoExtractBacklinks,
|
|
@@ -6886,6 +7121,7 @@ async function projectRoutes(app, opts) {
|
|
|
6886
7121
|
labels: body.labels ?? {},
|
|
6887
7122
|
providers: body.providers ?? [],
|
|
6888
7123
|
providerModels,
|
|
7124
|
+
measurement: nextMeasurement,
|
|
6889
7125
|
locations: nextLocations,
|
|
6890
7126
|
defaultLocation: nextDefaultLocation,
|
|
6891
7127
|
autoExtractBacklinks: nextAutoExtractBacklinks,
|
|
@@ -6919,12 +7155,12 @@ async function projectRoutes(app, opts) {
|
|
|
6919
7155
|
const project = resolveProject(app.db, request.params.name);
|
|
6920
7156
|
const pid = project.id;
|
|
6921
7157
|
const count2 = (n) => n ?? 0;
|
|
6922
|
-
const queryCount = app.db.select({ n:
|
|
6923
|
-
const competitorCount = app.db.select({ n:
|
|
6924
|
-
const runCount = app.db.select({ n:
|
|
6925
|
-
const snapshotCount = app.db.select({ n:
|
|
6926
|
-
const insightCount = app.db.select({ n:
|
|
6927
|
-
const auditLogCount = app.db.select({ n:
|
|
7158
|
+
const queryCount = app.db.select({ n: sql4`count(*)` }).from(queries).where(eq3(queries.projectId, pid)).get();
|
|
7159
|
+
const competitorCount = app.db.select({ n: sql4`count(*)` }).from(competitors).where(eq3(competitors.projectId, pid)).get();
|
|
7160
|
+
const runCount = app.db.select({ n: sql4`count(*)` }).from(runs).where(eq3(runs.projectId, pid)).get();
|
|
7161
|
+
const snapshotCount = app.db.select({ n: sql4`count(*)` }).from(querySnapshots).innerJoin(runs, eq3(querySnapshots.runId, runs.id)).where(eq3(runs.projectId, pid)).get();
|
|
7162
|
+
const insightCount = app.db.select({ n: sql4`count(*)` }).from(insights).where(eq3(insights.projectId, pid)).get();
|
|
7163
|
+
const auditLogCount = app.db.select({ n: sql4`count(*)` }).from(auditLog).where(eq3(auditLog.projectId, pid)).get();
|
|
6928
7164
|
return reply.send({
|
|
6929
7165
|
project: { id: project.id, name: project.name },
|
|
6930
7166
|
cascadeRows: {
|
|
@@ -7058,6 +7294,7 @@ async function projectRoutes(app, opts) {
|
|
|
7058
7294
|
competitors: comps.map((c) => c.domain),
|
|
7059
7295
|
providers: project.providers,
|
|
7060
7296
|
...Object.keys(project.providerModels).length > 0 ? { providerModels: project.providerModels } : {},
|
|
7297
|
+
measurement: project.measurement,
|
|
7061
7298
|
locations: project.locations,
|
|
7062
7299
|
...project.defaultLocation ? { defaultLocation: project.defaultLocation } : {},
|
|
7063
7300
|
...project.autoExtractBacklinks ? { autoExtractBacklinks: true } : {},
|
|
@@ -7106,6 +7343,7 @@ function formatProject(row) {
|
|
|
7106
7343
|
labels: row.labels,
|
|
7107
7344
|
providers: row.providers,
|
|
7108
7345
|
providerModels: row.providerModels,
|
|
7346
|
+
measurement: row.measurement,
|
|
7109
7347
|
locations: row.locations,
|
|
7110
7348
|
defaultLocation: row.defaultLocation,
|
|
7111
7349
|
autoExtractBacklinks: row.autoExtractBacklinks,
|
|
@@ -7125,7 +7363,7 @@ function aliasArraysEqual(a, b) {
|
|
|
7125
7363
|
|
|
7126
7364
|
// ../api-routes/src/queries.ts
|
|
7127
7365
|
import crypto6 from "crypto";
|
|
7128
|
-
import { and, eq as eq5, inArray as inArray2, sql as
|
|
7366
|
+
import { and, eq as eq5, inArray as inArray2, sql as sql5 } from "drizzle-orm";
|
|
7129
7367
|
|
|
7130
7368
|
// ../api-routes/src/query-replace.ts
|
|
7131
7369
|
import crypto5 from "crypto";
|
|
@@ -7247,9 +7485,9 @@ async function queryRoutes(app, opts) {
|
|
|
7247
7485
|
let snapshotsDetached = 0;
|
|
7248
7486
|
let affectedQueries = 0;
|
|
7249
7487
|
if (removedIds.length > 0) {
|
|
7250
|
-
const snapshotCount = app.db.select({ n:
|
|
7488
|
+
const snapshotCount = app.db.select({ n: sql5`count(*)` }).from(querySnapshots).where(inArray2(querySnapshots.queryId, removedIds)).get();
|
|
7251
7489
|
snapshotsDetached = snapshotCount?.n ?? 0;
|
|
7252
|
-
const distinctAffected = app.db.select({ n:
|
|
7490
|
+
const distinctAffected = app.db.select({ n: sql5`count(distinct ${querySnapshots.queryId})` }).from(querySnapshots).where(inArray2(querySnapshots.queryId, removedIds)).get();
|
|
7253
7491
|
affectedQueries = distinctAffected?.n ?? 0;
|
|
7254
7492
|
}
|
|
7255
7493
|
return reply.send({
|
|
@@ -7658,7 +7896,7 @@ function parseCompetitorBatch(value) {
|
|
|
7658
7896
|
|
|
7659
7897
|
// ../api-routes/src/runs.ts
|
|
7660
7898
|
import crypto9 from "crypto";
|
|
7661
|
-
import { and as and4, eq as eq8, asc, desc, or as or2, sql as
|
|
7899
|
+
import { and as and4, eq as eq8, asc, desc, or as or2, sql as sql6 } from "drizzle-orm";
|
|
7662
7900
|
import { gte } from "drizzle-orm";
|
|
7663
7901
|
|
|
7664
7902
|
// ../api-routes/src/run-queue.ts
|
|
@@ -7833,7 +8071,7 @@ async function runRoutes(app, opts) {
|
|
|
7833
8071
|
});
|
|
7834
8072
|
app.get("/projects/:name/runs/latest", async (request, reply) => {
|
|
7835
8073
|
const project = resolveProject(app.db, request.params.name);
|
|
7836
|
-
const countRow = app.db.select({ count:
|
|
8074
|
+
const countRow = app.db.select({ count: sql6`count(*)` }).from(runs).where(and4(eq8(runs.projectId, project.id), notProbeRun())).get();
|
|
7837
8075
|
const totalRuns = countRow?.count ?? 0;
|
|
7838
8076
|
const latestRun = app.db.select().from(runs).where(and4(eq8(runs.projectId, project.id), notProbeRun())).orderBy(desc(runs.createdAt), desc(runs.id)).limit(1).get();
|
|
7839
8077
|
if (!latestRun) {
|
|
@@ -8439,6 +8677,7 @@ async function applyRoutes(app, opts) {
|
|
|
8439
8677
|
labels: config.metadata.labels,
|
|
8440
8678
|
providers: config.spec.providers ?? [],
|
|
8441
8679
|
providerModels,
|
|
8680
|
+
measurement: config.spec.measurement,
|
|
8442
8681
|
locations: config.spec.locations ?? [],
|
|
8443
8682
|
defaultLocation: config.spec.defaultLocation ?? null,
|
|
8444
8683
|
autoExtractBacklinks: config.spec.autoExtractBacklinks ?? false,
|
|
@@ -8468,6 +8707,7 @@ async function applyRoutes(app, opts) {
|
|
|
8468
8707
|
labels: config.metadata.labels,
|
|
8469
8708
|
providers: config.spec.providers ?? [],
|
|
8470
8709
|
providerModels,
|
|
8710
|
+
measurement: config.spec.measurement,
|
|
8471
8711
|
locations: config.spec.locations ?? [],
|
|
8472
8712
|
defaultLocation: config.spec.defaultLocation ?? null,
|
|
8473
8713
|
autoExtractBacklinks: config.spec.autoExtractBacklinks ?? false,
|
|
@@ -8593,6 +8833,7 @@ async function applyRoutes(app, opts) {
|
|
|
8593
8833
|
labels: project.labels,
|
|
8594
8834
|
providers: project.providers,
|
|
8595
8835
|
providerModels: project.providerModels,
|
|
8836
|
+
measurement: project.measurement,
|
|
8596
8837
|
locations: project.locations,
|
|
8597
8838
|
defaultLocation: project.defaultLocation,
|
|
8598
8839
|
autoExtractBacklinks: project.autoExtractBacklinks,
|
|
@@ -9930,7 +10171,7 @@ async function intelligenceRoutes(app) {
|
|
|
9930
10171
|
}
|
|
9931
10172
|
|
|
9932
10173
|
// ../api-routes/src/report.ts
|
|
9933
|
-
import { and as and12, desc as desc7, eq as eq16, gte as gte4, inArray as inArray7, lt as lt2, lte, ne as ne2, or as or3, sql as
|
|
10174
|
+
import { and as and12, desc as desc7, eq as eq16, gte as gte4, inArray as inArray7, lt as lt2, lte, ne as ne2, or as or3, sql as sql8 } from "drizzle-orm";
|
|
9934
10175
|
|
|
9935
10176
|
// ../api-routes/src/content.ts
|
|
9936
10177
|
import crypto12 from "crypto";
|
|
@@ -10671,7 +10912,7 @@ function winnabilityClassRank(winnabilityClass) {
|
|
|
10671
10912
|
}
|
|
10672
10913
|
|
|
10673
10914
|
// ../api-routes/src/gsc-totals.ts
|
|
10674
|
-
import { and as and11, asc as asc3, eq as eq15, sql as
|
|
10915
|
+
import { and as and11, asc as asc3, eq as eq15, sql as sql7 } from "drizzle-orm";
|
|
10675
10916
|
function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
10676
10917
|
const rows = db.select({
|
|
10677
10918
|
date: gscDailyTotals.date,
|
|
@@ -10681,8 +10922,8 @@ function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
|
10681
10922
|
}).from(gscDailyTotals).where(
|
|
10682
10923
|
and11(
|
|
10683
10924
|
eq15(gscDailyTotals.projectId, projectId),
|
|
10684
|
-
|
|
10685
|
-
|
|
10925
|
+
sql7`${gscDailyTotals.date} >= ${startDate}`,
|
|
10926
|
+
sql7`${gscDailyTotals.date} <= ${endDate}`
|
|
10686
10927
|
)
|
|
10687
10928
|
).orderBy(asc3(gscDailyTotals.date)).all();
|
|
10688
10929
|
return rows.map((r) => {
|
|
@@ -10711,8 +10952,8 @@ function readGscQueryDailyRows(db, projectId, startDate, endDate) {
|
|
|
10711
10952
|
}).from(gscQueryDailyTotals).where(
|
|
10712
10953
|
and11(
|
|
10713
10954
|
eq15(gscQueryDailyTotals.projectId, projectId),
|
|
10714
|
-
|
|
10715
|
-
|
|
10955
|
+
sql7`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
10956
|
+
sql7`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
10716
10957
|
)
|
|
10717
10958
|
).all();
|
|
10718
10959
|
return rows.map((r) => {
|
|
@@ -13224,7 +13465,7 @@ function buildGaSection(db, projectId, windowDays) {
|
|
|
13224
13465
|
const aiSessionRows = db.select({
|
|
13225
13466
|
trafficClass: gaAiReferrals.trafficClass,
|
|
13226
13467
|
channelGroup: gaAiReferrals.channelGroup,
|
|
13227
|
-
sessions:
|
|
13468
|
+
sessions: sql8`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`
|
|
13228
13469
|
}).from(gaAiReferrals).where(and12(...aiConditions)).groupBy(gaAiReferrals.trafficClass, gaAiReferrals.channelGroup).all();
|
|
13229
13470
|
let paidAiSessions = 0;
|
|
13230
13471
|
let organicAiSessions = 0;
|
|
@@ -13318,8 +13559,8 @@ function buildAiReferrals(db, projectId, windowDays) {
|
|
|
13318
13559
|
const window = resolveAiReferralWindow(db, projectId, windowDays);
|
|
13319
13560
|
const rows = db.select().from(gaAiReferrals).where(and12(
|
|
13320
13561
|
eq16(gaAiReferrals.projectId, projectId),
|
|
13321
|
-
|
|
13322
|
-
|
|
13562
|
+
sql8`${gaAiReferrals.date} >= ${window.start}`,
|
|
13563
|
+
sql8`${gaAiReferrals.date} <= ${window.end}`
|
|
13323
13564
|
)).all();
|
|
13324
13565
|
if (rows.length === 0) return null;
|
|
13325
13566
|
const dimSessionsByTuple = /* @__PURE__ */ new Map();
|
|
@@ -13395,7 +13636,7 @@ function buildAiReferrals(db, projectId, windowDays) {
|
|
|
13395
13636
|
};
|
|
13396
13637
|
}
|
|
13397
13638
|
function nonSubresourceReferralPathCondition() {
|
|
13398
|
-
return
|
|
13639
|
+
return sql8`
|
|
13399
13640
|
LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/_next/static/%'
|
|
13400
13641
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/assets/%'
|
|
13401
13642
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/static/%'
|
|
@@ -13434,7 +13675,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13434
13675
|
const priorStart = new Date(priorStartMs).toISOString();
|
|
13435
13676
|
const trendStart = new Date(trendStartMs).toISOString();
|
|
13436
13677
|
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
13437
|
-
db.select({ total:
|
|
13678
|
+
db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
13438
13679
|
and12(
|
|
13439
13680
|
eq16(crawlerEventsHourly.projectId, projectId),
|
|
13440
13681
|
eq16(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -13444,7 +13685,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13444
13685
|
).get()?.total ?? 0
|
|
13445
13686
|
);
|
|
13446
13687
|
const sumUnverifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
13447
|
-
db.select({ total:
|
|
13688
|
+
db.select({ total: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
13448
13689
|
and12(
|
|
13449
13690
|
eq16(crawlerEventsHourly.projectId, projectId),
|
|
13450
13691
|
ne2(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -13455,9 +13696,9 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13455
13696
|
);
|
|
13456
13697
|
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => {
|
|
13457
13698
|
const row = db.select({
|
|
13458
|
-
total:
|
|
13459
|
-
paid:
|
|
13460
|
-
organic:
|
|
13699
|
+
total: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
13700
|
+
paid: sql8`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
13701
|
+
organic: sql8`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
13461
13702
|
}).from(aiReferralEventsHourly).where(
|
|
13462
13703
|
and12(
|
|
13463
13704
|
eq16(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -13469,7 +13710,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13469
13710
|
return aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0));
|
|
13470
13711
|
};
|
|
13471
13712
|
const sumUserFetches = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
13472
|
-
db.select({ total:
|
|
13713
|
+
db.select({ total: sql8`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
13473
13714
|
and12(
|
|
13474
13715
|
eq16(aiUserFetchEventsHourly.projectId, projectId),
|
|
13475
13716
|
gte4(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
@@ -13488,7 +13729,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13488
13729
|
const crawlerByOperatorRows = db.select({
|
|
13489
13730
|
operator: crawlerEventsHourly.operator,
|
|
13490
13731
|
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
13491
|
-
hits:
|
|
13732
|
+
hits: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
13492
13733
|
}).from(crawlerEventsHourly).where(
|
|
13493
13734
|
and12(
|
|
13494
13735
|
eq16(crawlerEventsHourly.projectId, projectId),
|
|
@@ -13498,7 +13739,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13498
13739
|
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
13499
13740
|
const crawlerByOperatorPriorRows = db.select({
|
|
13500
13741
|
operator: crawlerEventsHourly.operator,
|
|
13501
|
-
hits:
|
|
13742
|
+
hits: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
13502
13743
|
}).from(crawlerEventsHourly).where(
|
|
13503
13744
|
and12(
|
|
13504
13745
|
eq16(crawlerEventsHourly.projectId, projectId),
|
|
@@ -13509,7 +13750,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13509
13750
|
).groupBy(crawlerEventsHourly.operator).all();
|
|
13510
13751
|
const referralByOperatorRows = db.select({
|
|
13511
13752
|
operator: aiReferralEventsHourly.operator,
|
|
13512
|
-
hits:
|
|
13753
|
+
hits: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
13513
13754
|
}).from(aiReferralEventsHourly).where(
|
|
13514
13755
|
and12(
|
|
13515
13756
|
eq16(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -13520,7 +13761,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13520
13761
|
).groupBy(aiReferralEventsHourly.operator).all();
|
|
13521
13762
|
const userFetchByOperatorRows = db.select({
|
|
13522
13763
|
operator: aiUserFetchEventsHourly.operator,
|
|
13523
|
-
hits:
|
|
13764
|
+
hits: sql8`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
13524
13765
|
}).from(aiUserFetchEventsHourly).where(
|
|
13525
13766
|
and12(
|
|
13526
13767
|
eq16(aiUserFetchEventsHourly.projectId, projectId),
|
|
@@ -13563,8 +13804,8 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13563
13804
|
);
|
|
13564
13805
|
const topPathsRows = db.select({
|
|
13565
13806
|
path: crawlerEventsHourly.pathNormalized,
|
|
13566
|
-
hits:
|
|
13567
|
-
operators:
|
|
13807
|
+
hits: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
13808
|
+
operators: sql8`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
13568
13809
|
}).from(crawlerEventsHourly).where(
|
|
13569
13810
|
and12(
|
|
13570
13811
|
eq16(crawlerEventsHourly.projectId, projectId),
|
|
@@ -13572,7 +13813,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13572
13813
|
gte4(crawlerEventsHourly.tsHour, headlineStart),
|
|
13573
13814
|
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
13574
13815
|
)
|
|
13575
|
-
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc7(
|
|
13816
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc7(sql8`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
13576
13817
|
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
13577
13818
|
path: r.path,
|
|
13578
13819
|
verifiedHits: Number(r.hits),
|
|
@@ -13580,8 +13821,8 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13580
13821
|
}));
|
|
13581
13822
|
const referralProductsRows = db.select({
|
|
13582
13823
|
product: aiReferralEventsHourly.product,
|
|
13583
|
-
arrivals:
|
|
13584
|
-
landingPaths:
|
|
13824
|
+
arrivals: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
13825
|
+
landingPaths: sql8`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
13585
13826
|
}).from(aiReferralEventsHourly).where(
|
|
13586
13827
|
and12(
|
|
13587
13828
|
eq16(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -13589,7 +13830,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13589
13830
|
gte4(aiReferralEventsHourly.tsHour, headlineStart),
|
|
13590
13831
|
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
13591
13832
|
)
|
|
13592
|
-
).groupBy(aiReferralEventsHourly.product).orderBy(desc7(
|
|
13833
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc7(sql8`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
13593
13834
|
const referralProducts = referralProductsRows.map((r) => ({
|
|
13594
13835
|
product: r.product,
|
|
13595
13836
|
arrivals: Number(r.arrivals),
|
|
@@ -13597,8 +13838,8 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13597
13838
|
}));
|
|
13598
13839
|
const topReferralRows = db.select({
|
|
13599
13840
|
path: aiReferralEventsHourly.landingPathNormalized,
|
|
13600
|
-
arrivals:
|
|
13601
|
-
products:
|
|
13841
|
+
arrivals: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
13842
|
+
products: sql8`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
13602
13843
|
}).from(aiReferralEventsHourly).where(
|
|
13603
13844
|
and12(
|
|
13604
13845
|
eq16(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -13606,15 +13847,15 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13606
13847
|
gte4(aiReferralEventsHourly.tsHour, headlineStart),
|
|
13607
13848
|
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
13608
13849
|
)
|
|
13609
|
-
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc7(
|
|
13850
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc7(sql8`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
13610
13851
|
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
13611
13852
|
path: r.path,
|
|
13612
13853
|
arrivals: Number(r.arrivals),
|
|
13613
13854
|
distinctProducts: Number(r.products)
|
|
13614
13855
|
}));
|
|
13615
13856
|
const crawlerTrendRows = db.select({
|
|
13616
|
-
date:
|
|
13617
|
-
hits:
|
|
13857
|
+
date: sql8`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
13858
|
+
hits: sql8`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
13618
13859
|
}).from(crawlerEventsHourly).where(
|
|
13619
13860
|
and12(
|
|
13620
13861
|
eq16(crawlerEventsHourly.projectId, projectId),
|
|
@@ -13622,10 +13863,10 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13622
13863
|
gte4(crawlerEventsHourly.tsHour, trendStart),
|
|
13623
13864
|
lte(crawlerEventsHourly.tsHour, headlineEnd)
|
|
13624
13865
|
)
|
|
13625
|
-
).groupBy(
|
|
13866
|
+
).groupBy(sql8`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
13626
13867
|
const referralTrendRows = db.select({
|
|
13627
|
-
date:
|
|
13628
|
-
hits:
|
|
13868
|
+
date: sql8`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
13869
|
+
hits: sql8`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
13629
13870
|
}).from(aiReferralEventsHourly).where(
|
|
13630
13871
|
and12(
|
|
13631
13872
|
eq16(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -13633,17 +13874,17 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
13633
13874
|
gte4(aiReferralEventsHourly.tsHour, trendStart),
|
|
13634
13875
|
lte(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
13635
13876
|
)
|
|
13636
|
-
).groupBy(
|
|
13877
|
+
).groupBy(sql8`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
13637
13878
|
const userFetchTrendRows = db.select({
|
|
13638
|
-
date:
|
|
13639
|
-
hits:
|
|
13879
|
+
date: sql8`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`,
|
|
13880
|
+
hits: sql8`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
13640
13881
|
}).from(aiUserFetchEventsHourly).where(
|
|
13641
13882
|
and12(
|
|
13642
13883
|
eq16(aiUserFetchEventsHourly.projectId, projectId),
|
|
13643
13884
|
gte4(aiUserFetchEventsHourly.tsHour, trendStart),
|
|
13644
13885
|
lte(aiUserFetchEventsHourly.tsHour, headlineEnd)
|
|
13645
13886
|
)
|
|
13646
|
-
).groupBy(
|
|
13887
|
+
).groupBy(sql8`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`).all();
|
|
13647
13888
|
const emptyTrendEntry = () => ({ verifiedCrawlerHits: 0, userFetchHits: 0, referralArrivals: 0 });
|
|
13648
13889
|
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
13649
13890
|
for (const r of crawlerTrendRows) {
|
|
@@ -15495,7 +15736,7 @@ async function resultsExportRoutes(app) {
|
|
|
15495
15736
|
}
|
|
15496
15737
|
|
|
15497
15738
|
// ../api-routes/src/composites.ts
|
|
15498
|
-
import { eq as eq20, and as and16, desc as desc9, sql as
|
|
15739
|
+
import { eq as eq20, and as and16, desc as desc9, sql as sql9, like as like2, or as or4, inArray as inArray10 } from "drizzle-orm";
|
|
15499
15740
|
var TOP_INSIGHT_LIMIT = 5;
|
|
15500
15741
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
15501
15742
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -15672,9 +15913,9 @@ async function compositeRoutes(app) {
|
|
|
15672
15913
|
and16(
|
|
15673
15914
|
eq20(queries.projectId, project.id),
|
|
15674
15915
|
or4(
|
|
15675
|
-
|
|
15676
|
-
|
|
15677
|
-
|
|
15916
|
+
sql9`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
15917
|
+
sql9`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
15918
|
+
sql9`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
15678
15919
|
like2(queries.query, pattern)
|
|
15679
15920
|
)
|
|
15680
15921
|
)
|
|
@@ -15685,8 +15926,8 @@ async function compositeRoutes(app) {
|
|
|
15685
15926
|
or4(
|
|
15686
15927
|
like2(insights.title, pattern),
|
|
15687
15928
|
like2(insights.query, pattern),
|
|
15688
|
-
|
|
15689
|
-
|
|
15929
|
+
sql9`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
15930
|
+
sql9`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
15690
15931
|
)
|
|
15691
15932
|
)
|
|
15692
15933
|
).orderBy(desc9(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -15892,16 +16133,16 @@ function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
|
|
|
15892
16133
|
const dimensionedRows = app.db.select({
|
|
15893
16134
|
date: gscSearchData.date,
|
|
15894
16135
|
query: gscSearchData.query,
|
|
15895
|
-
impressions:
|
|
15896
|
-
clicks:
|
|
16136
|
+
impressions: sql9`COALESCE(SUM(${gscSearchData.impressions}), 0)`,
|
|
16137
|
+
clicks: sql9`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
15897
16138
|
// Weighted average: SUM(position * impressions) / SUM(impressions).
|
|
15898
16139
|
// NULLIF guards the degenerate impressions=0 case (SQLite returns NULL,
|
|
15899
16140
|
// which the JS coerces to 0 — caught by the impression floor anyway).
|
|
15900
|
-
avgPosition:
|
|
16141
|
+
avgPosition: sql9`COALESCE(SUM(${gscSearchData.position} * ${gscSearchData.impressions}) * 1.0 / NULLIF(SUM(${gscSearchData.impressions}), 0), 0)`
|
|
15901
16142
|
}).from(gscSearchData).where(and16(
|
|
15902
16143
|
eq20(gscSearchData.projectId, projectId),
|
|
15903
|
-
|
|
15904
|
-
|
|
16144
|
+
sql9`${gscSearchData.date} >= ${cutoff}`,
|
|
16145
|
+
sql9`${gscSearchData.impressions} > 0`
|
|
15905
16146
|
)).groupBy(gscSearchData.date, gscSearchData.query).all();
|
|
15906
16147
|
const todayIso = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
15907
16148
|
const merged = mergeGscQueryTotalsWithFallback(
|
|
@@ -16113,6 +16354,7 @@ function formatProject2(row) {
|
|
|
16113
16354
|
labels: row.labels,
|
|
16114
16355
|
providers: row.providers,
|
|
16115
16356
|
providerModels: row.providerModels,
|
|
16357
|
+
measurement: row.measurement,
|
|
16116
16358
|
locations: row.locations,
|
|
16117
16359
|
defaultLocation: row.defaultLocation,
|
|
16118
16360
|
autoExtractBacklinks: row.autoExtractBacklinks,
|
|
@@ -16264,6 +16506,9 @@ var SCHEMA_TABLE = {
|
|
|
16264
16506
|
DiscoveryPromoteResult: discoveryPromoteResultSchema,
|
|
16265
16507
|
DiscoverySessionDetailDto: discoverySessionDetailDtoSchema,
|
|
16266
16508
|
DiscoverySessionDto: discoverySessionDtoSchema,
|
|
16509
|
+
ResearchRunCreate: researchRunCreateSchema,
|
|
16510
|
+
ResearchRunDetailDto: researchRunDetailSchema,
|
|
16511
|
+
ResearchRunListDto: researchRunListSchema,
|
|
16267
16512
|
DoctorReportDto: doctorReportSchema,
|
|
16268
16513
|
GA4AiReferralHistoryEntry: ga4AiReferralHistoryEntrySchema,
|
|
16269
16514
|
GA4SessionHistoryEntry: ga4SessionHistoryEntrySchema,
|
|
@@ -16766,6 +17011,32 @@ var routeCatalog = [
|
|
|
16766
17011
|
404: errorResponse("Project not found.")
|
|
16767
17012
|
}
|
|
16768
17013
|
},
|
|
17014
|
+
{
|
|
17015
|
+
method: "post",
|
|
17016
|
+
path: "/api/v1/projects/{name}/research/runs",
|
|
17017
|
+
summary: "Start an isolated research query batch",
|
|
17018
|
+
description: "Runs one to fifty ad-hoc queries through one API provider and saves the answer evidence. Research never creates tracked queries, shared runs, snapshots, insights, or notifications.",
|
|
17019
|
+
tags: ["research"],
|
|
17020
|
+
parameters: [nameParameter],
|
|
17021
|
+
requestBody: { required: true, content: { "application/json": { schema: { $ref: "#/components/schemas/ResearchRunCreate" } } } },
|
|
17022
|
+
responses: { 200: jsonResponse("Idempotent request returned its existing research run.", "ResearchRunDetailDto"), 202: jsonResponse("Research batch queued.", "ResearchRunDetailDto"), 400: errorResponse("Invalid provider, model, location, or request."), 404: errorResponse("Project not found."), 409: errorResponse("Idempotency key was reused with a different payload."), 422: errorResponse("Research executor is unavailable on this deployment.") }
|
|
17023
|
+
},
|
|
17024
|
+
{
|
|
17025
|
+
method: "get",
|
|
17026
|
+
path: "/api/v1/projects/{name}/research/runs",
|
|
17027
|
+
summary: "List saved research query batches",
|
|
17028
|
+
tags: ["research"],
|
|
17029
|
+
parameters: [nameParameter, { name: "limit", in: "query", description: "Max runs, default 20 and maximum 100.", schema: integerSchema }],
|
|
17030
|
+
responses: { 200: jsonResponse("Research runs returned newest first.", "ResearchRunListDto"), 404: errorResponse("Project not found.") }
|
|
17031
|
+
},
|
|
17032
|
+
{
|
|
17033
|
+
method: "get",
|
|
17034
|
+
path: "/api/v1/projects/{name}/research/runs/{runId}",
|
|
17035
|
+
summary: "Get a research query batch and saved answers",
|
|
17036
|
+
tags: ["research"],
|
|
17037
|
+
parameters: [nameParameter, { name: "runId", in: "path", required: true, description: "Research run ID.", schema: stringSchema }],
|
|
17038
|
+
responses: { 200: jsonResponse("Research run detail returned.", "ResearchRunDetailDto"), 404: errorResponse("Project or research run not found.") }
|
|
17039
|
+
},
|
|
16769
17040
|
{
|
|
16770
17041
|
method: "post",
|
|
16771
17042
|
path: "/api/v1/projects/{name}/locations",
|
|
@@ -20791,15 +21062,16 @@ var routeCatalog = [
|
|
|
20791
21062
|
method: "get",
|
|
20792
21063
|
path: "/api/v1/projects/{name}/traffic/events",
|
|
20793
21064
|
summary: "List rolled-up crawler hits, AI user-fetch hits, and AI-referral sessions within a window",
|
|
20794
|
-
description: "Returns hourly
|
|
21065
|
+
description: "Returns full-window hourly or daily chart series plus newest-first detail rows from `crawler_events_hourly`, `ai_user_fetch_events_hourly`, and `ai_referral_events_hourly`. Defaults to the last 24h. Totals and `series.points` reflect the full window; only the `events` array is capped by `limit` (default 500, max 5000). `eventRows` reports the true pre-limit row count.",
|
|
20795
21066
|
tags: ["traffic"],
|
|
20796
21067
|
parameters: [
|
|
20797
21068
|
nameParameter,
|
|
20798
21069
|
{ name: "since", in: "query", description: "ISO-8601 window start (defaults to 24h ago).", schema: stringSchema },
|
|
20799
21070
|
{ name: "until", in: "query", description: "ISO-8601 window end (defaults to now).", schema: stringSchema },
|
|
20800
21071
|
{ name: "kind", in: "query", description: 'Filter to "crawler", "ai-user-fetch", "ai-referral", or "all" (default).', schema: stringSchema },
|
|
20801
|
-
{ name: "limit", in: "query", description: "Max
|
|
20802
|
-
{ name: "sourceId", in: "query", description: "Restrict to a single traffic source.", schema: stringSchema }
|
|
21072
|
+
{ name: "limit", in: "query", description: "Max newest combined detail rows in the events array (default 500, max 5000).", schema: stringSchema },
|
|
21073
|
+
{ name: "sourceId", in: "query", description: "Restrict to a single traffic source.", schema: stringSchema },
|
|
21074
|
+
{ name: "granularity", in: "query", description: 'Full-window series bucket size: "hour" (default) or "day".', schema: { type: "string", enum: ["hour", "day"] } }
|
|
20803
21075
|
],
|
|
20804
21076
|
responses: {
|
|
20805
21077
|
200: jsonResponse("Events returned with windowed totals.", "TrafficEventsResponse"),
|
|
@@ -21721,7 +21993,7 @@ function formatNotification(row) {
|
|
|
21721
21993
|
|
|
21722
21994
|
// ../api-routes/src/google.ts
|
|
21723
21995
|
import crypto19 from "crypto";
|
|
21724
|
-
import { eq as eq24, and as and18, desc as desc11, sql as
|
|
21996
|
+
import { eq as eq24, and as and18, desc as desc11, sql as sql10, inArray as inArray11 } from "drizzle-orm";
|
|
21725
21997
|
|
|
21726
21998
|
// ../api-routes/src/gbp-summary.ts
|
|
21727
21999
|
function computeMetricTotals(rows) {
|
|
@@ -23724,11 +23996,11 @@ async function googleRoutes(app, opts) {
|
|
|
23724
23996
|
const { startDate, endDate, query, page, limit, offset } = request.query;
|
|
23725
23997
|
const cutoffDate = !startDate ? windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null : null;
|
|
23726
23998
|
const conditions = [eq24(gscSearchData.projectId, project.id)];
|
|
23727
|
-
if (startDate) conditions.push(
|
|
23728
|
-
else if (cutoffDate) conditions.push(
|
|
23729
|
-
if (endDate) conditions.push(
|
|
23730
|
-
if (query) conditions.push(
|
|
23731
|
-
if (page) conditions.push(
|
|
23999
|
+
if (startDate) conditions.push(sql10`${gscSearchData.date} >= ${startDate}`);
|
|
24000
|
+
else if (cutoffDate) conditions.push(sql10`${gscSearchData.date} >= ${cutoffDate}`);
|
|
24001
|
+
if (endDate) conditions.push(sql10`${gscSearchData.date} <= ${endDate}`);
|
|
24002
|
+
if (query) conditions.push(sql10`${gscSearchData.query} LIKE ${"%" + escapeLikePattern(query) + "%"} ESCAPE '\\'`);
|
|
24003
|
+
if (page) conditions.push(sql10`${gscSearchData.page} LIKE ${"%" + escapeLikePattern(page) + "%"} ESCAPE '\\'`);
|
|
23732
24004
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
23733
24005
|
const offsetVal = Math.max(parseInt(offset ?? "0", 10) || 0, 0);
|
|
23734
24006
|
const rows = app.db.select().from(gscSearchData).where(and18(...conditions)).orderBy(desc11(gscSearchData.date)).limit(limitVal).offset(offsetVal).all();
|
|
@@ -23752,13 +24024,13 @@ async function googleRoutes(app, opts) {
|
|
|
23752
24024
|
const windowEnd = endDate ?? "9999-12-31";
|
|
23753
24025
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
23754
24026
|
const conditions = [eq24(gscSearchData.projectId, project.id)];
|
|
23755
|
-
if (startDate) conditions.push(
|
|
23756
|
-
else if (cutoffDate) conditions.push(
|
|
23757
|
-
if (endDate) conditions.push(
|
|
24027
|
+
if (startDate) conditions.push(sql10`${gscSearchData.date} >= ${startDate}`);
|
|
24028
|
+
else if (cutoffDate) conditions.push(sql10`${gscSearchData.date} >= ${cutoffDate}`);
|
|
24029
|
+
if (endDate) conditions.push(sql10`${gscSearchData.date} <= ${endDate}`);
|
|
23758
24030
|
const dimensionedRows = app.db.select({
|
|
23759
24031
|
date: gscSearchData.date,
|
|
23760
|
-
clicks:
|
|
23761
|
-
impressions:
|
|
24032
|
+
clicks: sql10`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
24033
|
+
impressions: sql10`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
23762
24034
|
}).from(gscSearchData).where(and18(...conditions)).groupBy(gscSearchData.date).orderBy(gscSearchData.date).all();
|
|
23763
24035
|
const daily = mergeGscDailyTotalsWithFallback(
|
|
23764
24036
|
dailyTotals,
|
|
@@ -24654,11 +24926,11 @@ async function googleRoutes(app, opts) {
|
|
|
24654
24926
|
|
|
24655
24927
|
// ../api-routes/src/ads.ts
|
|
24656
24928
|
import crypto22 from "crypto";
|
|
24657
|
-
import { eq as eq26, and as and20, asc as asc6, gt as gt2, gte as gte6, lt as lt3, lte as lte4, ne as ne3, inArray as inArray12, or as or6, isNull as isNull2, isNotNull as isNotNull2, sql as
|
|
24929
|
+
import { eq as eq26, and as and20, asc as asc6, gt as gt2, gte as gte6, lt as lt3, lte as lte4, ne as ne3, inArray as inArray12, or as or6, isNull as isNull2, isNotNull as isNotNull2, sql as sql12 } from "drizzle-orm";
|
|
24658
24930
|
|
|
24659
24931
|
// ../api-routes/src/ads-activation-routes.ts
|
|
24660
24932
|
import crypto21 from "crypto";
|
|
24661
|
-
import { and as and19, asc as asc5, eq as eq25, gt, isNotNull, isNull, lte as lte3, or as or5, sql as
|
|
24933
|
+
import { and as and19, asc as asc5, eq as eq25, gt, isNotNull, isNull, lte as lte3, or as or5, sql as sql11 } from "drizzle-orm";
|
|
24662
24934
|
|
|
24663
24935
|
// ../api-routes/src/ads-activation.ts
|
|
24664
24936
|
import crypto20 from "crypto";
|
|
@@ -26531,7 +26803,7 @@ function selectWatchdogOperations(app, nowIso, batchSize, allowLiveRevocationLea
|
|
|
26531
26803
|
),
|
|
26532
26804
|
leaseAvailable
|
|
26533
26805
|
);
|
|
26534
|
-
const freshRevocationPriority =
|
|
26806
|
+
const freshRevocationPriority = sql11`CASE
|
|
26535
26807
|
WHEN ${adsActivationGrants.revocationRequestedAt} IS NOT NULL
|
|
26536
26808
|
AND (
|
|
26537
26809
|
${adsOperations.lastReconciledAt} IS NULL
|
|
@@ -27627,7 +27899,7 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
27627
27899
|
state: AdsOperationStates.reconciling,
|
|
27628
27900
|
leaseOwner,
|
|
27629
27901
|
leaseExpiresAt,
|
|
27630
|
-
reconcileAttempts:
|
|
27902
|
+
reconcileAttempts: sql12`${adsOperations.reconcileAttempts} + 1`,
|
|
27631
27903
|
updatedAt: nowIso
|
|
27632
27904
|
}).where(and20(
|
|
27633
27905
|
eq26(adsOperations.id, row.id),
|
|
@@ -29699,7 +29971,7 @@ async function cdpRoutes(app, opts) {
|
|
|
29699
29971
|
|
|
29700
29972
|
// ../api-routes/src/ga.ts
|
|
29701
29973
|
import crypto24 from "crypto";
|
|
29702
|
-
import { eq as eq29, desc as desc13, and as and23, sql as
|
|
29974
|
+
import { eq as eq29, desc as desc13, and as and23, sql as sql13 } from "drizzle-orm";
|
|
29703
29975
|
|
|
29704
29976
|
// ../api-routes/src/ga-session-history.ts
|
|
29705
29977
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -30086,8 +30358,8 @@ async function ga4Routes(app, opts) {
|
|
|
30086
30358
|
tx.delete(gaTrafficSnapshots).where(
|
|
30087
30359
|
and23(
|
|
30088
30360
|
eq29(gaTrafficSnapshots.projectId, project.id),
|
|
30089
|
-
|
|
30090
|
-
|
|
30361
|
+
sql13`${gaTrafficSnapshots.date} >= ${summary.periodStart}`,
|
|
30362
|
+
sql13`${gaTrafficSnapshots.date} <= ${summary.periodEnd}`
|
|
30091
30363
|
)
|
|
30092
30364
|
).run();
|
|
30093
30365
|
for (const row of rows) {
|
|
@@ -30109,8 +30381,8 @@ async function ga4Routes(app, opts) {
|
|
|
30109
30381
|
tx.delete(gaDailyTotals).where(
|
|
30110
30382
|
and23(
|
|
30111
30383
|
eq29(gaDailyTotals.projectId, project.id),
|
|
30112
|
-
|
|
30113
|
-
|
|
30384
|
+
sql13`${gaDailyTotals.date} >= ${summary.periodStart}`,
|
|
30385
|
+
sql13`${gaDailyTotals.date} <= ${summary.periodEnd}`
|
|
30114
30386
|
)
|
|
30115
30387
|
).run();
|
|
30116
30388
|
for (const row of dailyTotals) {
|
|
@@ -30129,8 +30401,8 @@ async function ga4Routes(app, opts) {
|
|
|
30129
30401
|
tx.delete(gaAiReferrals).where(
|
|
30130
30402
|
and23(
|
|
30131
30403
|
eq29(gaAiReferrals.projectId, project.id),
|
|
30132
|
-
|
|
30133
|
-
|
|
30404
|
+
sql13`${gaAiReferrals.date} >= ${summary.periodStart}`,
|
|
30405
|
+
sql13`${gaAiReferrals.date} <= ${summary.periodEnd}`
|
|
30134
30406
|
)
|
|
30135
30407
|
).run();
|
|
30136
30408
|
for (const row of aiReferrals) {
|
|
@@ -30161,8 +30433,8 @@ async function ga4Routes(app, opts) {
|
|
|
30161
30433
|
tx.delete(gaSocialReferrals).where(
|
|
30162
30434
|
and23(
|
|
30163
30435
|
eq29(gaSocialReferrals.projectId, project.id),
|
|
30164
|
-
|
|
30165
|
-
|
|
30436
|
+
sql13`${gaSocialReferrals.date} >= ${summary.periodStart}`,
|
|
30437
|
+
sql13`${gaSocialReferrals.date} <= ${summary.periodEnd}`
|
|
30166
30438
|
)
|
|
30167
30439
|
).run();
|
|
30168
30440
|
for (const row of socialReferrals) {
|
|
@@ -30252,11 +30524,11 @@ async function ga4Routes(app, opts) {
|
|
|
30252
30524
|
const cutoff = windowCutoff(window);
|
|
30253
30525
|
const cutoffDate = cutoff?.slice(0, 10) ?? null;
|
|
30254
30526
|
const snapshotConditions = [eq29(gaTrafficSnapshots.projectId, project.id)];
|
|
30255
|
-
if (cutoffDate) snapshotConditions.push(
|
|
30527
|
+
if (cutoffDate) snapshotConditions.push(sql13`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
30256
30528
|
const aiConditions = [eq29(gaAiReferrals.projectId, project.id)];
|
|
30257
|
-
if (cutoffDate) aiConditions.push(
|
|
30529
|
+
if (cutoffDate) aiConditions.push(sql13`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
30258
30530
|
const socialConditions = [eq29(gaSocialReferrals.projectId, project.id)];
|
|
30259
|
-
if (cutoffDate) socialConditions.push(
|
|
30531
|
+
if (cutoffDate) socialConditions.push(sql13`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
30260
30532
|
const windowSummaryRow = cutoffDate ? app.db.select({
|
|
30261
30533
|
totalSessions: gaTrafficWindowSummaries.totalSessions,
|
|
30262
30534
|
totalOrganicSessions: gaTrafficWindowSummaries.totalOrganicSessions,
|
|
@@ -30269,9 +30541,9 @@ async function ga4Routes(app, opts) {
|
|
|
30269
30541
|
)
|
|
30270
30542
|
).get() : null;
|
|
30271
30543
|
const snapshotTotalsRow = cutoffDate && !windowSummaryRow ? app.db.select({
|
|
30272
|
-
totalSessions:
|
|
30273
|
-
totalOrganicSessions:
|
|
30274
|
-
totalUsers:
|
|
30544
|
+
totalSessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
30545
|
+
totalOrganicSessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
30546
|
+
totalUsers: sql13`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
30275
30547
|
}).from(gaTrafficSnapshots).where(and23(...snapshotConditions)).get() : null;
|
|
30276
30548
|
const summaryRow = cutoffDate ? windowSummaryRow ?? snapshotTotalsRow : app.db.select({
|
|
30277
30549
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
@@ -30279,41 +30551,41 @@ async function ga4Routes(app, opts) {
|
|
|
30279
30551
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
30280
30552
|
}).from(gaTrafficSummaries).where(eq29(gaTrafficSummaries.projectId, project.id)).get();
|
|
30281
30553
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
30282
|
-
totalDirectSessions:
|
|
30554
|
+
totalDirectSessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
30283
30555
|
}).from(gaTrafficSnapshots).where(and23(...snapshotConditions)).get();
|
|
30284
30556
|
const summaryMeta = app.db.select({
|
|
30285
30557
|
periodStart: gaTrafficSummaries.periodStart,
|
|
30286
30558
|
periodEnd: gaTrafficSummaries.periodEnd
|
|
30287
30559
|
}).from(gaTrafficSummaries).where(eq29(gaTrafficSummaries.projectId, project.id)).get();
|
|
30288
30560
|
const rows = app.db.select({
|
|
30289
|
-
landingPage:
|
|
30290
|
-
sessions:
|
|
30291
|
-
organicSessions:
|
|
30292
|
-
directSessions:
|
|
30293
|
-
users:
|
|
30294
|
-
}).from(gaTrafficSnapshots).where(and23(...snapshotConditions)).groupBy(
|
|
30561
|
+
landingPage: sql13`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
30562
|
+
sessions: sql13`SUM(${gaTrafficSnapshots.sessions})`,
|
|
30563
|
+
organicSessions: sql13`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
30564
|
+
directSessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
30565
|
+
users: sql13`SUM(${gaTrafficSnapshots.users})`
|
|
30566
|
+
}).from(gaTrafficSnapshots).where(and23(...snapshotConditions)).groupBy(sql13`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql13`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
30295
30567
|
const aiReferralRows = app.db.select({
|
|
30296
30568
|
source: gaAiReferrals.source,
|
|
30297
30569
|
medium: gaAiReferrals.medium,
|
|
30298
30570
|
trafficClass: gaAiReferrals.trafficClass,
|
|
30299
30571
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
30300
|
-
sessions:
|
|
30301
|
-
users:
|
|
30572
|
+
sessions: sql13`SUM(${gaAiReferrals.sessions})`,
|
|
30573
|
+
users: sql13`SUM(${gaAiReferrals.users})`
|
|
30302
30574
|
}).from(gaAiReferrals).where(and23(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
30303
30575
|
const aiReferralLandingPageRows = app.db.select({
|
|
30304
30576
|
source: gaAiReferrals.source,
|
|
30305
30577
|
medium: gaAiReferrals.medium,
|
|
30306
30578
|
trafficClass: gaAiReferrals.trafficClass,
|
|
30307
30579
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
30308
|
-
landingPage:
|
|
30309
|
-
sessions:
|
|
30310
|
-
users:
|
|
30580
|
+
landingPage: sql13`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
30581
|
+
sessions: sql13`SUM(${gaAiReferrals.sessions})`,
|
|
30582
|
+
users: sql13`SUM(${gaAiReferrals.users})`
|
|
30311
30583
|
}).from(gaAiReferrals).where(and23(...aiConditions)).groupBy(
|
|
30312
30584
|
gaAiReferrals.source,
|
|
30313
30585
|
gaAiReferrals.medium,
|
|
30314
30586
|
gaAiReferrals.trafficClass,
|
|
30315
30587
|
gaAiReferrals.sourceDimension,
|
|
30316
|
-
|
|
30588
|
+
sql13`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
30317
30589
|
).all();
|
|
30318
30590
|
const aiRowsForTotals = app.db.select({
|
|
30319
30591
|
date: gaAiReferrals.date,
|
|
@@ -30338,12 +30610,12 @@ async function ga4Routes(app, opts) {
|
|
|
30338
30610
|
source: gaSocialReferrals.source,
|
|
30339
30611
|
medium: gaSocialReferrals.medium,
|
|
30340
30612
|
channelGroup: gaSocialReferrals.channelGroup,
|
|
30341
|
-
sessions:
|
|
30342
|
-
users:
|
|
30343
|
-
}).from(gaSocialReferrals).where(and23(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
30613
|
+
sessions: sql13`SUM(${gaSocialReferrals.sessions})`,
|
|
30614
|
+
users: sql13`SUM(${gaSocialReferrals.users})`
|
|
30615
|
+
}).from(gaSocialReferrals).where(and23(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql13`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
30344
30616
|
const socialTotals = app.db.select({
|
|
30345
|
-
sessions:
|
|
30346
|
-
users:
|
|
30617
|
+
sessions: sql13`SUM(${gaSocialReferrals.sessions})`,
|
|
30618
|
+
users: sql13`SUM(${gaSocialReferrals.users})`
|
|
30347
30619
|
}).from(gaSocialReferrals).where(and23(...socialConditions)).get();
|
|
30348
30620
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq29(gaTrafficSummaries.projectId, project.id)).orderBy(desc13(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
30349
30621
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -30444,23 +30716,23 @@ async function ga4Routes(app, opts) {
|
|
|
30444
30716
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
30445
30717
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
30446
30718
|
const conditions = [eq29(gaAiReferrals.projectId, project.id)];
|
|
30447
|
-
if (cutoffDate) conditions.push(
|
|
30719
|
+
if (cutoffDate) conditions.push(sql13`${gaAiReferrals.date} >= ${cutoffDate}`);
|
|
30448
30720
|
const rows = app.db.select({
|
|
30449
30721
|
date: gaAiReferrals.date,
|
|
30450
30722
|
source: gaAiReferrals.source,
|
|
30451
30723
|
medium: gaAiReferrals.medium,
|
|
30452
30724
|
trafficClass: gaAiReferrals.trafficClass,
|
|
30453
|
-
landingPage:
|
|
30725
|
+
landingPage: sql13`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
30454
30726
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
30455
|
-
sessions:
|
|
30456
|
-
users:
|
|
30727
|
+
sessions: sql13`SUM(${gaAiReferrals.sessions})`,
|
|
30728
|
+
users: sql13`SUM(${gaAiReferrals.users})`
|
|
30457
30729
|
}).from(gaAiReferrals).where(and23(...conditions)).groupBy(
|
|
30458
30730
|
gaAiReferrals.date,
|
|
30459
30731
|
gaAiReferrals.source,
|
|
30460
30732
|
gaAiReferrals.medium,
|
|
30461
30733
|
gaAiReferrals.trafficClass,
|
|
30462
30734
|
gaAiReferrals.sourceDimension,
|
|
30463
|
-
|
|
30735
|
+
sql13`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
30464
30736
|
).orderBy(gaAiReferrals.date).all();
|
|
30465
30737
|
return rows;
|
|
30466
30738
|
});
|
|
@@ -30469,7 +30741,7 @@ async function ga4Routes(app, opts) {
|
|
|
30469
30741
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
30470
30742
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
30471
30743
|
const conditions = [eq29(gaSocialReferrals.projectId, project.id)];
|
|
30472
|
-
if (cutoffDate) conditions.push(
|
|
30744
|
+
if (cutoffDate) conditions.push(sql13`${gaSocialReferrals.date} >= ${cutoffDate}`);
|
|
30473
30745
|
const rows = app.db.select({
|
|
30474
30746
|
date: gaSocialReferrals.date,
|
|
30475
30747
|
source: gaSocialReferrals.source,
|
|
@@ -30490,10 +30762,10 @@ async function ga4Routes(app, opts) {
|
|
|
30490
30762
|
d.setDate(d.getDate() - n);
|
|
30491
30763
|
return fmt(d);
|
|
30492
30764
|
};
|
|
30493
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
30765
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql13`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and23(
|
|
30494
30766
|
eq29(gaSocialReferrals.projectId, project.id),
|
|
30495
|
-
|
|
30496
|
-
|
|
30767
|
+
sql13`${gaSocialReferrals.date} >= ${from}`,
|
|
30768
|
+
sql13`${gaSocialReferrals.date} < ${to}`
|
|
30497
30769
|
)).get();
|
|
30498
30770
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
30499
30771
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -30502,19 +30774,19 @@ async function ga4Routes(app, opts) {
|
|
|
30502
30774
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
30503
30775
|
const sourceCurrent = app.db.select({
|
|
30504
30776
|
source: gaSocialReferrals.source,
|
|
30505
|
-
sessions:
|
|
30777
|
+
sessions: sql13`SUM(${gaSocialReferrals.sessions})`
|
|
30506
30778
|
}).from(gaSocialReferrals).where(and23(
|
|
30507
30779
|
eq29(gaSocialReferrals.projectId, project.id),
|
|
30508
|
-
|
|
30509
|
-
|
|
30780
|
+
sql13`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
30781
|
+
sql13`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
30510
30782
|
)).groupBy(gaSocialReferrals.source).all();
|
|
30511
30783
|
const sourcePrev = app.db.select({
|
|
30512
30784
|
source: gaSocialReferrals.source,
|
|
30513
|
-
sessions:
|
|
30785
|
+
sessions: sql13`SUM(${gaSocialReferrals.sessions})`
|
|
30514
30786
|
}).from(gaSocialReferrals).where(and23(
|
|
30515
30787
|
eq29(gaSocialReferrals.projectId, project.id),
|
|
30516
|
-
|
|
30517
|
-
|
|
30788
|
+
sql13`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
30789
|
+
sql13`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
30518
30790
|
)).groupBy(gaSocialReferrals.source).all();
|
|
30519
30791
|
const prevMap = new Map(sourcePrev.map((r) => [r.source, r.sessions]));
|
|
30520
30792
|
let biggestMover = null;
|
|
@@ -30553,16 +30825,16 @@ async function ga4Routes(app, opts) {
|
|
|
30553
30825
|
return fmt(d);
|
|
30554
30826
|
};
|
|
30555
30827
|
const pct = (cur, prev) => prev === 0 ? null : Math.round((cur - prev) / prev * 100);
|
|
30556
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
30557
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
30558
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
30559
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
30828
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and23(eq29(gaTrafficSnapshots.projectId, project.id), sql13`${gaTrafficSnapshots.date} >= ${from}`, sql13`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
30829
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and23(eq29(gaTrafficSnapshots.projectId, project.id), sql13`${gaTrafficSnapshots.date} >= ${from}`, sql13`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
30830
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql13`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and23(eq29(gaTrafficSnapshots.projectId, project.id), sql13`${gaTrafficSnapshots.date} >= ${from}`, sql13`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
30831
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql13`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and23(
|
|
30560
30832
|
eq29(gaAiReferrals.projectId, project.id),
|
|
30561
|
-
|
|
30562
|
-
|
|
30833
|
+
sql13`${gaAiReferrals.date} >= ${from}`,
|
|
30834
|
+
sql13`${gaAiReferrals.date} < ${to}`,
|
|
30563
30835
|
eq29(gaAiReferrals.sourceDimension, "session")
|
|
30564
30836
|
)).get();
|
|
30565
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
30837
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql13`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and23(eq29(gaSocialReferrals.projectId, project.id), sql13`${gaSocialReferrals.date} >= ${from}`, sql13`${gaSocialReferrals.date} < ${to}`)).get();
|
|
30566
30838
|
const todayStr = fmt(today);
|
|
30567
30839
|
const buildTrend = (sum) => {
|
|
30568
30840
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -30571,16 +30843,16 @@ async function ga4Routes(app, opts) {
|
|
|
30571
30843
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
30572
30844
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: pct(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: pct(c30, p30) };
|
|
30573
30845
|
};
|
|
30574
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
30846
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql13`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and23(
|
|
30575
30847
|
eq29(gaAiReferrals.projectId, project.id),
|
|
30576
|
-
|
|
30577
|
-
|
|
30848
|
+
sql13`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
30849
|
+
sql13`${gaAiReferrals.date} < ${todayStr}`,
|
|
30578
30850
|
eq29(gaAiReferrals.sourceDimension, "session")
|
|
30579
30851
|
)).groupBy(gaAiReferrals.source).all();
|
|
30580
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
30852
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql13`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and23(
|
|
30581
30853
|
eq29(gaAiReferrals.projectId, project.id),
|
|
30582
|
-
|
|
30583
|
-
|
|
30854
|
+
sql13`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
30855
|
+
sql13`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
30584
30856
|
eq29(gaAiReferrals.sourceDimension, "session")
|
|
30585
30857
|
)).groupBy(gaAiReferrals.source).all();
|
|
30586
30858
|
const findBiggestMover = (current, prev) => {
|
|
@@ -30597,8 +30869,8 @@ async function ga4Routes(app, opts) {
|
|
|
30597
30869
|
}
|
|
30598
30870
|
return mover;
|
|
30599
30871
|
};
|
|
30600
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
30601
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
30872
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql13`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and23(eq29(gaSocialReferrals.projectId, project.id), sql13`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql13`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
30873
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql13`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and23(eq29(gaSocialReferrals.projectId, project.id), sql13`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql13`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
30602
30874
|
return {
|
|
30603
30875
|
total: buildTrend(sumTotal),
|
|
30604
30876
|
organic: buildTrend(sumOrganic),
|
|
@@ -30614,15 +30886,15 @@ async function ga4Routes(app, opts) {
|
|
|
30614
30886
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
30615
30887
|
const cutoffDate = windowCutoff(parseWindow(request.query.window))?.slice(0, 10) ?? null;
|
|
30616
30888
|
const conditions = [eq29(gaTrafficSnapshots.projectId, project.id)];
|
|
30617
|
-
if (cutoffDate) conditions.push(
|
|
30889
|
+
if (cutoffDate) conditions.push(sql13`${gaTrafficSnapshots.date} >= ${cutoffDate}`);
|
|
30618
30890
|
const rows = app.db.select({
|
|
30619
30891
|
date: gaTrafficSnapshots.date,
|
|
30620
|
-
sessions:
|
|
30621
|
-
organicSessions:
|
|
30622
|
-
users:
|
|
30892
|
+
sessions: sql13`SUM(${gaTrafficSnapshots.sessions})`,
|
|
30893
|
+
organicSessions: sql13`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
30894
|
+
users: sql13`SUM(${gaTrafficSnapshots.users})`
|
|
30623
30895
|
}).from(gaTrafficSnapshots).where(and23(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
30624
30896
|
const totalConditions = [eq29(gaDailyTotals.projectId, project.id)];
|
|
30625
|
-
if (cutoffDate) totalConditions.push(
|
|
30897
|
+
if (cutoffDate) totalConditions.push(sql13`${gaDailyTotals.date} >= ${cutoffDate}`);
|
|
30626
30898
|
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and23(...totalConditions)).all();
|
|
30627
30899
|
return buildSessionHistory(
|
|
30628
30900
|
rows.map((r) => ({
|
|
@@ -30638,11 +30910,11 @@ async function ga4Routes(app, opts) {
|
|
|
30638
30910
|
const project = resolveProject(app.db, request.params.name);
|
|
30639
30911
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
30640
30912
|
const trafficPages = app.db.select({
|
|
30641
|
-
landingPage:
|
|
30642
|
-
sessions:
|
|
30643
|
-
organicSessions:
|
|
30644
|
-
users:
|
|
30645
|
-
}).from(gaTrafficSnapshots).where(eq29(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
30913
|
+
landingPage: sql13`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
30914
|
+
sessions: sql13`SUM(${gaTrafficSnapshots.sessions})`,
|
|
30915
|
+
organicSessions: sql13`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
30916
|
+
users: sql13`SUM(${gaTrafficSnapshots.users})`
|
|
30917
|
+
}).from(gaTrafficSnapshots).where(eq29(gaTrafficSnapshots.projectId, project.id)).groupBy(sql13`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql13`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
30646
30918
|
return {
|
|
30647
30919
|
pages: trafficPages.map((r) => ({
|
|
30648
30920
|
landingPage: r.landingPage,
|
|
@@ -31750,9 +32022,9 @@ async function wordpressRoutes(app, opts) {
|
|
|
31750
32022
|
throw validationError("WordPress connection storage is not configured for this deployment");
|
|
31751
32023
|
}
|
|
31752
32024
|
async function assertWordpressUrlAllowed(rawUrl, field) {
|
|
31753
|
-
const
|
|
31754
|
-
if (!
|
|
31755
|
-
throw validationError(`${field} ${
|
|
32025
|
+
const check2 = await resolveWebhookTarget(rawUrl, { allowLoopback });
|
|
32026
|
+
if (!check2.ok) {
|
|
32027
|
+
throw validationError(`${field} ${check2.message.replace(/^"url" /, "")}`);
|
|
31756
32028
|
}
|
|
31757
32029
|
}
|
|
31758
32030
|
function requireConnection(store, projectName) {
|
|
@@ -32290,7 +32562,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
32290
32562
|
|
|
32291
32563
|
// ../api-routes/src/backlinks.ts
|
|
32292
32564
|
import crypto26 from "crypto";
|
|
32293
|
-
import { and as and25, asc as asc7, desc as desc14, eq as eq30, sql as
|
|
32565
|
+
import { and as and25, asc as asc7, desc as desc14, eq as eq30, sql as sql14 } from "drizzle-orm";
|
|
32294
32566
|
|
|
32295
32567
|
// ../integration-commoncrawl/src/constants.ts
|
|
32296
32568
|
import os2 from "os";
|
|
@@ -32580,7 +32852,7 @@ async function queryBacklinks(opts) {
|
|
|
32580
32852
|
const reversed = opts.targets.map(reverseDomain);
|
|
32581
32853
|
const targetList = reversed.map(quote).join(", ");
|
|
32582
32854
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
32583
|
-
const
|
|
32855
|
+
const sql17 = `
|
|
32584
32856
|
WITH vertices AS (
|
|
32585
32857
|
SELECT * FROM read_csv(
|
|
32586
32858
|
${quote(opts.vertexPath)},
|
|
@@ -32616,7 +32888,7 @@ async function queryBacklinks(opts) {
|
|
|
32616
32888
|
const conn = await instance.connect();
|
|
32617
32889
|
let rows;
|
|
32618
32890
|
try {
|
|
32619
|
-
const reader = await conn.runAndReadAll(
|
|
32891
|
+
const reader = await conn.runAndReadAll(sql17);
|
|
32620
32892
|
rows = reader.getRowObjects();
|
|
32621
32893
|
} finally {
|
|
32622
32894
|
conn.disconnectSync?.();
|
|
@@ -32806,12 +33078,12 @@ function computeFilteredSummary(db, base) {
|
|
|
32806
33078
|
);
|
|
32807
33079
|
const filteredCondition = and25(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
32808
33080
|
const unfilteredAgg = db.select({
|
|
32809
|
-
count:
|
|
32810
|
-
total:
|
|
33081
|
+
count: sql14`count(*)`,
|
|
33082
|
+
total: sql14`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
32811
33083
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
32812
33084
|
const filteredAgg = db.select({
|
|
32813
|
-
count:
|
|
32814
|
-
total:
|
|
33085
|
+
count: sql14`count(*)`,
|
|
33086
|
+
total: sql14`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
32815
33087
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
32816
33088
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc14(backlinkDomains.numHosts)).limit(10).all();
|
|
32817
33089
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -32837,7 +33109,7 @@ function buildSourceAvailability(db, projectId, source, connected, excludeCrawle
|
|
|
32837
33109
|
const summary = db.select().from(backlinkSummaries).where(and25(eq30(backlinkSummaries.projectId, projectId), eq30(backlinkSummaries.source, source))).orderBy(desc14(backlinkSummaries.queriedAt)).limit(1).get();
|
|
32838
33110
|
let totalLinkingDomains = summary?.totalLinkingDomains ?? 0;
|
|
32839
33111
|
if (summary && excludeCrawlers) {
|
|
32840
|
-
const filtered = db.select({ count:
|
|
33112
|
+
const filtered = db.select({ count: sql14`count(*)` }).from(backlinkDomains).where(and25(
|
|
32841
33113
|
eq30(backlinkDomains.projectId, projectId),
|
|
32842
33114
|
eq30(backlinkDomains.source, source),
|
|
32843
33115
|
eq30(backlinkDomains.release, summary.release),
|
|
@@ -33026,7 +33298,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
33026
33298
|
eq30(backlinkDomains.release, targetRelease)
|
|
33027
33299
|
);
|
|
33028
33300
|
const domainCondition = excludeCrawlers ? and25(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
33029
|
-
const totalRow = app.db.select({ count:
|
|
33301
|
+
const totalRow = app.db.select({ count: sql14`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
33030
33302
|
const rows = app.db.select({
|
|
33031
33303
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
33032
33304
|
numHosts: backlinkDomains.numHosts,
|
|
@@ -33105,7 +33377,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
33105
33377
|
// ../api-routes/src/traffic.ts
|
|
33106
33378
|
import crypto28 from "crypto";
|
|
33107
33379
|
import { Agent as UndiciAgent } from "undici";
|
|
33108
|
-
import { and as and26, desc as desc15, eq as eq31, gte as gte7, lte as lte5, sql as
|
|
33380
|
+
import { and as and26, desc as desc15, eq as eq31, gte as gte7, lte as lte5, sql as sql15 } from "drizzle-orm";
|
|
33109
33381
|
|
|
33110
33382
|
// ../integration-cloud-run/src/auth.ts
|
|
33111
33383
|
import crypto27 from "crypto";
|
|
@@ -36922,6 +37194,35 @@ var DEFAULT_TRAFFIC_SYNC_CRON = "*/30 * * * *";
|
|
|
36922
37194
|
var MAX_BACKFILL_DAYS = 90;
|
|
36923
37195
|
var BACKFILL_MAX_PAGES = 1e3;
|
|
36924
37196
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
37197
|
+
function emptyTrafficSeriesPoint(bucket) {
|
|
37198
|
+
return { bucket, crawlerHits: 0, aiUserFetchHits: 0, aiReferralHits: 0 };
|
|
37199
|
+
}
|
|
37200
|
+
function completeTrafficSeries(windowStart, windowEnd, granularity, pointsByBucket) {
|
|
37201
|
+
const cursor = new Date(windowStart);
|
|
37202
|
+
const last = new Date(windowEnd);
|
|
37203
|
+
if (granularity === TrafficSeriesGranularities.day) {
|
|
37204
|
+
cursor.setUTCHours(0, 0, 0, 0);
|
|
37205
|
+
last.setUTCHours(0, 0, 0, 0);
|
|
37206
|
+
} else {
|
|
37207
|
+
cursor.setUTCMinutes(0, 0, 0);
|
|
37208
|
+
last.setUTCMinutes(0, 0, 0);
|
|
37209
|
+
}
|
|
37210
|
+
const points = [];
|
|
37211
|
+
while (cursor <= last) {
|
|
37212
|
+
const bucket = granularity === TrafficSeriesGranularities.day ? cursor.toISOString().slice(0, 10) : cursor.toISOString();
|
|
37213
|
+
points.push(pointsByBucket.get(bucket) ?? emptyTrafficSeriesPoint(bucket));
|
|
37214
|
+
if (granularity === TrafficSeriesGranularities.day) cursor.setUTCDate(cursor.getUTCDate() + 1);
|
|
37215
|
+
else cursor.setUTCHours(cursor.getUTCHours() + 1);
|
|
37216
|
+
}
|
|
37217
|
+
return points;
|
|
37218
|
+
}
|
|
37219
|
+
function trafficSeriesPoint(pointsByBucket, bucket) {
|
|
37220
|
+
const existing = pointsByBucket.get(bucket);
|
|
37221
|
+
if (existing) return existing;
|
|
37222
|
+
const created = emptyTrafficSeriesPoint(bucket);
|
|
37223
|
+
pointsByBucket.set(bucket, created);
|
|
37224
|
+
return created;
|
|
37225
|
+
}
|
|
36925
37226
|
function parseSourceConfig(row) {
|
|
36926
37227
|
return row.configJson;
|
|
36927
37228
|
}
|
|
@@ -37137,11 +37438,11 @@ async function trafficRoutes(app, opts) {
|
|
|
37137
37438
|
const pullVercelEvents = opts.pullVercelTrafficEvents ?? listVercelTrafficEvents;
|
|
37138
37439
|
const allowLoopback = opts.allowLoopbackWebhooks === true;
|
|
37139
37440
|
async function assertWordpressTargetAllowed(baseUrl) {
|
|
37140
|
-
const
|
|
37141
|
-
if (!
|
|
37142
|
-
throw validationError(`WordPress baseUrl rejected: ${
|
|
37441
|
+
const check2 = await resolveWebhookTarget(baseUrl, { allowLoopback });
|
|
37442
|
+
if (!check2.ok) {
|
|
37443
|
+
throw validationError(`WordPress baseUrl rejected: ${check2.message}`);
|
|
37143
37444
|
}
|
|
37144
|
-
const { address, family } =
|
|
37445
|
+
const { address, family } = check2.target;
|
|
37145
37446
|
return new UndiciAgent({
|
|
37146
37447
|
connect: {
|
|
37147
37448
|
lookup: (_hostname, options, cb) => {
|
|
@@ -37777,7 +38078,7 @@ async function trafficRoutes(app, opts) {
|
|
|
37777
38078
|
crawlerEventsHourly.status
|
|
37778
38079
|
],
|
|
37779
38080
|
set: {
|
|
37780
|
-
hits:
|
|
38081
|
+
hits: sql15`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
37781
38082
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
37782
38083
|
updatedAt: finishedAt
|
|
37783
38084
|
}
|
|
@@ -37810,7 +38111,7 @@ async function trafficRoutes(app, opts) {
|
|
|
37810
38111
|
aiUserFetchEventsHourly.status
|
|
37811
38112
|
],
|
|
37812
38113
|
set: {
|
|
37813
|
-
hits:
|
|
38114
|
+
hits: sql15`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
37814
38115
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
37815
38116
|
updatedAt: finishedAt
|
|
37816
38117
|
}
|
|
@@ -37850,9 +38151,9 @@ async function trafficRoutes(app, opts) {
|
|
|
37850
38151
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
37851
38152
|
// already there rather than overwriting a single label.
|
|
37852
38153
|
set: {
|
|
37853
|
-
sessionsOrHits:
|
|
37854
|
-
paidSessionsOrHits:
|
|
37855
|
-
organicSessionsOrHits:
|
|
38154
|
+
sessionsOrHits: sql15`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
38155
|
+
paidSessionsOrHits: sql15`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
38156
|
+
organicSessionsOrHits: sql15`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
37856
38157
|
updatedAt: finishedAt
|
|
37857
38158
|
}
|
|
37858
38159
|
}).run();
|
|
@@ -38143,7 +38444,7 @@ async function trafficRoutes(app, opts) {
|
|
|
38143
38444
|
function buildSourceDetail(projectId, row, since) {
|
|
38144
38445
|
const crawlerPathRows = app.db.select({
|
|
38145
38446
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
38146
|
-
hits:
|
|
38447
|
+
hits: sql15`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
38147
38448
|
}).from(crawlerEventsHourly).where(
|
|
38148
38449
|
and26(
|
|
38149
38450
|
eq31(crawlerEventsHourly.sourceId, row.id),
|
|
@@ -38154,19 +38455,19 @@ async function trafficRoutes(app, opts) {
|
|
|
38154
38455
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
38155
38456
|
);
|
|
38156
38457
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
38157
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
38458
|
+
const aiUserFetchTotals = app.db.select({ total: sql15`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
38158
38459
|
and26(
|
|
38159
38460
|
eq31(aiUserFetchEventsHourly.sourceId, row.id),
|
|
38160
38461
|
gte7(aiUserFetchEventsHourly.tsHour, since)
|
|
38161
38462
|
)
|
|
38162
38463
|
).get();
|
|
38163
|
-
const aiTotals = app.db.select({ total:
|
|
38464
|
+
const aiTotals = app.db.select({ total: sql15`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
38164
38465
|
and26(
|
|
38165
38466
|
eq31(aiReferralEventsHourly.sourceId, row.id),
|
|
38166
38467
|
gte7(aiReferralEventsHourly.tsHour, since)
|
|
38167
38468
|
)
|
|
38168
38469
|
).get();
|
|
38169
|
-
const sampleTotals = app.db.select({ total:
|
|
38470
|
+
const sampleTotals = app.db.select({ total: sql15`COUNT(*)` }).from(rawEventSamples).where(
|
|
38170
38471
|
and26(
|
|
38171
38472
|
eq31(rawEventSamples.sourceId, row.id),
|
|
38172
38473
|
gte7(rawEventSamples.ts, since)
|
|
@@ -38297,6 +38598,17 @@ async function trafficRoutes(app, opts) {
|
|
|
38297
38598
|
throw validationError('"limit" must be a positive integer');
|
|
38298
38599
|
}
|
|
38299
38600
|
const limit = Math.min(requestedLimit, 5e3);
|
|
38601
|
+
const granularityParam = request.query?.granularity;
|
|
38602
|
+
let granularity = TrafficSeriesGranularities.hour;
|
|
38603
|
+
if (granularityParam !== void 0) {
|
|
38604
|
+
if (granularityParam === TrafficSeriesGranularities.hour || granularityParam === TrafficSeriesGranularities.day) {
|
|
38605
|
+
granularity = granularityParam;
|
|
38606
|
+
} else {
|
|
38607
|
+
throw validationError(
|
|
38608
|
+
`"granularity" must be one of: ${TrafficSeriesGranularities.hour}, ${TrafficSeriesGranularities.day}`
|
|
38609
|
+
);
|
|
38610
|
+
}
|
|
38611
|
+
}
|
|
38300
38612
|
const sourceIdParam = request.query?.sourceId;
|
|
38301
38613
|
const sinceIso = since.toISOString();
|
|
38302
38614
|
const untilIso = until.toISOString();
|
|
@@ -38305,6 +38617,8 @@ async function trafficRoutes(app, opts) {
|
|
|
38305
38617
|
let crawlerSegments = { content: 0, sitemap: 0, robots: 0, asset: 0, other: 0 };
|
|
38306
38618
|
let aiUserFetchTotal = 0;
|
|
38307
38619
|
let aiReferralCounts = aiReferralClassCounts(0, 0, 0);
|
|
38620
|
+
let totalEventRows = 0;
|
|
38621
|
+
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
38308
38622
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
38309
38623
|
const crawlerFilters = [
|
|
38310
38624
|
eq31(crawlerEventsHourly.projectId, project.id),
|
|
@@ -38315,12 +38629,22 @@ async function trafficRoutes(app, opts) {
|
|
|
38315
38629
|
const crawlerWhere = and26(...crawlerFilters);
|
|
38316
38630
|
const pathTotals = app.db.select({
|
|
38317
38631
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
38318
|
-
hits:
|
|
38632
|
+
hits: sql15`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
38633
|
+
rows: sql15`COUNT(*)`
|
|
38319
38634
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
38320
38635
|
crawlerSegments = segmentCrawlerHits(
|
|
38321
38636
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
38322
38637
|
);
|
|
38323
38638
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
38639
|
+
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
38640
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql15`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
38641
|
+
const crawlerSeries = app.db.select({
|
|
38642
|
+
bucket: crawlerSeriesBucket,
|
|
38643
|
+
hits: sql15`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
38644
|
+
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
38645
|
+
for (const row of crawlerSeries) {
|
|
38646
|
+
trafficSeriesPoint(seriesByBucket, row.bucket).crawlerHits = Number(row.hits);
|
|
38647
|
+
}
|
|
38324
38648
|
const rows = app.db.select().from(crawlerEventsHourly).where(crawlerWhere).orderBy(desc15(crawlerEventsHourly.tsHour)).limit(limit).all();
|
|
38325
38649
|
for (const r of rows) {
|
|
38326
38650
|
events.push({
|
|
@@ -38345,8 +38669,20 @@ async function trafficRoutes(app, opts) {
|
|
|
38345
38669
|
];
|
|
38346
38670
|
if (sourceIdParam) userFetchFilters.push(eq31(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
38347
38671
|
const userFetchWhere = and26(...userFetchFilters);
|
|
38348
|
-
const total = app.db.select({
|
|
38672
|
+
const total = app.db.select({
|
|
38673
|
+
total: sql15`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
38674
|
+
rows: sql15`COUNT(*)`
|
|
38675
|
+
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
38349
38676
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
38677
|
+
totalEventRows += Number(total?.rows ?? 0);
|
|
38678
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql15`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
38679
|
+
const userFetchSeries = app.db.select({
|
|
38680
|
+
bucket: userFetchSeriesBucket,
|
|
38681
|
+
hits: sql15`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
38682
|
+
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
38683
|
+
for (const row of userFetchSeries) {
|
|
38684
|
+
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
38685
|
+
}
|
|
38350
38686
|
const rows = app.db.select().from(aiUserFetchEventsHourly).where(userFetchWhere).orderBy(desc15(aiUserFetchEventsHourly.tsHour)).limit(limit).all();
|
|
38351
38687
|
for (const r of rows) {
|
|
38352
38688
|
events.push({
|
|
@@ -38371,15 +38707,25 @@ async function trafficRoutes(app, opts) {
|
|
|
38371
38707
|
if (sourceIdParam) aiFilters.push(eq31(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
38372
38708
|
const aiWhere = and26(...aiFilters);
|
|
38373
38709
|
const total = app.db.select({
|
|
38374
|
-
total:
|
|
38375
|
-
paid:
|
|
38376
|
-
organic:
|
|
38710
|
+
total: sql15`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
38711
|
+
paid: sql15`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
38712
|
+
organic: sql15`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`,
|
|
38713
|
+
rows: sql15`COUNT(*)`
|
|
38377
38714
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
38378
38715
|
aiReferralCounts = aiReferralClassCounts(
|
|
38379
38716
|
Number(total?.total ?? 0),
|
|
38380
38717
|
Number(total?.paid ?? 0),
|
|
38381
38718
|
Number(total?.organic ?? 0)
|
|
38382
38719
|
);
|
|
38720
|
+
totalEventRows += Number(total?.rows ?? 0);
|
|
38721
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql15`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
38722
|
+
const referralSeries = app.db.select({
|
|
38723
|
+
bucket: referralSeriesBucket,
|
|
38724
|
+
hits: sql15`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
38725
|
+
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
38726
|
+
for (const row of referralSeries) {
|
|
38727
|
+
trafficSeriesPoint(seriesByBucket, row.bucket).aiReferralHits = Number(row.hits);
|
|
38728
|
+
}
|
|
38383
38729
|
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc15(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
38384
38730
|
for (const r of rows) {
|
|
38385
38731
|
const counts2 = aiReferralClassCounts(r.sessionsOrHits, r.paidSessionsOrHits, r.organicSessionsOrHits);
|
|
@@ -38405,6 +38751,10 @@ async function trafficRoutes(app, opts) {
|
|
|
38405
38751
|
const response = {
|
|
38406
38752
|
windowStart: sinceIso,
|
|
38407
38753
|
windowEnd: untilIso,
|
|
38754
|
+
series: {
|
|
38755
|
+
granularity,
|
|
38756
|
+
points: completeTrafficSeries(since, until, granularity, seriesByBucket)
|
|
38757
|
+
},
|
|
38408
38758
|
totals: {
|
|
38409
38759
|
crawlerHits: crawlerTotal,
|
|
38410
38760
|
crawlerContentHits: crawlerSegments.content,
|
|
@@ -38416,6 +38766,11 @@ async function trafficRoutes(app, opts) {
|
|
|
38416
38766
|
aiReferralOrganicHits: aiReferralCounts.organic,
|
|
38417
38767
|
aiReferralUnknownHits: aiReferralCounts.unknown
|
|
38418
38768
|
},
|
|
38769
|
+
eventRows: {
|
|
38770
|
+
total: totalEventRows,
|
|
38771
|
+
returned: trimmed.length,
|
|
38772
|
+
truncated: trimmed.length < totalEventRows
|
|
38773
|
+
},
|
|
38419
38774
|
events: trimmed
|
|
38420
38775
|
};
|
|
38421
38776
|
return response;
|
|
@@ -38964,7 +39319,7 @@ var winnabilityCoverageCheck = {
|
|
|
38964
39319
|
};
|
|
38965
39320
|
var CONTENT_CHECKS = [winnabilityCoverageCheck];
|
|
38966
39321
|
var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
38967
|
-
CONTENT_CHECKS.map((
|
|
39322
|
+
CONTENT_CHECKS.map((check2) => [check2.id, check2])
|
|
38968
39323
|
);
|
|
38969
39324
|
|
|
38970
39325
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
@@ -39515,7 +39870,7 @@ var GBP_AUTH_CHECKS = [
|
|
|
39515
39870
|
recentSyncCheck
|
|
39516
39871
|
];
|
|
39517
39872
|
var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
39518
|
-
GBP_AUTH_CHECKS.map((
|
|
39873
|
+
GBP_AUTH_CHECKS.map((check2) => [check2.id, check2])
|
|
39519
39874
|
);
|
|
39520
39875
|
|
|
39521
39876
|
// ../api-routes/src/doctor/checks/places.ts
|
|
@@ -39593,7 +39948,7 @@ var apiKeyCheck = {
|
|
|
39593
39948
|
};
|
|
39594
39949
|
var PLACES_CHECKS = [apiKeyCheck];
|
|
39595
39950
|
var PLACES_CHECK_BY_ID = Object.fromEntries(
|
|
39596
|
-
PLACES_CHECKS.map((
|
|
39951
|
+
PLACES_CHECKS.map((check2) => [check2.id, check2])
|
|
39597
39952
|
);
|
|
39598
39953
|
|
|
39599
39954
|
// ../api-routes/src/doctor/checks/google-auth.ts
|
|
@@ -39900,7 +40255,7 @@ var GOOGLE_AUTH_CHECKS = [
|
|
|
39900
40255
|
scopesCheck2
|
|
39901
40256
|
];
|
|
39902
40257
|
var GOOGLE_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
39903
|
-
GOOGLE_AUTH_CHECKS.map((
|
|
40258
|
+
GOOGLE_AUTH_CHECKS.map((check2) => [check2.id, check2])
|
|
39904
40259
|
);
|
|
39905
40260
|
|
|
39906
40261
|
// ../api-routes/src/doctor/checks/providers.ts
|
|
@@ -40061,7 +40416,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
40061
40416
|
];
|
|
40062
40417
|
|
|
40063
40418
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
40064
|
-
import { and as and29, eq as eq37, gte as gte8, ne as ne5, sql as
|
|
40419
|
+
import { and as and29, eq as eq37, gte as gte8, ne as ne5, sql as sql16 } from "drizzle-orm";
|
|
40065
40420
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
40066
40421
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
40067
40422
|
function skippedNoProject5() {
|
|
@@ -40155,7 +40510,7 @@ var recentDataCheck = {
|
|
|
40155
40510
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
40156
40511
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
40157
40512
|
const recentCrawlers = Number(
|
|
40158
|
-
ctx.db.select({ total:
|
|
40513
|
+
ctx.db.select({ total: sql16`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
40159
40514
|
and29(
|
|
40160
40515
|
eq37(crawlerEventsHourly.projectId, ctx.project.id),
|
|
40161
40516
|
gte8(crawlerEventsHourly.tsHour, warnCutoff)
|
|
@@ -40163,7 +40518,7 @@ var recentDataCheck = {
|
|
|
40163
40518
|
).get()?.total ?? 0
|
|
40164
40519
|
);
|
|
40165
40520
|
const recentReferrals = Number(
|
|
40166
|
-
ctx.db.select({ total:
|
|
40521
|
+
ctx.db.select({ total: sql16`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
40167
40522
|
and29(
|
|
40168
40523
|
eq37(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
40169
40524
|
gte8(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
@@ -40179,7 +40534,7 @@ var recentDataCheck = {
|
|
|
40179
40534
|
};
|
|
40180
40535
|
}
|
|
40181
40536
|
const olderCrawlers = Number(
|
|
40182
|
-
ctx.db.select({ total:
|
|
40537
|
+
ctx.db.select({ total: sql16`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
40183
40538
|
and29(
|
|
40184
40539
|
eq37(crawlerEventsHourly.projectId, ctx.project.id),
|
|
40185
40540
|
gte8(crawlerEventsHourly.tsHour, failCutoff)
|
|
@@ -40187,7 +40542,7 @@ var recentDataCheck = {
|
|
|
40187
40542
|
).get()?.total ?? 0
|
|
40188
40543
|
);
|
|
40189
40544
|
const olderReferrals = Number(
|
|
40190
|
-
ctx.db.select({ total:
|
|
40545
|
+
ctx.db.select({ total: sql16`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
40191
40546
|
and29(
|
|
40192
40547
|
eq37(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
40193
40548
|
gte8(aiReferralEventsHourly.tsHour, failCutoff)
|
|
@@ -40499,9 +40854,9 @@ async function runChecks(ctx, checks, options = {}) {
|
|
|
40499
40854
|
const filters = options.checkIds ?? [];
|
|
40500
40855
|
const targetScope = ctx.project ? CheckScopes.project : CheckScopes.global;
|
|
40501
40856
|
const projectName = ctx.project?.name ?? null;
|
|
40502
|
-
const selected = checks.filter((
|
|
40503
|
-
if (
|
|
40504
|
-
return matchesCheckId(
|
|
40857
|
+
const selected = checks.filter((check2) => {
|
|
40858
|
+
if (check2.scope !== targetScope) return false;
|
|
40859
|
+
return matchesCheckId(check2.id, filters);
|
|
40505
40860
|
});
|
|
40506
40861
|
const results = [];
|
|
40507
40862
|
for (const definition of selected) {
|
|
@@ -41439,6 +41794,80 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
41439
41794
|
});
|
|
41440
41795
|
}
|
|
41441
41796
|
|
|
41797
|
+
// ../api-routes/src/research.ts
|
|
41798
|
+
import crypto33 from "crypto";
|
|
41799
|
+
import { and as and32, desc as desc18, eq as eq41 } from "drizzle-orm";
|
|
41800
|
+
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
41801
|
+
async function researchRoutes(app, opts) {
|
|
41802
|
+
app.post("/projects/:name/research/runs", async (request, reply) => {
|
|
41803
|
+
const project = resolveProject(app.db, request.params.name);
|
|
41804
|
+
if (!opts.onResearchRunRequested) throw missingDependency("Research execution is not available on this deployment.", { reason: "no-research-handler" });
|
|
41805
|
+
const parsed = researchRunCreateSchema.safeParse(request.body ?? {});
|
|
41806
|
+
if (!parsed.success) throw validationError("Invalid research run request", { issues: parsed.error.issues });
|
|
41807
|
+
const input = parsed.data;
|
|
41808
|
+
const adapters = opts.providerAdapters ?? [];
|
|
41809
|
+
const configured = new Set(opts.configuredProviderNames ?? []);
|
|
41810
|
+
const providerName = input.provider ?? project.providers.find((name) => configured.has(name) && adapters.some((adapter2) => adapter2.name === name && adapter2.mode === "api")) ?? adapters.find((adapter2) => adapter2.mode === "api" && configured.has(adapter2.name))?.name;
|
|
41811
|
+
const adapter = adapters.find((candidate) => candidate.name === providerName);
|
|
41812
|
+
if (!providerName || !adapter || adapter.mode !== "api" || isBrowserProvider(providerName) || !configured.has(providerName)) throw validationError("Research requires a configured API provider.", { provider: input.provider, validProviders: adapters.filter((a) => a.mode === "api" && configured.has(a.name)).map((a) => a.name) });
|
|
41813
|
+
if (input.model) {
|
|
41814
|
+
adapter.modelValidationPattern.lastIndex = 0;
|
|
41815
|
+
if (!adapter.modelConfigurable || !adapter.modelValidationPattern.test(input.model)) throw validationError(`Invalid model "${input.model}" for provider "${providerName}".`, { provider: providerName, model: input.model, hint: adapter.modelValidationHint });
|
|
41816
|
+
}
|
|
41817
|
+
const location = input.location === void 0 ? project.defaultLocation ? project.locations.find((item) => item.label === project.defaultLocation) ?? null : null : input.location;
|
|
41818
|
+
if (location && !project.locations.some((item) => sameLocation(item, location))) throw validationError("Research location must exactly match a configured project location.", { location });
|
|
41819
|
+
const requestedModel = input.model ?? null;
|
|
41820
|
+
const resolvedModel = requestedModel ?? (project.providerModels[providerName] || adapter.defaultModel);
|
|
41821
|
+
adapter.modelValidationPattern.lastIndex = 0;
|
|
41822
|
+
if (!adapter.modelValidationPattern.test(resolvedModel)) throw validationError(`Invalid resolved model "${resolvedModel}" for provider "${providerName}".`, { provider: providerName, model: resolvedModel, hint: adapter.modelValidationHint });
|
|
41823
|
+
if (new Set(input.queries.map((query) => query.toLocaleLowerCase())).size !== input.queries.length) throw validationError("Research queries must be unique within a batch.");
|
|
41824
|
+
const normalized = { queries: input.queries, provider: providerName, model: requestedModel, location: location ?? null };
|
|
41825
|
+
const requestHash2 = crypto33.createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
|
|
41826
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
41827
|
+
const decision = app.db.transaction((tx) => {
|
|
41828
|
+
if (input.idempotencyKey) {
|
|
41829
|
+
const existing = tx.select().from(researchRuns).where(and32(eq41(researchRuns.projectId, project.id), eq41(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
41830
|
+
if (existing) {
|
|
41831
|
+
if (existing.requestHash !== requestHash2) throw alreadyExists("Research idempotency key", input.idempotencyKey);
|
|
41832
|
+
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
41833
|
+
}
|
|
41834
|
+
}
|
|
41835
|
+
const id = crypto33.randomUUID();
|
|
41836
|
+
tx.insert(researchRuns).values({ id, projectId: project.id, status: ResearchRunStatuses.queued, provider: providerName, requestedModel, resolvedModel, location: location ?? null, totalQueries: input.queries.length, idempotencyKey: input.idempotencyKey ?? null, requestHash: input.idempotencyKey ? requestHash2 : null, createdAt: now }).run();
|
|
41837
|
+
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id: crypto33.randomUUID(), researchRunId: id, position, queryText: query, status: ResearchQueryStatuses.queued, requestedModel, resolvedModel, groundingSources: [], citedDomains: [], searchQueries: [], createdAt: now }).run();
|
|
41838
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
41839
|
+
return { reused: false, id, shouldDispatch: true };
|
|
41840
|
+
});
|
|
41841
|
+
const result = getDetail(app, project.id, decision.id);
|
|
41842
|
+
if (decision.shouldDispatch) opts.onResearchRunRequested(decision.id, project.id);
|
|
41843
|
+
if (decision.reused) return reply.status(200).send(result);
|
|
41844
|
+
return reply.status(202).send(result);
|
|
41845
|
+
});
|
|
41846
|
+
app.get("/projects/:name/research/runs", async (request) => {
|
|
41847
|
+
const project = resolveProject(app.db, request.params.name);
|
|
41848
|
+
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
41849
|
+
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
41850
|
+
const runs2 = app.db.select().from(researchRuns).where(eq41(researchRuns.projectId, project.id)).orderBy(desc18(researchRuns.createdAt)).limit(limit).all().map(serializeRun);
|
|
41851
|
+
return { runs: runs2 };
|
|
41852
|
+
});
|
|
41853
|
+
app.get("/projects/:name/research/runs/:runId", async (request) => {
|
|
41854
|
+
const project = resolveProject(app.db, request.params.name);
|
|
41855
|
+
return getDetail(app, project.id, request.params.runId);
|
|
41856
|
+
});
|
|
41857
|
+
}
|
|
41858
|
+
function getDetail(app, projectId, id) {
|
|
41859
|
+
const row = app.db.select().from(researchRuns).where(and32(eq41(researchRuns.id, id), eq41(researchRuns.projectId, projectId))).get();
|
|
41860
|
+
if (!row) throw notFound("Research run", id);
|
|
41861
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq41(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
41862
|
+
return { ...serializeRun(row), queries: queries2 };
|
|
41863
|
+
}
|
|
41864
|
+
function serializeRun(row) {
|
|
41865
|
+
return { id: row.id, projectId: row.projectId, status: row.status, provider: row.provider, requestedModel: row.requestedModel, resolvedModel: row.resolvedModel, location: row.location ?? null, totalQueries: row.totalQueries, completedQueries: row.completedQueries, failedQueries: row.failedQueries, error: row.error, startedAt: row.startedAt, finishedAt: row.finishedAt, createdAt: row.createdAt };
|
|
41866
|
+
}
|
|
41867
|
+
function serializeQuery(row) {
|
|
41868
|
+
return { id: row.id, position: row.position, query: row.queryText, status: row.status, requestedModel: row.requestedModel, resolvedModel: row.resolvedModel, servedModel: row.servedModel, answerText: row.answerText, groundingSources: row.groundingSources, citedDomains: row.citedDomains, searchQueries: row.searchQueries, answerMentioned: row.answerMentioned, citationState: row.citationState, error: row.error, startedAt: row.startedAt, finishedAt: row.finishedAt, createdAt: row.createdAt };
|
|
41869
|
+
}
|
|
41870
|
+
|
|
41442
41871
|
// ../api-routes/src/index.ts
|
|
41443
41872
|
async function apiRoutes(app, opts) {
|
|
41444
41873
|
app.decorate("db", opts.db);
|
|
@@ -41625,6 +42054,11 @@ async function apiRoutes(app, opts) {
|
|
|
41625
42054
|
harvestSearchQueries: opts.harvestSearchQueries,
|
|
41626
42055
|
embedQueries: opts.embedQueries
|
|
41627
42056
|
});
|
|
42057
|
+
await api.register(researchRoutes, {
|
|
42058
|
+
providerAdapters: opts.providerAdapters,
|
|
42059
|
+
configuredProviderNames: opts.providerSummary?.filter((provider) => provider.configured).map((provider) => provider.name),
|
|
42060
|
+
onResearchRunRequested: opts.onResearchRunRequested
|
|
42061
|
+
});
|
|
41628
42062
|
await api.register(technicalAeoRoutes, {
|
|
41629
42063
|
onSiteAuditRequested: opts.onSiteAuditRequested
|
|
41630
42064
|
});
|
|
@@ -41786,7 +42220,7 @@ function buildTrafficSourceValidators(opts) {
|
|
|
41786
42220
|
}
|
|
41787
42221
|
|
|
41788
42222
|
// src/intelligence-service.ts
|
|
41789
|
-
import
|
|
42223
|
+
import crypto34 from "crypto";
|
|
41790
42224
|
|
|
41791
42225
|
// src/logger.ts
|
|
41792
42226
|
var IS_TTY = process.stdout.isTTY === true;
|
|
@@ -42018,16 +42452,16 @@ var IntelligenceService = class {
|
|
|
42018
42452
|
*/
|
|
42019
42453
|
analyzeAndPersist(runId, projectId) {
|
|
42020
42454
|
const recentRuns = this.db.select().from(runs).where(
|
|
42021
|
-
|
|
42022
|
-
|
|
42023
|
-
or8(
|
|
42455
|
+
and33(
|
|
42456
|
+
eq42(runs.projectId, projectId),
|
|
42457
|
+
or8(eq42(runs.status, "completed"), eq42(runs.status, "partial")),
|
|
42024
42458
|
// Defensive: RunCoordinator already skips probes before this is
|
|
42025
42459
|
// called, but if a future call site invokes analyzeAndPersist
|
|
42026
42460
|
// directly for a probe, probes still must not pollute the
|
|
42027
42461
|
// intelligence window.
|
|
42028
42462
|
ne6(runs.trigger, RunTriggers.probe)
|
|
42029
42463
|
)
|
|
42030
|
-
).orderBy(
|
|
42464
|
+
).orderBy(desc19(runs.finishedAt), desc19(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
42031
42465
|
if (recentRuns.length === 0) {
|
|
42032
42466
|
log.info("intelligence.skip", { runId, reason: "no completed runs" });
|
|
42033
42467
|
return null;
|
|
@@ -42102,7 +42536,7 @@ var IntelligenceService = class {
|
|
|
42102
42536
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
42103
42537
|
*/
|
|
42104
42538
|
analyzeAndPersistGbp(runId, projectId) {
|
|
42105
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
42539
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq42(runs.id, runId)).get();
|
|
42106
42540
|
if (!runRow) {
|
|
42107
42541
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
42108
42542
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -42110,9 +42544,9 @@ var IntelligenceService = class {
|
|
|
42110
42544
|
}
|
|
42111
42545
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
42112
42546
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
42113
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
42114
|
-
|
|
42115
|
-
|
|
42547
|
+
const selected = this.db.select().from(gbpLocations).where(and33(
|
|
42548
|
+
eq42(gbpLocations.projectId, projectId),
|
|
42549
|
+
eq42(gbpLocations.selected, true),
|
|
42116
42550
|
gte10(gbpLocations.syncedAt, windowStart),
|
|
42117
42551
|
lte6(gbpLocations.syncedAt, windowEnd)
|
|
42118
42552
|
)).all();
|
|
@@ -42147,12 +42581,12 @@ var IntelligenceService = class {
|
|
|
42147
42581
|
}
|
|
42148
42582
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
42149
42583
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
42150
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
42151
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
42152
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
42153
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
42584
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and33(eq42(gbpDailyMetrics.projectId, projectId), eq42(gbpDailyMetrics.locationName, locationName))).all();
|
|
42585
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and33(eq42(gbpPlaceActions.projectId, projectId), eq42(gbpPlaceActions.locationName, locationName))).all();
|
|
42586
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and33(eq42(gbpLodgingSnapshots.projectId, projectId), eq42(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc19(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
42587
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and33(eq42(gbpLocations.projectId, projectId), eq42(gbpLocations.locationName, locationName))).get();
|
|
42154
42588
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
42155
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
42589
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and33(eq42(gbpPlaceDetails.projectId, projectId), eq42(gbpPlaceDetails.locationName, locationName))).orderBy(desc19(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
42156
42590
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
42157
42591
|
const summary = buildGbpSummary({
|
|
42158
42592
|
locationName,
|
|
@@ -42188,7 +42622,7 @@ var IntelligenceService = class {
|
|
|
42188
42622
|
/** Build the month-over-month keyword series for a location from the
|
|
42189
42623
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
42190
42624
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
42191
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
42625
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and33(eq42(gbpKeywordMonthly.projectId, projectId), eq42(gbpKeywordMonthly.locationName, locationName))).all();
|
|
42192
42626
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
42193
42627
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
42194
42628
|
const recentMonth = months[0] ?? null;
|
|
@@ -42219,7 +42653,7 @@ var IntelligenceService = class {
|
|
|
42219
42653
|
*/
|
|
42220
42654
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
42221
42655
|
const covered = new Set(coveredLocationNames);
|
|
42222
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
42656
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and33(eq42(insights.projectId, projectId), eq42(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
42223
42657
|
const staleIds = [];
|
|
42224
42658
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
42225
42659
|
for (const row of existing) {
|
|
@@ -42230,7 +42664,7 @@ var IntelligenceService = class {
|
|
|
42230
42664
|
}
|
|
42231
42665
|
this.db.transaction((tx) => {
|
|
42232
42666
|
for (const id of staleIds) {
|
|
42233
|
-
tx.delete(insights).where(
|
|
42667
|
+
tx.delete(insights).where(eq42(insights.id, id)).run();
|
|
42234
42668
|
}
|
|
42235
42669
|
for (const insight of gbpInsights) {
|
|
42236
42670
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -42308,7 +42742,7 @@ var IntelligenceService = class {
|
|
|
42308
42742
|
* create per run + aggregate). DB is left untouched.
|
|
42309
42743
|
*/
|
|
42310
42744
|
backfill(projectName, opts, onProgress) {
|
|
42311
|
-
const project = this.db.select().from(projects).where(
|
|
42745
|
+
const project = this.db.select().from(projects).where(eq42(projects.name, projectName)).get();
|
|
42312
42746
|
if (!project) {
|
|
42313
42747
|
throw new Error(`Project "${projectName}" not found`);
|
|
42314
42748
|
}
|
|
@@ -42321,9 +42755,9 @@ var IntelligenceService = class {
|
|
|
42321
42755
|
sinceTimestamp = parsed;
|
|
42322
42756
|
}
|
|
42323
42757
|
const allRuns = this.db.select().from(runs).where(
|
|
42324
|
-
|
|
42325
|
-
|
|
42326
|
-
or8(
|
|
42758
|
+
and33(
|
|
42759
|
+
eq42(runs.projectId, project.id),
|
|
42760
|
+
or8(eq42(runs.status, "completed"), eq42(runs.status, "partial")),
|
|
42327
42761
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
42328
42762
|
ne6(runs.trigger, RunTriggers.probe)
|
|
42329
42763
|
)
|
|
@@ -42402,7 +42836,7 @@ var IntelligenceService = class {
|
|
|
42402
42836
|
return { processed, skipped, totalInsights };
|
|
42403
42837
|
}
|
|
42404
42838
|
loadTrackedCompetitors(projectId) {
|
|
42405
|
-
return this.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
42839
|
+
return this.db.select({ domain: competitors.domain }).from(competitors).where(eq42(competitors.projectId, projectId)).all().map((r) => r.domain);
|
|
42406
42840
|
}
|
|
42407
42841
|
/**
|
|
42408
42842
|
* Wipe transition signals from an analysis result while keeping health.
|
|
@@ -42423,15 +42857,15 @@ var IntelligenceService = class {
|
|
|
42423
42857
|
}
|
|
42424
42858
|
persistResult(result, runId, projectId) {
|
|
42425
42859
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
42426
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
42860
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq42(insights.runId, runId)).all();
|
|
42427
42861
|
for (const row of existingInsights) {
|
|
42428
42862
|
if (row.dismissed) {
|
|
42429
42863
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
42430
42864
|
}
|
|
42431
42865
|
}
|
|
42432
42866
|
this.db.transaction((tx) => {
|
|
42433
|
-
tx.delete(insights).where(
|
|
42434
|
-
tx.delete(healthSnapshots).where(
|
|
42867
|
+
tx.delete(insights).where(eq42(insights.runId, runId)).run();
|
|
42868
|
+
tx.delete(healthSnapshots).where(eq42(healthSnapshots.runId, runId)).run();
|
|
42435
42869
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
42436
42870
|
for (const insight of result.insights) {
|
|
42437
42871
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -42451,7 +42885,7 @@ var IntelligenceService = class {
|
|
|
42451
42885
|
}).run();
|
|
42452
42886
|
}
|
|
42453
42887
|
tx.insert(healthSnapshots).values({
|
|
42454
|
-
id:
|
|
42888
|
+
id: crypto34.randomUUID(),
|
|
42455
42889
|
projectId,
|
|
42456
42890
|
runId,
|
|
42457
42891
|
overallCitedRate: String(result.health.overallCitedRate),
|
|
@@ -42484,28 +42918,28 @@ var IntelligenceService = class {
|
|
|
42484
42918
|
applySeverityTiering(rawInsights, excludeRunId, projectId) {
|
|
42485
42919
|
const regressions = rawInsights.filter((i) => i.type === "regression");
|
|
42486
42920
|
if (regressions.length === 0) return rawInsights;
|
|
42487
|
-
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(
|
|
42921
|
+
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(eq42(gscSearchData.projectId, projectId)).all();
|
|
42488
42922
|
const gscConnected = gscRows.length > 0;
|
|
42489
42923
|
const gscImpressionsByQuery = /* @__PURE__ */ new Map();
|
|
42490
42924
|
for (const row of gscRows) {
|
|
42491
42925
|
const key = row.query.toLowerCase();
|
|
42492
42926
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
42493
42927
|
}
|
|
42494
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
42928
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq42(projects.id, projectId)).get();
|
|
42495
42929
|
const locationCount = Math.max(
|
|
42496
42930
|
1,
|
|
42497
42931
|
(projectRow?.locations ?? []).length
|
|
42498
42932
|
);
|
|
42499
42933
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
42500
42934
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
42501
|
-
|
|
42502
|
-
|
|
42503
|
-
|
|
42504
|
-
or8(
|
|
42935
|
+
and33(
|
|
42936
|
+
eq42(runs.projectId, projectId),
|
|
42937
|
+
eq42(runs.kind, RunKinds["answer-visibility"]),
|
|
42938
|
+
or8(eq42(runs.status, "completed"), eq42(runs.status, "partial")),
|
|
42505
42939
|
// Defensive — see top of file.
|
|
42506
42940
|
ne6(runs.trigger, RunTriggers.probe)
|
|
42507
42941
|
)
|
|
42508
|
-
).orderBy(
|
|
42942
|
+
).orderBy(desc19(runs.createdAt), desc19(runs.id)).limit((RECURRENCE_LOOKBACK_RUNS + 1) * ROWS_PER_GROUP_BUDGET).all();
|
|
42509
42943
|
const recentGroups = groupRunsByCreatedAt(recentRunRows);
|
|
42510
42944
|
const recentRunIds = [];
|
|
42511
42945
|
const recentRunIdToCreatedAt = /* @__PURE__ */ new Map();
|
|
@@ -42521,7 +42955,7 @@ var IntelligenceService = class {
|
|
|
42521
42955
|
const haveHistory = recentRunIds.length > 0;
|
|
42522
42956
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
42523
42957
|
if (haveHistory) {
|
|
42524
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
42958
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and33(eq42(insights.type, "regression"), inArray15(insights.runId, recentRunIds))).all();
|
|
42525
42959
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
42526
42960
|
for (const row of priorRows) {
|
|
42527
42961
|
if (!row.runId) continue;
|
|
@@ -42550,7 +42984,7 @@ var IntelligenceService = class {
|
|
|
42550
42984
|
});
|
|
42551
42985
|
}
|
|
42552
42986
|
buildRunData(runId, projectId, completedAt, location = null) {
|
|
42553
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
42987
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq42(projects.id, projectId)).get();
|
|
42554
42988
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
42555
42989
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
42556
42990
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -42567,7 +43001,7 @@ var IntelligenceService = class {
|
|
|
42567
43001
|
citedDomains: querySnapshots.citedDomains,
|
|
42568
43002
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
42569
43003
|
snapshotLocation: querySnapshots.location
|
|
42570
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
43004
|
+
}).from(querySnapshots).leftJoin(queries, eq42(querySnapshots.queryId, queries.id)).where(eq42(querySnapshots.runId, runId)).all();
|
|
42571
43005
|
const snapshots = [];
|
|
42572
43006
|
let orphanCount = 0;
|
|
42573
43007
|
for (const r of rows) {
|
|
@@ -42643,6 +43077,8 @@ export {
|
|
|
42643
43077
|
aiUserFetchEventsHourly,
|
|
42644
43078
|
rawEventSamples,
|
|
42645
43079
|
discoverySessions,
|
|
43080
|
+
researchRuns,
|
|
43081
|
+
researchRunQueries,
|
|
42646
43082
|
gbpLocations,
|
|
42647
43083
|
gbpDailyMetrics,
|
|
42648
43084
|
gbpKeywordImpressions,
|