@ainyc/canonry 4.115.0 → 4.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{BacklinksPage-DhXsZ8Sb.js → BacklinksPage-Dy6Xc-LT.js} +1 -1
- package/assets/assets/{ChartPrimitives-Cdj0P8uG.js → ChartPrimitives-CHJh5M8x.js} +1 -1
- package/assets/assets/ProjectPage-Bjfa4CI5.js +7 -0
- package/assets/assets/{RunRow-GJ_2CEif.js → RunRow-DmvQnWPf.js} +1 -1
- package/assets/assets/{RunsPage-upaDiLFK.js → RunsPage-BxkxQEKE.js} +1 -1
- package/assets/assets/{SettingsPage-FbmEaN16.js → SettingsPage-DCyqHqDh.js} +1 -1
- package/assets/assets/{TrafficPage-Di0H5JRu.js → TrafficPage-BT_ae5aO.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-Bc2mxXSV.js → TrafficSourceDetailPage-Cps-tSHn.js} +1 -1
- package/assets/assets/{arrow-left-yOCIy3_x.js → arrow-left-D8vdqhx4.js} +1 -1
- package/assets/assets/{extract-error-message-tCNIzznL.js → extract-error-message-BOY5txUA.js} +1 -1
- package/assets/assets/{index-Bb5yUz7k.css → index-B4R1-8hR.css} +1 -1
- package/assets/assets/{index-D0ZdnFQf.js → index-DxjY63VC.js} +89 -89
- package/assets/assets/{trash-2-gRyTO-PS.js → trash-2-ap_Fa4zA.js} +1 -1
- package/assets/index.html +2 -2
- package/dist/{chunk-PGVGHGIX.js → chunk-GV665WOU.js} +4 -4
- package/dist/{chunk-VEUBAZRG.js → chunk-IQPMK36Y.js} +1352 -1199
- package/dist/{chunk-MO7GPMNV.js → chunk-RVH6QPTA.js} +1 -1
- package/dist/{chunk-BXNFOGDF.js → chunk-WEAXJ25Y.js} +420 -87
- package/dist/cli.js +57 -9
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-2C7H4C6A.js → intelligence-service-HEN5HLUM.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +7 -7
- package/assets/assets/ProjectPage-DAWtJQ3w.js +0 -7
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
AGENT_PROVIDER_IDS,
|
|
3
3
|
AI_ENGINE_DOMAINS,
|
|
4
4
|
AI_PROVIDER_INFRA_DOMAINS,
|
|
5
|
+
AiReferralTrafficClasses,
|
|
5
6
|
AppError,
|
|
6
7
|
BacklinkSources,
|
|
7
8
|
CcReleaseSyncStatuses,
|
|
@@ -49,6 +50,7 @@ import {
|
|
|
49
50
|
adsSyncResponseSchema,
|
|
50
51
|
agentProvidersResponseDtoSchema,
|
|
51
52
|
aggregateHarvestedQueries,
|
|
53
|
+
aiReferralClassCounts,
|
|
52
54
|
apiKeyDtoSchema,
|
|
53
55
|
apiKeyListDtoSchema,
|
|
54
56
|
applyHarvestSemanticNovelty,
|
|
@@ -86,6 +88,7 @@ import {
|
|
|
86
88
|
citationStateSchema,
|
|
87
89
|
citationStateToCited,
|
|
88
90
|
citationVisibilityResponseSchema,
|
|
91
|
+
classifyAiReferralTrafficClass,
|
|
89
92
|
classifyCitedSurface,
|
|
90
93
|
classifySkillFile,
|
|
91
94
|
classifySurfaceFromCategory,
|
|
@@ -130,6 +133,7 @@ import {
|
|
|
130
133
|
filterBrandedSeedCandidates,
|
|
131
134
|
findDuplicateLocationLabels,
|
|
132
135
|
forbidden,
|
|
136
|
+
formatAiReferralClassSummary,
|
|
133
137
|
formatAverageDelta,
|
|
134
138
|
formatDate,
|
|
135
139
|
formatDateRange,
|
|
@@ -273,7 +277,7 @@ import {
|
|
|
273
277
|
wordpressSchemaDeployResultDtoSchema,
|
|
274
278
|
wordpressSchemaStatusResultDtoSchema,
|
|
275
279
|
wordpressStatusDtoSchema
|
|
276
|
-
} from "./chunk-
|
|
280
|
+
} from "./chunk-IQPMK36Y.js";
|
|
277
281
|
|
|
278
282
|
// src/intelligence-service.ts
|
|
279
283
|
import { eq as eq38, desc as desc18, asc as asc6, and as and30, ne as ne5, or as or6, inArray as inArray14, gte as gte7, lte as lte4 } from "drizzle-orm";
|
|
@@ -758,6 +762,7 @@ var gaAiReferrals = sqliteTable("ga_ai_referrals", {
|
|
|
758
762
|
date: text("date").notNull(),
|
|
759
763
|
source: text("source").notNull(),
|
|
760
764
|
medium: text("medium").notNull(),
|
|
765
|
+
trafficClass: text("traffic_class").$type().notNull().default("organic"),
|
|
761
766
|
/** Which GA4 dimension produced this row: 'session' | 'first_user' | 'manual_utm' */
|
|
762
767
|
sourceDimension: text("source_dimension").notNull().default("session"),
|
|
763
768
|
/** GA4 default channel group for the session (e.g. 'Referral', 'Organic Social'). */
|
|
@@ -772,6 +777,7 @@ var gaAiReferrals = sqliteTable("ga_ai_referrals", {
|
|
|
772
777
|
index("idx_ga_ai_ref_project_date").on(table.projectId, table.date),
|
|
773
778
|
index("idx_ga_ai_ref_source").on(table.source),
|
|
774
779
|
index("idx_ga_ai_ref_landing_page").on(table.projectId, table.date, table.landingPageNormalized),
|
|
780
|
+
index("idx_ga_ai_ref_traffic_class").on(table.projectId, table.date, table.trafficClass),
|
|
775
781
|
uniqueIndex("idx_ga_ai_ref_unique_v4").on(table.projectId, table.date, table.source, table.medium, table.sourceDimension, table.channelGroup, table.landingPage),
|
|
776
782
|
index("idx_ga_ai_ref_run").on(table.syncRunId)
|
|
777
783
|
]);
|
|
@@ -1096,6 +1102,17 @@ var aiReferralEventsHourly = sqliteTable("ai_referral_events_hourly", {
|
|
|
1096
1102
|
landingPathNormalized: text("landing_path_normalized").notNull(),
|
|
1097
1103
|
status: integer("status").notNull(),
|
|
1098
1104
|
sessionsOrHits: integer("sessions_or_hits").notNull().default(0),
|
|
1105
|
+
/**
|
|
1106
|
+
* Paid/organic splits the MEASURE, not the primary key. The paid marker lives
|
|
1107
|
+
* in the request's UTM tags, which `landing_path_normalized` strips, so one
|
|
1108
|
+
* bucket can legitimately hold both kinds and the class is NOT functionally
|
|
1109
|
+
* determined by the key. Unclassified arrivals are the residual
|
|
1110
|
+
* `sessions_or_hits - paid - organic`, which leaves rows written before the
|
|
1111
|
+
* ingest classifier shipped (both columns 0) visibly unknown rather than
|
|
1112
|
+
* silently organic — the regression migration v95 shipped on the GA4 side.
|
|
1113
|
+
*/
|
|
1114
|
+
paidSessionsOrHits: integer("paid_sessions_or_hits").notNull().default(0),
|
|
1115
|
+
organicSessionsOrHits: integer("organic_sessions_or_hits").notNull().default(0),
|
|
1099
1116
|
usersEstimated: integer("users_estimated"),
|
|
1100
1117
|
createdAt: text("created_at").notNull(),
|
|
1101
1118
|
updatedAt: text("updated_at").notNull()
|
|
@@ -3482,6 +3499,62 @@ var MIGRATION_VERSIONS = [
|
|
|
3482
3499
|
// post-truncation). Additive + nullable (downgrade-safe).
|
|
3483
3500
|
`ALTER TABLE discovery_sessions ADD COLUMN canonical_count INTEGER`
|
|
3484
3501
|
]
|
|
3502
|
+
},
|
|
3503
|
+
{
|
|
3504
|
+
version: 95,
|
|
3505
|
+
name: "ga-ai-referral-traffic-class",
|
|
3506
|
+
statements: [],
|
|
3507
|
+
run: (tx) => {
|
|
3508
|
+
if (!tableExists(tx, "ga_ai_referrals")) return;
|
|
3509
|
+
if (!columnExists(tx, "ga_ai_referrals", "traffic_class")) {
|
|
3510
|
+
tx.run(sql.raw(`ALTER TABLE ga_ai_referrals ADD COLUMN traffic_class TEXT NOT NULL DEFAULT 'organic'`));
|
|
3511
|
+
}
|
|
3512
|
+
tx.run(sql.raw(`CREATE INDEX IF NOT EXISTS idx_ga_ai_ref_traffic_class ON ga_ai_referrals(project_id, date, traffic_class)`));
|
|
3513
|
+
}
|
|
3514
|
+
},
|
|
3515
|
+
{
|
|
3516
|
+
// v95 added `ga_ai_referrals.traffic_class` with `DEFAULT 'organic'` and did
|
|
3517
|
+
// not classify the rows that already existed. Every historical AI referral
|
|
3518
|
+
// was therefore stamped 'organic', including paid ChatGPT-ads traffic
|
|
3519
|
+
// carrying `medium='cpc'` / `channel_group='Paid Other'`. That silently
|
|
3520
|
+
// reports purchased sessions as organic wins.
|
|
3521
|
+
//
|
|
3522
|
+
// The class is a pure function of columns already stored on each row, so
|
|
3523
|
+
// re-derive it here instead of making an operator delete and re-fetch
|
|
3524
|
+
// identical data from GA4.
|
|
3525
|
+
//
|
|
3526
|
+
// Every row is re-classified, not only the "unclassified" ones: v95's
|
|
3527
|
+
// default is indistinguishable from a genuinely-organic classification, so
|
|
3528
|
+
// there is no way to tell the two apart. `classifyAiReferralTrafficClass`
|
|
3529
|
+
// is pure and deterministic, so re-running it over already-correct rows
|
|
3530
|
+
// writes nothing, which is what makes this version safe to replay.
|
|
3531
|
+
version: 96,
|
|
3532
|
+
name: "ga-ai-referral-traffic-class-backfill",
|
|
3533
|
+
statements: [],
|
|
3534
|
+
run: (tx) => {
|
|
3535
|
+
backfillGaAiReferralTrafficClass(tx);
|
|
3536
|
+
}
|
|
3537
|
+
},
|
|
3538
|
+
{
|
|
3539
|
+
// Splits the server-side AI-referral MEASURE by traffic class rather than
|
|
3540
|
+
// adding a class column. Paid-ness lives in the request's UTM tags, which
|
|
3541
|
+
// `landing_path_normalized` strips, so one hourly bucket can legitimately
|
|
3542
|
+
// hold both paid and organic arrivals: a class column outside the 8-column
|
|
3543
|
+
// primary key would silently stamp a mixed bucket with one label, and
|
|
3544
|
+
// SQLite cannot extend a composite key with ADD COLUMN anyway.
|
|
3545
|
+
//
|
|
3546
|
+
// Deliberately NO backfill. The discriminator was never persisted — the
|
|
3547
|
+
// request's query string is dropped before the hourly bucket and before
|
|
3548
|
+
// `raw_event_samples` — so no honest class exists for historical rows.
|
|
3549
|
+
// Leaving both counters at 0 surfaces their whole `sessions_or_hits` as the
|
|
3550
|
+
// unknown residual. An older binary writing rows after a downgrade lands in
|
|
3551
|
+
// that residual too, which is the truth about those rows.
|
|
3552
|
+
version: 97,
|
|
3553
|
+
name: "server-side-ai-referral-traffic-class",
|
|
3554
|
+
statements: [
|
|
3555
|
+
`ALTER TABLE ai_referral_events_hourly ADD COLUMN paid_sessions_or_hits INTEGER NOT NULL DEFAULT 0`,
|
|
3556
|
+
`ALTER TABLE ai_referral_events_hourly ADD COLUMN organic_sessions_or_hits INTEGER NOT NULL DEFAULT 0`
|
|
3557
|
+
]
|
|
3485
3558
|
}
|
|
3486
3559
|
];
|
|
3487
3560
|
function rebuildBacklinkTableWithSource(tx, table) {
|
|
@@ -3568,6 +3641,26 @@ function tableIsEmpty(db, table) {
|
|
|
3568
3641
|
const rows = db.all(sql.raw(`SELECT COUNT(*) as c FROM ${table}`));
|
|
3569
3642
|
return (rows[0]?.c ?? 0) === 0;
|
|
3570
3643
|
}
|
|
3644
|
+
function backfillGaAiReferralTrafficClass(tx) {
|
|
3645
|
+
if (!tableExists(tx, "ga_ai_referrals")) return 0;
|
|
3646
|
+
if (!columnExists(tx, "ga_ai_referrals", "traffic_class")) return 0;
|
|
3647
|
+
const rows = tx.all(sql.raw(
|
|
3648
|
+
`SELECT id, source, medium, channel_group, landing_page, traffic_class FROM ga_ai_referrals`
|
|
3649
|
+
));
|
|
3650
|
+
let updated = 0;
|
|
3651
|
+
for (const row of rows) {
|
|
3652
|
+
const next = classifyAiReferralTrafficClass({
|
|
3653
|
+
source: row.source,
|
|
3654
|
+
medium: row.medium,
|
|
3655
|
+
channelGroup: row.channel_group,
|
|
3656
|
+
landingPage: row.landing_page
|
|
3657
|
+
});
|
|
3658
|
+
if (next === row.traffic_class) continue;
|
|
3659
|
+
tx.run(sql`UPDATE ga_ai_referrals SET traffic_class = ${next} WHERE id = ${row.id}`);
|
|
3660
|
+
updated += 1;
|
|
3661
|
+
}
|
|
3662
|
+
return updated;
|
|
3663
|
+
}
|
|
3571
3664
|
function hasLegacyQuerySchema(db) {
|
|
3572
3665
|
return tableExists(db, "keywords") || columnExists(db, "query_snapshots", "keyword_id") || columnExists(db, "insights", "keyword");
|
|
3573
3666
|
}
|
|
@@ -8137,7 +8230,7 @@ async function analyticsRoutes(app) {
|
|
|
8137
8230
|
const project = resolveProject(app.db, request.params.name);
|
|
8138
8231
|
const window = parseWindow(request.query.window);
|
|
8139
8232
|
const cutoff = windowCutoff(window);
|
|
8140
|
-
const projectRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), notProbeRun())).orderBy(runs.createdAt).all().filter((r) => r.status === "completed" || r.status === "partial").filter((r) => !cutoff || r.createdAt >= cutoff);
|
|
8233
|
+
const projectRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), eq10(runs.kind, RunKinds["answer-visibility"]), notProbeRun())).orderBy(runs.createdAt).all().filter((r) => r.status === "completed" || r.status === "partial").filter((r) => !cutoff || r.createdAt >= cutoff);
|
|
8141
8234
|
if (projectRuns.length === 0) {
|
|
8142
8235
|
return reply.send({
|
|
8143
8236
|
window,
|
|
@@ -8189,14 +8282,14 @@ async function analyticsRoutes(app) {
|
|
|
8189
8282
|
const project = resolveProject(app.db, request.params.name);
|
|
8190
8283
|
const window = parseWindow(request.query.window);
|
|
8191
8284
|
const cutoff = windowCutoff(window);
|
|
8192
|
-
const completedRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), notProbeRun())).orderBy(desc3(runs.createdAt), desc3(runs.id)).all().filter((r) => r.status === "completed" || r.status === "partial");
|
|
8285
|
+
const completedRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), eq10(runs.kind, RunKinds["answer-visibility"]), notProbeRun())).orderBy(desc3(runs.createdAt), desc3(runs.id)).all().filter((r) => r.status === "completed" || r.status === "partial");
|
|
8193
8286
|
const latestGroup = groupRunsByCreatedAt(completedRuns)[0] ?? [];
|
|
8194
8287
|
const latestGroupRunIds = latestGroup.map((r) => r.id);
|
|
8195
8288
|
const latestRun = pickGroupRepresentative(latestGroup);
|
|
8196
8289
|
if (!latestRun) {
|
|
8197
8290
|
return reply.send({ cited: [], gap: [], uncited: [], mentionedQueries: [], mentionGap: [], notMentioned: [], runId: "", window });
|
|
8198
8291
|
}
|
|
8199
|
-
const windowRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), notProbeRun())).orderBy(runs.createdAt).all().filter((r) => r.status === "completed" || r.status === "partial").filter((r) => !cutoff || r.createdAt >= cutoff);
|
|
8292
|
+
const windowRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), eq10(runs.kind, RunKinds["answer-visibility"]), notProbeRun())).orderBy(runs.createdAt).all().filter((r) => r.status === "completed" || r.status === "partial").filter((r) => !cutoff || r.createdAt >= cutoff);
|
|
8200
8293
|
const windowRunIds = windowRuns.map((r) => r.id);
|
|
8201
8294
|
const runIdToCreatedAt = new Map(windowRuns.map((r) => [r.id, r.createdAt]));
|
|
8202
8295
|
const consistencyMap = /* @__PURE__ */ new Map();
|
|
@@ -8327,7 +8420,7 @@ async function analyticsRoutes(app) {
|
|
|
8327
8420
|
const mapped = surfaceClassFromCompetitorType(row.competitorType);
|
|
8328
8421
|
if (mapped) storedSurfaceClasses.set(normalizeProjectDomain(row.domain), mapped);
|
|
8329
8422
|
}
|
|
8330
|
-
const windowRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), notProbeRun())).orderBy(desc3(runs.createdAt), desc3(runs.id)).all().filter((r) => r.status === "completed" || r.status === "partial").filter((r) => !cutoff || r.createdAt >= cutoff);
|
|
8423
|
+
const windowRuns = app.db.select().from(runs).where(and7(eq10(runs.projectId, project.id), eq10(runs.kind, RunKinds["answer-visibility"]), notProbeRun())).orderBy(desc3(runs.createdAt), desc3(runs.id)).all().filter((r) => r.status === "completed" || r.status === "partial").filter((r) => !cutoff || r.createdAt >= cutoff);
|
|
8331
8424
|
if (windowRuns.length === 0) {
|
|
8332
8425
|
return reply.send({
|
|
8333
8426
|
overall: [],
|
|
@@ -11213,6 +11306,7 @@ function renderServerActivity(report, audience) {
|
|
|
11213
11306
|
if (!copy) return "";
|
|
11214
11307
|
return `<span class="tone-${deltaTone(d.deltaPct)}">${escapeHtml(copy)}</span>`;
|
|
11215
11308
|
};
|
|
11309
|
+
const referralSubtitle = [formatDelta(sa.referralArrivals, "sessions"), escapeHtml(sa.referralArrivalsClassSummary)].filter(Boolean).join(" \xB7 ");
|
|
11216
11310
|
if (isClient) {
|
|
11217
11311
|
const crawlerRequests = {
|
|
11218
11312
|
current: sa.verifiedCrawlerHits.current + sa.unverifiedCrawlerHits.current,
|
|
@@ -11251,7 +11345,7 @@ function renderServerActivity(report, audience) {
|
|
|
11251
11345
|
<div class="metric">
|
|
11252
11346
|
<div class="label">AI referral sessions</div>
|
|
11253
11347
|
<div class="value">${formatNumber(sa.referralArrivals.current)}</div>
|
|
11254
|
-
<div class="subtitle">${
|
|
11348
|
+
<div class="subtitle">${referralSubtitle}</div>
|
|
11255
11349
|
</div>
|
|
11256
11350
|
</div>
|
|
11257
11351
|
${clientOperatorRows ? `<div class="chart-card"><h3>By AI tool</h3>
|
|
@@ -11320,7 +11414,7 @@ function renderServerActivity(report, audience) {
|
|
|
11320
11414
|
<div class="metric">
|
|
11321
11415
|
<div class="label">AI-referral sessions (${windowLabel})</div>
|
|
11322
11416
|
<div class="value">${formatNumber(sa.referralArrivals.current)}</div>
|
|
11323
|
-
<div class="subtitle">${
|
|
11417
|
+
<div class="subtitle">${referralSubtitle}</div>
|
|
11324
11418
|
</div>
|
|
11325
11419
|
</div>
|
|
11326
11420
|
${trendChart}
|
|
@@ -11845,6 +11939,9 @@ function safeNum(value) {
|
|
|
11845
11939
|
}
|
|
11846
11940
|
return 0;
|
|
11847
11941
|
}
|
|
11942
|
+
function isPaidAiTrafficClass(value) {
|
|
11943
|
+
return value === AiReferralTrafficClasses.paid;
|
|
11944
|
+
}
|
|
11848
11945
|
function categorizeQuery(query, projectBrandNames, canonicalDomain) {
|
|
11849
11946
|
return categorizeQueryByIntent(query, buildBrandTokens(canonicalDomain, projectBrandNames));
|
|
11850
11947
|
}
|
|
@@ -11999,14 +12096,40 @@ function buildGaSection(db, projectId, windowDays) {
|
|
|
11999
12096
|
users: data.users,
|
|
12000
12097
|
organicSessions: data.organic
|
|
12001
12098
|
})).sort((a, b) => b.sessions - a.sessions).slice(0, TOP_LANDING_PAGES_LIMIT);
|
|
12099
|
+
const aiConditions = [
|
|
12100
|
+
eq15(gaAiReferrals.projectId, projectId),
|
|
12101
|
+
eq15(gaAiReferrals.sourceDimension, "session")
|
|
12102
|
+
];
|
|
12103
|
+
if (snapshotStartDate && snapshotMaxDate) {
|
|
12104
|
+
aiConditions.push(gte2(gaAiReferrals.date, snapshotStartDate));
|
|
12105
|
+
aiConditions.push(lte(gaAiReferrals.date, snapshotMaxDate));
|
|
12106
|
+
}
|
|
12107
|
+
const aiSessionRows = db.select({
|
|
12108
|
+
trafficClass: gaAiReferrals.trafficClass,
|
|
12109
|
+
channelGroup: gaAiReferrals.channelGroup,
|
|
12110
|
+
sessions: sql7`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`
|
|
12111
|
+
}).from(gaAiReferrals).where(and12(...aiConditions)).groupBy(gaAiReferrals.trafficClass, gaAiReferrals.channelGroup).all();
|
|
12112
|
+
let paidAiSessions = 0;
|
|
12113
|
+
let organicAiSessions = 0;
|
|
12114
|
+
let aiOrganicOverlap = 0;
|
|
12115
|
+
let aiDirectOverlap = 0;
|
|
12116
|
+
for (const row of aiSessionRows) {
|
|
12117
|
+
const sessions = Number(row.sessions ?? 0);
|
|
12118
|
+
if (isPaidAiTrafficClass(row.trafficClass)) paidAiSessions += sessions;
|
|
12119
|
+
else organicAiSessions += sessions;
|
|
12120
|
+
if (row.channelGroup === "Organic Search") aiOrganicOverlap += sessions;
|
|
12121
|
+
if (row.channelGroup === "Direct") aiDirectOverlap += sessions;
|
|
12122
|
+
}
|
|
12002
12123
|
const channelBreakdown = [];
|
|
12003
12124
|
if (totalSessions > 0) {
|
|
12004
|
-
const organic = totalOrganicSessions;
|
|
12005
|
-
const direct = directSessions;
|
|
12006
|
-
const other = Math.max(totalSessions - organic - direct, 0);
|
|
12125
|
+
const organic = Math.max(0, totalOrganicSessions - Math.min(totalOrganicSessions, aiOrganicOverlap));
|
|
12126
|
+
const direct = Math.max(0, directSessions - Math.min(directSessions, aiDirectOverlap));
|
|
12127
|
+
const other = Math.max(totalSessions - organic - direct - paidAiSessions - organicAiSessions, 0);
|
|
12007
12128
|
const buckets = [
|
|
12008
12129
|
{ channel: "Organic Search", sessions: organic },
|
|
12009
12130
|
{ channel: "Direct", sessions: direct },
|
|
12131
|
+
{ channel: "Paid AI", sessions: paidAiSessions },
|
|
12132
|
+
{ channel: "Organic AI referrals", sessions: organicAiSessions },
|
|
12010
12133
|
{ channel: "Other", sessions: other }
|
|
12011
12134
|
];
|
|
12012
12135
|
for (const b of buckets) {
|
|
@@ -12093,15 +12216,34 @@ function buildAiReferrals(db, projectId) {
|
|
|
12093
12216
|
);
|
|
12094
12217
|
let total = 0;
|
|
12095
12218
|
let totalUsers = 0;
|
|
12219
|
+
let paidSessions = 0;
|
|
12220
|
+
let paidUsers = 0;
|
|
12221
|
+
let organicSessions = 0;
|
|
12222
|
+
let organicUsers = 0;
|
|
12096
12223
|
const sourceAgg = /* @__PURE__ */ new Map();
|
|
12097
12224
|
const trendAgg = /* @__PURE__ */ new Map();
|
|
12098
12225
|
const pageAgg = /* @__PURE__ */ new Map();
|
|
12099
12226
|
for (const r of dedupedRows) {
|
|
12100
12227
|
total += r.sessions;
|
|
12101
12228
|
totalUsers += r.users;
|
|
12102
|
-
const
|
|
12229
|
+
const paid = isPaidAiTrafficClass(r.trafficClass);
|
|
12230
|
+
if (paid) {
|
|
12231
|
+
paidSessions += r.sessions;
|
|
12232
|
+
paidUsers += r.users;
|
|
12233
|
+
} else {
|
|
12234
|
+
organicSessions += r.sessions;
|
|
12235
|
+
organicUsers += r.users;
|
|
12236
|
+
}
|
|
12237
|
+
const s = sourceAgg.get(r.source) ?? {
|
|
12238
|
+
sessions: 0,
|
|
12239
|
+
users: 0,
|
|
12240
|
+
paidSessions: 0,
|
|
12241
|
+
organicSessions: 0
|
|
12242
|
+
};
|
|
12103
12243
|
s.sessions += r.sessions;
|
|
12104
12244
|
s.users += r.users;
|
|
12245
|
+
if (paid) s.paidSessions += r.sessions;
|
|
12246
|
+
else s.organicSessions += r.sessions;
|
|
12105
12247
|
sourceAgg.set(r.source, s);
|
|
12106
12248
|
trendAgg.set(r.date, (trendAgg.get(r.date) ?? 0) + r.sessions);
|
|
12107
12249
|
const page = r.landingPageNormalized ?? r.landingPage;
|
|
@@ -12114,11 +12256,23 @@ function buildAiReferrals(db, projectId) {
|
|
|
12114
12256
|
source,
|
|
12115
12257
|
sessions: data.sessions,
|
|
12116
12258
|
users: data.users,
|
|
12259
|
+
paidSessions: data.paidSessions,
|
|
12260
|
+
organicSessions: data.organicSessions,
|
|
12117
12261
|
sharePct: total > 0 ? Math.round(data.sessions / total * 100) : 0
|
|
12118
12262
|
})).sort((a, b) => b.sessions - a.sessions);
|
|
12119
12263
|
const trend = [...trendAgg.entries()].map(([date, sessions]) => ({ date, sessions })).sort((a, b) => a.date.localeCompare(b.date));
|
|
12120
12264
|
const topLandingPages = [...pageAgg.entries()].map(([page, data]) => ({ page, sessions: data.sessions, users: data.users })).sort((a, b) => b.sessions - a.sessions).slice(0, TOP_AI_REFERRAL_PAGES_LIMIT);
|
|
12121
|
-
return {
|
|
12265
|
+
return {
|
|
12266
|
+
totalSessions: total,
|
|
12267
|
+
totalUsers,
|
|
12268
|
+
paidSessions,
|
|
12269
|
+
paidUsers,
|
|
12270
|
+
organicSessions,
|
|
12271
|
+
organicUsers,
|
|
12272
|
+
bySource,
|
|
12273
|
+
trend,
|
|
12274
|
+
topLandingPages
|
|
12275
|
+
};
|
|
12122
12276
|
}
|
|
12123
12277
|
function nonSubresourceReferralPathCondition() {
|
|
12124
12278
|
return sql7`
|
|
@@ -12179,16 +12333,21 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
12179
12333
|
)
|
|
12180
12334
|
).get()?.total ?? 0
|
|
12181
12335
|
);
|
|
12182
|
-
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) =>
|
|
12183
|
-
db.select({
|
|
12336
|
+
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => {
|
|
12337
|
+
const row = db.select({
|
|
12338
|
+
total: sql7`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
12339
|
+
paid: sql7`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
12340
|
+
organic: sql7`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
12341
|
+
}).from(aiReferralEventsHourly).where(
|
|
12184
12342
|
and12(
|
|
12185
12343
|
eq15(aiReferralEventsHourly.projectId, projectId),
|
|
12186
12344
|
nonSubresourceReferralPathCondition(),
|
|
12187
12345
|
gte2(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
12188
12346
|
exclusiveEnd ? lt(aiReferralEventsHourly.tsHour, windowEndIso) : lte(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
12189
12347
|
)
|
|
12190
|
-
).get()
|
|
12191
|
-
|
|
12348
|
+
).get();
|
|
12349
|
+
return aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0));
|
|
12350
|
+
};
|
|
12192
12351
|
const sumUserFetches = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
12193
12352
|
db.select({ total: sql7`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
12194
12353
|
and12(
|
|
@@ -12386,7 +12545,7 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
12386
12545
|
return {
|
|
12387
12546
|
windowStart: headlineStart,
|
|
12388
12547
|
windowEnd: headlineEnd,
|
|
12389
|
-
hasData: verifiedCurrent + unverifiedCurrent + userFetchCurrent + referralCurrent + verifiedPrior + unverifiedPrior + userFetchPrior + referralPrior > 0 || byOperator.length > 0 || topCrawledPaths.length > 0 || referralProducts.length > 0,
|
|
12548
|
+
hasData: verifiedCurrent + unverifiedCurrent + userFetchCurrent + referralCurrent.total + verifiedPrior + unverifiedPrior + userFetchPrior + referralPrior.total > 0 || byOperator.length > 0 || topCrawledPaths.length > 0 || referralProducts.length > 0,
|
|
12390
12549
|
verifiedCrawlerHits: {
|
|
12391
12550
|
current: verifiedCurrent,
|
|
12392
12551
|
prior: verifiedPrior,
|
|
@@ -12403,10 +12562,28 @@ function buildServerActivity(db, projectId, windowDays) {
|
|
|
12403
12562
|
deltaPct: deltaPercent(userFetchCurrent, userFetchPrior)
|
|
12404
12563
|
},
|
|
12405
12564
|
referralArrivals: {
|
|
12406
|
-
current: referralCurrent,
|
|
12407
|
-
prior: referralPrior,
|
|
12408
|
-
deltaPct: deltaPercent(referralCurrent, referralPrior)
|
|
12565
|
+
current: referralCurrent.total,
|
|
12566
|
+
prior: referralPrior.total,
|
|
12567
|
+
deltaPct: deltaPercent(referralCurrent.total, referralPrior.total)
|
|
12568
|
+
},
|
|
12569
|
+
referralArrivalsByClass: {
|
|
12570
|
+
paid: {
|
|
12571
|
+
current: referralCurrent.paid,
|
|
12572
|
+
prior: referralPrior.paid,
|
|
12573
|
+
deltaPct: deltaPercent(referralCurrent.paid, referralPrior.paid)
|
|
12574
|
+
},
|
|
12575
|
+
organic: {
|
|
12576
|
+
current: referralCurrent.organic,
|
|
12577
|
+
prior: referralPrior.organic,
|
|
12578
|
+
deltaPct: deltaPercent(referralCurrent.organic, referralPrior.organic)
|
|
12579
|
+
},
|
|
12580
|
+
unclassified: {
|
|
12581
|
+
current: referralCurrent.unknown,
|
|
12582
|
+
prior: referralPrior.unknown,
|
|
12583
|
+
deltaPct: deltaPercent(referralCurrent.unknown, referralPrior.unknown)
|
|
12584
|
+
}
|
|
12409
12585
|
},
|
|
12586
|
+
referralArrivalsClassSummary: formatAiReferralClassSummary(referralCurrent),
|
|
12410
12587
|
byOperator,
|
|
12411
12588
|
topCrawledPaths,
|
|
12412
12589
|
referralProducts,
|
|
@@ -20423,16 +20600,23 @@ async function fetchAiReferrals(accessToken, propertyId, days) {
|
|
|
20423
20600
|
};
|
|
20424
20601
|
const response = await runReport(accessToken, propertyId, request);
|
|
20425
20602
|
if (!response) break;
|
|
20426
|
-
const pageRows = (response.rows ?? []).map((row) =>
|
|
20427
|
-
|
|
20428
|
-
|
|
20429
|
-
|
|
20430
|
-
|
|
20431
|
-
|
|
20432
|
-
|
|
20433
|
-
|
|
20434
|
-
|
|
20435
|
-
|
|
20603
|
+
const pageRows = (response.rows ?? []).map((row) => {
|
|
20604
|
+
const source = row.dimensionValues[1].value;
|
|
20605
|
+
const medium = row.dimensionValues[2].value;
|
|
20606
|
+
const channelGroup = row.dimensionValues[3]?.value ?? "(not set)";
|
|
20607
|
+
const landingPage = row.dimensionValues[4]?.value ?? "(not set)";
|
|
20608
|
+
return {
|
|
20609
|
+
date: row.dimensionValues[0].value,
|
|
20610
|
+
source,
|
|
20611
|
+
medium,
|
|
20612
|
+
trafficClass: classifyAiReferralTrafficClass({ source, medium, channelGroup, landingPage }),
|
|
20613
|
+
channelGroup,
|
|
20614
|
+
landingPage,
|
|
20615
|
+
sessions: parseInt(row.metricValues[0].value, 10) || 0,
|
|
20616
|
+
users: parseInt(row.metricValues[1].value, 10) || 0,
|
|
20617
|
+
sourceDimension: dimLabel
|
|
20618
|
+
};
|
|
20619
|
+
});
|
|
20436
20620
|
rows.push(...pageRows);
|
|
20437
20621
|
const totalRows = response.rowCount ?? 0;
|
|
20438
20622
|
offset += pageRows.length;
|
|
@@ -23400,6 +23584,76 @@ function pickWinningDimension(rows, tupleKey) {
|
|
|
23400
23584
|
}
|
|
23401
23585
|
return [...winners.values()].sort((a, b) => (b.sessions ?? 0) - (a.sessions ?? 0));
|
|
23402
23586
|
}
|
|
23587
|
+
function normalizeAiTrafficClass(value) {
|
|
23588
|
+
return value === AiReferralTrafficClasses.paid ? AiReferralTrafficClasses.paid : AiReferralTrafficClasses.organic;
|
|
23589
|
+
}
|
|
23590
|
+
function emptyAiCounts() {
|
|
23591
|
+
return { sessions: 0, users: 0 };
|
|
23592
|
+
}
|
|
23593
|
+
function summarizeAiReferralCounts(rows) {
|
|
23594
|
+
const dedupeGroups = /* @__PURE__ */ new Map();
|
|
23595
|
+
const bySessionChannelGroup = /* @__PURE__ */ new Map();
|
|
23596
|
+
const paidDeduped = emptyAiCounts();
|
|
23597
|
+
const organicDeduped = emptyAiCounts();
|
|
23598
|
+
const paidBySession = emptyAiCounts();
|
|
23599
|
+
const organicBySession = emptyAiCounts();
|
|
23600
|
+
for (const row of rows) {
|
|
23601
|
+
const isPaid = normalizeAiTrafficClass(row.trafficClass) === AiReferralTrafficClasses.paid;
|
|
23602
|
+
const sessions = row.sessions ?? 0;
|
|
23603
|
+
const users = row.users ?? 0;
|
|
23604
|
+
const key = `${row.date}\0${row.source}\0${row.medium}`;
|
|
23605
|
+
let byDimension = dedupeGroups.get(key);
|
|
23606
|
+
if (!byDimension) {
|
|
23607
|
+
byDimension = /* @__PURE__ */ new Map();
|
|
23608
|
+
dedupeGroups.set(key, byDimension);
|
|
23609
|
+
}
|
|
23610
|
+
let dim = byDimension.get(row.sourceDimension);
|
|
23611
|
+
if (!dim) {
|
|
23612
|
+
dim = { paidSessions: 0, organicSessions: 0, paidUsers: 0, organicUsers: 0 };
|
|
23613
|
+
byDimension.set(row.sourceDimension, dim);
|
|
23614
|
+
}
|
|
23615
|
+
if (isPaid) {
|
|
23616
|
+
dim.paidSessions += sessions;
|
|
23617
|
+
dim.paidUsers += users;
|
|
23618
|
+
} else {
|
|
23619
|
+
dim.organicSessions += sessions;
|
|
23620
|
+
dim.organicUsers += users;
|
|
23621
|
+
}
|
|
23622
|
+
if (row.sourceDimension === "session") {
|
|
23623
|
+
bySessionChannelGroup.set(
|
|
23624
|
+
row.channelGroup,
|
|
23625
|
+
(bySessionChannelGroup.get(row.channelGroup) ?? 0) + sessions
|
|
23626
|
+
);
|
|
23627
|
+
const sessionBucket = isPaid ? paidBySession : organicBySession;
|
|
23628
|
+
sessionBucket.sessions += sessions;
|
|
23629
|
+
sessionBucket.users += users;
|
|
23630
|
+
}
|
|
23631
|
+
}
|
|
23632
|
+
for (const byDimension of dedupeGroups.values()) {
|
|
23633
|
+
const dims = [...byDimension.values()];
|
|
23634
|
+
const bestSessions = dims.reduce((best, d) => d.paidSessions + d.organicSessions > best.paidSessions + best.organicSessions ? d : best);
|
|
23635
|
+
const bestUsers = dims.reduce((best, d) => d.paidUsers + d.organicUsers > best.paidUsers + best.organicUsers ? d : best);
|
|
23636
|
+
paidDeduped.sessions += bestSessions.paidSessions;
|
|
23637
|
+
organicDeduped.sessions += bestSessions.organicSessions;
|
|
23638
|
+
paidDeduped.users += bestUsers.paidUsers;
|
|
23639
|
+
organicDeduped.users += bestUsers.organicUsers;
|
|
23640
|
+
}
|
|
23641
|
+
return {
|
|
23642
|
+
paidDeduped,
|
|
23643
|
+
organicDeduped,
|
|
23644
|
+
paidBySession,
|
|
23645
|
+
organicBySession,
|
|
23646
|
+
bySessionChannelGroup,
|
|
23647
|
+
deduped: {
|
|
23648
|
+
sessions: paidDeduped.sessions + organicDeduped.sessions,
|
|
23649
|
+
users: paidDeduped.users + organicDeduped.users
|
|
23650
|
+
},
|
|
23651
|
+
bySession: {
|
|
23652
|
+
sessions: paidBySession.sessions + organicBySession.sessions,
|
|
23653
|
+
users: paidBySession.users + organicBySession.users
|
|
23654
|
+
}
|
|
23655
|
+
};
|
|
23656
|
+
}
|
|
23403
23657
|
async function refreshOAuthTokenIfNeeded(googleStore, authConfig, canonicalDomain, oauthConn) {
|
|
23404
23658
|
const expiresAt = oauthConn.tokenExpiresAt ? new Date(oauthConn.tokenExpiresAt).getTime() : 0;
|
|
23405
23659
|
const fiveMinutes = 5 * 60 * 1e3;
|
|
@@ -23676,6 +23930,12 @@ async function ga4Routes(app, opts) {
|
|
|
23676
23930
|
date: row.date,
|
|
23677
23931
|
source: row.source,
|
|
23678
23932
|
medium: row.medium,
|
|
23933
|
+
trafficClass: row.trafficClass ?? classifyAiReferralTrafficClass({
|
|
23934
|
+
source: row.source,
|
|
23935
|
+
medium: row.medium,
|
|
23936
|
+
channelGroup: row.channelGroup,
|
|
23937
|
+
landingPage: row.landingPage
|
|
23938
|
+
}),
|
|
23679
23939
|
sourceDimension: row.sourceDimension,
|
|
23680
23940
|
channelGroup: row.channelGroup,
|
|
23681
23941
|
landingPage: row.landingPage,
|
|
@@ -23825,13 +24085,15 @@ async function ga4Routes(app, opts) {
|
|
|
23825
24085
|
const aiReferralRows = app.db.select({
|
|
23826
24086
|
source: gaAiReferrals.source,
|
|
23827
24087
|
medium: gaAiReferrals.medium,
|
|
24088
|
+
trafficClass: gaAiReferrals.trafficClass,
|
|
23828
24089
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
23829
24090
|
sessions: sql10`SUM(${gaAiReferrals.sessions})`,
|
|
23830
24091
|
users: sql10`SUM(${gaAiReferrals.users})`
|
|
23831
|
-
}).from(gaAiReferrals).where(and21(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).all();
|
|
24092
|
+
}).from(gaAiReferrals).where(and21(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
23832
24093
|
const aiReferralLandingPageRows = app.db.select({
|
|
23833
24094
|
source: gaAiReferrals.source,
|
|
23834
24095
|
medium: gaAiReferrals.medium,
|
|
24096
|
+
trafficClass: gaAiReferrals.trafficClass,
|
|
23835
24097
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
23836
24098
|
landingPage: sql10`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
23837
24099
|
sessions: sql10`SUM(${gaAiReferrals.sessions})`,
|
|
@@ -23839,48 +24101,29 @@ async function ga4Routes(app, opts) {
|
|
|
23839
24101
|
}).from(gaAiReferrals).where(and21(...aiConditions)).groupBy(
|
|
23840
24102
|
gaAiReferrals.source,
|
|
23841
24103
|
gaAiReferrals.medium,
|
|
24104
|
+
gaAiReferrals.trafficClass,
|
|
23842
24105
|
gaAiReferrals.sourceDimension,
|
|
23843
24106
|
sql10`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
23844
24107
|
).all();
|
|
24108
|
+
const aiRowsForTotals = app.db.select({
|
|
24109
|
+
date: gaAiReferrals.date,
|
|
24110
|
+
source: gaAiReferrals.source,
|
|
24111
|
+
medium: gaAiReferrals.medium,
|
|
24112
|
+
trafficClass: gaAiReferrals.trafficClass,
|
|
24113
|
+
sourceDimension: gaAiReferrals.sourceDimension,
|
|
24114
|
+
channelGroup: gaAiReferrals.channelGroup,
|
|
24115
|
+
sessions: gaAiReferrals.sessions,
|
|
24116
|
+
users: gaAiReferrals.users
|
|
24117
|
+
}).from(gaAiReferrals).where(and21(...aiConditions)).all();
|
|
23845
24118
|
const aiReferrals = pickWinningDimension(
|
|
23846
24119
|
aiReferralRows,
|
|
23847
|
-
(r) => `${r.source}\0${r.medium}`
|
|
24120
|
+
(r) => `${r.source}\0${r.medium}\0${r.trafficClass}`
|
|
23848
24121
|
);
|
|
23849
24122
|
const aiReferralLandingPages = pickWinningDimension(
|
|
23850
24123
|
aiReferralLandingPageRows,
|
|
23851
|
-
(r) => `${r.source}\0${r.medium}\0${r.landingPage}`
|
|
24124
|
+
(r) => `${r.source}\0${r.medium}\0${r.trafficClass}\0${r.landingPage}`
|
|
23852
24125
|
);
|
|
23853
|
-
const
|
|
23854
|
-
sessions: sql10`COALESCE(SUM(max_sessions), 0)`,
|
|
23855
|
-
users: sql10`COALESCE(SUM(max_users), 0)`
|
|
23856
|
-
}).from(
|
|
23857
|
-
sql10`(
|
|
23858
|
-
SELECT date, source, medium,
|
|
23859
|
-
MAX(dimension_sessions) AS max_sessions,
|
|
23860
|
-
MAX(dimension_users) AS max_users
|
|
23861
|
-
FROM (
|
|
23862
|
-
SELECT date, source, medium, source_dimension,
|
|
23863
|
-
SUM(sessions) AS dimension_sessions,
|
|
23864
|
-
SUM(users) AS dimension_users
|
|
23865
|
-
FROM ga_ai_referrals
|
|
23866
|
-
WHERE project_id = ${project.id}${cutoffDate ? sql10` AND date >= ${cutoffDate}` : sql10``}
|
|
23867
|
-
GROUP BY date, source, medium, source_dimension
|
|
23868
|
-
)
|
|
23869
|
-
GROUP BY date, source, medium
|
|
23870
|
-
)`
|
|
23871
|
-
).get();
|
|
23872
|
-
const aiBySessionRows = app.db.select({
|
|
23873
|
-
channelGroup: gaAiReferrals.channelGroup,
|
|
23874
|
-
sessions: sql10`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`,
|
|
23875
|
-
users: sql10`COALESCE(SUM(${gaAiReferrals.users}), 0)`
|
|
23876
|
-
}).from(gaAiReferrals).where(and21(...aiConditions, eq26(gaAiReferrals.sourceDimension, "session"))).groupBy(gaAiReferrals.channelGroup).all();
|
|
23877
|
-
const aiSessionsByChannelGroup = /* @__PURE__ */ new Map();
|
|
23878
|
-
let aiBySessionUsers = 0;
|
|
23879
|
-
for (const row of aiBySessionRows) {
|
|
23880
|
-
aiSessionsByChannelGroup.set(row.channelGroup, row.sessions ?? 0);
|
|
23881
|
-
aiBySessionUsers += row.users ?? 0;
|
|
23882
|
-
}
|
|
23883
|
-
const aiBySessionSessions = [...aiSessionsByChannelGroup.values()].reduce((sum, sessions) => sum + sessions, 0);
|
|
24126
|
+
const aiSummary = summarizeAiReferralCounts(aiRowsForTotals);
|
|
23884
24127
|
const socialReferrals = app.db.select({
|
|
23885
24128
|
source: gaSocialReferrals.source,
|
|
23886
24129
|
medium: gaSocialReferrals.medium,
|
|
@@ -23902,7 +24145,7 @@ async function ga4Routes(app, opts) {
|
|
|
23902
24145
|
organicSessions: totalOrganicSessions,
|
|
23903
24146
|
socialSessions,
|
|
23904
24147
|
directSessions: totalDirectSessions,
|
|
23905
|
-
aiSessionsByChannelGroup
|
|
24148
|
+
aiSessionsByChannelGroup: aiSummary.bySessionChannelGroup
|
|
23906
24149
|
});
|
|
23907
24150
|
return {
|
|
23908
24151
|
totalSessions: total,
|
|
@@ -23919,6 +24162,7 @@ async function ga4Routes(app, opts) {
|
|
|
23919
24162
|
aiReferrals: aiReferrals.map((r) => ({
|
|
23920
24163
|
source: r.source,
|
|
23921
24164
|
medium: r.medium,
|
|
24165
|
+
trafficClass: normalizeAiTrafficClass(r.trafficClass),
|
|
23922
24166
|
sourceDimension: r.sourceDimension,
|
|
23923
24167
|
sessions: r.sessions ?? 0,
|
|
23924
24168
|
users: r.users ?? 0
|
|
@@ -23926,15 +24170,24 @@ async function ga4Routes(app, opts) {
|
|
|
23926
24170
|
aiReferralLandingPages: aiReferralLandingPages.map((r) => ({
|
|
23927
24171
|
source: r.source,
|
|
23928
24172
|
medium: r.medium,
|
|
24173
|
+
trafficClass: normalizeAiTrafficClass(r.trafficClass),
|
|
23929
24174
|
sourceDimension: r.sourceDimension,
|
|
23930
24175
|
landingPage: r.landingPage,
|
|
23931
24176
|
sessions: r.sessions ?? 0,
|
|
23932
24177
|
users: r.users ?? 0
|
|
23933
24178
|
})),
|
|
23934
|
-
aiSessionsDeduped:
|
|
23935
|
-
aiUsersDeduped:
|
|
23936
|
-
|
|
23937
|
-
|
|
24179
|
+
aiSessionsDeduped: aiSummary.deduped.sessions,
|
|
24180
|
+
aiUsersDeduped: aiSummary.deduped.users,
|
|
24181
|
+
paidAiSessionsDeduped: aiSummary.paidDeduped.sessions,
|
|
24182
|
+
paidAiUsersDeduped: aiSummary.paidDeduped.users,
|
|
24183
|
+
organicAiSessionsDeduped: aiSummary.organicDeduped.sessions,
|
|
24184
|
+
organicAiUsersDeduped: aiSummary.organicDeduped.users,
|
|
24185
|
+
aiSessionsBySession: aiSummary.bySession.sessions,
|
|
24186
|
+
aiUsersBySession: aiSummary.bySession.users,
|
|
24187
|
+
paidAiSessionsBySession: aiSummary.paidBySession.sessions,
|
|
24188
|
+
paidAiUsersBySession: aiSummary.paidBySession.users,
|
|
24189
|
+
organicAiSessionsBySession: aiSummary.organicBySession.sessions,
|
|
24190
|
+
organicAiUsersBySession: aiSummary.organicBySession.users,
|
|
23938
24191
|
socialReferrals: socialReferrals.map((r) => ({
|
|
23939
24192
|
source: r.source,
|
|
23940
24193
|
medium: r.medium,
|
|
@@ -23946,16 +24199,24 @@ async function ga4Routes(app, opts) {
|
|
|
23946
24199
|
socialUsers: socialTotals?.users ?? 0,
|
|
23947
24200
|
channelBreakdown,
|
|
23948
24201
|
organicSharePct: total > 0 ? Math.round(totalOrganicSessions / total * 100) : 0,
|
|
23949
|
-
aiSharePct: total > 0 ? Math.round(
|
|
23950
|
-
aiSharePctBySession: total > 0 ? Math.round(
|
|
24202
|
+
aiSharePct: total > 0 ? Math.round(aiSummary.deduped.sessions / total * 100) : 0,
|
|
24203
|
+
aiSharePctBySession: total > 0 ? Math.round(aiSummary.bySession.sessions / total * 100) : 0,
|
|
24204
|
+
paidAiSharePct: total > 0 ? Math.round(aiSummary.paidDeduped.sessions / total * 100) : 0,
|
|
24205
|
+
paidAiSharePctBySession: total > 0 ? Math.round(aiSummary.paidBySession.sessions / total * 100) : 0,
|
|
24206
|
+
organicAiSharePct: total > 0 ? Math.round(aiSummary.organicDeduped.sessions / total * 100) : 0,
|
|
24207
|
+
organicAiSharePctBySession: total > 0 ? Math.round(aiSummary.organicBySession.sessions / total * 100) : 0,
|
|
23951
24208
|
directSharePct: total > 0 ? Math.round(totalDirectSessions / total * 100) : 0,
|
|
23952
24209
|
socialSharePct: total > 0 ? Math.round(socialSessions / total * 100) : 0,
|
|
23953
24210
|
otherSessions: channelBreakdown.other.sessions,
|
|
23954
24211
|
otherSharePct: channelBreakdown.other.sharePct,
|
|
23955
24212
|
otherSharePctDisplay: channelBreakdown.other.sharePctDisplay,
|
|
23956
24213
|
organicSharePctDisplay: formatSharePct(totalOrganicSessions, total),
|
|
23957
|
-
aiSharePctDisplay: formatSharePct(
|
|
23958
|
-
aiSharePctBySessionDisplay: formatSharePct(
|
|
24214
|
+
aiSharePctDisplay: formatSharePct(aiSummary.deduped.sessions, total),
|
|
24215
|
+
aiSharePctBySessionDisplay: formatSharePct(aiSummary.bySession.sessions, total),
|
|
24216
|
+
paidAiSharePctDisplay: formatSharePct(aiSummary.paidDeduped.sessions, total),
|
|
24217
|
+
paidAiSharePctBySessionDisplay: formatSharePct(aiSummary.paidBySession.sessions, total),
|
|
24218
|
+
organicAiSharePctDisplay: formatSharePct(aiSummary.organicDeduped.sessions, total),
|
|
24219
|
+
organicAiSharePctBySessionDisplay: formatSharePct(aiSummary.organicBySession.sessions, total),
|
|
23959
24220
|
directSharePctDisplay: formatSharePct(totalDirectSessions, total),
|
|
23960
24221
|
socialSharePctDisplay: formatSharePct(socialSessions, total),
|
|
23961
24222
|
lastSyncedAt: latestSync?.syncedAt ?? null,
|
|
@@ -23978,6 +24239,7 @@ async function ga4Routes(app, opts) {
|
|
|
23978
24239
|
date: gaAiReferrals.date,
|
|
23979
24240
|
source: gaAiReferrals.source,
|
|
23980
24241
|
medium: gaAiReferrals.medium,
|
|
24242
|
+
trafficClass: gaAiReferrals.trafficClass,
|
|
23981
24243
|
landingPage: sql10`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
23982
24244
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
23983
24245
|
sessions: sql10`SUM(${gaAiReferrals.sessions})`,
|
|
@@ -23986,6 +24248,7 @@ async function ga4Routes(app, opts) {
|
|
|
23986
24248
|
gaAiReferrals.date,
|
|
23987
24249
|
gaAiReferrals.source,
|
|
23988
24250
|
gaAiReferrals.medium,
|
|
24251
|
+
gaAiReferrals.trafficClass,
|
|
23989
24252
|
gaAiReferrals.sourceDimension,
|
|
23990
24253
|
sql10`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
23991
24254
|
).orderBy(gaAiReferrals.date).all();
|
|
@@ -29445,6 +29708,33 @@ function utmSourceFromUrl(value) {
|
|
|
29445
29708
|
return null;
|
|
29446
29709
|
}
|
|
29447
29710
|
}
|
|
29711
|
+
function paramsFromQuery(queryString) {
|
|
29712
|
+
if (!queryString) return null;
|
|
29713
|
+
const params = new URLSearchParams(queryString);
|
|
29714
|
+
return [...params.keys()].length > 0 ? params : null;
|
|
29715
|
+
}
|
|
29716
|
+
function paramsFromUrl(value) {
|
|
29717
|
+
if (!value) return null;
|
|
29718
|
+
try {
|
|
29719
|
+
return paramsFromQuery(new URL(value).search.replace(/^\?/, ""));
|
|
29720
|
+
} catch {
|
|
29721
|
+
return null;
|
|
29722
|
+
}
|
|
29723
|
+
}
|
|
29724
|
+
function classifyAiReferralTrafficClassFromEvent(event) {
|
|
29725
|
+
for (const params of [paramsFromQuery(event.queryString), paramsFromUrl(event.referer)]) {
|
|
29726
|
+
if (!params) continue;
|
|
29727
|
+
const trafficClass = classifyAiReferralTrafficClass({
|
|
29728
|
+
source: params.get("utm_source"),
|
|
29729
|
+
medium: params.get("utm_medium"),
|
|
29730
|
+
channelGroup: null,
|
|
29731
|
+
// Carries utm_campaign / utm_content / utm_term to the shared rules.
|
|
29732
|
+
landingPage: `/?${params.toString()}`
|
|
29733
|
+
});
|
|
29734
|
+
if (trafficClass === AiReferralTrafficClasses.paid) return AiReferralTrafficClasses.paid;
|
|
29735
|
+
}
|
|
29736
|
+
return AiReferralTrafficClasses.organic;
|
|
29737
|
+
}
|
|
29448
29738
|
function isSelfTraffic(event) {
|
|
29449
29739
|
const userAgent = event.userAgent?.trim();
|
|
29450
29740
|
if (!userAgent) return false;
|
|
@@ -29488,6 +29778,7 @@ function classifyAiUserFetch(event) {
|
|
|
29488
29778
|
return null;
|
|
29489
29779
|
}
|
|
29490
29780
|
function classifyAiReferral(event) {
|
|
29781
|
+
const trafficClass = classifyAiReferralTrafficClassFromEvent(event);
|
|
29491
29782
|
const refererHost = hostFromUrl(event.referer);
|
|
29492
29783
|
if (refererHost) {
|
|
29493
29784
|
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(refererHost, candidate.domain));
|
|
@@ -29496,7 +29787,8 @@ function classifyAiReferral(event) {
|
|
|
29496
29787
|
operator: rule.operator,
|
|
29497
29788
|
product: rule.product,
|
|
29498
29789
|
sourceDomain: refererHost,
|
|
29499
|
-
evidenceType: "referer"
|
|
29790
|
+
evidenceType: "referer",
|
|
29791
|
+
trafficClass
|
|
29500
29792
|
};
|
|
29501
29793
|
}
|
|
29502
29794
|
}
|
|
@@ -29508,7 +29800,8 @@ function classifyAiReferral(event) {
|
|
|
29508
29800
|
operator: rule.operator,
|
|
29509
29801
|
product: rule.product,
|
|
29510
29802
|
sourceDomain: utmSource,
|
|
29511
|
-
evidenceType: "utm"
|
|
29803
|
+
evidenceType: "utm",
|
|
29804
|
+
trafficClass
|
|
29512
29805
|
};
|
|
29513
29806
|
}
|
|
29514
29807
|
}
|
|
@@ -29520,7 +29813,8 @@ function classifyAiReferral(event) {
|
|
|
29520
29813
|
operator: rule.operator,
|
|
29521
29814
|
product: rule.product,
|
|
29522
29815
|
sourceDomain: refererUtmSource,
|
|
29523
|
-
evidenceType: "referer-utm"
|
|
29816
|
+
evidenceType: "referer-utm",
|
|
29817
|
+
trafficClass
|
|
29524
29818
|
};
|
|
29525
29819
|
}
|
|
29526
29820
|
}
|
|
@@ -29623,8 +29917,10 @@ function evidenceRank(evidenceType) {
|
|
|
29623
29917
|
return 1;
|
|
29624
29918
|
}
|
|
29625
29919
|
}
|
|
29626
|
-
function
|
|
29627
|
-
|
|
29920
|
+
function mergeReferralSession(current, next) {
|
|
29921
|
+
const winner = evidenceRank(next.evidenceType) > evidenceRank(current.evidenceType) ? next : current;
|
|
29922
|
+
const paid = current.trafficClass === AiReferralTrafficClasses.paid || next.trafficClass === AiReferralTrafficClasses.paid;
|
|
29923
|
+
return { ...winner, trafficClass: paid ? AiReferralTrafficClasses.paid : winner.trafficClass };
|
|
29628
29924
|
}
|
|
29629
29925
|
function sortCrawlerBuckets(a, b) {
|
|
29630
29926
|
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
@@ -29657,6 +29953,8 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
29657
29953
|
let crawlerHits = 0;
|
|
29658
29954
|
let aiUserFetchHits = 0;
|
|
29659
29955
|
let aiReferralHits = 0;
|
|
29956
|
+
let aiReferralPaidSessions = 0;
|
|
29957
|
+
let aiReferralOrganicSessions = 0;
|
|
29660
29958
|
let unknownHits = 0;
|
|
29661
29959
|
const samples = [];
|
|
29662
29960
|
for (const event of includedEvents) {
|
|
@@ -29738,11 +30036,12 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
29738
30036
|
sourceDomain: aiReferral.sourceDomain,
|
|
29739
30037
|
evidenceType: aiReferral.evidenceType,
|
|
29740
30038
|
landingPathNormalized,
|
|
29741
|
-
status: event.status
|
|
30039
|
+
status: event.status,
|
|
30040
|
+
trafficClass: aiReferral.trafficClass
|
|
29742
30041
|
};
|
|
29743
30042
|
const key = aiReferralSessionKey(event, aiReferral, landingPathNormalized, aiReferralSessionWindowMs);
|
|
29744
30043
|
const existing = aiReferralSessions.get(key);
|
|
29745
|
-
aiReferralSessions.set(key, existing ?
|
|
30044
|
+
aiReferralSessions.set(key, existing ? mergeReferralSession(existing, session) : session);
|
|
29746
30045
|
}
|
|
29747
30046
|
}
|
|
29748
30047
|
if (!crawler && !aiUserFetch && !aiReferral) unknownHits += 1;
|
|
@@ -29770,13 +30069,21 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
29770
30069
|
session.landingPathNormalized,
|
|
29771
30070
|
session.status ?? "null"
|
|
29772
30071
|
].join(" ");
|
|
30072
|
+
const { trafficClass, ...bucketFields } = session;
|
|
30073
|
+
const paid = trafficClass === AiReferralTrafficClasses.paid;
|
|
30074
|
+
if (paid) aiReferralPaidSessions += 1;
|
|
30075
|
+
else aiReferralOrganicSessions += 1;
|
|
29773
30076
|
const existing = aiReferralBuckets.get(key);
|
|
29774
30077
|
if (existing) {
|
|
29775
30078
|
existing.hits += 1;
|
|
30079
|
+
if (paid) existing.paidHits += 1;
|
|
30080
|
+
else existing.organicHits += 1;
|
|
29776
30081
|
} else {
|
|
29777
30082
|
aiReferralBuckets.set(key, {
|
|
29778
|
-
...
|
|
29779
|
-
hits: 1
|
|
30083
|
+
...bucketFields,
|
|
30084
|
+
hits: 1,
|
|
30085
|
+
paidHits: paid ? 1 : 0,
|
|
30086
|
+
organicHits: paid ? 0 : 1
|
|
29780
30087
|
});
|
|
29781
30088
|
}
|
|
29782
30089
|
incrementBucket(topAiReferrers, session.sourceDomain, {
|
|
@@ -29795,6 +30102,8 @@ function buildTrafficProbeReport(events, options = {}) {
|
|
|
29795
30102
|
crawlerHits,
|
|
29796
30103
|
aiUserFetchHits,
|
|
29797
30104
|
aiReferralSessions: aiReferralSessions.size,
|
|
30105
|
+
aiReferralPaidSessions,
|
|
30106
|
+
aiReferralOrganicSessions,
|
|
29798
30107
|
aiReferralHits,
|
|
29799
30108
|
unknownHits
|
|
29800
30109
|
},
|
|
@@ -30556,6 +30865,8 @@ async function runBackfillTask(options) {
|
|
|
30556
30865
|
landingPathNormalized: bucket.landingPathNormalized,
|
|
30557
30866
|
status: bucket.status ?? 0,
|
|
30558
30867
|
sessionsOrHits: bucket.hits,
|
|
30868
|
+
paidSessionsOrHits: bucket.paidHits,
|
|
30869
|
+
organicSessionsOrHits: bucket.organicHits,
|
|
30559
30870
|
usersEstimated: null,
|
|
30560
30871
|
createdAt: finishedAt,
|
|
30561
30872
|
updatedAt: finishedAt
|
|
@@ -31303,6 +31614,8 @@ async function trafficRoutes(app, opts) {
|
|
|
31303
31614
|
landingPathNormalized: bucket.landingPathNormalized,
|
|
31304
31615
|
status,
|
|
31305
31616
|
sessionsOrHits: bucket.hits,
|
|
31617
|
+
paidSessionsOrHits: bucket.paidHits,
|
|
31618
|
+
organicSessionsOrHits: bucket.organicHits,
|
|
31306
31619
|
usersEstimated: null,
|
|
31307
31620
|
createdAt: finishedAt,
|
|
31308
31621
|
updatedAt: finishedAt
|
|
@@ -31317,8 +31630,13 @@ async function trafficRoutes(app, opts) {
|
|
|
31317
31630
|
aiReferralEventsHourly.landingPathNormalized,
|
|
31318
31631
|
aiReferralEventsHourly.status
|
|
31319
31632
|
],
|
|
31633
|
+
// All three counters accumulate. The class splits the measure, so a
|
|
31634
|
+
// colliding bucket adds its paid and organic arrivals to the ones
|
|
31635
|
+
// already there rather than overwriting a single label.
|
|
31320
31636
|
set: {
|
|
31321
31637
|
sessionsOrHits: sql12`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
31638
|
+
paidSessionsOrHits: sql12`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
31639
|
+
organicSessionsOrHits: sql12`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
31322
31640
|
updatedAt: finishedAt
|
|
31323
31641
|
}
|
|
31324
31642
|
}).run();
|
|
@@ -31770,7 +32088,7 @@ async function trafficRoutes(app, opts) {
|
|
|
31770
32088
|
let crawlerTotal = 0;
|
|
31771
32089
|
let crawlerSegments = { content: 0, sitemap: 0, robots: 0, asset: 0, other: 0 };
|
|
31772
32090
|
let aiUserFetchTotal = 0;
|
|
31773
|
-
let
|
|
32091
|
+
let aiReferralCounts = aiReferralClassCounts(0, 0, 0);
|
|
31774
32092
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
31775
32093
|
const crawlerFilters = [
|
|
31776
32094
|
eq28(crawlerEventsHourly.projectId, project.id),
|
|
@@ -31836,10 +32154,19 @@ async function trafficRoutes(app, opts) {
|
|
|
31836
32154
|
];
|
|
31837
32155
|
if (sourceIdParam) aiFilters.push(eq28(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
31838
32156
|
const aiWhere = and24(...aiFilters);
|
|
31839
|
-
const total = app.db.select({
|
|
31840
|
-
|
|
32157
|
+
const total = app.db.select({
|
|
32158
|
+
total: sql12`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
32159
|
+
paid: sql12`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
32160
|
+
organic: sql12`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
32161
|
+
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
32162
|
+
aiReferralCounts = aiReferralClassCounts(
|
|
32163
|
+
Number(total?.total ?? 0),
|
|
32164
|
+
Number(total?.paid ?? 0),
|
|
32165
|
+
Number(total?.organic ?? 0)
|
|
32166
|
+
);
|
|
31841
32167
|
const rows = app.db.select().from(aiReferralEventsHourly).where(aiWhere).orderBy(desc15(aiReferralEventsHourly.tsHour)).limit(limit).all();
|
|
31842
32168
|
for (const r of rows) {
|
|
32169
|
+
const counts2 = aiReferralClassCounts(r.sessionsOrHits, r.paidSessionsOrHits, r.organicSessionsOrHits);
|
|
31843
32170
|
events.push({
|
|
31844
32171
|
kind: TrafficEventKinds["ai-referral"],
|
|
31845
32172
|
sourceId: r.sourceId,
|
|
@@ -31850,7 +32177,10 @@ async function trafficRoutes(app, opts) {
|
|
|
31850
32177
|
evidenceType: r.evidenceType,
|
|
31851
32178
|
landingPathNormalized: r.landingPathNormalized,
|
|
31852
32179
|
status: r.status,
|
|
31853
|
-
hits:
|
|
32180
|
+
hits: counts2.total,
|
|
32181
|
+
paidHits: counts2.paid,
|
|
32182
|
+
organicHits: counts2.organic,
|
|
32183
|
+
unknownHits: counts2.unknown
|
|
31854
32184
|
});
|
|
31855
32185
|
}
|
|
31856
32186
|
}
|
|
@@ -31865,7 +32195,10 @@ async function trafficRoutes(app, opts) {
|
|
|
31865
32195
|
crawlerInfraHits: sumInfraHits(crawlerSegments),
|
|
31866
32196
|
crawlerSegments,
|
|
31867
32197
|
aiUserFetchHits: aiUserFetchTotal,
|
|
31868
|
-
aiReferralHits:
|
|
32198
|
+
aiReferralHits: aiReferralCounts.total,
|
|
32199
|
+
aiReferralPaidHits: aiReferralCounts.paid,
|
|
32200
|
+
aiReferralOrganicHits: aiReferralCounts.organic,
|
|
32201
|
+
aiReferralUnknownHits: aiReferralCounts.unknown
|
|
31869
32202
|
},
|
|
31870
32203
|
events: trimmed
|
|
31871
32204
|
};
|