@ainyc/canonry 1.33.0 → 1.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{index-BiIPqZJb.js → index-BPuIj1DV.js} +67 -67
- package/assets/assets/{index-B5Cg2H9M.css → index-CxMuEW6I.css} +1 -1
- package/assets/index.html +2 -2
- package/dist/{chunk-WRNSBFNQ.js → chunk-4SRBJCNX.js} +686 -627
- package/dist/cli.js +19 -16
- package/dist/index.js +1 -1
- package/package.json +6 -6
|
@@ -1123,7 +1123,12 @@ function extractAnswerMentions(answerText, displayName, domains) {
|
|
|
1123
1123
|
matchedTerms.push(...matchedTokens);
|
|
1124
1124
|
}
|
|
1125
1125
|
const unique = [...new Set(matchedTerms)];
|
|
1126
|
-
|
|
1126
|
+
const domainMatches3 = unique.filter((t) => t.includes("."));
|
|
1127
|
+
const dedupedFinal = unique.filter((term) => {
|
|
1128
|
+
if (term.includes(".")) return true;
|
|
1129
|
+
return !domainMatches3.some((d) => d.toLowerCase().startsWith(term.toLowerCase() + "."));
|
|
1130
|
+
});
|
|
1131
|
+
return { mentioned: dedupedFinal.length > 0, matchedTerms: dedupedFinal };
|
|
1127
1132
|
}
|
|
1128
1133
|
function determineAnswerMentioned(answerText, displayName, domains) {
|
|
1129
1134
|
return extractAnswerMentions(answerText, displayName, domains).mentioned;
|
|
@@ -1131,6 +1136,9 @@ function determineAnswerMentioned(answerText, displayName, domains) {
|
|
|
1131
1136
|
function visibilityStateFromAnswerMentioned(answerMentioned) {
|
|
1132
1137
|
return answerMentioned ? "visible" : "not-visible";
|
|
1133
1138
|
}
|
|
1139
|
+
function brandKeyFromText(value) {
|
|
1140
|
+
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
1141
|
+
}
|
|
1134
1142
|
function domainMentioned(lowerAnswer, normalizedDomain) {
|
|
1135
1143
|
const escapedDomain = escapeRegExp(normalizedDomain.toLowerCase());
|
|
1136
1144
|
const patterns = [
|
|
@@ -6241,7 +6249,7 @@ function formatNotification(row) {
|
|
|
6241
6249
|
}
|
|
6242
6250
|
|
|
6243
6251
|
// ../api-routes/src/google.ts
|
|
6244
|
-
import
|
|
6252
|
+
import crypto14 from "crypto";
|
|
6245
6253
|
import { eq as eq13, and as and2, desc as desc4, sql as sql3 } from "drizzle-orm";
|
|
6246
6254
|
|
|
6247
6255
|
// ../integration-google/src/constants.ts
|
|
@@ -6429,104 +6437,435 @@ async function inspectUrl(accessToken, inspectionUrl, siteUrl) {
|
|
|
6429
6437
|
);
|
|
6430
6438
|
}
|
|
6431
6439
|
|
|
6432
|
-
// ../
|
|
6433
|
-
|
|
6434
|
-
|
|
6440
|
+
// ../integration-google-analytics/src/ga4-client.ts
|
|
6441
|
+
import crypto13 from "crypto";
|
|
6442
|
+
|
|
6443
|
+
// ../integration-google-analytics/src/constants.ts
|
|
6444
|
+
var GA4_DATA_API_BASE = "https://analyticsdata.googleapis.com/v1beta";
|
|
6445
|
+
var GA4_SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
|
|
6446
|
+
var GOOGLE_TOKEN_URL2 = "https://oauth2.googleapis.com/token";
|
|
6447
|
+
var GA4_DEFAULT_SYNC_DAYS = 30;
|
|
6448
|
+
var GA4_MAX_SYNC_DAYS = 90;
|
|
6449
|
+
|
|
6450
|
+
// ../integration-google-analytics/src/types.ts
|
|
6451
|
+
var GA4ApiError = class extends Error {
|
|
6452
|
+
status;
|
|
6453
|
+
constructor(message, status) {
|
|
6454
|
+
super(message);
|
|
6455
|
+
this.name = "GA4ApiError";
|
|
6456
|
+
this.status = status;
|
|
6457
|
+
}
|
|
6458
|
+
};
|
|
6459
|
+
|
|
6460
|
+
// ../integration-google-analytics/src/ga4-client.ts
|
|
6461
|
+
function ga4Log(level, action, ctx) {
|
|
6462
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Client", action, ...ctx };
|
|
6463
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6464
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6435
6465
|
}
|
|
6436
|
-
function
|
|
6437
|
-
const
|
|
6438
|
-
const
|
|
6439
|
-
|
|
6466
|
+
function createServiceAccountJwt(clientEmail, privateKey, scope) {
|
|
6467
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
6468
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
6469
|
+
const payload = {
|
|
6470
|
+
iss: clientEmail,
|
|
6471
|
+
scope,
|
|
6472
|
+
aud: GOOGLE_TOKEN_URL2,
|
|
6473
|
+
iat: now,
|
|
6474
|
+
exp: now + 3600
|
|
6475
|
+
// 1 hour
|
|
6476
|
+
};
|
|
6477
|
+
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
6478
|
+
const headerB64 = encode(header);
|
|
6479
|
+
const payloadB64 = encode(payload);
|
|
6480
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
6481
|
+
const sign = crypto13.createSign("RSA-SHA256");
|
|
6482
|
+
sign.update(signingInput);
|
|
6483
|
+
const signature = sign.sign(privateKey, "base64url");
|
|
6484
|
+
return `${signingInput}.${signature}`;
|
|
6440
6485
|
}
|
|
6441
|
-
function
|
|
6442
|
-
|
|
6443
|
-
|
|
6444
|
-
|
|
6445
|
-
|
|
6446
|
-
|
|
6447
|
-
|
|
6448
|
-
|
|
6486
|
+
async function getAccessToken(clientEmail, privateKey) {
|
|
6487
|
+
const jwt = createServiceAccountJwt(clientEmail, privateKey, GA4_SCOPE);
|
|
6488
|
+
const res = await fetch(GOOGLE_TOKEN_URL2, {
|
|
6489
|
+
method: "POST",
|
|
6490
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
6491
|
+
body: new URLSearchParams({
|
|
6492
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
6493
|
+
assertion: jwt
|
|
6494
|
+
})
|
|
6495
|
+
});
|
|
6496
|
+
if (!res.ok) {
|
|
6497
|
+
const body = await res.text().catch(() => "");
|
|
6498
|
+
ga4Log("error", "token.failed", { httpStatus: res.status, responseBody: body });
|
|
6499
|
+
throw new GA4ApiError(`Failed to get access token: ${body}`, res.status);
|
|
6449
6500
|
}
|
|
6501
|
+
const data = await res.json();
|
|
6502
|
+
return data.access_token;
|
|
6450
6503
|
}
|
|
6451
|
-
async function
|
|
6452
|
-
const
|
|
6453
|
-
|
|
6454
|
-
|
|
6504
|
+
async function runReport(accessToken, propertyId, request) {
|
|
6505
|
+
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:runReport`;
|
|
6506
|
+
const res = await fetch(url, {
|
|
6507
|
+
method: "POST",
|
|
6508
|
+
headers: {
|
|
6509
|
+
"Authorization": `Bearer ${accessToken}`,
|
|
6510
|
+
"Content-Type": "application/json"
|
|
6511
|
+
},
|
|
6512
|
+
body: JSON.stringify(request)
|
|
6513
|
+
});
|
|
6514
|
+
if (res.status === 401 || res.status === 403) {
|
|
6515
|
+
const body = await res.text().catch(() => "");
|
|
6516
|
+
let detail = "";
|
|
6517
|
+
try {
|
|
6518
|
+
const parsed = JSON.parse(body);
|
|
6519
|
+
if (parsed.error?.status === "SERVICE_DISABLED") {
|
|
6520
|
+
detail = " The Google Analytics Data API is not enabled for this GCP project. Enable it at: https://console.developers.google.com/apis/api/analyticsdata.googleapis.com/overview";
|
|
6521
|
+
} else if (parsed.error?.message) {
|
|
6522
|
+
detail = ` ${parsed.error.message}`;
|
|
6523
|
+
}
|
|
6524
|
+
} catch {
|
|
6525
|
+
if (body.length < 200) detail = ` ${body}`;
|
|
6526
|
+
}
|
|
6527
|
+
ga4Log("error", "report.auth-failed", { propertyId, httpStatus: res.status, responseBody: body });
|
|
6528
|
+
throw new GA4ApiError(
|
|
6529
|
+
`GA4 API authentication failed \u2014 check service account permissions.${detail}`,
|
|
6530
|
+
res.status
|
|
6531
|
+
);
|
|
6455
6532
|
}
|
|
6456
|
-
if (
|
|
6457
|
-
|
|
6533
|
+
if (res.status === 429) {
|
|
6534
|
+
ga4Log("error", "report.rate-limited", { propertyId });
|
|
6535
|
+
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
6458
6536
|
}
|
|
6459
|
-
|
|
6460
|
-
|
|
6461
|
-
|
|
6462
|
-
|
|
6463
|
-
const newExpiresAt = new Date(Date.now() + tokens.expires_in * 1e3).toISOString();
|
|
6464
|
-
const updated = store.updateConnection(domain, connectionType, {
|
|
6465
|
-
accessToken: tokens.access_token,
|
|
6466
|
-
tokenExpiresAt: newExpiresAt,
|
|
6467
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6468
|
-
});
|
|
6469
|
-
return {
|
|
6470
|
-
accessToken: tokens.access_token,
|
|
6471
|
-
connectionId: `${domain}:${connectionType}`,
|
|
6472
|
-
propertyId: updated?.propertyId ?? conn.propertyId ?? null
|
|
6473
|
-
};
|
|
6537
|
+
if (!res.ok) {
|
|
6538
|
+
const body = await res.text();
|
|
6539
|
+
ga4Log("error", "report.error", { propertyId, httpStatus: res.status, responseBody: body });
|
|
6540
|
+
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
6474
6541
|
}
|
|
6475
|
-
return
|
|
6476
|
-
accessToken: conn.accessToken,
|
|
6477
|
-
connectionId: `${domain}:${connectionType}`,
|
|
6478
|
-
propertyId: conn.propertyId ?? null
|
|
6479
|
-
};
|
|
6542
|
+
return await res.json();
|
|
6480
6543
|
}
|
|
6481
|
-
async function
|
|
6482
|
-
const
|
|
6483
|
-
|
|
6484
|
-
|
|
6544
|
+
async function batchRunReports(accessToken, propertyId, requests) {
|
|
6545
|
+
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:batchRunReports`;
|
|
6546
|
+
const res = await fetch(url, {
|
|
6547
|
+
method: "POST",
|
|
6548
|
+
headers: {
|
|
6549
|
+
"Authorization": `Bearer ${accessToken}`,
|
|
6550
|
+
"Content-Type": "application/json"
|
|
6551
|
+
},
|
|
6552
|
+
body: JSON.stringify({ requests })
|
|
6553
|
+
});
|
|
6554
|
+
if (res.status === 401 || res.status === 403) {
|
|
6555
|
+
const body = await res.text().catch(() => "");
|
|
6556
|
+
ga4Log("error", "batch-report.auth-failed", { propertyId, httpStatus: res.status });
|
|
6557
|
+
throw new GA4ApiError(
|
|
6558
|
+
`GA4 API authentication failed \u2014 check service account permissions. ${body}`,
|
|
6559
|
+
res.status
|
|
6560
|
+
);
|
|
6485
6561
|
}
|
|
6486
|
-
|
|
6487
|
-
|
|
6488
|
-
|
|
6489
|
-
reply.status(err.statusCode).send(err.toJSON());
|
|
6490
|
-
return null;
|
|
6562
|
+
if (res.status === 429) {
|
|
6563
|
+
ga4Log("error", "batch-report.rate-limited", { propertyId });
|
|
6564
|
+
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
6491
6565
|
}
|
|
6492
|
-
|
|
6493
|
-
const
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6502
|
-
|
|
6503
|
-
|
|
6566
|
+
if (!res.ok) {
|
|
6567
|
+
const body = await res.text();
|
|
6568
|
+
ga4Log("error", "batch-report.error", { propertyId, httpStatus: res.status });
|
|
6569
|
+
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
6570
|
+
}
|
|
6571
|
+
const data = await res.json();
|
|
6572
|
+
return data.reports;
|
|
6573
|
+
}
|
|
6574
|
+
function formatDate(d) {
|
|
6575
|
+
return d.toISOString().split("T")[0];
|
|
6576
|
+
}
|
|
6577
|
+
var AI_REFERRAL_SOURCE_FILTERS = [
|
|
6578
|
+
{ matchType: "CONTAINS", value: "perplexity" },
|
|
6579
|
+
{ matchType: "CONTAINS", value: "gemini" },
|
|
6580
|
+
{ matchType: "CONTAINS", value: "chatgpt" },
|
|
6581
|
+
{ matchType: "CONTAINS", value: "openai" },
|
|
6582
|
+
{ matchType: "CONTAINS", value: "claude" },
|
|
6583
|
+
{ matchType: "CONTAINS", value: "anthropic" },
|
|
6584
|
+
{ matchType: "CONTAINS", value: "copilot" }
|
|
6585
|
+
];
|
|
6586
|
+
async function fetchTrafficByLandingPage(accessToken, propertyId, days) {
|
|
6587
|
+
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
6588
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
6589
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
6590
|
+
startDate.setDate(startDate.getDate() - syncDays);
|
|
6591
|
+
ga4Log("info", "fetch-traffic.start", { propertyId, days: syncDays });
|
|
6592
|
+
const PAGE_SIZE = 1e4;
|
|
6593
|
+
const rows = [];
|
|
6594
|
+
let offset = 0;
|
|
6595
|
+
while (true) {
|
|
6596
|
+
const request = {
|
|
6597
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
6598
|
+
dimensions: [
|
|
6599
|
+
{ name: "date" },
|
|
6600
|
+
{ name: "landingPagePlusQueryString" }
|
|
6601
|
+
],
|
|
6602
|
+
metrics: [
|
|
6603
|
+
{ name: "sessions" },
|
|
6604
|
+
{ name: "totalUsers" }
|
|
6605
|
+
],
|
|
6606
|
+
limit: PAGE_SIZE,
|
|
6607
|
+
offset
|
|
6608
|
+
};
|
|
6609
|
+
const response = await runReport(accessToken, propertyId, request);
|
|
6610
|
+
const pageRows = (response.rows ?? []).map((row) => ({
|
|
6611
|
+
date: row.dimensionValues[0].value,
|
|
6612
|
+
landingPage: row.dimensionValues[1].value,
|
|
6613
|
+
sessions: parseInt(row.metricValues[0].value, 10) || 0,
|
|
6614
|
+
organicSessions: 0,
|
|
6615
|
+
// populated by organic-only pass below
|
|
6616
|
+
users: parseInt(row.metricValues[1].value, 10) || 0
|
|
6504
6617
|
}));
|
|
6505
|
-
|
|
6506
|
-
|
|
6507
|
-
|
|
6508
|
-
if (
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
|
|
6516
|
-
|
|
6517
|
-
|
|
6518
|
-
|
|
6519
|
-
|
|
6520
|
-
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
|
|
6525
|
-
|
|
6526
|
-
|
|
6527
|
-
|
|
6528
|
-
|
|
6529
|
-
|
|
6618
|
+
rows.push(...pageRows);
|
|
6619
|
+
const totalRows = response.rowCount ?? 0;
|
|
6620
|
+
offset += pageRows.length;
|
|
6621
|
+
if (pageRows.length < PAGE_SIZE || offset >= totalRows) break;
|
|
6622
|
+
}
|
|
6623
|
+
const organicMap = /* @__PURE__ */ new Map();
|
|
6624
|
+
let organicOffset = 0;
|
|
6625
|
+
while (true) {
|
|
6626
|
+
const organicRequest = {
|
|
6627
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
6628
|
+
dimensions: [{ name: "date" }, { name: "landingPagePlusQueryString" }],
|
|
6629
|
+
metrics: [{ name: "sessions" }],
|
|
6630
|
+
dimensionFilter: {
|
|
6631
|
+
filter: {
|
|
6632
|
+
fieldName: "sessionDefaultChannelGrouping",
|
|
6633
|
+
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
6634
|
+
}
|
|
6635
|
+
},
|
|
6636
|
+
limit: 1e4,
|
|
6637
|
+
offset: organicOffset
|
|
6638
|
+
};
|
|
6639
|
+
const organicResponse = await runReport(accessToken, propertyId, organicRequest);
|
|
6640
|
+
for (const row of organicResponse.rows ?? []) {
|
|
6641
|
+
const key = `${row.dimensionValues[0].value}::${row.dimensionValues[1].value}`;
|
|
6642
|
+
organicMap.set(key, parseInt(row.metricValues[0].value, 10) || 0);
|
|
6643
|
+
}
|
|
6644
|
+
const total = organicResponse.rowCount ?? 0;
|
|
6645
|
+
organicOffset += (organicResponse.rows ?? []).length;
|
|
6646
|
+
if ((organicResponse.rows ?? []).length < 1e4 || organicOffset >= total) break;
|
|
6647
|
+
}
|
|
6648
|
+
for (const row of rows) {
|
|
6649
|
+
const key = `${row.date}::${row.landingPage}`;
|
|
6650
|
+
row.organicSessions = organicMap.get(key) ?? 0;
|
|
6651
|
+
}
|
|
6652
|
+
for (const row of rows) {
|
|
6653
|
+
if (row.date.length === 8 && !row.date.includes("-")) {
|
|
6654
|
+
row.date = `${row.date.slice(0, 4)}-${row.date.slice(4, 6)}-${row.date.slice(6, 8)}`;
|
|
6655
|
+
}
|
|
6656
|
+
}
|
|
6657
|
+
ga4Log("info", "fetch-traffic.done", { propertyId, rowCount: rows.length });
|
|
6658
|
+
return rows;
|
|
6659
|
+
}
|
|
6660
|
+
async function verifyConnection(clientEmail, privateKey, propertyId) {
|
|
6661
|
+
const accessToken = await getAccessToken(clientEmail, privateKey);
|
|
6662
|
+
return verifyConnectionWithToken(accessToken, propertyId);
|
|
6663
|
+
}
|
|
6664
|
+
async function verifyConnectionWithToken(accessToken, propertyId) {
|
|
6665
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
6666
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
6667
|
+
startDate.setDate(startDate.getDate() - 1);
|
|
6668
|
+
await runReport(accessToken, propertyId, {
|
|
6669
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
6670
|
+
dimensions: [{ name: "date" }],
|
|
6671
|
+
metrics: [{ name: "sessions" }],
|
|
6672
|
+
limit: 1
|
|
6673
|
+
});
|
|
6674
|
+
return true;
|
|
6675
|
+
}
|
|
6676
|
+
async function fetchAggregateSummary(accessToken, propertyId, days) {
|
|
6677
|
+
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
6678
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
6679
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
6680
|
+
startDate.setDate(startDate.getDate() - syncDays);
|
|
6681
|
+
ga4Log("info", "fetch-aggregate.start", { propertyId, days: syncDays });
|
|
6682
|
+
const dateRange = { startDate: formatDate(startDate), endDate: formatDate(endDate) };
|
|
6683
|
+
const batchRes = await batchRunReports(accessToken, propertyId, [
|
|
6684
|
+
{
|
|
6685
|
+
dateRanges: [dateRange],
|
|
6686
|
+
dimensions: [],
|
|
6687
|
+
metrics: [{ name: "sessions" }, { name: "totalUsers" }],
|
|
6688
|
+
limit: 1
|
|
6689
|
+
},
|
|
6690
|
+
{
|
|
6691
|
+
dateRanges: [dateRange],
|
|
6692
|
+
dimensions: [],
|
|
6693
|
+
metrics: [{ name: "sessions" }],
|
|
6694
|
+
dimensionFilter: {
|
|
6695
|
+
filter: {
|
|
6696
|
+
fieldName: "sessionDefaultChannelGrouping",
|
|
6697
|
+
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
6698
|
+
}
|
|
6699
|
+
},
|
|
6700
|
+
limit: 1
|
|
6701
|
+
}
|
|
6702
|
+
]);
|
|
6703
|
+
const totalRow = batchRes[0]?.rows?.[0];
|
|
6704
|
+
const organicRow = batchRes[1]?.rows?.[0];
|
|
6705
|
+
const summary = {
|
|
6706
|
+
periodStart: formatDate(startDate),
|
|
6707
|
+
periodEnd: formatDate(endDate),
|
|
6708
|
+
totalSessions: parseInt(totalRow?.metricValues[0]?.value ?? "0", 10) || 0,
|
|
6709
|
+
totalUsers: parseInt(totalRow?.metricValues[1]?.value ?? "0", 10) || 0,
|
|
6710
|
+
totalOrganicSessions: parseInt(organicRow?.metricValues[0]?.value ?? "0", 10) || 0
|
|
6711
|
+
};
|
|
6712
|
+
ga4Log("info", "fetch-aggregate.done", { propertyId, ...summary });
|
|
6713
|
+
return summary;
|
|
6714
|
+
}
|
|
6715
|
+
async function fetchAiReferrals(accessToken, propertyId, days) {
|
|
6716
|
+
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
6717
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
6718
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
6719
|
+
startDate.setDate(startDate.getDate() - syncDays);
|
|
6720
|
+
ga4Log("info", "fetch-ai-referrals.start", { propertyId, days: syncDays });
|
|
6721
|
+
const PAGE_SIZE = 1e3;
|
|
6722
|
+
const rows = [];
|
|
6723
|
+
let offset = 0;
|
|
6724
|
+
while (true) {
|
|
6725
|
+
const request = {
|
|
6726
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
6727
|
+
dimensions: [
|
|
6728
|
+
{ name: "date" },
|
|
6729
|
+
{ name: "sessionSource" },
|
|
6730
|
+
{ name: "sessionMedium" }
|
|
6731
|
+
],
|
|
6732
|
+
metrics: [
|
|
6733
|
+
{ name: "sessions" },
|
|
6734
|
+
{ name: "totalUsers" }
|
|
6735
|
+
],
|
|
6736
|
+
dimensionFilter: {
|
|
6737
|
+
orGroup: {
|
|
6738
|
+
expressions: AI_REFERRAL_SOURCE_FILTERS.map(({ matchType, value }) => ({
|
|
6739
|
+
filter: {
|
|
6740
|
+
fieldName: "sessionSource",
|
|
6741
|
+
stringFilter: { matchType, value }
|
|
6742
|
+
}
|
|
6743
|
+
}))
|
|
6744
|
+
}
|
|
6745
|
+
},
|
|
6746
|
+
limit: PAGE_SIZE,
|
|
6747
|
+
offset
|
|
6748
|
+
};
|
|
6749
|
+
const response = await runReport(accessToken, propertyId, request);
|
|
6750
|
+
const pageRows = (response.rows ?? []).map((row) => ({
|
|
6751
|
+
date: row.dimensionValues[0].value,
|
|
6752
|
+
source: row.dimensionValues[1].value,
|
|
6753
|
+
medium: row.dimensionValues[2].value,
|
|
6754
|
+
sessions: parseInt(row.metricValues[0].value, 10) || 0,
|
|
6755
|
+
users: parseInt(row.metricValues[1].value, 10) || 0
|
|
6756
|
+
}));
|
|
6757
|
+
rows.push(...pageRows);
|
|
6758
|
+
const totalRows = response.rowCount ?? 0;
|
|
6759
|
+
offset += pageRows.length;
|
|
6760
|
+
if (pageRows.length < PAGE_SIZE || offset >= totalRows) break;
|
|
6761
|
+
}
|
|
6762
|
+
for (const row of rows) {
|
|
6763
|
+
if (row.date.length === 8 && !row.date.includes("-")) {
|
|
6764
|
+
row.date = `${row.date.slice(0, 4)}-${row.date.slice(4, 6)}-${row.date.slice(6, 8)}`;
|
|
6765
|
+
}
|
|
6766
|
+
}
|
|
6767
|
+
ga4Log("info", "fetch-ai-referrals.done", { propertyId, rowCount: rows.length });
|
|
6768
|
+
return rows;
|
|
6769
|
+
}
|
|
6770
|
+
|
|
6771
|
+
// ../api-routes/src/google.ts
|
|
6772
|
+
function signState(payload, secret) {
|
|
6773
|
+
return crypto14.createHmac("sha256", secret).update(payload).digest("hex");
|
|
6774
|
+
}
|
|
6775
|
+
function buildSignedState(data, secret) {
|
|
6776
|
+
const payload = JSON.stringify(data);
|
|
6777
|
+
const sig = signState(payload, secret);
|
|
6778
|
+
return Buffer.from(JSON.stringify({ payload, sig })).toString("base64url");
|
|
6779
|
+
}
|
|
6780
|
+
function verifySignedState(encoded, secret) {
|
|
6781
|
+
try {
|
|
6782
|
+
const { payload, sig } = JSON.parse(Buffer.from(encoded, "base64url").toString());
|
|
6783
|
+
const expected = signState(payload, secret);
|
|
6784
|
+
if (!crypto14.timingSafeEqual(Buffer.from(sig, "hex"), Buffer.from(expected, "hex"))) return null;
|
|
6785
|
+
return JSON.parse(payload);
|
|
6786
|
+
} catch {
|
|
6787
|
+
return null;
|
|
6788
|
+
}
|
|
6789
|
+
}
|
|
6790
|
+
async function getValidToken(store, domain, connectionType, clientId, clientSecret) {
|
|
6791
|
+
const conn = store.getConnection(domain, connectionType);
|
|
6792
|
+
if (!conn) {
|
|
6793
|
+
throw notFound("Google connection", connectionType);
|
|
6794
|
+
}
|
|
6795
|
+
if (!conn.accessToken || !conn.refreshToken) {
|
|
6796
|
+
throw validationError("Google connection is incomplete \u2014 please reconnect");
|
|
6797
|
+
}
|
|
6798
|
+
const expiresAt = conn.tokenExpiresAt ? new Date(conn.tokenExpiresAt).getTime() : 0;
|
|
6799
|
+
const fiveMinutes = 5 * 60 * 1e3;
|
|
6800
|
+
if (Date.now() > expiresAt - fiveMinutes) {
|
|
6801
|
+
const tokens = await refreshAccessToken(clientId, clientSecret, conn.refreshToken);
|
|
6802
|
+
const newExpiresAt = new Date(Date.now() + tokens.expires_in * 1e3).toISOString();
|
|
6803
|
+
const updated = store.updateConnection(domain, connectionType, {
|
|
6804
|
+
accessToken: tokens.access_token,
|
|
6805
|
+
tokenExpiresAt: newExpiresAt,
|
|
6806
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6807
|
+
});
|
|
6808
|
+
return {
|
|
6809
|
+
accessToken: tokens.access_token,
|
|
6810
|
+
connectionId: `${domain}:${connectionType}`,
|
|
6811
|
+
propertyId: updated?.propertyId ?? conn.propertyId ?? null
|
|
6812
|
+
};
|
|
6813
|
+
}
|
|
6814
|
+
return {
|
|
6815
|
+
accessToken: conn.accessToken,
|
|
6816
|
+
connectionId: `${domain}:${connectionType}`,
|
|
6817
|
+
propertyId: conn.propertyId ?? null
|
|
6818
|
+
};
|
|
6819
|
+
}
|
|
6820
|
+
async function googleRoutes(app, opts) {
|
|
6821
|
+
const stateSecret = opts.googleStateSecret ?? "insecure-default-secret";
|
|
6822
|
+
function getAuthConfig() {
|
|
6823
|
+
return opts.getGoogleAuthConfig?.() ?? {};
|
|
6824
|
+
}
|
|
6825
|
+
function requireConnectionStore(reply) {
|
|
6826
|
+
if (opts.googleConnectionStore) return opts.googleConnectionStore;
|
|
6827
|
+
const err = validationError("Google auth storage is not configured for this deployment");
|
|
6828
|
+
reply.status(err.statusCode).send(err.toJSON());
|
|
6829
|
+
return null;
|
|
6830
|
+
}
|
|
6831
|
+
app.get("/projects/:name/google/connections", async (request) => {
|
|
6832
|
+
const project = resolveProject(app.db, request.params.name);
|
|
6833
|
+
const conns = opts.googleConnectionStore?.listConnections(project.canonicalDomain) ?? [];
|
|
6834
|
+
return conns.map((connection) => ({
|
|
6835
|
+
id: `${connection.domain}:${connection.connectionType}`,
|
|
6836
|
+
domain: connection.domain,
|
|
6837
|
+
connectionType: connection.connectionType,
|
|
6838
|
+
propertyId: connection.propertyId ?? null,
|
|
6839
|
+
sitemapUrl: connection.sitemapUrl ?? null,
|
|
6840
|
+
scopes: connection.scopes ?? [],
|
|
6841
|
+
createdAt: connection.createdAt,
|
|
6842
|
+
updatedAt: connection.updatedAt
|
|
6843
|
+
}));
|
|
6844
|
+
});
|
|
6845
|
+
app.post("/projects/:name/google/connect", async (request, reply) => {
|
|
6846
|
+
const { clientId: googleClientId, clientSecret: googleClientSecret } = getAuthConfig();
|
|
6847
|
+
if (!googleClientId || !googleClientSecret) {
|
|
6848
|
+
const err = validationError("Google OAuth is not configured. Set Google OAuth credentials in the local Canonry config.");
|
|
6849
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
6850
|
+
}
|
|
6851
|
+
const { type, propertyId, publicUrl } = request.body ?? {};
|
|
6852
|
+
if (!type || type !== "gsc" && type !== "ga4") {
|
|
6853
|
+
const err = validationError('type must be "gsc" or "ga4"');
|
|
6854
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
6855
|
+
}
|
|
6856
|
+
const project = resolveProject(app.db, request.params.name);
|
|
6857
|
+
let redirectUri;
|
|
6858
|
+
if (publicUrl) {
|
|
6859
|
+
redirectUri = publicUrl.replace(/\/$/, "") + "/api/v1/google/callback";
|
|
6860
|
+
} else if (opts.publicUrl) {
|
|
6861
|
+
redirectUri = opts.publicUrl.replace(/\/$/, "") + "/api/v1/google/callback";
|
|
6862
|
+
} else {
|
|
6863
|
+
const proto = request.headers["x-forwarded-proto"] ?? "http";
|
|
6864
|
+
const host = request.headers.host ?? "localhost:4100";
|
|
6865
|
+
redirectUri = `${proto}://${host}/api/v1/projects/${encodeURIComponent(request.params.name)}/google/callback`;
|
|
6866
|
+
}
|
|
6867
|
+
const scopes = type === "gsc" ? [GSC_SCOPE, INDEXING_SCOPE] : [GA4_SCOPE];
|
|
6868
|
+
const stateEncoded = buildSignedState(
|
|
6530
6869
|
{ domain: project.canonicalDomain, type, propertyId, redirectUri },
|
|
6531
6870
|
stateSecret
|
|
6532
6871
|
);
|
|
@@ -6669,7 +7008,7 @@ async function googleRoutes(app, opts) {
|
|
|
6669
7008
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6670
7009
|
}
|
|
6671
7010
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6672
|
-
const runId =
|
|
7011
|
+
const runId = crypto14.randomUUID();
|
|
6673
7012
|
app.db.insert(runs).values({
|
|
6674
7013
|
id: runId,
|
|
6675
7014
|
projectId: project.id,
|
|
@@ -6731,7 +7070,7 @@ async function googleRoutes(app, opts) {
|
|
|
6731
7070
|
const mob = ir.mobileUsabilityResult;
|
|
6732
7071
|
const rich = ir.richResultsResult;
|
|
6733
7072
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6734
|
-
const id =
|
|
7073
|
+
const id = crypto14.randomUUID();
|
|
6735
7074
|
app.db.insert(gscUrlInspections).values({
|
|
6736
7075
|
id,
|
|
6737
7076
|
projectId: project.id,
|
|
@@ -6971,7 +7310,7 @@ async function googleRoutes(app, opts) {
|
|
|
6971
7310
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
6972
7311
|
});
|
|
6973
7312
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6974
|
-
const runId =
|
|
7313
|
+
const runId = crypto14.randomUUID();
|
|
6975
7314
|
app.db.insert(runs).values({
|
|
6976
7315
|
id: runId,
|
|
6977
7316
|
projectId: project.id,
|
|
@@ -7000,7 +7339,7 @@ async function googleRoutes(app, opts) {
|
|
|
7000
7339
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
7001
7340
|
}
|
|
7002
7341
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7003
|
-
const runId =
|
|
7342
|
+
const runId = crypto14.randomUUID();
|
|
7004
7343
|
app.db.insert(runs).values({
|
|
7005
7344
|
id: runId,
|
|
7006
7345
|
projectId: project.id,
|
|
@@ -7142,7 +7481,7 @@ async function googleRoutes(app, opts) {
|
|
|
7142
7481
|
}
|
|
7143
7482
|
|
|
7144
7483
|
// ../api-routes/src/bing.ts
|
|
7145
|
-
import
|
|
7484
|
+
import crypto15 from "crypto";
|
|
7146
7485
|
import { eq as eq14, and as and3, desc as desc5 } from "drizzle-orm";
|
|
7147
7486
|
|
|
7148
7487
|
// ../integration-bing/src/constants.ts
|
|
@@ -7489,7 +7828,7 @@ async function bingRoutes(app, opts) {
|
|
|
7489
7828
|
throw e;
|
|
7490
7829
|
}
|
|
7491
7830
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7492
|
-
const id =
|
|
7831
|
+
const id = crypto15.randomUUID();
|
|
7493
7832
|
const httpCode = result.HttpStatus ?? result.HttpCode ?? null;
|
|
7494
7833
|
let derivedInIndex = null;
|
|
7495
7834
|
if (result.InIndex != null) {
|
|
@@ -7697,500 +8036,251 @@ async function cdpRoutes(app, opts) {
|
|
|
7697
8036
|
const err = validationError("query is required");
|
|
7698
8037
|
return reply.code(err.statusCode).send(err.toJSON());
|
|
7699
8038
|
}
|
|
7700
|
-
const results = await opts.onCdpScreenshot(query, targets);
|
|
7701
|
-
return reply.code(200).send({ results });
|
|
7702
|
-
});
|
|
7703
|
-
app.get(
|
|
7704
|
-
"/projects/:name/runs/:runId/browser-diff",
|
|
7705
|
-
async (request, reply) => {
|
|
7706
|
-
const project = resolveProject(app.db, request.params.name);
|
|
7707
|
-
const { runId } = request.params;
|
|
7708
|
-
const run = app.db.select().from(runs).where(and4(eq15(runs.id, runId), eq15(runs.projectId, project.id))).get();
|
|
7709
|
-
if (!run) {
|
|
7710
|
-
const err = notFound("Run", runId);
|
|
7711
|
-
return reply.code(err.statusCode).send(err.toJSON());
|
|
7712
|
-
}
|
|
7713
|
-
const snapshots = app.db.select({
|
|
7714
|
-
id: querySnapshots.id,
|
|
7715
|
-
keywordId: querySnapshots.keywordId,
|
|
7716
|
-
provider: querySnapshots.provider,
|
|
7717
|
-
citationState: querySnapshots.citationState,
|
|
7718
|
-
citedDomains: querySnapshots.citedDomains,
|
|
7719
|
-
screenshotPath: querySnapshots.screenshotPath,
|
|
7720
|
-
rawResponse: querySnapshots.rawResponse
|
|
7721
|
-
}).from(querySnapshots).where(eq15(querySnapshots.runId, runId)).all();
|
|
7722
|
-
const keywordRows = app.db.select({ id: keywords.id, keyword: keywords.keyword }).from(keywords).where(eq15(keywords.projectId, project.id)).all();
|
|
7723
|
-
const keywordMap = new Map(keywordRows.map((k) => [k.id, k.keyword]));
|
|
7724
|
-
const byKeyword = /* @__PURE__ */ new Map();
|
|
7725
|
-
for (const snap of snapshots) {
|
|
7726
|
-
const kwName = keywordMap.get(snap.keywordId) ?? snap.keywordId;
|
|
7727
|
-
if (!byKeyword.has(snap.keywordId)) {
|
|
7728
|
-
byKeyword.set(snap.keywordId, { keyword: kwName, api: null, browser: null });
|
|
7729
|
-
}
|
|
7730
|
-
const entry = byKeyword.get(snap.keywordId);
|
|
7731
|
-
if (snap.provider === "cdp:chatgpt") {
|
|
7732
|
-
entry.browser = snap;
|
|
7733
|
-
} else if (snap.provider === "openai") {
|
|
7734
|
-
entry.api = snap;
|
|
7735
|
-
}
|
|
7736
|
-
}
|
|
7737
|
-
let agreed = 0;
|
|
7738
|
-
let apiOnlyCited = 0;
|
|
7739
|
-
let browserOnlyCited = 0;
|
|
7740
|
-
let disagreed = 0;
|
|
7741
|
-
let total = 0;
|
|
7742
|
-
const keywordResults = [...byKeyword.values()].map(({ keyword, api, browser }) => {
|
|
7743
|
-
total++;
|
|
7744
|
-
const apiCited = api?.citationState === "cited";
|
|
7745
|
-
const browserCited = browser?.citationState === "cited";
|
|
7746
|
-
let agreement;
|
|
7747
|
-
if (!api && !browser) {
|
|
7748
|
-
agreement = "no-data";
|
|
7749
|
-
} else if (!api) {
|
|
7750
|
-
agreement = "no-api";
|
|
7751
|
-
} else if (!browser) {
|
|
7752
|
-
agreement = "no-browser";
|
|
7753
|
-
} else if (apiCited && browserCited) {
|
|
7754
|
-
agreement = "agree-cited";
|
|
7755
|
-
agreed++;
|
|
7756
|
-
} else if (!apiCited && !browserCited) {
|
|
7757
|
-
agreement = "agree-not-cited";
|
|
7758
|
-
agreed++;
|
|
7759
|
-
} else if (apiCited && !browserCited) {
|
|
7760
|
-
agreement = "api-only-cited";
|
|
7761
|
-
apiOnlyCited++;
|
|
7762
|
-
disagreed++;
|
|
7763
|
-
} else {
|
|
7764
|
-
agreement = "browser-only-cited";
|
|
7765
|
-
browserOnlyCited++;
|
|
7766
|
-
disagreed++;
|
|
7767
|
-
}
|
|
7768
|
-
const parseGroundingSources2 = (snap) => {
|
|
7769
|
-
if (!snap?.rawResponse) return [];
|
|
7770
|
-
try {
|
|
7771
|
-
const raw = JSON.parse(snap.rawResponse);
|
|
7772
|
-
return raw.groundingSources ?? [];
|
|
7773
|
-
} catch {
|
|
7774
|
-
return [];
|
|
7775
|
-
}
|
|
7776
|
-
};
|
|
7777
|
-
return {
|
|
7778
|
-
keyword,
|
|
7779
|
-
api: api ? {
|
|
7780
|
-
provider: api.provider,
|
|
7781
|
-
citationState: api.citationState,
|
|
7782
|
-
citedDomains: JSON.parse(api.citedDomains || "[]"),
|
|
7783
|
-
groundingSources: parseGroundingSources2(api)
|
|
7784
|
-
} : null,
|
|
7785
|
-
browser: browser ? {
|
|
7786
|
-
provider: browser.provider,
|
|
7787
|
-
citationState: browser.citationState,
|
|
7788
|
-
citedDomains: JSON.parse(browser.citedDomains || "[]"),
|
|
7789
|
-
groundingSources: parseGroundingSources2(browser),
|
|
7790
|
-
screenshotUrl: browser.screenshotPath ? `/api/v1/screenshots/${browser.id}` : void 0
|
|
7791
|
-
} : null,
|
|
7792
|
-
agreement
|
|
7793
|
-
};
|
|
7794
|
-
});
|
|
7795
|
-
return reply.send({
|
|
7796
|
-
summary: { total, agreed, apiOnly: apiOnlyCited, browserOnly: browserOnlyCited, disagreed },
|
|
7797
|
-
keywords: keywordResults
|
|
7798
|
-
});
|
|
7799
|
-
}
|
|
7800
|
-
);
|
|
7801
|
-
}
|
|
7802
|
-
|
|
7803
|
-
// ../api-routes/src/ga.ts
|
|
7804
|
-
import crypto16 from "crypto";
|
|
7805
|
-
import { eq as eq16, desc as desc6, and as and5, sql as sql4 } from "drizzle-orm";
|
|
7806
|
-
|
|
7807
|
-
// ../integration-google-analytics/src/ga4-client.ts
|
|
7808
|
-
import crypto15 from "crypto";
|
|
7809
|
-
|
|
7810
|
-
// ../integration-google-analytics/src/constants.ts
|
|
7811
|
-
var GA4_DATA_API_BASE = "https://analyticsdata.googleapis.com/v1beta";
|
|
7812
|
-
var GA4_SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
|
|
7813
|
-
var GOOGLE_TOKEN_URL2 = "https://oauth2.googleapis.com/token";
|
|
7814
|
-
var GA4_DEFAULT_SYNC_DAYS = 30;
|
|
7815
|
-
var GA4_MAX_SYNC_DAYS = 90;
|
|
7816
|
-
|
|
7817
|
-
// ../integration-google-analytics/src/types.ts
|
|
7818
|
-
var GA4ApiError = class extends Error {
|
|
7819
|
-
status;
|
|
7820
|
-
constructor(message, status) {
|
|
7821
|
-
super(message);
|
|
7822
|
-
this.name = "GA4ApiError";
|
|
7823
|
-
this.status = status;
|
|
7824
|
-
}
|
|
7825
|
-
};
|
|
7826
|
-
|
|
7827
|
-
// ../integration-google-analytics/src/ga4-client.ts
|
|
7828
|
-
function ga4Log(level, action, ctx) {
|
|
7829
|
-
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Client", action, ...ctx };
|
|
7830
|
-
const stream = level === "error" ? process.stderr : process.stdout;
|
|
7831
|
-
stream.write(JSON.stringify(entry) + "\n");
|
|
7832
|
-
}
|
|
7833
|
-
function createServiceAccountJwt(clientEmail, privateKey, scope) {
|
|
7834
|
-
const now = Math.floor(Date.now() / 1e3);
|
|
7835
|
-
const header = { alg: "RS256", typ: "JWT" };
|
|
7836
|
-
const payload = {
|
|
7837
|
-
iss: clientEmail,
|
|
7838
|
-
scope,
|
|
7839
|
-
aud: GOOGLE_TOKEN_URL2,
|
|
7840
|
-
iat: now,
|
|
7841
|
-
exp: now + 3600
|
|
7842
|
-
// 1 hour
|
|
7843
|
-
};
|
|
7844
|
-
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
7845
|
-
const headerB64 = encode(header);
|
|
7846
|
-
const payloadB64 = encode(payload);
|
|
7847
|
-
const signingInput = `${headerB64}.${payloadB64}`;
|
|
7848
|
-
const sign = crypto15.createSign("RSA-SHA256");
|
|
7849
|
-
sign.update(signingInput);
|
|
7850
|
-
const signature = sign.sign(privateKey, "base64url");
|
|
7851
|
-
return `${signingInput}.${signature}`;
|
|
7852
|
-
}
|
|
7853
|
-
async function getAccessToken(clientEmail, privateKey) {
|
|
7854
|
-
const jwt = createServiceAccountJwt(clientEmail, privateKey, GA4_SCOPE);
|
|
7855
|
-
const res = await fetch(GOOGLE_TOKEN_URL2, {
|
|
7856
|
-
method: "POST",
|
|
7857
|
-
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
7858
|
-
body: new URLSearchParams({
|
|
7859
|
-
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
7860
|
-
assertion: jwt
|
|
7861
|
-
})
|
|
7862
|
-
});
|
|
7863
|
-
if (!res.ok) {
|
|
7864
|
-
const body = await res.text().catch(() => "");
|
|
7865
|
-
ga4Log("error", "token.failed", { httpStatus: res.status, responseBody: body });
|
|
7866
|
-
throw new GA4ApiError(`Failed to get access token: ${body}`, res.status);
|
|
7867
|
-
}
|
|
7868
|
-
const data = await res.json();
|
|
7869
|
-
return data.access_token;
|
|
7870
|
-
}
|
|
7871
|
-
async function runReport(accessToken, propertyId, request) {
|
|
7872
|
-
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:runReport`;
|
|
7873
|
-
const res = await fetch(url, {
|
|
7874
|
-
method: "POST",
|
|
7875
|
-
headers: {
|
|
7876
|
-
"Authorization": `Bearer ${accessToken}`,
|
|
7877
|
-
"Content-Type": "application/json"
|
|
7878
|
-
},
|
|
7879
|
-
body: JSON.stringify(request)
|
|
7880
|
-
});
|
|
7881
|
-
if (res.status === 401 || res.status === 403) {
|
|
7882
|
-
const body = await res.text().catch(() => "");
|
|
7883
|
-
let detail = "";
|
|
7884
|
-
try {
|
|
7885
|
-
const parsed = JSON.parse(body);
|
|
7886
|
-
if (parsed.error?.status === "SERVICE_DISABLED") {
|
|
7887
|
-
detail = " The Google Analytics Data API is not enabled for this GCP project. Enable it at: https://console.developers.google.com/apis/api/analyticsdata.googleapis.com/overview";
|
|
7888
|
-
} else if (parsed.error?.message) {
|
|
7889
|
-
detail = ` ${parsed.error.message}`;
|
|
7890
|
-
}
|
|
7891
|
-
} catch {
|
|
7892
|
-
if (body.length < 200) detail = ` ${body}`;
|
|
7893
|
-
}
|
|
7894
|
-
ga4Log("error", "report.auth-failed", { propertyId, httpStatus: res.status, responseBody: body });
|
|
7895
|
-
throw new GA4ApiError(
|
|
7896
|
-
`GA4 API authentication failed \u2014 check service account permissions.${detail}`,
|
|
7897
|
-
res.status
|
|
7898
|
-
);
|
|
7899
|
-
}
|
|
7900
|
-
if (res.status === 429) {
|
|
7901
|
-
ga4Log("error", "report.rate-limited", { propertyId });
|
|
7902
|
-
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
7903
|
-
}
|
|
7904
|
-
if (!res.ok) {
|
|
7905
|
-
const body = await res.text();
|
|
7906
|
-
ga4Log("error", "report.error", { propertyId, httpStatus: res.status, responseBody: body });
|
|
7907
|
-
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
7908
|
-
}
|
|
7909
|
-
return await res.json();
|
|
7910
|
-
}
|
|
7911
|
-
async function batchRunReports(accessToken, propertyId, requests) {
|
|
7912
|
-
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:batchRunReports`;
|
|
7913
|
-
const res = await fetch(url, {
|
|
7914
|
-
method: "POST",
|
|
7915
|
-
headers: {
|
|
7916
|
-
"Authorization": `Bearer ${accessToken}`,
|
|
7917
|
-
"Content-Type": "application/json"
|
|
7918
|
-
},
|
|
7919
|
-
body: JSON.stringify({ requests })
|
|
7920
|
-
});
|
|
7921
|
-
if (res.status === 401 || res.status === 403) {
|
|
7922
|
-
const body = await res.text().catch(() => "");
|
|
7923
|
-
ga4Log("error", "batch-report.auth-failed", { propertyId, httpStatus: res.status });
|
|
7924
|
-
throw new GA4ApiError(
|
|
7925
|
-
`GA4 API authentication failed \u2014 check service account permissions. ${body}`,
|
|
7926
|
-
res.status
|
|
7927
|
-
);
|
|
7928
|
-
}
|
|
7929
|
-
if (res.status === 429) {
|
|
7930
|
-
ga4Log("error", "batch-report.rate-limited", { propertyId });
|
|
7931
|
-
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
7932
|
-
}
|
|
7933
|
-
if (!res.ok) {
|
|
7934
|
-
const body = await res.text();
|
|
7935
|
-
ga4Log("error", "batch-report.error", { propertyId, httpStatus: res.status });
|
|
7936
|
-
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
7937
|
-
}
|
|
7938
|
-
const data = await res.json();
|
|
7939
|
-
return data.reports;
|
|
7940
|
-
}
|
|
7941
|
-
function formatDate(d) {
|
|
7942
|
-
return d.toISOString().split("T")[0];
|
|
7943
|
-
}
|
|
7944
|
-
var AI_REFERRAL_SOURCE_FILTERS = [
|
|
7945
|
-
{ matchType: "CONTAINS", value: "perplexity" },
|
|
7946
|
-
{ matchType: "CONTAINS", value: "gemini" },
|
|
7947
|
-
{ matchType: "CONTAINS", value: "chatgpt" },
|
|
7948
|
-
{ matchType: "CONTAINS", value: "openai" },
|
|
7949
|
-
{ matchType: "CONTAINS", value: "claude" },
|
|
7950
|
-
{ matchType: "CONTAINS", value: "anthropic" },
|
|
7951
|
-
{ matchType: "CONTAINS", value: "copilot" }
|
|
7952
|
-
];
|
|
7953
|
-
async function fetchTrafficByLandingPage(accessToken, propertyId, days) {
|
|
7954
|
-
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
7955
|
-
const endDate = /* @__PURE__ */ new Date();
|
|
7956
|
-
const startDate = /* @__PURE__ */ new Date();
|
|
7957
|
-
startDate.setDate(startDate.getDate() - syncDays);
|
|
7958
|
-
ga4Log("info", "fetch-traffic.start", { propertyId, days: syncDays });
|
|
7959
|
-
const PAGE_SIZE = 1e4;
|
|
7960
|
-
const rows = [];
|
|
7961
|
-
let offset = 0;
|
|
7962
|
-
while (true) {
|
|
7963
|
-
const request = {
|
|
7964
|
-
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
7965
|
-
dimensions: [
|
|
7966
|
-
{ name: "date" },
|
|
7967
|
-
{ name: "landingPagePlusQueryString" }
|
|
7968
|
-
],
|
|
7969
|
-
metrics: [
|
|
7970
|
-
{ name: "sessions" },
|
|
7971
|
-
{ name: "totalUsers" }
|
|
7972
|
-
],
|
|
7973
|
-
limit: PAGE_SIZE,
|
|
7974
|
-
offset
|
|
7975
|
-
};
|
|
7976
|
-
const response = await runReport(accessToken, propertyId, request);
|
|
7977
|
-
const pageRows = (response.rows ?? []).map((row) => ({
|
|
7978
|
-
date: row.dimensionValues[0].value,
|
|
7979
|
-
landingPage: row.dimensionValues[1].value,
|
|
7980
|
-
sessions: parseInt(row.metricValues[0].value, 10) || 0,
|
|
7981
|
-
organicSessions: 0,
|
|
7982
|
-
// populated by organic-only pass below
|
|
7983
|
-
users: parseInt(row.metricValues[1].value, 10) || 0
|
|
7984
|
-
}));
|
|
7985
|
-
rows.push(...pageRows);
|
|
7986
|
-
const totalRows = response.rowCount ?? 0;
|
|
7987
|
-
offset += pageRows.length;
|
|
7988
|
-
if (pageRows.length < PAGE_SIZE || offset >= totalRows) break;
|
|
7989
|
-
}
|
|
7990
|
-
const organicMap = /* @__PURE__ */ new Map();
|
|
7991
|
-
let organicOffset = 0;
|
|
7992
|
-
while (true) {
|
|
7993
|
-
const organicRequest = {
|
|
7994
|
-
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
7995
|
-
dimensions: [{ name: "date" }, { name: "landingPagePlusQueryString" }],
|
|
7996
|
-
metrics: [{ name: "sessions" }],
|
|
7997
|
-
dimensionFilter: {
|
|
7998
|
-
filter: {
|
|
7999
|
-
fieldName: "sessionDefaultChannelGrouping",
|
|
8000
|
-
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
8001
|
-
}
|
|
8002
|
-
},
|
|
8003
|
-
limit: 1e4,
|
|
8004
|
-
offset: organicOffset
|
|
8005
|
-
};
|
|
8006
|
-
const organicResponse = await runReport(accessToken, propertyId, organicRequest);
|
|
8007
|
-
for (const row of organicResponse.rows ?? []) {
|
|
8008
|
-
const key = `${row.dimensionValues[0].value}::${row.dimensionValues[1].value}`;
|
|
8009
|
-
organicMap.set(key, parseInt(row.metricValues[0].value, 10) || 0);
|
|
8010
|
-
}
|
|
8011
|
-
const total = organicResponse.rowCount ?? 0;
|
|
8012
|
-
organicOffset += (organicResponse.rows ?? []).length;
|
|
8013
|
-
if ((organicResponse.rows ?? []).length < 1e4 || organicOffset >= total) break;
|
|
8014
|
-
}
|
|
8015
|
-
for (const row of rows) {
|
|
8016
|
-
const key = `${row.date}::${row.landingPage}`;
|
|
8017
|
-
row.organicSessions = organicMap.get(key) ?? 0;
|
|
8018
|
-
}
|
|
8019
|
-
for (const row of rows) {
|
|
8020
|
-
if (row.date.length === 8 && !row.date.includes("-")) {
|
|
8021
|
-
row.date = `${row.date.slice(0, 4)}-${row.date.slice(4, 6)}-${row.date.slice(6, 8)}`;
|
|
8022
|
-
}
|
|
8023
|
-
}
|
|
8024
|
-
ga4Log("info", "fetch-traffic.done", { propertyId, rowCount: rows.length });
|
|
8025
|
-
return rows;
|
|
8026
|
-
}
|
|
8027
|
-
async function verifyConnection(clientEmail, privateKey, propertyId) {
|
|
8028
|
-
const accessToken = await getAccessToken(clientEmail, privateKey);
|
|
8029
|
-
const endDate = /* @__PURE__ */ new Date();
|
|
8030
|
-
const startDate = /* @__PURE__ */ new Date();
|
|
8031
|
-
startDate.setDate(startDate.getDate() - 1);
|
|
8032
|
-
await runReport(accessToken, propertyId, {
|
|
8033
|
-
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
8034
|
-
dimensions: [{ name: "date" }],
|
|
8035
|
-
metrics: [{ name: "sessions" }],
|
|
8036
|
-
limit: 1
|
|
8037
|
-
});
|
|
8038
|
-
return true;
|
|
8039
|
-
}
|
|
8040
|
-
async function fetchAggregateSummary(accessToken, propertyId, days) {
|
|
8041
|
-
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
8042
|
-
const endDate = /* @__PURE__ */ new Date();
|
|
8043
|
-
const startDate = /* @__PURE__ */ new Date();
|
|
8044
|
-
startDate.setDate(startDate.getDate() - syncDays);
|
|
8045
|
-
ga4Log("info", "fetch-aggregate.start", { propertyId, days: syncDays });
|
|
8046
|
-
const dateRange = { startDate: formatDate(startDate), endDate: formatDate(endDate) };
|
|
8047
|
-
const batchRes = await batchRunReports(accessToken, propertyId, [
|
|
8048
|
-
{
|
|
8049
|
-
dateRanges: [dateRange],
|
|
8050
|
-
dimensions: [],
|
|
8051
|
-
metrics: [{ name: "sessions" }, { name: "totalUsers" }],
|
|
8052
|
-
limit: 1
|
|
8053
|
-
},
|
|
8054
|
-
{
|
|
8055
|
-
dateRanges: [dateRange],
|
|
8056
|
-
dimensions: [],
|
|
8057
|
-
metrics: [{ name: "sessions" }],
|
|
8058
|
-
dimensionFilter: {
|
|
8059
|
-
filter: {
|
|
8060
|
-
fieldName: "sessionDefaultChannelGrouping",
|
|
8061
|
-
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
8062
|
-
}
|
|
8063
|
-
},
|
|
8064
|
-
limit: 1
|
|
8065
|
-
}
|
|
8066
|
-
]);
|
|
8067
|
-
const totalRow = batchRes[0]?.rows?.[0];
|
|
8068
|
-
const organicRow = batchRes[1]?.rows?.[0];
|
|
8069
|
-
const summary = {
|
|
8070
|
-
periodStart: formatDate(startDate),
|
|
8071
|
-
periodEnd: formatDate(endDate),
|
|
8072
|
-
totalSessions: parseInt(totalRow?.metricValues[0]?.value ?? "0", 10) || 0,
|
|
8073
|
-
totalUsers: parseInt(totalRow?.metricValues[1]?.value ?? "0", 10) || 0,
|
|
8074
|
-
totalOrganicSessions: parseInt(organicRow?.metricValues[0]?.value ?? "0", 10) || 0
|
|
8075
|
-
};
|
|
8076
|
-
ga4Log("info", "fetch-aggregate.done", { propertyId, ...summary });
|
|
8077
|
-
return summary;
|
|
8078
|
-
}
|
|
8079
|
-
async function fetchAiReferrals(accessToken, propertyId, days) {
|
|
8080
|
-
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
8081
|
-
const endDate = /* @__PURE__ */ new Date();
|
|
8082
|
-
const startDate = /* @__PURE__ */ new Date();
|
|
8083
|
-
startDate.setDate(startDate.getDate() - syncDays);
|
|
8084
|
-
ga4Log("info", "fetch-ai-referrals.start", { propertyId, days: syncDays });
|
|
8085
|
-
const PAGE_SIZE = 1e3;
|
|
8086
|
-
const rows = [];
|
|
8087
|
-
let offset = 0;
|
|
8088
|
-
while (true) {
|
|
8089
|
-
const request = {
|
|
8090
|
-
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
8091
|
-
dimensions: [
|
|
8092
|
-
{ name: "date" },
|
|
8093
|
-
{ name: "sessionSource" },
|
|
8094
|
-
{ name: "sessionMedium" }
|
|
8095
|
-
],
|
|
8096
|
-
metrics: [
|
|
8097
|
-
{ name: "sessions" },
|
|
8098
|
-
{ name: "totalUsers" }
|
|
8099
|
-
],
|
|
8100
|
-
dimensionFilter: {
|
|
8101
|
-
orGroup: {
|
|
8102
|
-
expressions: AI_REFERRAL_SOURCE_FILTERS.map(({ matchType, value }) => ({
|
|
8103
|
-
filter: {
|
|
8104
|
-
fieldName: "sessionSource",
|
|
8105
|
-
stringFilter: { matchType, value }
|
|
8106
|
-
}
|
|
8107
|
-
}))
|
|
8039
|
+
const results = await opts.onCdpScreenshot(query, targets);
|
|
8040
|
+
return reply.code(200).send({ results });
|
|
8041
|
+
});
|
|
8042
|
+
app.get(
|
|
8043
|
+
"/projects/:name/runs/:runId/browser-diff",
|
|
8044
|
+
async (request, reply) => {
|
|
8045
|
+
const project = resolveProject(app.db, request.params.name);
|
|
8046
|
+
const { runId } = request.params;
|
|
8047
|
+
const run = app.db.select().from(runs).where(and4(eq15(runs.id, runId), eq15(runs.projectId, project.id))).get();
|
|
8048
|
+
if (!run) {
|
|
8049
|
+
const err = notFound("Run", runId);
|
|
8050
|
+
return reply.code(err.statusCode).send(err.toJSON());
|
|
8051
|
+
}
|
|
8052
|
+
const snapshots = app.db.select({
|
|
8053
|
+
id: querySnapshots.id,
|
|
8054
|
+
keywordId: querySnapshots.keywordId,
|
|
8055
|
+
provider: querySnapshots.provider,
|
|
8056
|
+
citationState: querySnapshots.citationState,
|
|
8057
|
+
citedDomains: querySnapshots.citedDomains,
|
|
8058
|
+
screenshotPath: querySnapshots.screenshotPath,
|
|
8059
|
+
rawResponse: querySnapshots.rawResponse
|
|
8060
|
+
}).from(querySnapshots).where(eq15(querySnapshots.runId, runId)).all();
|
|
8061
|
+
const keywordRows = app.db.select({ id: keywords.id, keyword: keywords.keyword }).from(keywords).where(eq15(keywords.projectId, project.id)).all();
|
|
8062
|
+
const keywordMap = new Map(keywordRows.map((k) => [k.id, k.keyword]));
|
|
8063
|
+
const byKeyword = /* @__PURE__ */ new Map();
|
|
8064
|
+
for (const snap of snapshots) {
|
|
8065
|
+
const kwName = keywordMap.get(snap.keywordId) ?? snap.keywordId;
|
|
8066
|
+
if (!byKeyword.has(snap.keywordId)) {
|
|
8067
|
+
byKeyword.set(snap.keywordId, { keyword: kwName, api: null, browser: null });
|
|
8108
8068
|
}
|
|
8109
|
-
|
|
8110
|
-
|
|
8111
|
-
|
|
8112
|
-
|
|
8113
|
-
|
|
8114
|
-
|
|
8115
|
-
|
|
8116
|
-
|
|
8117
|
-
|
|
8118
|
-
|
|
8119
|
-
|
|
8120
|
-
|
|
8121
|
-
|
|
8122
|
-
|
|
8123
|
-
|
|
8124
|
-
|
|
8125
|
-
|
|
8126
|
-
|
|
8127
|
-
|
|
8128
|
-
|
|
8069
|
+
const entry = byKeyword.get(snap.keywordId);
|
|
8070
|
+
if (snap.provider === "cdp:chatgpt") {
|
|
8071
|
+
entry.browser = snap;
|
|
8072
|
+
} else if (snap.provider === "openai") {
|
|
8073
|
+
entry.api = snap;
|
|
8074
|
+
}
|
|
8075
|
+
}
|
|
8076
|
+
let agreed = 0;
|
|
8077
|
+
let apiOnlyCited = 0;
|
|
8078
|
+
let browserOnlyCited = 0;
|
|
8079
|
+
let disagreed = 0;
|
|
8080
|
+
let total = 0;
|
|
8081
|
+
const keywordResults = [...byKeyword.values()].map(({ keyword, api, browser }) => {
|
|
8082
|
+
total++;
|
|
8083
|
+
const apiCited = api?.citationState === "cited";
|
|
8084
|
+
const browserCited = browser?.citationState === "cited";
|
|
8085
|
+
let agreement;
|
|
8086
|
+
if (!api && !browser) {
|
|
8087
|
+
agreement = "no-data";
|
|
8088
|
+
} else if (!api) {
|
|
8089
|
+
agreement = "no-api";
|
|
8090
|
+
} else if (!browser) {
|
|
8091
|
+
agreement = "no-browser";
|
|
8092
|
+
} else if (apiCited && browserCited) {
|
|
8093
|
+
agreement = "agree-cited";
|
|
8094
|
+
agreed++;
|
|
8095
|
+
} else if (!apiCited && !browserCited) {
|
|
8096
|
+
agreement = "agree-not-cited";
|
|
8097
|
+
agreed++;
|
|
8098
|
+
} else if (apiCited && !browserCited) {
|
|
8099
|
+
agreement = "api-only-cited";
|
|
8100
|
+
apiOnlyCited++;
|
|
8101
|
+
disagreed++;
|
|
8102
|
+
} else {
|
|
8103
|
+
agreement = "browser-only-cited";
|
|
8104
|
+
browserOnlyCited++;
|
|
8105
|
+
disagreed++;
|
|
8106
|
+
}
|
|
8107
|
+
const parseGroundingSources2 = (snap) => {
|
|
8108
|
+
if (!snap?.rawResponse) return [];
|
|
8109
|
+
try {
|
|
8110
|
+
const raw = JSON.parse(snap.rawResponse);
|
|
8111
|
+
return raw.groundingSources ?? [];
|
|
8112
|
+
} catch {
|
|
8113
|
+
return [];
|
|
8114
|
+
}
|
|
8115
|
+
};
|
|
8116
|
+
return {
|
|
8117
|
+
keyword,
|
|
8118
|
+
api: api ? {
|
|
8119
|
+
provider: api.provider,
|
|
8120
|
+
citationState: api.citationState,
|
|
8121
|
+
citedDomains: JSON.parse(api.citedDomains || "[]"),
|
|
8122
|
+
groundingSources: parseGroundingSources2(api)
|
|
8123
|
+
} : null,
|
|
8124
|
+
browser: browser ? {
|
|
8125
|
+
provider: browser.provider,
|
|
8126
|
+
citationState: browser.citationState,
|
|
8127
|
+
citedDomains: JSON.parse(browser.citedDomains || "[]"),
|
|
8128
|
+
groundingSources: parseGroundingSources2(browser),
|
|
8129
|
+
screenshotUrl: browser.screenshotPath ? `/api/v1/screenshots/${browser.id}` : void 0
|
|
8130
|
+
} : null,
|
|
8131
|
+
agreement
|
|
8132
|
+
};
|
|
8133
|
+
});
|
|
8134
|
+
return reply.send({
|
|
8135
|
+
summary: { total, agreed, apiOnly: apiOnlyCited, browserOnly: browserOnlyCited, disagreed },
|
|
8136
|
+
keywords: keywordResults
|
|
8137
|
+
});
|
|
8129
8138
|
}
|
|
8130
|
-
|
|
8131
|
-
ga4Log("info", "fetch-ai-referrals.done", { propertyId, rowCount: rows.length });
|
|
8132
|
-
return rows;
|
|
8139
|
+
);
|
|
8133
8140
|
}
|
|
8134
8141
|
|
|
8135
8142
|
// ../api-routes/src/ga.ts
|
|
8143
|
+
import crypto16 from "crypto";
|
|
8144
|
+
import { eq as eq16, desc as desc6, and as and5, sql as sql4 } from "drizzle-orm";
|
|
8136
8145
|
function gaLog(level, action, ctx) {
|
|
8137
8146
|
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
8138
8147
|
const stream = level === "error" ? process.stderr : process.stdout;
|
|
8139
8148
|
stream.write(JSON.stringify(entry) + "\n");
|
|
8140
8149
|
}
|
|
8141
|
-
async function
|
|
8142
|
-
|
|
8143
|
-
|
|
8144
|
-
|
|
8145
|
-
|
|
8146
|
-
|
|
8150
|
+
async function refreshOAuthTokenIfNeeded(googleStore, authConfig, canonicalDomain, oauthConn) {
|
|
8151
|
+
const expiresAt = oauthConn.tokenExpiresAt ? new Date(oauthConn.tokenExpiresAt).getTime() : 0;
|
|
8152
|
+
const fiveMinutes = 5 * 60 * 1e3;
|
|
8153
|
+
if (Date.now() > expiresAt - fiveMinutes) {
|
|
8154
|
+
if (!authConfig.clientId || !authConfig.clientSecret) {
|
|
8155
|
+
throw validationError("Google OAuth client credentials are not configured \u2014 cannot refresh GA4 token.");
|
|
8156
|
+
}
|
|
8157
|
+
const tokens = await refreshAccessToken(authConfig.clientId, authConfig.clientSecret, oauthConn.refreshToken);
|
|
8158
|
+
const newExpiresAt = new Date(Date.now() + tokens.expires_in * 1e3).toISOString();
|
|
8159
|
+
googleStore.updateConnection(canonicalDomain, "ga4", {
|
|
8160
|
+
accessToken: tokens.access_token,
|
|
8161
|
+
tokenExpiresAt: newExpiresAt,
|
|
8162
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8163
|
+
});
|
|
8164
|
+
return tokens.access_token;
|
|
8147
8165
|
}
|
|
8148
|
-
|
|
8149
|
-
|
|
8150
|
-
|
|
8166
|
+
return oauthConn.accessToken;
|
|
8167
|
+
}
|
|
8168
|
+
async function resolveGa4AccessToken(opts, projectName, canonicalDomain) {
|
|
8169
|
+
const saConn = opts.ga4CredentialStore?.getConnection(projectName);
|
|
8170
|
+
if (saConn?.clientEmail && saConn?.privateKey && saConn?.propertyId) {
|
|
8171
|
+
const token = await getAccessToken(saConn.clientEmail, saConn.privateKey);
|
|
8172
|
+
return { accessToken: token, propertyId: saConn.propertyId };
|
|
8173
|
+
}
|
|
8174
|
+
const googleStore = opts.googleConnectionStore;
|
|
8175
|
+
const authConfig = opts.getGoogleAuthConfig?.();
|
|
8176
|
+
if (!googleStore || !authConfig) {
|
|
8177
|
+
throw validationError(
|
|
8178
|
+
'No GA4 credentials found. Run "canonry ga connect <project> --key-file <path>" or "canonry google connect <project> --type ga4" to authenticate.'
|
|
8179
|
+
);
|
|
8180
|
+
}
|
|
8181
|
+
const oauthConn = googleStore.getConnection(canonicalDomain, "ga4");
|
|
8182
|
+
if (!oauthConn?.accessToken || !oauthConn?.refreshToken) {
|
|
8183
|
+
throw validationError(
|
|
8184
|
+
'No GA4 credentials found. Run "canonry ga connect <project> --key-file <path>" or "canonry google connect <project> --type ga4" to authenticate.'
|
|
8185
|
+
);
|
|
8186
|
+
}
|
|
8187
|
+
if (!oauthConn.propertyId) {
|
|
8188
|
+
throw validationError(
|
|
8189
|
+
'GA4 property ID not set. Run "canonry ga set-property <project> <propertyId>" to configure it.'
|
|
8190
|
+
);
|
|
8191
|
+
}
|
|
8192
|
+
const accessToken = await refreshOAuthTokenIfNeeded(googleStore, authConfig, canonicalDomain, {
|
|
8193
|
+
accessToken: oauthConn.accessToken,
|
|
8194
|
+
refreshToken: oauthConn.refreshToken,
|
|
8195
|
+
tokenExpiresAt: oauthConn.tokenExpiresAt
|
|
8196
|
+
});
|
|
8197
|
+
return { accessToken, propertyId: oauthConn.propertyId };
|
|
8198
|
+
}
|
|
8199
|
+
function requireGa4Connection(opts, projectName, canonicalDomain) {
|
|
8200
|
+
const saConn = opts.ga4CredentialStore?.getConnection(projectName);
|
|
8201
|
+
const oauthConn = opts.googleConnectionStore?.getConnection(canonicalDomain, "ga4");
|
|
8202
|
+
if (!saConn && !(oauthConn?.accessToken && oauthConn?.propertyId)) {
|
|
8203
|
+
throw validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
8204
|
+
}
|
|
8205
|
+
}
|
|
8206
|
+
async function ga4Routes(app, opts) {
|
|
8207
|
+
app.post("/projects/:name/ga/connect", async (request, _reply) => {
|
|
8151
8208
|
const project = resolveProject(app.db, request.params.name);
|
|
8152
8209
|
const { propertyId, keyJson } = request.body ?? {};
|
|
8153
8210
|
if (!propertyId || typeof propertyId !== "string") {
|
|
8154
|
-
|
|
8155
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8211
|
+
throw validationError("propertyId is required");
|
|
8156
8212
|
}
|
|
8157
|
-
let clientEmail;
|
|
8158
|
-
let privateKey;
|
|
8159
8213
|
if (keyJson && typeof keyJson === "string") {
|
|
8214
|
+
if (!opts.ga4CredentialStore) {
|
|
8215
|
+
throw validationError("GA4 credential storage is not configured for this deployment");
|
|
8216
|
+
}
|
|
8217
|
+
let parsed;
|
|
8160
8218
|
try {
|
|
8161
|
-
|
|
8162
|
-
if (!parsed.client_email || !parsed.private_key) {
|
|
8163
|
-
const err = validationError("Service account JSON must contain client_email and private_key");
|
|
8164
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8165
|
-
}
|
|
8166
|
-
clientEmail = parsed.client_email;
|
|
8167
|
-
privateKey = parsed.private_key;
|
|
8219
|
+
parsed = JSON.parse(keyJson);
|
|
8168
8220
|
} catch {
|
|
8169
|
-
|
|
8170
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8221
|
+
throw validationError("Invalid JSON in keyJson");
|
|
8171
8222
|
}
|
|
8172
|
-
|
|
8173
|
-
|
|
8174
|
-
|
|
8223
|
+
if (!parsed.client_email || !parsed.private_key) {
|
|
8224
|
+
throw validationError("Service account JSON must contain client_email and private_key");
|
|
8225
|
+
}
|
|
8226
|
+
const clientEmail = parsed.client_email;
|
|
8227
|
+
const privateKey = parsed.private_key;
|
|
8228
|
+
try {
|
|
8229
|
+
await verifyConnection(clientEmail, privateKey, propertyId);
|
|
8230
|
+
gaLog("info", "connect.verified.service-account", { projectId: project.id, propertyId });
|
|
8231
|
+
} catch (e) {
|
|
8232
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
8233
|
+
gaLog("error", "connect.verify-failed", { projectId: project.id, propertyId, error: msg });
|
|
8234
|
+
throw validationError(`Failed to verify GA4 credentials: ${msg}`);
|
|
8235
|
+
}
|
|
8236
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8237
|
+
const existing = opts.ga4CredentialStore.getConnection(project.name);
|
|
8238
|
+
opts.ga4CredentialStore.upsertConnection({
|
|
8239
|
+
projectName: project.name,
|
|
8240
|
+
propertyId,
|
|
8241
|
+
clientEmail,
|
|
8242
|
+
privateKey,
|
|
8243
|
+
createdAt: existing?.createdAt ?? now,
|
|
8244
|
+
updatedAt: now
|
|
8245
|
+
});
|
|
8246
|
+
writeAuditLog(app.db, {
|
|
8247
|
+
projectId: project.id,
|
|
8248
|
+
actor: "api",
|
|
8249
|
+
action: "ga4.connected",
|
|
8250
|
+
entityType: "ga_connection",
|
|
8251
|
+
entityId: propertyId
|
|
8252
|
+
});
|
|
8253
|
+
return { connected: true, propertyId, authMethod: "service-account", clientEmail };
|
|
8254
|
+
}
|
|
8255
|
+
const googleStore = opts.googleConnectionStore;
|
|
8256
|
+
const authConfig = opts.getGoogleAuthConfig?.();
|
|
8257
|
+
if (!googleStore || !authConfig) {
|
|
8258
|
+
throw validationError(
|
|
8259
|
+
'No service account key provided and OAuth storage is not configured. Pass --key-file or run "canonry google connect <project> --type ga4" first.'
|
|
8260
|
+
);
|
|
8261
|
+
}
|
|
8262
|
+
const oauthConn = googleStore.getConnection(project.canonicalDomain, "ga4");
|
|
8263
|
+
if (!oauthConn?.accessToken || !oauthConn?.refreshToken) {
|
|
8264
|
+
throw validationError(
|
|
8265
|
+
'No GA4 OAuth token found. Run "canonry google connect <project> --type ga4" first, or pass --key-file to use a service account.'
|
|
8266
|
+
);
|
|
8175
8267
|
}
|
|
8268
|
+
const accessToken = await refreshOAuthTokenIfNeeded(googleStore, authConfig, project.canonicalDomain, {
|
|
8269
|
+
accessToken: oauthConn.accessToken,
|
|
8270
|
+
refreshToken: oauthConn.refreshToken,
|
|
8271
|
+
tokenExpiresAt: oauthConn.tokenExpiresAt
|
|
8272
|
+
});
|
|
8176
8273
|
try {
|
|
8177
|
-
await
|
|
8178
|
-
gaLog("info", "connect.verified", { projectId: project.id, propertyId });
|
|
8274
|
+
await verifyConnectionWithToken(accessToken, propertyId);
|
|
8275
|
+
gaLog("info", "connect.verified.oauth", { projectId: project.id, propertyId });
|
|
8179
8276
|
} catch (e) {
|
|
8180
8277
|
const msg = e instanceof Error ? e.message : String(e);
|
|
8181
|
-
gaLog("error", "connect.verify-failed", { projectId: project.id, propertyId, error: msg });
|
|
8182
|
-
|
|
8183
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8278
|
+
gaLog("error", "connect.verify-failed.oauth", { projectId: project.id, propertyId, error: msg });
|
|
8279
|
+
throw validationError(`Failed to verify GA4 access: ${msg}`);
|
|
8184
8280
|
}
|
|
8185
|
-
|
|
8186
|
-
const existing = store.getConnection(project.name);
|
|
8187
|
-
store.upsertConnection({
|
|
8188
|
-
projectName: project.name,
|
|
8281
|
+
googleStore.updateConnection(project.canonicalDomain, "ga4", {
|
|
8189
8282
|
propertyId,
|
|
8190
|
-
|
|
8191
|
-
privateKey,
|
|
8192
|
-
createdAt: existing?.createdAt ?? now,
|
|
8193
|
-
updatedAt: now
|
|
8283
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
8194
8284
|
});
|
|
8195
8285
|
writeAuditLog(app.db, {
|
|
8196
8286
|
projectId: project.id,
|
|
@@ -8199,80 +8289,62 @@ async function ga4Routes(app, opts) {
|
|
|
8199
8289
|
entityType: "ga_connection",
|
|
8200
8290
|
entityId: propertyId
|
|
8201
8291
|
});
|
|
8202
|
-
return {
|
|
8203
|
-
connected: true,
|
|
8204
|
-
propertyId,
|
|
8205
|
-
clientEmail
|
|
8206
|
-
};
|
|
8292
|
+
return { connected: true, propertyId, authMethod: "oauth" };
|
|
8207
8293
|
});
|
|
8208
8294
|
app.delete("/projects/:name/ga/disconnect", async (request, reply) => {
|
|
8209
|
-
const store = requireCredentialStore(reply);
|
|
8210
|
-
if (!store) return;
|
|
8211
8295
|
const project = resolveProject(app.db, request.params.name);
|
|
8212
|
-
const
|
|
8213
|
-
|
|
8214
|
-
|
|
8215
|
-
|
|
8296
|
+
const saConn = opts.ga4CredentialStore?.getConnection(project.name);
|
|
8297
|
+
const oauthConn = opts.googleConnectionStore?.getConnection(project.canonicalDomain, "ga4");
|
|
8298
|
+
if (!saConn && !oauthConn) {
|
|
8299
|
+
throw notFound("GA4 connection", project.name);
|
|
8216
8300
|
}
|
|
8217
8301
|
app.db.delete(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).run();
|
|
8218
8302
|
app.db.delete(gaTrafficSummaries).where(eq16(gaTrafficSummaries.projectId, project.id)).run();
|
|
8219
8303
|
app.db.delete(gaAiReferrals).where(eq16(gaAiReferrals.projectId, project.id)).run();
|
|
8220
|
-
|
|
8304
|
+
const propertyId = saConn?.propertyId ?? oauthConn?.propertyId ?? null;
|
|
8305
|
+
opts.ga4CredentialStore?.deleteConnection(project.name);
|
|
8306
|
+
opts.googleConnectionStore?.deleteConnection(project.canonicalDomain, "ga4");
|
|
8221
8307
|
writeAuditLog(app.db, {
|
|
8222
8308
|
projectId: project.id,
|
|
8223
8309
|
actor: "api",
|
|
8224
8310
|
action: "ga4.disconnected",
|
|
8225
8311
|
entityType: "ga_connection",
|
|
8226
|
-
entityId:
|
|
8312
|
+
entityId: propertyId
|
|
8227
8313
|
});
|
|
8228
8314
|
return reply.status(204).send();
|
|
8229
8315
|
});
|
|
8230
|
-
app.get("/projects/:name/ga/status", async (request,
|
|
8231
|
-
const store = requireCredentialStore(reply);
|
|
8232
|
-
if (!store) return;
|
|
8316
|
+
app.get("/projects/:name/ga/status", async (request, _reply) => {
|
|
8233
8317
|
const project = resolveProject(app.db, request.params.name);
|
|
8234
|
-
const
|
|
8235
|
-
|
|
8236
|
-
|
|
8318
|
+
const saConn = opts.ga4CredentialStore?.getConnection(project.name);
|
|
8319
|
+
const oauthConn = opts.googleConnectionStore?.getConnection(project.canonicalDomain, "ga4");
|
|
8320
|
+
const connected = !!(saConn || oauthConn?.accessToken && oauthConn?.propertyId);
|
|
8321
|
+
if (!connected) {
|
|
8322
|
+
return { connected: false, propertyId: null, clientEmail: null, authMethod: null, lastSyncedAt: null };
|
|
8237
8323
|
}
|
|
8238
8324
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq16(gaTrafficSummaries.projectId, project.id)).orderBy(desc6(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
8239
8325
|
return {
|
|
8240
8326
|
connected: true,
|
|
8241
|
-
propertyId:
|
|
8242
|
-
clientEmail:
|
|
8327
|
+
propertyId: saConn?.propertyId ?? oauthConn?.propertyId ?? null,
|
|
8328
|
+
clientEmail: saConn?.clientEmail ?? null,
|
|
8329
|
+
authMethod: saConn ? "service-account" : "oauth",
|
|
8243
8330
|
lastSyncedAt: latestSync?.syncedAt ?? null,
|
|
8244
|
-
createdAt:
|
|
8245
|
-
updatedAt:
|
|
8331
|
+
createdAt: saConn?.createdAt ?? oauthConn?.createdAt ?? null,
|
|
8332
|
+
updatedAt: saConn?.updatedAt ?? oauthConn?.updatedAt ?? null
|
|
8246
8333
|
};
|
|
8247
8334
|
});
|
|
8248
|
-
app.post("/projects/:name/ga/sync", async (request,
|
|
8249
|
-
const store = requireCredentialStore(reply);
|
|
8250
|
-
if (!store) return;
|
|
8335
|
+
app.post("/projects/:name/ga/sync", async (request, _reply) => {
|
|
8251
8336
|
const project = resolveProject(app.db, request.params.name);
|
|
8252
|
-
const conn = store.getConnection(project.name);
|
|
8253
|
-
if (!conn) {
|
|
8254
|
-
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
8255
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8256
|
-
}
|
|
8257
8337
|
const days = request.body?.days ?? 30;
|
|
8258
|
-
|
|
8259
|
-
try {
|
|
8260
|
-
accessToken = await getAccessToken(conn.clientEmail, conn.privateKey);
|
|
8261
|
-
} catch (e) {
|
|
8262
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
8263
|
-
gaLog("error", "sync.auth-failed", { projectId: project.id, error: msg });
|
|
8264
|
-
const err = validationError(`GA4 authentication failed: ${msg}`);
|
|
8265
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8266
|
-
}
|
|
8338
|
+
const { accessToken, propertyId } = await resolveGa4AccessToken(opts, project.name, project.canonicalDomain);
|
|
8267
8339
|
let rows;
|
|
8268
8340
|
let summary;
|
|
8269
8341
|
let aiReferrals;
|
|
8270
8342
|
try {
|
|
8271
8343
|
;
|
|
8272
8344
|
[rows, summary, aiReferrals] = await Promise.all([
|
|
8273
|
-
fetchTrafficByLandingPage(accessToken,
|
|
8274
|
-
fetchAggregateSummary(accessToken,
|
|
8275
|
-
fetchAiReferrals(accessToken,
|
|
8345
|
+
fetchTrafficByLandingPage(accessToken, propertyId, days),
|
|
8346
|
+
fetchAggregateSummary(accessToken, propertyId, days),
|
|
8347
|
+
fetchAiReferrals(accessToken, propertyId, days)
|
|
8276
8348
|
]);
|
|
8277
8349
|
} catch (e) {
|
|
8278
8350
|
const msg = e instanceof Error ? e.message : String(e);
|
|
@@ -8350,15 +8422,9 @@ async function ga4Routes(app, opts) {
|
|
|
8350
8422
|
syncedAt: now
|
|
8351
8423
|
};
|
|
8352
8424
|
});
|
|
8353
|
-
app.get("/projects/:name/ga/traffic", async (request,
|
|
8354
|
-
const store = requireCredentialStore(reply);
|
|
8355
|
-
if (!store) return;
|
|
8425
|
+
app.get("/projects/:name/ga/traffic", async (request, _reply) => {
|
|
8356
8426
|
const project = resolveProject(app.db, request.params.name);
|
|
8357
|
-
|
|
8358
|
-
if (!conn) {
|
|
8359
|
-
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
8360
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8361
|
-
}
|
|
8427
|
+
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
8362
8428
|
const limit = Math.max(1, Math.min(parseInt(request.query.limit ?? "50", 10) || 50, 500));
|
|
8363
8429
|
const summary = app.db.select({
|
|
8364
8430
|
totalSessions: gaTrafficSummaries.totalSessions,
|
|
@@ -8397,15 +8463,9 @@ async function ga4Routes(app, opts) {
|
|
|
8397
8463
|
lastSyncedAt: latestSync?.syncedAt ?? null
|
|
8398
8464
|
};
|
|
8399
8465
|
});
|
|
8400
|
-
app.get("/projects/:name/ga/coverage", async (request,
|
|
8401
|
-
const store = requireCredentialStore(reply);
|
|
8402
|
-
if (!store) return;
|
|
8466
|
+
app.get("/projects/:name/ga/coverage", async (request, _reply) => {
|
|
8403
8467
|
const project = resolveProject(app.db, request.params.name);
|
|
8404
|
-
|
|
8405
|
-
if (!conn) {
|
|
8406
|
-
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
8407
|
-
return reply.status(err.statusCode).send(err.toJSON());
|
|
8408
|
-
}
|
|
8468
|
+
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
8409
8469
|
const trafficPages = app.db.select({
|
|
8410
8470
|
landingPage: gaTrafficSnapshots.landingPage,
|
|
8411
8471
|
sessions: sql4`SUM(${gaTrafficSnapshots.sessions})`,
|
|
@@ -10162,7 +10222,9 @@ async function apiRoutes(app, opts) {
|
|
|
10162
10222
|
onCdpConfigure: opts.onCdpConfigure
|
|
10163
10223
|
});
|
|
10164
10224
|
await api.register(ga4Routes, {
|
|
10165
|
-
ga4CredentialStore: opts.ga4CredentialStore
|
|
10225
|
+
ga4CredentialStore: opts.ga4CredentialStore,
|
|
10226
|
+
googleConnectionStore: opts.googleConnectionStore,
|
|
10227
|
+
getGoogleAuthConfig: opts.getGoogleAuthConfig
|
|
10166
10228
|
});
|
|
10167
10229
|
}, { prefix: opts.routePrefix ?? "/api/v1" });
|
|
10168
10230
|
}
|
|
@@ -12706,9 +12768,6 @@ function extractRecommendedCompetitors(answerText, ownDomains, citedDomains, com
|
|
|
12706
12768
|
function cleanCandidateName(candidate) {
|
|
12707
12769
|
return candidate.replace(/^[\s"'`]+|[\s"'`.,:;!?]+$/g, "").replace(/\s+/g, " ").trim();
|
|
12708
12770
|
}
|
|
12709
|
-
function brandKeyFromText(value) {
|
|
12710
|
-
return value.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
12711
|
-
}
|
|
12712
12771
|
function collectBrandKeysFromDomain(domain) {
|
|
12713
12772
|
const hostname = normalizeProjectDomain(domain).split("/")[0] ?? "";
|
|
12714
12773
|
const labels = hostname.split(".").filter(Boolean);
|