@ainyc/canonry 1.22.0 → 1.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -10
- package/assets/assets/index-BUzyFRxr.js +246 -0
- package/assets/assets/index-BidvmvWJ.css +1 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-SN5AMQJE.js → chunk-QMUO2JYU.js} +847 -72
- package/dist/cli.js +331 -67
- package/dist/index.d.ts +12 -0
- package/dist/index.js +1 -1
- package/package.json +6 -6
- package/assets/assets/index-7DjD4Oje.css +0 -1
- package/assets/assets/index-sl--69xx.js +0 -246
|
@@ -170,7 +170,7 @@ function trackEvent(event, properties) {
|
|
|
170
170
|
|
|
171
171
|
// src/server.ts
|
|
172
172
|
import { createRequire as createRequire2 } from "module";
|
|
173
|
-
import
|
|
173
|
+
import crypto21 from "crypto";
|
|
174
174
|
import fs5 from "fs";
|
|
175
175
|
import path6 from "path";
|
|
176
176
|
import { fileURLToPath } from "url";
|
|
@@ -711,6 +711,37 @@ function categoryLabel(category) {
|
|
|
711
711
|
return CATEGORY_LABELS[category];
|
|
712
712
|
}
|
|
713
713
|
|
|
714
|
+
// ../contracts/src/ga.ts
|
|
715
|
+
import { z as z9 } from "zod";
|
|
716
|
+
var ga4ConnectionDtoSchema = z9.object({
|
|
717
|
+
id: z9.string(),
|
|
718
|
+
projectId: z9.string(),
|
|
719
|
+
propertyId: z9.string(),
|
|
720
|
+
clientEmail: z9.string(),
|
|
721
|
+
connected: z9.boolean(),
|
|
722
|
+
createdAt: z9.string(),
|
|
723
|
+
updatedAt: z9.string()
|
|
724
|
+
});
|
|
725
|
+
var ga4TrafficSnapshotDtoSchema = z9.object({
|
|
726
|
+
date: z9.string(),
|
|
727
|
+
landingPage: z9.string(),
|
|
728
|
+
sessions: z9.number(),
|
|
729
|
+
organicSessions: z9.number(),
|
|
730
|
+
users: z9.number()
|
|
731
|
+
});
|
|
732
|
+
var ga4TrafficSummaryDtoSchema = z9.object({
|
|
733
|
+
totalSessions: z9.number(),
|
|
734
|
+
totalOrganicSessions: z9.number(),
|
|
735
|
+
totalUsers: z9.number(),
|
|
736
|
+
topPages: z9.array(z9.object({
|
|
737
|
+
landingPage: z9.string(),
|
|
738
|
+
sessions: z9.number(),
|
|
739
|
+
organicSessions: z9.number(),
|
|
740
|
+
users: z9.number()
|
|
741
|
+
})),
|
|
742
|
+
lastSyncedAt: z9.string().nullable()
|
|
743
|
+
});
|
|
744
|
+
|
|
714
745
|
// ../api-routes/src/auth.ts
|
|
715
746
|
import crypto2 from "crypto";
|
|
716
747
|
import { eq } from "drizzle-orm";
|
|
@@ -730,6 +761,9 @@ __export(schema_exports, {
|
|
|
730
761
|
bingKeywordStats: () => bingKeywordStats,
|
|
731
762
|
bingUrlInspections: () => bingUrlInspections,
|
|
732
763
|
competitors: () => competitors,
|
|
764
|
+
gaConnections: () => gaConnections,
|
|
765
|
+
gaTrafficSnapshots: () => gaTrafficSnapshots,
|
|
766
|
+
gaTrafficSummaries: () => gaTrafficSummaries,
|
|
733
767
|
googleConnections: () => googleConnections,
|
|
734
768
|
gscCoverageSnapshots: () => gscCoverageSnapshots,
|
|
735
769
|
gscSearchData: () => gscSearchData,
|
|
@@ -970,6 +1004,42 @@ var bingKeywordStats = sqliteTable("bing_keyword_stats", {
|
|
|
970
1004
|
index("idx_bing_keyword_project").on(table.projectId),
|
|
971
1005
|
index("idx_bing_keyword_query").on(table.query)
|
|
972
1006
|
]);
|
|
1007
|
+
var gaConnections = sqliteTable("ga_connections", {
|
|
1008
|
+
id: text("id").primaryKey(),
|
|
1009
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1010
|
+
propertyId: text("property_id").notNull(),
|
|
1011
|
+
clientEmail: text("client_email").notNull(),
|
|
1012
|
+
privateKey: text("private_key").notNull(),
|
|
1013
|
+
createdAt: text("created_at").notNull(),
|
|
1014
|
+
updatedAt: text("updated_at").notNull()
|
|
1015
|
+
}, (table) => [
|
|
1016
|
+
uniqueIndex("idx_ga_conn_project").on(table.projectId)
|
|
1017
|
+
]);
|
|
1018
|
+
var gaTrafficSnapshots = sqliteTable("ga_traffic_snapshots", {
|
|
1019
|
+
id: text("id").primaryKey(),
|
|
1020
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1021
|
+
date: text("date").notNull(),
|
|
1022
|
+
landingPage: text("landing_page").notNull(),
|
|
1023
|
+
sessions: integer("sessions").notNull().default(0),
|
|
1024
|
+
organicSessions: integer("organic_sessions").notNull().default(0),
|
|
1025
|
+
users: integer("users").notNull().default(0),
|
|
1026
|
+
syncedAt: text("synced_at").notNull()
|
|
1027
|
+
}, (table) => [
|
|
1028
|
+
index("idx_ga_traffic_project_date").on(table.projectId, table.date),
|
|
1029
|
+
index("idx_ga_traffic_page").on(table.landingPage)
|
|
1030
|
+
]);
|
|
1031
|
+
var gaTrafficSummaries = sqliteTable("ga_traffic_summaries", {
|
|
1032
|
+
id: text("id").primaryKey(),
|
|
1033
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1034
|
+
periodStart: text("period_start").notNull(),
|
|
1035
|
+
periodEnd: text("period_end").notNull(),
|
|
1036
|
+
totalSessions: integer("total_sessions").notNull().default(0),
|
|
1037
|
+
totalOrganicSessions: integer("total_organic_sessions").notNull().default(0),
|
|
1038
|
+
totalUsers: integer("total_users").notNull().default(0),
|
|
1039
|
+
syncedAt: text("synced_at").notNull()
|
|
1040
|
+
}, (table) => [
|
|
1041
|
+
index("idx_ga_summary_project").on(table.projectId)
|
|
1042
|
+
]);
|
|
973
1043
|
var usageCounters = sqliteTable("usage_counters", {
|
|
974
1044
|
id: text("id").primaryKey(),
|
|
975
1045
|
scope: text("scope").notNull(),
|
|
@@ -1247,7 +1317,43 @@ var MIGRATIONS = [
|
|
|
1247
1317
|
created_at TEXT NOT NULL
|
|
1248
1318
|
)`,
|
|
1249
1319
|
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_project ON bing_keyword_stats(project_id)`,
|
|
1250
|
-
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_query ON bing_keyword_stats(query)
|
|
1320
|
+
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_query ON bing_keyword_stats(query)`,
|
|
1321
|
+
// v13: Google Analytics 4 — ga_connections table (service account auth)
|
|
1322
|
+
`CREATE TABLE IF NOT EXISTS ga_connections (
|
|
1323
|
+
id TEXT PRIMARY KEY,
|
|
1324
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1325
|
+
property_id TEXT NOT NULL,
|
|
1326
|
+
client_email TEXT NOT NULL,
|
|
1327
|
+
private_key TEXT NOT NULL,
|
|
1328
|
+
created_at TEXT NOT NULL,
|
|
1329
|
+
updated_at TEXT NOT NULL
|
|
1330
|
+
)`,
|
|
1331
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_conn_project ON ga_connections(project_id)`,
|
|
1332
|
+
// v13: Google Analytics 4 — ga_traffic_snapshots table
|
|
1333
|
+
`CREATE TABLE IF NOT EXISTS ga_traffic_snapshots (
|
|
1334
|
+
id TEXT PRIMARY KEY,
|
|
1335
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1336
|
+
date TEXT NOT NULL,
|
|
1337
|
+
landing_page TEXT NOT NULL,
|
|
1338
|
+
sessions INTEGER NOT NULL DEFAULT 0,
|
|
1339
|
+
organic_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1340
|
+
users INTEGER NOT NULL DEFAULT 0,
|
|
1341
|
+
synced_at TEXT NOT NULL
|
|
1342
|
+
)`,
|
|
1343
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_traffic_project_date ON ga_traffic_snapshots(project_id, date)`,
|
|
1344
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_traffic_page ON ga_traffic_snapshots(landing_page)`,
|
|
1345
|
+
// v14: GA4 aggregate summaries — stores true unique user count per sync period
|
|
1346
|
+
`CREATE TABLE IF NOT EXISTS ga_traffic_summaries (
|
|
1347
|
+
id TEXT PRIMARY KEY,
|
|
1348
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1349
|
+
period_start TEXT NOT NULL,
|
|
1350
|
+
period_end TEXT NOT NULL,
|
|
1351
|
+
total_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1352
|
+
total_organic_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1353
|
+
total_users INTEGER NOT NULL DEFAULT 0,
|
|
1354
|
+
synced_at TEXT NOT NULL
|
|
1355
|
+
)`,
|
|
1356
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_summary_project ON ga_traffic_summaries(project_id)`
|
|
1251
1357
|
];
|
|
1252
1358
|
function migrate(db) {
|
|
1253
1359
|
const statements = MIGRATION_SQL.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
@@ -4639,6 +4745,105 @@ var routeCatalog = [
|
|
|
4639
4745
|
400: { description: "Bing is not configured for this project." },
|
|
4640
4746
|
404: { description: "Project not found." }
|
|
4641
4747
|
}
|
|
4748
|
+
},
|
|
4749
|
+
// GA4 routes
|
|
4750
|
+
{
|
|
4751
|
+
method: "post",
|
|
4752
|
+
path: "/api/v1/projects/{name}/ga/connect",
|
|
4753
|
+
summary: "Connect Google Analytics 4 via service account",
|
|
4754
|
+
tags: ["ga4"],
|
|
4755
|
+
parameters: [nameParameter],
|
|
4756
|
+
requestBody: {
|
|
4757
|
+
required: true,
|
|
4758
|
+
content: {
|
|
4759
|
+
"application/json": {
|
|
4760
|
+
schema: {
|
|
4761
|
+
type: "object",
|
|
4762
|
+
required: ["propertyId", "keyJson"],
|
|
4763
|
+
properties: {
|
|
4764
|
+
propertyId: stringSchema,
|
|
4765
|
+
keyJson: stringSchema
|
|
4766
|
+
}
|
|
4767
|
+
}
|
|
4768
|
+
}
|
|
4769
|
+
}
|
|
4770
|
+
},
|
|
4771
|
+
responses: {
|
|
4772
|
+
200: { description: "GA4 connection established." },
|
|
4773
|
+
400: { description: "Invalid GA4 connection request." },
|
|
4774
|
+
404: { description: "Project not found." }
|
|
4775
|
+
}
|
|
4776
|
+
},
|
|
4777
|
+
{
|
|
4778
|
+
method: "delete",
|
|
4779
|
+
path: "/api/v1/projects/{name}/ga/disconnect",
|
|
4780
|
+
summary: "Disconnect Google Analytics 4",
|
|
4781
|
+
tags: ["ga4"],
|
|
4782
|
+
parameters: [nameParameter],
|
|
4783
|
+
responses: {
|
|
4784
|
+
204: { description: "GA4 connection deleted." },
|
|
4785
|
+
404: { description: "Project or connection not found." }
|
|
4786
|
+
}
|
|
4787
|
+
},
|
|
4788
|
+
{
|
|
4789
|
+
method: "get",
|
|
4790
|
+
path: "/api/v1/projects/{name}/ga/status",
|
|
4791
|
+
summary: "Get GA4 connection status",
|
|
4792
|
+
tags: ["ga4"],
|
|
4793
|
+
parameters: [nameParameter],
|
|
4794
|
+
responses: {
|
|
4795
|
+
200: { description: "GA4 status returned." },
|
|
4796
|
+
404: { description: "Project not found." }
|
|
4797
|
+
}
|
|
4798
|
+
},
|
|
4799
|
+
{
|
|
4800
|
+
method: "post",
|
|
4801
|
+
path: "/api/v1/projects/{name}/ga/sync",
|
|
4802
|
+
summary: "Sync GA4 traffic data",
|
|
4803
|
+
tags: ["ga4"],
|
|
4804
|
+
parameters: [nameParameter],
|
|
4805
|
+
requestBody: {
|
|
4806
|
+
required: false,
|
|
4807
|
+
content: {
|
|
4808
|
+
"application/json": {
|
|
4809
|
+
schema: {
|
|
4810
|
+
type: "object",
|
|
4811
|
+
properties: {
|
|
4812
|
+
days: integerSchema
|
|
4813
|
+
}
|
|
4814
|
+
}
|
|
4815
|
+
}
|
|
4816
|
+
}
|
|
4817
|
+
},
|
|
4818
|
+
responses: {
|
|
4819
|
+
200: { description: "GA4 sync completed." },
|
|
4820
|
+
400: { description: "GA4 is not connected." },
|
|
4821
|
+
404: { description: "Project not found." }
|
|
4822
|
+
}
|
|
4823
|
+
},
|
|
4824
|
+
{
|
|
4825
|
+
method: "get",
|
|
4826
|
+
path: "/api/v1/projects/{name}/ga/traffic",
|
|
4827
|
+
summary: "Get GA4 landing page traffic",
|
|
4828
|
+
tags: ["ga4"],
|
|
4829
|
+
parameters: [nameParameter, limitQueryParameter],
|
|
4830
|
+
responses: {
|
|
4831
|
+
200: { description: "GA4 traffic data returned." },
|
|
4832
|
+
400: { description: "GA4 is not connected." },
|
|
4833
|
+
404: { description: "Project not found." }
|
|
4834
|
+
}
|
|
4835
|
+
},
|
|
4836
|
+
{
|
|
4837
|
+
method: "get",
|
|
4838
|
+
path: "/api/v1/projects/{name}/ga/coverage",
|
|
4839
|
+
summary: "Get GA4 page coverage with traffic overlay",
|
|
4840
|
+
tags: ["ga4"],
|
|
4841
|
+
parameters: [nameParameter],
|
|
4842
|
+
responses: {
|
|
4843
|
+
200: { description: "GA4 coverage data returned." },
|
|
4844
|
+
400: { description: "GA4 is not connected." },
|
|
4845
|
+
404: { description: "Project not found." }
|
|
4846
|
+
}
|
|
4642
4847
|
}
|
|
4643
4848
|
];
|
|
4644
4849
|
function buildOpenApiDocument(info = {}) {
|
|
@@ -6639,6 +6844,525 @@ async function cdpRoutes(app, opts) {
|
|
|
6639
6844
|
);
|
|
6640
6845
|
}
|
|
6641
6846
|
|
|
6847
|
+
// ../api-routes/src/ga.ts
|
|
6848
|
+
import crypto16 from "crypto";
|
|
6849
|
+
import { eq as eq16, desc as desc6, and as and6, sql as sql3 } from "drizzle-orm";
|
|
6850
|
+
|
|
6851
|
+
// ../integration-google-analytics/src/ga4-client.ts
|
|
6852
|
+
import crypto15 from "crypto";
|
|
6853
|
+
|
|
6854
|
+
// ../integration-google-analytics/src/constants.ts
|
|
6855
|
+
var GA4_DATA_API_BASE = "https://analyticsdata.googleapis.com/v1beta";
|
|
6856
|
+
var GA4_SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
|
|
6857
|
+
var GOOGLE_TOKEN_URL2 = "https://oauth2.googleapis.com/token";
|
|
6858
|
+
var GA4_DEFAULT_SYNC_DAYS = 30;
|
|
6859
|
+
var GA4_MAX_SYNC_DAYS = 90;
|
|
6860
|
+
|
|
6861
|
+
// ../integration-google-analytics/src/types.ts
|
|
6862
|
+
var GA4ApiError = class extends Error {
|
|
6863
|
+
status;
|
|
6864
|
+
constructor(message, status) {
|
|
6865
|
+
super(message);
|
|
6866
|
+
this.name = "GA4ApiError";
|
|
6867
|
+
this.status = status;
|
|
6868
|
+
}
|
|
6869
|
+
};
|
|
6870
|
+
|
|
6871
|
+
// ../integration-google-analytics/src/ga4-client.ts
|
|
6872
|
+
function ga4Log(level, action, ctx) {
|
|
6873
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Client", action, ...ctx };
|
|
6874
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6875
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6876
|
+
}
|
|
6877
|
+
function createServiceAccountJwt(clientEmail, privateKey, scope) {
|
|
6878
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
6879
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
6880
|
+
const payload = {
|
|
6881
|
+
iss: clientEmail,
|
|
6882
|
+
scope,
|
|
6883
|
+
aud: GOOGLE_TOKEN_URL2,
|
|
6884
|
+
iat: now,
|
|
6885
|
+
exp: now + 3600
|
|
6886
|
+
// 1 hour
|
|
6887
|
+
};
|
|
6888
|
+
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
6889
|
+
const headerB64 = encode(header);
|
|
6890
|
+
const payloadB64 = encode(payload);
|
|
6891
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
6892
|
+
const sign = crypto15.createSign("RSA-SHA256");
|
|
6893
|
+
sign.update(signingInput);
|
|
6894
|
+
const signature = sign.sign(privateKey, "base64url");
|
|
6895
|
+
return `${signingInput}.${signature}`;
|
|
6896
|
+
}
|
|
6897
|
+
async function getAccessToken(clientEmail, privateKey) {
|
|
6898
|
+
const jwt = createServiceAccountJwt(clientEmail, privateKey, GA4_SCOPE);
|
|
6899
|
+
const res = await fetch(GOOGLE_TOKEN_URL2, {
|
|
6900
|
+
method: "POST",
|
|
6901
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
6902
|
+
body: new URLSearchParams({
|
|
6903
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
6904
|
+
assertion: jwt
|
|
6905
|
+
})
|
|
6906
|
+
});
|
|
6907
|
+
if (!res.ok) {
|
|
6908
|
+
const body = await res.text().catch(() => "");
|
|
6909
|
+
ga4Log("error", "token.failed", { httpStatus: res.status, responseBody: body });
|
|
6910
|
+
throw new GA4ApiError(`Failed to get access token: ${body}`, res.status);
|
|
6911
|
+
}
|
|
6912
|
+
const data = await res.json();
|
|
6913
|
+
return data.access_token;
|
|
6914
|
+
}
|
|
6915
|
+
async function runReport(accessToken, propertyId, request) {
|
|
6916
|
+
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:runReport`;
|
|
6917
|
+
const res = await fetch(url, {
|
|
6918
|
+
method: "POST",
|
|
6919
|
+
headers: {
|
|
6920
|
+
"Authorization": `Bearer ${accessToken}`,
|
|
6921
|
+
"Content-Type": "application/json"
|
|
6922
|
+
},
|
|
6923
|
+
body: JSON.stringify(request)
|
|
6924
|
+
});
|
|
6925
|
+
if (res.status === 401 || res.status === 403) {
|
|
6926
|
+
const body = await res.text().catch(() => "");
|
|
6927
|
+
let detail = "";
|
|
6928
|
+
try {
|
|
6929
|
+
const parsed = JSON.parse(body);
|
|
6930
|
+
if (parsed.error?.status === "SERVICE_DISABLED") {
|
|
6931
|
+
detail = " The Google Analytics Data API is not enabled for this GCP project. Enable it at: https://console.developers.google.com/apis/api/analyticsdata.googleapis.com/overview";
|
|
6932
|
+
} else if (parsed.error?.message) {
|
|
6933
|
+
detail = ` ${parsed.error.message}`;
|
|
6934
|
+
}
|
|
6935
|
+
} catch {
|
|
6936
|
+
if (body.length < 200) detail = ` ${body}`;
|
|
6937
|
+
}
|
|
6938
|
+
ga4Log("error", "report.auth-failed", { propertyId, httpStatus: res.status, responseBody: body });
|
|
6939
|
+
throw new GA4ApiError(
|
|
6940
|
+
`GA4 API authentication failed \u2014 check service account permissions.${detail}`,
|
|
6941
|
+
res.status
|
|
6942
|
+
);
|
|
6943
|
+
}
|
|
6944
|
+
if (res.status === 429) {
|
|
6945
|
+
ga4Log("error", "report.rate-limited", { propertyId });
|
|
6946
|
+
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
6947
|
+
}
|
|
6948
|
+
if (!res.ok) {
|
|
6949
|
+
const body = await res.text();
|
|
6950
|
+
ga4Log("error", "report.error", { propertyId, httpStatus: res.status, responseBody: body });
|
|
6951
|
+
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
6952
|
+
}
|
|
6953
|
+
return await res.json();
|
|
6954
|
+
}
|
|
6955
|
+
async function batchRunReports(accessToken, propertyId, requests) {
|
|
6956
|
+
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:batchRunReports`;
|
|
6957
|
+
const res = await fetch(url, {
|
|
6958
|
+
method: "POST",
|
|
6959
|
+
headers: {
|
|
6960
|
+
"Authorization": `Bearer ${accessToken}`,
|
|
6961
|
+
"Content-Type": "application/json"
|
|
6962
|
+
},
|
|
6963
|
+
body: JSON.stringify({ requests })
|
|
6964
|
+
});
|
|
6965
|
+
if (res.status === 401 || res.status === 403) {
|
|
6966
|
+
const body = await res.text().catch(() => "");
|
|
6967
|
+
ga4Log("error", "batch-report.auth-failed", { propertyId, httpStatus: res.status });
|
|
6968
|
+
throw new GA4ApiError(
|
|
6969
|
+
`GA4 API authentication failed \u2014 check service account permissions. ${body}`,
|
|
6970
|
+
res.status
|
|
6971
|
+
);
|
|
6972
|
+
}
|
|
6973
|
+
if (res.status === 429) {
|
|
6974
|
+
ga4Log("error", "batch-report.rate-limited", { propertyId });
|
|
6975
|
+
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
6976
|
+
}
|
|
6977
|
+
if (!res.ok) {
|
|
6978
|
+
const body = await res.text();
|
|
6979
|
+
ga4Log("error", "batch-report.error", { propertyId, httpStatus: res.status });
|
|
6980
|
+
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
6981
|
+
}
|
|
6982
|
+
const data = await res.json();
|
|
6983
|
+
return data.reports;
|
|
6984
|
+
}
|
|
6985
|
+
function formatDate(d) {
|
|
6986
|
+
return d.toISOString().split("T")[0];
|
|
6987
|
+
}
|
|
6988
|
+
async function fetchTrafficByLandingPage(accessToken, propertyId, days) {
|
|
6989
|
+
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
6990
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
6991
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
6992
|
+
startDate.setDate(startDate.getDate() - syncDays);
|
|
6993
|
+
ga4Log("info", "fetch-traffic.start", { propertyId, days: syncDays });
|
|
6994
|
+
const PAGE_SIZE = 1e4;
|
|
6995
|
+
const rows = [];
|
|
6996
|
+
let offset = 0;
|
|
6997
|
+
while (true) {
|
|
6998
|
+
const request = {
|
|
6999
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
7000
|
+
dimensions: [
|
|
7001
|
+
{ name: "date" },
|
|
7002
|
+
{ name: "landingPagePlusQueryString" }
|
|
7003
|
+
],
|
|
7004
|
+
metrics: [
|
|
7005
|
+
{ name: "sessions" },
|
|
7006
|
+
{ name: "totalUsers" }
|
|
7007
|
+
],
|
|
7008
|
+
limit: PAGE_SIZE,
|
|
7009
|
+
offset
|
|
7010
|
+
};
|
|
7011
|
+
const response = await runReport(accessToken, propertyId, request);
|
|
7012
|
+
const pageRows = (response.rows ?? []).map((row) => ({
|
|
7013
|
+
date: row.dimensionValues[0].value,
|
|
7014
|
+
landingPage: row.dimensionValues[1].value,
|
|
7015
|
+
sessions: parseInt(row.metricValues[0].value, 10) || 0,
|
|
7016
|
+
organicSessions: 0,
|
|
7017
|
+
// populated by organic-only pass below
|
|
7018
|
+
users: parseInt(row.metricValues[1].value, 10) || 0
|
|
7019
|
+
}));
|
|
7020
|
+
rows.push(...pageRows);
|
|
7021
|
+
const totalRows = response.rowCount ?? 0;
|
|
7022
|
+
offset += pageRows.length;
|
|
7023
|
+
if (pageRows.length < PAGE_SIZE || offset >= totalRows) break;
|
|
7024
|
+
}
|
|
7025
|
+
const organicMap = /* @__PURE__ */ new Map();
|
|
7026
|
+
let organicOffset = 0;
|
|
7027
|
+
while (true) {
|
|
7028
|
+
const organicRequest = {
|
|
7029
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
7030
|
+
dimensions: [{ name: "date" }, { name: "landingPagePlusQueryString" }],
|
|
7031
|
+
metrics: [{ name: "sessions" }],
|
|
7032
|
+
dimensionFilter: {
|
|
7033
|
+
filter: {
|
|
7034
|
+
fieldName: "sessionDefaultChannelGrouping",
|
|
7035
|
+
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
7036
|
+
}
|
|
7037
|
+
},
|
|
7038
|
+
limit: 1e4,
|
|
7039
|
+
offset: organicOffset
|
|
7040
|
+
};
|
|
7041
|
+
const organicResponse = await runReport(accessToken, propertyId, organicRequest);
|
|
7042
|
+
for (const row of organicResponse.rows ?? []) {
|
|
7043
|
+
const key = `${row.dimensionValues[0].value}::${row.dimensionValues[1].value}`;
|
|
7044
|
+
organicMap.set(key, parseInt(row.metricValues[0].value, 10) || 0);
|
|
7045
|
+
}
|
|
7046
|
+
const total = organicResponse.rowCount ?? 0;
|
|
7047
|
+
organicOffset += (organicResponse.rows ?? []).length;
|
|
7048
|
+
if ((organicResponse.rows ?? []).length < 1e4 || organicOffset >= total) break;
|
|
7049
|
+
}
|
|
7050
|
+
for (const row of rows) {
|
|
7051
|
+
const key = `${row.date}::${row.landingPage}`;
|
|
7052
|
+
row.organicSessions = organicMap.get(key) ?? 0;
|
|
7053
|
+
}
|
|
7054
|
+
for (const row of rows) {
|
|
7055
|
+
if (row.date.length === 8 && !row.date.includes("-")) {
|
|
7056
|
+
row.date = `${row.date.slice(0, 4)}-${row.date.slice(4, 6)}-${row.date.slice(6, 8)}`;
|
|
7057
|
+
}
|
|
7058
|
+
}
|
|
7059
|
+
ga4Log("info", "fetch-traffic.done", { propertyId, rowCount: rows.length });
|
|
7060
|
+
return rows;
|
|
7061
|
+
}
|
|
7062
|
+
async function verifyConnection(clientEmail, privateKey, propertyId) {
|
|
7063
|
+
const accessToken = await getAccessToken(clientEmail, privateKey);
|
|
7064
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
7065
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
7066
|
+
startDate.setDate(startDate.getDate() - 1);
|
|
7067
|
+
await runReport(accessToken, propertyId, {
|
|
7068
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
7069
|
+
dimensions: [{ name: "date" }],
|
|
7070
|
+
metrics: [{ name: "sessions" }],
|
|
7071
|
+
limit: 1
|
|
7072
|
+
});
|
|
7073
|
+
return true;
|
|
7074
|
+
}
|
|
7075
|
+
async function fetchAggregateSummary(accessToken, propertyId, days) {
|
|
7076
|
+
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
7077
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
7078
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
7079
|
+
startDate.setDate(startDate.getDate() - syncDays);
|
|
7080
|
+
ga4Log("info", "fetch-aggregate.start", { propertyId, days: syncDays });
|
|
7081
|
+
const dateRange = { startDate: formatDate(startDate), endDate: formatDate(endDate) };
|
|
7082
|
+
const batchRes = await batchRunReports(accessToken, propertyId, [
|
|
7083
|
+
{
|
|
7084
|
+
dateRanges: [dateRange],
|
|
7085
|
+
dimensions: [],
|
|
7086
|
+
metrics: [{ name: "sessions" }, { name: "totalUsers" }],
|
|
7087
|
+
limit: 1
|
|
7088
|
+
},
|
|
7089
|
+
{
|
|
7090
|
+
dateRanges: [dateRange],
|
|
7091
|
+
dimensions: [],
|
|
7092
|
+
metrics: [{ name: "sessions" }],
|
|
7093
|
+
dimensionFilter: {
|
|
7094
|
+
filter: {
|
|
7095
|
+
fieldName: "sessionDefaultChannelGrouping",
|
|
7096
|
+
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
7097
|
+
}
|
|
7098
|
+
},
|
|
7099
|
+
limit: 1
|
|
7100
|
+
}
|
|
7101
|
+
]);
|
|
7102
|
+
const totalRow = batchRes[0]?.rows?.[0];
|
|
7103
|
+
const organicRow = batchRes[1]?.rows?.[0];
|
|
7104
|
+
const summary = {
|
|
7105
|
+
periodStart: formatDate(startDate),
|
|
7106
|
+
periodEnd: formatDate(endDate),
|
|
7107
|
+
totalSessions: parseInt(totalRow?.metricValues[0]?.value ?? "0", 10) || 0,
|
|
7108
|
+
totalUsers: parseInt(totalRow?.metricValues[1]?.value ?? "0", 10) || 0,
|
|
7109
|
+
totalOrganicSessions: parseInt(organicRow?.metricValues[0]?.value ?? "0", 10) || 0
|
|
7110
|
+
};
|
|
7111
|
+
ga4Log("info", "fetch-aggregate.done", { propertyId, ...summary });
|
|
7112
|
+
return summary;
|
|
7113
|
+
}
|
|
7114
|
+
|
|
7115
|
+
// ../api-routes/src/ga.ts
|
|
7116
|
+
function gaLog(level, action, ctx) {
|
|
7117
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
7118
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
7119
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
7120
|
+
}
|
|
7121
|
+
async function ga4Routes(app, opts) {
|
|
7122
|
+
function requireCredentialStore(reply) {
|
|
7123
|
+
if (opts.ga4CredentialStore) return opts.ga4CredentialStore;
|
|
7124
|
+
const err = validationError("GA4 credential storage is not configured for this deployment");
|
|
7125
|
+
reply.status(err.statusCode).send(err.toJSON());
|
|
7126
|
+
return null;
|
|
7127
|
+
}
|
|
7128
|
+
app.post("/projects/:name/ga/connect", async (request, reply) => {
|
|
7129
|
+
const store = requireCredentialStore(reply);
|
|
7130
|
+
if (!store) return;
|
|
7131
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7132
|
+
const { propertyId, keyJson } = request.body ?? {};
|
|
7133
|
+
if (!propertyId || typeof propertyId !== "string") {
|
|
7134
|
+
const err = validationError("propertyId is required");
|
|
7135
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7136
|
+
}
|
|
7137
|
+
let clientEmail;
|
|
7138
|
+
let privateKey;
|
|
7139
|
+
if (keyJson && typeof keyJson === "string") {
|
|
7140
|
+
try {
|
|
7141
|
+
const parsed = JSON.parse(keyJson);
|
|
7142
|
+
if (!parsed.client_email || !parsed.private_key) {
|
|
7143
|
+
const err = validationError("Service account JSON must contain client_email and private_key");
|
|
7144
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7145
|
+
}
|
|
7146
|
+
clientEmail = parsed.client_email;
|
|
7147
|
+
privateKey = parsed.private_key;
|
|
7148
|
+
} catch {
|
|
7149
|
+
const err = validationError("Invalid JSON in keyJson");
|
|
7150
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7151
|
+
}
|
|
7152
|
+
} else {
|
|
7153
|
+
const err = validationError("keyJson is required");
|
|
7154
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7155
|
+
}
|
|
7156
|
+
try {
|
|
7157
|
+
await verifyConnection(clientEmail, privateKey, propertyId);
|
|
7158
|
+
gaLog("info", "connect.verified", { projectId: project.id, propertyId });
|
|
7159
|
+
} catch (e) {
|
|
7160
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
7161
|
+
gaLog("error", "connect.verify-failed", { projectId: project.id, propertyId, error: msg });
|
|
7162
|
+
const err = validationError(`Failed to verify GA4 credentials: ${msg}`);
|
|
7163
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7164
|
+
}
|
|
7165
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7166
|
+
const existing = store.getConnection(project.name);
|
|
7167
|
+
store.upsertConnection({
|
|
7168
|
+
projectName: project.name,
|
|
7169
|
+
propertyId,
|
|
7170
|
+
clientEmail,
|
|
7171
|
+
privateKey,
|
|
7172
|
+
createdAt: existing?.createdAt ?? now,
|
|
7173
|
+
updatedAt: now
|
|
7174
|
+
});
|
|
7175
|
+
writeAuditLog(app.db, {
|
|
7176
|
+
projectId: project.id,
|
|
7177
|
+
actor: "api",
|
|
7178
|
+
action: "ga4.connected",
|
|
7179
|
+
entityType: "ga_connection",
|
|
7180
|
+
entityId: propertyId
|
|
7181
|
+
});
|
|
7182
|
+
return {
|
|
7183
|
+
connected: true,
|
|
7184
|
+
propertyId,
|
|
7185
|
+
clientEmail
|
|
7186
|
+
};
|
|
7187
|
+
});
|
|
7188
|
+
app.delete("/projects/:name/ga/disconnect", async (request, reply) => {
|
|
7189
|
+
const store = requireCredentialStore(reply);
|
|
7190
|
+
if (!store) return;
|
|
7191
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7192
|
+
const conn = store.getConnection(project.name);
|
|
7193
|
+
if (!conn) {
|
|
7194
|
+
const err = notFound("GA4 connection", project.name);
|
|
7195
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7196
|
+
}
|
|
7197
|
+
app.db.delete(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).run();
|
|
7198
|
+
app.db.delete(gaTrafficSummaries).where(eq16(gaTrafficSummaries.projectId, project.id)).run();
|
|
7199
|
+
store.deleteConnection(project.name);
|
|
7200
|
+
writeAuditLog(app.db, {
|
|
7201
|
+
projectId: project.id,
|
|
7202
|
+
actor: "api",
|
|
7203
|
+
action: "ga4.disconnected",
|
|
7204
|
+
entityType: "ga_connection",
|
|
7205
|
+
entityId: conn.propertyId
|
|
7206
|
+
});
|
|
7207
|
+
return reply.status(204).send();
|
|
7208
|
+
});
|
|
7209
|
+
app.get("/projects/:name/ga/status", async (request, reply) => {
|
|
7210
|
+
const store = requireCredentialStore(reply);
|
|
7211
|
+
if (!store) return;
|
|
7212
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7213
|
+
const conn = store.getConnection(project.name);
|
|
7214
|
+
if (!conn) {
|
|
7215
|
+
return { connected: false, propertyId: null, clientEmail: null, lastSyncedAt: null };
|
|
7216
|
+
}
|
|
7217
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSnapshots.syncedAt }).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).orderBy(desc6(gaTrafficSnapshots.syncedAt)).limit(1).get();
|
|
7218
|
+
return {
|
|
7219
|
+
connected: true,
|
|
7220
|
+
propertyId: conn.propertyId,
|
|
7221
|
+
clientEmail: conn.clientEmail,
|
|
7222
|
+
lastSyncedAt: latestSync?.syncedAt ?? null,
|
|
7223
|
+
createdAt: conn.createdAt,
|
|
7224
|
+
updatedAt: conn.updatedAt
|
|
7225
|
+
};
|
|
7226
|
+
});
|
|
7227
|
+
app.post("/projects/:name/ga/sync", async (request, reply) => {
|
|
7228
|
+
const store = requireCredentialStore(reply);
|
|
7229
|
+
if (!store) return;
|
|
7230
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7231
|
+
const conn = store.getConnection(project.name);
|
|
7232
|
+
if (!conn) {
|
|
7233
|
+
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
7234
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7235
|
+
}
|
|
7236
|
+
const days = request.body?.days ?? 30;
|
|
7237
|
+
let accessToken;
|
|
7238
|
+
try {
|
|
7239
|
+
accessToken = await getAccessToken(conn.clientEmail, conn.privateKey);
|
|
7240
|
+
} catch (e) {
|
|
7241
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
7242
|
+
gaLog("error", "sync.auth-failed", { projectId: project.id, error: msg });
|
|
7243
|
+
const err = validationError(`GA4 authentication failed: ${msg}`);
|
|
7244
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7245
|
+
}
|
|
7246
|
+
let rows;
|
|
7247
|
+
let summary;
|
|
7248
|
+
try {
|
|
7249
|
+
;
|
|
7250
|
+
[rows, summary] = await Promise.all([
|
|
7251
|
+
fetchTrafficByLandingPage(accessToken, conn.propertyId, days),
|
|
7252
|
+
fetchAggregateSummary(accessToken, conn.propertyId, days)
|
|
7253
|
+
]);
|
|
7254
|
+
} catch (e) {
|
|
7255
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
7256
|
+
gaLog("error", "sync.fetch-failed", { projectId: project.id, error: msg });
|
|
7257
|
+
throw e;
|
|
7258
|
+
}
|
|
7259
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7260
|
+
app.db.transaction((tx) => {
|
|
7261
|
+
if (rows.length > 0) {
|
|
7262
|
+
const dates = rows.map((r) => r.date);
|
|
7263
|
+
const minDate = dates.reduce((a, b) => a < b ? a : b);
|
|
7264
|
+
const maxDate = dates.reduce((a, b) => a > b ? a : b);
|
|
7265
|
+
tx.delete(gaTrafficSnapshots).where(
|
|
7266
|
+
and6(
|
|
7267
|
+
eq16(gaTrafficSnapshots.projectId, project.id),
|
|
7268
|
+
sql3`${gaTrafficSnapshots.date} >= ${minDate}`,
|
|
7269
|
+
sql3`${gaTrafficSnapshots.date} <= ${maxDate}`
|
|
7270
|
+
)
|
|
7271
|
+
).run();
|
|
7272
|
+
for (const row of rows) {
|
|
7273
|
+
tx.insert(gaTrafficSnapshots).values({
|
|
7274
|
+
id: crypto16.randomUUID(),
|
|
7275
|
+
projectId: project.id,
|
|
7276
|
+
date: row.date,
|
|
7277
|
+
landingPage: row.landingPage,
|
|
7278
|
+
sessions: row.sessions,
|
|
7279
|
+
organicSessions: row.organicSessions,
|
|
7280
|
+
users: row.users,
|
|
7281
|
+
syncedAt: now
|
|
7282
|
+
}).run();
|
|
7283
|
+
}
|
|
7284
|
+
}
|
|
7285
|
+
tx.delete(gaTrafficSummaries).where(eq16(gaTrafficSummaries.projectId, project.id)).run();
|
|
7286
|
+
tx.insert(gaTrafficSummaries).values({
|
|
7287
|
+
id: crypto16.randomUUID(),
|
|
7288
|
+
projectId: project.id,
|
|
7289
|
+
periodStart: summary.periodStart,
|
|
7290
|
+
periodEnd: summary.periodEnd,
|
|
7291
|
+
totalSessions: summary.totalSessions,
|
|
7292
|
+
totalOrganicSessions: summary.totalOrganicSessions,
|
|
7293
|
+
totalUsers: summary.totalUsers,
|
|
7294
|
+
syncedAt: now
|
|
7295
|
+
}).run();
|
|
7296
|
+
});
|
|
7297
|
+
gaLog("info", "sync.complete", { projectId: project.id, rowCount: rows.length, days, totalUsers: summary.totalUsers });
|
|
7298
|
+
return {
|
|
7299
|
+
synced: true,
|
|
7300
|
+
rowCount: rows.length,
|
|
7301
|
+
days,
|
|
7302
|
+
syncedAt: now
|
|
7303
|
+
};
|
|
7304
|
+
});
|
|
7305
|
+
app.get("/projects/:name/ga/traffic", async (request, reply) => {
|
|
7306
|
+
const store = requireCredentialStore(reply);
|
|
7307
|
+
if (!store) return;
|
|
7308
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7309
|
+
const conn = store.getConnection(project.name);
|
|
7310
|
+
if (!conn) {
|
|
7311
|
+
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
7312
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7313
|
+
}
|
|
7314
|
+
const limit = Math.max(1, Math.min(parseInt(request.query.limit ?? "50", 10) || 50, 500));
|
|
7315
|
+
const summary = app.db.select({
|
|
7316
|
+
totalSessions: gaTrafficSummaries.totalSessions,
|
|
7317
|
+
totalOrganicSessions: gaTrafficSummaries.totalOrganicSessions,
|
|
7318
|
+
totalUsers: gaTrafficSummaries.totalUsers
|
|
7319
|
+
}).from(gaTrafficSummaries).where(eq16(gaTrafficSummaries.projectId, project.id)).get();
|
|
7320
|
+
const rows = app.db.select({
|
|
7321
|
+
landingPage: gaTrafficSnapshots.landingPage,
|
|
7322
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
7323
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
7324
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
7325
|
+
}).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql3`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
7326
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSnapshots.syncedAt }).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).orderBy(desc6(gaTrafficSnapshots.syncedAt)).limit(1).get();
|
|
7327
|
+
return {
|
|
7328
|
+
totalSessions: summary?.totalSessions ?? 0,
|
|
7329
|
+
totalOrganicSessions: summary?.totalOrganicSessions ?? 0,
|
|
7330
|
+
totalUsers: summary?.totalUsers ?? 0,
|
|
7331
|
+
topPages: rows.map((r) => ({
|
|
7332
|
+
landingPage: r.landingPage,
|
|
7333
|
+
sessions: r.sessions ?? 0,
|
|
7334
|
+
organicSessions: r.organicSessions ?? 0,
|
|
7335
|
+
users: r.users ?? 0
|
|
7336
|
+
})),
|
|
7337
|
+
lastSyncedAt: latestSync?.syncedAt ?? null
|
|
7338
|
+
};
|
|
7339
|
+
});
|
|
7340
|
+
app.get("/projects/:name/ga/coverage", async (request, reply) => {
|
|
7341
|
+
const store = requireCredentialStore(reply);
|
|
7342
|
+
if (!store) return;
|
|
7343
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7344
|
+
const conn = store.getConnection(project.name);
|
|
7345
|
+
if (!conn) {
|
|
7346
|
+
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
7347
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7348
|
+
}
|
|
7349
|
+
const trafficPages = app.db.select({
|
|
7350
|
+
landingPage: gaTrafficSnapshots.landingPage,
|
|
7351
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
7352
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
7353
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
7354
|
+
}).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql3`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
7355
|
+
return {
|
|
7356
|
+
pages: trafficPages.map((r) => ({
|
|
7357
|
+
landingPage: r.landingPage,
|
|
7358
|
+
sessions: r.sessions ?? 0,
|
|
7359
|
+
organicSessions: r.organicSessions ?? 0,
|
|
7360
|
+
users: r.users ?? 0
|
|
7361
|
+
}))
|
|
7362
|
+
};
|
|
7363
|
+
});
|
|
7364
|
+
}
|
|
7365
|
+
|
|
6642
7366
|
// ../api-routes/src/index.ts
|
|
6643
7367
|
async function apiRoutes(app, opts) {
|
|
6644
7368
|
app.decorate("db", opts.db);
|
|
@@ -6727,6 +7451,9 @@ async function apiRoutes(app, opts) {
|
|
|
6727
7451
|
onCdpScreenshot: opts.onCdpScreenshot,
|
|
6728
7452
|
onCdpConfigure: opts.onCdpConfigure
|
|
6729
7453
|
});
|
|
7454
|
+
await api.register(ga4Routes, {
|
|
7455
|
+
ga4CredentialStore: opts.ga4CredentialStore
|
|
7456
|
+
});
|
|
6730
7457
|
}, { prefix: opts.routePrefix ?? "/api/v1" });
|
|
6731
7458
|
}
|
|
6732
7459
|
|
|
@@ -8558,12 +9285,44 @@ function removeGoogleConnection(config, domain, connectionType) {
|
|
|
8558
9285
|
return true;
|
|
8559
9286
|
}
|
|
8560
9287
|
|
|
9288
|
+
// src/ga4-config.ts
|
|
9289
|
+
function ensureConnections2(config) {
|
|
9290
|
+
if (!config.ga4) config.ga4 = {};
|
|
9291
|
+
if (!config.ga4.connections) config.ga4.connections = [];
|
|
9292
|
+
return config.ga4.connections;
|
|
9293
|
+
}
|
|
9294
|
+
function getGa4Connection(config, projectName) {
|
|
9295
|
+
return (config.ga4?.connections ?? []).find((c) => c.projectName === projectName);
|
|
9296
|
+
}
|
|
9297
|
+
function upsertGa4Connection(config, connection) {
|
|
9298
|
+
const connections = ensureConnections2(config);
|
|
9299
|
+
const index2 = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
9300
|
+
if (index2 === -1) {
|
|
9301
|
+
connections.push(connection);
|
|
9302
|
+
return connection;
|
|
9303
|
+
}
|
|
9304
|
+
connections[index2] = connection;
|
|
9305
|
+
return connection;
|
|
9306
|
+
}
|
|
9307
|
+
function removeGa4Connection(config, projectName) {
|
|
9308
|
+
const connections = config.ga4?.connections;
|
|
9309
|
+
if (!connections?.length) return false;
|
|
9310
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
9311
|
+
if (next.length === connections.length) return false;
|
|
9312
|
+
if (!config.ga4) return false;
|
|
9313
|
+
config.ga4.connections = next;
|
|
9314
|
+
if (next.length === 0) {
|
|
9315
|
+
delete config.ga4;
|
|
9316
|
+
}
|
|
9317
|
+
return true;
|
|
9318
|
+
}
|
|
9319
|
+
|
|
8561
9320
|
// src/job-runner.ts
|
|
8562
|
-
import
|
|
9321
|
+
import crypto17 from "crypto";
|
|
8563
9322
|
import fs4 from "fs";
|
|
8564
9323
|
import path5 from "path";
|
|
8565
9324
|
import os4 from "os";
|
|
8566
|
-
import { and as
|
|
9325
|
+
import { and as and7, eq as eq17, inArray as inArray3 } from "drizzle-orm";
|
|
8567
9326
|
|
|
8568
9327
|
// src/logger.ts
|
|
8569
9328
|
var IS_TTY = process.stdout.isTTY === true;
|
|
@@ -8685,7 +9444,7 @@ var JobRunner = class {
|
|
|
8685
9444
|
if (stale.length === 0) return;
|
|
8686
9445
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8687
9446
|
for (const run of stale) {
|
|
8688
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
9447
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq17(runs.id, run.id)).run();
|
|
8689
9448
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
8690
9449
|
}
|
|
8691
9450
|
}
|
|
@@ -8713,10 +9472,10 @@ var JobRunner = class {
|
|
|
8713
9472
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
8714
9473
|
}
|
|
8715
9474
|
if (existingRun.status === "queued") {
|
|
8716
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
9475
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and7(eq17(runs.id, runId), eq17(runs.status, "queued"))).run();
|
|
8717
9476
|
}
|
|
8718
9477
|
this.throwIfRunCancelled(runId);
|
|
8719
|
-
const project = this.db.select().from(projects).where(
|
|
9478
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
8720
9479
|
if (!project) {
|
|
8721
9480
|
throw new Error(`Project ${projectId} not found`);
|
|
8722
9481
|
}
|
|
@@ -8736,8 +9495,8 @@ var JobRunner = class {
|
|
|
8736
9495
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
8737
9496
|
}
|
|
8738
9497
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
8739
|
-
projectKeywords = this.db.select().from(keywords).where(
|
|
8740
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
9498
|
+
projectKeywords = this.db.select().from(keywords).where(eq17(keywords.projectId, projectId)).all();
|
|
9499
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq17(competitors.projectId, projectId)).all();
|
|
8741
9500
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
8742
9501
|
const allDomains = effectiveDomains({
|
|
8743
9502
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -8753,7 +9512,7 @@ var JobRunner = class {
|
|
|
8753
9512
|
const todayPeriod = getCurrentUsageDay();
|
|
8754
9513
|
for (const p of activeProviders) {
|
|
8755
9514
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
8756
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
9515
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq17(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
8757
9516
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
8758
9517
|
if (providerUsage + queriesPerProvider > limit) {
|
|
8759
9518
|
throw new Error(
|
|
@@ -8802,7 +9561,7 @@ var JobRunner = class {
|
|
|
8802
9561
|
const overlap = computeCompetitorOverlap(normalized, competitorDomains);
|
|
8803
9562
|
let screenshotRelPath = null;
|
|
8804
9563
|
if (raw.screenshotPath && fs4.existsSync(raw.screenshotPath)) {
|
|
8805
|
-
const snapshotId =
|
|
9564
|
+
const snapshotId = crypto17.randomUUID();
|
|
8806
9565
|
const screenshotDir = path5.join(os4.homedir(), ".canonry", "screenshots", runId);
|
|
8807
9566
|
if (!fs4.existsSync(screenshotDir)) fs4.mkdirSync(screenshotDir, { recursive: true });
|
|
8808
9567
|
const destPath = path5.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -8830,7 +9589,7 @@ var JobRunner = class {
|
|
|
8830
9589
|
}).run();
|
|
8831
9590
|
} else {
|
|
8832
9591
|
this.db.insert(querySnapshots).values({
|
|
8833
|
-
id:
|
|
9592
|
+
id: crypto17.randomUUID(),
|
|
8834
9593
|
runId,
|
|
8835
9594
|
keywordId: kw.id,
|
|
8836
9595
|
provider: providerName,
|
|
@@ -8879,12 +9638,12 @@ var JobRunner = class {
|
|
|
8879
9638
|
const someFailed = providerErrors.size > 0;
|
|
8880
9639
|
if (allFailed) {
|
|
8881
9640
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
8882
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
9641
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq17(runs.id, runId)).run();
|
|
8883
9642
|
} else if (someFailed) {
|
|
8884
9643
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
8885
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
9644
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq17(runs.id, runId)).run();
|
|
8886
9645
|
} else {
|
|
8887
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9646
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq17(runs.id, runId)).run();
|
|
8888
9647
|
}
|
|
8889
9648
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
8890
9649
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -8919,7 +9678,7 @@ var JobRunner = class {
|
|
|
8919
9678
|
status: "failed",
|
|
8920
9679
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8921
9680
|
error: errorMessage
|
|
8922
|
-
}).where(
|
|
9681
|
+
}).where(eq17(runs.id, runId)).run();
|
|
8923
9682
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
8924
9683
|
trackEvent("run.completed", {
|
|
8925
9684
|
status: "failed",
|
|
@@ -8939,10 +9698,10 @@ var JobRunner = class {
|
|
|
8939
9698
|
incrementUsage(scope, metric, count) {
|
|
8940
9699
|
const now = /* @__PURE__ */ new Date();
|
|
8941
9700
|
const period = now.toISOString().slice(0, 10);
|
|
8942
|
-
const id =
|
|
8943
|
-
const existing = this.db.select().from(usageCounters).where(
|
|
9701
|
+
const id = crypto17.randomUUID();
|
|
9702
|
+
const existing = this.db.select().from(usageCounters).where(eq17(usageCounters.scope, scope)).all().find((r) => r.period === period && r.metric === metric);
|
|
8944
9703
|
if (existing) {
|
|
8945
|
-
this.db.update(usageCounters).set({ count: existing.count + count, updatedAt: now.toISOString() }).where(
|
|
9704
|
+
this.db.update(usageCounters).set({ count: existing.count + count, updatedAt: now.toISOString() }).where(eq17(usageCounters.id, existing.id)).run();
|
|
8946
9705
|
} else {
|
|
8947
9706
|
this.db.insert(usageCounters).values({
|
|
8948
9707
|
id,
|
|
@@ -8965,7 +9724,7 @@ var JobRunner = class {
|
|
|
8965
9724
|
status: runs.status,
|
|
8966
9725
|
finishedAt: runs.finishedAt,
|
|
8967
9726
|
error: runs.error
|
|
8968
|
-
}).from(runs).where(
|
|
9727
|
+
}).from(runs).where(eq17(runs.id, runId)).get();
|
|
8969
9728
|
}
|
|
8970
9729
|
isRunCancelled(runId) {
|
|
8971
9730
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -8981,7 +9740,7 @@ var JobRunner = class {
|
|
|
8981
9740
|
this.db.update(runs).set({
|
|
8982
9741
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8983
9742
|
error: currentRun.error ?? "Cancelled by user"
|
|
8984
|
-
}).where(
|
|
9743
|
+
}).where(eq17(runs.id, runId)).run();
|
|
8985
9744
|
}
|
|
8986
9745
|
trackEvent("run.completed", {
|
|
8987
9746
|
status: "cancelled",
|
|
@@ -9064,10 +9823,10 @@ function computeCompetitorOverlap(normalized, competitorDomains) {
|
|
|
9064
9823
|
}
|
|
9065
9824
|
|
|
9066
9825
|
// src/gsc-sync.ts
|
|
9067
|
-
import
|
|
9068
|
-
import { eq as
|
|
9826
|
+
import crypto18 from "crypto";
|
|
9827
|
+
import { eq as eq18, and as and8, sql as sql4 } from "drizzle-orm";
|
|
9069
9828
|
var log2 = createLogger("GscSync");
|
|
9070
|
-
function
|
|
9829
|
+
function formatDate2(d) {
|
|
9071
9830
|
return d.toISOString().split("T")[0];
|
|
9072
9831
|
}
|
|
9073
9832
|
function daysAgo(n) {
|
|
@@ -9077,13 +9836,13 @@ function daysAgo(n) {
|
|
|
9077
9836
|
}
|
|
9078
9837
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
9079
9838
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9080
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
9839
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq18(runs.id, runId)).run();
|
|
9081
9840
|
try {
|
|
9082
9841
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
9083
9842
|
if (!googleClientId || !googleClientSecret) {
|
|
9084
9843
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
9085
9844
|
}
|
|
9086
|
-
const project = db.select().from(projects).where(
|
|
9845
|
+
const project = db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
9087
9846
|
if (!project) {
|
|
9088
9847
|
throw new Error(`Project not found: ${projectId}`);
|
|
9089
9848
|
}
|
|
@@ -9107,9 +9866,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9107
9866
|
saveConfig(opts.config);
|
|
9108
9867
|
}
|
|
9109
9868
|
const lagOffset = GSC_DATA_LAG_DAYS;
|
|
9110
|
-
const endDate =
|
|
9869
|
+
const endDate = formatDate2(daysAgo(lagOffset));
|
|
9111
9870
|
const days = opts.full ? 480 : opts.days ?? 30;
|
|
9112
|
-
const startDate =
|
|
9871
|
+
const startDate = formatDate2(daysAgo(days + lagOffset));
|
|
9113
9872
|
log2.info("fetch.start", { runId, projectId, propertyId: conn.propertyId, startDate, endDate });
|
|
9114
9873
|
const rows = await fetchSearchAnalytics(accessToken, conn.propertyId, {
|
|
9115
9874
|
startDate,
|
|
@@ -9117,10 +9876,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9117
9876
|
});
|
|
9118
9877
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
9119
9878
|
db.delete(gscSearchData).where(
|
|
9120
|
-
|
|
9121
|
-
|
|
9122
|
-
|
|
9123
|
-
|
|
9879
|
+
and8(
|
|
9880
|
+
eq18(gscSearchData.projectId, projectId),
|
|
9881
|
+
sql4`${gscSearchData.date} >= ${startDate}`,
|
|
9882
|
+
sql4`${gscSearchData.date} <= ${endDate}`
|
|
9124
9883
|
)
|
|
9125
9884
|
).run();
|
|
9126
9885
|
const batchSize = 500;
|
|
@@ -9130,7 +9889,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9130
9889
|
for (const row of batch) {
|
|
9131
9890
|
const [query, page, country, device, date] = row.keys;
|
|
9132
9891
|
db.insert(gscSearchData).values({
|
|
9133
|
-
id:
|
|
9892
|
+
id: crypto18.randomUUID(),
|
|
9134
9893
|
projectId,
|
|
9135
9894
|
syncRunId: runId,
|
|
9136
9895
|
date: date ?? "",
|
|
@@ -9164,7 +9923,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9164
9923
|
const rich = ir.richResultsResult;
|
|
9165
9924
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9166
9925
|
db.insert(gscUrlInspections).values({
|
|
9167
|
-
id:
|
|
9926
|
+
id: crypto18.randomUUID(),
|
|
9168
9927
|
projectId,
|
|
9169
9928
|
syncRunId: runId,
|
|
9170
9929
|
url: pageUrl,
|
|
@@ -9185,7 +9944,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9185
9944
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
9186
9945
|
}
|
|
9187
9946
|
}
|
|
9188
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
9947
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq18(gscUrlInspections.projectId, projectId)).all();
|
|
9189
9948
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
9190
9949
|
for (const row of allInspections) {
|
|
9191
9950
|
const existing = latestByUrl.get(row.url);
|
|
@@ -9205,10 +9964,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9205
9964
|
reasonCounts[reason] = (reasonCounts[reason] ?? 0) + 1;
|
|
9206
9965
|
}
|
|
9207
9966
|
}
|
|
9208
|
-
const snapshotDate =
|
|
9209
|
-
db.delete(gscCoverageSnapshots).where(
|
|
9967
|
+
const snapshotDate = formatDate2(/* @__PURE__ */ new Date());
|
|
9968
|
+
db.delete(gscCoverageSnapshots).where(and8(eq18(gscCoverageSnapshots.projectId, projectId), eq18(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
9210
9969
|
db.insert(gscCoverageSnapshots).values({
|
|
9211
|
-
id:
|
|
9970
|
+
id: crypto18.randomUUID(),
|
|
9212
9971
|
projectId,
|
|
9213
9972
|
syncRunId: runId,
|
|
9214
9973
|
date: snapshotDate,
|
|
@@ -9217,19 +9976,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9217
9976
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
9218
9977
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9219
9978
|
}).run();
|
|
9220
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9979
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(runs.id, runId)).run();
|
|
9221
9980
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
9222
9981
|
} catch (err) {
|
|
9223
9982
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
9224
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9983
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(runs.id, runId)).run();
|
|
9225
9984
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
9226
9985
|
throw err;
|
|
9227
9986
|
}
|
|
9228
9987
|
}
|
|
9229
9988
|
|
|
9230
9989
|
// src/gsc-inspect-sitemap.ts
|
|
9231
|
-
import
|
|
9232
|
-
import { eq as
|
|
9990
|
+
import crypto19 from "crypto";
|
|
9991
|
+
import { eq as eq19, and as and9 } from "drizzle-orm";
|
|
9233
9992
|
|
|
9234
9993
|
// src/sitemap-parser.ts
|
|
9235
9994
|
var LOC_REGEX = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
|
|
@@ -9298,13 +10057,13 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
9298
10057
|
var log3 = createLogger("InspectSitemap");
|
|
9299
10058
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
9300
10059
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9301
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
10060
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq19(runs.id, runId)).run();
|
|
9302
10061
|
try {
|
|
9303
10062
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
9304
10063
|
if (!googleClientId || !googleClientSecret) {
|
|
9305
10064
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
9306
10065
|
}
|
|
9307
|
-
const project = db.select().from(projects).where(
|
|
10066
|
+
const project = db.select().from(projects).where(eq19(projects.id, projectId)).get();
|
|
9308
10067
|
if (!project) {
|
|
9309
10068
|
throw new Error(`Project not found: ${projectId}`);
|
|
9310
10069
|
}
|
|
@@ -9345,7 +10104,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9345
10104
|
const rich = ir.richResultsResult;
|
|
9346
10105
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9347
10106
|
db.insert(gscUrlInspections).values({
|
|
9348
|
-
id:
|
|
10107
|
+
id: crypto19.randomUUID(),
|
|
9349
10108
|
projectId,
|
|
9350
10109
|
syncRunId: runId,
|
|
9351
10110
|
url: pageUrl,
|
|
@@ -9372,7 +10131,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9372
10131
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
9373
10132
|
}
|
|
9374
10133
|
}
|
|
9375
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
10134
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq19(gscUrlInspections.projectId, projectId)).all();
|
|
9376
10135
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
9377
10136
|
for (const row of allInspections) {
|
|
9378
10137
|
const existing = latestByUrl.get(row.url);
|
|
@@ -9393,9 +10152,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9393
10152
|
}
|
|
9394
10153
|
}
|
|
9395
10154
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
9396
|
-
db.delete(gscCoverageSnapshots).where(
|
|
10155
|
+
db.delete(gscCoverageSnapshots).where(and9(eq19(gscCoverageSnapshots.projectId, projectId), eq19(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
9397
10156
|
db.insert(gscCoverageSnapshots).values({
|
|
9398
|
-
id:
|
|
10157
|
+
id: crypto19.randomUUID(),
|
|
9399
10158
|
projectId,
|
|
9400
10159
|
syncRunId: runId,
|
|
9401
10160
|
date: snapshotDate,
|
|
@@ -9405,11 +10164,11 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9405
10164
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9406
10165
|
}).run();
|
|
9407
10166
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
9408
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10167
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(runs.id, runId)).run();
|
|
9409
10168
|
log3.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
9410
10169
|
} catch (err) {
|
|
9411
10170
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
9412
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
10171
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(runs.id, runId)).run();
|
|
9413
10172
|
log3.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
9414
10173
|
throw err;
|
|
9415
10174
|
}
|
|
@@ -9468,7 +10227,7 @@ var ProviderRegistry = class {
|
|
|
9468
10227
|
|
|
9469
10228
|
// src/scheduler.ts
|
|
9470
10229
|
import cron from "node-cron";
|
|
9471
|
-
import { eq as
|
|
10230
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
9472
10231
|
var log4 = createLogger("Scheduler");
|
|
9473
10232
|
var Scheduler = class {
|
|
9474
10233
|
db;
|
|
@@ -9480,7 +10239,7 @@ var Scheduler = class {
|
|
|
9480
10239
|
}
|
|
9481
10240
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
9482
10241
|
start() {
|
|
9483
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
10242
|
+
const allSchedules = this.db.select().from(schedules).where(eq20(schedules.enabled, 1)).all();
|
|
9484
10243
|
for (const schedule of allSchedules) {
|
|
9485
10244
|
const missedRunAt = schedule.nextRunAt;
|
|
9486
10245
|
this.registerCronTask(schedule);
|
|
@@ -9505,7 +10264,7 @@ var Scheduler = class {
|
|
|
9505
10264
|
this.stopTask(projectId, existing, "Stopped");
|
|
9506
10265
|
this.tasks.delete(projectId);
|
|
9507
10266
|
}
|
|
9508
|
-
const schedule = this.db.select().from(schedules).where(
|
|
10267
|
+
const schedule = this.db.select().from(schedules).where(eq20(schedules.projectId, projectId)).get();
|
|
9509
10268
|
if (schedule && schedule.enabled === 1) {
|
|
9510
10269
|
this.registerCronTask(schedule);
|
|
9511
10270
|
}
|
|
@@ -9538,13 +10297,13 @@ var Scheduler = class {
|
|
|
9538
10297
|
this.db.update(schedules).set({
|
|
9539
10298
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
9540
10299
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9541
|
-
}).where(
|
|
10300
|
+
}).where(eq20(schedules.id, scheduleId)).run();
|
|
9542
10301
|
const label = schedule.preset ?? cronExpr;
|
|
9543
10302
|
log4.info("cron.registered", { projectId, schedule: label, timezone });
|
|
9544
10303
|
}
|
|
9545
10304
|
triggerRun(scheduleId, projectId) {
|
|
9546
10305
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9547
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
10306
|
+
const currentSchedule = this.db.select().from(schedules).where(eq20(schedules.id, scheduleId)).get();
|
|
9548
10307
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
9549
10308
|
log4.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
9550
10309
|
this.remove(projectId);
|
|
@@ -9552,7 +10311,7 @@ var Scheduler = class {
|
|
|
9552
10311
|
}
|
|
9553
10312
|
const task = this.tasks.get(projectId);
|
|
9554
10313
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
9555
|
-
const project = this.db.select().from(projects).where(
|
|
10314
|
+
const project = this.db.select().from(projects).where(eq20(projects.id, projectId)).get();
|
|
9556
10315
|
if (!project) {
|
|
9557
10316
|
log4.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
9558
10317
|
this.remove(projectId);
|
|
@@ -9569,7 +10328,7 @@ var Scheduler = class {
|
|
|
9569
10328
|
this.db.update(schedules).set({
|
|
9570
10329
|
nextRunAt,
|
|
9571
10330
|
updatedAt: now
|
|
9572
|
-
}).where(
|
|
10331
|
+
}).where(eq20(schedules.id, currentSchedule.id)).run();
|
|
9573
10332
|
return;
|
|
9574
10333
|
}
|
|
9575
10334
|
const runId = queueResult.runId;
|
|
@@ -9577,7 +10336,7 @@ var Scheduler = class {
|
|
|
9577
10336
|
lastRunAt: now,
|
|
9578
10337
|
nextRunAt,
|
|
9579
10338
|
updatedAt: now
|
|
9580
|
-
}).where(
|
|
10339
|
+
}).where(eq20(schedules.id, currentSchedule.id)).run();
|
|
9581
10340
|
const scheduleProviders = JSON.parse(currentSchedule.providers);
|
|
9582
10341
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
9583
10342
|
log4.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -9586,8 +10345,8 @@ var Scheduler = class {
|
|
|
9586
10345
|
};
|
|
9587
10346
|
|
|
9588
10347
|
// src/notifier.ts
|
|
9589
|
-
import { eq as
|
|
9590
|
-
import
|
|
10348
|
+
import { eq as eq21, desc as desc7, and as and10, or as or2 } from "drizzle-orm";
|
|
10349
|
+
import crypto20 from "crypto";
|
|
9591
10350
|
var log5 = createLogger("Notifier");
|
|
9592
10351
|
var Notifier = class {
|
|
9593
10352
|
db;
|
|
@@ -9599,18 +10358,18 @@ var Notifier = class {
|
|
|
9599
10358
|
/** Called after a run completes (success, partial, or failed). */
|
|
9600
10359
|
async onRunCompleted(runId, projectId) {
|
|
9601
10360
|
log5.info("run.completed", { runId, projectId });
|
|
9602
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10361
|
+
const notifs = this.db.select().from(notifications).where(eq21(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
9603
10362
|
if (notifs.length === 0) {
|
|
9604
10363
|
log5.info("notifications.none-enabled", { projectId });
|
|
9605
10364
|
return;
|
|
9606
10365
|
}
|
|
9607
10366
|
log5.info("notifications.found", { projectId, count: notifs.length });
|
|
9608
|
-
const run = this.db.select().from(runs).where(
|
|
10367
|
+
const run = this.db.select().from(runs).where(eq21(runs.id, runId)).get();
|
|
9609
10368
|
if (!run) {
|
|
9610
10369
|
log5.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
9611
10370
|
return;
|
|
9612
10371
|
}
|
|
9613
|
-
const project = this.db.select().from(projects).where(
|
|
10372
|
+
const project = this.db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
9614
10373
|
if (!project) {
|
|
9615
10374
|
log5.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
9616
10375
|
return;
|
|
@@ -9650,11 +10409,11 @@ var Notifier = class {
|
|
|
9650
10409
|
}
|
|
9651
10410
|
computeTransitions(runId, projectId) {
|
|
9652
10411
|
const recentRuns = this.db.select().from(runs).where(
|
|
9653
|
-
|
|
9654
|
-
|
|
9655
|
-
or2(
|
|
10412
|
+
and10(
|
|
10413
|
+
eq21(runs.projectId, projectId),
|
|
10414
|
+
or2(eq21(runs.status, "completed"), eq21(runs.status, "partial"))
|
|
9656
10415
|
)
|
|
9657
|
-
).orderBy(
|
|
10416
|
+
).orderBy(desc7(runs.createdAt)).limit(2).all();
|
|
9658
10417
|
if (recentRuns.length < 2) return [];
|
|
9659
10418
|
const currentRunId = recentRuns[0].id;
|
|
9660
10419
|
const previousRunId = recentRuns[1].id;
|
|
@@ -9664,12 +10423,12 @@ var Notifier = class {
|
|
|
9664
10423
|
keyword: keywords.keyword,
|
|
9665
10424
|
provider: querySnapshots.provider,
|
|
9666
10425
|
citationState: querySnapshots.citationState
|
|
9667
|
-
}).from(querySnapshots).leftJoin(keywords,
|
|
10426
|
+
}).from(querySnapshots).leftJoin(keywords, eq21(querySnapshots.keywordId, keywords.id)).where(eq21(querySnapshots.runId, currentRunId)).all();
|
|
9668
10427
|
const previousSnapshots = this.db.select({
|
|
9669
10428
|
keywordId: querySnapshots.keywordId,
|
|
9670
10429
|
provider: querySnapshots.provider,
|
|
9671
10430
|
citationState: querySnapshots.citationState
|
|
9672
|
-
}).from(querySnapshots).where(
|
|
10431
|
+
}).from(querySnapshots).where(eq21(querySnapshots.runId, previousRunId)).all();
|
|
9673
10432
|
const prevMap = /* @__PURE__ */ new Map();
|
|
9674
10433
|
for (const s of previousSnapshots) {
|
|
9675
10434
|
prevMap.set(`${s.keywordId}:${s.provider}`, s.citationState);
|
|
@@ -9726,7 +10485,7 @@ var Notifier = class {
|
|
|
9726
10485
|
}
|
|
9727
10486
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
9728
10487
|
this.db.insert(auditLog).values({
|
|
9729
|
-
id:
|
|
10488
|
+
id: crypto20.randomUUID(),
|
|
9730
10489
|
projectId,
|
|
9731
10490
|
actor: "scheduler",
|
|
9732
10491
|
action: `notification.${status}`,
|
|
@@ -9997,7 +10756,22 @@ async function createServer(opts) {
|
|
|
9997
10756
|
return true;
|
|
9998
10757
|
}
|
|
9999
10758
|
};
|
|
10000
|
-
const
|
|
10759
|
+
const ga4CredentialStore = {
|
|
10760
|
+
getConnection: (projectName) => {
|
|
10761
|
+
return getGa4Connection(opts.config, projectName);
|
|
10762
|
+
},
|
|
10763
|
+
upsertConnection: (connection) => {
|
|
10764
|
+
const updated = upsertGa4Connection(opts.config, connection);
|
|
10765
|
+
saveConfig(opts.config);
|
|
10766
|
+
return updated;
|
|
10767
|
+
},
|
|
10768
|
+
deleteConnection: (projectName) => {
|
|
10769
|
+
const removed = removeGa4Connection(opts.config, projectName);
|
|
10770
|
+
if (removed) saveConfig(opts.config);
|
|
10771
|
+
return removed;
|
|
10772
|
+
}
|
|
10773
|
+
};
|
|
10774
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto21.randomBytes(32).toString("hex");
|
|
10001
10775
|
const googleConnectionStore = {
|
|
10002
10776
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
10003
10777
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -10070,6 +10844,7 @@ async function createServer(opts) {
|
|
|
10070
10844
|
googleSettingsSummary,
|
|
10071
10845
|
bingSettingsSummary,
|
|
10072
10846
|
bingConnectionStore,
|
|
10847
|
+
ga4CredentialStore,
|
|
10073
10848
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
10074
10849
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
10075
10850
|
app.log.error({ runId, err }, "Job runner failed");
|
|
@@ -10125,7 +10900,7 @@ async function createServer(opts) {
|
|
|
10125
10900
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
10126
10901
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
10127
10902
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
10128
|
-
id:
|
|
10903
|
+
id: crypto21.randomUUID(),
|
|
10129
10904
|
projectId,
|
|
10130
10905
|
actor: "api",
|
|
10131
10906
|
action: existing ? "provider.updated" : "provider.created",
|