@ainyc/canonry 1.21.1 → 1.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -10
- package/assets/assets/index-BidvmvWJ.css +1 -0
- package/assets/assets/index-YuErKoaN.js +246 -0
- package/assets/index.html +2 -2
- package/dist/{chunk-2JBQ7NMO.js → chunk-Q5REKIL6.js} +866 -115
- package/dist/cli.js +331 -67
- package/dist/index.d.ts +12 -0
- package/dist/index.js +1 -1
- package/package.json +7 -7
- package/assets/assets/index-7DjD4Oje.css +0 -1
- package/assets/assets/index-BFA6dYNt.js +0 -246
|
@@ -170,7 +170,7 @@ function trackEvent(event, properties) {
|
|
|
170
170
|
|
|
171
171
|
// src/server.ts
|
|
172
172
|
import { createRequire as createRequire2 } from "module";
|
|
173
|
-
import
|
|
173
|
+
import crypto21 from "crypto";
|
|
174
174
|
import fs5 from "fs";
|
|
175
175
|
import path6 from "path";
|
|
176
176
|
import { fileURLToPath } from "url";
|
|
@@ -711,6 +711,37 @@ function categoryLabel(category) {
|
|
|
711
711
|
return CATEGORY_LABELS[category];
|
|
712
712
|
}
|
|
713
713
|
|
|
714
|
+
// ../contracts/src/ga.ts
|
|
715
|
+
import { z as z9 } from "zod";
|
|
716
|
+
var ga4ConnectionDtoSchema = z9.object({
|
|
717
|
+
id: z9.string(),
|
|
718
|
+
projectId: z9.string(),
|
|
719
|
+
propertyId: z9.string(),
|
|
720
|
+
clientEmail: z9.string(),
|
|
721
|
+
connected: z9.boolean(),
|
|
722
|
+
createdAt: z9.string(),
|
|
723
|
+
updatedAt: z9.string()
|
|
724
|
+
});
|
|
725
|
+
var ga4TrafficSnapshotDtoSchema = z9.object({
|
|
726
|
+
date: z9.string(),
|
|
727
|
+
landingPage: z9.string(),
|
|
728
|
+
sessions: z9.number(),
|
|
729
|
+
organicSessions: z9.number(),
|
|
730
|
+
users: z9.number()
|
|
731
|
+
});
|
|
732
|
+
var ga4TrafficSummaryDtoSchema = z9.object({
|
|
733
|
+
totalSessions: z9.number(),
|
|
734
|
+
totalOrganicSessions: z9.number(),
|
|
735
|
+
totalUsers: z9.number(),
|
|
736
|
+
topPages: z9.array(z9.object({
|
|
737
|
+
landingPage: z9.string(),
|
|
738
|
+
sessions: z9.number(),
|
|
739
|
+
organicSessions: z9.number(),
|
|
740
|
+
users: z9.number()
|
|
741
|
+
})),
|
|
742
|
+
lastSyncedAt: z9.string().nullable()
|
|
743
|
+
});
|
|
744
|
+
|
|
714
745
|
// ../api-routes/src/auth.ts
|
|
715
746
|
import crypto2 from "crypto";
|
|
716
747
|
import { eq } from "drizzle-orm";
|
|
@@ -730,6 +761,8 @@ __export(schema_exports, {
|
|
|
730
761
|
bingKeywordStats: () => bingKeywordStats,
|
|
731
762
|
bingUrlInspections: () => bingUrlInspections,
|
|
732
763
|
competitors: () => competitors,
|
|
764
|
+
gaConnections: () => gaConnections,
|
|
765
|
+
gaTrafficSnapshots: () => gaTrafficSnapshots,
|
|
733
766
|
googleConnections: () => googleConnections,
|
|
734
767
|
gscCoverageSnapshots: () => gscCoverageSnapshots,
|
|
735
768
|
gscSearchData: () => gscSearchData,
|
|
@@ -970,6 +1003,30 @@ var bingKeywordStats = sqliteTable("bing_keyword_stats", {
|
|
|
970
1003
|
index("idx_bing_keyword_project").on(table.projectId),
|
|
971
1004
|
index("idx_bing_keyword_query").on(table.query)
|
|
972
1005
|
]);
|
|
1006
|
+
var gaConnections = sqliteTable("ga_connections", {
|
|
1007
|
+
id: text("id").primaryKey(),
|
|
1008
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1009
|
+
propertyId: text("property_id").notNull(),
|
|
1010
|
+
clientEmail: text("client_email").notNull(),
|
|
1011
|
+
privateKey: text("private_key").notNull(),
|
|
1012
|
+
createdAt: text("created_at").notNull(),
|
|
1013
|
+
updatedAt: text("updated_at").notNull()
|
|
1014
|
+
}, (table) => [
|
|
1015
|
+
uniqueIndex("idx_ga_conn_project").on(table.projectId)
|
|
1016
|
+
]);
|
|
1017
|
+
var gaTrafficSnapshots = sqliteTable("ga_traffic_snapshots", {
|
|
1018
|
+
id: text("id").primaryKey(),
|
|
1019
|
+
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
1020
|
+
date: text("date").notNull(),
|
|
1021
|
+
landingPage: text("landing_page").notNull(),
|
|
1022
|
+
sessions: integer("sessions").notNull().default(0),
|
|
1023
|
+
organicSessions: integer("organic_sessions").notNull().default(0),
|
|
1024
|
+
users: integer("users").notNull().default(0),
|
|
1025
|
+
syncedAt: text("synced_at").notNull()
|
|
1026
|
+
}, (table) => [
|
|
1027
|
+
index("idx_ga_traffic_project_date").on(table.projectId, table.date),
|
|
1028
|
+
index("idx_ga_traffic_page").on(table.landingPage)
|
|
1029
|
+
]);
|
|
973
1030
|
var usageCounters = sqliteTable("usage_counters", {
|
|
974
1031
|
id: text("id").primaryKey(),
|
|
975
1032
|
scope: text("scope").notNull(),
|
|
@@ -1247,7 +1304,31 @@ var MIGRATIONS = [
|
|
|
1247
1304
|
created_at TEXT NOT NULL
|
|
1248
1305
|
)`,
|
|
1249
1306
|
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_project ON bing_keyword_stats(project_id)`,
|
|
1250
|
-
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_query ON bing_keyword_stats(query)
|
|
1307
|
+
`CREATE INDEX IF NOT EXISTS idx_bing_keyword_query ON bing_keyword_stats(query)`,
|
|
1308
|
+
// v13: Google Analytics 4 — ga_connections table (service account auth)
|
|
1309
|
+
`CREATE TABLE IF NOT EXISTS ga_connections (
|
|
1310
|
+
id TEXT PRIMARY KEY,
|
|
1311
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1312
|
+
property_id TEXT NOT NULL,
|
|
1313
|
+
client_email TEXT NOT NULL,
|
|
1314
|
+
private_key TEXT NOT NULL,
|
|
1315
|
+
created_at TEXT NOT NULL,
|
|
1316
|
+
updated_at TEXT NOT NULL
|
|
1317
|
+
)`,
|
|
1318
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS idx_ga_conn_project ON ga_connections(project_id)`,
|
|
1319
|
+
// v13: Google Analytics 4 — ga_traffic_snapshots table
|
|
1320
|
+
`CREATE TABLE IF NOT EXISTS ga_traffic_snapshots (
|
|
1321
|
+
id TEXT PRIMARY KEY,
|
|
1322
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1323
|
+
date TEXT NOT NULL,
|
|
1324
|
+
landing_page TEXT NOT NULL,
|
|
1325
|
+
sessions INTEGER NOT NULL DEFAULT 0,
|
|
1326
|
+
organic_sessions INTEGER NOT NULL DEFAULT 0,
|
|
1327
|
+
users INTEGER NOT NULL DEFAULT 0,
|
|
1328
|
+
synced_at TEXT NOT NULL
|
|
1329
|
+
)`,
|
|
1330
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_traffic_project_date ON ga_traffic_snapshots(project_id, date)`,
|
|
1331
|
+
`CREATE INDEX IF NOT EXISTS idx_ga_traffic_page ON ga_traffic_snapshots(landing_page)`
|
|
1251
1332
|
];
|
|
1252
1333
|
function migrate(db) {
|
|
1253
1334
|
const statements = MIGRATION_SQL.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
@@ -4639,6 +4720,105 @@ var routeCatalog = [
|
|
|
4639
4720
|
400: { description: "Bing is not configured for this project." },
|
|
4640
4721
|
404: { description: "Project not found." }
|
|
4641
4722
|
}
|
|
4723
|
+
},
|
|
4724
|
+
// GA4 routes
|
|
4725
|
+
{
|
|
4726
|
+
method: "post",
|
|
4727
|
+
path: "/api/v1/projects/{name}/ga/connect",
|
|
4728
|
+
summary: "Connect Google Analytics 4 via service account",
|
|
4729
|
+
tags: ["ga4"],
|
|
4730
|
+
parameters: [nameParameter],
|
|
4731
|
+
requestBody: {
|
|
4732
|
+
required: true,
|
|
4733
|
+
content: {
|
|
4734
|
+
"application/json": {
|
|
4735
|
+
schema: {
|
|
4736
|
+
type: "object",
|
|
4737
|
+
required: ["propertyId", "keyJson"],
|
|
4738
|
+
properties: {
|
|
4739
|
+
propertyId: stringSchema,
|
|
4740
|
+
keyJson: stringSchema
|
|
4741
|
+
}
|
|
4742
|
+
}
|
|
4743
|
+
}
|
|
4744
|
+
}
|
|
4745
|
+
},
|
|
4746
|
+
responses: {
|
|
4747
|
+
200: { description: "GA4 connection established." },
|
|
4748
|
+
400: { description: "Invalid GA4 connection request." },
|
|
4749
|
+
404: { description: "Project not found." }
|
|
4750
|
+
}
|
|
4751
|
+
},
|
|
4752
|
+
{
|
|
4753
|
+
method: "delete",
|
|
4754
|
+
path: "/api/v1/projects/{name}/ga/disconnect",
|
|
4755
|
+
summary: "Disconnect Google Analytics 4",
|
|
4756
|
+
tags: ["ga4"],
|
|
4757
|
+
parameters: [nameParameter],
|
|
4758
|
+
responses: {
|
|
4759
|
+
204: { description: "GA4 connection deleted." },
|
|
4760
|
+
404: { description: "Project or connection not found." }
|
|
4761
|
+
}
|
|
4762
|
+
},
|
|
4763
|
+
{
|
|
4764
|
+
method: "get",
|
|
4765
|
+
path: "/api/v1/projects/{name}/ga/status",
|
|
4766
|
+
summary: "Get GA4 connection status",
|
|
4767
|
+
tags: ["ga4"],
|
|
4768
|
+
parameters: [nameParameter],
|
|
4769
|
+
responses: {
|
|
4770
|
+
200: { description: "GA4 status returned." },
|
|
4771
|
+
404: { description: "Project not found." }
|
|
4772
|
+
}
|
|
4773
|
+
},
|
|
4774
|
+
{
|
|
4775
|
+
method: "post",
|
|
4776
|
+
path: "/api/v1/projects/{name}/ga/sync",
|
|
4777
|
+
summary: "Sync GA4 traffic data",
|
|
4778
|
+
tags: ["ga4"],
|
|
4779
|
+
parameters: [nameParameter],
|
|
4780
|
+
requestBody: {
|
|
4781
|
+
required: false,
|
|
4782
|
+
content: {
|
|
4783
|
+
"application/json": {
|
|
4784
|
+
schema: {
|
|
4785
|
+
type: "object",
|
|
4786
|
+
properties: {
|
|
4787
|
+
days: integerSchema
|
|
4788
|
+
}
|
|
4789
|
+
}
|
|
4790
|
+
}
|
|
4791
|
+
}
|
|
4792
|
+
},
|
|
4793
|
+
responses: {
|
|
4794
|
+
200: { description: "GA4 sync completed." },
|
|
4795
|
+
400: { description: "GA4 is not connected." },
|
|
4796
|
+
404: { description: "Project not found." }
|
|
4797
|
+
}
|
|
4798
|
+
},
|
|
4799
|
+
{
|
|
4800
|
+
method: "get",
|
|
4801
|
+
path: "/api/v1/projects/{name}/ga/traffic",
|
|
4802
|
+
summary: "Get GA4 landing page traffic",
|
|
4803
|
+
tags: ["ga4"],
|
|
4804
|
+
parameters: [nameParameter, limitQueryParameter],
|
|
4805
|
+
responses: {
|
|
4806
|
+
200: { description: "GA4 traffic data returned." },
|
|
4807
|
+
400: { description: "GA4 is not connected." },
|
|
4808
|
+
404: { description: "Project not found." }
|
|
4809
|
+
}
|
|
4810
|
+
},
|
|
4811
|
+
{
|
|
4812
|
+
method: "get",
|
|
4813
|
+
path: "/api/v1/projects/{name}/ga/coverage",
|
|
4814
|
+
summary: "Get GA4 page coverage with traffic overlay",
|
|
4815
|
+
tags: ["ga4"],
|
|
4816
|
+
parameters: [nameParameter],
|
|
4817
|
+
responses: {
|
|
4818
|
+
200: { description: "GA4 coverage data returned." },
|
|
4819
|
+
400: { description: "GA4 is not connected." },
|
|
4820
|
+
404: { description: "Project not found." }
|
|
4821
|
+
}
|
|
4642
4822
|
}
|
|
4643
4823
|
];
|
|
4644
4824
|
function buildOpenApiDocument(info = {}) {
|
|
@@ -5221,6 +5401,11 @@ async function refreshAccessToken(clientId, clientSecret, currentRefreshToken) {
|
|
|
5221
5401
|
}
|
|
5222
5402
|
|
|
5223
5403
|
// ../integration-google/src/gsc-client.ts
|
|
5404
|
+
function gscClientLog(level, action, ctx) {
|
|
5405
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GscClient", action, ...ctx };
|
|
5406
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
5407
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
5408
|
+
}
|
|
5224
5409
|
async function gscFetch(accessToken, url, opts) {
|
|
5225
5410
|
const method = opts?.method ?? "GET";
|
|
5226
5411
|
const headers = {
|
|
@@ -5233,13 +5418,18 @@ async function gscFetch(accessToken, url, opts) {
|
|
|
5233
5418
|
body: opts?.body != null ? JSON.stringify(opts.body) : void 0
|
|
5234
5419
|
});
|
|
5235
5420
|
if (res.status === 401) {
|
|
5421
|
+
const body = await res.text().catch(() => "");
|
|
5422
|
+
gscClientLog("error", "http.auth-expired", { url, method, httpStatus: 401, responseBody: body });
|
|
5236
5423
|
throw new GoogleApiError("Access token expired or revoked", 401);
|
|
5237
5424
|
}
|
|
5238
5425
|
if (res.status === 429) {
|
|
5426
|
+
const body = await res.text().catch(() => "");
|
|
5427
|
+
gscClientLog("error", "http.rate-limited", { url, method, httpStatus: 429, responseBody: body });
|
|
5239
5428
|
throw new GoogleApiError("Google API rate limit exceeded", 429);
|
|
5240
5429
|
}
|
|
5241
5430
|
if (!res.ok) {
|
|
5242
5431
|
const body = await res.text();
|
|
5432
|
+
gscClientLog("error", "http.error", { url, method, httpStatus: res.status, responseBody: body });
|
|
5243
5433
|
throw new GoogleApiError(`GSC API error (${res.status}): ${body}`, res.status);
|
|
5244
5434
|
}
|
|
5245
5435
|
return await res.json();
|
|
@@ -6045,6 +6235,11 @@ var BingApiError = class extends Error {
|
|
|
6045
6235
|
};
|
|
6046
6236
|
|
|
6047
6237
|
// ../integration-bing/src/bing-client.ts
|
|
6238
|
+
function bingClientLog(level, action, ctx) {
|
|
6239
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "BingClient", action, ...ctx };
|
|
6240
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6241
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6242
|
+
}
|
|
6048
6243
|
async function bingFetch(apiKey, endpoint, opts) {
|
|
6049
6244
|
const method = opts?.method ?? "GET";
|
|
6050
6245
|
const separator = endpoint.includes("?") ? "&" : "?";
|
|
@@ -6058,13 +6253,18 @@ async function bingFetch(apiKey, endpoint, opts) {
|
|
|
6058
6253
|
body: opts?.body != null ? JSON.stringify(opts.body) : void 0
|
|
6059
6254
|
});
|
|
6060
6255
|
if (res.status === 401 || res.status === 403) {
|
|
6256
|
+
const body = await res.text().catch(() => "");
|
|
6257
|
+
bingClientLog("error", "http.auth-failed", { endpoint, method, httpStatus: res.status, responseBody: body });
|
|
6061
6258
|
throw new BingApiError("Bing API key is invalid or unauthorized", res.status);
|
|
6062
6259
|
}
|
|
6063
6260
|
if (res.status === 429) {
|
|
6261
|
+
const body = await res.text().catch(() => "");
|
|
6262
|
+
bingClientLog("error", "http.rate-limited", { endpoint, method, httpStatus: 429, responseBody: body });
|
|
6064
6263
|
throw new BingApiError("Bing API rate limit exceeded", 429);
|
|
6065
6264
|
}
|
|
6066
6265
|
if (!res.ok) {
|
|
6067
6266
|
const body = await res.text();
|
|
6267
|
+
bingClientLog("error", "http.error", { endpoint, method, httpStatus: res.status, responseBody: body });
|
|
6068
6268
|
throw new BingApiError(`Bing API error (${res.status}): ${body}`, res.status);
|
|
6069
6269
|
}
|
|
6070
6270
|
const text2 = await res.text();
|
|
@@ -6112,6 +6312,11 @@ async function getKeywordStats(apiKey, siteUrl) {
|
|
|
6112
6312
|
}
|
|
6113
6313
|
|
|
6114
6314
|
// ../api-routes/src/bing.ts
|
|
6315
|
+
function bingLog(level, action, ctx) {
|
|
6316
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "BingRoutes", action, ...ctx };
|
|
6317
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6318
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6319
|
+
}
|
|
6115
6320
|
async function bingRoutes(app, opts) {
|
|
6116
6321
|
function requireConnectionStore(reply) {
|
|
6117
6322
|
if (opts.bingConnectionStore) return opts.bingConnectionStore;
|
|
@@ -6140,8 +6345,10 @@ async function bingRoutes(app, opts) {
|
|
|
6140
6345
|
let sites;
|
|
6141
6346
|
try {
|
|
6142
6347
|
sites = await getSites(apiKey);
|
|
6348
|
+
bingLog("info", "connect.verify-key", { domain: project.canonicalDomain, siteCount: sites.length });
|
|
6143
6349
|
} catch (e) {
|
|
6144
6350
|
const msg = e instanceof Error ? e.message : String(e);
|
|
6351
|
+
bingLog("error", "connect.verify-key-failed", { domain: project.canonicalDomain, error: msg });
|
|
6145
6352
|
const err = validationError(`Failed to verify Bing API key: ${msg}`);
|
|
6146
6353
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6147
6354
|
}
|
|
@@ -6307,7 +6514,15 @@ async function bingRoutes(app, opts) {
|
|
|
6307
6514
|
const err = validationError("url is required");
|
|
6308
6515
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6309
6516
|
}
|
|
6310
|
-
|
|
6517
|
+
let result;
|
|
6518
|
+
try {
|
|
6519
|
+
result = await getUrlInfo(conn.apiKey, conn.siteUrl, url);
|
|
6520
|
+
bingLog("info", "inspect-url.result", { domain: project.canonicalDomain, url, httpCode: result.HttpCode ?? null, inIndex: result.InIndex ?? null, lastCrawledDate: result.LastCrawledDate ?? null });
|
|
6521
|
+
} catch (e) {
|
|
6522
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
6523
|
+
bingLog("error", "inspect-url.failed", { domain: project.canonicalDomain, url, error: msg });
|
|
6524
|
+
throw e;
|
|
6525
|
+
}
|
|
6311
6526
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6312
6527
|
const id = crypto14.randomUUID();
|
|
6313
6528
|
app.db.insert(bingUrlInspections).values({
|
|
@@ -6371,6 +6586,7 @@ async function bingRoutes(app, opts) {
|
|
|
6371
6586
|
return reply.status(err.statusCode).send(err.toJSON());
|
|
6372
6587
|
}
|
|
6373
6588
|
const results = [];
|
|
6589
|
+
bingLog("info", "index-submit.start", { domain: project.canonicalDomain, siteUrl: conn.siteUrl, urlCount: urlsToSubmit.length, allUnindexed: !!request.body?.allUnindexed });
|
|
6374
6590
|
if (urlsToSubmit.length > 1) {
|
|
6375
6591
|
for (let i = 0; i < urlsToSubmit.length; i += BING_SUBMIT_URL_BATCH_LIMIT) {
|
|
6376
6592
|
const batch = urlsToSubmit.slice(i, i + BING_SUBMIT_URL_BATCH_LIMIT);
|
|
@@ -6380,12 +6596,14 @@ async function bingRoutes(app, opts) {
|
|
|
6380
6596
|
for (const url of batch) {
|
|
6381
6597
|
results.push({ url, status: "success", submittedAt: now });
|
|
6382
6598
|
}
|
|
6599
|
+
bingLog("info", "index-submit.batch-ok", { domain: project.canonicalDomain, batchSize: batch.length, urls: batch });
|
|
6383
6600
|
} catch (e) {
|
|
6384
6601
|
const msg = e instanceof Error ? e.message : String(e);
|
|
6385
6602
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6386
6603
|
for (const url of batch) {
|
|
6387
6604
|
results.push({ url, status: "error", submittedAt: now, error: msg });
|
|
6388
6605
|
}
|
|
6606
|
+
bingLog("error", "index-submit.batch-failed", { domain: project.canonicalDomain, batchSize: batch.length, urls: batch, error: msg });
|
|
6389
6607
|
}
|
|
6390
6608
|
}
|
|
6391
6609
|
} else {
|
|
@@ -6393,13 +6611,16 @@ async function bingRoutes(app, opts) {
|
|
|
6393
6611
|
try {
|
|
6394
6612
|
await submitUrl(conn.apiKey, conn.siteUrl, url);
|
|
6395
6613
|
results.push({ url, status: "success", submittedAt: (/* @__PURE__ */ new Date()).toISOString() });
|
|
6614
|
+
bingLog("info", "index-submit.ok", { domain: project.canonicalDomain, url });
|
|
6396
6615
|
} catch (e) {
|
|
6397
6616
|
const msg = e instanceof Error ? e.message : String(e);
|
|
6398
6617
|
results.push({ url, status: "error", submittedAt: (/* @__PURE__ */ new Date()).toISOString(), error: msg });
|
|
6618
|
+
bingLog("error", "index-submit.failed", { domain: project.canonicalDomain, url, error: msg });
|
|
6399
6619
|
}
|
|
6400
6620
|
}
|
|
6401
6621
|
const succeeded = results.filter((r) => r.status === "success").length;
|
|
6402
6622
|
const failed = results.filter((r) => r.status === "error").length;
|
|
6623
|
+
bingLog("info", "index-submit.complete", { domain: project.canonicalDomain, total: results.length, succeeded, failed });
|
|
6403
6624
|
return {
|
|
6404
6625
|
summary: { total: results.length, succeeded, failed },
|
|
6405
6626
|
results
|
|
@@ -6598,6 +6819,439 @@ async function cdpRoutes(app, opts) {
|
|
|
6598
6819
|
);
|
|
6599
6820
|
}
|
|
6600
6821
|
|
|
6822
|
+
// ../api-routes/src/ga.ts
|
|
6823
|
+
import crypto16 from "crypto";
|
|
6824
|
+
import { eq as eq16, desc as desc6, and as and6, sql as sql3 } from "drizzle-orm";
|
|
6825
|
+
|
|
6826
|
+
// ../integration-google-analytics/src/ga4-client.ts
|
|
6827
|
+
import crypto15 from "crypto";
|
|
6828
|
+
|
|
6829
|
+
// ../integration-google-analytics/src/constants.ts
|
|
6830
|
+
var GA4_DATA_API_BASE = "https://analyticsdata.googleapis.com/v1beta";
|
|
6831
|
+
var GA4_SCOPE = "https://www.googleapis.com/auth/analytics.readonly";
|
|
6832
|
+
var GOOGLE_TOKEN_URL2 = "https://oauth2.googleapis.com/token";
|
|
6833
|
+
var GA4_DEFAULT_SYNC_DAYS = 30;
|
|
6834
|
+
var GA4_MAX_SYNC_DAYS = 90;
|
|
6835
|
+
|
|
6836
|
+
// ../integration-google-analytics/src/types.ts
|
|
6837
|
+
var GA4ApiError = class extends Error {
|
|
6838
|
+
status;
|
|
6839
|
+
constructor(message, status) {
|
|
6840
|
+
super(message);
|
|
6841
|
+
this.name = "GA4ApiError";
|
|
6842
|
+
this.status = status;
|
|
6843
|
+
}
|
|
6844
|
+
};
|
|
6845
|
+
|
|
6846
|
+
// ../integration-google-analytics/src/ga4-client.ts
|
|
6847
|
+
function ga4Log(level, action, ctx) {
|
|
6848
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Client", action, ...ctx };
|
|
6849
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
6850
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
6851
|
+
}
|
|
6852
|
+
function createServiceAccountJwt(clientEmail, privateKey, scope) {
|
|
6853
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
6854
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
6855
|
+
const payload = {
|
|
6856
|
+
iss: clientEmail,
|
|
6857
|
+
scope,
|
|
6858
|
+
aud: GOOGLE_TOKEN_URL2,
|
|
6859
|
+
iat: now,
|
|
6860
|
+
exp: now + 3600
|
|
6861
|
+
// 1 hour
|
|
6862
|
+
};
|
|
6863
|
+
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
6864
|
+
const headerB64 = encode(header);
|
|
6865
|
+
const payloadB64 = encode(payload);
|
|
6866
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
6867
|
+
const sign = crypto15.createSign("RSA-SHA256");
|
|
6868
|
+
sign.update(signingInput);
|
|
6869
|
+
const signature = sign.sign(privateKey, "base64url");
|
|
6870
|
+
return `${signingInput}.${signature}`;
|
|
6871
|
+
}
|
|
6872
|
+
async function getAccessToken(clientEmail, privateKey) {
|
|
6873
|
+
const jwt = createServiceAccountJwt(clientEmail, privateKey, GA4_SCOPE);
|
|
6874
|
+
const res = await fetch(GOOGLE_TOKEN_URL2, {
|
|
6875
|
+
method: "POST",
|
|
6876
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
6877
|
+
body: new URLSearchParams({
|
|
6878
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
6879
|
+
assertion: jwt
|
|
6880
|
+
})
|
|
6881
|
+
});
|
|
6882
|
+
if (!res.ok) {
|
|
6883
|
+
const body = await res.text().catch(() => "");
|
|
6884
|
+
ga4Log("error", "token.failed", { httpStatus: res.status, responseBody: body });
|
|
6885
|
+
throw new GA4ApiError(`Failed to get access token: ${body}`, res.status);
|
|
6886
|
+
}
|
|
6887
|
+
const data = await res.json();
|
|
6888
|
+
return data.access_token;
|
|
6889
|
+
}
|
|
6890
|
+
async function runReport(accessToken, propertyId, request) {
|
|
6891
|
+
const url = `${GA4_DATA_API_BASE}/properties/${propertyId}:runReport`;
|
|
6892
|
+
const res = await fetch(url, {
|
|
6893
|
+
method: "POST",
|
|
6894
|
+
headers: {
|
|
6895
|
+
"Authorization": `Bearer ${accessToken}`,
|
|
6896
|
+
"Content-Type": "application/json"
|
|
6897
|
+
},
|
|
6898
|
+
body: JSON.stringify(request)
|
|
6899
|
+
});
|
|
6900
|
+
if (res.status === 401 || res.status === 403) {
|
|
6901
|
+
const body = await res.text().catch(() => "");
|
|
6902
|
+
let detail = "";
|
|
6903
|
+
try {
|
|
6904
|
+
const parsed = JSON.parse(body);
|
|
6905
|
+
if (parsed.error?.status === "SERVICE_DISABLED") {
|
|
6906
|
+
detail = " The Google Analytics Data API is not enabled for this GCP project. Enable it at: https://console.developers.google.com/apis/api/analyticsdata.googleapis.com/overview";
|
|
6907
|
+
} else if (parsed.error?.message) {
|
|
6908
|
+
detail = ` ${parsed.error.message}`;
|
|
6909
|
+
}
|
|
6910
|
+
} catch {
|
|
6911
|
+
if (body.length < 200) detail = ` ${body}`;
|
|
6912
|
+
}
|
|
6913
|
+
ga4Log("error", "report.auth-failed", { propertyId, httpStatus: res.status, responseBody: body });
|
|
6914
|
+
throw new GA4ApiError(
|
|
6915
|
+
`GA4 API authentication failed \u2014 check service account permissions.${detail}`,
|
|
6916
|
+
res.status
|
|
6917
|
+
);
|
|
6918
|
+
}
|
|
6919
|
+
if (res.status === 429) {
|
|
6920
|
+
ga4Log("error", "report.rate-limited", { propertyId });
|
|
6921
|
+
throw new GA4ApiError("GA4 API rate limit exceeded", 429);
|
|
6922
|
+
}
|
|
6923
|
+
if (!res.ok) {
|
|
6924
|
+
const body = await res.text();
|
|
6925
|
+
ga4Log("error", "report.error", { propertyId, httpStatus: res.status, responseBody: body });
|
|
6926
|
+
throw new GA4ApiError(`GA4 API error (${res.status}): ${body}`, res.status);
|
|
6927
|
+
}
|
|
6928
|
+
return await res.json();
|
|
6929
|
+
}
|
|
6930
|
+
function formatDate(d) {
|
|
6931
|
+
return d.toISOString().split("T")[0];
|
|
6932
|
+
}
|
|
6933
|
+
async function fetchTrafficByLandingPage(accessToken, propertyId, days) {
|
|
6934
|
+
const syncDays = Math.min(Math.max(1, days ?? GA4_DEFAULT_SYNC_DAYS), GA4_MAX_SYNC_DAYS);
|
|
6935
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
6936
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
6937
|
+
startDate.setDate(startDate.getDate() - syncDays);
|
|
6938
|
+
ga4Log("info", "fetch-traffic.start", { propertyId, days: syncDays });
|
|
6939
|
+
const PAGE_SIZE = 1e4;
|
|
6940
|
+
const rows = [];
|
|
6941
|
+
let offset = 0;
|
|
6942
|
+
while (true) {
|
|
6943
|
+
const request = {
|
|
6944
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
6945
|
+
dimensions: [
|
|
6946
|
+
{ name: "date" },
|
|
6947
|
+
{ name: "landingPagePlusQueryString" }
|
|
6948
|
+
],
|
|
6949
|
+
metrics: [
|
|
6950
|
+
{ name: "sessions" },
|
|
6951
|
+
{ name: "totalUsers" }
|
|
6952
|
+
],
|
|
6953
|
+
limit: PAGE_SIZE,
|
|
6954
|
+
offset
|
|
6955
|
+
};
|
|
6956
|
+
const response = await runReport(accessToken, propertyId, request);
|
|
6957
|
+
const pageRows = (response.rows ?? []).map((row) => ({
|
|
6958
|
+
date: row.dimensionValues[0].value,
|
|
6959
|
+
landingPage: row.dimensionValues[1].value,
|
|
6960
|
+
sessions: parseInt(row.metricValues[0].value, 10) || 0,
|
|
6961
|
+
organicSessions: 0,
|
|
6962
|
+
// populated by organic-only pass below
|
|
6963
|
+
users: parseInt(row.metricValues[1].value, 10) || 0
|
|
6964
|
+
}));
|
|
6965
|
+
rows.push(...pageRows);
|
|
6966
|
+
const totalRows = response.rowCount ?? 0;
|
|
6967
|
+
offset += pageRows.length;
|
|
6968
|
+
if (pageRows.length < PAGE_SIZE || offset >= totalRows) break;
|
|
6969
|
+
}
|
|
6970
|
+
const organicMap = /* @__PURE__ */ new Map();
|
|
6971
|
+
let organicOffset = 0;
|
|
6972
|
+
while (true) {
|
|
6973
|
+
const organicRequest = {
|
|
6974
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
6975
|
+
dimensions: [{ name: "date" }, { name: "landingPagePlusQueryString" }],
|
|
6976
|
+
metrics: [{ name: "sessions" }],
|
|
6977
|
+
dimensionFilter: {
|
|
6978
|
+
filter: {
|
|
6979
|
+
fieldName: "sessionDefaultChannelGrouping",
|
|
6980
|
+
stringFilter: { matchType: "EXACT", value: "Organic Search" }
|
|
6981
|
+
}
|
|
6982
|
+
},
|
|
6983
|
+
limit: 1e4,
|
|
6984
|
+
offset: organicOffset
|
|
6985
|
+
};
|
|
6986
|
+
const organicResponse = await runReport(accessToken, propertyId, organicRequest);
|
|
6987
|
+
for (const row of organicResponse.rows ?? []) {
|
|
6988
|
+
const key = `${row.dimensionValues[0].value}::${row.dimensionValues[1].value}`;
|
|
6989
|
+
organicMap.set(key, parseInt(row.metricValues[0].value, 10) || 0);
|
|
6990
|
+
}
|
|
6991
|
+
const total = organicResponse.rowCount ?? 0;
|
|
6992
|
+
organicOffset += (organicResponse.rows ?? []).length;
|
|
6993
|
+
if ((organicResponse.rows ?? []).length < 1e4 || organicOffset >= total) break;
|
|
6994
|
+
}
|
|
6995
|
+
for (const row of rows) {
|
|
6996
|
+
const key = `${row.date}::${row.landingPage}`;
|
|
6997
|
+
row.organicSessions = organicMap.get(key) ?? 0;
|
|
6998
|
+
}
|
|
6999
|
+
for (const row of rows) {
|
|
7000
|
+
if (row.date.length === 8 && !row.date.includes("-")) {
|
|
7001
|
+
row.date = `${row.date.slice(0, 4)}-${row.date.slice(4, 6)}-${row.date.slice(6, 8)}`;
|
|
7002
|
+
}
|
|
7003
|
+
}
|
|
7004
|
+
ga4Log("info", "fetch-traffic.done", { propertyId, rowCount: rows.length });
|
|
7005
|
+
return rows;
|
|
7006
|
+
}
|
|
7007
|
+
async function verifyConnection(clientEmail, privateKey, propertyId) {
|
|
7008
|
+
const accessToken = await getAccessToken(clientEmail, privateKey);
|
|
7009
|
+
const endDate = /* @__PURE__ */ new Date();
|
|
7010
|
+
const startDate = /* @__PURE__ */ new Date();
|
|
7011
|
+
startDate.setDate(startDate.getDate() - 1);
|
|
7012
|
+
await runReport(accessToken, propertyId, {
|
|
7013
|
+
dateRanges: [{ startDate: formatDate(startDate), endDate: formatDate(endDate) }],
|
|
7014
|
+
dimensions: [{ name: "date" }],
|
|
7015
|
+
metrics: [{ name: "sessions" }],
|
|
7016
|
+
limit: 1
|
|
7017
|
+
});
|
|
7018
|
+
return true;
|
|
7019
|
+
}
|
|
7020
|
+
|
|
7021
|
+
// ../api-routes/src/ga.ts
|
|
7022
|
+
function gaLog(level, action, ctx) {
|
|
7023
|
+
const entry = { ts: (/* @__PURE__ */ new Date()).toISOString(), level, module: "GA4Routes", action, ...ctx };
|
|
7024
|
+
const stream = level === "error" ? process.stderr : process.stdout;
|
|
7025
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
7026
|
+
}
|
|
7027
|
+
async function ga4Routes(app, opts) {
|
|
7028
|
+
function requireCredentialStore(reply) {
|
|
7029
|
+
if (opts.ga4CredentialStore) return opts.ga4CredentialStore;
|
|
7030
|
+
const err = validationError("GA4 credential storage is not configured for this deployment");
|
|
7031
|
+
reply.status(err.statusCode).send(err.toJSON());
|
|
7032
|
+
return null;
|
|
7033
|
+
}
|
|
7034
|
+
app.post("/projects/:name/ga/connect", async (request, reply) => {
|
|
7035
|
+
const store = requireCredentialStore(reply);
|
|
7036
|
+
if (!store) return;
|
|
7037
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7038
|
+
const { propertyId, keyJson } = request.body ?? {};
|
|
7039
|
+
if (!propertyId || typeof propertyId !== "string") {
|
|
7040
|
+
const err = validationError("propertyId is required");
|
|
7041
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7042
|
+
}
|
|
7043
|
+
let clientEmail;
|
|
7044
|
+
let privateKey;
|
|
7045
|
+
if (keyJson && typeof keyJson === "string") {
|
|
7046
|
+
try {
|
|
7047
|
+
const parsed = JSON.parse(keyJson);
|
|
7048
|
+
if (!parsed.client_email || !parsed.private_key) {
|
|
7049
|
+
const err = validationError("Service account JSON must contain client_email and private_key");
|
|
7050
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7051
|
+
}
|
|
7052
|
+
clientEmail = parsed.client_email;
|
|
7053
|
+
privateKey = parsed.private_key;
|
|
7054
|
+
} catch {
|
|
7055
|
+
const err = validationError("Invalid JSON in keyJson");
|
|
7056
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7057
|
+
}
|
|
7058
|
+
} else {
|
|
7059
|
+
const err = validationError("keyJson is required");
|
|
7060
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7061
|
+
}
|
|
7062
|
+
try {
|
|
7063
|
+
await verifyConnection(clientEmail, privateKey, propertyId);
|
|
7064
|
+
gaLog("info", "connect.verified", { projectId: project.id, propertyId });
|
|
7065
|
+
} catch (e) {
|
|
7066
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
7067
|
+
gaLog("error", "connect.verify-failed", { projectId: project.id, propertyId, error: msg });
|
|
7068
|
+
const err = validationError(`Failed to verify GA4 credentials: ${msg}`);
|
|
7069
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7070
|
+
}
|
|
7071
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7072
|
+
const existing = store.getConnection(project.name);
|
|
7073
|
+
store.upsertConnection({
|
|
7074
|
+
projectName: project.name,
|
|
7075
|
+
propertyId,
|
|
7076
|
+
clientEmail,
|
|
7077
|
+
privateKey,
|
|
7078
|
+
createdAt: existing?.createdAt ?? now,
|
|
7079
|
+
updatedAt: now
|
|
7080
|
+
});
|
|
7081
|
+
writeAuditLog(app.db, {
|
|
7082
|
+
projectId: project.id,
|
|
7083
|
+
actor: "api",
|
|
7084
|
+
action: "ga4.connected",
|
|
7085
|
+
entityType: "ga_connection",
|
|
7086
|
+
entityId: propertyId
|
|
7087
|
+
});
|
|
7088
|
+
return {
|
|
7089
|
+
connected: true,
|
|
7090
|
+
propertyId,
|
|
7091
|
+
clientEmail
|
|
7092
|
+
};
|
|
7093
|
+
});
|
|
7094
|
+
app.delete("/projects/:name/ga/disconnect", async (request, reply) => {
|
|
7095
|
+
const store = requireCredentialStore(reply);
|
|
7096
|
+
if (!store) return;
|
|
7097
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7098
|
+
const conn = store.getConnection(project.name);
|
|
7099
|
+
if (!conn) {
|
|
7100
|
+
const err = notFound("GA4 connection", project.name);
|
|
7101
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7102
|
+
}
|
|
7103
|
+
app.db.delete(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).run();
|
|
7104
|
+
store.deleteConnection(project.name);
|
|
7105
|
+
writeAuditLog(app.db, {
|
|
7106
|
+
projectId: project.id,
|
|
7107
|
+
actor: "api",
|
|
7108
|
+
action: "ga4.disconnected",
|
|
7109
|
+
entityType: "ga_connection",
|
|
7110
|
+
entityId: conn.propertyId
|
|
7111
|
+
});
|
|
7112
|
+
return reply.status(204).send();
|
|
7113
|
+
});
|
|
7114
|
+
app.get("/projects/:name/ga/status", async (request, reply) => {
|
|
7115
|
+
const store = requireCredentialStore(reply);
|
|
7116
|
+
if (!store) return;
|
|
7117
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7118
|
+
const conn = store.getConnection(project.name);
|
|
7119
|
+
if (!conn) {
|
|
7120
|
+
return { connected: false, propertyId: null, clientEmail: null, lastSyncedAt: null };
|
|
7121
|
+
}
|
|
7122
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSnapshots.syncedAt }).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).orderBy(desc6(gaTrafficSnapshots.syncedAt)).limit(1).get();
|
|
7123
|
+
return {
|
|
7124
|
+
connected: true,
|
|
7125
|
+
propertyId: conn.propertyId,
|
|
7126
|
+
clientEmail: conn.clientEmail,
|
|
7127
|
+
lastSyncedAt: latestSync?.syncedAt ?? null,
|
|
7128
|
+
createdAt: conn.createdAt,
|
|
7129
|
+
updatedAt: conn.updatedAt
|
|
7130
|
+
};
|
|
7131
|
+
});
|
|
7132
|
+
app.post("/projects/:name/ga/sync", async (request, reply) => {
|
|
7133
|
+
const store = requireCredentialStore(reply);
|
|
7134
|
+
if (!store) return;
|
|
7135
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7136
|
+
const conn = store.getConnection(project.name);
|
|
7137
|
+
if (!conn) {
|
|
7138
|
+
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
7139
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7140
|
+
}
|
|
7141
|
+
const days = request.body?.days ?? 30;
|
|
7142
|
+
let accessToken;
|
|
7143
|
+
try {
|
|
7144
|
+
accessToken = await getAccessToken(conn.clientEmail, conn.privateKey);
|
|
7145
|
+
} catch (e) {
|
|
7146
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
7147
|
+
gaLog("error", "sync.auth-failed", { projectId: project.id, error: msg });
|
|
7148
|
+
const err = validationError(`GA4 authentication failed: ${msg}`);
|
|
7149
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7150
|
+
}
|
|
7151
|
+
let rows;
|
|
7152
|
+
try {
|
|
7153
|
+
rows = await fetchTrafficByLandingPage(accessToken, conn.propertyId, days);
|
|
7154
|
+
} catch (e) {
|
|
7155
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
7156
|
+
gaLog("error", "sync.fetch-failed", { projectId: project.id, error: msg });
|
|
7157
|
+
throw e;
|
|
7158
|
+
}
|
|
7159
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7160
|
+
if (rows.length > 0) {
|
|
7161
|
+
const dates = rows.map((r) => r.date);
|
|
7162
|
+
const minDate = dates.reduce((a, b) => a < b ? a : b);
|
|
7163
|
+
const maxDate = dates.reduce((a, b) => a > b ? a : b);
|
|
7164
|
+
app.db.transaction((tx) => {
|
|
7165
|
+
tx.delete(gaTrafficSnapshots).where(
|
|
7166
|
+
and6(
|
|
7167
|
+
eq16(gaTrafficSnapshots.projectId, project.id),
|
|
7168
|
+
sql3`${gaTrafficSnapshots.date} >= ${minDate}`,
|
|
7169
|
+
sql3`${gaTrafficSnapshots.date} <= ${maxDate}`
|
|
7170
|
+
)
|
|
7171
|
+
).run();
|
|
7172
|
+
for (const row of rows) {
|
|
7173
|
+
tx.insert(gaTrafficSnapshots).values({
|
|
7174
|
+
id: crypto16.randomUUID(),
|
|
7175
|
+
projectId: project.id,
|
|
7176
|
+
date: row.date,
|
|
7177
|
+
landingPage: row.landingPage,
|
|
7178
|
+
sessions: row.sessions,
|
|
7179
|
+
organicSessions: row.organicSessions,
|
|
7180
|
+
users: row.users,
|
|
7181
|
+
syncedAt: now
|
|
7182
|
+
}).run();
|
|
7183
|
+
}
|
|
7184
|
+
});
|
|
7185
|
+
}
|
|
7186
|
+
gaLog("info", "sync.complete", { projectId: project.id, rowCount: rows.length, days });
|
|
7187
|
+
return {
|
|
7188
|
+
synced: true,
|
|
7189
|
+
rowCount: rows.length,
|
|
7190
|
+
days,
|
|
7191
|
+
syncedAt: now
|
|
7192
|
+
};
|
|
7193
|
+
});
|
|
7194
|
+
app.get("/projects/:name/ga/traffic", async (request, reply) => {
|
|
7195
|
+
const store = requireCredentialStore(reply);
|
|
7196
|
+
if (!store) return;
|
|
7197
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7198
|
+
const conn = store.getConnection(project.name);
|
|
7199
|
+
if (!conn) {
|
|
7200
|
+
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
7201
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7202
|
+
}
|
|
7203
|
+
const limit = Math.max(1, Math.min(parseInt(request.query.limit ?? "50", 10) || 50, 500));
|
|
7204
|
+
const totals = app.db.select({
|
|
7205
|
+
totalSessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
7206
|
+
totalOrganicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
7207
|
+
totalUsers: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
7208
|
+
}).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).get();
|
|
7209
|
+
const rows = app.db.select({
|
|
7210
|
+
landingPage: gaTrafficSnapshots.landingPage,
|
|
7211
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
7212
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
7213
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
7214
|
+
}).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql3`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
7215
|
+
const latestSync = app.db.select({ syncedAt: gaTrafficSnapshots.syncedAt }).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).orderBy(desc6(gaTrafficSnapshots.syncedAt)).limit(1).get();
|
|
7216
|
+
return {
|
|
7217
|
+
totalSessions: totals?.totalSessions ?? 0,
|
|
7218
|
+
totalOrganicSessions: totals?.totalOrganicSessions ?? 0,
|
|
7219
|
+
totalUsers: totals?.totalUsers ?? 0,
|
|
7220
|
+
topPages: rows.map((r) => ({
|
|
7221
|
+
landingPage: r.landingPage,
|
|
7222
|
+
sessions: r.sessions ?? 0,
|
|
7223
|
+
organicSessions: r.organicSessions ?? 0,
|
|
7224
|
+
users: r.users ?? 0
|
|
7225
|
+
})),
|
|
7226
|
+
lastSyncedAt: latestSync?.syncedAt ?? null
|
|
7227
|
+
};
|
|
7228
|
+
});
|
|
7229
|
+
app.get("/projects/:name/ga/coverage", async (request, reply) => {
|
|
7230
|
+
const store = requireCredentialStore(reply);
|
|
7231
|
+
if (!store) return;
|
|
7232
|
+
const project = resolveProject(app.db, request.params.name);
|
|
7233
|
+
const conn = store.getConnection(project.name);
|
|
7234
|
+
if (!conn) {
|
|
7235
|
+
const err = validationError('No GA4 connection found. Run "canonry ga connect <project>" first.');
|
|
7236
|
+
return reply.status(err.statusCode).send(err.toJSON());
|
|
7237
|
+
}
|
|
7238
|
+
const trafficPages = app.db.select({
|
|
7239
|
+
landingPage: gaTrafficSnapshots.landingPage,
|
|
7240
|
+
sessions: sql3`SUM(${gaTrafficSnapshots.sessions})`,
|
|
7241
|
+
organicSessions: sql3`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
7242
|
+
users: sql3`SUM(${gaTrafficSnapshots.users})`
|
|
7243
|
+
}).from(gaTrafficSnapshots).where(eq16(gaTrafficSnapshots.projectId, project.id)).groupBy(gaTrafficSnapshots.landingPage).orderBy(sql3`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
7244
|
+
return {
|
|
7245
|
+
pages: trafficPages.map((r) => ({
|
|
7246
|
+
landingPage: r.landingPage,
|
|
7247
|
+
sessions: r.sessions ?? 0,
|
|
7248
|
+
organicSessions: r.organicSessions ?? 0,
|
|
7249
|
+
users: r.users ?? 0
|
|
7250
|
+
}))
|
|
7251
|
+
};
|
|
7252
|
+
});
|
|
7253
|
+
}
|
|
7254
|
+
|
|
6601
7255
|
// ../api-routes/src/index.ts
|
|
6602
7256
|
async function apiRoutes(app, opts) {
|
|
6603
7257
|
app.decorate("db", opts.db);
|
|
@@ -6686,6 +7340,9 @@ async function apiRoutes(app, opts) {
|
|
|
6686
7340
|
onCdpScreenshot: opts.onCdpScreenshot,
|
|
6687
7341
|
onCdpConfigure: opts.onCdpConfigure
|
|
6688
7342
|
});
|
|
7343
|
+
await api.register(ga4Routes, {
|
|
7344
|
+
ga4CredentialStore: opts.ga4CredentialStore
|
|
7345
|
+
});
|
|
6689
7346
|
}, { prefix: opts.routePrefix ?? "/api/v1" });
|
|
6690
7347
|
}
|
|
6691
7348
|
|
|
@@ -8517,12 +9174,84 @@ function removeGoogleConnection(config, domain, connectionType) {
|
|
|
8517
9174
|
return true;
|
|
8518
9175
|
}
|
|
8519
9176
|
|
|
9177
|
+
// src/ga4-config.ts
|
|
9178
|
+
function ensureConnections2(config) {
|
|
9179
|
+
if (!config.ga4) config.ga4 = {};
|
|
9180
|
+
if (!config.ga4.connections) config.ga4.connections = [];
|
|
9181
|
+
return config.ga4.connections;
|
|
9182
|
+
}
|
|
9183
|
+
function getGa4Connection(config, projectName) {
|
|
9184
|
+
return (config.ga4?.connections ?? []).find((c) => c.projectName === projectName);
|
|
9185
|
+
}
|
|
9186
|
+
function upsertGa4Connection(config, connection) {
|
|
9187
|
+
const connections = ensureConnections2(config);
|
|
9188
|
+
const index2 = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
9189
|
+
if (index2 === -1) {
|
|
9190
|
+
connections.push(connection);
|
|
9191
|
+
return connection;
|
|
9192
|
+
}
|
|
9193
|
+
connections[index2] = connection;
|
|
9194
|
+
return connection;
|
|
9195
|
+
}
|
|
9196
|
+
function removeGa4Connection(config, projectName) {
|
|
9197
|
+
const connections = config.ga4?.connections;
|
|
9198
|
+
if (!connections?.length) return false;
|
|
9199
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
9200
|
+
if (next.length === connections.length) return false;
|
|
9201
|
+
if (!config.ga4) return false;
|
|
9202
|
+
config.ga4.connections = next;
|
|
9203
|
+
if (next.length === 0) {
|
|
9204
|
+
delete config.ga4;
|
|
9205
|
+
}
|
|
9206
|
+
return true;
|
|
9207
|
+
}
|
|
9208
|
+
|
|
8520
9209
|
// src/job-runner.ts
|
|
8521
|
-
import
|
|
9210
|
+
import crypto17 from "crypto";
|
|
8522
9211
|
import fs4 from "fs";
|
|
8523
9212
|
import path5 from "path";
|
|
8524
9213
|
import os4 from "os";
|
|
8525
|
-
import { and as
|
|
9214
|
+
import { and as and7, eq as eq17, inArray as inArray3 } from "drizzle-orm";
|
|
9215
|
+
|
|
9216
|
+
// src/logger.ts
|
|
9217
|
+
var IS_TTY = process.stdout.isTTY === true;
|
|
9218
|
+
function formatTTY(entry) {
|
|
9219
|
+
const { ts, level, module, action, msg, ...ctx } = entry;
|
|
9220
|
+
const time = ts.slice(11, 19);
|
|
9221
|
+
const levelTag = level === "error" ? "\x1B[31mERR\x1B[0m" : level === "warn" ? "\x1B[33mWRN\x1B[0m" : "\x1B[36mINF\x1B[0m";
|
|
9222
|
+
const ctxParts = Object.entries(ctx).filter(([, v]) => v !== void 0 && v !== null).map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`).join(" ");
|
|
9223
|
+
const msgPart = msg ? ` ${msg}` : "";
|
|
9224
|
+
const ctxPart = ctxParts ? ` ${ctxParts}` : "";
|
|
9225
|
+
return `${time} ${levelTag} [${module}] ${action}${msgPart}${ctxPart}`;
|
|
9226
|
+
}
|
|
9227
|
+
function emit(entry) {
|
|
9228
|
+
const stream = entry.level === "error" ? process.stderr : process.stdout;
|
|
9229
|
+
if (IS_TTY) {
|
|
9230
|
+
stream.write(formatTTY(entry) + "\n");
|
|
9231
|
+
} else {
|
|
9232
|
+
stream.write(JSON.stringify(entry) + "\n");
|
|
9233
|
+
}
|
|
9234
|
+
}
|
|
9235
|
+
function createLogger(module) {
|
|
9236
|
+
function log7(level, action, ctx) {
|
|
9237
|
+
const entry = {
|
|
9238
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9239
|
+
level,
|
|
9240
|
+
module,
|
|
9241
|
+
action,
|
|
9242
|
+
...ctx
|
|
9243
|
+
};
|
|
9244
|
+
emit(entry);
|
|
9245
|
+
}
|
|
9246
|
+
return {
|
|
9247
|
+
info: (action, ctx) => log7("info", action, ctx),
|
|
9248
|
+
warn: (action, ctx) => log7("warn", action, ctx),
|
|
9249
|
+
error: (action, ctx) => log7("error", action, ctx)
|
|
9250
|
+
};
|
|
9251
|
+
}
|
|
9252
|
+
|
|
9253
|
+
// src/job-runner.ts
|
|
9254
|
+
var log = createLogger("JobRunner");
|
|
8526
9255
|
var RunCancelledError = class extends Error {
|
|
8527
9256
|
constructor(runId) {
|
|
8528
9257
|
super(`Run ${runId} was cancelled`);
|
|
@@ -8604,8 +9333,8 @@ var JobRunner = class {
|
|
|
8604
9333
|
if (stale.length === 0) return;
|
|
8605
9334
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8606
9335
|
for (const run of stale) {
|
|
8607
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
8608
|
-
|
|
9336
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq17(runs.id, run.id)).run();
|
|
9337
|
+
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
8609
9338
|
}
|
|
8610
9339
|
}
|
|
8611
9340
|
async executeRun(runId, projectId, providerOverride, locationOverride) {
|
|
@@ -8632,10 +9361,10 @@ var JobRunner = class {
|
|
|
8632
9361
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
8633
9362
|
}
|
|
8634
9363
|
if (existingRun.status === "queued") {
|
|
8635
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
9364
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and7(eq17(runs.id, runId), eq17(runs.status, "queued"))).run();
|
|
8636
9365
|
}
|
|
8637
9366
|
this.throwIfRunCancelled(runId);
|
|
8638
|
-
const project = this.db.select().from(projects).where(
|
|
9367
|
+
const project = this.db.select().from(projects).where(eq17(projects.id, projectId)).get();
|
|
8639
9368
|
if (!project) {
|
|
8640
9369
|
throw new Error(`Project ${projectId} not found`);
|
|
8641
9370
|
}
|
|
@@ -8654,9 +9383,9 @@ var JobRunner = class {
|
|
|
8654
9383
|
if (activeProviders.length === 0) {
|
|
8655
9384
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
8656
9385
|
}
|
|
8657
|
-
|
|
8658
|
-
projectKeywords = this.db.select().from(keywords).where(
|
|
8659
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
9386
|
+
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
9387
|
+
projectKeywords = this.db.select().from(keywords).where(eq17(keywords.projectId, projectId)).all();
|
|
9388
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq17(competitors.projectId, projectId)).all();
|
|
8660
9389
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
8661
9390
|
const allDomains = effectiveDomains({
|
|
8662
9391
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -8672,7 +9401,7 @@ var JobRunner = class {
|
|
|
8672
9401
|
const todayPeriod = getCurrentUsageDay();
|
|
8673
9402
|
for (const p of activeProviders) {
|
|
8674
9403
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
8675
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
9404
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq17(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
8676
9405
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
8677
9406
|
if (providerUsage + queriesPerProvider > limit) {
|
|
8678
9407
|
throw new Error(
|
|
@@ -8716,12 +9445,12 @@ var JobRunner = class {
|
|
|
8716
9445
|
);
|
|
8717
9446
|
this.throwIfRunCancelled(runId);
|
|
8718
9447
|
const normalized = adapter.normalizeResult(raw);
|
|
8719
|
-
|
|
9448
|
+
log.info("query.result", { runId, provider: providerName, keyword: kw.keyword, citedDomains: normalized.citedDomains, groundingSources: normalized.groundingSources.map((s) => s.uri), matchDomains: allDomains });
|
|
8720
9449
|
const citationState = determineCitationState(normalized, allDomains);
|
|
8721
9450
|
const overlap = computeCompetitorOverlap(normalized, competitorDomains);
|
|
8722
9451
|
let screenshotRelPath = null;
|
|
8723
9452
|
if (raw.screenshotPath && fs4.existsSync(raw.screenshotPath)) {
|
|
8724
|
-
const snapshotId =
|
|
9453
|
+
const snapshotId = crypto17.randomUUID();
|
|
8725
9454
|
const screenshotDir = path5.join(os4.homedir(), ".canonry", "screenshots", runId);
|
|
8726
9455
|
if (!fs4.existsSync(screenshotDir)) fs4.mkdirSync(screenshotDir, { recursive: true });
|
|
8727
9456
|
const destPath = path5.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -8749,7 +9478,7 @@ var JobRunner = class {
|
|
|
8749
9478
|
}).run();
|
|
8750
9479
|
} else {
|
|
8751
9480
|
this.db.insert(querySnapshots).values({
|
|
8752
|
-
id:
|
|
9481
|
+
id: crypto17.randomUUID(),
|
|
8753
9482
|
runId,
|
|
8754
9483
|
keywordId: kw.id,
|
|
8755
9484
|
provider: providerName,
|
|
@@ -8769,14 +9498,15 @@ var JobRunner = class {
|
|
|
8769
9498
|
}).run();
|
|
8770
9499
|
}
|
|
8771
9500
|
totalSnapshotsInserted++;
|
|
8772
|
-
|
|
9501
|
+
log.info("query.citation", { runId, provider: providerName, keyword: kw.keyword, citationState });
|
|
8773
9502
|
});
|
|
8774
9503
|
} catch (err) {
|
|
8775
9504
|
if (err instanceof RunCancelledError) {
|
|
8776
9505
|
throw err;
|
|
8777
9506
|
}
|
|
8778
9507
|
const msg = err instanceof Error ? err.message : String(err);
|
|
8779
|
-
|
|
9508
|
+
const stack = err instanceof Error ? err.stack : void 0;
|
|
9509
|
+
log.error("query.failed", { runId, provider: providerName, keyword: kw.keyword, error: msg, stack });
|
|
8780
9510
|
if (!providerErrors.has(providerName)) {
|
|
8781
9511
|
providerErrors.set(providerName, msg);
|
|
8782
9512
|
}
|
|
@@ -8797,12 +9527,12 @@ var JobRunner = class {
|
|
|
8797
9527
|
const someFailed = providerErrors.size > 0;
|
|
8798
9528
|
if (allFailed) {
|
|
8799
9529
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
8800
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
9530
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq17(runs.id, runId)).run();
|
|
8801
9531
|
} else if (someFailed) {
|
|
8802
9532
|
const errorDetail = JSON.stringify(Object.fromEntries(providerErrors));
|
|
8803
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
9533
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq17(runs.id, runId)).run();
|
|
8804
9534
|
} else {
|
|
8805
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9535
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq17(runs.id, runId)).run();
|
|
8806
9536
|
}
|
|
8807
9537
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
8808
9538
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -8817,7 +9547,7 @@ var JobRunner = class {
|
|
|
8817
9547
|
this.incrementUsage(projectId, "runs", 1);
|
|
8818
9548
|
if (this.onRunCompleted) {
|
|
8819
9549
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
8820
|
-
|
|
9550
|
+
log.error("notification.callback-failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8821
9551
|
});
|
|
8822
9552
|
}
|
|
8823
9553
|
} catch (err) {
|
|
@@ -8837,7 +9567,7 @@ var JobRunner = class {
|
|
|
8837
9567
|
status: "failed",
|
|
8838
9568
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8839
9569
|
error: errorMessage
|
|
8840
|
-
}).where(
|
|
9570
|
+
}).where(eq17(runs.id, runId)).run();
|
|
8841
9571
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
8842
9572
|
trackEvent("run.completed", {
|
|
8843
9573
|
status: "failed",
|
|
@@ -8857,10 +9587,10 @@ var JobRunner = class {
|
|
|
8857
9587
|
incrementUsage(scope, metric, count) {
|
|
8858
9588
|
const now = /* @__PURE__ */ new Date();
|
|
8859
9589
|
const period = now.toISOString().slice(0, 10);
|
|
8860
|
-
const id =
|
|
8861
|
-
const existing = this.db.select().from(usageCounters).where(
|
|
9590
|
+
const id = crypto17.randomUUID();
|
|
9591
|
+
const existing = this.db.select().from(usageCounters).where(eq17(usageCounters.scope, scope)).all().find((r) => r.period === period && r.metric === metric);
|
|
8862
9592
|
if (existing) {
|
|
8863
|
-
this.db.update(usageCounters).set({ count: existing.count + count, updatedAt: now.toISOString() }).where(
|
|
9593
|
+
this.db.update(usageCounters).set({ count: existing.count + count, updatedAt: now.toISOString() }).where(eq17(usageCounters.id, existing.id)).run();
|
|
8864
9594
|
} else {
|
|
8865
9595
|
this.db.insert(usageCounters).values({
|
|
8866
9596
|
id,
|
|
@@ -8883,7 +9613,7 @@ var JobRunner = class {
|
|
|
8883
9613
|
status: runs.status,
|
|
8884
9614
|
finishedAt: runs.finishedAt,
|
|
8885
9615
|
error: runs.error
|
|
8886
|
-
}).from(runs).where(
|
|
9616
|
+
}).from(runs).where(eq17(runs.id, runId)).get();
|
|
8887
9617
|
}
|
|
8888
9618
|
isRunCancelled(runId) {
|
|
8889
9619
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -8899,7 +9629,7 @@ var JobRunner = class {
|
|
|
8899
9629
|
this.db.update(runs).set({
|
|
8900
9630
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8901
9631
|
error: currentRun.error ?? "Cancelled by user"
|
|
8902
|
-
}).where(
|
|
9632
|
+
}).where(eq17(runs.id, runId)).run();
|
|
8903
9633
|
}
|
|
8904
9634
|
trackEvent("run.completed", {
|
|
8905
9635
|
status: "cancelled",
|
|
@@ -8911,7 +9641,7 @@ var JobRunner = class {
|
|
|
8911
9641
|
});
|
|
8912
9642
|
if (this.onRunCompleted) {
|
|
8913
9643
|
this.onRunCompleted(runId, projectId).catch((err) => {
|
|
8914
|
-
|
|
9644
|
+
log.error("notification.callback-failed", { runId, error: err instanceof Error ? err.message : String(err) });
|
|
8915
9645
|
});
|
|
8916
9646
|
}
|
|
8917
9647
|
}
|
|
@@ -8982,9 +9712,10 @@ function computeCompetitorOverlap(normalized, competitorDomains) {
|
|
|
8982
9712
|
}
|
|
8983
9713
|
|
|
8984
9714
|
// src/gsc-sync.ts
|
|
8985
|
-
import
|
|
8986
|
-
import { eq as
|
|
8987
|
-
|
|
9715
|
+
import crypto18 from "crypto";
|
|
9716
|
+
import { eq as eq18, and as and8, sql as sql4 } from "drizzle-orm";
|
|
9717
|
+
var log2 = createLogger("GscSync");
|
|
9718
|
+
function formatDate2(d) {
|
|
8988
9719
|
return d.toISOString().split("T")[0];
|
|
8989
9720
|
}
|
|
8990
9721
|
function daysAgo(n) {
|
|
@@ -8994,13 +9725,13 @@ function daysAgo(n) {
|
|
|
8994
9725
|
}
|
|
8995
9726
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
8996
9727
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
8997
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
9728
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq18(runs.id, runId)).run();
|
|
8998
9729
|
try {
|
|
8999
9730
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
9000
9731
|
if (!googleClientId || !googleClientSecret) {
|
|
9001
9732
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
9002
9733
|
}
|
|
9003
|
-
const project = db.select().from(projects).where(
|
|
9734
|
+
const project = db.select().from(projects).where(eq18(projects.id, projectId)).get();
|
|
9004
9735
|
if (!project) {
|
|
9005
9736
|
throw new Error(`Project not found: ${projectId}`);
|
|
9006
9737
|
}
|
|
@@ -9024,20 +9755,20 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9024
9755
|
saveConfig(opts.config);
|
|
9025
9756
|
}
|
|
9026
9757
|
const lagOffset = GSC_DATA_LAG_DAYS;
|
|
9027
|
-
const endDate =
|
|
9758
|
+
const endDate = formatDate2(daysAgo(lagOffset));
|
|
9028
9759
|
const days = opts.full ? 480 : opts.days ?? 30;
|
|
9029
|
-
const startDate =
|
|
9030
|
-
|
|
9760
|
+
const startDate = formatDate2(daysAgo(days + lagOffset));
|
|
9761
|
+
log2.info("fetch.start", { runId, projectId, propertyId: conn.propertyId, startDate, endDate });
|
|
9031
9762
|
const rows = await fetchSearchAnalytics(accessToken, conn.propertyId, {
|
|
9032
9763
|
startDate,
|
|
9033
9764
|
endDate
|
|
9034
9765
|
});
|
|
9035
|
-
|
|
9766
|
+
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
9036
9767
|
db.delete(gscSearchData).where(
|
|
9037
|
-
|
|
9038
|
-
|
|
9039
|
-
|
|
9040
|
-
|
|
9768
|
+
and8(
|
|
9769
|
+
eq18(gscSearchData.projectId, projectId),
|
|
9770
|
+
sql4`${gscSearchData.date} >= ${startDate}`,
|
|
9771
|
+
sql4`${gscSearchData.date} <= ${endDate}`
|
|
9041
9772
|
)
|
|
9042
9773
|
).run();
|
|
9043
9774
|
const batchSize = 500;
|
|
@@ -9047,7 +9778,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9047
9778
|
for (const row of batch) {
|
|
9048
9779
|
const [query, page, country, device, date] = row.keys;
|
|
9049
9780
|
db.insert(gscSearchData).values({
|
|
9050
|
-
id:
|
|
9781
|
+
id: crypto18.randomUUID(),
|
|
9051
9782
|
projectId,
|
|
9052
9783
|
syncRunId: runId,
|
|
9053
9784
|
date: date ?? "",
|
|
@@ -9071,7 +9802,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9071
9802
|
}
|
|
9072
9803
|
}
|
|
9073
9804
|
const topPages = [...pageClicks.entries()].sort((a, b) => b[1] - a[1]).slice(0, 50).map(([page]) => page);
|
|
9074
|
-
|
|
9805
|
+
log2.info("inspect.start", { runId, projectId, urlCount: topPages.length });
|
|
9075
9806
|
for (const pageUrl of topPages) {
|
|
9076
9807
|
try {
|
|
9077
9808
|
const result = await inspectUrl(accessToken, pageUrl, conn.propertyId);
|
|
@@ -9081,7 +9812,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9081
9812
|
const rich = ir.richResultsResult;
|
|
9082
9813
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9083
9814
|
db.insert(gscUrlInspections).values({
|
|
9084
|
-
id:
|
|
9815
|
+
id: crypto18.randomUUID(),
|
|
9085
9816
|
projectId,
|
|
9086
9817
|
syncRunId: runId,
|
|
9087
9818
|
url: pageUrl,
|
|
@@ -9099,10 +9830,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9099
9830
|
createdAt: inspectedAt
|
|
9100
9831
|
}).run();
|
|
9101
9832
|
} catch (err) {
|
|
9102
|
-
|
|
9833
|
+
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
9103
9834
|
}
|
|
9104
9835
|
}
|
|
9105
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
9836
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq18(gscUrlInspections.projectId, projectId)).all();
|
|
9106
9837
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
9107
9838
|
for (const row of allInspections) {
|
|
9108
9839
|
const existing = latestByUrl.get(row.url);
|
|
@@ -9122,10 +9853,10 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9122
9853
|
reasonCounts[reason] = (reasonCounts[reason] ?? 0) + 1;
|
|
9123
9854
|
}
|
|
9124
9855
|
}
|
|
9125
|
-
const snapshotDate =
|
|
9126
|
-
db.delete(gscCoverageSnapshots).where(
|
|
9856
|
+
const snapshotDate = formatDate2(/* @__PURE__ */ new Date());
|
|
9857
|
+
db.delete(gscCoverageSnapshots).where(and8(eq18(gscCoverageSnapshots.projectId, projectId), eq18(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
9127
9858
|
db.insert(gscCoverageSnapshots).values({
|
|
9128
|
-
id:
|
|
9859
|
+
id: crypto18.randomUUID(),
|
|
9129
9860
|
projectId,
|
|
9130
9861
|
syncRunId: runId,
|
|
9131
9862
|
date: snapshotDate,
|
|
@@ -9134,19 +9865,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
9134
9865
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
9135
9866
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9136
9867
|
}).run();
|
|
9137
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9138
|
-
|
|
9868
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(runs.id, runId)).run();
|
|
9869
|
+
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
9139
9870
|
} catch (err) {
|
|
9140
9871
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
9141
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9142
|
-
|
|
9872
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq18(runs.id, runId)).run();
|
|
9873
|
+
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
9143
9874
|
throw err;
|
|
9144
9875
|
}
|
|
9145
9876
|
}
|
|
9146
9877
|
|
|
9147
9878
|
// src/gsc-inspect-sitemap.ts
|
|
9148
|
-
import
|
|
9149
|
-
import { eq as
|
|
9879
|
+
import crypto19 from "crypto";
|
|
9880
|
+
import { eq as eq19, and as and9 } from "drizzle-orm";
|
|
9150
9881
|
|
|
9151
9882
|
// src/sitemap-parser.ts
|
|
9152
9883
|
var LOC_REGEX = /<loc>\s*([^<]+?)\s*<\/loc>/gi;
|
|
@@ -9212,15 +9943,16 @@ async function parseSitemapRecursive(url, urls, depth) {
|
|
|
9212
9943
|
}
|
|
9213
9944
|
|
|
9214
9945
|
// src/gsc-inspect-sitemap.ts
|
|
9946
|
+
var log3 = createLogger("InspectSitemap");
|
|
9215
9947
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
9216
9948
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9217
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
9949
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq19(runs.id, runId)).run();
|
|
9218
9950
|
try {
|
|
9219
9951
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
9220
9952
|
if (!googleClientId || !googleClientSecret) {
|
|
9221
9953
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
9222
9954
|
}
|
|
9223
|
-
const project = db.select().from(projects).where(
|
|
9955
|
+
const project = db.select().from(projects).where(eq19(projects.id, projectId)).get();
|
|
9224
9956
|
if (!project) {
|
|
9225
9957
|
throw new Error(`Project not found: ${projectId}`);
|
|
9226
9958
|
}
|
|
@@ -9244,9 +9976,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9244
9976
|
saveConfig(opts.config);
|
|
9245
9977
|
}
|
|
9246
9978
|
const sitemapUrl = opts.sitemapUrl || conn.sitemapUrl || `https://${project.canonicalDomain}/sitemap.xml`;
|
|
9247
|
-
|
|
9979
|
+
log3.info("sitemap.fetch", { runId, projectId, sitemapUrl });
|
|
9248
9980
|
const urls = await fetchAndParseSitemap(sitemapUrl);
|
|
9249
|
-
|
|
9981
|
+
log3.info("sitemap.parsed", { runId, projectId, urlCount: urls.length, sitemapUrl });
|
|
9250
9982
|
if (urls.length === 0) {
|
|
9251
9983
|
throw new Error("No URLs found in sitemap");
|
|
9252
9984
|
}
|
|
@@ -9261,7 +9993,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9261
9993
|
const rich = ir.richResultsResult;
|
|
9262
9994
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
9263
9995
|
db.insert(gscUrlInspections).values({
|
|
9264
|
-
id:
|
|
9996
|
+
id: crypto19.randomUUID(),
|
|
9265
9997
|
projectId,
|
|
9266
9998
|
syncRunId: runId,
|
|
9267
9999
|
url: pageUrl,
|
|
@@ -9279,16 +10011,16 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9279
10011
|
createdAt: inspectedAt
|
|
9280
10012
|
}).run();
|
|
9281
10013
|
inspected++;
|
|
9282
|
-
|
|
10014
|
+
log3.info("inspect.url-done", { runId, projectId, url: pageUrl, progress: `${inspected}/${urls.length}` });
|
|
9283
10015
|
} catch (err) {
|
|
9284
10016
|
errors++;
|
|
9285
|
-
|
|
10017
|
+
log3.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
9286
10018
|
}
|
|
9287
10019
|
if (inspected + errors < urls.length) {
|
|
9288
10020
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
9289
10021
|
}
|
|
9290
10022
|
}
|
|
9291
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
10023
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq19(gscUrlInspections.projectId, projectId)).all();
|
|
9292
10024
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
9293
10025
|
for (const row of allInspections) {
|
|
9294
10026
|
const existing = latestByUrl.get(row.url);
|
|
@@ -9309,9 +10041,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9309
10041
|
}
|
|
9310
10042
|
}
|
|
9311
10043
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
9312
|
-
db.delete(gscCoverageSnapshots).where(
|
|
10044
|
+
db.delete(gscCoverageSnapshots).where(and9(eq19(gscCoverageSnapshots.projectId, projectId), eq19(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
9313
10045
|
db.insert(gscCoverageSnapshots).values({
|
|
9314
|
-
id:
|
|
10046
|
+
id: crypto19.randomUUID(),
|
|
9315
10047
|
projectId,
|
|
9316
10048
|
syncRunId: runId,
|
|
9317
10049
|
date: snapshotDate,
|
|
@@ -9321,12 +10053,12 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
9321
10053
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9322
10054
|
}).run();
|
|
9323
10055
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
9324
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9325
|
-
|
|
10056
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(runs.id, runId)).run();
|
|
10057
|
+
log3.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
9326
10058
|
} catch (err) {
|
|
9327
10059
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
9328
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
9329
|
-
|
|
10060
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq19(runs.id, runId)).run();
|
|
10061
|
+
log3.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
9330
10062
|
throw err;
|
|
9331
10063
|
}
|
|
9332
10064
|
}
|
|
@@ -9384,7 +10116,8 @@ var ProviderRegistry = class {
|
|
|
9384
10116
|
|
|
9385
10117
|
// src/scheduler.ts
|
|
9386
10118
|
import cron from "node-cron";
|
|
9387
|
-
import { eq as
|
|
10119
|
+
import { eq as eq20 } from "drizzle-orm";
|
|
10120
|
+
var log4 = createLogger("Scheduler");
|
|
9388
10121
|
var Scheduler = class {
|
|
9389
10122
|
db;
|
|
9390
10123
|
callbacks;
|
|
@@ -9395,16 +10128,16 @@ var Scheduler = class {
|
|
|
9395
10128
|
}
|
|
9396
10129
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
9397
10130
|
start() {
|
|
9398
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
10131
|
+
const allSchedules = this.db.select().from(schedules).where(eq20(schedules.enabled, 1)).all();
|
|
9399
10132
|
for (const schedule of allSchedules) {
|
|
9400
10133
|
const missedRunAt = schedule.nextRunAt;
|
|
9401
10134
|
this.registerCronTask(schedule);
|
|
9402
10135
|
if (missedRunAt && new Date(missedRunAt) < /* @__PURE__ */ new Date()) {
|
|
9403
|
-
|
|
10136
|
+
log4.info("run.catch-up", { projectId: schedule.projectId, missedRunAt });
|
|
9404
10137
|
this.triggerRun(schedule.id, schedule.projectId);
|
|
9405
10138
|
}
|
|
9406
10139
|
}
|
|
9407
|
-
|
|
10140
|
+
log4.info("started", { scheduleCount: allSchedules.length });
|
|
9408
10141
|
}
|
|
9409
10142
|
/** Stop all cron tasks for graceful shutdown. */
|
|
9410
10143
|
stop() {
|
|
@@ -9420,7 +10153,7 @@ var Scheduler = class {
|
|
|
9420
10153
|
this.stopTask(projectId, existing, "Stopped");
|
|
9421
10154
|
this.tasks.delete(projectId);
|
|
9422
10155
|
}
|
|
9423
|
-
const schedule = this.db.select().from(schedules).where(
|
|
10156
|
+
const schedule = this.db.select().from(schedules).where(eq20(schedules.projectId, projectId)).get();
|
|
9424
10157
|
if (schedule && schedule.enabled === 1) {
|
|
9425
10158
|
this.registerCronTask(schedule);
|
|
9426
10159
|
}
|
|
@@ -9436,12 +10169,12 @@ var Scheduler = class {
|
|
|
9436
10169
|
stopTask(projectId, task, verb) {
|
|
9437
10170
|
task.stop();
|
|
9438
10171
|
task.destroy();
|
|
9439
|
-
|
|
10172
|
+
log4.info(`task.${verb.toLowerCase()}`, { projectId });
|
|
9440
10173
|
}
|
|
9441
10174
|
registerCronTask(schedule) {
|
|
9442
10175
|
const { id: scheduleId, projectId, cronExpr, timezone } = schedule;
|
|
9443
10176
|
if (!cron.validate(cronExpr)) {
|
|
9444
|
-
|
|
10177
|
+
log4.error("cron.invalid", { projectId, cronExpr });
|
|
9445
10178
|
return;
|
|
9446
10179
|
}
|
|
9447
10180
|
const task = cron.schedule(cronExpr, () => {
|
|
@@ -9453,23 +10186,23 @@ var Scheduler = class {
|
|
|
9453
10186
|
this.db.update(schedules).set({
|
|
9454
10187
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
9455
10188
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
9456
|
-
}).where(
|
|
10189
|
+
}).where(eq20(schedules.id, scheduleId)).run();
|
|
9457
10190
|
const label = schedule.preset ?? cronExpr;
|
|
9458
|
-
|
|
10191
|
+
log4.info("cron.registered", { projectId, schedule: label, timezone });
|
|
9459
10192
|
}
|
|
9460
10193
|
triggerRun(scheduleId, projectId) {
|
|
9461
10194
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
9462
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
10195
|
+
const currentSchedule = this.db.select().from(schedules).where(eq20(schedules.id, scheduleId)).get();
|
|
9463
10196
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
9464
|
-
|
|
10197
|
+
log4.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
9465
10198
|
this.remove(projectId);
|
|
9466
10199
|
return;
|
|
9467
10200
|
}
|
|
9468
10201
|
const task = this.tasks.get(projectId);
|
|
9469
10202
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
9470
|
-
const project = this.db.select().from(projects).where(
|
|
10203
|
+
const project = this.db.select().from(projects).where(eq20(projects.id, projectId)).get();
|
|
9471
10204
|
if (!project) {
|
|
9472
|
-
|
|
10205
|
+
log4.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
9473
10206
|
this.remove(projectId);
|
|
9474
10207
|
return;
|
|
9475
10208
|
}
|
|
@@ -9480,11 +10213,11 @@ var Scheduler = class {
|
|
|
9480
10213
|
trigger: "scheduled"
|
|
9481
10214
|
});
|
|
9482
10215
|
if (queueResult.conflict) {
|
|
9483
|
-
|
|
10216
|
+
log4.info("run.skipped-active", { projectName: project.name, activeRunId: queueResult.activeRunId });
|
|
9484
10217
|
this.db.update(schedules).set({
|
|
9485
10218
|
nextRunAt,
|
|
9486
10219
|
updatedAt: now
|
|
9487
|
-
}).where(
|
|
10220
|
+
}).where(eq20(schedules.id, currentSchedule.id)).run();
|
|
9488
10221
|
return;
|
|
9489
10222
|
}
|
|
9490
10223
|
const runId = queueResult.runId;
|
|
@@ -9492,17 +10225,18 @@ var Scheduler = class {
|
|
|
9492
10225
|
lastRunAt: now,
|
|
9493
10226
|
nextRunAt,
|
|
9494
10227
|
updatedAt: now
|
|
9495
|
-
}).where(
|
|
10228
|
+
}).where(eq20(schedules.id, currentSchedule.id)).run();
|
|
9496
10229
|
const scheduleProviders = JSON.parse(currentSchedule.providers);
|
|
9497
10230
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
9498
|
-
|
|
10231
|
+
log4.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
9499
10232
|
this.callbacks.onRunCreated(runId, projectId, providers);
|
|
9500
10233
|
}
|
|
9501
10234
|
};
|
|
9502
10235
|
|
|
9503
10236
|
// src/notifier.ts
|
|
9504
|
-
import { eq as
|
|
9505
|
-
import
|
|
10237
|
+
import { eq as eq21, desc as desc7, and as and10, or as or2 } from "drizzle-orm";
|
|
10238
|
+
import crypto20 from "crypto";
|
|
10239
|
+
var log5 = createLogger("Notifier");
|
|
9506
10240
|
var Notifier = class {
|
|
9507
10241
|
db;
|
|
9508
10242
|
serverUrl;
|
|
@@ -9512,26 +10246,26 @@ var Notifier = class {
|
|
|
9512
10246
|
}
|
|
9513
10247
|
/** Called after a run completes (success, partial, or failed). */
|
|
9514
10248
|
async onRunCompleted(runId, projectId) {
|
|
9515
|
-
|
|
9516
|
-
const notifs = this.db.select().from(notifications).where(
|
|
10249
|
+
log5.info("run.completed", { runId, projectId });
|
|
10250
|
+
const notifs = this.db.select().from(notifications).where(eq21(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
9517
10251
|
if (notifs.length === 0) {
|
|
9518
|
-
|
|
10252
|
+
log5.info("notifications.none-enabled", { projectId });
|
|
9519
10253
|
return;
|
|
9520
10254
|
}
|
|
9521
|
-
|
|
9522
|
-
const run = this.db.select().from(runs).where(
|
|
10255
|
+
log5.info("notifications.found", { projectId, count: notifs.length });
|
|
10256
|
+
const run = this.db.select().from(runs).where(eq21(runs.id, runId)).get();
|
|
9523
10257
|
if (!run) {
|
|
9524
|
-
|
|
10258
|
+
log5.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
9525
10259
|
return;
|
|
9526
10260
|
}
|
|
9527
|
-
const project = this.db.select().from(projects).where(
|
|
10261
|
+
const project = this.db.select().from(projects).where(eq21(projects.id, projectId)).get();
|
|
9528
10262
|
if (!project) {
|
|
9529
|
-
|
|
10263
|
+
log5.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
9530
10264
|
return;
|
|
9531
10265
|
}
|
|
9532
10266
|
const transitions = this.computeTransitions(runId, projectId);
|
|
9533
10267
|
const events = [];
|
|
9534
|
-
|
|
10268
|
+
log5.info("run.status", { runId: run.id, status: run.status, projectId });
|
|
9535
10269
|
if (run.status === "completed" || run.status === "partial") {
|
|
9536
10270
|
events.push("run.completed");
|
|
9537
10271
|
}
|
|
@@ -9546,7 +10280,7 @@ var Notifier = class {
|
|
|
9546
10280
|
const config = JSON.parse(notif.config);
|
|
9547
10281
|
const subscribedEvents = config.events;
|
|
9548
10282
|
const matchingEvents = events.filter((e) => subscribedEvents.includes(e));
|
|
9549
|
-
|
|
10283
|
+
log5.info("notification.match", { notificationId: notif.id, subscribedEvents, matchedEvents: matchingEvents });
|
|
9550
10284
|
if (matchingEvents.length === 0) continue;
|
|
9551
10285
|
for (const event of matchingEvents) {
|
|
9552
10286
|
const relevantTransitions = event === "citation.lost" ? lostTransitions : event === "citation.gained" ? gainedTransitions : transitions;
|
|
@@ -9564,11 +10298,11 @@ var Notifier = class {
|
|
|
9564
10298
|
}
|
|
9565
10299
|
computeTransitions(runId, projectId) {
|
|
9566
10300
|
const recentRuns = this.db.select().from(runs).where(
|
|
9567
|
-
|
|
9568
|
-
|
|
9569
|
-
or2(
|
|
10301
|
+
and10(
|
|
10302
|
+
eq21(runs.projectId, projectId),
|
|
10303
|
+
or2(eq21(runs.status, "completed"), eq21(runs.status, "partial"))
|
|
9570
10304
|
)
|
|
9571
|
-
).orderBy(
|
|
10305
|
+
).orderBy(desc7(runs.createdAt)).limit(2).all();
|
|
9572
10306
|
if (recentRuns.length < 2) return [];
|
|
9573
10307
|
const currentRunId = recentRuns[0].id;
|
|
9574
10308
|
const previousRunId = recentRuns[1].id;
|
|
@@ -9578,12 +10312,12 @@ var Notifier = class {
|
|
|
9578
10312
|
keyword: keywords.keyword,
|
|
9579
10313
|
provider: querySnapshots.provider,
|
|
9580
10314
|
citationState: querySnapshots.citationState
|
|
9581
|
-
}).from(querySnapshots).leftJoin(keywords,
|
|
10315
|
+
}).from(querySnapshots).leftJoin(keywords, eq21(querySnapshots.keywordId, keywords.id)).where(eq21(querySnapshots.runId, currentRunId)).all();
|
|
9582
10316
|
const previousSnapshots = this.db.select({
|
|
9583
10317
|
keywordId: querySnapshots.keywordId,
|
|
9584
10318
|
provider: querySnapshots.provider,
|
|
9585
10319
|
citationState: querySnapshots.citationState
|
|
9586
|
-
}).from(querySnapshots).where(
|
|
10320
|
+
}).from(querySnapshots).where(eq21(querySnapshots.runId, previousRunId)).all();
|
|
9587
10321
|
const prevMap = /* @__PURE__ */ new Map();
|
|
9588
10322
|
for (const s of previousSnapshots) {
|
|
9589
10323
|
prevMap.set(`${s.keywordId}:${s.provider}`, s.citationState);
|
|
@@ -9606,23 +10340,23 @@ var Notifier = class {
|
|
|
9606
10340
|
async sendWebhook(url, payload, notificationId, projectId, webhookSecret) {
|
|
9607
10341
|
const targetCheck = await resolveWebhookTarget(url);
|
|
9608
10342
|
if (!targetCheck.ok) {
|
|
9609
|
-
|
|
10343
|
+
log5.error("webhook.ssrf-blocked", { url, reason: targetCheck.message });
|
|
9610
10344
|
this.logDelivery(projectId, notificationId, payload.event, "failed", `SSRF: ${targetCheck.message}`);
|
|
9611
10345
|
return;
|
|
9612
10346
|
}
|
|
9613
|
-
|
|
10347
|
+
log5.info("webhook.send", { event: payload.event, url });
|
|
9614
10348
|
const maxRetries = 3;
|
|
9615
10349
|
const delays = [1e3, 4e3, 16e3];
|
|
9616
10350
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
9617
10351
|
try {
|
|
9618
10352
|
const response = await deliverWebhook(targetCheck.target, payload, webhookSecret);
|
|
9619
10353
|
if (response.status >= 200 && response.status < 300) {
|
|
9620
|
-
|
|
10354
|
+
log5.info("webhook.delivered", { event: payload.event, url, httpStatus: response.status });
|
|
9621
10355
|
this.logDelivery(projectId, notificationId, payload.event, "sent", null);
|
|
9622
10356
|
return;
|
|
9623
10357
|
}
|
|
9624
10358
|
const errorDetail = response.error ?? `HTTP ${response.status}`;
|
|
9625
|
-
|
|
10359
|
+
log5.warn("webhook.attempt-failed", { event: payload.event, url, attempt: attempt + 1, maxRetries, httpStatus: response.status, error: errorDetail });
|
|
9626
10360
|
if (attempt === maxRetries - 1) {
|
|
9627
10361
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
9628
10362
|
}
|
|
@@ -9630,7 +10364,7 @@ var Notifier = class {
|
|
|
9630
10364
|
const errorDetail = err instanceof Error ? err.message : String(err);
|
|
9631
10365
|
if (attempt === maxRetries - 1) {
|
|
9632
10366
|
this.logDelivery(projectId, notificationId, payload.event, "failed", errorDetail);
|
|
9633
|
-
|
|
10367
|
+
log5.error("webhook.exhausted", { event: payload.event, url, maxRetries, error: errorDetail });
|
|
9634
10368
|
}
|
|
9635
10369
|
}
|
|
9636
10370
|
if (attempt < maxRetries - 1) {
|
|
@@ -9640,7 +10374,7 @@ var Notifier = class {
|
|
|
9640
10374
|
}
|
|
9641
10375
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
9642
10376
|
this.db.insert(auditLog).values({
|
|
9643
|
-
id:
|
|
10377
|
+
id: crypto20.randomUUID(),
|
|
9644
10378
|
projectId,
|
|
9645
10379
|
actor: "scheduler",
|
|
9646
10380
|
action: `notification.${status}`,
|
|
@@ -9766,6 +10500,7 @@ function stripHtml(html) {
|
|
|
9766
10500
|
// src/server.ts
|
|
9767
10501
|
var _require2 = createRequire2(import.meta.url);
|
|
9768
10502
|
var { version: PKG_VERSION } = _require2("../package.json");
|
|
10503
|
+
var log6 = createLogger("Server");
|
|
9769
10504
|
var DEFAULT_QUOTA = {
|
|
9770
10505
|
maxConcurrency: 2,
|
|
9771
10506
|
maxRequestsPerMinute: 10,
|
|
@@ -9817,10 +10552,10 @@ async function createServer(opts) {
|
|
|
9817
10552
|
quota: opts.config.geminiQuota
|
|
9818
10553
|
};
|
|
9819
10554
|
}
|
|
9820
|
-
|
|
10555
|
+
log6.info("providers.configured", { providers: Object.keys(providers).filter((k) => {
|
|
9821
10556
|
const p = providers[k];
|
|
9822
10557
|
return p?.apiKey || p?.baseUrl;
|
|
9823
|
-
}));
|
|
10558
|
+
}) });
|
|
9824
10559
|
for (const adapter of API_ADAPTERS) {
|
|
9825
10560
|
const entry = providers[adapter.name];
|
|
9826
10561
|
if (!entry) continue;
|
|
@@ -9910,7 +10645,22 @@ async function createServer(opts) {
|
|
|
9910
10645
|
return true;
|
|
9911
10646
|
}
|
|
9912
10647
|
};
|
|
9913
|
-
const
|
|
10648
|
+
const ga4CredentialStore = {
|
|
10649
|
+
getConnection: (projectName) => {
|
|
10650
|
+
return getGa4Connection(opts.config, projectName);
|
|
10651
|
+
},
|
|
10652
|
+
upsertConnection: (connection) => {
|
|
10653
|
+
const updated = upsertGa4Connection(opts.config, connection);
|
|
10654
|
+
saveConfig(opts.config);
|
|
10655
|
+
return updated;
|
|
10656
|
+
},
|
|
10657
|
+
deleteConnection: (projectName) => {
|
|
10658
|
+
const removed = removeGa4Connection(opts.config, projectName);
|
|
10659
|
+
if (removed) saveConfig(opts.config);
|
|
10660
|
+
return removed;
|
|
10661
|
+
}
|
|
10662
|
+
};
|
|
10663
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto21.randomBytes(32).toString("hex");
|
|
9914
10664
|
const googleConnectionStore = {
|
|
9915
10665
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
9916
10666
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -9983,6 +10733,7 @@ async function createServer(opts) {
|
|
|
9983
10733
|
googleSettingsSummary,
|
|
9984
10734
|
bingSettingsSummary,
|
|
9985
10735
|
bingConnectionStore,
|
|
10736
|
+
ga4CredentialStore,
|
|
9986
10737
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
9987
10738
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
9988
10739
|
app.log.error({ runId, err }, "Job runner failed");
|
|
@@ -10038,7 +10789,7 @@ async function createServer(opts) {
|
|
|
10038
10789
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
10039
10790
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
10040
10791
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
10041
|
-
id:
|
|
10792
|
+
id: crypto21.randomUUID(),
|
|
10042
10793
|
projectId,
|
|
10043
10794
|
actor: "api",
|
|
10044
10795
|
action: existing ? "provider.updated" : "provider.created",
|