@ainyc/canonry 4.11.0 → 4.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{index-DOcemxPD.js → index-CCC1E6ji.js} +106 -106
- package/assets/index.html +1 -1
- package/dist/{chunk-5G6WYP2S.js → chunk-DCE3B6KD.js} +177 -2
- package/dist/{chunk-NG2PJHVL.js → chunk-L4KKHRVQ.js} +1222 -143
- package/dist/{chunk-ZR4AVT4T.js → chunk-LNRDWAG3.js} +16 -1
- package/dist/{chunk-WWU65YPN.js → chunk-YDGT5CAY.js} +65 -2
- package/dist/cli.js +269 -103
- package/dist/index.d.ts +19 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-ZFIYBHLQ.js → intelligence-service-NT24OLLA.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +6 -4
|
@@ -4,13 +4,14 @@ import {
|
|
|
4
4
|
configExists,
|
|
5
5
|
loadConfig,
|
|
6
6
|
saveConfigPatch
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-LNRDWAG3.js";
|
|
8
8
|
import {
|
|
9
9
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
10
10
|
IntelligenceService,
|
|
11
11
|
MIN_TREND_POINTS,
|
|
12
12
|
agentMemory,
|
|
13
13
|
agentSessions,
|
|
14
|
+
aiReferralEventsHourly,
|
|
14
15
|
apiKeys,
|
|
15
16
|
auditLog,
|
|
16
17
|
backlinkDomains,
|
|
@@ -36,6 +37,7 @@ import {
|
|
|
36
37
|
categorizeQueryByIntent,
|
|
37
38
|
ccReleaseSyncs,
|
|
38
39
|
competitors,
|
|
40
|
+
crawlerEventsHourly,
|
|
39
41
|
createLogger,
|
|
40
42
|
dropLegacyCredentialColumns,
|
|
41
43
|
extractLegacyCredentials,
|
|
@@ -58,10 +60,12 @@ import {
|
|
|
58
60
|
projects,
|
|
59
61
|
queries,
|
|
60
62
|
querySnapshots,
|
|
63
|
+
rawEventSamples,
|
|
61
64
|
runs,
|
|
62
65
|
schedules,
|
|
66
|
+
trafficSources,
|
|
63
67
|
usageCounters
|
|
64
|
-
} from "./chunk-
|
|
68
|
+
} from "./chunk-DCE3B6KD.js";
|
|
65
69
|
import {
|
|
66
70
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
67
71
|
AGENT_PROVIDER_IDS,
|
|
@@ -76,6 +80,11 @@ import {
|
|
|
76
80
|
RunKinds,
|
|
77
81
|
RunStatuses,
|
|
78
82
|
RunTriggers,
|
|
83
|
+
TrafficEventConfidences,
|
|
84
|
+
TrafficEvidenceKinds,
|
|
85
|
+
TrafficSourceAuthModes,
|
|
86
|
+
TrafficSourceStatuses,
|
|
87
|
+
TrafficSourceTypes,
|
|
79
88
|
absolutizeProjectUrl,
|
|
80
89
|
actionConfidenceLabel,
|
|
81
90
|
agentBusy,
|
|
@@ -137,7 +146,7 @@ import {
|
|
|
137
146
|
visibilityStateFromAnswerMentioned,
|
|
138
147
|
windowCutoff,
|
|
139
148
|
wordpressEnvSchema
|
|
140
|
-
} from "./chunk-
|
|
149
|
+
} from "./chunk-YDGT5CAY.js";
|
|
141
150
|
|
|
142
151
|
// src/telemetry.ts
|
|
143
152
|
import crypto from "crypto";
|
|
@@ -213,11 +222,11 @@ function trackEvent(event, properties) {
|
|
|
213
222
|
|
|
214
223
|
// src/server.ts
|
|
215
224
|
import { createRequire as createRequire3 } from "module";
|
|
216
|
-
import
|
|
225
|
+
import crypto30 from "crypto";
|
|
217
226
|
import fs12 from "fs";
|
|
218
227
|
import path14 from "path";
|
|
219
228
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
220
|
-
import { eq as
|
|
229
|
+
import { eq as eq35 } from "drizzle-orm";
|
|
221
230
|
import Fastify from "fastify";
|
|
222
231
|
|
|
223
232
|
// ../api-routes/src/auth.ts
|
|
@@ -9160,6 +9169,66 @@ var routeCatalog = [
|
|
|
9160
9169
|
200: { description: "History returned oldest-first by queriedAt." },
|
|
9161
9170
|
404: { description: "Project not found." }
|
|
9162
9171
|
}
|
|
9172
|
+
},
|
|
9173
|
+
{
|
|
9174
|
+
method: "post",
|
|
9175
|
+
path: "/api/v1/projects/{name}/traffic/connect/cloud-run",
|
|
9176
|
+
summary: "Connect a Cloud Run traffic source",
|
|
9177
|
+
description: "Stores the service-account JSON in `~/.canonry/config.yaml` and creates a `traffic_sources` row for the project. Reconnecting updates the existing active source rather than creating a duplicate.",
|
|
9178
|
+
tags: ["traffic"],
|
|
9179
|
+
parameters: [nameParameter],
|
|
9180
|
+
requestBody: {
|
|
9181
|
+
required: true,
|
|
9182
|
+
content: {
|
|
9183
|
+
"application/json": {
|
|
9184
|
+
schema: {
|
|
9185
|
+
type: "object",
|
|
9186
|
+
required: ["gcpProjectId", "keyJson"],
|
|
9187
|
+
properties: {
|
|
9188
|
+
gcpProjectId: stringSchema,
|
|
9189
|
+
serviceName: stringSchema,
|
|
9190
|
+
location: stringSchema,
|
|
9191
|
+
displayName: stringSchema,
|
|
9192
|
+
keyJson: { ...stringSchema, description: "Service-account JSON content." }
|
|
9193
|
+
}
|
|
9194
|
+
}
|
|
9195
|
+
}
|
|
9196
|
+
}
|
|
9197
|
+
},
|
|
9198
|
+
responses: {
|
|
9199
|
+
200: { description: "Traffic source DTO returned." },
|
|
9200
|
+
400: { description: "Invalid Cloud Run connection request." },
|
|
9201
|
+
404: { description: "Project not found." }
|
|
9202
|
+
}
|
|
9203
|
+
},
|
|
9204
|
+
{
|
|
9205
|
+
method: "post",
|
|
9206
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
9207
|
+
summary: "Trigger a sync run for a traffic source",
|
|
9208
|
+
description: "Pulls request logs from the configured Cloud Run service for the lookback window, classifies crawler / AI-referral hits, and upserts hourly buckets and a bounded sample tail.",
|
|
9209
|
+
tags: ["traffic"],
|
|
9210
|
+
parameters: [
|
|
9211
|
+
nameParameter,
|
|
9212
|
+
{ name: "id", in: "path", required: true, description: "Traffic source ID.", schema: stringSchema }
|
|
9213
|
+
],
|
|
9214
|
+
requestBody: {
|
|
9215
|
+
required: false,
|
|
9216
|
+
content: {
|
|
9217
|
+
"application/json": {
|
|
9218
|
+
schema: {
|
|
9219
|
+
type: "object",
|
|
9220
|
+
properties: {
|
|
9221
|
+
sinceMinutes: { ...integerSchema, description: "Lookback window in minutes (default 60)." }
|
|
9222
|
+
}
|
|
9223
|
+
}
|
|
9224
|
+
}
|
|
9225
|
+
}
|
|
9226
|
+
},
|
|
9227
|
+
responses: {
|
|
9228
|
+
200: { description: "Sync summary returned." },
|
|
9229
|
+
400: { description: "Invalid sync request, missing credentials, or upstream pull error." },
|
|
9230
|
+
404: { description: "Project or traffic source not found." }
|
|
9231
|
+
}
|
|
9163
9232
|
}
|
|
9164
9233
|
];
|
|
9165
9234
|
var canonryLocalRouteCatalog = [
|
|
@@ -12235,6 +12304,17 @@ function formatSharePct(numerator, total) {
|
|
|
12235
12304
|
if (rounded === 0) return "<1%";
|
|
12236
12305
|
return `${rounded}%`;
|
|
12237
12306
|
}
|
|
12307
|
+
function pickWinningDimension(rows, tupleKey) {
|
|
12308
|
+
const winners = /* @__PURE__ */ new Map();
|
|
12309
|
+
for (const row of rows) {
|
|
12310
|
+
const key = tupleKey(row);
|
|
12311
|
+
const existing = winners.get(key);
|
|
12312
|
+
if (!existing || (row.sessions ?? 0) > (existing.sessions ?? 0)) {
|
|
12313
|
+
winners.set(key, row);
|
|
12314
|
+
}
|
|
12315
|
+
}
|
|
12316
|
+
return [...winners.values()].sort((a, b) => (b.sessions ?? 0) - (a.sessions ?? 0));
|
|
12317
|
+
}
|
|
12238
12318
|
async function refreshOAuthTokenIfNeeded(googleStore, authConfig, canonicalDomain, oauthConn) {
|
|
12239
12319
|
const expiresAt = oauthConn.tokenExpiresAt ? new Date(oauthConn.tokenExpiresAt).getTime() : 0;
|
|
12240
12320
|
const fiveMinutes = 5 * 60 * 1e3;
|
|
@@ -12656,14 +12736,14 @@ async function ga4Routes(app, opts) {
|
|
|
12656
12736
|
directSessions: sql5`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
12657
12737
|
users: sql5`SUM(${gaTrafficSnapshots.users})`
|
|
12658
12738
|
}).from(gaTrafficSnapshots).where(and9(...snapshotConditions)).groupBy(sql5`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql5`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
12659
|
-
const
|
|
12739
|
+
const aiReferralRows = app.db.select({
|
|
12660
12740
|
source: gaAiReferrals.source,
|
|
12661
12741
|
medium: gaAiReferrals.medium,
|
|
12662
12742
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
12663
12743
|
sessions: sql5`SUM(${gaAiReferrals.sessions})`,
|
|
12664
12744
|
users: sql5`SUM(${gaAiReferrals.users})`
|
|
12665
|
-
}).from(gaAiReferrals).where(and9(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).
|
|
12666
|
-
const
|
|
12745
|
+
}).from(gaAiReferrals).where(and9(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.sourceDimension).all();
|
|
12746
|
+
const aiReferralLandingPageRows = app.db.select({
|
|
12667
12747
|
source: gaAiReferrals.source,
|
|
12668
12748
|
medium: gaAiReferrals.medium,
|
|
12669
12749
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
@@ -12675,7 +12755,15 @@ async function ga4Routes(app, opts) {
|
|
|
12675
12755
|
gaAiReferrals.medium,
|
|
12676
12756
|
gaAiReferrals.sourceDimension,
|
|
12677
12757
|
sql5`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
12678
|
-
).
|
|
12758
|
+
).all();
|
|
12759
|
+
const aiReferrals = pickWinningDimension(
|
|
12760
|
+
aiReferralRows,
|
|
12761
|
+
(r) => `${r.source}\0${r.medium}`
|
|
12762
|
+
);
|
|
12763
|
+
const aiReferralLandingPages = pickWinningDimension(
|
|
12764
|
+
aiReferralLandingPageRows,
|
|
12765
|
+
(r) => `${r.source}\0${r.medium}\0${r.landingPage}`
|
|
12766
|
+
);
|
|
12679
12767
|
const aiDeduped = app.db.select({
|
|
12680
12768
|
sessions: sql5`COALESCE(SUM(max_sessions), 0)`,
|
|
12681
12769
|
users: sql5`COALESCE(SUM(max_users), 0)`
|
|
@@ -14889,7 +14977,7 @@ async function queryBacklinks(opts) {
|
|
|
14889
14977
|
const reversed = opts.targets.map(reverseDomain);
|
|
14890
14978
|
const targetList = reversed.map(quote).join(", ");
|
|
14891
14979
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
14892
|
-
const
|
|
14980
|
+
const sql12 = `
|
|
14893
14981
|
WITH vertices AS (
|
|
14894
14982
|
SELECT * FROM read_csv(
|
|
14895
14983
|
${quote(opts.vertexPath)},
|
|
@@ -14925,7 +15013,7 @@ async function queryBacklinks(opts) {
|
|
|
14925
15013
|
const conn = await instance.connect();
|
|
14926
15014
|
let rows;
|
|
14927
15015
|
try {
|
|
14928
|
-
const reader = await conn.runAndReadAll(
|
|
15016
|
+
const reader = await conn.runAndReadAll(sql12);
|
|
14929
15017
|
rows = reader.getRowObjects();
|
|
14930
15018
|
} finally {
|
|
14931
15019
|
conn.disconnectSync?.();
|
|
@@ -15313,6 +15401,944 @@ async function backlinksRoutes(app, opts) {
|
|
|
15313
15401
|
);
|
|
15314
15402
|
}
|
|
15315
15403
|
|
|
15404
|
+
// ../api-routes/src/traffic.ts
|
|
15405
|
+
import crypto20 from "crypto";
|
|
15406
|
+
import { eq as eq23, sql as sql7 } from "drizzle-orm";
|
|
15407
|
+
|
|
15408
|
+
// ../integration-cloud-run/src/auth.ts
|
|
15409
|
+
import crypto19 from "crypto";
|
|
15410
|
+
var GOOGLE_TOKEN_URL3 = "https://oauth2.googleapis.com/token";
|
|
15411
|
+
var CLOUD_LOGGING_READ_SCOPE = "https://www.googleapis.com/auth/logging.read";
|
|
15412
|
+
var TOKEN_REQUEST_TIMEOUT_MS = 3e4;
|
|
15413
|
+
var CloudRunAuthError = class extends Error {
|
|
15414
|
+
constructor(message, httpStatus, body) {
|
|
15415
|
+
super(message);
|
|
15416
|
+
this.httpStatus = httpStatus;
|
|
15417
|
+
this.body = body;
|
|
15418
|
+
this.name = "CloudRunAuthError";
|
|
15419
|
+
}
|
|
15420
|
+
};
|
|
15421
|
+
function createServiceAccountJwt2(clientEmail, privateKey, scope) {
|
|
15422
|
+
if (!clientEmail) throw new CloudRunAuthError("clientEmail is required");
|
|
15423
|
+
if (!privateKey) throw new CloudRunAuthError("privateKey is required");
|
|
15424
|
+
if (!scope) throw new CloudRunAuthError("scope is required");
|
|
15425
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
15426
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
15427
|
+
const payload = {
|
|
15428
|
+
iss: clientEmail,
|
|
15429
|
+
scope,
|
|
15430
|
+
aud: GOOGLE_TOKEN_URL3,
|
|
15431
|
+
iat: now,
|
|
15432
|
+
exp: now + 3600
|
|
15433
|
+
};
|
|
15434
|
+
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
15435
|
+
const headerB64 = encode(header);
|
|
15436
|
+
const payloadB64 = encode(payload);
|
|
15437
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
15438
|
+
const sign = crypto19.createSign("RSA-SHA256");
|
|
15439
|
+
sign.update(signingInput);
|
|
15440
|
+
const signature = sign.sign(privateKey, "base64url");
|
|
15441
|
+
return `${signingInput}.${signature}`;
|
|
15442
|
+
}
|
|
15443
|
+
async function getCloudLoggingAccessToken(clientEmail, privateKey) {
|
|
15444
|
+
const jwt = createServiceAccountJwt2(clientEmail, privateKey, CLOUD_LOGGING_READ_SCOPE);
|
|
15445
|
+
const res = await fetch(GOOGLE_TOKEN_URL3, {
|
|
15446
|
+
method: "POST",
|
|
15447
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
15448
|
+
body: new URLSearchParams({
|
|
15449
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
15450
|
+
assertion: jwt
|
|
15451
|
+
}),
|
|
15452
|
+
signal: AbortSignal.timeout(TOKEN_REQUEST_TIMEOUT_MS)
|
|
15453
|
+
});
|
|
15454
|
+
if (!res.ok) {
|
|
15455
|
+
const body = await res.text().catch(() => "");
|
|
15456
|
+
throw new CloudRunAuthError(
|
|
15457
|
+
`Service-account token exchange failed (HTTP ${res.status})`,
|
|
15458
|
+
res.status,
|
|
15459
|
+
body.slice(0, 500)
|
|
15460
|
+
);
|
|
15461
|
+
}
|
|
15462
|
+
const data = await res.json();
|
|
15463
|
+
if (!data.access_token) {
|
|
15464
|
+
throw new CloudRunAuthError("Service-account token response missing access_token", res.status);
|
|
15465
|
+
}
|
|
15466
|
+
return data.access_token;
|
|
15467
|
+
}
|
|
15468
|
+
|
|
15469
|
+
// ../integration-cloud-run/src/filter.ts
|
|
15470
|
+
function assertNonEmpty(name, value) {
|
|
15471
|
+
if (!value.trim()) {
|
|
15472
|
+
throw new Error(`${name} must be a non-empty string`);
|
|
15473
|
+
}
|
|
15474
|
+
}
|
|
15475
|
+
function quoteLogFilterValue(value) {
|
|
15476
|
+
return JSON.stringify(value);
|
|
15477
|
+
}
|
|
15478
|
+
function normalizeTimestamp(value) {
|
|
15479
|
+
const date = value instanceof Date ? value : new Date(value);
|
|
15480
|
+
if (Number.isNaN(date.getTime())) {
|
|
15481
|
+
throw new Error(`Invalid timestamp: ${String(value)}`);
|
|
15482
|
+
}
|
|
15483
|
+
return date.toISOString();
|
|
15484
|
+
}
|
|
15485
|
+
function buildCloudRunLogFilter(options = {}) {
|
|
15486
|
+
const clauses = ['resource.type="cloud_run_revision"'];
|
|
15487
|
+
if (options.serviceName !== void 0) {
|
|
15488
|
+
assertNonEmpty("serviceName", options.serviceName);
|
|
15489
|
+
clauses.push(`resource.labels.service_name=${quoteLogFilterValue(options.serviceName)}`);
|
|
15490
|
+
}
|
|
15491
|
+
if (options.location !== void 0) {
|
|
15492
|
+
assertNonEmpty("location", options.location);
|
|
15493
|
+
clauses.push(`resource.labels.location=${quoteLogFilterValue(options.location)}`);
|
|
15494
|
+
}
|
|
15495
|
+
if (options.startTime !== void 0) {
|
|
15496
|
+
clauses.push(`timestamp >= ${quoteLogFilterValue(normalizeTimestamp(options.startTime))}`);
|
|
15497
|
+
}
|
|
15498
|
+
if (options.endTime !== void 0) {
|
|
15499
|
+
clauses.push(`timestamp < ${quoteLogFilterValue(normalizeTimestamp(options.endTime))}`);
|
|
15500
|
+
}
|
|
15501
|
+
const userAgentSubstrings = (options.userAgentSubstrings ?? []).map((pattern) => pattern.trim()).filter(Boolean);
|
|
15502
|
+
if (userAgentSubstrings.length > 0) {
|
|
15503
|
+
const uaClauses = userAgentSubstrings.map((pattern) => `httpRequest.userAgent:${quoteLogFilterValue(pattern)}`);
|
|
15504
|
+
clauses.push(`(${uaClauses.join(" OR ")})`);
|
|
15505
|
+
}
|
|
15506
|
+
const requestUrlSubstrings = (options.requestUrlSubstrings ?? []).map((pattern) => pattern.trim()).filter(Boolean);
|
|
15507
|
+
if (requestUrlSubstrings.length > 0) {
|
|
15508
|
+
const urlClauses = requestUrlSubstrings.map((pattern) => `httpRequest.requestUrl:${quoteLogFilterValue(pattern)}`);
|
|
15509
|
+
clauses.push(`(${urlClauses.join(" OR ")})`);
|
|
15510
|
+
}
|
|
15511
|
+
return clauses.join(" AND ");
|
|
15512
|
+
}
|
|
15513
|
+
|
|
15514
|
+
// ../integration-cloud-run/src/normalize.ts
|
|
15515
|
+
function numberOrNull(value) {
|
|
15516
|
+
if (value === void 0 || value === null) return null;
|
|
15517
|
+
const parsed = typeof value === "number" ? value : Number(value);
|
|
15518
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
15519
|
+
}
|
|
15520
|
+
function latencyToMs(value) {
|
|
15521
|
+
if (!value) return null;
|
|
15522
|
+
const secondsMatch = /^([0-9]+(?:\.[0-9]+)?)s$/.exec(value.trim());
|
|
15523
|
+
if (!secondsMatch) return null;
|
|
15524
|
+
const seconds = Number(secondsMatch[1]);
|
|
15525
|
+
return Number.isFinite(seconds) ? Math.round(seconds * 1e6) / 1e3 : null;
|
|
15526
|
+
}
|
|
15527
|
+
function normalizeLabels(labels) {
|
|
15528
|
+
if (!labels) return {};
|
|
15529
|
+
return Object.fromEntries(
|
|
15530
|
+
Object.entries(labels).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
|
|
15531
|
+
);
|
|
15532
|
+
}
|
|
15533
|
+
function parseRequestUrl(requestUrl) {
|
|
15534
|
+
try {
|
|
15535
|
+
const url = requestUrl.startsWith("/") ? new URL(requestUrl, "https://canonry.local") : new URL(requestUrl);
|
|
15536
|
+
return {
|
|
15537
|
+
host: url.hostname === "canonry.local" ? null : url.hostname,
|
|
15538
|
+
path: url.pathname || "/",
|
|
15539
|
+
queryString: url.search ? url.search.slice(1) : null
|
|
15540
|
+
};
|
|
15541
|
+
} catch {
|
|
15542
|
+
return null;
|
|
15543
|
+
}
|
|
15544
|
+
}
|
|
15545
|
+
function buildEventId(entry, observedAt, requestUrl) {
|
|
15546
|
+
if (entry.insertId?.trim()) {
|
|
15547
|
+
return `cloud-run:${observedAt}:${entry.insertId}`;
|
|
15548
|
+
}
|
|
15549
|
+
return `cloud-run:${observedAt}:${requestUrl}`;
|
|
15550
|
+
}
|
|
15551
|
+
function normalizeCloudRunLogEntry(entry) {
|
|
15552
|
+
const request = entry.httpRequest;
|
|
15553
|
+
if (!request?.requestUrl) return null;
|
|
15554
|
+
const observedAt = entry.timestamp ?? entry.receiveTimestamp;
|
|
15555
|
+
if (!observedAt) return null;
|
|
15556
|
+
const urlParts = parseRequestUrl(request.requestUrl);
|
|
15557
|
+
if (!urlParts) return null;
|
|
15558
|
+
return {
|
|
15559
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
15560
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
15561
|
+
confidence: TrafficEventConfidences.observed,
|
|
15562
|
+
eventId: buildEventId(entry, observedAt, request.requestUrl),
|
|
15563
|
+
observedAt,
|
|
15564
|
+
method: request.requestMethod ?? null,
|
|
15565
|
+
requestUrl: request.requestUrl,
|
|
15566
|
+
host: urlParts.host,
|
|
15567
|
+
path: urlParts.path,
|
|
15568
|
+
queryString: urlParts.queryString,
|
|
15569
|
+
status: numberOrNull(request.status),
|
|
15570
|
+
userAgent: request.userAgent ?? null,
|
|
15571
|
+
remoteIp: request.remoteIp ?? null,
|
|
15572
|
+
referer: request.referer ?? null,
|
|
15573
|
+
latencyMs: latencyToMs(request.latency),
|
|
15574
|
+
requestSizeBytes: numberOrNull(request.requestSize),
|
|
15575
|
+
responseSizeBytes: numberOrNull(request.responseSize),
|
|
15576
|
+
providerResource: {
|
|
15577
|
+
type: entry.resource?.type ?? null,
|
|
15578
|
+
labels: normalizeLabels(entry.resource?.labels)
|
|
15579
|
+
},
|
|
15580
|
+
providerLabels: normalizeLabels(entry.labels)
|
|
15581
|
+
};
|
|
15582
|
+
}
|
|
15583
|
+
|
|
15584
|
+
// ../integration-cloud-run/src/client.ts
|
|
15585
|
+
var CLOUD_LOGGING_ENTRIES_LIST_URL = "https://logging.googleapis.com/v2/entries:list";
|
|
15586
|
+
var DEFAULT_PAGE_SIZE = 1e3;
|
|
15587
|
+
var DEFAULT_MAX_PAGES = 1;
|
|
15588
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
15589
|
+
var CloudRunLoggingApiError = class extends Error {
|
|
15590
|
+
constructor(message, status, body) {
|
|
15591
|
+
super(message);
|
|
15592
|
+
this.status = status;
|
|
15593
|
+
this.body = body;
|
|
15594
|
+
this.name = "CloudRunLoggingApiError";
|
|
15595
|
+
}
|
|
15596
|
+
};
|
|
15597
|
+
function validateAccessToken3(accessToken) {
|
|
15598
|
+
if (!accessToken.trim()) {
|
|
15599
|
+
throw new CloudRunLoggingApiError("Cloud Logging access token is required", 400);
|
|
15600
|
+
}
|
|
15601
|
+
}
|
|
15602
|
+
function validateProjectId(gcpProjectId) {
|
|
15603
|
+
if (!gcpProjectId.trim()) {
|
|
15604
|
+
throw new CloudRunLoggingApiError("GCP project ID is required", 400);
|
|
15605
|
+
}
|
|
15606
|
+
}
|
|
15607
|
+
function normalizePageSize(pageSize) {
|
|
15608
|
+
if (pageSize === void 0) return DEFAULT_PAGE_SIZE;
|
|
15609
|
+
if (!Number.isInteger(pageSize) || pageSize < 1) {
|
|
15610
|
+
throw new CloudRunLoggingApiError("pageSize must be a positive integer", 400);
|
|
15611
|
+
}
|
|
15612
|
+
return pageSize;
|
|
15613
|
+
}
|
|
15614
|
+
function normalizeMaxPages(maxPages) {
|
|
15615
|
+
if (maxPages === void 0) return DEFAULT_MAX_PAGES;
|
|
15616
|
+
if (!Number.isInteger(maxPages) || maxPages < 1) {
|
|
15617
|
+
throw new CloudRunLoggingApiError("maxPages must be a positive integer", 400);
|
|
15618
|
+
}
|
|
15619
|
+
return maxPages;
|
|
15620
|
+
}
|
|
15621
|
+
async function readErrorBody(response) {
|
|
15622
|
+
const text = await response.text().catch(() => "");
|
|
15623
|
+
if (!text) return void 0;
|
|
15624
|
+
return text.length <= 500 ? text : `${text.slice(0, 500)}... [truncated]`;
|
|
15625
|
+
}
|
|
15626
|
+
async function listCloudRunTrafficEvents(accessToken, options) {
|
|
15627
|
+
validateAccessToken3(accessToken);
|
|
15628
|
+
validateProjectId(options.gcpProjectId);
|
|
15629
|
+
const filter = buildCloudRunLogFilter(options);
|
|
15630
|
+
const pageSize = normalizePageSize(options.pageSize);
|
|
15631
|
+
const maxPages = normalizeMaxPages(options.maxPages);
|
|
15632
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
15633
|
+
let pageToken = options.pageToken;
|
|
15634
|
+
let rawEntryCount = 0;
|
|
15635
|
+
let skippedEntryCount = 0;
|
|
15636
|
+
const events = [];
|
|
15637
|
+
for (let page = 0; page < maxPages; page += 1) {
|
|
15638
|
+
const requestBody = {
|
|
15639
|
+
resourceNames: [`projects/${options.gcpProjectId}`],
|
|
15640
|
+
filter,
|
|
15641
|
+
orderBy: options.orderBy ?? "timestamp asc",
|
|
15642
|
+
pageSize
|
|
15643
|
+
};
|
|
15644
|
+
if (pageToken) {
|
|
15645
|
+
requestBody.pageToken = pageToken;
|
|
15646
|
+
}
|
|
15647
|
+
const response = await fetch(CLOUD_LOGGING_ENTRIES_LIST_URL, {
|
|
15648
|
+
method: "POST",
|
|
15649
|
+
headers: {
|
|
15650
|
+
Authorization: `Bearer ${accessToken}`,
|
|
15651
|
+
"Content-Type": "application/json"
|
|
15652
|
+
},
|
|
15653
|
+
body: JSON.stringify(requestBody),
|
|
15654
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
15655
|
+
});
|
|
15656
|
+
if (!response.ok) {
|
|
15657
|
+
const body2 = await readErrorBody(response);
|
|
15658
|
+
throw new CloudRunLoggingApiError(
|
|
15659
|
+
`Cloud Logging entries.list failed with HTTP ${response.status}`,
|
|
15660
|
+
response.status,
|
|
15661
|
+
body2
|
|
15662
|
+
);
|
|
15663
|
+
}
|
|
15664
|
+
const body = await response.json();
|
|
15665
|
+
const entries = body.entries ?? [];
|
|
15666
|
+
rawEntryCount += entries.length;
|
|
15667
|
+
for (const entry of entries) {
|
|
15668
|
+
const event = normalizeCloudRunLogEntry(entry);
|
|
15669
|
+
if (event) {
|
|
15670
|
+
events.push(event);
|
|
15671
|
+
} else {
|
|
15672
|
+
skippedEntryCount += 1;
|
|
15673
|
+
}
|
|
15674
|
+
}
|
|
15675
|
+
pageToken = body.nextPageToken;
|
|
15676
|
+
if (!pageToken) break;
|
|
15677
|
+
}
|
|
15678
|
+
return {
|
|
15679
|
+
events,
|
|
15680
|
+
rawEntryCount,
|
|
15681
|
+
skippedEntryCount,
|
|
15682
|
+
nextPageToken: pageToken,
|
|
15683
|
+
filter
|
|
15684
|
+
};
|
|
15685
|
+
}
|
|
15686
|
+
|
|
15687
|
+
// ../integration-traffic/src/rules.ts
|
|
15688
|
+
var DEFAULT_AI_CRAWLER_RULES = [
|
|
15689
|
+
{
|
|
15690
|
+
id: "openai-gptbot",
|
|
15691
|
+
operator: "OpenAI",
|
|
15692
|
+
product: "GPTBot",
|
|
15693
|
+
purpose: "training",
|
|
15694
|
+
userAgentPatterns: [/GPTBot\//i]
|
|
15695
|
+
},
|
|
15696
|
+
{
|
|
15697
|
+
id: "openai-searchbot",
|
|
15698
|
+
operator: "OpenAI",
|
|
15699
|
+
product: "OAI-SearchBot",
|
|
15700
|
+
purpose: "search",
|
|
15701
|
+
userAgentPatterns: [/OAI-SearchBot\//i]
|
|
15702
|
+
},
|
|
15703
|
+
{
|
|
15704
|
+
id: "openai-chatgpt-user",
|
|
15705
|
+
operator: "OpenAI",
|
|
15706
|
+
product: "ChatGPT-User",
|
|
15707
|
+
purpose: "user-agent",
|
|
15708
|
+
userAgentPatterns: [/ChatGPT-User\//i]
|
|
15709
|
+
},
|
|
15710
|
+
{
|
|
15711
|
+
id: "anthropic-claudebot",
|
|
15712
|
+
operator: "Anthropic",
|
|
15713
|
+
product: "ClaudeBot",
|
|
15714
|
+
purpose: "training",
|
|
15715
|
+
userAgentPatterns: [/ClaudeBot\//i, /Claude-Web\//i, /anthropic-ai/i]
|
|
15716
|
+
},
|
|
15717
|
+
{
|
|
15718
|
+
id: "perplexity-bot",
|
|
15719
|
+
operator: "Perplexity",
|
|
15720
|
+
product: "PerplexityBot",
|
|
15721
|
+
purpose: "search",
|
|
15722
|
+
userAgentPatterns: [/PerplexityBot\//i]
|
|
15723
|
+
},
|
|
15724
|
+
{
|
|
15725
|
+
id: "google-extended",
|
|
15726
|
+
operator: "Google",
|
|
15727
|
+
product: "Google-Extended",
|
|
15728
|
+
purpose: "training-control",
|
|
15729
|
+
userAgentPatterns: [/Google-Extended/i]
|
|
15730
|
+
},
|
|
15731
|
+
{
|
|
15732
|
+
id: "bytespider",
|
|
15733
|
+
operator: "ByteDance",
|
|
15734
|
+
product: "Bytespider",
|
|
15735
|
+
purpose: "training",
|
|
15736
|
+
userAgentPatterns: [/Bytespider/i]
|
|
15737
|
+
},
|
|
15738
|
+
{
|
|
15739
|
+
id: "applebot-extended",
|
|
15740
|
+
operator: "Apple",
|
|
15741
|
+
product: "Applebot-Extended",
|
|
15742
|
+
purpose: "training",
|
|
15743
|
+
userAgentPatterns: [/Applebot-Extended/i]
|
|
15744
|
+
},
|
|
15745
|
+
{
|
|
15746
|
+
id: "meta-externalagent",
|
|
15747
|
+
operator: "Meta",
|
|
15748
|
+
product: "meta-externalagent",
|
|
15749
|
+
purpose: "training",
|
|
15750
|
+
userAgentPatterns: [/meta-externalagent/i]
|
|
15751
|
+
},
|
|
15752
|
+
{
|
|
15753
|
+
id: "ccbot",
|
|
15754
|
+
operator: "Common Crawl",
|
|
15755
|
+
product: "CCBot",
|
|
15756
|
+
purpose: "crawl",
|
|
15757
|
+
userAgentPatterns: [/CCBot\//i]
|
|
15758
|
+
},
|
|
15759
|
+
{
|
|
15760
|
+
id: "cohere-ai",
|
|
15761
|
+
operator: "Cohere",
|
|
15762
|
+
product: "cohere-ai",
|
|
15763
|
+
purpose: "training",
|
|
15764
|
+
userAgentPatterns: [/cohere-ai/i]
|
|
15765
|
+
},
|
|
15766
|
+
{
|
|
15767
|
+
id: "diffbot",
|
|
15768
|
+
operator: "Diffbot",
|
|
15769
|
+
product: "Diffbot",
|
|
15770
|
+
purpose: "crawl",
|
|
15771
|
+
userAgentPatterns: [/Diffbot/i]
|
|
15772
|
+
},
|
|
15773
|
+
{
|
|
15774
|
+
id: "mistral-ai",
|
|
15775
|
+
operator: "Mistral AI",
|
|
15776
|
+
product: "MistralAI-User",
|
|
15777
|
+
purpose: "crawl",
|
|
15778
|
+
userAgentPatterns: [/MistralAI/i]
|
|
15779
|
+
}
|
|
15780
|
+
];
|
|
15781
|
+
var DEFAULT_AI_REFERRER_RULES = [
|
|
15782
|
+
{ domain: "chatgpt.com", operator: "OpenAI", product: "ChatGPT" },
|
|
15783
|
+
{ domain: "chat.openai.com", operator: "OpenAI", product: "ChatGPT" },
|
|
15784
|
+
{ domain: "perplexity.ai", operator: "Perplexity", product: "Perplexity" },
|
|
15785
|
+
{ domain: "claude.ai", operator: "Anthropic", product: "Claude" },
|
|
15786
|
+
{ domain: "gemini.google.com", operator: "Google", product: "Gemini" },
|
|
15787
|
+
{ domain: "copilot.microsoft.com", operator: "Microsoft", product: "Copilot" },
|
|
15788
|
+
{ domain: "phind.com", operator: "Phind", product: "Phind" },
|
|
15789
|
+
{ domain: "you.com", operator: "You.com", product: "You.com" },
|
|
15790
|
+
{ domain: "meta.ai", operator: "Meta", product: "Meta AI" }
|
|
15791
|
+
];
|
|
15792
|
+
|
|
15793
|
+
// ../integration-traffic/src/classifier.ts
|
|
15794
|
+
function normalizeHost(host) {
|
|
15795
|
+
return host.trim().toLowerCase().replace(/^www\./, "");
|
|
15796
|
+
}
|
|
15797
|
+
function hostMatches(host, domain) {
|
|
15798
|
+
const normalizedHost = normalizeHost(host);
|
|
15799
|
+
const normalizedDomain = normalizeHost(domain);
|
|
15800
|
+
return normalizedHost === normalizedDomain || normalizedHost.endsWith(`.${normalizedDomain}`);
|
|
15801
|
+
}
|
|
15802
|
+
function hostFromUrl(value) {
|
|
15803
|
+
if (!value) return null;
|
|
15804
|
+
try {
|
|
15805
|
+
return normalizeHost(new URL(value).hostname);
|
|
15806
|
+
} catch {
|
|
15807
|
+
return null;
|
|
15808
|
+
}
|
|
15809
|
+
}
|
|
15810
|
+
function utmSourceFromQuery(queryString) {
|
|
15811
|
+
if (!queryString) return null;
|
|
15812
|
+
const params = new URLSearchParams(queryString);
|
|
15813
|
+
const source = params.get("utm_source");
|
|
15814
|
+
return source ? normalizeHost(source) : null;
|
|
15815
|
+
}
|
|
15816
|
+
function classifyCrawler(event) {
|
|
15817
|
+
const userAgent = event.userAgent?.trim();
|
|
15818
|
+
if (!userAgent) return null;
|
|
15819
|
+
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
15820
|
+
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
15821
|
+
return {
|
|
15822
|
+
botId: rule.id,
|
|
15823
|
+
operator: rule.operator,
|
|
15824
|
+
product: rule.product,
|
|
15825
|
+
purpose: rule.purpose,
|
|
15826
|
+
verificationStatus: "claimed_unverified",
|
|
15827
|
+
matchedUserAgent: userAgent
|
|
15828
|
+
};
|
|
15829
|
+
}
|
|
15830
|
+
}
|
|
15831
|
+
return null;
|
|
15832
|
+
}
|
|
15833
|
+
function classifyAiReferral(event) {
|
|
15834
|
+
const refererHost = hostFromUrl(event.referer);
|
|
15835
|
+
if (refererHost) {
|
|
15836
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(refererHost, candidate.domain));
|
|
15837
|
+
if (rule) {
|
|
15838
|
+
return {
|
|
15839
|
+
operator: rule.operator,
|
|
15840
|
+
product: rule.product,
|
|
15841
|
+
sourceDomain: refererHost,
|
|
15842
|
+
evidenceType: "referer"
|
|
15843
|
+
};
|
|
15844
|
+
}
|
|
15845
|
+
}
|
|
15846
|
+
const utmSource = utmSourceFromQuery(event.queryString);
|
|
15847
|
+
if (utmSource) {
|
|
15848
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(utmSource, candidate.domain));
|
|
15849
|
+
if (rule) {
|
|
15850
|
+
return {
|
|
15851
|
+
operator: rule.operator,
|
|
15852
|
+
product: rule.product,
|
|
15853
|
+
sourceDomain: utmSource,
|
|
15854
|
+
evidenceType: "utm"
|
|
15855
|
+
};
|
|
15856
|
+
}
|
|
15857
|
+
}
|
|
15858
|
+
return null;
|
|
15859
|
+
}
|
|
15860
|
+
|
|
15861
|
+
// ../integration-traffic/src/rollup.ts
|
|
15862
|
+
var DEFAULT_SAMPLE_LIMIT = 25;
|
|
15863
|
+
var UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
15864
|
+
var LONG_HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
|
|
15865
|
+
var NUMERIC_SEGMENT = /^\d+$/;
|
|
15866
|
+
function normalizeTrafficPathPattern(path15) {
|
|
15867
|
+
const cleanPath = path15.trim() || "/";
|
|
15868
|
+
const pathOnly = cleanPath.split("?")[0] || "/";
|
|
15869
|
+
const segments = pathOnly.split("/").map((segment) => {
|
|
15870
|
+
if (!segment) return segment;
|
|
15871
|
+
if (UUID_SEGMENT.test(segment) || LONG_HEX_SEGMENT.test(segment) || NUMERIC_SEGMENT.test(segment)) {
|
|
15872
|
+
return ":id";
|
|
15873
|
+
}
|
|
15874
|
+
return segment;
|
|
15875
|
+
});
|
|
15876
|
+
const normalized = segments.join("/");
|
|
15877
|
+
return normalized.startsWith("/") ? normalized : `/${normalized}`;
|
|
15878
|
+
}
|
|
15879
|
+
function hourBucket(value) {
|
|
15880
|
+
const date = new Date(value);
|
|
15881
|
+
if (Number.isNaN(date.getTime())) return value;
|
|
15882
|
+
date.setUTCMinutes(0, 0, 0);
|
|
15883
|
+
return date.toISOString();
|
|
15884
|
+
}
|
|
15885
|
+
function sortCrawlerBuckets(a, b) {
|
|
15886
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
15887
|
+
}
|
|
15888
|
+
function sortReferralBuckets(a, b) {
|
|
15889
|
+
return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
15890
|
+
}
|
|
15891
|
+
function topEntries(map, limit) {
|
|
15892
|
+
return [...map.values()].sort((a, b) => b.hits - a.hits || JSON.stringify(a.fields).localeCompare(JSON.stringify(b.fields))).slice(0, limit).map((entry) => ({ ...entry.fields, hits: entry.hits }));
|
|
15893
|
+
}
|
|
15894
|
+
function buildTrafficProbeReport(events, options = {}) {
|
|
15895
|
+
const sampleLimit = options.sampleLimit ?? DEFAULT_SAMPLE_LIMIT;
|
|
15896
|
+
const crawlerBuckets = /* @__PURE__ */ new Map();
|
|
15897
|
+
const aiReferralBuckets = /* @__PURE__ */ new Map();
|
|
15898
|
+
const topBots = /* @__PURE__ */ new Map();
|
|
15899
|
+
const topCrawlerPaths = /* @__PURE__ */ new Map();
|
|
15900
|
+
const topAiReferrers = /* @__PURE__ */ new Map();
|
|
15901
|
+
const topAiReferralLandingPaths = /* @__PURE__ */ new Map();
|
|
15902
|
+
let crawlerHits = 0;
|
|
15903
|
+
let aiReferralHits = 0;
|
|
15904
|
+
let unknownHits = 0;
|
|
15905
|
+
const samples = [];
|
|
15906
|
+
for (const event of events) {
|
|
15907
|
+
const tsHour = hourBucket(event.observedAt);
|
|
15908
|
+
const pathNormalized = normalizeTrafficPathPattern(event.path);
|
|
15909
|
+
const crawler = classifyCrawler(event);
|
|
15910
|
+
const aiReferral = classifyAiReferral(event);
|
|
15911
|
+
if (crawler) {
|
|
15912
|
+
crawlerHits += 1;
|
|
15913
|
+
const key = [
|
|
15914
|
+
tsHour,
|
|
15915
|
+
crawler.botId,
|
|
15916
|
+
crawler.verificationStatus,
|
|
15917
|
+
pathNormalized,
|
|
15918
|
+
event.status ?? "null"
|
|
15919
|
+
].join(" ");
|
|
15920
|
+
const existing = crawlerBuckets.get(key);
|
|
15921
|
+
if (existing) {
|
|
15922
|
+
existing.hits += 1;
|
|
15923
|
+
} else {
|
|
15924
|
+
crawlerBuckets.set(key, {
|
|
15925
|
+
tsHour,
|
|
15926
|
+
botId: crawler.botId,
|
|
15927
|
+
operator: crawler.operator,
|
|
15928
|
+
product: crawler.product,
|
|
15929
|
+
verificationStatus: crawler.verificationStatus,
|
|
15930
|
+
pathNormalized,
|
|
15931
|
+
status: event.status,
|
|
15932
|
+
hits: 1,
|
|
15933
|
+
sampledUserAgent: event.userAgent
|
|
15934
|
+
});
|
|
15935
|
+
}
|
|
15936
|
+
const botKey = `${crawler.botId} ${crawler.operator}`;
|
|
15937
|
+
const botEntry = topBots.get(botKey);
|
|
15938
|
+
if (botEntry) botEntry.hits += 1;
|
|
15939
|
+
else topBots.set(botKey, { fields: { botId: crawler.botId, operator: crawler.operator }, hits: 1 });
|
|
15940
|
+
incrementBucket(topCrawlerPaths, pathNormalized, { pathNormalized });
|
|
15941
|
+
}
|
|
15942
|
+
if (aiReferral) {
|
|
15943
|
+
aiReferralHits += 1;
|
|
15944
|
+
const key = [
|
|
15945
|
+
tsHour,
|
|
15946
|
+
aiReferral.product,
|
|
15947
|
+
aiReferral.sourceDomain,
|
|
15948
|
+
aiReferral.evidenceType,
|
|
15949
|
+
pathNormalized,
|
|
15950
|
+
event.status ?? "null"
|
|
15951
|
+
].join(" ");
|
|
15952
|
+
const existing = aiReferralBuckets.get(key);
|
|
15953
|
+
if (existing) {
|
|
15954
|
+
existing.hits += 1;
|
|
15955
|
+
} else {
|
|
15956
|
+
aiReferralBuckets.set(key, {
|
|
15957
|
+
tsHour,
|
|
15958
|
+
operator: aiReferral.operator,
|
|
15959
|
+
product: aiReferral.product,
|
|
15960
|
+
sourceDomain: aiReferral.sourceDomain,
|
|
15961
|
+
evidenceType: aiReferral.evidenceType,
|
|
15962
|
+
landingPathNormalized: pathNormalized,
|
|
15963
|
+
status: event.status,
|
|
15964
|
+
hits: 1
|
|
15965
|
+
});
|
|
15966
|
+
}
|
|
15967
|
+
incrementBucket(topAiReferrers, aiReferral.sourceDomain, {
|
|
15968
|
+
sourceDomain: aiReferral.sourceDomain,
|
|
15969
|
+
product: aiReferral.product
|
|
15970
|
+
});
|
|
15971
|
+
incrementBucket(topAiReferralLandingPaths, pathNormalized, { landingPathNormalized: pathNormalized });
|
|
15972
|
+
}
|
|
15973
|
+
if (!crawler && !aiReferral) unknownHits += 1;
|
|
15974
|
+
if (samples.length < sampleLimit) {
|
|
15975
|
+
samples.push({
|
|
15976
|
+
eventId: event.eventId,
|
|
15977
|
+
observedAt: event.observedAt,
|
|
15978
|
+
sourceType: event.sourceType,
|
|
15979
|
+
path: event.path,
|
|
15980
|
+
pathNormalized,
|
|
15981
|
+
status: event.status,
|
|
15982
|
+
userAgent: event.userAgent,
|
|
15983
|
+
referer: event.referer,
|
|
15984
|
+
crawler,
|
|
15985
|
+
aiReferral
|
|
15986
|
+
});
|
|
15987
|
+
}
|
|
15988
|
+
}
|
|
15989
|
+
return {
|
|
15990
|
+
generatedAt: options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
15991
|
+
totals: {
|
|
15992
|
+
normalizedEvents: events.length,
|
|
15993
|
+
crawlerHits,
|
|
15994
|
+
aiReferralHits,
|
|
15995
|
+
unknownHits
|
|
15996
|
+
},
|
|
15997
|
+
crawlerEventsHourly: [...crawlerBuckets.values()].sort(sortCrawlerBuckets),
|
|
15998
|
+
aiReferralEventsHourly: [...aiReferralBuckets.values()].sort(sortReferralBuckets),
|
|
15999
|
+
topBots: topEntries(topBots, 10),
|
|
16000
|
+
topCrawlerPaths: topEntries(topCrawlerPaths, 10),
|
|
16001
|
+
topAiReferrers: topEntries(topAiReferrers, 10),
|
|
16002
|
+
topAiReferralLandingPaths: topEntries(topAiReferralLandingPaths, 10),
|
|
16003
|
+
samples
|
|
16004
|
+
};
|
|
16005
|
+
}
|
|
16006
|
+
function incrementBucket(map, key, fields) {
|
|
16007
|
+
const existing = map.get(key);
|
|
16008
|
+
if (existing) existing.hits += 1;
|
|
16009
|
+
else map.set(key, { fields, hits: 1 });
|
|
16010
|
+
}
|
|
16011
|
+
|
|
16012
|
+
// ../api-routes/src/traffic.ts
|
|
16013
|
+
var DEFAULT_SYNC_WINDOW_MINUTES = 60;
|
|
16014
|
+
var DEFAULT_PAGE_SIZE2 = 1e3;
|
|
16015
|
+
var DEFAULT_MAX_PAGES2 = 5;
|
|
16016
|
+
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
16017
|
+
function parseSourceConfig(row) {
|
|
16018
|
+
return parseJsonColumn(row.configJson, {});
|
|
16019
|
+
}
|
|
16020
|
+
function rowToDto(row) {
|
|
16021
|
+
return {
|
|
16022
|
+
id: row.id,
|
|
16023
|
+
projectId: row.projectId,
|
|
16024
|
+
sourceType: row.sourceType,
|
|
16025
|
+
displayName: row.displayName,
|
|
16026
|
+
status: row.status,
|
|
16027
|
+
lastSyncedAt: row.lastSyncedAt ?? null,
|
|
16028
|
+
lastCursor: row.lastCursor ?? null,
|
|
16029
|
+
lastError: row.lastError ?? null,
|
|
16030
|
+
archivedAt: row.archivedAt ?? null,
|
|
16031
|
+
config: parseSourceConfig(row),
|
|
16032
|
+
createdAt: row.createdAt,
|
|
16033
|
+
updatedAt: row.updatedAt
|
|
16034
|
+
};
|
|
16035
|
+
}
|
|
16036
|
+
async function defaultResolveAccessToken(record) {
|
|
16037
|
+
if (record.authMode === TrafficSourceAuthModes["service-account"]) {
|
|
16038
|
+
if (!record.clientEmail || !record.privateKey) {
|
|
16039
|
+
throw validationError("Service-account credentials missing client_email or private_key");
|
|
16040
|
+
}
|
|
16041
|
+
return getCloudLoggingAccessToken(record.clientEmail, record.privateKey);
|
|
16042
|
+
}
|
|
16043
|
+
throw validationError(
|
|
16044
|
+
"OAuth-mode Cloud Run sync is not yet supported in v1. Provide a service-account key file."
|
|
16045
|
+
);
|
|
16046
|
+
}
|
|
16047
|
+
async function trafficRoutes(app, opts) {
|
|
16048
|
+
const pullEvents = opts.pullCloudRunEvents ?? listCloudRunTrafficEvents;
|
|
16049
|
+
const resolveAccessToken2 = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
16050
|
+
const syncWindowMinutes = opts.defaultSyncWindowMinutes ?? DEFAULT_SYNC_WINDOW_MINUTES;
|
|
16051
|
+
const pageSize = opts.defaultPageSize ?? DEFAULT_PAGE_SIZE2;
|
|
16052
|
+
const maxPages = opts.defaultMaxPages ?? DEFAULT_MAX_PAGES2;
|
|
16053
|
+
const sampleLimit = opts.defaultSampleLimit ?? DEFAULT_SAMPLE_LIMIT2;
|
|
16054
|
+
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
16055
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16056
|
+
const body = request.body ?? {};
|
|
16057
|
+
const { gcpProjectId, serviceName, location, displayName, keyJson } = body;
|
|
16058
|
+
if (!gcpProjectId || typeof gcpProjectId !== "string") {
|
|
16059
|
+
throw validationError("gcpProjectId is required");
|
|
16060
|
+
}
|
|
16061
|
+
if (!keyJson) {
|
|
16062
|
+
throw validationError(
|
|
16063
|
+
"keyJson is required for v1 (service-account JSON content). OAuth-mode Cloud Run is not yet supported."
|
|
16064
|
+
);
|
|
16065
|
+
}
|
|
16066
|
+
if (!opts.cloudRunCredentialStore) {
|
|
16067
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
16068
|
+
}
|
|
16069
|
+
let parsed;
|
|
16070
|
+
try {
|
|
16071
|
+
parsed = JSON.parse(keyJson);
|
|
16072
|
+
} catch {
|
|
16073
|
+
throw validationError("Invalid JSON in keyJson");
|
|
16074
|
+
}
|
|
16075
|
+
if (!parsed.client_email || !parsed.private_key) {
|
|
16076
|
+
throw validationError("Service-account JSON must contain client_email and private_key");
|
|
16077
|
+
}
|
|
16078
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16079
|
+
const existing = opts.cloudRunCredentialStore.getConnection(project.name);
|
|
16080
|
+
opts.cloudRunCredentialStore.upsertConnection({
|
|
16081
|
+
projectName: project.name,
|
|
16082
|
+
gcpProjectId,
|
|
16083
|
+
serviceName: serviceName ?? void 0,
|
|
16084
|
+
location: location ?? void 0,
|
|
16085
|
+
authMode: TrafficSourceAuthModes["service-account"],
|
|
16086
|
+
clientEmail: parsed.client_email,
|
|
16087
|
+
privateKey: parsed.private_key,
|
|
16088
|
+
createdAt: existing?.createdAt ?? now,
|
|
16089
|
+
updatedAt: now
|
|
16090
|
+
});
|
|
16091
|
+
const activeSource = app.db.select().from(trafficSources).where(eq23(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
16092
|
+
const config = {
|
|
16093
|
+
gcpProjectId,
|
|
16094
|
+
serviceName: serviceName ?? null,
|
|
16095
|
+
location: location ?? null,
|
|
16096
|
+
authMode: TrafficSourceAuthModes["service-account"]
|
|
16097
|
+
};
|
|
16098
|
+
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
16099
|
+
let sourceRow;
|
|
16100
|
+
if (activeSource) {
|
|
16101
|
+
app.db.update(trafficSources).set({
|
|
16102
|
+
displayName: fallbackName,
|
|
16103
|
+
status: TrafficSourceStatuses.connected,
|
|
16104
|
+
lastError: null,
|
|
16105
|
+
configJson: JSON.stringify(config),
|
|
16106
|
+
updatedAt: now
|
|
16107
|
+
}).where(eq23(trafficSources.id, activeSource.id)).run();
|
|
16108
|
+
sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, activeSource.id)).get();
|
|
16109
|
+
} else {
|
|
16110
|
+
const newId = crypto20.randomUUID();
|
|
16111
|
+
app.db.insert(trafficSources).values({
|
|
16112
|
+
id: newId,
|
|
16113
|
+
projectId: project.id,
|
|
16114
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
16115
|
+
displayName: fallbackName,
|
|
16116
|
+
status: TrafficSourceStatuses.connected,
|
|
16117
|
+
lastSyncedAt: null,
|
|
16118
|
+
lastCursor: null,
|
|
16119
|
+
lastError: null,
|
|
16120
|
+
archivedAt: null,
|
|
16121
|
+
configJson: JSON.stringify(config),
|
|
16122
|
+
createdAt: now,
|
|
16123
|
+
updatedAt: now
|
|
16124
|
+
}).run();
|
|
16125
|
+
sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, newId)).get();
|
|
16126
|
+
}
|
|
16127
|
+
writeAuditLog(app.db, {
|
|
16128
|
+
projectId: project.id,
|
|
16129
|
+
actor: "api",
|
|
16130
|
+
action: "traffic.cloud-run.connected",
|
|
16131
|
+
entityType: "traffic_source",
|
|
16132
|
+
entityId: sourceRow.id
|
|
16133
|
+
});
|
|
16134
|
+
return rowToDto(sourceRow);
|
|
16135
|
+
});
|
|
16136
|
+
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
16137
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16138
|
+
const sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, request.params.id)).get();
|
|
16139
|
+
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
16140
|
+
throw notFound("Traffic source", request.params.id);
|
|
16141
|
+
}
|
|
16142
|
+
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"]) {
|
|
16143
|
+
throw validationError(
|
|
16144
|
+
`Sync for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run is supported in v1.`
|
|
16145
|
+
);
|
|
16146
|
+
}
|
|
16147
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
16148
|
+
if (!credentialStore) {
|
|
16149
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
16150
|
+
}
|
|
16151
|
+
const credential = credentialStore.getConnection(project.name);
|
|
16152
|
+
if (!credential) {
|
|
16153
|
+
throw validationError(
|
|
16154
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
16155
|
+
);
|
|
16156
|
+
}
|
|
16157
|
+
const config = parseSourceConfig(sourceRow);
|
|
16158
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
16159
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
16160
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
16161
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
16162
|
+
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
16163
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
16164
|
+
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
16165
|
+
const lastSyncedMs = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
16166
|
+
const windowStart = new Date(
|
|
16167
|
+
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
16168
|
+
);
|
|
16169
|
+
const startedAt = windowEnd.toISOString();
|
|
16170
|
+
const runId = crypto20.randomUUID();
|
|
16171
|
+
app.db.insert(runs).values({
|
|
16172
|
+
id: runId,
|
|
16173
|
+
projectId: project.id,
|
|
16174
|
+
kind: RunKinds["traffic-sync"],
|
|
16175
|
+
status: RunStatuses.running,
|
|
16176
|
+
trigger: RunTriggers.manual,
|
|
16177
|
+
startedAt,
|
|
16178
|
+
createdAt: startedAt
|
|
16179
|
+
}).run();
|
|
16180
|
+
let accessToken;
|
|
16181
|
+
try {
|
|
16182
|
+
accessToken = await resolveAccessToken2(credential);
|
|
16183
|
+
} catch (e) {
|
|
16184
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
16185
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(runs.id, runId)).run();
|
|
16186
|
+
app.db.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16187
|
+
throw validationError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
16188
|
+
}
|
|
16189
|
+
let allEvents = [];
|
|
16190
|
+
try {
|
|
16191
|
+
const page = await pullEvents(accessToken, {
|
|
16192
|
+
gcpProjectId,
|
|
16193
|
+
serviceName,
|
|
16194
|
+
location,
|
|
16195
|
+
startTime: windowStart.toISOString(),
|
|
16196
|
+
endTime: windowEnd.toISOString(),
|
|
16197
|
+
pageSize,
|
|
16198
|
+
maxPages
|
|
16199
|
+
});
|
|
16200
|
+
allEvents = page.events;
|
|
16201
|
+
} catch (e) {
|
|
16202
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
16203
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(runs.id, runId)).run();
|
|
16204
|
+
app.db.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16205
|
+
throw validationError(`Cloud Run pull failed: ${msg}`);
|
|
16206
|
+
}
|
|
16207
|
+
const report = buildTrafficProbeReport(allEvents, { sampleLimit });
|
|
16208
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16209
|
+
let crawlerBucketRows = 0;
|
|
16210
|
+
let aiReferralBucketRows = 0;
|
|
16211
|
+
let sampleRows = 0;
|
|
16212
|
+
app.db.transaction((tx) => {
|
|
16213
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
16214
|
+
const status = bucket.status ?? 0;
|
|
16215
|
+
tx.insert(crawlerEventsHourly).values({
|
|
16216
|
+
projectId: project.id,
|
|
16217
|
+
sourceId: sourceRow.id,
|
|
16218
|
+
tsHour: bucket.tsHour,
|
|
16219
|
+
botId: bucket.botId,
|
|
16220
|
+
operator: bucket.operator,
|
|
16221
|
+
verificationStatus: bucket.verificationStatus,
|
|
16222
|
+
pathNormalized: bucket.pathNormalized,
|
|
16223
|
+
status,
|
|
16224
|
+
hits: bucket.hits,
|
|
16225
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
16226
|
+
createdAt: finishedAt,
|
|
16227
|
+
updatedAt: finishedAt
|
|
16228
|
+
}).onConflictDoUpdate({
|
|
16229
|
+
target: [
|
|
16230
|
+
crawlerEventsHourly.projectId,
|
|
16231
|
+
crawlerEventsHourly.sourceId,
|
|
16232
|
+
crawlerEventsHourly.tsHour,
|
|
16233
|
+
crawlerEventsHourly.botId,
|
|
16234
|
+
crawlerEventsHourly.verificationStatus,
|
|
16235
|
+
crawlerEventsHourly.pathNormalized,
|
|
16236
|
+
crawlerEventsHourly.status
|
|
16237
|
+
],
|
|
16238
|
+
set: {
|
|
16239
|
+
hits: sql7`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
16240
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
16241
|
+
updatedAt: finishedAt
|
|
16242
|
+
}
|
|
16243
|
+
}).run();
|
|
16244
|
+
crawlerBucketRows += 1;
|
|
16245
|
+
}
|
|
16246
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
16247
|
+
const status = bucket.status ?? 0;
|
|
16248
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
16249
|
+
projectId: project.id,
|
|
16250
|
+
sourceId: sourceRow.id,
|
|
16251
|
+
tsHour: bucket.tsHour,
|
|
16252
|
+
product: bucket.product,
|
|
16253
|
+
operator: bucket.operator,
|
|
16254
|
+
sourceDomain: bucket.sourceDomain,
|
|
16255
|
+
evidenceType: bucket.evidenceType,
|
|
16256
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
16257
|
+
status,
|
|
16258
|
+
sessionsOrHits: bucket.hits,
|
|
16259
|
+
usersEstimated: null,
|
|
16260
|
+
createdAt: finishedAt,
|
|
16261
|
+
updatedAt: finishedAt
|
|
16262
|
+
}).onConflictDoUpdate({
|
|
16263
|
+
target: [
|
|
16264
|
+
aiReferralEventsHourly.projectId,
|
|
16265
|
+
aiReferralEventsHourly.sourceId,
|
|
16266
|
+
aiReferralEventsHourly.tsHour,
|
|
16267
|
+
aiReferralEventsHourly.product,
|
|
16268
|
+
aiReferralEventsHourly.sourceDomain,
|
|
16269
|
+
aiReferralEventsHourly.evidenceType,
|
|
16270
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
16271
|
+
aiReferralEventsHourly.status
|
|
16272
|
+
],
|
|
16273
|
+
set: {
|
|
16274
|
+
sessionsOrHits: sql7`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
16275
|
+
updatedAt: finishedAt
|
|
16276
|
+
}
|
|
16277
|
+
}).run();
|
|
16278
|
+
aiReferralBucketRows += 1;
|
|
16279
|
+
}
|
|
16280
|
+
for (const sample of report.samples) {
|
|
16281
|
+
const eventType = sample.crawler ? "crawler" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
16282
|
+
const refererHost = (() => {
|
|
16283
|
+
if (!sample.referer) return null;
|
|
16284
|
+
try {
|
|
16285
|
+
return new URL(sample.referer).hostname;
|
|
16286
|
+
} catch {
|
|
16287
|
+
return null;
|
|
16288
|
+
}
|
|
16289
|
+
})();
|
|
16290
|
+
tx.insert(rawEventSamples).values({
|
|
16291
|
+
id: crypto20.randomUUID(),
|
|
16292
|
+
projectId: project.id,
|
|
16293
|
+
sourceId: sourceRow.id,
|
|
16294
|
+
ts: sample.observedAt,
|
|
16295
|
+
eventType,
|
|
16296
|
+
ipHash: null,
|
|
16297
|
+
userAgent: sample.userAgent,
|
|
16298
|
+
pathNormalized: sample.pathNormalized,
|
|
16299
|
+
status: sample.status,
|
|
16300
|
+
refererHost,
|
|
16301
|
+
classifierDetailsJson: JSON.stringify({
|
|
16302
|
+
crawler: sample.crawler,
|
|
16303
|
+
aiReferral: sample.aiReferral
|
|
16304
|
+
}),
|
|
16305
|
+
createdAt: finishedAt
|
|
16306
|
+
}).run();
|
|
16307
|
+
sampleRows += 1;
|
|
16308
|
+
}
|
|
16309
|
+
tx.update(trafficSources).set({
|
|
16310
|
+
status: TrafficSourceStatuses.connected,
|
|
16311
|
+
lastSyncedAt: finishedAt,
|
|
16312
|
+
lastError: null,
|
|
16313
|
+
updatedAt: finishedAt
|
|
16314
|
+
}).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16315
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
16316
|
+
});
|
|
16317
|
+
writeAuditLog(app.db, {
|
|
16318
|
+
projectId: project.id,
|
|
16319
|
+
actor: "api",
|
|
16320
|
+
action: "traffic.cloud-run.synced",
|
|
16321
|
+
entityType: "traffic_source",
|
|
16322
|
+
entityId: sourceRow.id
|
|
16323
|
+
});
|
|
16324
|
+
const response = {
|
|
16325
|
+
sourceId: sourceRow.id,
|
|
16326
|
+
runId,
|
|
16327
|
+
syncedAt: finishedAt,
|
|
16328
|
+
pulledEvents: report.totals.normalizedEvents,
|
|
16329
|
+
crawlerHits: report.totals.crawlerHits,
|
|
16330
|
+
aiReferralHits: report.totals.aiReferralHits,
|
|
16331
|
+
unknownHits: report.totals.unknownHits,
|
|
16332
|
+
crawlerBucketRows,
|
|
16333
|
+
aiReferralBucketRows,
|
|
16334
|
+
sampleRows,
|
|
16335
|
+
windowStart: windowStart.toISOString(),
|
|
16336
|
+
windowEnd: windowEnd.toISOString()
|
|
16337
|
+
};
|
|
16338
|
+
return response;
|
|
16339
|
+
});
|
|
16340
|
+
}
|
|
16341
|
+
|
|
15316
16342
|
// ../api-routes/src/doctor/checks/bing-auth.ts
|
|
15317
16343
|
var BING_AUTH_CHECKS = [
|
|
15318
16344
|
{
|
|
@@ -16164,6 +17190,11 @@ async function apiRoutes(app, opts) {
|
|
|
16164
17190
|
googleConnectionStore: opts.googleConnectionStore,
|
|
16165
17191
|
getGoogleAuthConfig: opts.getGoogleAuthConfig
|
|
16166
17192
|
});
|
|
17193
|
+
await api.register(trafficRoutes, {
|
|
17194
|
+
cloudRunCredentialStore: opts.cloudRunCredentialStore,
|
|
17195
|
+
pullCloudRunEvents: opts.pullCloudRunEvents,
|
|
17196
|
+
resolveCloudRunAccessToken: opts.resolveCloudRunAccessToken
|
|
17197
|
+
});
|
|
16167
17198
|
await api.register(backlinksRoutes, {
|
|
16168
17199
|
getBacklinksStatus: opts.getBacklinksStatus,
|
|
16169
17200
|
onInstallBacklinks: opts.onInstallBacklinks,
|
|
@@ -18589,8 +19620,40 @@ function removeGa4Connection(config, projectName) {
|
|
|
18589
19620
|
return true;
|
|
18590
19621
|
}
|
|
18591
19622
|
|
|
18592
|
-
// src/
|
|
19623
|
+
// src/cloud-run-config.ts
|
|
18593
19624
|
function ensureConnections3(config) {
|
|
19625
|
+
if (!config.cloudRun) config.cloudRun = {};
|
|
19626
|
+
if (!config.cloudRun.connections) config.cloudRun.connections = [];
|
|
19627
|
+
return config.cloudRun.connections;
|
|
19628
|
+
}
|
|
19629
|
+
function getCloudRunConnection(config, projectName) {
|
|
19630
|
+
return (config.cloudRun?.connections ?? []).find((c) => c.projectName === projectName);
|
|
19631
|
+
}
|
|
19632
|
+
function upsertCloudRunConnection(config, connection) {
|
|
19633
|
+
const connections = ensureConnections3(config);
|
|
19634
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
19635
|
+
if (index === -1) {
|
|
19636
|
+
connections.push(connection);
|
|
19637
|
+
return connection;
|
|
19638
|
+
}
|
|
19639
|
+
connections[index] = connection;
|
|
19640
|
+
return connection;
|
|
19641
|
+
}
|
|
19642
|
+
function removeCloudRunConnection(config, projectName) {
|
|
19643
|
+
const connections = config.cloudRun?.connections;
|
|
19644
|
+
if (!connections?.length) return false;
|
|
19645
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
19646
|
+
if (next.length === connections.length) return false;
|
|
19647
|
+
if (!config.cloudRun) return false;
|
|
19648
|
+
config.cloudRun.connections = next;
|
|
19649
|
+
if (next.length === 0) {
|
|
19650
|
+
delete config.cloudRun;
|
|
19651
|
+
}
|
|
19652
|
+
return true;
|
|
19653
|
+
}
|
|
19654
|
+
|
|
19655
|
+
// src/wordpress-config.ts
|
|
19656
|
+
function ensureConnections4(config) {
|
|
18594
19657
|
if (!config.wordpress) config.wordpress = {};
|
|
18595
19658
|
if (!config.wordpress.connections) config.wordpress.connections = [];
|
|
18596
19659
|
return config.wordpress.connections;
|
|
@@ -18607,7 +19670,7 @@ function getWordpressConnection(config, projectName) {
|
|
|
18607
19670
|
return (config.wordpress?.connections ?? []).find((connection) => connection.projectName === projectName);
|
|
18608
19671
|
}
|
|
18609
19672
|
function upsertWordpressConnection(config, connection) {
|
|
18610
|
-
const connections =
|
|
19673
|
+
const connections = ensureConnections4(config);
|
|
18611
19674
|
const normalized = normalizeConnection(connection);
|
|
18612
19675
|
const index = connections.findIndex((entry) => entry.projectName === connection.projectName);
|
|
18613
19676
|
if (index === -1) {
|
|
@@ -18641,11 +19704,11 @@ function removeWordpressConnection(config, projectName) {
|
|
|
18641
19704
|
}
|
|
18642
19705
|
|
|
18643
19706
|
// src/job-runner.ts
|
|
18644
|
-
import
|
|
19707
|
+
import crypto21 from "crypto";
|
|
18645
19708
|
import fs7 from "fs";
|
|
18646
19709
|
import path9 from "path";
|
|
18647
19710
|
import os4 from "os";
|
|
18648
|
-
import { and as and12, eq as
|
|
19711
|
+
import { and as and12, eq as eq24, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
18649
19712
|
|
|
18650
19713
|
// src/citation-utils.ts
|
|
18651
19714
|
function domainMatches(domain, canonicalDomain) {
|
|
@@ -18906,7 +19969,7 @@ var JobRunner = class {
|
|
|
18906
19969
|
if (stale.length === 0) return;
|
|
18907
19970
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
18908
19971
|
for (const run of stale) {
|
|
18909
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
19972
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq24(runs.id, run.id)).run();
|
|
18910
19973
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
18911
19974
|
}
|
|
18912
19975
|
}
|
|
@@ -18934,10 +19997,10 @@ var JobRunner = class {
|
|
|
18934
19997
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
18935
19998
|
}
|
|
18936
19999
|
if (existingRun.status === "queued") {
|
|
18937
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(and12(
|
|
20000
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and12(eq24(runs.id, runId), eq24(runs.status, "queued"))).run();
|
|
18938
20001
|
}
|
|
18939
20002
|
this.throwIfRunCancelled(runId);
|
|
18940
|
-
const project = this.db.select().from(projects).where(
|
|
20003
|
+
const project = this.db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
18941
20004
|
if (!project) {
|
|
18942
20005
|
throw new Error(`Project ${projectId} not found`);
|
|
18943
20006
|
}
|
|
@@ -18957,8 +20020,8 @@ var JobRunner = class {
|
|
|
18957
20020
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
18958
20021
|
}
|
|
18959
20022
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
18960
|
-
projectQueries = this.db.select().from(queries).where(
|
|
18961
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
20023
|
+
projectQueries = this.db.select().from(queries).where(eq24(queries.projectId, projectId)).all();
|
|
20024
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq24(competitors.projectId, projectId)).all();
|
|
18962
20025
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
18963
20026
|
const allDomains = effectiveDomains({
|
|
18964
20027
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -18974,7 +20037,7 @@ var JobRunner = class {
|
|
|
18974
20037
|
const todayPeriod = getCurrentUsageDay();
|
|
18975
20038
|
for (const p of activeProviders) {
|
|
18976
20039
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
18977
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
20040
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq24(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
18978
20041
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
18979
20042
|
if (providerUsage + queriesPerProvider > limit) {
|
|
18980
20043
|
throw new Error(
|
|
@@ -19034,7 +20097,7 @@ var JobRunner = class {
|
|
|
19034
20097
|
);
|
|
19035
20098
|
let screenshotRelPath = null;
|
|
19036
20099
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
19037
|
-
const snapshotId =
|
|
20100
|
+
const snapshotId = crypto21.randomUUID();
|
|
19038
20101
|
const screenshotDir = path9.join(os4.homedir(), ".canonry", "screenshots", runId);
|
|
19039
20102
|
if (!fs7.existsSync(screenshotDir)) fs7.mkdirSync(screenshotDir, { recursive: true });
|
|
19040
20103
|
const destPath = path9.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -19064,7 +20127,7 @@ var JobRunner = class {
|
|
|
19064
20127
|
}).run();
|
|
19065
20128
|
} else {
|
|
19066
20129
|
this.db.insert(querySnapshots).values({
|
|
19067
|
-
id:
|
|
20130
|
+
id: crypto21.randomUUID(),
|
|
19068
20131
|
runId,
|
|
19069
20132
|
queryId: q.id,
|
|
19070
20133
|
provider: providerName,
|
|
@@ -19115,12 +20178,12 @@ var JobRunner = class {
|
|
|
19115
20178
|
const someFailed = providerErrors.size > 0;
|
|
19116
20179
|
if (allFailed) {
|
|
19117
20180
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
19118
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
20181
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq24(runs.id, runId)).run();
|
|
19119
20182
|
} else if (someFailed) {
|
|
19120
20183
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
19121
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
20184
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq24(runs.id, runId)).run();
|
|
19122
20185
|
} else {
|
|
19123
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20186
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq24(runs.id, runId)).run();
|
|
19124
20187
|
}
|
|
19125
20188
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
19126
20189
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -19155,7 +20218,7 @@ var JobRunner = class {
|
|
|
19155
20218
|
status: "failed",
|
|
19156
20219
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19157
20220
|
error: errorMessage
|
|
19158
|
-
}).where(
|
|
20221
|
+
}).where(eq24(runs.id, runId)).run();
|
|
19159
20222
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
19160
20223
|
trackEvent("run.completed", {
|
|
19161
20224
|
status: "failed",
|
|
@@ -19176,7 +20239,7 @@ var JobRunner = class {
|
|
|
19176
20239
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19177
20240
|
const period = now.slice(0, 10);
|
|
19178
20241
|
this.db.insert(usageCounters).values({
|
|
19179
|
-
id:
|
|
20242
|
+
id: crypto21.randomUUID(),
|
|
19180
20243
|
scope,
|
|
19181
20244
|
period,
|
|
19182
20245
|
metric,
|
|
@@ -19184,7 +20247,7 @@ var JobRunner = class {
|
|
|
19184
20247
|
updatedAt: now
|
|
19185
20248
|
}).onConflictDoUpdate({
|
|
19186
20249
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
19187
|
-
set: { count:
|
|
20250
|
+
set: { count: sql8`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
19188
20251
|
}).run();
|
|
19189
20252
|
}
|
|
19190
20253
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -19198,7 +20261,7 @@ var JobRunner = class {
|
|
|
19198
20261
|
status: runs.status,
|
|
19199
20262
|
finishedAt: runs.finishedAt,
|
|
19200
20263
|
error: runs.error
|
|
19201
|
-
}).from(runs).where(
|
|
20264
|
+
}).from(runs).where(eq24(runs.id, runId)).get();
|
|
19202
20265
|
}
|
|
19203
20266
|
isRunCancelled(runId) {
|
|
19204
20267
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -19214,7 +20277,7 @@ var JobRunner = class {
|
|
|
19214
20277
|
this.db.update(runs).set({
|
|
19215
20278
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19216
20279
|
error: currentRun.error ?? "Cancelled by user"
|
|
19217
|
-
}).where(
|
|
20280
|
+
}).where(eq24(runs.id, runId)).run();
|
|
19218
20281
|
}
|
|
19219
20282
|
trackEvent("run.completed", {
|
|
19220
20283
|
status: "cancelled",
|
|
@@ -19236,8 +20299,8 @@ function getCurrentUsageDay() {
|
|
|
19236
20299
|
}
|
|
19237
20300
|
|
|
19238
20301
|
// src/gsc-sync.ts
|
|
19239
|
-
import
|
|
19240
|
-
import { eq as
|
|
20302
|
+
import crypto22 from "crypto";
|
|
20303
|
+
import { eq as eq25, and as and13, sql as sql9 } from "drizzle-orm";
|
|
19241
20304
|
var log2 = createLogger("GscSync");
|
|
19242
20305
|
function formatDate3(d) {
|
|
19243
20306
|
return d.toISOString().split("T")[0];
|
|
@@ -19249,13 +20312,13 @@ function daysAgo(n) {
|
|
|
19249
20312
|
}
|
|
19250
20313
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
19251
20314
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19252
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20315
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq25(runs.id, runId)).run();
|
|
19253
20316
|
try {
|
|
19254
20317
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
19255
20318
|
if (!googleClientId || !googleClientSecret) {
|
|
19256
20319
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
19257
20320
|
}
|
|
19258
|
-
const project = db.select().from(projects).where(
|
|
20321
|
+
const project = db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
19259
20322
|
if (!project) {
|
|
19260
20323
|
throw new Error(`Project not found: ${projectId}`);
|
|
19261
20324
|
}
|
|
@@ -19290,9 +20353,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19290
20353
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
19291
20354
|
db.delete(gscSearchData).where(
|
|
19292
20355
|
and13(
|
|
19293
|
-
|
|
19294
|
-
|
|
19295
|
-
|
|
20356
|
+
eq25(gscSearchData.projectId, projectId),
|
|
20357
|
+
sql9`${gscSearchData.date} >= ${startDate}`,
|
|
20358
|
+
sql9`${gscSearchData.date} <= ${endDate}`
|
|
19296
20359
|
)
|
|
19297
20360
|
).run();
|
|
19298
20361
|
const batchSize = 500;
|
|
@@ -19302,7 +20365,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19302
20365
|
for (const row of batch) {
|
|
19303
20366
|
const [query, page, country, device, date] = row.keys;
|
|
19304
20367
|
db.insert(gscSearchData).values({
|
|
19305
|
-
id:
|
|
20368
|
+
id: crypto22.randomUUID(),
|
|
19306
20369
|
projectId,
|
|
19307
20370
|
syncRunId: runId,
|
|
19308
20371
|
date: date ?? "",
|
|
@@ -19336,7 +20399,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19336
20399
|
const rich = ir.richResultsResult;
|
|
19337
20400
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19338
20401
|
db.insert(gscUrlInspections).values({
|
|
19339
|
-
id:
|
|
20402
|
+
id: crypto22.randomUUID(),
|
|
19340
20403
|
projectId,
|
|
19341
20404
|
syncRunId: runId,
|
|
19342
20405
|
url: pageUrl,
|
|
@@ -19357,7 +20420,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19357
20420
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
19358
20421
|
}
|
|
19359
20422
|
}
|
|
19360
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
20423
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq25(gscUrlInspections.projectId, projectId)).all();
|
|
19361
20424
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19362
20425
|
for (const row of allInspections) {
|
|
19363
20426
|
const existing = latestByUrl.get(row.url);
|
|
@@ -19378,9 +20441,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19378
20441
|
}
|
|
19379
20442
|
}
|
|
19380
20443
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
19381
|
-
db.delete(gscCoverageSnapshots).where(and13(
|
|
20444
|
+
db.delete(gscCoverageSnapshots).where(and13(eq25(gscCoverageSnapshots.projectId, projectId), eq25(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
19382
20445
|
db.insert(gscCoverageSnapshots).values({
|
|
19383
|
-
id:
|
|
20446
|
+
id: crypto22.randomUUID(),
|
|
19384
20447
|
projectId,
|
|
19385
20448
|
syncRunId: runId,
|
|
19386
20449
|
date: snapshotDate,
|
|
@@ -19389,19 +20452,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19389
20452
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
19390
20453
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19391
20454
|
}).run();
|
|
19392
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20455
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
19393
20456
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
19394
20457
|
} catch (err) {
|
|
19395
20458
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19396
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20459
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
19397
20460
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
19398
20461
|
throw err;
|
|
19399
20462
|
}
|
|
19400
20463
|
}
|
|
19401
20464
|
|
|
19402
20465
|
// src/gsc-inspect-sitemap.ts
|
|
19403
|
-
import
|
|
19404
|
-
import { eq as
|
|
20466
|
+
import crypto23 from "crypto";
|
|
20467
|
+
import { eq as eq26, and as and14 } from "drizzle-orm";
|
|
19405
20468
|
|
|
19406
20469
|
// src/sitemap-parser.ts
|
|
19407
20470
|
var log3 = createLogger("SitemapParser");
|
|
@@ -19522,13 +20585,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
19522
20585
|
var log4 = createLogger("InspectSitemap");
|
|
19523
20586
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
19524
20587
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19525
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20588
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq26(runs.id, runId)).run();
|
|
19526
20589
|
try {
|
|
19527
20590
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
19528
20591
|
if (!googleClientId || !googleClientSecret) {
|
|
19529
20592
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
19530
20593
|
}
|
|
19531
|
-
const project = db.select().from(projects).where(
|
|
20594
|
+
const project = db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
19532
20595
|
if (!project) {
|
|
19533
20596
|
throw new Error(`Project not found: ${projectId}`);
|
|
19534
20597
|
}
|
|
@@ -19569,7 +20632,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19569
20632
|
const rich = ir.richResultsResult;
|
|
19570
20633
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19571
20634
|
db.insert(gscUrlInspections).values({
|
|
19572
|
-
id:
|
|
20635
|
+
id: crypto23.randomUUID(),
|
|
19573
20636
|
projectId,
|
|
19574
20637
|
syncRunId: runId,
|
|
19575
20638
|
url: pageUrl,
|
|
@@ -19596,7 +20659,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19596
20659
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
19597
20660
|
}
|
|
19598
20661
|
}
|
|
19599
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
20662
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq26(gscUrlInspections.projectId, projectId)).all();
|
|
19600
20663
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19601
20664
|
for (const row of allInspections) {
|
|
19602
20665
|
const existing = latestByUrl.get(row.url);
|
|
@@ -19617,9 +20680,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19617
20680
|
}
|
|
19618
20681
|
}
|
|
19619
20682
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
19620
|
-
db.delete(gscCoverageSnapshots).where(and14(
|
|
20683
|
+
db.delete(gscCoverageSnapshots).where(and14(eq26(gscCoverageSnapshots.projectId, projectId), eq26(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
19621
20684
|
db.insert(gscCoverageSnapshots).values({
|
|
19622
|
-
id:
|
|
20685
|
+
id: crypto23.randomUUID(),
|
|
19623
20686
|
projectId,
|
|
19624
20687
|
syncRunId: runId,
|
|
19625
20688
|
date: snapshotDate,
|
|
@@ -19629,19 +20692,19 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19629
20692
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19630
20693
|
}).run();
|
|
19631
20694
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
19632
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20695
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
19633
20696
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
19634
20697
|
} catch (err) {
|
|
19635
20698
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19636
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20699
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
19637
20700
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
19638
20701
|
throw err;
|
|
19639
20702
|
}
|
|
19640
20703
|
}
|
|
19641
20704
|
|
|
19642
20705
|
// src/bing-inspect-sitemap.ts
|
|
19643
|
-
import
|
|
19644
|
-
import { eq as
|
|
20706
|
+
import crypto24 from "crypto";
|
|
20707
|
+
import { eq as eq27, desc as desc12 } from "drizzle-orm";
|
|
19645
20708
|
var log5 = createLogger("BingInspectSitemap");
|
|
19646
20709
|
function parseBingDate2(value) {
|
|
19647
20710
|
if (!value) return null;
|
|
@@ -19659,9 +20722,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
19659
20722
|
}
|
|
19660
20723
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
19661
20724
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19662
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
20725
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq27(runs.id, runId)).run();
|
|
19663
20726
|
try {
|
|
19664
|
-
const project = db.select().from(projects).where(
|
|
20727
|
+
const project = db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
19665
20728
|
if (!project) {
|
|
19666
20729
|
throw new Error(`Project not found: ${projectId}`);
|
|
19667
20730
|
}
|
|
@@ -19679,7 +20742,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19679
20742
|
if (sitemapUrls.length === 0) {
|
|
19680
20743
|
throw new Error("No URLs found in sitemap");
|
|
19681
20744
|
}
|
|
19682
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
20745
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).all();
|
|
19683
20746
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
19684
20747
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
19685
20748
|
log5.info("sitemap.diff", {
|
|
@@ -19728,7 +20791,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19728
20791
|
derivedInIndex = false;
|
|
19729
20792
|
}
|
|
19730
20793
|
db.insert(bingUrlInspections).values({
|
|
19731
|
-
id:
|
|
20794
|
+
id: crypto24.randomUUID(),
|
|
19732
20795
|
projectId,
|
|
19733
20796
|
url: pageUrl,
|
|
19734
20797
|
httpCode,
|
|
@@ -19762,7 +20825,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19762
20825
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
19763
20826
|
}
|
|
19764
20827
|
}
|
|
19765
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
20828
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).orderBy(desc12(bingUrlInspections.inspectedAt)).all();
|
|
19766
20829
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19767
20830
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
19768
20831
|
for (const row of allInspections) {
|
|
@@ -19786,7 +20849,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19786
20849
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
19787
20850
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
19788
20851
|
db.insert(bingCoverageSnapshots).values({
|
|
19789
|
-
id:
|
|
20852
|
+
id: crypto24.randomUUID(),
|
|
19790
20853
|
projectId,
|
|
19791
20854
|
syncRunId: runId,
|
|
19792
20855
|
date: snapshotDate,
|
|
@@ -19805,7 +20868,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19805
20868
|
}
|
|
19806
20869
|
}).run();
|
|
19807
20870
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
19808
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20871
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
19809
20872
|
log5.info("inspect.completed", {
|
|
19810
20873
|
runId,
|
|
19811
20874
|
projectId,
|
|
@@ -19819,16 +20882,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19819
20882
|
});
|
|
19820
20883
|
} catch (err) {
|
|
19821
20884
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19822
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20885
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
19823
20886
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
19824
20887
|
throw err;
|
|
19825
20888
|
}
|
|
19826
20889
|
}
|
|
19827
20890
|
|
|
19828
20891
|
// src/commoncrawl-sync.ts
|
|
19829
|
-
import
|
|
20892
|
+
import crypto25 from "crypto";
|
|
19830
20893
|
import path10 from "path";
|
|
19831
|
-
import { and as and15, eq as
|
|
20894
|
+
import { and as and15, eq as eq28, sql as sql10 } from "drizzle-orm";
|
|
19832
20895
|
var log6 = createLogger("CommonCrawlSync");
|
|
19833
20896
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
19834
20897
|
function defaultDeps() {
|
|
@@ -19854,7 +20917,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19854
20917
|
phaseDetail: "downloading vertices + edges",
|
|
19855
20918
|
updatedAt: downloadStartedAt,
|
|
19856
20919
|
error: null
|
|
19857
|
-
}).where(
|
|
20920
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19858
20921
|
const paths = ccReleasePaths(release);
|
|
19859
20922
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
19860
20923
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -19877,7 +20940,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19877
20940
|
vertexSha256: vertex.sha256,
|
|
19878
20941
|
edgesSha256: edges.sha256,
|
|
19879
20942
|
updatedAt: downloadFinishedAt
|
|
19880
|
-
}).where(
|
|
20943
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19881
20944
|
const allProjects = db.select().from(projects).all();
|
|
19882
20945
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
19883
20946
|
let rows = [];
|
|
@@ -19893,15 +20956,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19893
20956
|
}
|
|
19894
20957
|
const queriedAt = deps.now().toISOString();
|
|
19895
20958
|
db.transaction((tx) => {
|
|
19896
|
-
tx.delete(backlinkDomains).where(
|
|
19897
|
-
tx.delete(backlinkSummaries).where(
|
|
20959
|
+
tx.delete(backlinkDomains).where(eq28(backlinkDomains.releaseSyncId, syncId)).run();
|
|
20960
|
+
tx.delete(backlinkSummaries).where(eq28(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
19898
20961
|
const expanded = [];
|
|
19899
20962
|
for (const r of rows) {
|
|
19900
20963
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
19901
20964
|
if (!projectIds) continue;
|
|
19902
20965
|
for (const projectId of projectIds) {
|
|
19903
20966
|
expanded.push({
|
|
19904
|
-
id:
|
|
20967
|
+
id: crypto25.randomUUID(),
|
|
19905
20968
|
projectId,
|
|
19906
20969
|
releaseSyncId: syncId,
|
|
19907
20970
|
release,
|
|
@@ -19921,7 +20984,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19921
20984
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
19922
20985
|
const summary = computeSummary(projectRows);
|
|
19923
20986
|
tx.insert(backlinkSummaries).values({
|
|
19924
|
-
id:
|
|
20987
|
+
id: crypto25.randomUUID(),
|
|
19925
20988
|
projectId: p.id,
|
|
19926
20989
|
releaseSyncId: syncId,
|
|
19927
20990
|
release,
|
|
@@ -19953,7 +21016,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19953
21016
|
domainsDiscovered: rows.length,
|
|
19954
21017
|
updatedAt: finishedAt,
|
|
19955
21018
|
error: null
|
|
19956
|
-
}).where(
|
|
21019
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19957
21020
|
log6.info("sync.completed", {
|
|
19958
21021
|
syncId,
|
|
19959
21022
|
release,
|
|
@@ -19983,7 +21046,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19983
21046
|
error: errorMsg,
|
|
19984
21047
|
phaseDetail: null,
|
|
19985
21048
|
updatedAt: finishedAt
|
|
19986
|
-
}).where(
|
|
21049
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19987
21050
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
19988
21051
|
throw err;
|
|
19989
21052
|
}
|
|
@@ -20017,9 +21080,9 @@ function computeSummary(rows) {
|
|
|
20017
21080
|
}
|
|
20018
21081
|
|
|
20019
21082
|
// src/backlink-extract.ts
|
|
20020
|
-
import
|
|
21083
|
+
import crypto26 from "crypto";
|
|
20021
21084
|
import fs8 from "fs";
|
|
20022
|
-
import { and as and16, desc as desc13, eq as
|
|
21085
|
+
import { and as and16, desc as desc13, eq as eq29 } from "drizzle-orm";
|
|
20023
21086
|
var log7 = createLogger("BacklinkExtract");
|
|
20024
21087
|
function defaultDeps2() {
|
|
20025
21088
|
return {
|
|
@@ -20031,13 +21094,13 @@ function defaultDeps2() {
|
|
|
20031
21094
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
20032
21095
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
20033
21096
|
const startedAt = deps.now().toISOString();
|
|
20034
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
21097
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq29(runs.id, runId)).run();
|
|
20035
21098
|
try {
|
|
20036
|
-
const project = db.select().from(projects).where(
|
|
21099
|
+
const project = db.select().from(projects).where(eq29(projects.id, projectId)).get();
|
|
20037
21100
|
if (!project) {
|
|
20038
21101
|
throw new Error(`Project not found: ${projectId}`);
|
|
20039
21102
|
}
|
|
20040
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
21103
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc13(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
20041
21104
|
if (!sync) {
|
|
20042
21105
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
20043
21106
|
}
|
|
@@ -20065,11 +21128,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20065
21128
|
const targetDomain = project.canonicalDomain;
|
|
20066
21129
|
db.transaction((tx) => {
|
|
20067
21130
|
tx.delete(backlinkDomains).where(
|
|
20068
|
-
and16(
|
|
21131
|
+
and16(eq29(backlinkDomains.projectId, projectId), eq29(backlinkDomains.release, release))
|
|
20069
21132
|
).run();
|
|
20070
21133
|
if (rows.length > 0) {
|
|
20071
21134
|
const values = rows.map((r) => ({
|
|
20072
|
-
id:
|
|
21135
|
+
id: crypto26.randomUUID(),
|
|
20073
21136
|
projectId,
|
|
20074
21137
|
releaseSyncId: syncId,
|
|
20075
21138
|
release,
|
|
@@ -20082,7 +21145,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20082
21145
|
}
|
|
20083
21146
|
const summary = computeSummary2(rows);
|
|
20084
21147
|
tx.insert(backlinkSummaries).values({
|
|
20085
|
-
id:
|
|
21148
|
+
id: crypto26.randomUUID(),
|
|
20086
21149
|
projectId,
|
|
20087
21150
|
releaseSyncId: syncId,
|
|
20088
21151
|
release,
|
|
@@ -20105,7 +21168,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20105
21168
|
}).run();
|
|
20106
21169
|
});
|
|
20107
21170
|
const finishedAt = deps.now().toISOString();
|
|
20108
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
21171
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq29(runs.id, runId)).run();
|
|
20109
21172
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
20110
21173
|
} catch (err) {
|
|
20111
21174
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -20114,7 +21177,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20114
21177
|
status: RunStatuses.failed,
|
|
20115
21178
|
error: errorMsg,
|
|
20116
21179
|
finishedAt
|
|
20117
|
-
}).where(
|
|
21180
|
+
}).where(eq29(runs.id, runId)).run();
|
|
20118
21181
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
20119
21182
|
throw err;
|
|
20120
21183
|
}
|
|
@@ -20187,7 +21250,7 @@ var ProviderRegistry = class {
|
|
|
20187
21250
|
|
|
20188
21251
|
// src/scheduler.ts
|
|
20189
21252
|
import cron from "node-cron";
|
|
20190
|
-
import { eq as
|
|
21253
|
+
import { eq as eq30 } from "drizzle-orm";
|
|
20191
21254
|
var log8 = createLogger("Scheduler");
|
|
20192
21255
|
var Scheduler = class {
|
|
20193
21256
|
db;
|
|
@@ -20199,7 +21262,7 @@ var Scheduler = class {
|
|
|
20199
21262
|
}
|
|
20200
21263
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
20201
21264
|
start() {
|
|
20202
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
21265
|
+
const allSchedules = this.db.select().from(schedules).where(eq30(schedules.enabled, 1)).all();
|
|
20203
21266
|
for (const schedule of allSchedules) {
|
|
20204
21267
|
const missedRunAt = schedule.nextRunAt;
|
|
20205
21268
|
this.registerCronTask(schedule);
|
|
@@ -20224,7 +21287,7 @@ var Scheduler = class {
|
|
|
20224
21287
|
this.stopTask(projectId, existing, "Stopped");
|
|
20225
21288
|
this.tasks.delete(projectId);
|
|
20226
21289
|
}
|
|
20227
|
-
const schedule = this.db.select().from(schedules).where(
|
|
21290
|
+
const schedule = this.db.select().from(schedules).where(eq30(schedules.projectId, projectId)).get();
|
|
20228
21291
|
if (schedule && schedule.enabled === 1) {
|
|
20229
21292
|
this.registerCronTask(schedule);
|
|
20230
21293
|
}
|
|
@@ -20257,14 +21320,14 @@ var Scheduler = class {
|
|
|
20257
21320
|
this.db.update(schedules).set({
|
|
20258
21321
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
20259
21322
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
20260
|
-
}).where(
|
|
21323
|
+
}).where(eq30(schedules.id, scheduleId)).run();
|
|
20261
21324
|
const label = schedule.preset ?? cronExpr;
|
|
20262
21325
|
log8.info("cron.registered", { projectId, schedule: label, timezone });
|
|
20263
21326
|
}
|
|
20264
21327
|
triggerRun(scheduleId, projectId) {
|
|
20265
21328
|
try {
|
|
20266
21329
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20267
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
21330
|
+
const currentSchedule = this.db.select().from(schedules).where(eq30(schedules.id, scheduleId)).get();
|
|
20268
21331
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
20269
21332
|
log8.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
20270
21333
|
this.remove(projectId);
|
|
@@ -20272,7 +21335,7 @@ var Scheduler = class {
|
|
|
20272
21335
|
}
|
|
20273
21336
|
const task = this.tasks.get(projectId);
|
|
20274
21337
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
20275
|
-
const project = this.db.select().from(projects).where(
|
|
21338
|
+
const project = this.db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
20276
21339
|
if (!project) {
|
|
20277
21340
|
log8.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
20278
21341
|
this.remove(projectId);
|
|
@@ -20301,7 +21364,7 @@ var Scheduler = class {
|
|
|
20301
21364
|
this.db.update(schedules).set({
|
|
20302
21365
|
nextRunAt,
|
|
20303
21366
|
updatedAt: now
|
|
20304
|
-
}).where(
|
|
21367
|
+
}).where(eq30(schedules.id, currentSchedule.id)).run();
|
|
20305
21368
|
return;
|
|
20306
21369
|
}
|
|
20307
21370
|
const runId = queueResult.runId;
|
|
@@ -20309,7 +21372,7 @@ var Scheduler = class {
|
|
|
20309
21372
|
lastRunAt: now,
|
|
20310
21373
|
nextRunAt,
|
|
20311
21374
|
updatedAt: now
|
|
20312
|
-
}).where(
|
|
21375
|
+
}).where(eq30(schedules.id, currentSchedule.id)).run();
|
|
20313
21376
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
20314
21377
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
20315
21378
|
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -20321,8 +21384,8 @@ var Scheduler = class {
|
|
|
20321
21384
|
};
|
|
20322
21385
|
|
|
20323
21386
|
// src/notifier.ts
|
|
20324
|
-
import { eq as
|
|
20325
|
-
import
|
|
21387
|
+
import { eq as eq31, desc as desc14, and as and17, or as or4 } from "drizzle-orm";
|
|
21388
|
+
import crypto27 from "crypto";
|
|
20326
21389
|
var log9 = createLogger("Notifier");
|
|
20327
21390
|
var Notifier = class {
|
|
20328
21391
|
db;
|
|
@@ -20334,18 +21397,18 @@ var Notifier = class {
|
|
|
20334
21397
|
/** Called after a run completes (success, partial, or failed). */
|
|
20335
21398
|
async onRunCompleted(runId, projectId) {
|
|
20336
21399
|
log9.info("run.completed", { runId, projectId });
|
|
20337
|
-
const notifs = this.db.select().from(notifications).where(
|
|
21400
|
+
const notifs = this.db.select().from(notifications).where(eq31(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
20338
21401
|
if (notifs.length === 0) {
|
|
20339
21402
|
log9.info("notifications.none-enabled", { projectId });
|
|
20340
21403
|
return;
|
|
20341
21404
|
}
|
|
20342
21405
|
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
20343
|
-
const run = this.db.select().from(runs).where(
|
|
21406
|
+
const run = this.db.select().from(runs).where(eq31(runs.id, runId)).get();
|
|
20344
21407
|
if (!run) {
|
|
20345
21408
|
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
20346
21409
|
return;
|
|
20347
21410
|
}
|
|
20348
|
-
const project = this.db.select().from(projects).where(
|
|
21411
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
20349
21412
|
if (!project) {
|
|
20350
21413
|
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
20351
21414
|
return;
|
|
@@ -20392,11 +21455,11 @@ var Notifier = class {
|
|
|
20392
21455
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
20393
21456
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
20394
21457
|
if (insightEvents.length === 0) return;
|
|
20395
|
-
const notifs = this.db.select().from(notifications).where(
|
|
21458
|
+
const notifs = this.db.select().from(notifications).where(eq31(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
20396
21459
|
if (notifs.length === 0) return;
|
|
20397
|
-
const run = this.db.select().from(runs).where(
|
|
21460
|
+
const run = this.db.select().from(runs).where(eq31(runs.id, runId)).get();
|
|
20398
21461
|
if (!run) return;
|
|
20399
|
-
const project = this.db.select().from(projects).where(
|
|
21462
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
20400
21463
|
if (!project) return;
|
|
20401
21464
|
for (const notif of notifs) {
|
|
20402
21465
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -20428,8 +21491,8 @@ var Notifier = class {
|
|
|
20428
21491
|
computeTransitions(runId, projectId) {
|
|
20429
21492
|
const recentRuns = this.db.select().from(runs).where(
|
|
20430
21493
|
and17(
|
|
20431
|
-
|
|
20432
|
-
or4(
|
|
21494
|
+
eq31(runs.projectId, projectId),
|
|
21495
|
+
or4(eq31(runs.status, "completed"), eq31(runs.status, "partial"))
|
|
20433
21496
|
)
|
|
20434
21497
|
).orderBy(desc14(runs.createdAt)).limit(2).all();
|
|
20435
21498
|
if (recentRuns.length < 2) return [];
|
|
@@ -20441,12 +21504,12 @@ var Notifier = class {
|
|
|
20441
21504
|
query: queries.query,
|
|
20442
21505
|
provider: querySnapshots.provider,
|
|
20443
21506
|
citationState: querySnapshots.citationState
|
|
20444
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
21507
|
+
}).from(querySnapshots).leftJoin(queries, eq31(querySnapshots.queryId, queries.id)).where(eq31(querySnapshots.runId, currentRunId)).all();
|
|
20445
21508
|
const previousSnapshots = this.db.select({
|
|
20446
21509
|
queryId: querySnapshots.queryId,
|
|
20447
21510
|
provider: querySnapshots.provider,
|
|
20448
21511
|
citationState: querySnapshots.citationState
|
|
20449
|
-
}).from(querySnapshots).where(
|
|
21512
|
+
}).from(querySnapshots).where(eq31(querySnapshots.runId, previousRunId)).all();
|
|
20450
21513
|
const prevMap = /* @__PURE__ */ new Map();
|
|
20451
21514
|
for (const s of previousSnapshots) {
|
|
20452
21515
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -20504,7 +21567,7 @@ var Notifier = class {
|
|
|
20504
21567
|
}
|
|
20505
21568
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
20506
21569
|
this.db.insert(auditLog).values({
|
|
20507
|
-
id:
|
|
21570
|
+
id: crypto27.randomUUID(),
|
|
20508
21571
|
projectId,
|
|
20509
21572
|
actor: "scheduler",
|
|
20510
21573
|
action: `notification.${status}`,
|
|
@@ -20562,8 +21625,8 @@ var RunCoordinator = class {
|
|
|
20562
21625
|
};
|
|
20563
21626
|
|
|
20564
21627
|
// src/agent/session-registry.ts
|
|
20565
|
-
import
|
|
20566
|
-
import { eq as
|
|
21628
|
+
import crypto29 from "crypto";
|
|
21629
|
+
import { eq as eq33 } from "drizzle-orm";
|
|
20567
21630
|
|
|
20568
21631
|
// src/agent/session.ts
|
|
20569
21632
|
import fs11 from "fs";
|
|
@@ -20912,11 +21975,11 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
20912
21975
|
}
|
|
20913
21976
|
|
|
20914
21977
|
// src/agent/memory-store.ts
|
|
20915
|
-
import
|
|
20916
|
-
import { and as and18, desc as desc15, eq as
|
|
21978
|
+
import crypto28 from "crypto";
|
|
21979
|
+
import { and as and18, desc as desc15, eq as eq32, like as like2, sql as sql11 } from "drizzle-orm";
|
|
20917
21980
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
20918
21981
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
20919
|
-
function
|
|
21982
|
+
function rowToDto2(row) {
|
|
20920
21983
|
return {
|
|
20921
21984
|
id: row.id,
|
|
20922
21985
|
key: row.key,
|
|
@@ -20927,9 +21990,9 @@ function rowToDto(row) {
|
|
|
20927
21990
|
};
|
|
20928
21991
|
}
|
|
20929
21992
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
20930
|
-
const query = db.select().from(agentMemory).where(
|
|
21993
|
+
const query = db.select().from(agentMemory).where(eq32(agentMemory.projectId, projectId)).orderBy(desc15(agentMemory.updatedAt));
|
|
20931
21994
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
20932
|
-
return rows.map(
|
|
21995
|
+
return rows.map(rowToDto2);
|
|
20933
21996
|
}
|
|
20934
21997
|
function upsertMemoryEntry(db, args) {
|
|
20935
21998
|
if (Buffer.byteLength(args.value, "utf8") > AGENT_MEMORY_VALUE_MAX_BYTES) {
|
|
@@ -20941,7 +22004,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
20941
22004
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
20942
22005
|
}
|
|
20943
22006
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20944
|
-
const id =
|
|
22007
|
+
const id = crypto28.randomUUID();
|
|
20945
22008
|
db.insert(agentMemory).values({
|
|
20946
22009
|
id,
|
|
20947
22010
|
projectId: args.projectId,
|
|
@@ -20958,12 +22021,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
20958
22021
|
updatedAt: now
|
|
20959
22022
|
}
|
|
20960
22023
|
}).run();
|
|
20961
|
-
const row = db.select().from(agentMemory).where(and18(
|
|
22024
|
+
const row = db.select().from(agentMemory).where(and18(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, args.key))).get();
|
|
20962
22025
|
if (!row) throw new Error("memory upsert produced no row");
|
|
20963
|
-
return
|
|
22026
|
+
return rowToDto2(row);
|
|
20964
22027
|
}
|
|
20965
22028
|
function deleteMemoryEntry(db, projectId, key) {
|
|
20966
|
-
const result = db.delete(agentMemory).where(and18(
|
|
22029
|
+
const result = db.delete(agentMemory).where(and18(eq32(agentMemory.projectId, projectId), eq32(agentMemory.key, key))).run();
|
|
20967
22030
|
const changes = result.changes ?? 0;
|
|
20968
22031
|
return changes > 0;
|
|
20969
22032
|
}
|
|
@@ -20978,7 +22041,7 @@ function writeCompactionNote(db, args) {
|
|
|
20978
22041
|
}
|
|
20979
22042
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20980
22043
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
20981
|
-
const id =
|
|
22044
|
+
const id = crypto28.randomUUID();
|
|
20982
22045
|
let inserted;
|
|
20983
22046
|
db.transaction((tx) => {
|
|
20984
22047
|
tx.insert(agentMemory).values({
|
|
@@ -20993,16 +22056,16 @@ function writeCompactionNote(db, args) {
|
|
|
20993
22056
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
20994
22057
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
20995
22058
|
and18(
|
|
20996
|
-
|
|
22059
|
+
eq32(agentMemory.projectId, args.projectId),
|
|
20997
22060
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
20998
22061
|
)
|
|
20999
22062
|
).orderBy(desc15(agentMemory.updatedAt)).all();
|
|
21000
22063
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
21001
22064
|
if (stale.length > 0) {
|
|
21002
|
-
tx.delete(agentMemory).where(
|
|
22065
|
+
tx.delete(agentMemory).where(sql11`${agentMemory.id} IN (${sql11.join(stale.map((s) => sql11`${s}`), sql11`, `)})`).run();
|
|
21003
22066
|
}
|
|
21004
|
-
const row = tx.select().from(agentMemory).where(and18(
|
|
21005
|
-
if (row) inserted =
|
|
22067
|
+
const row = tx.select().from(agentMemory).where(and18(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, key))).get();
|
|
22068
|
+
if (row) inserted = rowToDto2(row);
|
|
21006
22069
|
});
|
|
21007
22070
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
21008
22071
|
return inserted;
|
|
@@ -21183,7 +22246,7 @@ var SessionRegistry = class {
|
|
|
21183
22246
|
modelProvider: effectiveProvider,
|
|
21184
22247
|
modelId: effectiveModelId,
|
|
21185
22248
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21186
|
-
}).where(
|
|
22249
|
+
}).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21187
22250
|
}
|
|
21188
22251
|
const agent2 = createAeroSession({
|
|
21189
22252
|
projectName,
|
|
@@ -21397,7 +22460,7 @@ ${lines.join("\n")}
|
|
|
21397
22460
|
modelProvider: nextProvider,
|
|
21398
22461
|
modelId: nextModelId,
|
|
21399
22462
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21400
|
-
}).where(
|
|
22463
|
+
}).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21401
22464
|
}
|
|
21402
22465
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
21403
22466
|
save(projectName) {
|
|
@@ -21559,17 +22622,17 @@ ${lines.join("\n")}
|
|
|
21559
22622
|
return id;
|
|
21560
22623
|
}
|
|
21561
22624
|
tryResolveProjectId(projectName) {
|
|
21562
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
22625
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq33(projects.name, projectName)).get();
|
|
21563
22626
|
return row?.id;
|
|
21564
22627
|
}
|
|
21565
22628
|
loadRow(projectId) {
|
|
21566
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
22629
|
+
const row = this.opts.db.select().from(agentSessions).where(eq33(agentSessions.projectId, projectId)).get();
|
|
21567
22630
|
return row ?? null;
|
|
21568
22631
|
}
|
|
21569
22632
|
insertRow(params) {
|
|
21570
22633
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21571
22634
|
this.opts.db.insert(agentSessions).values({
|
|
21572
|
-
id:
|
|
22635
|
+
id: crypto29.randomUUID(),
|
|
21573
22636
|
projectId: params.projectId,
|
|
21574
22637
|
systemPrompt: params.systemPrompt,
|
|
21575
22638
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -21582,14 +22645,14 @@ ${lines.join("\n")}
|
|
|
21582
22645
|
}
|
|
21583
22646
|
updateRow(projectId, patch) {
|
|
21584
22647
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21585
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
22648
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21586
22649
|
}
|
|
21587
22650
|
};
|
|
21588
22651
|
|
|
21589
22652
|
// src/agent/agent-routes.ts
|
|
21590
|
-
import { eq as
|
|
22653
|
+
import { eq as eq34 } from "drizzle-orm";
|
|
21591
22654
|
function resolveProject2(db, name) {
|
|
21592
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
22655
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq34(projects.name, name)).get();
|
|
21593
22656
|
if (!row) throw notFound("project", name);
|
|
21594
22657
|
return row;
|
|
21595
22658
|
}
|
|
@@ -21598,7 +22661,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
21598
22661
|
"/projects/:name/agent/transcript",
|
|
21599
22662
|
async (request) => {
|
|
21600
22663
|
const project = resolveProject2(opts.db, request.params.name);
|
|
21601
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
22664
|
+
const row = opts.db.select().from(agentSessions).where(eq34(agentSessions.projectId, project.id)).get();
|
|
21602
22665
|
if (!row) {
|
|
21603
22666
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
21604
22667
|
}
|
|
@@ -21622,7 +22685,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
21622
22685
|
async (request) => {
|
|
21623
22686
|
const project = resolveProject2(opts.db, request.params.name);
|
|
21624
22687
|
opts.sessionRegistry.reset(project.name);
|
|
21625
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22688
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq34(agentSessions.projectId, project.id)).run();
|
|
21626
22689
|
return { status: "reset" };
|
|
21627
22690
|
}
|
|
21628
22691
|
);
|
|
@@ -22486,7 +23549,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
22486
23549
|
};
|
|
22487
23550
|
}
|
|
22488
23551
|
function hashApiKey(key) {
|
|
22489
|
-
return
|
|
23552
|
+
return crypto30.createHash("sha256").update(key).digest("hex");
|
|
22490
23553
|
}
|
|
22491
23554
|
function parseCookies2(header) {
|
|
22492
23555
|
if (!header) return {};
|
|
@@ -22644,7 +23707,7 @@ async function createServer(opts) {
|
|
|
22644
23707
|
intelligenceService,
|
|
22645
23708
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
22646
23709
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
22647
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
23710
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq35(projects.id, projectId)).get();
|
|
22648
23711
|
if (!project) return;
|
|
22649
23712
|
sessionRegistry.queueFollowUp(project.name, {
|
|
22650
23713
|
role: "user",
|
|
@@ -22738,7 +23801,22 @@ async function createServer(opts) {
|
|
|
22738
23801
|
return removed;
|
|
22739
23802
|
}
|
|
22740
23803
|
};
|
|
22741
|
-
const
|
|
23804
|
+
const cloudRunCredentialStore = {
|
|
23805
|
+
getConnection: (projectName) => {
|
|
23806
|
+
return getCloudRunConnection(opts.config, projectName);
|
|
23807
|
+
},
|
|
23808
|
+
upsertConnection: (record) => {
|
|
23809
|
+
const updated = upsertCloudRunConnection(opts.config, record);
|
|
23810
|
+
saveConfigPatch(opts.config);
|
|
23811
|
+
return updated;
|
|
23812
|
+
},
|
|
23813
|
+
deleteConnection: (projectName) => {
|
|
23814
|
+
const removed = removeCloudRunConnection(opts.config, projectName);
|
|
23815
|
+
if (removed) saveConfigPatch(opts.config);
|
|
23816
|
+
return removed;
|
|
23817
|
+
}
|
|
23818
|
+
};
|
|
23819
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto30.randomBytes(32).toString("hex");
|
|
22742
23820
|
const googleConnectionStore = {
|
|
22743
23821
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
22744
23822
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -22784,11 +23862,11 @@ async function createServer(opts) {
|
|
|
22784
23862
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
22785
23863
|
if (opts.config.apiKey) {
|
|
22786
23864
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
22787
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
23865
|
+
const existing = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, keyHash)).get();
|
|
22788
23866
|
if (!existing) {
|
|
22789
23867
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
22790
23868
|
opts.db.insert(apiKeys).values({
|
|
22791
|
-
id: `key_${
|
|
23869
|
+
id: `key_${crypto30.randomBytes(8).toString("hex")}`,
|
|
22792
23870
|
name: "default",
|
|
22793
23871
|
keyHash,
|
|
22794
23872
|
keyPrefix: prefix,
|
|
@@ -22812,7 +23890,7 @@ async function createServer(opts) {
|
|
|
22812
23890
|
};
|
|
22813
23891
|
const createSession = (apiKeyId) => {
|
|
22814
23892
|
pruneExpiredSessions();
|
|
22815
|
-
const sessionId =
|
|
23893
|
+
const sessionId = crypto30.randomBytes(32).toString("hex");
|
|
22816
23894
|
sessions.set(sessionId, {
|
|
22817
23895
|
apiKeyId,
|
|
22818
23896
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -22836,7 +23914,7 @@ async function createServer(opts) {
|
|
|
22836
23914
|
};
|
|
22837
23915
|
const getDefaultApiKey = () => {
|
|
22838
23916
|
if (!opts.config.apiKey) return void 0;
|
|
22839
|
-
return opts.db.select().from(apiKeys).where(
|
|
23917
|
+
return opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
22840
23918
|
};
|
|
22841
23919
|
const createPasswordSession = (reply) => {
|
|
22842
23920
|
const key = getDefaultApiKey();
|
|
@@ -22893,12 +23971,12 @@ async function createServer(opts) {
|
|
|
22893
23971
|
return reply.send({ authenticated: true });
|
|
22894
23972
|
}
|
|
22895
23973
|
if (apiKey) {
|
|
22896
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
23974
|
+
const key = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
22897
23975
|
if (!key || key.revokedAt) {
|
|
22898
23976
|
const err2 = authInvalid();
|
|
22899
23977
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
22900
23978
|
}
|
|
22901
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23979
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq35(apiKeys.id, key.id)).run();
|
|
22902
23980
|
const sessionId = createSession(key.id);
|
|
22903
23981
|
reply.header("set-cookie", serializeSessionCookie({
|
|
22904
23982
|
name: SESSION_COOKIE_NAME,
|
|
@@ -23008,7 +24086,7 @@ async function createServer(opts) {
|
|
|
23008
24086
|
deps: {
|
|
23009
24087
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
23010
24088
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23011
|
-
const runId =
|
|
24089
|
+
const runId = crypto30.randomUUID();
|
|
23012
24090
|
opts.db.insert(runs).values({
|
|
23013
24091
|
id: runId,
|
|
23014
24092
|
projectId,
|
|
@@ -23081,6 +24159,7 @@ async function createServer(opts) {
|
|
|
23081
24159
|
},
|
|
23082
24160
|
wordpressConnectionStore,
|
|
23083
24161
|
ga4CredentialStore,
|
|
24162
|
+
cloudRunCredentialStore,
|
|
23084
24163
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
23085
24164
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
23086
24165
|
app.log.error({ runId, err }, "Job runner failed");
|
|
@@ -23143,7 +24222,7 @@ async function createServer(opts) {
|
|
|
23143
24222
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
23144
24223
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23145
24224
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
23146
|
-
id:
|
|
24225
|
+
id: crypto30.randomUUID(),
|
|
23147
24226
|
projectId,
|
|
23148
24227
|
actor: "api",
|
|
23149
24228
|
action: existing ? "provider.updated" : "provider.created",
|