@ainyc/canonry 4.11.1 → 4.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/assets/{index-DOcemxPD.js → index-CCC1E6ji.js} +106 -106
- package/assets/index.html +1 -1
- package/dist/{chunk-3SFDZPKU.js → chunk-DCE3B6KD.js} +177 -2
- package/dist/{chunk-PBQ4Z4P4.js → chunk-L4KKHRVQ.js} +1199 -139
- package/dist/{chunk-5J5BVJF7.js → chunk-LNRDWAG3.js} +16 -1
- package/dist/{chunk-565T7PMC.js → chunk-YDGT5CAY.js} +54 -2
- package/dist/cli.js +269 -103
- package/dist/index.d.ts +19 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-XWOFLEHJ.js → intelligence-service-NT24OLLA.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +8 -6
|
@@ -4,13 +4,14 @@ import {
|
|
|
4
4
|
configExists,
|
|
5
5
|
loadConfig,
|
|
6
6
|
saveConfigPatch
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-LNRDWAG3.js";
|
|
8
8
|
import {
|
|
9
9
|
DEFAULT_RUN_HISTORY_LIMIT,
|
|
10
10
|
IntelligenceService,
|
|
11
11
|
MIN_TREND_POINTS,
|
|
12
12
|
agentMemory,
|
|
13
13
|
agentSessions,
|
|
14
|
+
aiReferralEventsHourly,
|
|
14
15
|
apiKeys,
|
|
15
16
|
auditLog,
|
|
16
17
|
backlinkDomains,
|
|
@@ -36,6 +37,7 @@ import {
|
|
|
36
37
|
categorizeQueryByIntent,
|
|
37
38
|
ccReleaseSyncs,
|
|
38
39
|
competitors,
|
|
40
|
+
crawlerEventsHourly,
|
|
39
41
|
createLogger,
|
|
40
42
|
dropLegacyCredentialColumns,
|
|
41
43
|
extractLegacyCredentials,
|
|
@@ -58,10 +60,12 @@ import {
|
|
|
58
60
|
projects,
|
|
59
61
|
queries,
|
|
60
62
|
querySnapshots,
|
|
63
|
+
rawEventSamples,
|
|
61
64
|
runs,
|
|
62
65
|
schedules,
|
|
66
|
+
trafficSources,
|
|
63
67
|
usageCounters
|
|
64
|
-
} from "./chunk-
|
|
68
|
+
} from "./chunk-DCE3B6KD.js";
|
|
65
69
|
import {
|
|
66
70
|
AGENT_MEMORY_VALUE_MAX_BYTES,
|
|
67
71
|
AGENT_PROVIDER_IDS,
|
|
@@ -76,6 +80,11 @@ import {
|
|
|
76
80
|
RunKinds,
|
|
77
81
|
RunStatuses,
|
|
78
82
|
RunTriggers,
|
|
83
|
+
TrafficEventConfidences,
|
|
84
|
+
TrafficEvidenceKinds,
|
|
85
|
+
TrafficSourceAuthModes,
|
|
86
|
+
TrafficSourceStatuses,
|
|
87
|
+
TrafficSourceTypes,
|
|
79
88
|
absolutizeProjectUrl,
|
|
80
89
|
actionConfidenceLabel,
|
|
81
90
|
agentBusy,
|
|
@@ -137,7 +146,7 @@ import {
|
|
|
137
146
|
visibilityStateFromAnswerMentioned,
|
|
138
147
|
windowCutoff,
|
|
139
148
|
wordpressEnvSchema
|
|
140
|
-
} from "./chunk-
|
|
149
|
+
} from "./chunk-YDGT5CAY.js";
|
|
141
150
|
|
|
142
151
|
// src/telemetry.ts
|
|
143
152
|
import crypto from "crypto";
|
|
@@ -213,11 +222,11 @@ function trackEvent(event, properties) {
|
|
|
213
222
|
|
|
214
223
|
// src/server.ts
|
|
215
224
|
import { createRequire as createRequire3 } from "module";
|
|
216
|
-
import
|
|
225
|
+
import crypto30 from "crypto";
|
|
217
226
|
import fs12 from "fs";
|
|
218
227
|
import path14 from "path";
|
|
219
228
|
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
220
|
-
import { eq as
|
|
229
|
+
import { eq as eq35 } from "drizzle-orm";
|
|
221
230
|
import Fastify from "fastify";
|
|
222
231
|
|
|
223
232
|
// ../api-routes/src/auth.ts
|
|
@@ -9160,6 +9169,66 @@ var routeCatalog = [
|
|
|
9160
9169
|
200: { description: "History returned oldest-first by queriedAt." },
|
|
9161
9170
|
404: { description: "Project not found." }
|
|
9162
9171
|
}
|
|
9172
|
+
},
|
|
9173
|
+
{
|
|
9174
|
+
method: "post",
|
|
9175
|
+
path: "/api/v1/projects/{name}/traffic/connect/cloud-run",
|
|
9176
|
+
summary: "Connect a Cloud Run traffic source",
|
|
9177
|
+
description: "Stores the service-account JSON in `~/.canonry/config.yaml` and creates a `traffic_sources` row for the project. Reconnecting updates the existing active source rather than creating a duplicate.",
|
|
9178
|
+
tags: ["traffic"],
|
|
9179
|
+
parameters: [nameParameter],
|
|
9180
|
+
requestBody: {
|
|
9181
|
+
required: true,
|
|
9182
|
+
content: {
|
|
9183
|
+
"application/json": {
|
|
9184
|
+
schema: {
|
|
9185
|
+
type: "object",
|
|
9186
|
+
required: ["gcpProjectId", "keyJson"],
|
|
9187
|
+
properties: {
|
|
9188
|
+
gcpProjectId: stringSchema,
|
|
9189
|
+
serviceName: stringSchema,
|
|
9190
|
+
location: stringSchema,
|
|
9191
|
+
displayName: stringSchema,
|
|
9192
|
+
keyJson: { ...stringSchema, description: "Service-account JSON content." }
|
|
9193
|
+
}
|
|
9194
|
+
}
|
|
9195
|
+
}
|
|
9196
|
+
}
|
|
9197
|
+
},
|
|
9198
|
+
responses: {
|
|
9199
|
+
200: { description: "Traffic source DTO returned." },
|
|
9200
|
+
400: { description: "Invalid Cloud Run connection request." },
|
|
9201
|
+
404: { description: "Project not found." }
|
|
9202
|
+
}
|
|
9203
|
+
},
|
|
9204
|
+
{
|
|
9205
|
+
method: "post",
|
|
9206
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
9207
|
+
summary: "Trigger a sync run for a traffic source",
|
|
9208
|
+
description: "Pulls request logs from the configured Cloud Run service for the lookback window, classifies crawler / AI-referral hits, and upserts hourly buckets and a bounded sample tail.",
|
|
9209
|
+
tags: ["traffic"],
|
|
9210
|
+
parameters: [
|
|
9211
|
+
nameParameter,
|
|
9212
|
+
{ name: "id", in: "path", required: true, description: "Traffic source ID.", schema: stringSchema }
|
|
9213
|
+
],
|
|
9214
|
+
requestBody: {
|
|
9215
|
+
required: false,
|
|
9216
|
+
content: {
|
|
9217
|
+
"application/json": {
|
|
9218
|
+
schema: {
|
|
9219
|
+
type: "object",
|
|
9220
|
+
properties: {
|
|
9221
|
+
sinceMinutes: { ...integerSchema, description: "Lookback window in minutes (default 60)." }
|
|
9222
|
+
}
|
|
9223
|
+
}
|
|
9224
|
+
}
|
|
9225
|
+
}
|
|
9226
|
+
},
|
|
9227
|
+
responses: {
|
|
9228
|
+
200: { description: "Sync summary returned." },
|
|
9229
|
+
400: { description: "Invalid sync request, missing credentials, or upstream pull error." },
|
|
9230
|
+
404: { description: "Project or traffic source not found." }
|
|
9231
|
+
}
|
|
9163
9232
|
}
|
|
9164
9233
|
];
|
|
9165
9234
|
var canonryLocalRouteCatalog = [
|
|
@@ -14908,7 +14977,7 @@ async function queryBacklinks(opts) {
|
|
|
14908
14977
|
const reversed = opts.targets.map(reverseDomain);
|
|
14909
14978
|
const targetList = reversed.map(quote).join(", ");
|
|
14910
14979
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
14911
|
-
const
|
|
14980
|
+
const sql12 = `
|
|
14912
14981
|
WITH vertices AS (
|
|
14913
14982
|
SELECT * FROM read_csv(
|
|
14914
14983
|
${quote(opts.vertexPath)},
|
|
@@ -14944,7 +15013,7 @@ async function queryBacklinks(opts) {
|
|
|
14944
15013
|
const conn = await instance.connect();
|
|
14945
15014
|
let rows;
|
|
14946
15015
|
try {
|
|
14947
|
-
const reader = await conn.runAndReadAll(
|
|
15016
|
+
const reader = await conn.runAndReadAll(sql12);
|
|
14948
15017
|
rows = reader.getRowObjects();
|
|
14949
15018
|
} finally {
|
|
14950
15019
|
conn.disconnectSync?.();
|
|
@@ -15332,6 +15401,944 @@ async function backlinksRoutes(app, opts) {
|
|
|
15332
15401
|
);
|
|
15333
15402
|
}
|
|
15334
15403
|
|
|
15404
|
+
// ../api-routes/src/traffic.ts
|
|
15405
|
+
import crypto20 from "crypto";
|
|
15406
|
+
import { eq as eq23, sql as sql7 } from "drizzle-orm";
|
|
15407
|
+
|
|
15408
|
+
// ../integration-cloud-run/src/auth.ts
|
|
15409
|
+
import crypto19 from "crypto";
|
|
15410
|
+
var GOOGLE_TOKEN_URL3 = "https://oauth2.googleapis.com/token";
|
|
15411
|
+
var CLOUD_LOGGING_READ_SCOPE = "https://www.googleapis.com/auth/logging.read";
|
|
15412
|
+
var TOKEN_REQUEST_TIMEOUT_MS = 3e4;
|
|
15413
|
+
var CloudRunAuthError = class extends Error {
|
|
15414
|
+
constructor(message, httpStatus, body) {
|
|
15415
|
+
super(message);
|
|
15416
|
+
this.httpStatus = httpStatus;
|
|
15417
|
+
this.body = body;
|
|
15418
|
+
this.name = "CloudRunAuthError";
|
|
15419
|
+
}
|
|
15420
|
+
};
|
|
15421
|
+
function createServiceAccountJwt2(clientEmail, privateKey, scope) {
|
|
15422
|
+
if (!clientEmail) throw new CloudRunAuthError("clientEmail is required");
|
|
15423
|
+
if (!privateKey) throw new CloudRunAuthError("privateKey is required");
|
|
15424
|
+
if (!scope) throw new CloudRunAuthError("scope is required");
|
|
15425
|
+
const now = Math.floor(Date.now() / 1e3);
|
|
15426
|
+
const header = { alg: "RS256", typ: "JWT" };
|
|
15427
|
+
const payload = {
|
|
15428
|
+
iss: clientEmail,
|
|
15429
|
+
scope,
|
|
15430
|
+
aud: GOOGLE_TOKEN_URL3,
|
|
15431
|
+
iat: now,
|
|
15432
|
+
exp: now + 3600
|
|
15433
|
+
};
|
|
15434
|
+
const encode = (obj) => Buffer.from(JSON.stringify(obj)).toString("base64url");
|
|
15435
|
+
const headerB64 = encode(header);
|
|
15436
|
+
const payloadB64 = encode(payload);
|
|
15437
|
+
const signingInput = `${headerB64}.${payloadB64}`;
|
|
15438
|
+
const sign = crypto19.createSign("RSA-SHA256");
|
|
15439
|
+
sign.update(signingInput);
|
|
15440
|
+
const signature = sign.sign(privateKey, "base64url");
|
|
15441
|
+
return `${signingInput}.${signature}`;
|
|
15442
|
+
}
|
|
15443
|
+
async function getCloudLoggingAccessToken(clientEmail, privateKey) {
|
|
15444
|
+
const jwt = createServiceAccountJwt2(clientEmail, privateKey, CLOUD_LOGGING_READ_SCOPE);
|
|
15445
|
+
const res = await fetch(GOOGLE_TOKEN_URL3, {
|
|
15446
|
+
method: "POST",
|
|
15447
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
15448
|
+
body: new URLSearchParams({
|
|
15449
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
15450
|
+
assertion: jwt
|
|
15451
|
+
}),
|
|
15452
|
+
signal: AbortSignal.timeout(TOKEN_REQUEST_TIMEOUT_MS)
|
|
15453
|
+
});
|
|
15454
|
+
if (!res.ok) {
|
|
15455
|
+
const body = await res.text().catch(() => "");
|
|
15456
|
+
throw new CloudRunAuthError(
|
|
15457
|
+
`Service-account token exchange failed (HTTP ${res.status})`,
|
|
15458
|
+
res.status,
|
|
15459
|
+
body.slice(0, 500)
|
|
15460
|
+
);
|
|
15461
|
+
}
|
|
15462
|
+
const data = await res.json();
|
|
15463
|
+
if (!data.access_token) {
|
|
15464
|
+
throw new CloudRunAuthError("Service-account token response missing access_token", res.status);
|
|
15465
|
+
}
|
|
15466
|
+
return data.access_token;
|
|
15467
|
+
}
|
|
15468
|
+
|
|
15469
|
+
// ../integration-cloud-run/src/filter.ts
|
|
15470
|
+
function assertNonEmpty(name, value) {
|
|
15471
|
+
if (!value.trim()) {
|
|
15472
|
+
throw new Error(`${name} must be a non-empty string`);
|
|
15473
|
+
}
|
|
15474
|
+
}
|
|
15475
|
+
function quoteLogFilterValue(value) {
|
|
15476
|
+
return JSON.stringify(value);
|
|
15477
|
+
}
|
|
15478
|
+
function normalizeTimestamp(value) {
|
|
15479
|
+
const date = value instanceof Date ? value : new Date(value);
|
|
15480
|
+
if (Number.isNaN(date.getTime())) {
|
|
15481
|
+
throw new Error(`Invalid timestamp: ${String(value)}`);
|
|
15482
|
+
}
|
|
15483
|
+
return date.toISOString();
|
|
15484
|
+
}
|
|
15485
|
+
function buildCloudRunLogFilter(options = {}) {
|
|
15486
|
+
const clauses = ['resource.type="cloud_run_revision"'];
|
|
15487
|
+
if (options.serviceName !== void 0) {
|
|
15488
|
+
assertNonEmpty("serviceName", options.serviceName);
|
|
15489
|
+
clauses.push(`resource.labels.service_name=${quoteLogFilterValue(options.serviceName)}`);
|
|
15490
|
+
}
|
|
15491
|
+
if (options.location !== void 0) {
|
|
15492
|
+
assertNonEmpty("location", options.location);
|
|
15493
|
+
clauses.push(`resource.labels.location=${quoteLogFilterValue(options.location)}`);
|
|
15494
|
+
}
|
|
15495
|
+
if (options.startTime !== void 0) {
|
|
15496
|
+
clauses.push(`timestamp >= ${quoteLogFilterValue(normalizeTimestamp(options.startTime))}`);
|
|
15497
|
+
}
|
|
15498
|
+
if (options.endTime !== void 0) {
|
|
15499
|
+
clauses.push(`timestamp < ${quoteLogFilterValue(normalizeTimestamp(options.endTime))}`);
|
|
15500
|
+
}
|
|
15501
|
+
const userAgentSubstrings = (options.userAgentSubstrings ?? []).map((pattern) => pattern.trim()).filter(Boolean);
|
|
15502
|
+
if (userAgentSubstrings.length > 0) {
|
|
15503
|
+
const uaClauses = userAgentSubstrings.map((pattern) => `httpRequest.userAgent:${quoteLogFilterValue(pattern)}`);
|
|
15504
|
+
clauses.push(`(${uaClauses.join(" OR ")})`);
|
|
15505
|
+
}
|
|
15506
|
+
const requestUrlSubstrings = (options.requestUrlSubstrings ?? []).map((pattern) => pattern.trim()).filter(Boolean);
|
|
15507
|
+
if (requestUrlSubstrings.length > 0) {
|
|
15508
|
+
const urlClauses = requestUrlSubstrings.map((pattern) => `httpRequest.requestUrl:${quoteLogFilterValue(pattern)}`);
|
|
15509
|
+
clauses.push(`(${urlClauses.join(" OR ")})`);
|
|
15510
|
+
}
|
|
15511
|
+
return clauses.join(" AND ");
|
|
15512
|
+
}
|
|
15513
|
+
|
|
15514
|
+
// ../integration-cloud-run/src/normalize.ts
|
|
15515
|
+
function numberOrNull(value) {
|
|
15516
|
+
if (value === void 0 || value === null) return null;
|
|
15517
|
+
const parsed = typeof value === "number" ? value : Number(value);
|
|
15518
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
15519
|
+
}
|
|
15520
|
+
function latencyToMs(value) {
|
|
15521
|
+
if (!value) return null;
|
|
15522
|
+
const secondsMatch = /^([0-9]+(?:\.[0-9]+)?)s$/.exec(value.trim());
|
|
15523
|
+
if (!secondsMatch) return null;
|
|
15524
|
+
const seconds = Number(secondsMatch[1]);
|
|
15525
|
+
return Number.isFinite(seconds) ? Math.round(seconds * 1e6) / 1e3 : null;
|
|
15526
|
+
}
|
|
15527
|
+
function normalizeLabels(labels) {
|
|
15528
|
+
if (!labels) return {};
|
|
15529
|
+
return Object.fromEntries(
|
|
15530
|
+
Object.entries(labels).filter((entry) => typeof entry[0] === "string" && typeof entry[1] === "string")
|
|
15531
|
+
);
|
|
15532
|
+
}
|
|
15533
|
+
function parseRequestUrl(requestUrl) {
|
|
15534
|
+
try {
|
|
15535
|
+
const url = requestUrl.startsWith("/") ? new URL(requestUrl, "https://canonry.local") : new URL(requestUrl);
|
|
15536
|
+
return {
|
|
15537
|
+
host: url.hostname === "canonry.local" ? null : url.hostname,
|
|
15538
|
+
path: url.pathname || "/",
|
|
15539
|
+
queryString: url.search ? url.search.slice(1) : null
|
|
15540
|
+
};
|
|
15541
|
+
} catch {
|
|
15542
|
+
return null;
|
|
15543
|
+
}
|
|
15544
|
+
}
|
|
15545
|
+
function buildEventId(entry, observedAt, requestUrl) {
|
|
15546
|
+
if (entry.insertId?.trim()) {
|
|
15547
|
+
return `cloud-run:${observedAt}:${entry.insertId}`;
|
|
15548
|
+
}
|
|
15549
|
+
return `cloud-run:${observedAt}:${requestUrl}`;
|
|
15550
|
+
}
|
|
15551
|
+
function normalizeCloudRunLogEntry(entry) {
|
|
15552
|
+
const request = entry.httpRequest;
|
|
15553
|
+
if (!request?.requestUrl) return null;
|
|
15554
|
+
const observedAt = entry.timestamp ?? entry.receiveTimestamp;
|
|
15555
|
+
if (!observedAt) return null;
|
|
15556
|
+
const urlParts = parseRequestUrl(request.requestUrl);
|
|
15557
|
+
if (!urlParts) return null;
|
|
15558
|
+
return {
|
|
15559
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
15560
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
15561
|
+
confidence: TrafficEventConfidences.observed,
|
|
15562
|
+
eventId: buildEventId(entry, observedAt, request.requestUrl),
|
|
15563
|
+
observedAt,
|
|
15564
|
+
method: request.requestMethod ?? null,
|
|
15565
|
+
requestUrl: request.requestUrl,
|
|
15566
|
+
host: urlParts.host,
|
|
15567
|
+
path: urlParts.path,
|
|
15568
|
+
queryString: urlParts.queryString,
|
|
15569
|
+
status: numberOrNull(request.status),
|
|
15570
|
+
userAgent: request.userAgent ?? null,
|
|
15571
|
+
remoteIp: request.remoteIp ?? null,
|
|
15572
|
+
referer: request.referer ?? null,
|
|
15573
|
+
latencyMs: latencyToMs(request.latency),
|
|
15574
|
+
requestSizeBytes: numberOrNull(request.requestSize),
|
|
15575
|
+
responseSizeBytes: numberOrNull(request.responseSize),
|
|
15576
|
+
providerResource: {
|
|
15577
|
+
type: entry.resource?.type ?? null,
|
|
15578
|
+
labels: normalizeLabels(entry.resource?.labels)
|
|
15579
|
+
},
|
|
15580
|
+
providerLabels: normalizeLabels(entry.labels)
|
|
15581
|
+
};
|
|
15582
|
+
}
|
|
15583
|
+
|
|
15584
|
+
// ../integration-cloud-run/src/client.ts
|
|
15585
|
+
var CLOUD_LOGGING_ENTRIES_LIST_URL = "https://logging.googleapis.com/v2/entries:list";
|
|
15586
|
+
var DEFAULT_PAGE_SIZE = 1e3;
|
|
15587
|
+
var DEFAULT_MAX_PAGES = 1;
|
|
15588
|
+
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
15589
|
+
var CloudRunLoggingApiError = class extends Error {
|
|
15590
|
+
constructor(message, status, body) {
|
|
15591
|
+
super(message);
|
|
15592
|
+
this.status = status;
|
|
15593
|
+
this.body = body;
|
|
15594
|
+
this.name = "CloudRunLoggingApiError";
|
|
15595
|
+
}
|
|
15596
|
+
};
|
|
15597
|
+
function validateAccessToken3(accessToken) {
|
|
15598
|
+
if (!accessToken.trim()) {
|
|
15599
|
+
throw new CloudRunLoggingApiError("Cloud Logging access token is required", 400);
|
|
15600
|
+
}
|
|
15601
|
+
}
|
|
15602
|
+
function validateProjectId(gcpProjectId) {
|
|
15603
|
+
if (!gcpProjectId.trim()) {
|
|
15604
|
+
throw new CloudRunLoggingApiError("GCP project ID is required", 400);
|
|
15605
|
+
}
|
|
15606
|
+
}
|
|
15607
|
+
function normalizePageSize(pageSize) {
|
|
15608
|
+
if (pageSize === void 0) return DEFAULT_PAGE_SIZE;
|
|
15609
|
+
if (!Number.isInteger(pageSize) || pageSize < 1) {
|
|
15610
|
+
throw new CloudRunLoggingApiError("pageSize must be a positive integer", 400);
|
|
15611
|
+
}
|
|
15612
|
+
return pageSize;
|
|
15613
|
+
}
|
|
15614
|
+
function normalizeMaxPages(maxPages) {
|
|
15615
|
+
if (maxPages === void 0) return DEFAULT_MAX_PAGES;
|
|
15616
|
+
if (!Number.isInteger(maxPages) || maxPages < 1) {
|
|
15617
|
+
throw new CloudRunLoggingApiError("maxPages must be a positive integer", 400);
|
|
15618
|
+
}
|
|
15619
|
+
return maxPages;
|
|
15620
|
+
}
|
|
15621
|
+
async function readErrorBody(response) {
|
|
15622
|
+
const text = await response.text().catch(() => "");
|
|
15623
|
+
if (!text) return void 0;
|
|
15624
|
+
return text.length <= 500 ? text : `${text.slice(0, 500)}... [truncated]`;
|
|
15625
|
+
}
|
|
15626
|
+
async function listCloudRunTrafficEvents(accessToken, options) {
|
|
15627
|
+
validateAccessToken3(accessToken);
|
|
15628
|
+
validateProjectId(options.gcpProjectId);
|
|
15629
|
+
const filter = buildCloudRunLogFilter(options);
|
|
15630
|
+
const pageSize = normalizePageSize(options.pageSize);
|
|
15631
|
+
const maxPages = normalizeMaxPages(options.maxPages);
|
|
15632
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
15633
|
+
let pageToken = options.pageToken;
|
|
15634
|
+
let rawEntryCount = 0;
|
|
15635
|
+
let skippedEntryCount = 0;
|
|
15636
|
+
const events = [];
|
|
15637
|
+
for (let page = 0; page < maxPages; page += 1) {
|
|
15638
|
+
const requestBody = {
|
|
15639
|
+
resourceNames: [`projects/${options.gcpProjectId}`],
|
|
15640
|
+
filter,
|
|
15641
|
+
orderBy: options.orderBy ?? "timestamp asc",
|
|
15642
|
+
pageSize
|
|
15643
|
+
};
|
|
15644
|
+
if (pageToken) {
|
|
15645
|
+
requestBody.pageToken = pageToken;
|
|
15646
|
+
}
|
|
15647
|
+
const response = await fetch(CLOUD_LOGGING_ENTRIES_LIST_URL, {
|
|
15648
|
+
method: "POST",
|
|
15649
|
+
headers: {
|
|
15650
|
+
Authorization: `Bearer ${accessToken}`,
|
|
15651
|
+
"Content-Type": "application/json"
|
|
15652
|
+
},
|
|
15653
|
+
body: JSON.stringify(requestBody),
|
|
15654
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
15655
|
+
});
|
|
15656
|
+
if (!response.ok) {
|
|
15657
|
+
const body2 = await readErrorBody(response);
|
|
15658
|
+
throw new CloudRunLoggingApiError(
|
|
15659
|
+
`Cloud Logging entries.list failed with HTTP ${response.status}`,
|
|
15660
|
+
response.status,
|
|
15661
|
+
body2
|
|
15662
|
+
);
|
|
15663
|
+
}
|
|
15664
|
+
const body = await response.json();
|
|
15665
|
+
const entries = body.entries ?? [];
|
|
15666
|
+
rawEntryCount += entries.length;
|
|
15667
|
+
for (const entry of entries) {
|
|
15668
|
+
const event = normalizeCloudRunLogEntry(entry);
|
|
15669
|
+
if (event) {
|
|
15670
|
+
events.push(event);
|
|
15671
|
+
} else {
|
|
15672
|
+
skippedEntryCount += 1;
|
|
15673
|
+
}
|
|
15674
|
+
}
|
|
15675
|
+
pageToken = body.nextPageToken;
|
|
15676
|
+
if (!pageToken) break;
|
|
15677
|
+
}
|
|
15678
|
+
return {
|
|
15679
|
+
events,
|
|
15680
|
+
rawEntryCount,
|
|
15681
|
+
skippedEntryCount,
|
|
15682
|
+
nextPageToken: pageToken,
|
|
15683
|
+
filter
|
|
15684
|
+
};
|
|
15685
|
+
}
|
|
15686
|
+
|
|
15687
|
+
// ../integration-traffic/src/rules.ts
|
|
15688
|
+
var DEFAULT_AI_CRAWLER_RULES = [
|
|
15689
|
+
{
|
|
15690
|
+
id: "openai-gptbot",
|
|
15691
|
+
operator: "OpenAI",
|
|
15692
|
+
product: "GPTBot",
|
|
15693
|
+
purpose: "training",
|
|
15694
|
+
userAgentPatterns: [/GPTBot\//i]
|
|
15695
|
+
},
|
|
15696
|
+
{
|
|
15697
|
+
id: "openai-searchbot",
|
|
15698
|
+
operator: "OpenAI",
|
|
15699
|
+
product: "OAI-SearchBot",
|
|
15700
|
+
purpose: "search",
|
|
15701
|
+
userAgentPatterns: [/OAI-SearchBot\//i]
|
|
15702
|
+
},
|
|
15703
|
+
{
|
|
15704
|
+
id: "openai-chatgpt-user",
|
|
15705
|
+
operator: "OpenAI",
|
|
15706
|
+
product: "ChatGPT-User",
|
|
15707
|
+
purpose: "user-agent",
|
|
15708
|
+
userAgentPatterns: [/ChatGPT-User\//i]
|
|
15709
|
+
},
|
|
15710
|
+
{
|
|
15711
|
+
id: "anthropic-claudebot",
|
|
15712
|
+
operator: "Anthropic",
|
|
15713
|
+
product: "ClaudeBot",
|
|
15714
|
+
purpose: "training",
|
|
15715
|
+
userAgentPatterns: [/ClaudeBot\//i, /Claude-Web\//i, /anthropic-ai/i]
|
|
15716
|
+
},
|
|
15717
|
+
{
|
|
15718
|
+
id: "perplexity-bot",
|
|
15719
|
+
operator: "Perplexity",
|
|
15720
|
+
product: "PerplexityBot",
|
|
15721
|
+
purpose: "search",
|
|
15722
|
+
userAgentPatterns: [/PerplexityBot\//i]
|
|
15723
|
+
},
|
|
15724
|
+
{
|
|
15725
|
+
id: "google-extended",
|
|
15726
|
+
operator: "Google",
|
|
15727
|
+
product: "Google-Extended",
|
|
15728
|
+
purpose: "training-control",
|
|
15729
|
+
userAgentPatterns: [/Google-Extended/i]
|
|
15730
|
+
},
|
|
15731
|
+
{
|
|
15732
|
+
id: "bytespider",
|
|
15733
|
+
operator: "ByteDance",
|
|
15734
|
+
product: "Bytespider",
|
|
15735
|
+
purpose: "training",
|
|
15736
|
+
userAgentPatterns: [/Bytespider/i]
|
|
15737
|
+
},
|
|
15738
|
+
{
|
|
15739
|
+
id: "applebot-extended",
|
|
15740
|
+
operator: "Apple",
|
|
15741
|
+
product: "Applebot-Extended",
|
|
15742
|
+
purpose: "training",
|
|
15743
|
+
userAgentPatterns: [/Applebot-Extended/i]
|
|
15744
|
+
},
|
|
15745
|
+
{
|
|
15746
|
+
id: "meta-externalagent",
|
|
15747
|
+
operator: "Meta",
|
|
15748
|
+
product: "meta-externalagent",
|
|
15749
|
+
purpose: "training",
|
|
15750
|
+
userAgentPatterns: [/meta-externalagent/i]
|
|
15751
|
+
},
|
|
15752
|
+
{
|
|
15753
|
+
id: "ccbot",
|
|
15754
|
+
operator: "Common Crawl",
|
|
15755
|
+
product: "CCBot",
|
|
15756
|
+
purpose: "crawl",
|
|
15757
|
+
userAgentPatterns: [/CCBot\//i]
|
|
15758
|
+
},
|
|
15759
|
+
{
|
|
15760
|
+
id: "cohere-ai",
|
|
15761
|
+
operator: "Cohere",
|
|
15762
|
+
product: "cohere-ai",
|
|
15763
|
+
purpose: "training",
|
|
15764
|
+
userAgentPatterns: [/cohere-ai/i]
|
|
15765
|
+
},
|
|
15766
|
+
{
|
|
15767
|
+
id: "diffbot",
|
|
15768
|
+
operator: "Diffbot",
|
|
15769
|
+
product: "Diffbot",
|
|
15770
|
+
purpose: "crawl",
|
|
15771
|
+
userAgentPatterns: [/Diffbot/i]
|
|
15772
|
+
},
|
|
15773
|
+
{
|
|
15774
|
+
id: "mistral-ai",
|
|
15775
|
+
operator: "Mistral AI",
|
|
15776
|
+
product: "MistralAI-User",
|
|
15777
|
+
purpose: "crawl",
|
|
15778
|
+
userAgentPatterns: [/MistralAI/i]
|
|
15779
|
+
}
|
|
15780
|
+
];
|
|
15781
|
+
var DEFAULT_AI_REFERRER_RULES = [
|
|
15782
|
+
{ domain: "chatgpt.com", operator: "OpenAI", product: "ChatGPT" },
|
|
15783
|
+
{ domain: "chat.openai.com", operator: "OpenAI", product: "ChatGPT" },
|
|
15784
|
+
{ domain: "perplexity.ai", operator: "Perplexity", product: "Perplexity" },
|
|
15785
|
+
{ domain: "claude.ai", operator: "Anthropic", product: "Claude" },
|
|
15786
|
+
{ domain: "gemini.google.com", operator: "Google", product: "Gemini" },
|
|
15787
|
+
{ domain: "copilot.microsoft.com", operator: "Microsoft", product: "Copilot" },
|
|
15788
|
+
{ domain: "phind.com", operator: "Phind", product: "Phind" },
|
|
15789
|
+
{ domain: "you.com", operator: "You.com", product: "You.com" },
|
|
15790
|
+
{ domain: "meta.ai", operator: "Meta", product: "Meta AI" }
|
|
15791
|
+
];
|
|
15792
|
+
|
|
15793
|
+
// ../integration-traffic/src/classifier.ts
|
|
15794
|
+
function normalizeHost(host) {
|
|
15795
|
+
return host.trim().toLowerCase().replace(/^www\./, "");
|
|
15796
|
+
}
|
|
15797
|
+
function hostMatches(host, domain) {
|
|
15798
|
+
const normalizedHost = normalizeHost(host);
|
|
15799
|
+
const normalizedDomain = normalizeHost(domain);
|
|
15800
|
+
return normalizedHost === normalizedDomain || normalizedHost.endsWith(`.${normalizedDomain}`);
|
|
15801
|
+
}
|
|
15802
|
+
function hostFromUrl(value) {
|
|
15803
|
+
if (!value) return null;
|
|
15804
|
+
try {
|
|
15805
|
+
return normalizeHost(new URL(value).hostname);
|
|
15806
|
+
} catch {
|
|
15807
|
+
return null;
|
|
15808
|
+
}
|
|
15809
|
+
}
|
|
15810
|
+
function utmSourceFromQuery(queryString) {
|
|
15811
|
+
if (!queryString) return null;
|
|
15812
|
+
const params = new URLSearchParams(queryString);
|
|
15813
|
+
const source = params.get("utm_source");
|
|
15814
|
+
return source ? normalizeHost(source) : null;
|
|
15815
|
+
}
|
|
15816
|
+
function classifyCrawler(event) {
|
|
15817
|
+
const userAgent = event.userAgent?.trim();
|
|
15818
|
+
if (!userAgent) return null;
|
|
15819
|
+
for (const rule of DEFAULT_AI_CRAWLER_RULES) {
|
|
15820
|
+
if (rule.userAgentPatterns.some((pattern) => pattern.test(userAgent))) {
|
|
15821
|
+
return {
|
|
15822
|
+
botId: rule.id,
|
|
15823
|
+
operator: rule.operator,
|
|
15824
|
+
product: rule.product,
|
|
15825
|
+
purpose: rule.purpose,
|
|
15826
|
+
verificationStatus: "claimed_unverified",
|
|
15827
|
+
matchedUserAgent: userAgent
|
|
15828
|
+
};
|
|
15829
|
+
}
|
|
15830
|
+
}
|
|
15831
|
+
return null;
|
|
15832
|
+
}
|
|
15833
|
+
function classifyAiReferral(event) {
|
|
15834
|
+
const refererHost = hostFromUrl(event.referer);
|
|
15835
|
+
if (refererHost) {
|
|
15836
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(refererHost, candidate.domain));
|
|
15837
|
+
if (rule) {
|
|
15838
|
+
return {
|
|
15839
|
+
operator: rule.operator,
|
|
15840
|
+
product: rule.product,
|
|
15841
|
+
sourceDomain: refererHost,
|
|
15842
|
+
evidenceType: "referer"
|
|
15843
|
+
};
|
|
15844
|
+
}
|
|
15845
|
+
}
|
|
15846
|
+
const utmSource = utmSourceFromQuery(event.queryString);
|
|
15847
|
+
if (utmSource) {
|
|
15848
|
+
const rule = DEFAULT_AI_REFERRER_RULES.find((candidate) => hostMatches(utmSource, candidate.domain));
|
|
15849
|
+
if (rule) {
|
|
15850
|
+
return {
|
|
15851
|
+
operator: rule.operator,
|
|
15852
|
+
product: rule.product,
|
|
15853
|
+
sourceDomain: utmSource,
|
|
15854
|
+
evidenceType: "utm"
|
|
15855
|
+
};
|
|
15856
|
+
}
|
|
15857
|
+
}
|
|
15858
|
+
return null;
|
|
15859
|
+
}
|
|
15860
|
+
|
|
15861
|
+
// ../integration-traffic/src/rollup.ts
|
|
15862
|
+
var DEFAULT_SAMPLE_LIMIT = 25;
|
|
15863
|
+
var UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
15864
|
+
var LONG_HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
|
|
15865
|
+
var NUMERIC_SEGMENT = /^\d+$/;
|
|
15866
|
+
function normalizeTrafficPathPattern(path15) {
|
|
15867
|
+
const cleanPath = path15.trim() || "/";
|
|
15868
|
+
const pathOnly = cleanPath.split("?")[0] || "/";
|
|
15869
|
+
const segments = pathOnly.split("/").map((segment) => {
|
|
15870
|
+
if (!segment) return segment;
|
|
15871
|
+
if (UUID_SEGMENT.test(segment) || LONG_HEX_SEGMENT.test(segment) || NUMERIC_SEGMENT.test(segment)) {
|
|
15872
|
+
return ":id";
|
|
15873
|
+
}
|
|
15874
|
+
return segment;
|
|
15875
|
+
});
|
|
15876
|
+
const normalized = segments.join("/");
|
|
15877
|
+
return normalized.startsWith("/") ? normalized : `/${normalized}`;
|
|
15878
|
+
}
|
|
15879
|
+
function hourBucket(value) {
|
|
15880
|
+
const date = new Date(value);
|
|
15881
|
+
if (Number.isNaN(date.getTime())) return value;
|
|
15882
|
+
date.setUTCMinutes(0, 0, 0);
|
|
15883
|
+
return date.toISOString();
|
|
15884
|
+
}
|
|
15885
|
+
function sortCrawlerBuckets(a, b) {
|
|
15886
|
+
return a.tsHour.localeCompare(b.tsHour) || a.botId.localeCompare(b.botId) || a.pathNormalized.localeCompare(b.pathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
15887
|
+
}
|
|
15888
|
+
function sortReferralBuckets(a, b) {
|
|
15889
|
+
return a.tsHour.localeCompare(b.tsHour) || a.product.localeCompare(b.product) || a.sourceDomain.localeCompare(b.sourceDomain) || a.landingPathNormalized.localeCompare(b.landingPathNormalized) || String(a.status).localeCompare(String(b.status));
|
|
15890
|
+
}
|
|
15891
|
+
function topEntries(map, limit) {
|
|
15892
|
+
return [...map.values()].sort((a, b) => b.hits - a.hits || JSON.stringify(a.fields).localeCompare(JSON.stringify(b.fields))).slice(0, limit).map((entry) => ({ ...entry.fields, hits: entry.hits }));
|
|
15893
|
+
}
|
|
15894
|
+
function buildTrafficProbeReport(events, options = {}) {
|
|
15895
|
+
const sampleLimit = options.sampleLimit ?? DEFAULT_SAMPLE_LIMIT;
|
|
15896
|
+
const crawlerBuckets = /* @__PURE__ */ new Map();
|
|
15897
|
+
const aiReferralBuckets = /* @__PURE__ */ new Map();
|
|
15898
|
+
const topBots = /* @__PURE__ */ new Map();
|
|
15899
|
+
const topCrawlerPaths = /* @__PURE__ */ new Map();
|
|
15900
|
+
const topAiReferrers = /* @__PURE__ */ new Map();
|
|
15901
|
+
const topAiReferralLandingPaths = /* @__PURE__ */ new Map();
|
|
15902
|
+
let crawlerHits = 0;
|
|
15903
|
+
let aiReferralHits = 0;
|
|
15904
|
+
let unknownHits = 0;
|
|
15905
|
+
const samples = [];
|
|
15906
|
+
for (const event of events) {
|
|
15907
|
+
const tsHour = hourBucket(event.observedAt);
|
|
15908
|
+
const pathNormalized = normalizeTrafficPathPattern(event.path);
|
|
15909
|
+
const crawler = classifyCrawler(event);
|
|
15910
|
+
const aiReferral = classifyAiReferral(event);
|
|
15911
|
+
if (crawler) {
|
|
15912
|
+
crawlerHits += 1;
|
|
15913
|
+
const key = [
|
|
15914
|
+
tsHour,
|
|
15915
|
+
crawler.botId,
|
|
15916
|
+
crawler.verificationStatus,
|
|
15917
|
+
pathNormalized,
|
|
15918
|
+
event.status ?? "null"
|
|
15919
|
+
].join(" ");
|
|
15920
|
+
const existing = crawlerBuckets.get(key);
|
|
15921
|
+
if (existing) {
|
|
15922
|
+
existing.hits += 1;
|
|
15923
|
+
} else {
|
|
15924
|
+
crawlerBuckets.set(key, {
|
|
15925
|
+
tsHour,
|
|
15926
|
+
botId: crawler.botId,
|
|
15927
|
+
operator: crawler.operator,
|
|
15928
|
+
product: crawler.product,
|
|
15929
|
+
verificationStatus: crawler.verificationStatus,
|
|
15930
|
+
pathNormalized,
|
|
15931
|
+
status: event.status,
|
|
15932
|
+
hits: 1,
|
|
15933
|
+
sampledUserAgent: event.userAgent
|
|
15934
|
+
});
|
|
15935
|
+
}
|
|
15936
|
+
const botKey = `${crawler.botId} ${crawler.operator}`;
|
|
15937
|
+
const botEntry = topBots.get(botKey);
|
|
15938
|
+
if (botEntry) botEntry.hits += 1;
|
|
15939
|
+
else topBots.set(botKey, { fields: { botId: crawler.botId, operator: crawler.operator }, hits: 1 });
|
|
15940
|
+
incrementBucket(topCrawlerPaths, pathNormalized, { pathNormalized });
|
|
15941
|
+
}
|
|
15942
|
+
if (aiReferral) {
|
|
15943
|
+
aiReferralHits += 1;
|
|
15944
|
+
const key = [
|
|
15945
|
+
tsHour,
|
|
15946
|
+
aiReferral.product,
|
|
15947
|
+
aiReferral.sourceDomain,
|
|
15948
|
+
aiReferral.evidenceType,
|
|
15949
|
+
pathNormalized,
|
|
15950
|
+
event.status ?? "null"
|
|
15951
|
+
].join(" ");
|
|
15952
|
+
const existing = aiReferralBuckets.get(key);
|
|
15953
|
+
if (existing) {
|
|
15954
|
+
existing.hits += 1;
|
|
15955
|
+
} else {
|
|
15956
|
+
aiReferralBuckets.set(key, {
|
|
15957
|
+
tsHour,
|
|
15958
|
+
operator: aiReferral.operator,
|
|
15959
|
+
product: aiReferral.product,
|
|
15960
|
+
sourceDomain: aiReferral.sourceDomain,
|
|
15961
|
+
evidenceType: aiReferral.evidenceType,
|
|
15962
|
+
landingPathNormalized: pathNormalized,
|
|
15963
|
+
status: event.status,
|
|
15964
|
+
hits: 1
|
|
15965
|
+
});
|
|
15966
|
+
}
|
|
15967
|
+
incrementBucket(topAiReferrers, aiReferral.sourceDomain, {
|
|
15968
|
+
sourceDomain: aiReferral.sourceDomain,
|
|
15969
|
+
product: aiReferral.product
|
|
15970
|
+
});
|
|
15971
|
+
incrementBucket(topAiReferralLandingPaths, pathNormalized, { landingPathNormalized: pathNormalized });
|
|
15972
|
+
}
|
|
15973
|
+
if (!crawler && !aiReferral) unknownHits += 1;
|
|
15974
|
+
if (samples.length < sampleLimit) {
|
|
15975
|
+
samples.push({
|
|
15976
|
+
eventId: event.eventId,
|
|
15977
|
+
observedAt: event.observedAt,
|
|
15978
|
+
sourceType: event.sourceType,
|
|
15979
|
+
path: event.path,
|
|
15980
|
+
pathNormalized,
|
|
15981
|
+
status: event.status,
|
|
15982
|
+
userAgent: event.userAgent,
|
|
15983
|
+
referer: event.referer,
|
|
15984
|
+
crawler,
|
|
15985
|
+
aiReferral
|
|
15986
|
+
});
|
|
15987
|
+
}
|
|
15988
|
+
}
|
|
15989
|
+
return {
|
|
15990
|
+
generatedAt: options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
15991
|
+
totals: {
|
|
15992
|
+
normalizedEvents: events.length,
|
|
15993
|
+
crawlerHits,
|
|
15994
|
+
aiReferralHits,
|
|
15995
|
+
unknownHits
|
|
15996
|
+
},
|
|
15997
|
+
crawlerEventsHourly: [...crawlerBuckets.values()].sort(sortCrawlerBuckets),
|
|
15998
|
+
aiReferralEventsHourly: [...aiReferralBuckets.values()].sort(sortReferralBuckets),
|
|
15999
|
+
topBots: topEntries(topBots, 10),
|
|
16000
|
+
topCrawlerPaths: topEntries(topCrawlerPaths, 10),
|
|
16001
|
+
topAiReferrers: topEntries(topAiReferrers, 10),
|
|
16002
|
+
topAiReferralLandingPaths: topEntries(topAiReferralLandingPaths, 10),
|
|
16003
|
+
samples
|
|
16004
|
+
};
|
|
16005
|
+
}
|
|
16006
|
+
function incrementBucket(map, key, fields) {
|
|
16007
|
+
const existing = map.get(key);
|
|
16008
|
+
if (existing) existing.hits += 1;
|
|
16009
|
+
else map.set(key, { fields, hits: 1 });
|
|
16010
|
+
}
|
|
16011
|
+
|
|
16012
|
+
// ../api-routes/src/traffic.ts
|
|
16013
|
+
var DEFAULT_SYNC_WINDOW_MINUTES = 60;
|
|
16014
|
+
var DEFAULT_PAGE_SIZE2 = 1e3;
|
|
16015
|
+
var DEFAULT_MAX_PAGES2 = 5;
|
|
16016
|
+
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
16017
|
+
function parseSourceConfig(row) {
|
|
16018
|
+
return parseJsonColumn(row.configJson, {});
|
|
16019
|
+
}
|
|
16020
|
+
function rowToDto(row) {
|
|
16021
|
+
return {
|
|
16022
|
+
id: row.id,
|
|
16023
|
+
projectId: row.projectId,
|
|
16024
|
+
sourceType: row.sourceType,
|
|
16025
|
+
displayName: row.displayName,
|
|
16026
|
+
status: row.status,
|
|
16027
|
+
lastSyncedAt: row.lastSyncedAt ?? null,
|
|
16028
|
+
lastCursor: row.lastCursor ?? null,
|
|
16029
|
+
lastError: row.lastError ?? null,
|
|
16030
|
+
archivedAt: row.archivedAt ?? null,
|
|
16031
|
+
config: parseSourceConfig(row),
|
|
16032
|
+
createdAt: row.createdAt,
|
|
16033
|
+
updatedAt: row.updatedAt
|
|
16034
|
+
};
|
|
16035
|
+
}
|
|
16036
|
+
async function defaultResolveAccessToken(record) {
|
|
16037
|
+
if (record.authMode === TrafficSourceAuthModes["service-account"]) {
|
|
16038
|
+
if (!record.clientEmail || !record.privateKey) {
|
|
16039
|
+
throw validationError("Service-account credentials missing client_email or private_key");
|
|
16040
|
+
}
|
|
16041
|
+
return getCloudLoggingAccessToken(record.clientEmail, record.privateKey);
|
|
16042
|
+
}
|
|
16043
|
+
throw validationError(
|
|
16044
|
+
"OAuth-mode Cloud Run sync is not yet supported in v1. Provide a service-account key file."
|
|
16045
|
+
);
|
|
16046
|
+
}
|
|
16047
|
+
async function trafficRoutes(app, opts) {
|
|
16048
|
+
const pullEvents = opts.pullCloudRunEvents ?? listCloudRunTrafficEvents;
|
|
16049
|
+
const resolveAccessToken2 = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
16050
|
+
const syncWindowMinutes = opts.defaultSyncWindowMinutes ?? DEFAULT_SYNC_WINDOW_MINUTES;
|
|
16051
|
+
const pageSize = opts.defaultPageSize ?? DEFAULT_PAGE_SIZE2;
|
|
16052
|
+
const maxPages = opts.defaultMaxPages ?? DEFAULT_MAX_PAGES2;
|
|
16053
|
+
const sampleLimit = opts.defaultSampleLimit ?? DEFAULT_SAMPLE_LIMIT2;
|
|
16054
|
+
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
16055
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16056
|
+
const body = request.body ?? {};
|
|
16057
|
+
const { gcpProjectId, serviceName, location, displayName, keyJson } = body;
|
|
16058
|
+
if (!gcpProjectId || typeof gcpProjectId !== "string") {
|
|
16059
|
+
throw validationError("gcpProjectId is required");
|
|
16060
|
+
}
|
|
16061
|
+
if (!keyJson) {
|
|
16062
|
+
throw validationError(
|
|
16063
|
+
"keyJson is required for v1 (service-account JSON content). OAuth-mode Cloud Run is not yet supported."
|
|
16064
|
+
);
|
|
16065
|
+
}
|
|
16066
|
+
if (!opts.cloudRunCredentialStore) {
|
|
16067
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
16068
|
+
}
|
|
16069
|
+
let parsed;
|
|
16070
|
+
try {
|
|
16071
|
+
parsed = JSON.parse(keyJson);
|
|
16072
|
+
} catch {
|
|
16073
|
+
throw validationError("Invalid JSON in keyJson");
|
|
16074
|
+
}
|
|
16075
|
+
if (!parsed.client_email || !parsed.private_key) {
|
|
16076
|
+
throw validationError("Service-account JSON must contain client_email and private_key");
|
|
16077
|
+
}
|
|
16078
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
16079
|
+
const existing = opts.cloudRunCredentialStore.getConnection(project.name);
|
|
16080
|
+
opts.cloudRunCredentialStore.upsertConnection({
|
|
16081
|
+
projectName: project.name,
|
|
16082
|
+
gcpProjectId,
|
|
16083
|
+
serviceName: serviceName ?? void 0,
|
|
16084
|
+
location: location ?? void 0,
|
|
16085
|
+
authMode: TrafficSourceAuthModes["service-account"],
|
|
16086
|
+
clientEmail: parsed.client_email,
|
|
16087
|
+
privateKey: parsed.private_key,
|
|
16088
|
+
createdAt: existing?.createdAt ?? now,
|
|
16089
|
+
updatedAt: now
|
|
16090
|
+
});
|
|
16091
|
+
const activeSource = app.db.select().from(trafficSources).where(eq23(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
16092
|
+
const config = {
|
|
16093
|
+
gcpProjectId,
|
|
16094
|
+
serviceName: serviceName ?? null,
|
|
16095
|
+
location: location ?? null,
|
|
16096
|
+
authMode: TrafficSourceAuthModes["service-account"]
|
|
16097
|
+
};
|
|
16098
|
+
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
16099
|
+
let sourceRow;
|
|
16100
|
+
if (activeSource) {
|
|
16101
|
+
app.db.update(trafficSources).set({
|
|
16102
|
+
displayName: fallbackName,
|
|
16103
|
+
status: TrafficSourceStatuses.connected,
|
|
16104
|
+
lastError: null,
|
|
16105
|
+
configJson: JSON.stringify(config),
|
|
16106
|
+
updatedAt: now
|
|
16107
|
+
}).where(eq23(trafficSources.id, activeSource.id)).run();
|
|
16108
|
+
sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, activeSource.id)).get();
|
|
16109
|
+
} else {
|
|
16110
|
+
const newId = crypto20.randomUUID();
|
|
16111
|
+
app.db.insert(trafficSources).values({
|
|
16112
|
+
id: newId,
|
|
16113
|
+
projectId: project.id,
|
|
16114
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
16115
|
+
displayName: fallbackName,
|
|
16116
|
+
status: TrafficSourceStatuses.connected,
|
|
16117
|
+
lastSyncedAt: null,
|
|
16118
|
+
lastCursor: null,
|
|
16119
|
+
lastError: null,
|
|
16120
|
+
archivedAt: null,
|
|
16121
|
+
configJson: JSON.stringify(config),
|
|
16122
|
+
createdAt: now,
|
|
16123
|
+
updatedAt: now
|
|
16124
|
+
}).run();
|
|
16125
|
+
sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, newId)).get();
|
|
16126
|
+
}
|
|
16127
|
+
writeAuditLog(app.db, {
|
|
16128
|
+
projectId: project.id,
|
|
16129
|
+
actor: "api",
|
|
16130
|
+
action: "traffic.cloud-run.connected",
|
|
16131
|
+
entityType: "traffic_source",
|
|
16132
|
+
entityId: sourceRow.id
|
|
16133
|
+
});
|
|
16134
|
+
return rowToDto(sourceRow);
|
|
16135
|
+
});
|
|
16136
|
+
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
16137
|
+
const project = resolveProject(app.db, request.params.name);
|
|
16138
|
+
const sourceRow = app.db.select().from(trafficSources).where(eq23(trafficSources.id, request.params.id)).get();
|
|
16139
|
+
if (!sourceRow || sourceRow.projectId !== project.id) {
|
|
16140
|
+
throw notFound("Traffic source", request.params.id);
|
|
16141
|
+
}
|
|
16142
|
+
if (sourceRow.sourceType !== TrafficSourceTypes["cloud-run"]) {
|
|
16143
|
+
throw validationError(
|
|
16144
|
+
`Sync for source type "${sourceRow.sourceType}" is not implemented yet \u2014 only cloud-run is supported in v1.`
|
|
16145
|
+
);
|
|
16146
|
+
}
|
|
16147
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
16148
|
+
if (!credentialStore) {
|
|
16149
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
16150
|
+
}
|
|
16151
|
+
const credential = credentialStore.getConnection(project.name);
|
|
16152
|
+
if (!credential) {
|
|
16153
|
+
throw validationError(
|
|
16154
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
16155
|
+
);
|
|
16156
|
+
}
|
|
16157
|
+
const config = parseSourceConfig(sourceRow);
|
|
16158
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
16159
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
16160
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
16161
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
16162
|
+
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
16163
|
+
const windowEnd = /* @__PURE__ */ new Date();
|
|
16164
|
+
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
16165
|
+
const lastSyncedMs = sourceRow.lastSyncedAt ? new Date(sourceRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
16166
|
+
const windowStart = new Date(
|
|
16167
|
+
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
16168
|
+
);
|
|
16169
|
+
const startedAt = windowEnd.toISOString();
|
|
16170
|
+
const runId = crypto20.randomUUID();
|
|
16171
|
+
app.db.insert(runs).values({
|
|
16172
|
+
id: runId,
|
|
16173
|
+
projectId: project.id,
|
|
16174
|
+
kind: RunKinds["traffic-sync"],
|
|
16175
|
+
status: RunStatuses.running,
|
|
16176
|
+
trigger: RunTriggers.manual,
|
|
16177
|
+
startedAt,
|
|
16178
|
+
createdAt: startedAt
|
|
16179
|
+
}).run();
|
|
16180
|
+
let accessToken;
|
|
16181
|
+
try {
|
|
16182
|
+
accessToken = await resolveAccessToken2(credential);
|
|
16183
|
+
} catch (e) {
|
|
16184
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
16185
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(runs.id, runId)).run();
|
|
16186
|
+
app.db.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16187
|
+
throw validationError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
16188
|
+
}
|
|
16189
|
+
let allEvents = [];
|
|
16190
|
+
try {
|
|
16191
|
+
const page = await pullEvents(accessToken, {
|
|
16192
|
+
gcpProjectId,
|
|
16193
|
+
serviceName,
|
|
16194
|
+
location,
|
|
16195
|
+
startTime: windowStart.toISOString(),
|
|
16196
|
+
endTime: windowEnd.toISOString(),
|
|
16197
|
+
pageSize,
|
|
16198
|
+
maxPages
|
|
16199
|
+
});
|
|
16200
|
+
allEvents = page.events;
|
|
16201
|
+
} catch (e) {
|
|
16202
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
16203
|
+
app.db.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(runs.id, runId)).run();
|
|
16204
|
+
app.db.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16205
|
+
throw validationError(`Cloud Run pull failed: ${msg}`);
|
|
16206
|
+
}
|
|
16207
|
+
const report = buildTrafficProbeReport(allEvents, { sampleLimit });
|
|
16208
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16209
|
+
let crawlerBucketRows = 0;
|
|
16210
|
+
let aiReferralBucketRows = 0;
|
|
16211
|
+
let sampleRows = 0;
|
|
16212
|
+
app.db.transaction((tx) => {
|
|
16213
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
16214
|
+
const status = bucket.status ?? 0;
|
|
16215
|
+
tx.insert(crawlerEventsHourly).values({
|
|
16216
|
+
projectId: project.id,
|
|
16217
|
+
sourceId: sourceRow.id,
|
|
16218
|
+
tsHour: bucket.tsHour,
|
|
16219
|
+
botId: bucket.botId,
|
|
16220
|
+
operator: bucket.operator,
|
|
16221
|
+
verificationStatus: bucket.verificationStatus,
|
|
16222
|
+
pathNormalized: bucket.pathNormalized,
|
|
16223
|
+
status,
|
|
16224
|
+
hits: bucket.hits,
|
|
16225
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
16226
|
+
createdAt: finishedAt,
|
|
16227
|
+
updatedAt: finishedAt
|
|
16228
|
+
}).onConflictDoUpdate({
|
|
16229
|
+
target: [
|
|
16230
|
+
crawlerEventsHourly.projectId,
|
|
16231
|
+
crawlerEventsHourly.sourceId,
|
|
16232
|
+
crawlerEventsHourly.tsHour,
|
|
16233
|
+
crawlerEventsHourly.botId,
|
|
16234
|
+
crawlerEventsHourly.verificationStatus,
|
|
16235
|
+
crawlerEventsHourly.pathNormalized,
|
|
16236
|
+
crawlerEventsHourly.status
|
|
16237
|
+
],
|
|
16238
|
+
set: {
|
|
16239
|
+
hits: sql7`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
16240
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
16241
|
+
updatedAt: finishedAt
|
|
16242
|
+
}
|
|
16243
|
+
}).run();
|
|
16244
|
+
crawlerBucketRows += 1;
|
|
16245
|
+
}
|
|
16246
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
16247
|
+
const status = bucket.status ?? 0;
|
|
16248
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
16249
|
+
projectId: project.id,
|
|
16250
|
+
sourceId: sourceRow.id,
|
|
16251
|
+
tsHour: bucket.tsHour,
|
|
16252
|
+
product: bucket.product,
|
|
16253
|
+
operator: bucket.operator,
|
|
16254
|
+
sourceDomain: bucket.sourceDomain,
|
|
16255
|
+
evidenceType: bucket.evidenceType,
|
|
16256
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
16257
|
+
status,
|
|
16258
|
+
sessionsOrHits: bucket.hits,
|
|
16259
|
+
usersEstimated: null,
|
|
16260
|
+
createdAt: finishedAt,
|
|
16261
|
+
updatedAt: finishedAt
|
|
16262
|
+
}).onConflictDoUpdate({
|
|
16263
|
+
target: [
|
|
16264
|
+
aiReferralEventsHourly.projectId,
|
|
16265
|
+
aiReferralEventsHourly.sourceId,
|
|
16266
|
+
aiReferralEventsHourly.tsHour,
|
|
16267
|
+
aiReferralEventsHourly.product,
|
|
16268
|
+
aiReferralEventsHourly.sourceDomain,
|
|
16269
|
+
aiReferralEventsHourly.evidenceType,
|
|
16270
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
16271
|
+
aiReferralEventsHourly.status
|
|
16272
|
+
],
|
|
16273
|
+
set: {
|
|
16274
|
+
sessionsOrHits: sql7`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
16275
|
+
updatedAt: finishedAt
|
|
16276
|
+
}
|
|
16277
|
+
}).run();
|
|
16278
|
+
aiReferralBucketRows += 1;
|
|
16279
|
+
}
|
|
16280
|
+
for (const sample of report.samples) {
|
|
16281
|
+
const eventType = sample.crawler ? "crawler" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
16282
|
+
const refererHost = (() => {
|
|
16283
|
+
if (!sample.referer) return null;
|
|
16284
|
+
try {
|
|
16285
|
+
return new URL(sample.referer).hostname;
|
|
16286
|
+
} catch {
|
|
16287
|
+
return null;
|
|
16288
|
+
}
|
|
16289
|
+
})();
|
|
16290
|
+
tx.insert(rawEventSamples).values({
|
|
16291
|
+
id: crypto20.randomUUID(),
|
|
16292
|
+
projectId: project.id,
|
|
16293
|
+
sourceId: sourceRow.id,
|
|
16294
|
+
ts: sample.observedAt,
|
|
16295
|
+
eventType,
|
|
16296
|
+
ipHash: null,
|
|
16297
|
+
userAgent: sample.userAgent,
|
|
16298
|
+
pathNormalized: sample.pathNormalized,
|
|
16299
|
+
status: sample.status,
|
|
16300
|
+
refererHost,
|
|
16301
|
+
classifierDetailsJson: JSON.stringify({
|
|
16302
|
+
crawler: sample.crawler,
|
|
16303
|
+
aiReferral: sample.aiReferral
|
|
16304
|
+
}),
|
|
16305
|
+
createdAt: finishedAt
|
|
16306
|
+
}).run();
|
|
16307
|
+
sampleRows += 1;
|
|
16308
|
+
}
|
|
16309
|
+
tx.update(trafficSources).set({
|
|
16310
|
+
status: TrafficSourceStatuses.connected,
|
|
16311
|
+
lastSyncedAt: finishedAt,
|
|
16312
|
+
lastError: null,
|
|
16313
|
+
updatedAt: finishedAt
|
|
16314
|
+
}).where(eq23(trafficSources.id, sourceRow.id)).run();
|
|
16315
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq23(runs.id, runId)).run();
|
|
16316
|
+
});
|
|
16317
|
+
writeAuditLog(app.db, {
|
|
16318
|
+
projectId: project.id,
|
|
16319
|
+
actor: "api",
|
|
16320
|
+
action: "traffic.cloud-run.synced",
|
|
16321
|
+
entityType: "traffic_source",
|
|
16322
|
+
entityId: sourceRow.id
|
|
16323
|
+
});
|
|
16324
|
+
const response = {
|
|
16325
|
+
sourceId: sourceRow.id,
|
|
16326
|
+
runId,
|
|
16327
|
+
syncedAt: finishedAt,
|
|
16328
|
+
pulledEvents: report.totals.normalizedEvents,
|
|
16329
|
+
crawlerHits: report.totals.crawlerHits,
|
|
16330
|
+
aiReferralHits: report.totals.aiReferralHits,
|
|
16331
|
+
unknownHits: report.totals.unknownHits,
|
|
16332
|
+
crawlerBucketRows,
|
|
16333
|
+
aiReferralBucketRows,
|
|
16334
|
+
sampleRows,
|
|
16335
|
+
windowStart: windowStart.toISOString(),
|
|
16336
|
+
windowEnd: windowEnd.toISOString()
|
|
16337
|
+
};
|
|
16338
|
+
return response;
|
|
16339
|
+
});
|
|
16340
|
+
}
|
|
16341
|
+
|
|
15335
16342
|
// ../api-routes/src/doctor/checks/bing-auth.ts
|
|
15336
16343
|
var BING_AUTH_CHECKS = [
|
|
15337
16344
|
{
|
|
@@ -16183,6 +17190,11 @@ async function apiRoutes(app, opts) {
|
|
|
16183
17190
|
googleConnectionStore: opts.googleConnectionStore,
|
|
16184
17191
|
getGoogleAuthConfig: opts.getGoogleAuthConfig
|
|
16185
17192
|
});
|
|
17193
|
+
await api.register(trafficRoutes, {
|
|
17194
|
+
cloudRunCredentialStore: opts.cloudRunCredentialStore,
|
|
17195
|
+
pullCloudRunEvents: opts.pullCloudRunEvents,
|
|
17196
|
+
resolveCloudRunAccessToken: opts.resolveCloudRunAccessToken
|
|
17197
|
+
});
|
|
16186
17198
|
await api.register(backlinksRoutes, {
|
|
16187
17199
|
getBacklinksStatus: opts.getBacklinksStatus,
|
|
16188
17200
|
onInstallBacklinks: opts.onInstallBacklinks,
|
|
@@ -18608,8 +19620,40 @@ function removeGa4Connection(config, projectName) {
|
|
|
18608
19620
|
return true;
|
|
18609
19621
|
}
|
|
18610
19622
|
|
|
18611
|
-
// src/
|
|
19623
|
+
// src/cloud-run-config.ts
|
|
18612
19624
|
function ensureConnections3(config) {
|
|
19625
|
+
if (!config.cloudRun) config.cloudRun = {};
|
|
19626
|
+
if (!config.cloudRun.connections) config.cloudRun.connections = [];
|
|
19627
|
+
return config.cloudRun.connections;
|
|
19628
|
+
}
|
|
19629
|
+
function getCloudRunConnection(config, projectName) {
|
|
19630
|
+
return (config.cloudRun?.connections ?? []).find((c) => c.projectName === projectName);
|
|
19631
|
+
}
|
|
19632
|
+
function upsertCloudRunConnection(config, connection) {
|
|
19633
|
+
const connections = ensureConnections3(config);
|
|
19634
|
+
const index = connections.findIndex((c) => c.projectName === connection.projectName);
|
|
19635
|
+
if (index === -1) {
|
|
19636
|
+
connections.push(connection);
|
|
19637
|
+
return connection;
|
|
19638
|
+
}
|
|
19639
|
+
connections[index] = connection;
|
|
19640
|
+
return connection;
|
|
19641
|
+
}
|
|
19642
|
+
function removeCloudRunConnection(config, projectName) {
|
|
19643
|
+
const connections = config.cloudRun?.connections;
|
|
19644
|
+
if (!connections?.length) return false;
|
|
19645
|
+
const next = connections.filter((c) => c.projectName !== projectName);
|
|
19646
|
+
if (next.length === connections.length) return false;
|
|
19647
|
+
if (!config.cloudRun) return false;
|
|
19648
|
+
config.cloudRun.connections = next;
|
|
19649
|
+
if (next.length === 0) {
|
|
19650
|
+
delete config.cloudRun;
|
|
19651
|
+
}
|
|
19652
|
+
return true;
|
|
19653
|
+
}
|
|
19654
|
+
|
|
19655
|
+
// src/wordpress-config.ts
|
|
19656
|
+
function ensureConnections4(config) {
|
|
18613
19657
|
if (!config.wordpress) config.wordpress = {};
|
|
18614
19658
|
if (!config.wordpress.connections) config.wordpress.connections = [];
|
|
18615
19659
|
return config.wordpress.connections;
|
|
@@ -18626,7 +19670,7 @@ function getWordpressConnection(config, projectName) {
|
|
|
18626
19670
|
return (config.wordpress?.connections ?? []).find((connection) => connection.projectName === projectName);
|
|
18627
19671
|
}
|
|
18628
19672
|
function upsertWordpressConnection(config, connection) {
|
|
18629
|
-
const connections =
|
|
19673
|
+
const connections = ensureConnections4(config);
|
|
18630
19674
|
const normalized = normalizeConnection(connection);
|
|
18631
19675
|
const index = connections.findIndex((entry) => entry.projectName === connection.projectName);
|
|
18632
19676
|
if (index === -1) {
|
|
@@ -18660,11 +19704,11 @@ function removeWordpressConnection(config, projectName) {
|
|
|
18660
19704
|
}
|
|
18661
19705
|
|
|
18662
19706
|
// src/job-runner.ts
|
|
18663
|
-
import
|
|
19707
|
+
import crypto21 from "crypto";
|
|
18664
19708
|
import fs7 from "fs";
|
|
18665
19709
|
import path9 from "path";
|
|
18666
19710
|
import os4 from "os";
|
|
18667
|
-
import { and as and12, eq as
|
|
19711
|
+
import { and as and12, eq as eq24, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
18668
19712
|
|
|
18669
19713
|
// src/citation-utils.ts
|
|
18670
19714
|
function domainMatches(domain, canonicalDomain) {
|
|
@@ -18925,7 +19969,7 @@ var JobRunner = class {
|
|
|
18925
19969
|
if (stale.length === 0) return;
|
|
18926
19970
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
18927
19971
|
for (const run of stale) {
|
|
18928
|
-
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(
|
|
19972
|
+
this.db.update(runs).set({ status: "failed", finishedAt: now, error: "Server restarted while run was in progress" }).where(eq24(runs.id, run.id)).run();
|
|
18929
19973
|
log.warn("run.recovered-stale", { runId: run.id, previousStatus: run.status });
|
|
18930
19974
|
}
|
|
18931
19975
|
}
|
|
@@ -18953,10 +19997,10 @@ var JobRunner = class {
|
|
|
18953
19997
|
throw new Error(`Run ${runId} is not executable from status '${existingRun.status}'`);
|
|
18954
19998
|
}
|
|
18955
19999
|
if (existingRun.status === "queued") {
|
|
18956
|
-
this.db.update(runs).set({ status: "running", startedAt: now }).where(and12(
|
|
20000
|
+
this.db.update(runs).set({ status: "running", startedAt: now }).where(and12(eq24(runs.id, runId), eq24(runs.status, "queued"))).run();
|
|
18957
20001
|
}
|
|
18958
20002
|
this.throwIfRunCancelled(runId);
|
|
18959
|
-
const project = this.db.select().from(projects).where(
|
|
20003
|
+
const project = this.db.select().from(projects).where(eq24(projects.id, projectId)).get();
|
|
18960
20004
|
if (!project) {
|
|
18961
20005
|
throw new Error(`Project ${projectId} not found`);
|
|
18962
20006
|
}
|
|
@@ -18976,8 +20020,8 @@ var JobRunner = class {
|
|
|
18976
20020
|
throw new Error("No providers configured. Add at least one provider API key.");
|
|
18977
20021
|
}
|
|
18978
20022
|
log.info("run.dispatch", { runId, providerCount: activeProviders.length, providers: activeProviders.map((p) => p.adapter.name) });
|
|
18979
|
-
projectQueries = this.db.select().from(queries).where(
|
|
18980
|
-
const projectCompetitors = this.db.select().from(competitors).where(
|
|
20023
|
+
projectQueries = this.db.select().from(queries).where(eq24(queries.projectId, projectId)).all();
|
|
20024
|
+
const projectCompetitors = this.db.select().from(competitors).where(eq24(competitors.projectId, projectId)).all();
|
|
18981
20025
|
const competitorDomains = projectCompetitors.map((c) => c.domain);
|
|
18982
20026
|
const allDomains = effectiveDomains({
|
|
18983
20027
|
canonicalDomain: project.canonicalDomain,
|
|
@@ -18993,7 +20037,7 @@ var JobRunner = class {
|
|
|
18993
20037
|
const todayPeriod = getCurrentUsageDay();
|
|
18994
20038
|
for (const p of activeProviders) {
|
|
18995
20039
|
const providerScope = `${projectId}:${p.adapter.name}`;
|
|
18996
|
-
const providerUsage = this.db.select().from(usageCounters).where(
|
|
20040
|
+
const providerUsage = this.db.select().from(usageCounters).where(eq24(usageCounters.scope, providerScope)).all().filter((r) => r.period === todayPeriod && r.metric === "queries").reduce((sum, r) => sum + r.count, 0);
|
|
18997
20041
|
const limit = p.config.quotaPolicy.maxRequestsPerDay;
|
|
18998
20042
|
if (providerUsage + queriesPerProvider > limit) {
|
|
18999
20043
|
throw new Error(
|
|
@@ -19053,7 +20097,7 @@ var JobRunner = class {
|
|
|
19053
20097
|
);
|
|
19054
20098
|
let screenshotRelPath = null;
|
|
19055
20099
|
if (raw.screenshotPath && fs7.existsSync(raw.screenshotPath)) {
|
|
19056
|
-
const snapshotId =
|
|
20100
|
+
const snapshotId = crypto21.randomUUID();
|
|
19057
20101
|
const screenshotDir = path9.join(os4.homedir(), ".canonry", "screenshots", runId);
|
|
19058
20102
|
if (!fs7.existsSync(screenshotDir)) fs7.mkdirSync(screenshotDir, { recursive: true });
|
|
19059
20103
|
const destPath = path9.join(screenshotDir, `${snapshotId}.png`);
|
|
@@ -19083,7 +20127,7 @@ var JobRunner = class {
|
|
|
19083
20127
|
}).run();
|
|
19084
20128
|
} else {
|
|
19085
20129
|
this.db.insert(querySnapshots).values({
|
|
19086
|
-
id:
|
|
20130
|
+
id: crypto21.randomUUID(),
|
|
19087
20131
|
runId,
|
|
19088
20132
|
queryId: q.id,
|
|
19089
20133
|
provider: providerName,
|
|
@@ -19134,12 +20178,12 @@ var JobRunner = class {
|
|
|
19134
20178
|
const someFailed = providerErrors.size > 0;
|
|
19135
20179
|
if (allFailed) {
|
|
19136
20180
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
19137
|
-
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
20181
|
+
this.db.update(runs).set({ status: "failed", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq24(runs.id, runId)).run();
|
|
19138
20182
|
} else if (someFailed) {
|
|
19139
20183
|
const errorDetail = serializeRunError(buildRunErrorFromMessages(providerErrors));
|
|
19140
|
-
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(
|
|
20184
|
+
this.db.update(runs).set({ status: "partial", finishedAt: (/* @__PURE__ */ new Date()).toISOString(), error: errorDetail }).where(eq24(runs.id, runId)).run();
|
|
19141
20185
|
} else {
|
|
19142
|
-
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20186
|
+
this.db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq24(runs.id, runId)).run();
|
|
19143
20187
|
}
|
|
19144
20188
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
19145
20189
|
const finalStatus = allFailed ? "failed" : someFailed ? "partial" : "completed";
|
|
@@ -19174,7 +20218,7 @@ var JobRunner = class {
|
|
|
19174
20218
|
status: "failed",
|
|
19175
20219
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19176
20220
|
error: errorMessage
|
|
19177
|
-
}).where(
|
|
20221
|
+
}).where(eq24(runs.id, runId)).run();
|
|
19178
20222
|
this.flushProviderUsage(projectId, providerDispatchCounts);
|
|
19179
20223
|
trackEvent("run.completed", {
|
|
19180
20224
|
status: "failed",
|
|
@@ -19195,7 +20239,7 @@ var JobRunner = class {
|
|
|
19195
20239
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19196
20240
|
const period = now.slice(0, 10);
|
|
19197
20241
|
this.db.insert(usageCounters).values({
|
|
19198
|
-
id:
|
|
20242
|
+
id: crypto21.randomUUID(),
|
|
19199
20243
|
scope,
|
|
19200
20244
|
period,
|
|
19201
20245
|
metric,
|
|
@@ -19203,7 +20247,7 @@ var JobRunner = class {
|
|
|
19203
20247
|
updatedAt: now
|
|
19204
20248
|
}).onConflictDoUpdate({
|
|
19205
20249
|
target: [usageCounters.scope, usageCounters.period, usageCounters.metric],
|
|
19206
|
-
set: { count:
|
|
20250
|
+
set: { count: sql8`${usageCounters.count} + ${count}`, updatedAt: now }
|
|
19207
20251
|
}).run();
|
|
19208
20252
|
}
|
|
19209
20253
|
flushProviderUsage(projectId, providerDispatchCounts) {
|
|
@@ -19217,7 +20261,7 @@ var JobRunner = class {
|
|
|
19217
20261
|
status: runs.status,
|
|
19218
20262
|
finishedAt: runs.finishedAt,
|
|
19219
20263
|
error: runs.error
|
|
19220
|
-
}).from(runs).where(
|
|
20264
|
+
}).from(runs).where(eq24(runs.id, runId)).get();
|
|
19221
20265
|
}
|
|
19222
20266
|
isRunCancelled(runId) {
|
|
19223
20267
|
return this.getRunState(runId)?.status === "cancelled";
|
|
@@ -19233,7 +20277,7 @@ var JobRunner = class {
|
|
|
19233
20277
|
this.db.update(runs).set({
|
|
19234
20278
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19235
20279
|
error: currentRun.error ?? "Cancelled by user"
|
|
19236
|
-
}).where(
|
|
20280
|
+
}).where(eq24(runs.id, runId)).run();
|
|
19237
20281
|
}
|
|
19238
20282
|
trackEvent("run.completed", {
|
|
19239
20283
|
status: "cancelled",
|
|
@@ -19255,8 +20299,8 @@ function getCurrentUsageDay() {
|
|
|
19255
20299
|
}
|
|
19256
20300
|
|
|
19257
20301
|
// src/gsc-sync.ts
|
|
19258
|
-
import
|
|
19259
|
-
import { eq as
|
|
20302
|
+
import crypto22 from "crypto";
|
|
20303
|
+
import { eq as eq25, and as and13, sql as sql9 } from "drizzle-orm";
|
|
19260
20304
|
var log2 = createLogger("GscSync");
|
|
19261
20305
|
function formatDate3(d) {
|
|
19262
20306
|
return d.toISOString().split("T")[0];
|
|
@@ -19268,13 +20312,13 @@ function daysAgo(n) {
|
|
|
19268
20312
|
}
|
|
19269
20313
|
async function executeGscSync(db, runId, projectId, opts) {
|
|
19270
20314
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19271
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20315
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq25(runs.id, runId)).run();
|
|
19272
20316
|
try {
|
|
19273
20317
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
19274
20318
|
if (!googleClientId || !googleClientSecret) {
|
|
19275
20319
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
19276
20320
|
}
|
|
19277
|
-
const project = db.select().from(projects).where(
|
|
20321
|
+
const project = db.select().from(projects).where(eq25(projects.id, projectId)).get();
|
|
19278
20322
|
if (!project) {
|
|
19279
20323
|
throw new Error(`Project not found: ${projectId}`);
|
|
19280
20324
|
}
|
|
@@ -19309,9 +20353,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19309
20353
|
log2.info("fetch.complete", { runId, projectId, rowCount: rows.length });
|
|
19310
20354
|
db.delete(gscSearchData).where(
|
|
19311
20355
|
and13(
|
|
19312
|
-
|
|
19313
|
-
|
|
19314
|
-
|
|
20356
|
+
eq25(gscSearchData.projectId, projectId),
|
|
20357
|
+
sql9`${gscSearchData.date} >= ${startDate}`,
|
|
20358
|
+
sql9`${gscSearchData.date} <= ${endDate}`
|
|
19315
20359
|
)
|
|
19316
20360
|
).run();
|
|
19317
20361
|
const batchSize = 500;
|
|
@@ -19321,7 +20365,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19321
20365
|
for (const row of batch) {
|
|
19322
20366
|
const [query, page, country, device, date] = row.keys;
|
|
19323
20367
|
db.insert(gscSearchData).values({
|
|
19324
|
-
id:
|
|
20368
|
+
id: crypto22.randomUUID(),
|
|
19325
20369
|
projectId,
|
|
19326
20370
|
syncRunId: runId,
|
|
19327
20371
|
date: date ?? "",
|
|
@@ -19355,7 +20399,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19355
20399
|
const rich = ir.richResultsResult;
|
|
19356
20400
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19357
20401
|
db.insert(gscUrlInspections).values({
|
|
19358
|
-
id:
|
|
20402
|
+
id: crypto22.randomUUID(),
|
|
19359
20403
|
projectId,
|
|
19360
20404
|
syncRunId: runId,
|
|
19361
20405
|
url: pageUrl,
|
|
@@ -19376,7 +20420,7 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19376
20420
|
log2.error("inspect.url-failed", { runId, projectId, url: pageUrl, error: err instanceof Error ? err.message : String(err) });
|
|
19377
20421
|
}
|
|
19378
20422
|
}
|
|
19379
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
20423
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq25(gscUrlInspections.projectId, projectId)).all();
|
|
19380
20424
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19381
20425
|
for (const row of allInspections) {
|
|
19382
20426
|
const existing = latestByUrl.get(row.url);
|
|
@@ -19397,9 +20441,9 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19397
20441
|
}
|
|
19398
20442
|
}
|
|
19399
20443
|
const snapshotDate = formatDate3(/* @__PURE__ */ new Date());
|
|
19400
|
-
db.delete(gscCoverageSnapshots).where(and13(
|
|
20444
|
+
db.delete(gscCoverageSnapshots).where(and13(eq25(gscCoverageSnapshots.projectId, projectId), eq25(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
19401
20445
|
db.insert(gscCoverageSnapshots).values({
|
|
19402
|
-
id:
|
|
20446
|
+
id: crypto22.randomUUID(),
|
|
19403
20447
|
projectId,
|
|
19404
20448
|
syncRunId: runId,
|
|
19405
20449
|
date: snapshotDate,
|
|
@@ -19408,19 +20452,19 @@ async function executeGscSync(db, runId, projectId, opts) {
|
|
|
19408
20452
|
reasonBreakdown: JSON.stringify(reasonCounts),
|
|
19409
20453
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19410
20454
|
}).run();
|
|
19411
|
-
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20455
|
+
db.update(runs).set({ status: "completed", finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
19412
20456
|
log2.info("sync.completed", { runId, projectId, searchDataRows: rows.length, urlInspections: topPages.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
19413
20457
|
} catch (err) {
|
|
19414
20458
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19415
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20459
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq25(runs.id, runId)).run();
|
|
19416
20460
|
log2.error("sync.failed", { runId, projectId, error: errorMsg });
|
|
19417
20461
|
throw err;
|
|
19418
20462
|
}
|
|
19419
20463
|
}
|
|
19420
20464
|
|
|
19421
20465
|
// src/gsc-inspect-sitemap.ts
|
|
19422
|
-
import
|
|
19423
|
-
import { eq as
|
|
20466
|
+
import crypto23 from "crypto";
|
|
20467
|
+
import { eq as eq26, and as and14 } from "drizzle-orm";
|
|
19424
20468
|
|
|
19425
20469
|
// src/sitemap-parser.ts
|
|
19426
20470
|
var log3 = createLogger("SitemapParser");
|
|
@@ -19541,13 +20585,13 @@ async function parseSitemapRecursive(url, urls, visited, depth, isChild) {
|
|
|
19541
20585
|
var log4 = createLogger("InspectSitemap");
|
|
19542
20586
|
async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
19543
20587
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
19544
|
-
db.update(runs).set({ status: "running", startedAt: now }).where(
|
|
20588
|
+
db.update(runs).set({ status: "running", startedAt: now }).where(eq26(runs.id, runId)).run();
|
|
19545
20589
|
try {
|
|
19546
20590
|
const { clientId: googleClientId, clientSecret: googleClientSecret } = getGoogleAuthConfig(opts.config);
|
|
19547
20591
|
if (!googleClientId || !googleClientSecret) {
|
|
19548
20592
|
throw new Error("Google OAuth is not configured in the local Canonry config");
|
|
19549
20593
|
}
|
|
19550
|
-
const project = db.select().from(projects).where(
|
|
20594
|
+
const project = db.select().from(projects).where(eq26(projects.id, projectId)).get();
|
|
19551
20595
|
if (!project) {
|
|
19552
20596
|
throw new Error(`Project not found: ${projectId}`);
|
|
19553
20597
|
}
|
|
@@ -19588,7 +20632,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19588
20632
|
const rich = ir.richResultsResult;
|
|
19589
20633
|
const inspectedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19590
20634
|
db.insert(gscUrlInspections).values({
|
|
19591
|
-
id:
|
|
20635
|
+
id: crypto23.randomUUID(),
|
|
19592
20636
|
projectId,
|
|
19593
20637
|
syncRunId: runId,
|
|
19594
20638
|
url: pageUrl,
|
|
@@ -19615,7 +20659,7 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19615
20659
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
19616
20660
|
}
|
|
19617
20661
|
}
|
|
19618
|
-
const allInspections = db.select().from(gscUrlInspections).where(
|
|
20662
|
+
const allInspections = db.select().from(gscUrlInspections).where(eq26(gscUrlInspections.projectId, projectId)).all();
|
|
19619
20663
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19620
20664
|
for (const row of allInspections) {
|
|
19621
20665
|
const existing = latestByUrl.get(row.url);
|
|
@@ -19636,9 +20680,9 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19636
20680
|
}
|
|
19637
20681
|
}
|
|
19638
20682
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
19639
|
-
db.delete(gscCoverageSnapshots).where(and14(
|
|
20683
|
+
db.delete(gscCoverageSnapshots).where(and14(eq26(gscCoverageSnapshots.projectId, projectId), eq26(gscCoverageSnapshots.date, snapshotDate))).run();
|
|
19640
20684
|
db.insert(gscCoverageSnapshots).values({
|
|
19641
|
-
id:
|
|
20685
|
+
id: crypto23.randomUUID(),
|
|
19642
20686
|
projectId,
|
|
19643
20687
|
syncRunId: runId,
|
|
19644
20688
|
date: snapshotDate,
|
|
@@ -19648,19 +20692,19 @@ async function executeInspectSitemap(db, runId, projectId, opts) {
|
|
|
19648
20692
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
19649
20693
|
}).run();
|
|
19650
20694
|
const status = errors > 0 && inspected > 0 ? "partial" : errors === urls.length ? "failed" : "completed";
|
|
19651
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20695
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
19652
20696
|
log4.info("inspect.completed", { runId, projectId, inspected, errors, total: urls.length, indexed: snapIndexed, notIndexed: snapNotIndexed });
|
|
19653
20697
|
} catch (err) {
|
|
19654
20698
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19655
|
-
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20699
|
+
db.update(runs).set({ status: "failed", error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq26(runs.id, runId)).run();
|
|
19656
20700
|
log4.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
19657
20701
|
throw err;
|
|
19658
20702
|
}
|
|
19659
20703
|
}
|
|
19660
20704
|
|
|
19661
20705
|
// src/bing-inspect-sitemap.ts
|
|
19662
|
-
import
|
|
19663
|
-
import { eq as
|
|
20706
|
+
import crypto24 from "crypto";
|
|
20707
|
+
import { eq as eq27, desc as desc12 } from "drizzle-orm";
|
|
19664
20708
|
var log5 = createLogger("BingInspectSitemap");
|
|
19665
20709
|
function parseBingDate2(value) {
|
|
19666
20710
|
if (!value) return null;
|
|
@@ -19678,9 +20722,9 @@ function isBlockingIssueType2(issueType) {
|
|
|
19678
20722
|
}
|
|
19679
20723
|
async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
19680
20724
|
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
19681
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
20725
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq27(runs.id, runId)).run();
|
|
19682
20726
|
try {
|
|
19683
|
-
const project = db.select().from(projects).where(
|
|
20727
|
+
const project = db.select().from(projects).where(eq27(projects.id, projectId)).get();
|
|
19684
20728
|
if (!project) {
|
|
19685
20729
|
throw new Error(`Project not found: ${projectId}`);
|
|
19686
20730
|
}
|
|
@@ -19698,7 +20742,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19698
20742
|
if (sitemapUrls.length === 0) {
|
|
19699
20743
|
throw new Error("No URLs found in sitemap");
|
|
19700
20744
|
}
|
|
19701
|
-
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(
|
|
20745
|
+
const trackedRows = db.select({ url: bingUrlInspections.url }).from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).all();
|
|
19702
20746
|
const trackedUrls = new Set(trackedRows.map((r) => r.url));
|
|
19703
20747
|
const discovered = sitemapUrls.filter((u) => !trackedUrls.has(u));
|
|
19704
20748
|
log5.info("sitemap.diff", {
|
|
@@ -19747,7 +20791,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19747
20791
|
derivedInIndex = false;
|
|
19748
20792
|
}
|
|
19749
20793
|
db.insert(bingUrlInspections).values({
|
|
19750
|
-
id:
|
|
20794
|
+
id: crypto24.randomUUID(),
|
|
19751
20795
|
projectId,
|
|
19752
20796
|
url: pageUrl,
|
|
19753
20797
|
httpCode,
|
|
@@ -19781,7 +20825,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19781
20825
|
await new Promise((r) => setTimeout(r, 1e3));
|
|
19782
20826
|
}
|
|
19783
20827
|
}
|
|
19784
|
-
const allInspections = db.select().from(bingUrlInspections).where(
|
|
20828
|
+
const allInspections = db.select().from(bingUrlInspections).where(eq27(bingUrlInspections.projectId, projectId)).orderBy(desc12(bingUrlInspections.inspectedAt)).all();
|
|
19785
20829
|
const latestByUrl = /* @__PURE__ */ new Map();
|
|
19786
20830
|
const definitiveByUrl = /* @__PURE__ */ new Map();
|
|
19787
20831
|
for (const row of allInspections) {
|
|
@@ -19805,7 +20849,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19805
20849
|
const snapshotDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
19806
20850
|
const snapNow = (/* @__PURE__ */ new Date()).toISOString();
|
|
19807
20851
|
db.insert(bingCoverageSnapshots).values({
|
|
19808
|
-
id:
|
|
20852
|
+
id: crypto24.randomUUID(),
|
|
19809
20853
|
projectId,
|
|
19810
20854
|
syncRunId: runId,
|
|
19811
20855
|
date: snapshotDate,
|
|
@@ -19824,7 +20868,7 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19824
20868
|
}
|
|
19825
20869
|
}).run();
|
|
19826
20870
|
const status = errors === sitemapUrls.length ? RunStatuses.failed : errors > 0 ? RunStatuses.partial : RunStatuses.completed;
|
|
19827
|
-
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20871
|
+
db.update(runs).set({ status, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
19828
20872
|
log5.info("inspect.completed", {
|
|
19829
20873
|
runId,
|
|
19830
20874
|
projectId,
|
|
@@ -19838,16 +20882,16 @@ async function executeBingInspectSitemap(db, runId, projectId, opts) {
|
|
|
19838
20882
|
});
|
|
19839
20883
|
} catch (err) {
|
|
19840
20884
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
19841
|
-
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
20885
|
+
db.update(runs).set({ status: RunStatuses.failed, error: errorMsg, finishedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq27(runs.id, runId)).run();
|
|
19842
20886
|
log5.error("inspect.failed", { runId, projectId, error: errorMsg });
|
|
19843
20887
|
throw err;
|
|
19844
20888
|
}
|
|
19845
20889
|
}
|
|
19846
20890
|
|
|
19847
20891
|
// src/commoncrawl-sync.ts
|
|
19848
|
-
import
|
|
20892
|
+
import crypto25 from "crypto";
|
|
19849
20893
|
import path10 from "path";
|
|
19850
|
-
import { and as and15, eq as
|
|
20894
|
+
import { and as and15, eq as eq28, sql as sql10 } from "drizzle-orm";
|
|
19851
20895
|
var log6 = createLogger("CommonCrawlSync");
|
|
19852
20896
|
var INSERT_CHUNK_SIZE = 1e4;
|
|
19853
20897
|
function defaultDeps() {
|
|
@@ -19873,7 +20917,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19873
20917
|
phaseDetail: "downloading vertices + edges",
|
|
19874
20918
|
updatedAt: downloadStartedAt,
|
|
19875
20919
|
error: null
|
|
19876
|
-
}).where(
|
|
20920
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19877
20921
|
const paths = ccReleasePaths(release);
|
|
19878
20922
|
const releaseCacheDir = path10.join(deps.cacheDir, release);
|
|
19879
20923
|
const vertexPath = path10.join(releaseCacheDir, paths.vertexFilename);
|
|
@@ -19896,7 +20940,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19896
20940
|
vertexSha256: vertex.sha256,
|
|
19897
20941
|
edgesSha256: edges.sha256,
|
|
19898
20942
|
updatedAt: downloadFinishedAt
|
|
19899
|
-
}).where(
|
|
20943
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19900
20944
|
const allProjects = db.select().from(projects).all();
|
|
19901
20945
|
const targets = Array.from(new Set(allProjects.map((p) => p.canonicalDomain)));
|
|
19902
20946
|
let rows = [];
|
|
@@ -19912,15 +20956,15 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19912
20956
|
}
|
|
19913
20957
|
const queriedAt = deps.now().toISOString();
|
|
19914
20958
|
db.transaction((tx) => {
|
|
19915
|
-
tx.delete(backlinkDomains).where(
|
|
19916
|
-
tx.delete(backlinkSummaries).where(
|
|
20959
|
+
tx.delete(backlinkDomains).where(eq28(backlinkDomains.releaseSyncId, syncId)).run();
|
|
20960
|
+
tx.delete(backlinkSummaries).where(eq28(backlinkSummaries.releaseSyncId, syncId)).run();
|
|
19917
20961
|
const expanded = [];
|
|
19918
20962
|
for (const r of rows) {
|
|
19919
20963
|
const projectIds = projectsByDomain.get(r.targetDomain);
|
|
19920
20964
|
if (!projectIds) continue;
|
|
19921
20965
|
for (const projectId of projectIds) {
|
|
19922
20966
|
expanded.push({
|
|
19923
|
-
id:
|
|
20967
|
+
id: crypto25.randomUUID(),
|
|
19924
20968
|
projectId,
|
|
19925
20969
|
releaseSyncId: syncId,
|
|
19926
20970
|
release,
|
|
@@ -19940,7 +20984,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19940
20984
|
const projectRows = rowsByProject.get(p.id) ?? [];
|
|
19941
20985
|
const summary = computeSummary(projectRows);
|
|
19942
20986
|
tx.insert(backlinkSummaries).values({
|
|
19943
|
-
id:
|
|
20987
|
+
id: crypto25.randomUUID(),
|
|
19944
20988
|
projectId: p.id,
|
|
19945
20989
|
releaseSyncId: syncId,
|
|
19946
20990
|
release,
|
|
@@ -19972,7 +21016,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
19972
21016
|
domainsDiscovered: rows.length,
|
|
19973
21017
|
updatedAt: finishedAt,
|
|
19974
21018
|
error: null
|
|
19975
|
-
}).where(
|
|
21019
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
19976
21020
|
log6.info("sync.completed", {
|
|
19977
21021
|
syncId,
|
|
19978
21022
|
release,
|
|
@@ -20002,7 +21046,7 @@ async function executeReleaseSync(db, syncId, opts) {
|
|
|
20002
21046
|
error: errorMsg,
|
|
20003
21047
|
phaseDetail: null,
|
|
20004
21048
|
updatedAt: finishedAt
|
|
20005
|
-
}).where(
|
|
21049
|
+
}).where(eq28(ccReleaseSyncs.id, syncId)).run();
|
|
20006
21050
|
log6.error("sync.failed", { syncId, release, error: errorMsg });
|
|
20007
21051
|
throw err;
|
|
20008
21052
|
}
|
|
@@ -20036,9 +21080,9 @@ function computeSummary(rows) {
|
|
|
20036
21080
|
}
|
|
20037
21081
|
|
|
20038
21082
|
// src/backlink-extract.ts
|
|
20039
|
-
import
|
|
21083
|
+
import crypto26 from "crypto";
|
|
20040
21084
|
import fs8 from "fs";
|
|
20041
|
-
import { and as and16, desc as desc13, eq as
|
|
21085
|
+
import { and as and16, desc as desc13, eq as eq29 } from "drizzle-orm";
|
|
20042
21086
|
var log7 = createLogger("BacklinkExtract");
|
|
20043
21087
|
function defaultDeps2() {
|
|
20044
21088
|
return {
|
|
@@ -20050,13 +21094,13 @@ function defaultDeps2() {
|
|
|
20050
21094
|
async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
20051
21095
|
const deps = { ...defaultDeps2(), ...opts.deps };
|
|
20052
21096
|
const startedAt = deps.now().toISOString();
|
|
20053
|
-
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(
|
|
21097
|
+
db.update(runs).set({ status: RunStatuses.running, startedAt }).where(eq29(runs.id, runId)).run();
|
|
20054
21098
|
try {
|
|
20055
|
-
const project = db.select().from(projects).where(
|
|
21099
|
+
const project = db.select().from(projects).where(eq29(projects.id, projectId)).get();
|
|
20056
21100
|
if (!project) {
|
|
20057
21101
|
throw new Error(`Project not found: ${projectId}`);
|
|
20058
21102
|
}
|
|
20059
|
-
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(
|
|
21103
|
+
const sync = opts.release ? db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.release, opts.release)).get() : db.select().from(ccReleaseSyncs).where(eq29(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).orderBy(desc13(ccReleaseSyncs.createdAt)).limit(1).get();
|
|
20060
21104
|
if (!sync) {
|
|
20061
21105
|
throw new Error("No ready release sync available \u2014 run `canonry backlinks sync` first");
|
|
20062
21106
|
}
|
|
@@ -20084,11 +21128,11 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20084
21128
|
const targetDomain = project.canonicalDomain;
|
|
20085
21129
|
db.transaction((tx) => {
|
|
20086
21130
|
tx.delete(backlinkDomains).where(
|
|
20087
|
-
and16(
|
|
21131
|
+
and16(eq29(backlinkDomains.projectId, projectId), eq29(backlinkDomains.release, release))
|
|
20088
21132
|
).run();
|
|
20089
21133
|
if (rows.length > 0) {
|
|
20090
21134
|
const values = rows.map((r) => ({
|
|
20091
|
-
id:
|
|
21135
|
+
id: crypto26.randomUUID(),
|
|
20092
21136
|
projectId,
|
|
20093
21137
|
releaseSyncId: syncId,
|
|
20094
21138
|
release,
|
|
@@ -20101,7 +21145,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20101
21145
|
}
|
|
20102
21146
|
const summary = computeSummary2(rows);
|
|
20103
21147
|
tx.insert(backlinkSummaries).values({
|
|
20104
|
-
id:
|
|
21148
|
+
id: crypto26.randomUUID(),
|
|
20105
21149
|
projectId,
|
|
20106
21150
|
releaseSyncId: syncId,
|
|
20107
21151
|
release,
|
|
@@ -20124,7 +21168,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20124
21168
|
}).run();
|
|
20125
21169
|
});
|
|
20126
21170
|
const finishedAt = deps.now().toISOString();
|
|
20127
|
-
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
21171
|
+
db.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq29(runs.id, runId)).run();
|
|
20128
21172
|
log7.info("extract.completed", { runId, projectId, release, rows: rows.length });
|
|
20129
21173
|
} catch (err) {
|
|
20130
21174
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -20133,7 +21177,7 @@ async function executeBacklinkExtract(db, runId, projectId, opts = {}) {
|
|
|
20133
21177
|
status: RunStatuses.failed,
|
|
20134
21178
|
error: errorMsg,
|
|
20135
21179
|
finishedAt
|
|
20136
|
-
}).where(
|
|
21180
|
+
}).where(eq29(runs.id, runId)).run();
|
|
20137
21181
|
log7.error("extract.failed", { runId, projectId, error: errorMsg });
|
|
20138
21182
|
throw err;
|
|
20139
21183
|
}
|
|
@@ -20206,7 +21250,7 @@ var ProviderRegistry = class {
|
|
|
20206
21250
|
|
|
20207
21251
|
// src/scheduler.ts
|
|
20208
21252
|
import cron from "node-cron";
|
|
20209
|
-
import { eq as
|
|
21253
|
+
import { eq as eq30 } from "drizzle-orm";
|
|
20210
21254
|
var log8 = createLogger("Scheduler");
|
|
20211
21255
|
var Scheduler = class {
|
|
20212
21256
|
db;
|
|
@@ -20218,7 +21262,7 @@ var Scheduler = class {
|
|
|
20218
21262
|
}
|
|
20219
21263
|
/** Load all enabled schedules from DB and register cron jobs. */
|
|
20220
21264
|
start() {
|
|
20221
|
-
const allSchedules = this.db.select().from(schedules).where(
|
|
21265
|
+
const allSchedules = this.db.select().from(schedules).where(eq30(schedules.enabled, 1)).all();
|
|
20222
21266
|
for (const schedule of allSchedules) {
|
|
20223
21267
|
const missedRunAt = schedule.nextRunAt;
|
|
20224
21268
|
this.registerCronTask(schedule);
|
|
@@ -20243,7 +21287,7 @@ var Scheduler = class {
|
|
|
20243
21287
|
this.stopTask(projectId, existing, "Stopped");
|
|
20244
21288
|
this.tasks.delete(projectId);
|
|
20245
21289
|
}
|
|
20246
|
-
const schedule = this.db.select().from(schedules).where(
|
|
21290
|
+
const schedule = this.db.select().from(schedules).where(eq30(schedules.projectId, projectId)).get();
|
|
20247
21291
|
if (schedule && schedule.enabled === 1) {
|
|
20248
21292
|
this.registerCronTask(schedule);
|
|
20249
21293
|
}
|
|
@@ -20276,14 +21320,14 @@ var Scheduler = class {
|
|
|
20276
21320
|
this.db.update(schedules).set({
|
|
20277
21321
|
nextRunAt: task.getNextRun()?.toISOString() ?? null,
|
|
20278
21322
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
20279
|
-
}).where(
|
|
21323
|
+
}).where(eq30(schedules.id, scheduleId)).run();
|
|
20280
21324
|
const label = schedule.preset ?? cronExpr;
|
|
20281
21325
|
log8.info("cron.registered", { projectId, schedule: label, timezone });
|
|
20282
21326
|
}
|
|
20283
21327
|
triggerRun(scheduleId, projectId) {
|
|
20284
21328
|
try {
|
|
20285
21329
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20286
|
-
const currentSchedule = this.db.select().from(schedules).where(
|
|
21330
|
+
const currentSchedule = this.db.select().from(schedules).where(eq30(schedules.id, scheduleId)).get();
|
|
20287
21331
|
if (!currentSchedule || currentSchedule.enabled !== 1) {
|
|
20288
21332
|
log8.warn("schedule.stale", { scheduleId, projectId, msg: "schedule no longer exists or is disabled" });
|
|
20289
21333
|
this.remove(projectId);
|
|
@@ -20291,7 +21335,7 @@ var Scheduler = class {
|
|
|
20291
21335
|
}
|
|
20292
21336
|
const task = this.tasks.get(projectId);
|
|
20293
21337
|
const nextRunAt = task?.getNextRun()?.toISOString() ?? null;
|
|
20294
|
-
const project = this.db.select().from(projects).where(
|
|
21338
|
+
const project = this.db.select().from(projects).where(eq30(projects.id, projectId)).get();
|
|
20295
21339
|
if (!project) {
|
|
20296
21340
|
log8.error("project.not-found", { projectId, msg: "skipping scheduled run" });
|
|
20297
21341
|
this.remove(projectId);
|
|
@@ -20320,7 +21364,7 @@ var Scheduler = class {
|
|
|
20320
21364
|
this.db.update(schedules).set({
|
|
20321
21365
|
nextRunAt,
|
|
20322
21366
|
updatedAt: now
|
|
20323
|
-
}).where(
|
|
21367
|
+
}).where(eq30(schedules.id, currentSchedule.id)).run();
|
|
20324
21368
|
return;
|
|
20325
21369
|
}
|
|
20326
21370
|
const runId = queueResult.runId;
|
|
@@ -20328,7 +21372,7 @@ var Scheduler = class {
|
|
|
20328
21372
|
lastRunAt: now,
|
|
20329
21373
|
nextRunAt,
|
|
20330
21374
|
updatedAt: now
|
|
20331
|
-
}).where(
|
|
21375
|
+
}).where(eq30(schedules.id, currentSchedule.id)).run();
|
|
20332
21376
|
const scheduleProviders = parseJsonColumn(currentSchedule.providers, []);
|
|
20333
21377
|
const providers = scheduleProviders.length > 0 ? scheduleProviders : void 0;
|
|
20334
21378
|
log8.info("run.triggered", { runId, projectName: project.name, providers: providers ?? "all" });
|
|
@@ -20340,8 +21384,8 @@ var Scheduler = class {
|
|
|
20340
21384
|
};
|
|
20341
21385
|
|
|
20342
21386
|
// src/notifier.ts
|
|
20343
|
-
import { eq as
|
|
20344
|
-
import
|
|
21387
|
+
import { eq as eq31, desc as desc14, and as and17, or as or4 } from "drizzle-orm";
|
|
21388
|
+
import crypto27 from "crypto";
|
|
20345
21389
|
var log9 = createLogger("Notifier");
|
|
20346
21390
|
var Notifier = class {
|
|
20347
21391
|
db;
|
|
@@ -20353,18 +21397,18 @@ var Notifier = class {
|
|
|
20353
21397
|
/** Called after a run completes (success, partial, or failed). */
|
|
20354
21398
|
async onRunCompleted(runId, projectId) {
|
|
20355
21399
|
log9.info("run.completed", { runId, projectId });
|
|
20356
|
-
const notifs = this.db.select().from(notifications).where(
|
|
21400
|
+
const notifs = this.db.select().from(notifications).where(eq31(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
20357
21401
|
if (notifs.length === 0) {
|
|
20358
21402
|
log9.info("notifications.none-enabled", { projectId });
|
|
20359
21403
|
return;
|
|
20360
21404
|
}
|
|
20361
21405
|
log9.info("notifications.found", { projectId, count: notifs.length });
|
|
20362
|
-
const run = this.db.select().from(runs).where(
|
|
21406
|
+
const run = this.db.select().from(runs).where(eq31(runs.id, runId)).get();
|
|
20363
21407
|
if (!run) {
|
|
20364
21408
|
log9.error("run.not-found", { runId, msg: "skipping notification dispatch" });
|
|
20365
21409
|
return;
|
|
20366
21410
|
}
|
|
20367
|
-
const project = this.db.select().from(projects).where(
|
|
21411
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
20368
21412
|
if (!project) {
|
|
20369
21413
|
log9.error("project.not-found", { projectId, msg: "skipping notification dispatch" });
|
|
20370
21414
|
return;
|
|
@@ -20411,11 +21455,11 @@ var Notifier = class {
|
|
|
20411
21455
|
if (criticalInsights.length > 0) insightEvents.push("insight.critical");
|
|
20412
21456
|
if (highInsights.length > 0) insightEvents.push("insight.high");
|
|
20413
21457
|
if (insightEvents.length === 0) return;
|
|
20414
|
-
const notifs = this.db.select().from(notifications).where(
|
|
21458
|
+
const notifs = this.db.select().from(notifications).where(eq31(notifications.projectId, projectId)).all().filter((n) => n.enabled === 1);
|
|
20415
21459
|
if (notifs.length === 0) return;
|
|
20416
|
-
const run = this.db.select().from(runs).where(
|
|
21460
|
+
const run = this.db.select().from(runs).where(eq31(runs.id, runId)).get();
|
|
20417
21461
|
if (!run) return;
|
|
20418
|
-
const project = this.db.select().from(projects).where(
|
|
21462
|
+
const project = this.db.select().from(projects).where(eq31(projects.id, projectId)).get();
|
|
20419
21463
|
if (!project) return;
|
|
20420
21464
|
for (const notif of notifs) {
|
|
20421
21465
|
const config = parseJsonColumn(notif.config, { url: "", events: [] });
|
|
@@ -20447,8 +21491,8 @@ var Notifier = class {
|
|
|
20447
21491
|
computeTransitions(runId, projectId) {
|
|
20448
21492
|
const recentRuns = this.db.select().from(runs).where(
|
|
20449
21493
|
and17(
|
|
20450
|
-
|
|
20451
|
-
or4(
|
|
21494
|
+
eq31(runs.projectId, projectId),
|
|
21495
|
+
or4(eq31(runs.status, "completed"), eq31(runs.status, "partial"))
|
|
20452
21496
|
)
|
|
20453
21497
|
).orderBy(desc14(runs.createdAt)).limit(2).all();
|
|
20454
21498
|
if (recentRuns.length < 2) return [];
|
|
@@ -20460,12 +21504,12 @@ var Notifier = class {
|
|
|
20460
21504
|
query: queries.query,
|
|
20461
21505
|
provider: querySnapshots.provider,
|
|
20462
21506
|
citationState: querySnapshots.citationState
|
|
20463
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
21507
|
+
}).from(querySnapshots).leftJoin(queries, eq31(querySnapshots.queryId, queries.id)).where(eq31(querySnapshots.runId, currentRunId)).all();
|
|
20464
21508
|
const previousSnapshots = this.db.select({
|
|
20465
21509
|
queryId: querySnapshots.queryId,
|
|
20466
21510
|
provider: querySnapshots.provider,
|
|
20467
21511
|
citationState: querySnapshots.citationState
|
|
20468
|
-
}).from(querySnapshots).where(
|
|
21512
|
+
}).from(querySnapshots).where(eq31(querySnapshots.runId, previousRunId)).all();
|
|
20469
21513
|
const prevMap = /* @__PURE__ */ new Map();
|
|
20470
21514
|
for (const s of previousSnapshots) {
|
|
20471
21515
|
prevMap.set(`${s.queryId}:${s.provider}`, s.citationState);
|
|
@@ -20523,7 +21567,7 @@ var Notifier = class {
|
|
|
20523
21567
|
}
|
|
20524
21568
|
logDelivery(projectId, notificationId, event, status, error) {
|
|
20525
21569
|
this.db.insert(auditLog).values({
|
|
20526
|
-
id:
|
|
21570
|
+
id: crypto27.randomUUID(),
|
|
20527
21571
|
projectId,
|
|
20528
21572
|
actor: "scheduler",
|
|
20529
21573
|
action: `notification.${status}`,
|
|
@@ -20581,8 +21625,8 @@ var RunCoordinator = class {
|
|
|
20581
21625
|
};
|
|
20582
21626
|
|
|
20583
21627
|
// src/agent/session-registry.ts
|
|
20584
|
-
import
|
|
20585
|
-
import { eq as
|
|
21628
|
+
import crypto29 from "crypto";
|
|
21629
|
+
import { eq as eq33 } from "drizzle-orm";
|
|
20586
21630
|
|
|
20587
21631
|
// src/agent/session.ts
|
|
20588
21632
|
import fs11 from "fs";
|
|
@@ -20931,11 +21975,11 @@ function resolveSessionProviderAndModel(config, opts) {
|
|
|
20931
21975
|
}
|
|
20932
21976
|
|
|
20933
21977
|
// src/agent/memory-store.ts
|
|
20934
|
-
import
|
|
20935
|
-
import { and as and18, desc as desc15, eq as
|
|
21978
|
+
import crypto28 from "crypto";
|
|
21979
|
+
import { and as and18, desc as desc15, eq as eq32, like as like2, sql as sql11 } from "drizzle-orm";
|
|
20936
21980
|
var COMPACTION_KEY_PREFIX = "compaction:";
|
|
20937
21981
|
var COMPACTION_NOTES_PER_SESSION = 3;
|
|
20938
|
-
function
|
|
21982
|
+
function rowToDto2(row) {
|
|
20939
21983
|
return {
|
|
20940
21984
|
id: row.id,
|
|
20941
21985
|
key: row.key,
|
|
@@ -20946,9 +21990,9 @@ function rowToDto(row) {
|
|
|
20946
21990
|
};
|
|
20947
21991
|
}
|
|
20948
21992
|
function listMemoryEntries(db, projectId, opts = {}) {
|
|
20949
|
-
const query = db.select().from(agentMemory).where(
|
|
21993
|
+
const query = db.select().from(agentMemory).where(eq32(agentMemory.projectId, projectId)).orderBy(desc15(agentMemory.updatedAt));
|
|
20950
21994
|
const rows = opts.limit === void 0 ? query.all() : query.limit(opts.limit).all();
|
|
20951
|
-
return rows.map(
|
|
21995
|
+
return rows.map(rowToDto2);
|
|
20952
21996
|
}
|
|
20953
21997
|
function upsertMemoryEntry(db, args) {
|
|
20954
21998
|
if (Buffer.byteLength(args.value, "utf8") > AGENT_MEMORY_VALUE_MAX_BYTES) {
|
|
@@ -20960,7 +22004,7 @@ function upsertMemoryEntry(db, args) {
|
|
|
20960
22004
|
throw new Error(`memory key prefix "${COMPACTION_KEY_PREFIX}" is reserved for compaction notes`);
|
|
20961
22005
|
}
|
|
20962
22006
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20963
|
-
const id =
|
|
22007
|
+
const id = crypto28.randomUUID();
|
|
20964
22008
|
db.insert(agentMemory).values({
|
|
20965
22009
|
id,
|
|
20966
22010
|
projectId: args.projectId,
|
|
@@ -20977,12 +22021,12 @@ function upsertMemoryEntry(db, args) {
|
|
|
20977
22021
|
updatedAt: now
|
|
20978
22022
|
}
|
|
20979
22023
|
}).run();
|
|
20980
|
-
const row = db.select().from(agentMemory).where(and18(
|
|
22024
|
+
const row = db.select().from(agentMemory).where(and18(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, args.key))).get();
|
|
20981
22025
|
if (!row) throw new Error("memory upsert produced no row");
|
|
20982
|
-
return
|
|
22026
|
+
return rowToDto2(row);
|
|
20983
22027
|
}
|
|
20984
22028
|
function deleteMemoryEntry(db, projectId, key) {
|
|
20985
|
-
const result = db.delete(agentMemory).where(and18(
|
|
22029
|
+
const result = db.delete(agentMemory).where(and18(eq32(agentMemory.projectId, projectId), eq32(agentMemory.key, key))).run();
|
|
20986
22030
|
const changes = result.changes ?? 0;
|
|
20987
22031
|
return changes > 0;
|
|
20988
22032
|
}
|
|
@@ -20997,7 +22041,7 @@ function writeCompactionNote(db, args) {
|
|
|
20997
22041
|
}
|
|
20998
22042
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
20999
22043
|
const key = `${COMPACTION_KEY_PREFIX}${args.sessionId}:${now}`;
|
|
21000
|
-
const id =
|
|
22044
|
+
const id = crypto28.randomUUID();
|
|
21001
22045
|
let inserted;
|
|
21002
22046
|
db.transaction((tx) => {
|
|
21003
22047
|
tx.insert(agentMemory).values({
|
|
@@ -21012,16 +22056,16 @@ function writeCompactionNote(db, args) {
|
|
|
21012
22056
|
const sessionPrefix = `${COMPACTION_KEY_PREFIX}${args.sessionId}:`;
|
|
21013
22057
|
const existing = tx.select({ id: agentMemory.id, updatedAt: agentMemory.updatedAt }).from(agentMemory).where(
|
|
21014
22058
|
and18(
|
|
21015
|
-
|
|
22059
|
+
eq32(agentMemory.projectId, args.projectId),
|
|
21016
22060
|
like2(agentMemory.key, `${sessionPrefix}%`)
|
|
21017
22061
|
)
|
|
21018
22062
|
).orderBy(desc15(agentMemory.updatedAt)).all();
|
|
21019
22063
|
const stale = existing.slice(COMPACTION_NOTES_PER_SESSION).map((r) => r.id);
|
|
21020
22064
|
if (stale.length > 0) {
|
|
21021
|
-
tx.delete(agentMemory).where(
|
|
22065
|
+
tx.delete(agentMemory).where(sql11`${agentMemory.id} IN (${sql11.join(stale.map((s) => sql11`${s}`), sql11`, `)})`).run();
|
|
21022
22066
|
}
|
|
21023
|
-
const row = tx.select().from(agentMemory).where(and18(
|
|
21024
|
-
if (row) inserted =
|
|
22067
|
+
const row = tx.select().from(agentMemory).where(and18(eq32(agentMemory.projectId, args.projectId), eq32(agentMemory.key, key))).get();
|
|
22068
|
+
if (row) inserted = rowToDto2(row);
|
|
21025
22069
|
});
|
|
21026
22070
|
if (!inserted) throw new Error("compaction note write produced no row");
|
|
21027
22071
|
return inserted;
|
|
@@ -21202,7 +22246,7 @@ var SessionRegistry = class {
|
|
|
21202
22246
|
modelProvider: effectiveProvider,
|
|
21203
22247
|
modelId: effectiveModelId,
|
|
21204
22248
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21205
|
-
}).where(
|
|
22249
|
+
}).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21206
22250
|
}
|
|
21207
22251
|
const agent2 = createAeroSession({
|
|
21208
22252
|
projectName,
|
|
@@ -21416,7 +22460,7 @@ ${lines.join("\n")}
|
|
|
21416
22460
|
modelProvider: nextProvider,
|
|
21417
22461
|
modelId: nextModelId,
|
|
21418
22462
|
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21419
|
-
}).where(
|
|
22463
|
+
}).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21420
22464
|
}
|
|
21421
22465
|
/** Persist a session's transcript back to the DB. Call after any run settles. */
|
|
21422
22466
|
save(projectName) {
|
|
@@ -21578,17 +22622,17 @@ ${lines.join("\n")}
|
|
|
21578
22622
|
return id;
|
|
21579
22623
|
}
|
|
21580
22624
|
tryResolveProjectId(projectName) {
|
|
21581
|
-
const row = this.opts.db.select({ id: projects.id }).from(projects).where(
|
|
22625
|
+
const row = this.opts.db.select({ id: projects.id }).from(projects).where(eq33(projects.name, projectName)).get();
|
|
21582
22626
|
return row?.id;
|
|
21583
22627
|
}
|
|
21584
22628
|
loadRow(projectId) {
|
|
21585
|
-
const row = this.opts.db.select().from(agentSessions).where(
|
|
22629
|
+
const row = this.opts.db.select().from(agentSessions).where(eq33(agentSessions.projectId, projectId)).get();
|
|
21586
22630
|
return row ?? null;
|
|
21587
22631
|
}
|
|
21588
22632
|
insertRow(params) {
|
|
21589
22633
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21590
22634
|
this.opts.db.insert(agentSessions).values({
|
|
21591
|
-
id:
|
|
22635
|
+
id: crypto29.randomUUID(),
|
|
21592
22636
|
projectId: params.projectId,
|
|
21593
22637
|
systemPrompt: params.systemPrompt,
|
|
21594
22638
|
modelProvider: params.provider ?? params.modelProvider ?? AgentProviderIds.claude,
|
|
@@ -21601,14 +22645,14 @@ ${lines.join("\n")}
|
|
|
21601
22645
|
}
|
|
21602
22646
|
updateRow(projectId, patch) {
|
|
21603
22647
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
21604
|
-
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(
|
|
22648
|
+
this.opts.db.update(agentSessions).set({ ...patch, updatedAt: now }).where(eq33(agentSessions.projectId, projectId)).run();
|
|
21605
22649
|
}
|
|
21606
22650
|
};
|
|
21607
22651
|
|
|
21608
22652
|
// src/agent/agent-routes.ts
|
|
21609
|
-
import { eq as
|
|
22653
|
+
import { eq as eq34 } from "drizzle-orm";
|
|
21610
22654
|
function resolveProject2(db, name) {
|
|
21611
|
-
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(
|
|
22655
|
+
const row = db.select({ id: projects.id, name: projects.name }).from(projects).where(eq34(projects.name, name)).get();
|
|
21612
22656
|
if (!row) throw notFound("project", name);
|
|
21613
22657
|
return row;
|
|
21614
22658
|
}
|
|
@@ -21617,7 +22661,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
21617
22661
|
"/projects/:name/agent/transcript",
|
|
21618
22662
|
async (request) => {
|
|
21619
22663
|
const project = resolveProject2(opts.db, request.params.name);
|
|
21620
|
-
const row = opts.db.select().from(agentSessions).where(
|
|
22664
|
+
const row = opts.db.select().from(agentSessions).where(eq34(agentSessions.projectId, project.id)).get();
|
|
21621
22665
|
if (!row) {
|
|
21622
22666
|
return { messages: [], modelProvider: null, modelId: null, updatedAt: null };
|
|
21623
22667
|
}
|
|
@@ -21641,7 +22685,7 @@ function registerAgentRoutes(app, opts) {
|
|
|
21641
22685
|
async (request) => {
|
|
21642
22686
|
const project = resolveProject2(opts.db, request.params.name);
|
|
21643
22687
|
opts.sessionRegistry.reset(project.name);
|
|
21644
|
-
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
22688
|
+
opts.db.update(agentSessions).set({ messages: "[]", followUpQueue: "[]", updatedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq34(agentSessions.projectId, project.id)).run();
|
|
21645
22689
|
return { status: "reset" };
|
|
21646
22690
|
}
|
|
21647
22691
|
);
|
|
@@ -22505,7 +23549,7 @@ function summarizeProviderConfig(provider, config) {
|
|
|
22505
23549
|
};
|
|
22506
23550
|
}
|
|
22507
23551
|
function hashApiKey(key) {
|
|
22508
|
-
return
|
|
23552
|
+
return crypto30.createHash("sha256").update(key).digest("hex");
|
|
22509
23553
|
}
|
|
22510
23554
|
function parseCookies2(header) {
|
|
22511
23555
|
if (!header) return {};
|
|
@@ -22663,7 +23707,7 @@ async function createServer(opts) {
|
|
|
22663
23707
|
intelligenceService,
|
|
22664
23708
|
(runId, projectId, result) => notifier.dispatchInsightWebhooks(runId, projectId, result),
|
|
22665
23709
|
async ({ runId, projectId, insightCount, criticalOrHigh }) => {
|
|
22666
|
-
const project = opts.db.select({ name: projects.name }).from(projects).where(
|
|
23710
|
+
const project = opts.db.select({ name: projects.name }).from(projects).where(eq35(projects.id, projectId)).get();
|
|
22667
23711
|
if (!project) return;
|
|
22668
23712
|
sessionRegistry.queueFollowUp(project.name, {
|
|
22669
23713
|
role: "user",
|
|
@@ -22757,7 +23801,22 @@ async function createServer(opts) {
|
|
|
22757
23801
|
return removed;
|
|
22758
23802
|
}
|
|
22759
23803
|
};
|
|
22760
|
-
const
|
|
23804
|
+
const cloudRunCredentialStore = {
|
|
23805
|
+
getConnection: (projectName) => {
|
|
23806
|
+
return getCloudRunConnection(opts.config, projectName);
|
|
23807
|
+
},
|
|
23808
|
+
upsertConnection: (record) => {
|
|
23809
|
+
const updated = upsertCloudRunConnection(opts.config, record);
|
|
23810
|
+
saveConfigPatch(opts.config);
|
|
23811
|
+
return updated;
|
|
23812
|
+
},
|
|
23813
|
+
deleteConnection: (projectName) => {
|
|
23814
|
+
const removed = removeCloudRunConnection(opts.config, projectName);
|
|
23815
|
+
if (removed) saveConfigPatch(opts.config);
|
|
23816
|
+
return removed;
|
|
23817
|
+
}
|
|
23818
|
+
};
|
|
23819
|
+
const googleStateSecret = process.env.GOOGLE_STATE_SECRET ?? crypto30.randomBytes(32).toString("hex");
|
|
22761
23820
|
const googleConnectionStore = {
|
|
22762
23821
|
listConnections: (domain) => listGoogleConnections(opts.config, domain),
|
|
22763
23822
|
getConnection: (domain, connectionType) => getGoogleConnection(opts.config, domain, connectionType),
|
|
@@ -22803,11 +23862,11 @@ async function createServer(opts) {
|
|
|
22803
23862
|
const apiPrefix = basePath ? `${basePath}api/v1` : "/api/v1";
|
|
22804
23863
|
if (opts.config.apiKey) {
|
|
22805
23864
|
const keyHash = hashApiKey(opts.config.apiKey);
|
|
22806
|
-
const existing = opts.db.select().from(apiKeys).where(
|
|
23865
|
+
const existing = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, keyHash)).get();
|
|
22807
23866
|
if (!existing) {
|
|
22808
23867
|
const prefix = opts.config.apiKey.slice(0, 12);
|
|
22809
23868
|
opts.db.insert(apiKeys).values({
|
|
22810
|
-
id: `key_${
|
|
23869
|
+
id: `key_${crypto30.randomBytes(8).toString("hex")}`,
|
|
22811
23870
|
name: "default",
|
|
22812
23871
|
keyHash,
|
|
22813
23872
|
keyPrefix: prefix,
|
|
@@ -22831,7 +23890,7 @@ async function createServer(opts) {
|
|
|
22831
23890
|
};
|
|
22832
23891
|
const createSession = (apiKeyId) => {
|
|
22833
23892
|
pruneExpiredSessions();
|
|
22834
|
-
const sessionId =
|
|
23893
|
+
const sessionId = crypto30.randomBytes(32).toString("hex");
|
|
22835
23894
|
sessions.set(sessionId, {
|
|
22836
23895
|
apiKeyId,
|
|
22837
23896
|
expiresAt: Date.now() + SESSION_TTL_MS
|
|
@@ -22855,7 +23914,7 @@ async function createServer(opts) {
|
|
|
22855
23914
|
};
|
|
22856
23915
|
const getDefaultApiKey = () => {
|
|
22857
23916
|
if (!opts.config.apiKey) return void 0;
|
|
22858
|
-
return opts.db.select().from(apiKeys).where(
|
|
23917
|
+
return opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(opts.config.apiKey))).get();
|
|
22859
23918
|
};
|
|
22860
23919
|
const createPasswordSession = (reply) => {
|
|
22861
23920
|
const key = getDefaultApiKey();
|
|
@@ -22912,12 +23971,12 @@ async function createServer(opts) {
|
|
|
22912
23971
|
return reply.send({ authenticated: true });
|
|
22913
23972
|
}
|
|
22914
23973
|
if (apiKey) {
|
|
22915
|
-
const key = opts.db.select().from(apiKeys).where(
|
|
23974
|
+
const key = opts.db.select().from(apiKeys).where(eq35(apiKeys.keyHash, hashApiKey(apiKey))).get();
|
|
22916
23975
|
if (!key || key.revokedAt) {
|
|
22917
23976
|
const err2 = authInvalid();
|
|
22918
23977
|
return reply.status(err2.statusCode).send(err2.toJSON());
|
|
22919
23978
|
}
|
|
22920
|
-
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(
|
|
23979
|
+
opts.db.update(apiKeys).set({ lastUsedAt: (/* @__PURE__ */ new Date()).toISOString() }).where(eq35(apiKeys.id, key.id)).run();
|
|
22921
23980
|
const sessionId = createSession(key.id);
|
|
22922
23981
|
reply.header("set-cookie", serializeSessionCookie({
|
|
22923
23982
|
name: SESSION_COOKIE_NAME,
|
|
@@ -23027,7 +24086,7 @@ async function createServer(opts) {
|
|
|
23027
24086
|
deps: {
|
|
23028
24087
|
enqueueAutoExtract: ({ projectId, release: r }) => {
|
|
23029
24088
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
23030
|
-
const runId =
|
|
24089
|
+
const runId = crypto30.randomUUID();
|
|
23031
24090
|
opts.db.insert(runs).values({
|
|
23032
24091
|
id: runId,
|
|
23033
24092
|
projectId,
|
|
@@ -23100,6 +24159,7 @@ async function createServer(opts) {
|
|
|
23100
24159
|
},
|
|
23101
24160
|
wordpressConnectionStore,
|
|
23102
24161
|
ga4CredentialStore,
|
|
24162
|
+
cloudRunCredentialStore,
|
|
23103
24163
|
onRunCreated: (runId, projectId, providers2, location) => {
|
|
23104
24164
|
jobRunner.executeRun(runId, projectId, providers2, location).catch((err) => {
|
|
23105
24165
|
app.log.error({ runId, err }, "Job runner failed");
|
|
@@ -23162,7 +24222,7 @@ async function createServer(opts) {
|
|
|
23162
24222
|
const targetProjectIds = affectedProjectIds.length > 0 ? affectedProjectIds : [null];
|
|
23163
24223
|
const createdAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23164
24224
|
opts.db.insert(auditLog).values(targetProjectIds.map((projectId) => ({
|
|
23165
|
-
id:
|
|
24225
|
+
id: crypto30.randomUUID(),
|
|
23166
24226
|
projectId,
|
|
23167
24227
|
actor: "api",
|
|
23168
24228
|
action: existing ? "provider.updated" : "provider.created",
|