@ainyc/canonry 4.193.2 → 4.195.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +81 -1
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +41 -0
- package/assets/assets/{AuditHistoryPanel-BBXiKuD1.js → AuditHistoryPanel-vWUtVXOI.js} +1 -1
- package/assets/assets/{BacklinksPage-CE_-DKWO.js → BacklinksPage-CSzuMEuv.js} +1 -1
- package/assets/assets/{HistoryPage-D-l4jMt5.js → HistoryPage-DtCTQ7CQ.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-fCJm3sW6.js → MeasurementPropertyPage-B7gTMWNs.js} +1 -1
- package/assets/assets/{ProjectPage-Cxrd8lZM.js → ProjectPage-Bsm0_jUY.js} +1 -1
- package/assets/assets/{RunRow-C2kmB04V.js → RunRow-D5k4Rdo4.js} +1 -1
- package/assets/assets/{RunsPage-B73hjh_l.js → RunsPage-DIXBB2Om.js} +1 -1
- package/assets/assets/{SettingsPage-Ba30Jfnn.js → SettingsPage-wYhBm6Az.js} +1 -1
- package/assets/assets/{SiteHealthSection-mjpxy6N9.js → SiteHealthSection-YebzsgDh.js} +3 -3
- package/assets/assets/{TrafficPage-BZ8H_c8u.js → TrafficPage-DtnSRFO8.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-sCUyzPpH.js → TrafficSourceDetailPage-DYQ6mpYq.js} +1 -1
- package/assets/assets/{extract-error-message-G-MvSIBA.js → extract-error-message-BdChrkmo.js} +1 -1
- package/assets/assets/{index-CSktlkUv.js → index-C_E2NRYY.js} +26 -26
- package/assets/assets/{react-sigma_core.esm.min-CgN2Dnvk.js → react-sigma_core.esm.min-sKWUk_tG.js} +1 -1
- package/assets/assets/{v2-overview-adapter-OmeP2l8O.js → v2-overview-adapter-CkPt-sRK.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-5CXUV3ZX.js → chunk-3WODA4JF.js} +288 -7
- package/dist/{chunk-24JHDZ7H.js → chunk-B2XM6AQ6.js} +318 -33
- package/dist/{chunk-KJJMIWVD.js → chunk-H3RVNMOO.js} +91 -38
- package/dist/{chunk-LIMWYFHS.js → chunk-LJKAB4CX.js} +2 -2
- package/dist/{chunk-Q5KGSOT7.js → chunk-UMU3IBXM.js} +1231 -484
- package/dist/cli.js +252 -156
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-6ORSU2M6.js → intelligence-service-MGQRJW4V.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +10 -10
|
@@ -280,6 +280,7 @@ import {
|
|
|
280
280
|
deriveWinnabilityClass,
|
|
281
281
|
describeError,
|
|
282
282
|
determineAnswerMentioned,
|
|
283
|
+
diagnosticIdentity,
|
|
283
284
|
discoveryBucketSchema,
|
|
284
285
|
discoveryHarvestDtoSchema,
|
|
285
286
|
discoveryPromotePreviewSchema,
|
|
@@ -382,6 +383,7 @@ import {
|
|
|
382
383
|
hostOf,
|
|
383
384
|
inclusiveDayCount,
|
|
384
385
|
indexingRequestResponseDtoSchema,
|
|
386
|
+
integrationSettingsSummaryDtoSchema,
|
|
385
387
|
internalError,
|
|
386
388
|
intersectScopes,
|
|
387
389
|
isBrowserProvider,
|
|
@@ -397,6 +399,7 @@ import {
|
|
|
397
399
|
latestProjectRunDtoSchema,
|
|
398
400
|
linearTrend,
|
|
399
401
|
locationContextSchema,
|
|
402
|
+
logQuerySchema,
|
|
400
403
|
loginRequestSchema,
|
|
401
404
|
mapWithConcurrency,
|
|
402
405
|
matchedAliasKeys,
|
|
@@ -512,6 +515,7 @@ import {
|
|
|
512
515
|
normalizeProjectName,
|
|
513
516
|
normalizeQueryText,
|
|
514
517
|
normalizeSiteAuditRunRequest,
|
|
518
|
+
normalizeTelemetryStatus,
|
|
515
519
|
normalizeUrlPath,
|
|
516
520
|
normalizeUserName,
|
|
517
521
|
notFound,
|
|
@@ -520,6 +524,8 @@ import {
|
|
|
520
524
|
observeTemplateLinkEdges,
|
|
521
525
|
onboardingTelemetryEventSchema,
|
|
522
526
|
operationInProgress,
|
|
527
|
+
operationalLogEntryDtoSchema,
|
|
528
|
+
operationalLogListDtoSchema,
|
|
523
529
|
orderLocationsDefaultFirst,
|
|
524
530
|
organicEvidenceDtoSchema,
|
|
525
531
|
organicEvidencePeriodSchema,
|
|
@@ -542,6 +548,7 @@ import {
|
|
|
542
548
|
projectUpsertRequestSchema,
|
|
543
549
|
providerAuthError,
|
|
544
550
|
providerError,
|
|
551
|
+
providerSummaryEntryDtoSchema,
|
|
545
552
|
queryDtoSchema,
|
|
546
553
|
queryGenerateRequestSchema,
|
|
547
554
|
queryTrackingCommitRequestSchema,
|
|
@@ -555,6 +562,8 @@ import {
|
|
|
555
562
|
recommendationBriefDtoSchema,
|
|
556
563
|
recommendationExplainRequestSchema,
|
|
557
564
|
recommendationExplanationDtoSchema,
|
|
565
|
+
redactLogString,
|
|
566
|
+
redactLogValue,
|
|
558
567
|
registrableDomain,
|
|
559
568
|
relativeChangeRatio,
|
|
560
569
|
reportActionCategoryLabel,
|
|
@@ -639,6 +648,7 @@ import {
|
|
|
639
648
|
snapshotDiffResponseSchema,
|
|
640
649
|
snapshotListResponseSchema,
|
|
641
650
|
snapshotReportSchema,
|
|
651
|
+
snapshotRequestInputSchema,
|
|
642
652
|
snapshotRequestSchema,
|
|
643
653
|
sourceBreakdownDtoSchema,
|
|
644
654
|
splitList,
|
|
@@ -647,6 +657,7 @@ import {
|
|
|
647
657
|
surfaceClassFromCompetitorType,
|
|
648
658
|
surfaceClassLabel,
|
|
649
659
|
telemetryEventAcceptedDtoSchema,
|
|
660
|
+
telemetryStatusDtoSchema,
|
|
650
661
|
templateLinkRatio,
|
|
651
662
|
templateLinkSource,
|
|
652
663
|
templateLinkUbiquityAvailable,
|
|
@@ -691,10 +702,10 @@ import {
|
|
|
691
702
|
wordpressSchemaDeployResultDtoSchema,
|
|
692
703
|
wordpressSchemaStatusResultDtoSchema,
|
|
693
704
|
wordpressStatusDtoSchema
|
|
694
|
-
} from "./chunk-
|
|
705
|
+
} from "./chunk-3WODA4JF.js";
|
|
695
706
|
|
|
696
707
|
// src/intelligence-service.ts
|
|
697
|
-
import { eq as eq67, desc as desc29, asc as asc12, and as and57, ne as ne10, or as
|
|
708
|
+
import { eq as eq67, desc as desc29, asc as asc12, and as and57, ne as ne10, or as or16, inArray as inArray25, gte as gte18, lte as lte15, isNull as isNull12, sql as sql27, exists } from "drizzle-orm";
|
|
698
709
|
|
|
699
710
|
// ../db/src/client.ts
|
|
700
711
|
import { mkdirSync } from "fs";
|
|
@@ -794,6 +805,8 @@ __export(schema_exports, {
|
|
|
794
805
|
researchRunQueries: () => researchRunQueries,
|
|
795
806
|
researchRuns: () => researchRuns,
|
|
796
807
|
runs: () => runs,
|
|
808
|
+
runtimeLogMetadata: () => runtimeLogMetadata,
|
|
809
|
+
runtimeLogs: () => runtimeLogs,
|
|
797
810
|
schedules: () => schedules,
|
|
798
811
|
simpleMeasurementDefinitions: () => simpleMeasurementDefinitions,
|
|
799
812
|
siteAuditPages: () => siteAuditPages,
|
|
@@ -1157,10 +1170,9 @@ var auditLog = sqliteTable("audit_log", {
|
|
|
1157
1170
|
// reader could see it (the deletion would erase the only evidence it
|
|
1158
1171
|
// happened). Detached rows surface in audit queries with project_id=NULL.
|
|
1159
1172
|
projectId: text("project_id").references(() => projects.id, { onDelete: "set null" }),
|
|
1160
|
-
//
|
|
1161
|
-
//
|
|
1162
|
-
//
|
|
1163
|
-
// `userAgent` and `actorSession`.
|
|
1173
|
+
// Authenticated user:<id> / api-key:<id> for HTTP callers; explicit business
|
|
1174
|
+
// actors (scheduler, system, etc.) remain unchanged. Client hints below
|
|
1175
|
+
// are untrusted correlation metadata, never authorization identities.
|
|
1164
1176
|
actor: text("actor").notNull(),
|
|
1165
1177
|
action: text("action").notNull(),
|
|
1166
1178
|
entityType: text("entity_type").notNull(),
|
|
@@ -1177,11 +1189,46 @@ var auditLog = sqliteTable("audit_log", {
|
|
|
1177
1189
|
// with its session id so post-mortems can group a related sequence of
|
|
1178
1190
|
// mutations. NULL when the caller didn't provide one.
|
|
1179
1191
|
actorSession: text("actor_session"),
|
|
1192
|
+
// Stable credential id, intentionally not an FK: audit evidence must remain
|
|
1193
|
+
// intelligible after a key is revoked or deleted.
|
|
1194
|
+
credentialId: text("credential_id"),
|
|
1195
|
+
// Server-issued request correlation id, separate from caller actorSession.
|
|
1196
|
+
requestId: text("request_id"),
|
|
1180
1197
|
createdAt: text("created_at").notNull()
|
|
1181
1198
|
}, (table) => [
|
|
1182
1199
|
index("idx_audit_log_project").on(table.projectId),
|
|
1183
|
-
index("idx_audit_log_created").on(table.createdAt)
|
|
1200
|
+
index("idx_audit_log_created").on(table.createdAt),
|
|
1201
|
+
index("idx_audit_log_credential_created").on(table.credentialId, table.createdAt),
|
|
1202
|
+
index("idx_audit_log_request_created").on(table.requestId, table.createdAt)
|
|
1184
1203
|
]);
|
|
1204
|
+
var runtimeLogs = sqliteTable("runtime_logs", {
|
|
1205
|
+
sequence: integer("sequence").primaryKey(),
|
|
1206
|
+
ts: text("ts").notNull(),
|
|
1207
|
+
level: text("level").notNull(),
|
|
1208
|
+
module: text("module").notNull(),
|
|
1209
|
+
action: text("action").notNull(),
|
|
1210
|
+
msg: text("msg"),
|
|
1211
|
+
projectId: text("project_id"),
|
|
1212
|
+
runId: text("run_id"),
|
|
1213
|
+
actor: text("actor"),
|
|
1214
|
+
requestId: text("request_id"),
|
|
1215
|
+
context: text("context", { mode: "json" }).$type().notNull().default({}),
|
|
1216
|
+
entryBytes: integer("entry_bytes").notNull()
|
|
1217
|
+
}, (table) => [
|
|
1218
|
+
index("idx_runtime_logs_project_ts").on(table.projectId, table.ts, table.sequence),
|
|
1219
|
+
index("idx_runtime_logs_run_ts").on(table.runId, table.ts, table.sequence),
|
|
1220
|
+
index("idx_runtime_logs_actor_ts").on(table.actor, table.ts, table.sequence),
|
|
1221
|
+
index("idx_runtime_logs_request_ts").on(table.requestId, table.ts, table.sequence),
|
|
1222
|
+
index("idx_runtime_logs_module_level_ts").on(table.module, table.level, table.ts, table.sequence),
|
|
1223
|
+
index("idx_runtime_logs_ts").on(table.ts, table.sequence)
|
|
1224
|
+
]);
|
|
1225
|
+
var runtimeLogMetadata = sqliteTable("runtime_log_metadata", {
|
|
1226
|
+
id: text("id").primaryKey(),
|
|
1227
|
+
cursorNamespace: text("cursor_namespace").notNull(),
|
|
1228
|
+
nextSequence: integer("next_sequence").notNull(),
|
|
1229
|
+
dropped: integer("dropped").notNull().default(0),
|
|
1230
|
+
captureErrors: integer("capture_errors").notNull().default(0)
|
|
1231
|
+
});
|
|
1185
1232
|
var apiKeys = sqliteTable("api_keys", {
|
|
1186
1233
|
id: text("id").primaryKey(),
|
|
1187
1234
|
name: text("name").notNull(),
|
|
@@ -7201,6 +7248,45 @@ var MIGRATION_VERSIONS = [
|
|
|
7201
7248
|
statements: [
|
|
7202
7249
|
`ALTER TABLE api_keys ADD COLUMN delegated_user_id TEXT REFERENCES users(id) ON DELETE CASCADE`
|
|
7203
7250
|
]
|
|
7251
|
+
},
|
|
7252
|
+
{
|
|
7253
|
+
version: 155,
|
|
7254
|
+
name: "durable-operational-logs",
|
|
7255
|
+
statements: [
|
|
7256
|
+
// Audit attribution is intentionally a plain nullable identity string:
|
|
7257
|
+
// deleting/revoking a credential must not delete its historical trail.
|
|
7258
|
+
`ALTER TABLE audit_log ADD COLUMN credential_id TEXT`,
|
|
7259
|
+
`ALTER TABLE audit_log ADD COLUMN request_id TEXT`,
|
|
7260
|
+
`CREATE INDEX IF NOT EXISTS idx_audit_log_credential_created ON audit_log(credential_id, created_at)`,
|
|
7261
|
+
`CREATE INDEX IF NOT EXISTS idx_audit_log_request_created ON audit_log(request_id, created_at)`,
|
|
7262
|
+
`CREATE TABLE IF NOT EXISTS runtime_logs (
|
|
7263
|
+
sequence INTEGER PRIMARY KEY,
|
|
7264
|
+
ts TEXT NOT NULL,
|
|
7265
|
+
level TEXT NOT NULL,
|
|
7266
|
+
module TEXT NOT NULL,
|
|
7267
|
+
action TEXT NOT NULL,
|
|
7268
|
+
msg TEXT,
|
|
7269
|
+
project_id TEXT,
|
|
7270
|
+
run_id TEXT,
|
|
7271
|
+
actor TEXT,
|
|
7272
|
+
request_id TEXT,
|
|
7273
|
+
context TEXT NOT NULL DEFAULT '{}',
|
|
7274
|
+
entry_bytes INTEGER NOT NULL
|
|
7275
|
+
)`,
|
|
7276
|
+
`CREATE TABLE IF NOT EXISTS runtime_log_metadata (
|
|
7277
|
+
id TEXT PRIMARY KEY,
|
|
7278
|
+
cursor_namespace TEXT NOT NULL,
|
|
7279
|
+
next_sequence INTEGER NOT NULL,
|
|
7280
|
+
dropped INTEGER NOT NULL DEFAULT 0,
|
|
7281
|
+
capture_errors INTEGER NOT NULL DEFAULT 0
|
|
7282
|
+
)`,
|
|
7283
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_project_ts ON runtime_logs(project_id, ts, sequence)`,
|
|
7284
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_run_ts ON runtime_logs(run_id, ts, sequence)`,
|
|
7285
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_actor_ts ON runtime_logs(actor, ts, sequence)`,
|
|
7286
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_request_ts ON runtime_logs(request_id, ts, sequence)`,
|
|
7287
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_module_level_ts ON runtime_logs(module, level, ts, sequence)`,
|
|
7288
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_ts ON runtime_logs(ts, sequence)`
|
|
7289
|
+
]
|
|
7204
7290
|
}
|
|
7205
7291
|
];
|
|
7206
7292
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -7475,6 +7561,293 @@ function migrate(db, versions = MIGRATION_VERSIONS) {
|
|
|
7475
7561
|
}
|
|
7476
7562
|
}
|
|
7477
7563
|
|
|
7564
|
+
// ../db/src/operational-logs.ts
|
|
7565
|
+
import { createHash, randomUUID } from "crypto";
|
|
7566
|
+
import { sql as sql3 } from "drizzle-orm";
|
|
7567
|
+
var METADATA_ID = "runtime-log-store";
|
|
7568
|
+
var DEFAULT_MAX_ENTRIES = 1e4;
|
|
7569
|
+
var MAX_MAX_ENTRIES = 1e5;
|
|
7570
|
+
var DEFAULT_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
7571
|
+
var MIN_MAX_AGE_MS = 1e3;
|
|
7572
|
+
var MAX_MAX_AGE_MS = 31 * 24 * 60 * 60 * 1e3;
|
|
7573
|
+
var MAX_ENTRY_BYTES = 8192;
|
|
7574
|
+
var OperationalLogStore = class {
|
|
7575
|
+
constructor(db, options = {}) {
|
|
7576
|
+
this.db = db;
|
|
7577
|
+
this.maxEntries = boundedInteger(options.maxEntries ?? DEFAULT_MAX_ENTRIES, 1, MAX_MAX_ENTRIES, "maxEntries");
|
|
7578
|
+
this.maxAgeMs = boundedInteger(options.maxAgeMs ?? DEFAULT_MAX_AGE_MS, MIN_MAX_AGE_MS, MAX_MAX_AGE_MS, "maxAgeMs");
|
|
7579
|
+
this.retention = options.retention ?? "durable";
|
|
7580
|
+
this.now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
7581
|
+
this.ensureMetadata();
|
|
7582
|
+
}
|
|
7583
|
+
db;
|
|
7584
|
+
maxEntries;
|
|
7585
|
+
maxAgeMs;
|
|
7586
|
+
retention;
|
|
7587
|
+
now;
|
|
7588
|
+
localCaptureErrors = 0;
|
|
7589
|
+
append(event) {
|
|
7590
|
+
try {
|
|
7591
|
+
const entry = projectEvent(event);
|
|
7592
|
+
if (Buffer.byteLength(JSON.stringify(entry), "utf8") > MAX_ENTRY_BYTES) {
|
|
7593
|
+
throw new RangeError(`Operational log entry exceeds ${MAX_ENTRY_BYTES} bytes.`);
|
|
7594
|
+
}
|
|
7595
|
+
this.withoutLockWait(() => this.db.transaction((tx) => {
|
|
7596
|
+
this.prune(tx);
|
|
7597
|
+
const metadata2 = readMetadata(tx);
|
|
7598
|
+
tx.run(sql3`
|
|
7599
|
+
INSERT INTO runtime_logs (
|
|
7600
|
+
sequence, ts, level, module, action, msg, project_id, run_id, actor, request_id, context, entry_bytes
|
|
7601
|
+
) VALUES (
|
|
7602
|
+
${metadata2.nextSequence}, ${entry.ts}, ${entry.level}, ${entry.module}, ${entry.action}, ${entry.message ?? null},
|
|
7603
|
+
${entry.projectId ?? null}, ${entry.runId ?? null}, ${entry.context.actor ?? null}, ${entry.context.requestId ?? null},
|
|
7604
|
+
${JSON.stringify(entry.context)}, ${Buffer.byteLength(JSON.stringify(entry), "utf8")}
|
|
7605
|
+
)
|
|
7606
|
+
`);
|
|
7607
|
+
tx.run(sql3`UPDATE runtime_log_metadata SET next_sequence = ${metadata2.nextSequence + 1} WHERE id = ${METADATA_ID}`);
|
|
7608
|
+
this.prune(tx);
|
|
7609
|
+
}));
|
|
7610
|
+
} catch {
|
|
7611
|
+
this.recordCaptureError();
|
|
7612
|
+
}
|
|
7613
|
+
}
|
|
7614
|
+
list(query) {
|
|
7615
|
+
const parsed = logQuerySchema.safeParse(query);
|
|
7616
|
+
if (!parsed.success) throw validationError("Invalid operational logs query.", { issues: parsed.error.issues });
|
|
7617
|
+
const filters = normalizeLogQuery(parsed.data);
|
|
7618
|
+
this.withoutLockWait(() => this.db.transaction((tx) => {
|
|
7619
|
+
this.prune(tx);
|
|
7620
|
+
}));
|
|
7621
|
+
const metadata2 = readMetadata(this.db);
|
|
7622
|
+
const after = filters.cursor === void 0 ? 0 : this.parseCursor(filters.cursor, metadata2, filters);
|
|
7623
|
+
if (filters.cursor !== void 0) this.assertCursorCurrent(after);
|
|
7624
|
+
const where = buildWhere(filters, after);
|
|
7625
|
+
const rows = this.db.all(sql3`
|
|
7626
|
+
SELECT sequence, ts, level, module, action, msg,
|
|
7627
|
+
project_id AS projectId, run_id AS runId, actor, request_id AS requestId, context
|
|
7628
|
+
FROM runtime_logs
|
|
7629
|
+
WHERE ${sql3.join(where, sql3` AND `)}
|
|
7630
|
+
ORDER BY sequence ASC
|
|
7631
|
+
LIMIT ${filters.limit}
|
|
7632
|
+
`);
|
|
7633
|
+
const countRows = this.db.all(sql3`
|
|
7634
|
+
SELECT COUNT(*) AS count FROM runtime_logs WHERE ${sql3.join(where, sql3` AND `)}
|
|
7635
|
+
`);
|
|
7636
|
+
const matching = countRows[0]?.count;
|
|
7637
|
+
if (!Number.isSafeInteger(matching) || matching < rows.length) throw new Error("Operational log storage returned an invalid count.");
|
|
7638
|
+
const truncated = matching - rows.length;
|
|
7639
|
+
return {
|
|
7640
|
+
entries: rows.map((row) => this.toDto(row, metadata2.cursorNamespace, filters)),
|
|
7641
|
+
nextCursor: truncated > 0 ? this.cursorFor(metadata2.cursorNamespace, rows.at(-1).sequence, filters) : null,
|
|
7642
|
+
truncated,
|
|
7643
|
+
dropped: metadata2.dropped,
|
|
7644
|
+
retention: this.retention,
|
|
7645
|
+
retentionPolicy: { maxEntries: this.maxEntries, maxAgeSeconds: Math.ceil(this.maxAgeMs / 1e3) },
|
|
7646
|
+
captureErrors: metadata2.captureErrors + this.localCaptureErrors,
|
|
7647
|
+
observedAt: this.now().toISOString()
|
|
7648
|
+
};
|
|
7649
|
+
}
|
|
7650
|
+
ensureMetadata() {
|
|
7651
|
+
const existing = this.db.all(sql3`SELECT id FROM runtime_log_metadata WHERE id = ${METADATA_ID}`);
|
|
7652
|
+
if (existing.length > 0) return;
|
|
7653
|
+
this.db.run(sql3`
|
|
7654
|
+
INSERT OR IGNORE INTO runtime_log_metadata (id, cursor_namespace, next_sequence, dropped, capture_errors)
|
|
7655
|
+
VALUES (${METADATA_ID}, ${randomUUID()}, 1, 0, 0)
|
|
7656
|
+
`);
|
|
7657
|
+
}
|
|
7658
|
+
prune(db) {
|
|
7659
|
+
const expiry = new Date(this.now().getTime() - this.maxAgeMs).toISOString();
|
|
7660
|
+
db.run(sql3`DELETE FROM runtime_logs WHERE ts < ${expiry}`);
|
|
7661
|
+
db.run(sql3`UPDATE runtime_log_metadata SET dropped = dropped + changes() WHERE id = ${METADATA_ID}`);
|
|
7662
|
+
const metadata2 = readMetadata(db);
|
|
7663
|
+
const firstRetainedSequence = Math.max(1, metadata2.nextSequence - this.maxEntries);
|
|
7664
|
+
db.run(sql3`DELETE FROM runtime_logs WHERE sequence < ${firstRetainedSequence}`);
|
|
7665
|
+
db.run(sql3`UPDATE runtime_log_metadata SET dropped = dropped + changes() WHERE id = ${METADATA_ID}`);
|
|
7666
|
+
}
|
|
7667
|
+
recordCaptureError() {
|
|
7668
|
+
try {
|
|
7669
|
+
this.withoutLockWait(() => this.db.run(sql3`UPDATE runtime_log_metadata SET capture_errors = capture_errors + 1 WHERE id = ${METADATA_ID}`));
|
|
7670
|
+
} catch {
|
|
7671
|
+
this.localCaptureErrors++;
|
|
7672
|
+
}
|
|
7673
|
+
}
|
|
7674
|
+
/**
|
|
7675
|
+
* Diagnostics must not spend the application's busy timeout waiting for a
|
|
7676
|
+
* writer. All work here is synchronous, so restore the connection's exact
|
|
7677
|
+
* policy before any other application work can run, including on failure.
|
|
7678
|
+
*/
|
|
7679
|
+
withoutLockWait(work) {
|
|
7680
|
+
const timeout = this.db.$client.pragma("busy_timeout", { simple: true });
|
|
7681
|
+
try {
|
|
7682
|
+
this.db.$client.pragma("busy_timeout = 0");
|
|
7683
|
+
return work();
|
|
7684
|
+
} finally {
|
|
7685
|
+
this.db.$client.pragma(`busy_timeout = ${timeout}`);
|
|
7686
|
+
}
|
|
7687
|
+
}
|
|
7688
|
+
parseCursor(cursor, metadata2, filters) {
|
|
7689
|
+
let decoded;
|
|
7690
|
+
try {
|
|
7691
|
+
decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
|
7692
|
+
} catch {
|
|
7693
|
+
throw validationError("The operational-log cursor is invalid.");
|
|
7694
|
+
}
|
|
7695
|
+
if (!isCursorPayload(decoded)) throw validationError("The operational-log cursor is invalid.");
|
|
7696
|
+
if (decoded.namespace !== metadata2.cursorNamespace) throw validationError("The operational-log cursor belongs to a different database.");
|
|
7697
|
+
if (decoded.filters !== filterHash(filters)) throw validationError("The operational-log cursor does not match these query filters.");
|
|
7698
|
+
if (decoded.sequence < 1 || decoded.sequence >= metadata2.nextSequence) throw validationError("The operational-log cursor is invalid.");
|
|
7699
|
+
return decoded.sequence;
|
|
7700
|
+
}
|
|
7701
|
+
assertCursorCurrent(sequence) {
|
|
7702
|
+
const rows = this.db.all(sql3`SELECT sequence FROM runtime_logs WHERE sequence = ${sequence}`);
|
|
7703
|
+
if (rows.length === 0) throw validationError("The operational-log cursor is stale because retention evicted its entry.");
|
|
7704
|
+
}
|
|
7705
|
+
cursorFor(namespace, sequence, filters) {
|
|
7706
|
+
return Buffer.from(JSON.stringify({ namespace, sequence, filters: filterHash(filters) }), "utf8").toString("base64url");
|
|
7707
|
+
}
|
|
7708
|
+
toDto(row, namespace, filters) {
|
|
7709
|
+
const context = parseContext(row.context);
|
|
7710
|
+
const entry = {
|
|
7711
|
+
cursor: this.cursorFor(namespace, row.sequence, filters),
|
|
7712
|
+
ts: redactLogString(row.ts),
|
|
7713
|
+
level: redactLogString(row.level),
|
|
7714
|
+
module: redactLogString(row.module),
|
|
7715
|
+
action: redactLogString(row.action),
|
|
7716
|
+
...row.msg === null ? {} : { message: clip(redactLogString(row.msg), 4096) },
|
|
7717
|
+
...row.runId === null ? {} : { runId: clip(redactLogString(row.runId), 256) },
|
|
7718
|
+
...row.projectId === null ? {} : { projectId: clip(redactLogString(row.projectId), 256) },
|
|
7719
|
+
context
|
|
7720
|
+
};
|
|
7721
|
+
return operationalLogEntryDtoSchema.parse(entry);
|
|
7722
|
+
}
|
|
7723
|
+
};
|
|
7724
|
+
function projectEvent(event) {
|
|
7725
|
+
const value = redactLogValue(event);
|
|
7726
|
+
if (!isRecord(value)) throw new TypeError("Operational log event must be an object.");
|
|
7727
|
+
const contextSource = isRecord(value.context) ? value.context : {};
|
|
7728
|
+
const context = projectContext(value, contextSource);
|
|
7729
|
+
const entry = {
|
|
7730
|
+
ts: new Date(requiredString(value.ts, 64, "timestamp")).toISOString(),
|
|
7731
|
+
level: requiredString(value.level, 16, "level"),
|
|
7732
|
+
module: requiredString(value.module, 256, "module"),
|
|
7733
|
+
action: requiredString(value.action, 256, "action"),
|
|
7734
|
+
...optionalString(value.msg, 4096) === void 0 ? {} : { message: optionalString(value.msg, 4096) },
|
|
7735
|
+
...context.runId === void 0 ? {} : { runId: context.runId },
|
|
7736
|
+
...context.projectId === void 0 ? {} : { projectId: context.projectId },
|
|
7737
|
+
context
|
|
7738
|
+
};
|
|
7739
|
+
return operationalLogEntryDtoSchema.omit({ cursor: true }).parse(entry);
|
|
7740
|
+
}
|
|
7741
|
+
function projectContext(event, nested) {
|
|
7742
|
+
const context = {};
|
|
7743
|
+
for (const [key, max2] of ID_FIELDS) {
|
|
7744
|
+
const raw = event[key] ?? nested[key];
|
|
7745
|
+
const value = diagnosticIdentity(key, raw) ?? optionalString(raw, max2);
|
|
7746
|
+
if (value !== void 0) context[key] = value;
|
|
7747
|
+
}
|
|
7748
|
+
for (const key of NUMBER_FIELDS) {
|
|
7749
|
+
const value = event[key] ?? nested[key];
|
|
7750
|
+
if (typeof value === "number" && Number.isFinite(value)) context[key] = value;
|
|
7751
|
+
}
|
|
7752
|
+
for (const key of BOOLEAN_FIELDS) {
|
|
7753
|
+
const value = event[key] ?? nested[key];
|
|
7754
|
+
if (typeof value === "boolean") context[key] = value;
|
|
7755
|
+
}
|
|
7756
|
+
const errorCode = optionalString(event.errorCode ?? nested.errorCode, 128);
|
|
7757
|
+
if (errorCode !== void 0) context.errorCode = errorCode;
|
|
7758
|
+
return context;
|
|
7759
|
+
}
|
|
7760
|
+
var ID_FIELDS = [
|
|
7761
|
+
["runId", 256],
|
|
7762
|
+
["projectId", 256],
|
|
7763
|
+
["requestId", 256],
|
|
7764
|
+
["actor", 512],
|
|
7765
|
+
["credentialId", 512],
|
|
7766
|
+
["userAgent", 512],
|
|
7767
|
+
["actorSession", 512],
|
|
7768
|
+
["operationId", 256],
|
|
7769
|
+
["jobId", 256],
|
|
7770
|
+
["taskId", 256],
|
|
7771
|
+
["traceId", 256],
|
|
7772
|
+
["method", 16],
|
|
7773
|
+
["route", 256]
|
|
7774
|
+
];
|
|
7775
|
+
var NUMBER_FIELDS = ["attempt", "count", "total", "progress", "httpStatus", "statusCode", "durationMs", "bytes", "retryAfterMs"];
|
|
7776
|
+
var BOOLEAN_FIELDS = ["retriable", "retrying", "cancelled", "success"];
|
|
7777
|
+
function buildWhere(query, after) {
|
|
7778
|
+
const where = [sql3`sequence > ${after}`];
|
|
7779
|
+
if (query.level !== void 0) where.push(sql3`level = ${query.level}`);
|
|
7780
|
+
if (query.module !== void 0) where.push(sql3`module = ${query.module}`);
|
|
7781
|
+
if (query.runId !== void 0) where.push(sql3`run_id = ${query.runId}`);
|
|
7782
|
+
if (query.projectId !== void 0) where.push(sql3`project_id = ${query.projectId}`);
|
|
7783
|
+
if (query.actor !== void 0) where.push(sql3`actor = ${query.actor}`);
|
|
7784
|
+
if (query.requestId !== void 0) where.push(sql3`request_id = ${query.requestId}`);
|
|
7785
|
+
if (query.since !== void 0) where.push(sql3`ts >= ${query.since}`);
|
|
7786
|
+
if (query.until !== void 0) where.push(sql3`ts <= ${query.until}`);
|
|
7787
|
+
return where;
|
|
7788
|
+
}
|
|
7789
|
+
function readMetadata(db) {
|
|
7790
|
+
const rows = db.all(sql3`
|
|
7791
|
+
SELECT cursor_namespace AS cursorNamespace, next_sequence AS nextSequence, dropped, capture_errors AS captureErrors
|
|
7792
|
+
FROM runtime_log_metadata WHERE id = ${METADATA_ID}
|
|
7793
|
+
`);
|
|
7794
|
+
const metadata2 = rows[0];
|
|
7795
|
+
if (metadata2 === void 0 || !Number.isSafeInteger(metadata2.nextSequence) || metadata2.nextSequence < 1) {
|
|
7796
|
+
throw new Error("Operational log metadata is missing or corrupt.");
|
|
7797
|
+
}
|
|
7798
|
+
return metadata2;
|
|
7799
|
+
}
|
|
7800
|
+
function parseContext(value) {
|
|
7801
|
+
const parsed = typeof value === "string" ? parseJsonColumn(value, void 0) : value;
|
|
7802
|
+
const redacted = redactLogValue(parsed);
|
|
7803
|
+
if (!isRecord(redacted)) throw new Error("Operational log context is corrupt.");
|
|
7804
|
+
return redacted;
|
|
7805
|
+
}
|
|
7806
|
+
function filterHash(query) {
|
|
7807
|
+
const stable = JSON.stringify({
|
|
7808
|
+
level: query.level ?? null,
|
|
7809
|
+
module: query.module ?? null,
|
|
7810
|
+
runId: query.runId ?? null,
|
|
7811
|
+
projectId: query.projectId ?? null,
|
|
7812
|
+
actor: query.actor ?? null,
|
|
7813
|
+
requestId: query.requestId ?? null,
|
|
7814
|
+
since: query.since ?? null,
|
|
7815
|
+
until: query.until ?? null
|
|
7816
|
+
});
|
|
7817
|
+
return createHash("sha256").update(stable).digest("base64url");
|
|
7818
|
+
}
|
|
7819
|
+
function normalizeLogQuery(query) {
|
|
7820
|
+
return {
|
|
7821
|
+
...query,
|
|
7822
|
+
...query.since === void 0 ? {} : { since: new Date(query.since).toISOString() },
|
|
7823
|
+
...query.until === void 0 ? {} : { until: new Date(query.until).toISOString() }
|
|
7824
|
+
};
|
|
7825
|
+
}
|
|
7826
|
+
function isCursorPayload(value) {
|
|
7827
|
+
return isRecord(value) && typeof value.namespace === "string" && Number.isSafeInteger(value.sequence) && typeof value.filters === "string";
|
|
7828
|
+
}
|
|
7829
|
+
function isRecord(value) {
|
|
7830
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
7831
|
+
}
|
|
7832
|
+
function requiredString(value, max2, label) {
|
|
7833
|
+
const string = optionalString(value, max2);
|
|
7834
|
+
if (string === void 0) throw new TypeError(`Operational log ${label} must be a non-empty string.`);
|
|
7835
|
+
return string;
|
|
7836
|
+
}
|
|
7837
|
+
function optionalString(value, max2) {
|
|
7838
|
+
if (typeof value !== "string" || value.length === 0) return void 0;
|
|
7839
|
+
return clip(redactLogString(value), max2);
|
|
7840
|
+
}
|
|
7841
|
+
function clip(value, max2) {
|
|
7842
|
+
return value.length <= max2 ? value : value.slice(0, max2);
|
|
7843
|
+
}
|
|
7844
|
+
function boundedInteger(value, min2, max2, label) {
|
|
7845
|
+
if (!Number.isInteger(value) || value < min2 || value > max2) {
|
|
7846
|
+
throw new RangeError(`${label} must be an integer from ${min2} to ${max2}.`);
|
|
7847
|
+
}
|
|
7848
|
+
return value;
|
|
7849
|
+
}
|
|
7850
|
+
|
|
7478
7851
|
// ../db/src/run-helpers.ts
|
|
7479
7852
|
function groupRunsByCreatedAt(rows) {
|
|
7480
7853
|
const groups = [];
|
|
@@ -7663,12 +8036,12 @@ function analyzeCause(regression, currentSnapshots) {
|
|
|
7663
8036
|
}
|
|
7664
8037
|
|
|
7665
8038
|
// ../intelligence/src/insights.ts
|
|
7666
|
-
import { randomUUID } from "crypto";
|
|
8039
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
7667
8040
|
var QUERY_LEVEL_PROVIDER = "all";
|
|
7668
8041
|
function generateInsights(input) {
|
|
7669
8042
|
const insights2 = [];
|
|
7670
8043
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7671
|
-
const id = () => `ins_${
|
|
8044
|
+
const id = () => `ins_${randomUUID2().slice(0, 8)}`;
|
|
7672
8045
|
for (const reg of input.regressions) {
|
|
7673
8046
|
const cause = input.causes.get(`${reg.query}:${reg.provider}`);
|
|
7674
8047
|
insights2.push({
|
|
@@ -11147,6 +11520,70 @@ async function authPlugin(app, opts = {}) {
|
|
|
11147
11520
|
});
|
|
11148
11521
|
}
|
|
11149
11522
|
|
|
11523
|
+
// ../api-routes/src/request-context.ts
|
|
11524
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
11525
|
+
var requestContext = new AsyncLocalStorage();
|
|
11526
|
+
var MAX_REQUEST_CONTEXT_LENGTH = 512;
|
|
11527
|
+
function getRequestContext(boundRequestId) {
|
|
11528
|
+
const context = requestContext.getStore();
|
|
11529
|
+
if (!context || context.completed && context.requestId !== boundRequestId) return void 0;
|
|
11530
|
+
const { completed: _completed, ...safeContext } = context;
|
|
11531
|
+
return safeContext;
|
|
11532
|
+
}
|
|
11533
|
+
function requestAuditIdentity(request) {
|
|
11534
|
+
const principal = request.principal;
|
|
11535
|
+
if (!principal) return {};
|
|
11536
|
+
if (principal.delegatedUser) {
|
|
11537
|
+
return {
|
|
11538
|
+
actor: `user:${principal.delegatedUser.id}`,
|
|
11539
|
+
credentialId: request.apiKey?.id ?? principal.id
|
|
11540
|
+
};
|
|
11541
|
+
}
|
|
11542
|
+
if (principal.kind === "user") return { actor: `user:${principal.id}` };
|
|
11543
|
+
return {
|
|
11544
|
+
actor: `api-key:${principal.id}`,
|
|
11545
|
+
credentialId: request.apiKey?.id ?? principal.id
|
|
11546
|
+
};
|
|
11547
|
+
}
|
|
11548
|
+
function registerRequestContext(app) {
|
|
11549
|
+
app.addHook("onRequest", (request, reply, done) => {
|
|
11550
|
+
reply.header("x-request-id", request.id);
|
|
11551
|
+
requestContext.run({
|
|
11552
|
+
requestId: sanitizeRequestContext(request.id) ?? void 0,
|
|
11553
|
+
userAgent: sanitizeRequestContext(request.headers["user-agent"]) ?? void 0,
|
|
11554
|
+
actorSession: sanitizeRequestContext(request.headers["x-canonry-actor-session"]) ?? void 0,
|
|
11555
|
+
method: request.method,
|
|
11556
|
+
// Route template only, never a raw URL, query string, or route parameter.
|
|
11557
|
+
route: request.routeOptions.url
|
|
11558
|
+
}, done);
|
|
11559
|
+
});
|
|
11560
|
+
app.addHook("preHandler", (request, _reply, done) => {
|
|
11561
|
+
populateIdentity(request);
|
|
11562
|
+
done();
|
|
11563
|
+
});
|
|
11564
|
+
app.addHook("onError", (request, _reply, _error, done) => {
|
|
11565
|
+
populateIdentity(request);
|
|
11566
|
+
done();
|
|
11567
|
+
});
|
|
11568
|
+
app.addHook("onResponse", (request, reply, done) => {
|
|
11569
|
+
populateIdentity(request);
|
|
11570
|
+
const context = requestContext.getStore();
|
|
11571
|
+
if (context) context.statusCode = reply.statusCode;
|
|
11572
|
+
done();
|
|
11573
|
+
if (context) context.completed = true;
|
|
11574
|
+
});
|
|
11575
|
+
}
|
|
11576
|
+
function populateIdentity(request) {
|
|
11577
|
+
const context = requestContext.getStore();
|
|
11578
|
+
if (context) Object.assign(context, requestAuditIdentity(request));
|
|
11579
|
+
}
|
|
11580
|
+
function sanitizeRequestContext(value) {
|
|
11581
|
+
const joined = Array.isArray(value) ? value.join(", ") : value;
|
|
11582
|
+
if (!joined) return null;
|
|
11583
|
+
const sanitized = redactLogString(joined).replace(/[\r\n]+/g, " ").trim().slice(0, MAX_REQUEST_CONTEXT_LENGTH);
|
|
11584
|
+
return sanitized || null;
|
|
11585
|
+
}
|
|
11586
|
+
|
|
11150
11587
|
// ../api-routes/src/oauth.ts
|
|
11151
11588
|
import crypto5 from "crypto";
|
|
11152
11589
|
import { and, eq as eq3, isNull, lt } from "drizzle-orm";
|
|
@@ -11608,11 +12045,11 @@ function registerOAuthAdminRoutes(app, opts) {
|
|
|
11608
12045
|
|
|
11609
12046
|
// ../api-routes/src/projects.ts
|
|
11610
12047
|
import crypto7 from "crypto";
|
|
11611
|
-
import { and as and2, eq as eq5, sql as
|
|
12048
|
+
import { and as and2, eq as eq5, sql as sql5 } from "drizzle-orm";
|
|
11612
12049
|
|
|
11613
12050
|
// ../api-routes/src/helpers.ts
|
|
11614
12051
|
import crypto6 from "crypto";
|
|
11615
|
-
import { eq as eq4, ne, sql as
|
|
12052
|
+
import { eq as eq4, ne, sql as sql4 } from "drizzle-orm";
|
|
11616
12053
|
function notProbeRun() {
|
|
11617
12054
|
return ne(runs.trigger, RunTriggers.probe);
|
|
11618
12055
|
}
|
|
@@ -11624,29 +12061,37 @@ function resolveProject(db, name) {
|
|
|
11624
12061
|
return project;
|
|
11625
12062
|
}
|
|
11626
12063
|
function writeAuditLog(db, entry) {
|
|
12064
|
+
const context = getRequestContext();
|
|
12065
|
+
const actor = entry.actor === "api" ? context?.actor ?? entry.actor : entry.actor;
|
|
11627
12066
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11628
12067
|
db.insert(auditLog).values({
|
|
11629
12068
|
id: crypto6.randomUUID(),
|
|
11630
12069
|
projectId: entry.projectId ?? null,
|
|
11631
|
-
actor
|
|
12070
|
+
actor,
|
|
11632
12071
|
action: entry.action,
|
|
11633
12072
|
entityType: entry.entityType,
|
|
11634
12073
|
entityId: entry.entityId ?? null,
|
|
11635
12074
|
diff: entry.diff != null ? JSON.stringify(entry.diff) : null,
|
|
11636
|
-
userAgent: entry.userAgent ??
|
|
11637
|
-
actorSession: entry.actorSession ??
|
|
12075
|
+
userAgent: sanitizeRequestContext(entry.userAgent ?? context?.userAgent),
|
|
12076
|
+
actorSession: sanitizeRequestContext(entry.actorSession ?? context?.actorSession),
|
|
12077
|
+
requestId: entry.requestId ?? context?.requestId ?? null,
|
|
12078
|
+
credentialId: entry.credentialId ?? context?.credentialId ?? null,
|
|
11638
12079
|
createdAt: now
|
|
11639
12080
|
}).run();
|
|
11640
12081
|
}
|
|
11641
12082
|
function auditFromRequest(request, entry) {
|
|
11642
|
-
const
|
|
11643
|
-
const
|
|
11644
|
-
const
|
|
11645
|
-
const
|
|
12083
|
+
const userAgent = sanitizeRequestContext(request.headers["user-agent"]);
|
|
12084
|
+
const actorSession = sanitizeRequestContext(request.headers["x-canonry-actor-session"]);
|
|
12085
|
+
const identity = requestAuditIdentity(request);
|
|
12086
|
+
const context = getRequestContext();
|
|
12087
|
+
const actor = entry.actor === "api" ? identity.actor ?? entry.actor : entry.actor;
|
|
11646
12088
|
return {
|
|
11647
12089
|
...entry,
|
|
11648
|
-
|
|
11649
|
-
|
|
12090
|
+
actor,
|
|
12091
|
+
credentialId: entry.credentialId ?? identity.credentialId ?? context?.credentialId ?? null,
|
|
12092
|
+
userAgent: sanitizeRequestContext(entry.userAgent ?? userAgent),
|
|
12093
|
+
actorSession: sanitizeRequestContext(entry.actorSession ?? actorSession),
|
|
12094
|
+
requestId: entry.requestId ?? context?.requestId ?? null
|
|
11650
12095
|
};
|
|
11651
12096
|
}
|
|
11652
12097
|
function resolveSnapshotMentionResult(snapshot, project) {
|
|
@@ -11726,18 +12171,32 @@ async function settingsRoutes(app, opts) {
|
|
|
11726
12171
|
validProviders: validNames
|
|
11727
12172
|
});
|
|
11728
12173
|
}
|
|
11729
|
-
if (
|
|
11730
|
-
|
|
12174
|
+
if (apiKey !== void 0 && (typeof apiKey !== "string" || !apiKey.trim())) {
|
|
12175
|
+
throw validationError("apiKey must be a non-empty string when provided");
|
|
12176
|
+
}
|
|
12177
|
+
if (baseUrl !== void 0 && (typeof baseUrl !== "string" || !baseUrl.trim())) {
|
|
12178
|
+
throw validationError("baseUrl must be a non-empty string when provided");
|
|
12179
|
+
}
|
|
12180
|
+
if (model !== void 0 && (typeof model !== "string" || !model.trim())) {
|
|
12181
|
+
throw validationError("model must be a non-empty string when provided");
|
|
12182
|
+
}
|
|
12183
|
+
if (apiKey === void 0 && baseUrl === void 0 && model === void 0 && quota === void 0) {
|
|
12184
|
+
throw validationError("at least one provider setting must be supplied");
|
|
12185
|
+
}
|
|
12186
|
+
const existing = (opts.providerSummary ?? []).find((provider) => provider.name === name);
|
|
12187
|
+
const configured = Boolean(existing?.configured);
|
|
12188
|
+
if (!configured && name === "local") {
|
|
12189
|
+
if (!baseUrl) {
|
|
11731
12190
|
throw validationError("baseUrl is required for local provider");
|
|
11732
12191
|
}
|
|
11733
|
-
} else if (name === "gemini" && !apiKey) {
|
|
12192
|
+
} else if (!configured && name === "gemini" && !apiKey) {
|
|
11734
12193
|
const geminiSummary = (opts.providerSummary ?? []).find((p) => p.name === "gemini");
|
|
11735
12194
|
if (!geminiSummary?.vertexConfigured) {
|
|
11736
12195
|
throw validationError(
|
|
11737
12196
|
"apiKey is required for Gemini unless Vertex AI is configured (set GEMINI_VERTEX_PROJECT env var or vertexProject in config file)"
|
|
11738
12197
|
);
|
|
11739
12198
|
}
|
|
11740
|
-
} else {
|
|
12199
|
+
} else if (!configured) {
|
|
11741
12200
|
if (!apiKey || typeof apiKey !== "string") {
|
|
11742
12201
|
throw validationError("apiKey is required");
|
|
11743
12202
|
}
|
|
@@ -11756,6 +12215,9 @@ async function settingsRoutes(app, opts) {
|
|
|
11756
12215
|
if (typeof quota !== "object" || quota === null) {
|
|
11757
12216
|
throw validationError("quota must be an object");
|
|
11758
12217
|
}
|
|
12218
|
+
if (Object.keys(quota).length === 0) {
|
|
12219
|
+
throw validationError("quota must include at least one field");
|
|
12220
|
+
}
|
|
11759
12221
|
for (const [key, val] of Object.entries(quota)) {
|
|
11760
12222
|
if (!["maxConcurrency", "maxRequestsPerMinute", "maxRequestsPerDay"].includes(key)) {
|
|
11761
12223
|
throw validationError(`Unknown quota field: ${key}`);
|
|
@@ -11765,7 +12227,12 @@ async function settingsRoutes(app, opts) {
|
|
|
11765
12227
|
}
|
|
11766
12228
|
}
|
|
11767
12229
|
}
|
|
11768
|
-
const
|
|
12230
|
+
const { actor, userAgent, actorSession, requestId, credentialId } = auditFromRequest(request, {
|
|
12231
|
+
actor: "api",
|
|
12232
|
+
action: "provider.updated",
|
|
12233
|
+
entityType: "provider"
|
|
12234
|
+
});
|
|
12235
|
+
const result = opts.onProviderUpdate(name, apiKey ?? "", model, baseUrl, quota, { actor, userAgent, actorSession, requestId, credentialId });
|
|
11769
12236
|
if (!result) {
|
|
11770
12237
|
throw internalError("Failed to update provider configuration");
|
|
11771
12238
|
}
|
|
@@ -12091,12 +12558,12 @@ async function projectRoutes(app, opts) {
|
|
|
12091
12558
|
const project = resolveProject(app.db, request.params.name);
|
|
12092
12559
|
const pid = project.id;
|
|
12093
12560
|
const count4 = (n) => n ?? 0;
|
|
12094
|
-
const queryCount = app.db.select({ n:
|
|
12095
|
-
const competitorCount = app.db.select({ n:
|
|
12096
|
-
const runCount = app.db.select({ n:
|
|
12097
|
-
const snapshotCount = app.db.select({ n:
|
|
12098
|
-
const insightCount = app.db.select({ n:
|
|
12099
|
-
const auditLogCount = app.db.select({ n:
|
|
12561
|
+
const queryCount = app.db.select({ n: sql5`count(*)` }).from(queries).where(eq5(queries.projectId, pid)).get();
|
|
12562
|
+
const competitorCount = app.db.select({ n: sql5`count(*)` }).from(competitors).where(eq5(competitors.projectId, pid)).get();
|
|
12563
|
+
const runCount = app.db.select({ n: sql5`count(*)` }).from(runs).where(eq5(runs.projectId, pid)).get();
|
|
12564
|
+
const snapshotCount = app.db.select({ n: sql5`count(*)` }).from(querySnapshots).innerJoin(runs, eq5(querySnapshots.runId, runs.id)).where(eq5(runs.projectId, pid)).get();
|
|
12565
|
+
const insightCount = app.db.select({ n: sql5`count(*)` }).from(insights).where(eq5(insights.projectId, pid)).get();
|
|
12566
|
+
const auditLogCount = app.db.select({ n: sql5`count(*)` }).from(auditLog).where(eq5(auditLog.projectId, pid)).get();
|
|
12100
12567
|
return reply.send({
|
|
12101
12568
|
project: { id: project.id, name: project.name },
|
|
12102
12569
|
cascadeRows: {
|
|
@@ -12311,7 +12778,7 @@ function aliasArraysEqual(a, b) {
|
|
|
12311
12778
|
|
|
12312
12779
|
// ../api-routes/src/queries.ts
|
|
12313
12780
|
import crypto10 from "crypto";
|
|
12314
|
-
import { and as and5, eq as eq8, inArray as inArray2, sql as
|
|
12781
|
+
import { and as and5, eq as eq8, inArray as inArray2, sql as sql6 } from "drizzle-orm";
|
|
12315
12782
|
|
|
12316
12783
|
// ../api-routes/src/query-replace.ts
|
|
12317
12784
|
import crypto9 from "crypto";
|
|
@@ -12674,9 +13141,9 @@ async function queryRoutes(app, opts) {
|
|
|
12674
13141
|
let snapshotsDetached = 0;
|
|
12675
13142
|
let affectedQueries = 0;
|
|
12676
13143
|
if (removedIds.length > 0) {
|
|
12677
|
-
const snapshotCount = app.db.select({ n:
|
|
13144
|
+
const snapshotCount = app.db.select({ n: sql6`count(*)` }).from(querySnapshots).where(inArray2(querySnapshots.queryId, removedIds)).get();
|
|
12678
13145
|
snapshotsDetached = snapshotCount?.n ?? 0;
|
|
12679
|
-
const distinctAffected = app.db.select({ n:
|
|
13146
|
+
const distinctAffected = app.db.select({ n: sql6`count(distinct ${querySnapshots.queryId})` }).from(querySnapshots).where(inArray2(querySnapshots.queryId, removedIds)).get();
|
|
12680
13147
|
affectedQueries = distinctAffected?.n ?? 0;
|
|
12681
13148
|
}
|
|
12682
13149
|
return reply.send({
|
|
@@ -13198,7 +13665,7 @@ function shareOfVoiceFromLandscape(landscape, queryClass) {
|
|
|
13198
13665
|
}
|
|
13199
13666
|
|
|
13200
13667
|
// ../api-routes/src/measurement-overview.ts
|
|
13201
|
-
import { createHash } from "crypto";
|
|
13668
|
+
import { createHash as createHash2 } from "crypto";
|
|
13202
13669
|
import { and as and8, eq as eq13 } from "drizzle-orm";
|
|
13203
13670
|
|
|
13204
13671
|
// ../api-routes/src/measurement-report.ts
|
|
@@ -14585,7 +15052,7 @@ function overviewFilterFingerprint(query) {
|
|
|
14585
15052
|
to: query.to ?? null,
|
|
14586
15053
|
search: query.search === void 0 ? null : normalizedText(query.search)
|
|
14587
15054
|
};
|
|
14588
|
-
return
|
|
15055
|
+
return createHash2("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
14589
15056
|
}
|
|
14590
15057
|
function overviewAggregateFingerprint(query) {
|
|
14591
15058
|
const filters = {
|
|
@@ -14598,11 +15065,11 @@ function overviewAggregateFingerprint(query) {
|
|
|
14598
15065
|
from: query.from ?? null,
|
|
14599
15066
|
to: query.to ?? null
|
|
14600
15067
|
};
|
|
14601
|
-
return
|
|
15068
|
+
return createHash2("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
14602
15069
|
}
|
|
14603
15070
|
function overviewEvidenceFingerprint(snapshots) {
|
|
14604
15071
|
const canonical = [...snapshots].sort((left, right) => left.id.localeCompare(right.id)).map((snapshot) => JSON.stringify(snapshot)).join("\n");
|
|
14605
|
-
return
|
|
15072
|
+
return createHash2("sha256").update(canonical).digest("base64url");
|
|
14606
15073
|
}
|
|
14607
15074
|
function compareLabels(left, right) {
|
|
14608
15075
|
const leftLabel = normalizedText(left.label);
|
|
@@ -15454,7 +15921,7 @@ function mergePins(...collections) {
|
|
|
15454
15921
|
|
|
15455
15922
|
// ../api-routes/src/runs.ts
|
|
15456
15923
|
import crypto14 from "crypto";
|
|
15457
|
-
import { and as and12, eq as eq17, asc, desc as desc3, inArray as inArray5, or as or2, sql as
|
|
15924
|
+
import { and as and12, eq as eq17, asc, desc as desc3, inArray as inArray5, or as or2, sql as sql7 } from "drizzle-orm";
|
|
15458
15925
|
import { gte as gte3 } from "drizzle-orm";
|
|
15459
15926
|
|
|
15460
15927
|
// ../api-routes/src/run-queue.ts
|
|
@@ -16030,7 +16497,7 @@ async function runRoutes(app, opts) {
|
|
|
16030
16497
|
});
|
|
16031
16498
|
app.get("/projects/:name/runs/latest", async (request, reply) => {
|
|
16032
16499
|
const project = resolveProject(app.db, request.params.name);
|
|
16033
|
-
const countRow = app.db.select({ count:
|
|
16500
|
+
const countRow = app.db.select({ count: sql7`count(*)` }).from(runs).where(and12(eq17(runs.projectId, project.id), notProbeRun())).get();
|
|
16034
16501
|
const totalRuns = countRow?.count ?? 0;
|
|
16035
16502
|
const latestRun = app.db.select().from(runs).where(and12(eq17(runs.projectId, project.id), notProbeRun())).orderBy(desc3(runs.createdAt), desc3(runs.id)).limit(1).get();
|
|
16036
16503
|
if (!latestRun) {
|
|
@@ -20961,7 +21428,7 @@ async function measurementDiscoveryV2Routes(app) {
|
|
|
20961
21428
|
}
|
|
20962
21429
|
|
|
20963
21430
|
// ../api-routes/src/measurement-property-evidence.ts
|
|
20964
|
-
import { createHash as
|
|
21431
|
+
import { createHash as createHash3 } from "crypto";
|
|
20965
21432
|
import { and as and16, eq as eq21 } from "drizzle-orm";
|
|
20966
21433
|
function parseEvidenceQuery(raw) {
|
|
20967
21434
|
const candidate = { ...raw, ...raw.limit === void 0 ? {} : { limit: Number(raw.limit) } };
|
|
@@ -20981,11 +21448,11 @@ function filterFingerprint(query) {
|
|
|
20981
21448
|
provider: query.provider === void 0 ? null : normalizedText2(query.provider),
|
|
20982
21449
|
location: query.location === void 0 ? null : normalizedText2(query.location)
|
|
20983
21450
|
};
|
|
20984
|
-
return
|
|
21451
|
+
return createHash3("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
20985
21452
|
}
|
|
20986
21453
|
function evidenceFingerprintOf(snapshots) {
|
|
20987
21454
|
const canonical = [...snapshots].sort((left, right) => left.id.localeCompare(right.id)).map((snapshot) => JSON.stringify(snapshot)).join("\n");
|
|
20988
|
-
return
|
|
21455
|
+
return createHash3("sha256").update(canonical).digest("base64url");
|
|
20989
21456
|
}
|
|
20990
21457
|
function sourceRowKey(row) {
|
|
20991
21458
|
return [row.expectedSlotId, row.usageEdgeId, row.sourceUrl].join("\0");
|
|
@@ -21152,7 +21619,7 @@ async function measurementPropertyEvidenceRoutes(app) {
|
|
|
21152
21619
|
import { and as and18, desc as desc6, eq as eq23, inArray as inArray8, lt as lt3, or as or3 } from "drizzle-orm";
|
|
21153
21620
|
|
|
21154
21621
|
// ../api-routes/src/measurement-question-reads.ts
|
|
21155
|
-
import { and as and17, eq as eq22, getTableColumns, inArray as inArray7, sql as
|
|
21622
|
+
import { and as and17, eq as eq22, getTableColumns, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
21156
21623
|
var DEFAULT_LIMIT2 = 50;
|
|
21157
21624
|
var TERMINAL_STATUSES = /* @__PURE__ */ new Set([
|
|
21158
21625
|
RunStatuses.completed,
|
|
@@ -21271,7 +21738,7 @@ function selectedQuestionSnapshots(db, runId, manifest, scope) {
|
|
|
21271
21738
|
conditions.push(inArray7(querySnapshots.measurementExecutionId, executionIds));
|
|
21272
21739
|
}
|
|
21273
21740
|
if (scope?.provider !== void 0) {
|
|
21274
|
-
conditions.push(
|
|
21741
|
+
conditions.push(sql8`lower(trim(${querySnapshots.provider})) = ${normalizeText(scope.provider)}`);
|
|
21275
21742
|
}
|
|
21276
21743
|
return db.select(measurementQuestionSnapshotColumns).from(querySnapshots).where(and17(...conditions)).all();
|
|
21277
21744
|
}
|
|
@@ -22226,11 +22693,11 @@ async function measurementPortfolioReadRoutes(app) {
|
|
|
22226
22693
|
}
|
|
22227
22694
|
|
|
22228
22695
|
// ../api-routes/src/visibility-report.ts
|
|
22229
|
-
import { createHash as
|
|
22696
|
+
import { createHash as createHash5 } from "crypto";
|
|
22230
22697
|
import { and as and19, desc as desc7, eq as eq24, gte as gte4, inArray as inArray9, isNotNull, isNull as isNull4, lte as lte4, ne as ne3 } from "drizzle-orm";
|
|
22231
22698
|
|
|
22232
22699
|
// ../api-routes/src/visibility-report-reader.ts
|
|
22233
|
-
import { createHash as
|
|
22700
|
+
import { createHash as createHash4 } from "crypto";
|
|
22234
22701
|
var VisibilityReportCursorError = class extends Error {
|
|
22235
22702
|
constructor(message) {
|
|
22236
22703
|
super(message);
|
|
@@ -22605,7 +23072,7 @@ function breakdown(candidates, definition, populationTargetKeys = selectedTarget
|
|
|
22605
23072
|
return { properties, groups };
|
|
22606
23073
|
}
|
|
22607
23074
|
function fingerprint(value) {
|
|
22608
|
-
return
|
|
23075
|
+
return createHash4("sha256").update(JSON.stringify(value)).digest("base64url");
|
|
22609
23076
|
}
|
|
22610
23077
|
function cursorOf2(cursor) {
|
|
22611
23078
|
return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
|
|
@@ -23098,11 +23565,11 @@ function stableSimpleDefinitionId(definition) {
|
|
|
23098
23565
|
// not freeze a competitor set, whereas a newly captured empty list did.
|
|
23099
23566
|
competitors: definition.competitors === void 0 ? null : [...definition.competitors].map((competitor) => ({ domain: competitor.domain, label: competitor.label, aliases: [...competitor.aliases].sort() })).sort((left, right) => left.domain.localeCompare(right.domain, "en"))
|
|
23100
23567
|
};
|
|
23101
|
-
return `simple:${
|
|
23568
|
+
return `simple:${createHash5("sha256").update(JSON.stringify(source)).digest("hex")}`;
|
|
23102
23569
|
}
|
|
23103
23570
|
function simpleModelFingerprint(definition) {
|
|
23104
23571
|
const engines = [...definition.engines].map((engine) => ({ provider: normalizedText3(engine.provider), requestedModel: engine.requestedModel })).sort((left, right) => left.provider.localeCompare(right.provider, "en"));
|
|
23105
|
-
return `simple-model:${
|
|
23572
|
+
return `simple-model:${createHash5("sha256").update(JSON.stringify(engines)).digest("hex")}`;
|
|
23106
23573
|
}
|
|
23107
23574
|
function frozenSimpleRun(run, definition, snapshots, includeEvidence = true) {
|
|
23108
23575
|
const matcher = compileBrandAliases(effectiveBrandNames({
|
|
@@ -25625,7 +26092,7 @@ function normalizeCompetitorList2(domains) {
|
|
|
25625
26092
|
}
|
|
25626
26093
|
|
|
25627
26094
|
// ../api-routes/src/history.ts
|
|
25628
|
-
import { and as and22, asc as asc3, count, desc as desc9, eq as eq27, gte as gte5, inArray as inArray11 } from "drizzle-orm";
|
|
26095
|
+
import { and as and22, asc as asc3, count, desc as desc9, eq as eq27, gte as gte5, inArray as inArray11, like, or as or4 } from "drizzle-orm";
|
|
25629
26096
|
|
|
25630
26097
|
// ../api-routes/src/notification-redaction.ts
|
|
25631
26098
|
var REDACTED_URL = {
|
|
@@ -25696,7 +26163,7 @@ async function historyRoutes(app) {
|
|
|
25696
26163
|
if (input.confirm) {
|
|
25697
26164
|
tx.delete(researchRuns).where(and22(eq27(researchRuns.projectId, project.id), inArray11(researchRuns.id, input.researchRunIds))).run();
|
|
25698
26165
|
tx.delete(runs).where(and22(eq27(runs.projectId, project.id), inArray11(runs.id, input.runIds))).run();
|
|
25699
|
-
writeAuditLog(tx, { projectId: project.id, actor:
|
|
26166
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "results.cleared", entityType: "project", entityId: project.id, diff: result });
|
|
25700
26167
|
}
|
|
25701
26168
|
return result;
|
|
25702
26169
|
});
|
|
@@ -25705,14 +26172,14 @@ async function historyRoutes(app) {
|
|
|
25705
26172
|
const project = resolveProject(app.db, request.params.name);
|
|
25706
26173
|
const filters = [eq27(auditLog.projectId, project.id)];
|
|
25707
26174
|
addAuditHistoryFilters(filters, request.query);
|
|
25708
|
-
const rows = app.db.select().from(auditLog).where(and22(...filters)).orderBy(desc9(auditLog.createdAt)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
26175
|
+
const rows = app.db.select().from(auditLog).where(and22(...filters)).orderBy(desc9(auditLog.createdAt), desc9(auditLog.id)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
25709
26176
|
return reply.send(rows.map(formatAuditEntry));
|
|
25710
26177
|
});
|
|
25711
26178
|
app.get("/history", async (request, reply) => {
|
|
25712
26179
|
const scopedProjectId = request.apiKey?.projectId;
|
|
25713
26180
|
const filters = scopedProjectId ? [eq27(auditLog.projectId, scopedProjectId)] : [];
|
|
25714
26181
|
addAuditHistoryFilters(filters, request.query);
|
|
25715
|
-
const rows = app.db.select().from(auditLog).where(filters.length > 0 ? and22(...filters) : void 0).orderBy(desc9(auditLog.createdAt)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
26182
|
+
const rows = app.db.select().from(auditLog).where(filters.length > 0 ? and22(...filters) : void 0).orderBy(desc9(auditLog.createdAt), desc9(auditLog.id)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
25716
26183
|
return reply.send(rows.map(formatAuditEntry));
|
|
25717
26184
|
});
|
|
25718
26185
|
app.get("/projects/:name/snapshots", async (request, reply) => {
|
|
@@ -26003,13 +26470,20 @@ function formatAuditEntry(row) {
|
|
|
26003
26470
|
diff: row.diff ? row.entityType === "notification" ? redactNotificationDiff(parseJsonColumn(row.diff, null)) : parseJsonColumn(row.diff, null) : null,
|
|
26004
26471
|
userAgent: row.userAgent,
|
|
26005
26472
|
actorSession: row.actorSession,
|
|
26473
|
+
requestId: row.requestId,
|
|
26474
|
+
credentialId: row.credentialId,
|
|
26006
26475
|
createdAt: row.createdAt
|
|
26007
26476
|
};
|
|
26008
26477
|
}
|
|
26009
26478
|
function addAuditHistoryFilters(filters, query) {
|
|
26010
26479
|
if (query.since && !Number.isNaN(Date.parse(query.since))) filters.push(gte5(auditLog.createdAt, query.since));
|
|
26011
26480
|
if (query.action) filters.push(eq27(auditLog.action, query.action));
|
|
26012
|
-
if (query.actor) filters.push(
|
|
26481
|
+
if (query.actor === "api") filters.push(or4(
|
|
26482
|
+
eq27(auditLog.actor, "api"),
|
|
26483
|
+
like(auditLog.actor, "api-key:%"),
|
|
26484
|
+
like(auditLog.actor, "user:%")
|
|
26485
|
+
));
|
|
26486
|
+
else if (query.actor) filters.push(eq27(auditLog.actor, query.actor));
|
|
26013
26487
|
if (query.entityType) filters.push(eq27(auditLog.entityType, query.entityType));
|
|
26014
26488
|
}
|
|
26015
26489
|
function parseBoundedInt(value, fallback, max2) {
|
|
@@ -26818,7 +27292,7 @@ function buildCategoryCounts(counts2) {
|
|
|
26818
27292
|
}
|
|
26819
27293
|
|
|
26820
27294
|
// ../api-routes/src/intelligence.ts
|
|
26821
|
-
import { eq as eq29, desc as desc11, and as and24, inArray as inArray13, like } from "drizzle-orm";
|
|
27295
|
+
import { eq as eq29, desc as desc11, and as and24, inArray as inArray13, like as like2 } from "drizzle-orm";
|
|
26822
27296
|
var SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
|
|
26823
27297
|
function severitiesAtOrAbove(min2) {
|
|
26824
27298
|
const floor = SEVERITY_RANK[min2];
|
|
@@ -26943,7 +27417,7 @@ async function intelligenceRoutes(app) {
|
|
|
26943
27417
|
const typeFilter = request.query.type?.trim();
|
|
26944
27418
|
if (typeFilter) {
|
|
26945
27419
|
conditions.push(
|
|
26946
|
-
typeFilter.endsWith("*") ?
|
|
27420
|
+
typeFilter.endsWith("*") ? like2(insights.type, `${typeFilter.slice(0, -1)}%`) : eq29(insights.type, typeFilter)
|
|
26947
27421
|
);
|
|
26948
27422
|
}
|
|
26949
27423
|
const severityFilter = request.query.severity?.trim();
|
|
@@ -27018,16 +27492,16 @@ async function intelligenceRoutes(app) {
|
|
|
27018
27492
|
}
|
|
27019
27493
|
|
|
27020
27494
|
// ../api-routes/src/ai-referral-status.ts
|
|
27021
|
-
import { and as and25, sql as
|
|
27022
|
-
var REDIRECT_LIST =
|
|
27495
|
+
import { and as and25, sql as sql9 } from "drizzle-orm";
|
|
27496
|
+
var REDIRECT_LIST = sql9.join([...LOCATION_REDIRECT_STATUSES].map((s) => sql9`${s}`), sql9`, `);
|
|
27023
27497
|
function referralLandedCondition() {
|
|
27024
|
-
return
|
|
27498
|
+
return sql9`${aiReferralEventsHourly.status} NOT IN (${REDIRECT_LIST})`;
|
|
27025
27499
|
}
|
|
27026
27500
|
function countableReferralCondition() {
|
|
27027
27501
|
return and25(nonSubresourceReferralPathCondition(), referralLandedCondition());
|
|
27028
27502
|
}
|
|
27029
27503
|
function nonSubresourceReferralPathCondition() {
|
|
27030
|
-
return
|
|
27504
|
+
return sql9`
|
|
27031
27505
|
LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/_next/static/%'
|
|
27032
27506
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/assets/%'
|
|
27033
27507
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/static/%'
|
|
@@ -27051,7 +27525,7 @@ function nonSubresourceReferralPathCondition() {
|
|
|
27051
27525
|
}
|
|
27052
27526
|
|
|
27053
27527
|
// ../api-routes/src/report.ts
|
|
27054
|
-
import { and as and29, desc as desc14, eq as eq33, gte as gte8, inArray as inArray15, lt as lt5, lte as lte6, ne as ne5, or as
|
|
27528
|
+
import { and as and29, desc as desc14, eq as eq33, gte as gte8, inArray as inArray15, lt as lt5, lte as lte6, ne as ne5, or as or5, sql as sql11 } from "drizzle-orm";
|
|
27055
27529
|
|
|
27056
27530
|
// ../api-routes/src/content.ts
|
|
27057
27531
|
import crypto23 from "crypto";
|
|
@@ -27061,7 +27535,7 @@ import { and as and28, desc as desc13, eq as eq32 } from "drizzle-orm";
|
|
|
27061
27535
|
import { and as and27, eq as eq31, desc as desc12, gte as gte7, inArray as inArray14, lte as lte5 } from "drizzle-orm";
|
|
27062
27536
|
|
|
27063
27537
|
// ../api-routes/src/gsc-totals.ts
|
|
27064
|
-
import { and as and26, asc as asc4, eq as eq30, sql as
|
|
27538
|
+
import { and as and26, asc as asc4, eq as eq30, sql as sql10, max, min } from "drizzle-orm";
|
|
27065
27539
|
var WINDOW_DAYS = { "7d": 7, "30d": 30, "90d": 90 };
|
|
27066
27540
|
function resolveGscWindowRange(window, latestDataDate, today) {
|
|
27067
27541
|
if (window === "all") {
|
|
@@ -27116,8 +27590,8 @@ function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
|
27116
27590
|
}).from(gscDailyTotals).where(
|
|
27117
27591
|
and26(
|
|
27118
27592
|
eq30(gscDailyTotals.projectId, projectId),
|
|
27119
|
-
|
|
27120
|
-
|
|
27593
|
+
sql10`${gscDailyTotals.date} >= ${startDate}`,
|
|
27594
|
+
sql10`${gscDailyTotals.date} <= ${endDate}`
|
|
27121
27595
|
)
|
|
27122
27596
|
).orderBy(asc4(gscDailyTotals.date)).all();
|
|
27123
27597
|
return rows.map((r) => {
|
|
@@ -27146,8 +27620,8 @@ function readGscQueryDailyRows(db, projectId, startDate, endDate) {
|
|
|
27146
27620
|
}).from(gscQueryDailyTotals).where(
|
|
27147
27621
|
and26(
|
|
27148
27622
|
eq30(gscQueryDailyTotals.projectId, projectId),
|
|
27149
|
-
|
|
27150
|
-
|
|
27623
|
+
sql10`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
27624
|
+
sql10`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
27151
27625
|
)
|
|
27152
27626
|
).all();
|
|
27153
27627
|
return rows.map((r) => {
|
|
@@ -30508,8 +30982,8 @@ function loadQueryLookup(db, projectId) {
|
|
|
30508
30982
|
return { byId };
|
|
30509
30983
|
}
|
|
30510
30984
|
function buildGscSection(db, projectId, projectBrandNames, canonicalDomain, trackedQueries, windowDays2) {
|
|
30511
|
-
const maxSearch = db.select({ m:
|
|
30512
|
-
const maxDaily = db.select({ m:
|
|
30985
|
+
const maxSearch = db.select({ m: sql11`MAX(${gscSearchData.date})` }).from(gscSearchData).where(eq33(gscSearchData.projectId, projectId)).get()?.m ?? "";
|
|
30986
|
+
const maxDaily = db.select({ m: sql11`MAX(${gscDailyTotals.date})` }).from(gscDailyTotals).where(eq33(gscDailyTotals.projectId, projectId)).get()?.m ?? "";
|
|
30513
30987
|
const maxDate = maxSearch > maxDaily ? maxSearch : maxDaily;
|
|
30514
30988
|
if (!maxDate) return null;
|
|
30515
30989
|
const startDate = windowStartDate(maxDate, windowDays2);
|
|
@@ -30620,7 +31094,7 @@ function buildGaSection(db, projectId, windowDays2) {
|
|
|
30620
31094
|
).limit(1).get() : void 0;
|
|
30621
31095
|
const fallbackSummary = windowSummary ? null : db.select().from(gaTrafficSummaries).where(eq33(gaTrafficSummaries.projectId, projectId)).orderBy(desc14(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
30622
31096
|
const matchingFallbackSummary = fallbackSummary && inclusiveDayCount(fallbackSummary.periodStart, fallbackSummary.periodEnd) === windowDays2 ? fallbackSummary : null;
|
|
30623
|
-
const snapshotMaxDate = db.select({ m:
|
|
31097
|
+
const snapshotMaxDate = db.select({ m: sql11`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq33(gaTrafficSnapshots.projectId, projectId)).get()?.m ?? "";
|
|
30624
31098
|
if (!windowSummary && !matchingFallbackSummary && !snapshotMaxDate) return null;
|
|
30625
31099
|
const summaryPeriod = windowSummary ?? matchingFallbackSummary;
|
|
30626
31100
|
const snapshotMaxInWindow = summaryPeriod?.periodEnd || snapshotMaxDate;
|
|
@@ -30662,7 +31136,7 @@ function buildGaSection(db, projectId, windowDays2) {
|
|
|
30662
31136
|
const aiSessionRows = db.select({
|
|
30663
31137
|
trafficClass: gaAiReferrals.trafficClass,
|
|
30664
31138
|
channelGroup: gaAiReferrals.channelGroup,
|
|
30665
|
-
sessions:
|
|
31139
|
+
sessions: sql11`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`
|
|
30666
31140
|
}).from(gaAiReferrals).where(and29(...aiConditions)).groupBy(gaAiReferrals.trafficClass, gaAiReferrals.channelGroup).all();
|
|
30667
31141
|
let paidAiSessions = 0;
|
|
30668
31142
|
let organicAiSessions = 0;
|
|
@@ -30761,8 +31235,8 @@ function buildAiReferrals(db, projectId, windowDays2) {
|
|
|
30761
31235
|
const window = resolveAiReferralWindow(db, projectId, windowDays2);
|
|
30762
31236
|
const rows = db.select().from(gaAiReferrals).where(and29(
|
|
30763
31237
|
eq33(gaAiReferrals.projectId, projectId),
|
|
30764
|
-
|
|
30765
|
-
|
|
31238
|
+
sql11`${gaAiReferrals.date} >= ${window.start}`,
|
|
31239
|
+
sql11`${gaAiReferrals.date} <= ${window.end}`
|
|
30766
31240
|
)).all();
|
|
30767
31241
|
if (rows.length === 0) return null;
|
|
30768
31242
|
const dimSessionsByTuple = /* @__PURE__ */ new Map();
|
|
@@ -30847,7 +31321,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30847
31321
|
const priorStart = new Date(priorStartMs).toISOString();
|
|
30848
31322
|
const trendStart = new Date(trendStartMs).toISOString();
|
|
30849
31323
|
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
30850
|
-
db.select({ total:
|
|
31324
|
+
db.select({ total: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
30851
31325
|
and29(
|
|
30852
31326
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
30853
31327
|
eq33(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -30857,7 +31331,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30857
31331
|
).get()?.total ?? 0
|
|
30858
31332
|
);
|
|
30859
31333
|
const sumUnverifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
30860
|
-
db.select({ total:
|
|
31334
|
+
db.select({ total: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
30861
31335
|
and29(
|
|
30862
31336
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
30863
31337
|
ne5(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -30868,9 +31342,9 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30868
31342
|
);
|
|
30869
31343
|
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => {
|
|
30870
31344
|
const row = db.select({
|
|
30871
|
-
total:
|
|
30872
|
-
paid:
|
|
30873
|
-
organic:
|
|
31345
|
+
total: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
31346
|
+
paid: sql11`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
31347
|
+
organic: sql11`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
30874
31348
|
}).from(aiReferralEventsHourly).where(
|
|
30875
31349
|
and29(
|
|
30876
31350
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -30882,7 +31356,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30882
31356
|
return aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0));
|
|
30883
31357
|
};
|
|
30884
31358
|
const sumUserFetches = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
30885
|
-
db.select({ total:
|
|
31359
|
+
db.select({ total: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
30886
31360
|
and29(
|
|
30887
31361
|
eq33(aiUserFetchEventsHourly.projectId, projectId),
|
|
30888
31362
|
gte8(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
@@ -30898,7 +31372,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30898
31372
|
const userFetchPrior = sumUserFetches(priorStart, headlineStart, true);
|
|
30899
31373
|
const referralCurrent = sumReferrals(headlineStart, headlineEnd);
|
|
30900
31374
|
const referralAllCurrent = Number(
|
|
30901
|
-
db.select({ total:
|
|
31375
|
+
db.select({ total: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
30902
31376
|
and29(
|
|
30903
31377
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
30904
31378
|
nonSubresourceReferralPathCondition(),
|
|
@@ -30912,7 +31386,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30912
31386
|
const crawlerByOperatorRows = db.select({
|
|
30913
31387
|
operator: crawlerEventsHourly.operator,
|
|
30914
31388
|
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
30915
|
-
hits:
|
|
31389
|
+
hits: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
30916
31390
|
}).from(crawlerEventsHourly).where(
|
|
30917
31391
|
and29(
|
|
30918
31392
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
@@ -30922,7 +31396,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30922
31396
|
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
30923
31397
|
const crawlerByOperatorPriorRows = db.select({
|
|
30924
31398
|
operator: crawlerEventsHourly.operator,
|
|
30925
|
-
hits:
|
|
31399
|
+
hits: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
30926
31400
|
}).from(crawlerEventsHourly).where(
|
|
30927
31401
|
and29(
|
|
30928
31402
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
@@ -30932,7 +31406,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30932
31406
|
).groupBy(crawlerEventsHourly.operator).all();
|
|
30933
31407
|
const referralByOperatorRows = db.select({
|
|
30934
31408
|
operator: aiReferralEventsHourly.operator,
|
|
30935
|
-
hits:
|
|
31409
|
+
hits: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
30936
31410
|
}).from(aiReferralEventsHourly).where(
|
|
30937
31411
|
and29(
|
|
30938
31412
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -30943,7 +31417,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30943
31417
|
).groupBy(aiReferralEventsHourly.operator).all();
|
|
30944
31418
|
const userFetchByOperatorRows = db.select({
|
|
30945
31419
|
operator: aiUserFetchEventsHourly.operator,
|
|
30946
|
-
hits:
|
|
31420
|
+
hits: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
30947
31421
|
}).from(aiUserFetchEventsHourly).where(
|
|
30948
31422
|
and29(
|
|
30949
31423
|
eq33(aiUserFetchEventsHourly.projectId, projectId),
|
|
@@ -30986,16 +31460,16 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30986
31460
|
);
|
|
30987
31461
|
const topPathsRows = db.select({
|
|
30988
31462
|
path: crawlerEventsHourly.pathNormalized,
|
|
30989
|
-
verifiedHits:
|
|
30990
|
-
unverifiedHits:
|
|
30991
|
-
operators:
|
|
31463
|
+
verifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
31464
|
+
unverifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
31465
|
+
operators: sql11`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
30992
31466
|
}).from(crawlerEventsHourly).where(
|
|
30993
31467
|
and29(
|
|
30994
31468
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
30995
31469
|
gte8(crawlerEventsHourly.tsHour, headlineStart),
|
|
30996
31470
|
lte6(crawlerEventsHourly.tsHour, headlineEnd)
|
|
30997
31471
|
)
|
|
30998
|
-
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc14(
|
|
31472
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc14(sql11`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
30999
31473
|
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
31000
31474
|
path: r.path,
|
|
31001
31475
|
verifiedHits: Number(r.verifiedHits),
|
|
@@ -31004,8 +31478,8 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31004
31478
|
}));
|
|
31005
31479
|
const referralProductsRows = db.select({
|
|
31006
31480
|
product: aiReferralEventsHourly.product,
|
|
31007
|
-
arrivals:
|
|
31008
|
-
landingPaths:
|
|
31481
|
+
arrivals: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
31482
|
+
landingPaths: sql11`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
31009
31483
|
}).from(aiReferralEventsHourly).where(
|
|
31010
31484
|
and29(
|
|
31011
31485
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -31013,7 +31487,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31013
31487
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
31014
31488
|
lte6(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
31015
31489
|
)
|
|
31016
|
-
).groupBy(aiReferralEventsHourly.product).orderBy(desc14(
|
|
31490
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc14(sql11`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
31017
31491
|
const referralProducts = referralProductsRows.map((r) => ({
|
|
31018
31492
|
product: r.product,
|
|
31019
31493
|
arrivals: Number(r.arrivals),
|
|
@@ -31021,8 +31495,8 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31021
31495
|
}));
|
|
31022
31496
|
const topReferralRows = db.select({
|
|
31023
31497
|
path: aiReferralEventsHourly.landingPathNormalized,
|
|
31024
|
-
arrivals:
|
|
31025
|
-
products:
|
|
31498
|
+
arrivals: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
31499
|
+
products: sql11`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
31026
31500
|
}).from(aiReferralEventsHourly).where(
|
|
31027
31501
|
and29(
|
|
31028
31502
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -31030,26 +31504,26 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31030
31504
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
31031
31505
|
lte6(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
31032
31506
|
)
|
|
31033
|
-
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc14(
|
|
31507
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc14(sql11`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
31034
31508
|
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
31035
31509
|
path: r.path,
|
|
31036
31510
|
arrivals: Number(r.arrivals),
|
|
31037
31511
|
distinctProducts: Number(r.products)
|
|
31038
31512
|
}));
|
|
31039
31513
|
const crawlerTrendRows = db.select({
|
|
31040
|
-
date:
|
|
31041
|
-
verifiedHits:
|
|
31042
|
-
unverifiedHits:
|
|
31514
|
+
date: sql11`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
31515
|
+
verifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
31516
|
+
unverifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`
|
|
31043
31517
|
}).from(crawlerEventsHourly).where(
|
|
31044
31518
|
and29(
|
|
31045
31519
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
31046
31520
|
gte8(crawlerEventsHourly.tsHour, trendStart),
|
|
31047
31521
|
lte6(crawlerEventsHourly.tsHour, headlineEnd)
|
|
31048
31522
|
)
|
|
31049
|
-
).groupBy(
|
|
31523
|
+
).groupBy(sql11`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
31050
31524
|
const referralTrendRows = db.select({
|
|
31051
|
-
date:
|
|
31052
|
-
hits:
|
|
31525
|
+
date: sql11`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
31526
|
+
hits: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
31053
31527
|
}).from(aiReferralEventsHourly).where(
|
|
31054
31528
|
and29(
|
|
31055
31529
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -31057,17 +31531,17 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31057
31531
|
gte8(aiReferralEventsHourly.tsHour, trendStart),
|
|
31058
31532
|
lte6(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
31059
31533
|
)
|
|
31060
|
-
).groupBy(
|
|
31534
|
+
).groupBy(sql11`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
31061
31535
|
const userFetchTrendRows = db.select({
|
|
31062
|
-
date:
|
|
31063
|
-
hits:
|
|
31536
|
+
date: sql11`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`,
|
|
31537
|
+
hits: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
31064
31538
|
}).from(aiUserFetchEventsHourly).where(
|
|
31065
31539
|
and29(
|
|
31066
31540
|
eq33(aiUserFetchEventsHourly.projectId, projectId),
|
|
31067
31541
|
gte8(aiUserFetchEventsHourly.tsHour, trendStart),
|
|
31068
31542
|
lte6(aiUserFetchEventsHourly.tsHour, headlineEnd)
|
|
31069
31543
|
)
|
|
31070
|
-
).groupBy(
|
|
31544
|
+
).groupBy(sql11`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`).all();
|
|
31071
31545
|
const emptyTrendEntry = () => ({ verifiedCrawlerHits: 0, unverifiedCrawlerHits: 0, userFetchHits: 0, referralArrivals: 0 });
|
|
31072
31546
|
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
31073
31547
|
for (const r of crawlerTrendRows) {
|
|
@@ -31218,7 +31692,7 @@ function buildInsightList(db, projectId, locationFilter) {
|
|
|
31218
31692
|
and29(
|
|
31219
31693
|
eq33(runs.projectId, projectId),
|
|
31220
31694
|
eq33(runs.kind, RunKinds["answer-visibility"]),
|
|
31221
|
-
|
|
31695
|
+
or5(eq33(runs.status, RunStatuses.completed), eq33(runs.status, RunStatuses.partial)),
|
|
31222
31696
|
notProbeRun()
|
|
31223
31697
|
)
|
|
31224
31698
|
).orderBy(desc14(runs.createdAt)).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter).slice(0, INSIGHT_LOOKBACK_RUNS).map((r) => r.id);
|
|
@@ -32100,10 +32574,10 @@ async function reportRoutes(app) {
|
|
|
32100
32574
|
}
|
|
32101
32575
|
|
|
32102
32576
|
// ../api-routes/src/organic-evidence.ts
|
|
32103
|
-
import { and as and31, desc as desc15, eq as eq35, gte as gte10, lte as lte8, or as
|
|
32577
|
+
import { and as and31, desc as desc15, eq as eq35, gte as gte10, lte as lte8, or as or7, sql as sql13 } from "drizzle-orm";
|
|
32104
32578
|
|
|
32105
32579
|
// ../api-routes/src/ga-measurement-analysis.ts
|
|
32106
|
-
import { and as and30, eq as eq34, gte as gte9, lte as lte7, or as
|
|
32580
|
+
import { and as and30, eq as eq34, gte as gte9, lte as lte7, or as or6, sql as sql12 } from "drizzle-orm";
|
|
32107
32581
|
var UNKNOWN_LANDING_PAGE = "(not set)";
|
|
32108
32582
|
function addDays(date, offset) {
|
|
32109
32583
|
const value = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
@@ -32275,24 +32749,24 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32275
32749
|
const scopedConditions = (hostColumn, normalizedPathColumn, rawPathColumn) => {
|
|
32276
32750
|
const conditions = [];
|
|
32277
32751
|
if (parsedHostScope.data === "marketing") {
|
|
32278
|
-
const lowerHost =
|
|
32279
|
-
const normalizedHost2 =
|
|
32280
|
-
conditions.push(
|
|
32281
|
-
|
|
32282
|
-
|
|
32752
|
+
const lowerHost = sql12`lower(${hostColumn})`;
|
|
32753
|
+
const normalizedHost2 = sql12`case when ${lowerHost} like 'www.%' then substr(${lowerHost}, 5) else ${lowerHost} end`;
|
|
32754
|
+
conditions.push(or6(...marketingHosts.flatMap((host) => [
|
|
32755
|
+
sql12`${normalizedHost2} = ${host}`,
|
|
32756
|
+
sql12`${normalizedHost2} like ${`%.${host}`}`
|
|
32283
32757
|
])));
|
|
32284
32758
|
}
|
|
32285
32759
|
if (pathPrefix && pathPrefix !== "/") {
|
|
32286
|
-
const rawPath =
|
|
32287
|
-
const landingPath =
|
|
32288
|
-
conditions.push(
|
|
32760
|
+
const rawPath = sql12`case when instr(${rawPathColumn}, '?') = 0 then ${rawPathColumn} else substr(${rawPathColumn}, 1, instr(${rawPathColumn}, '?') - 1) end`;
|
|
32761
|
+
const landingPath = sql12`case when trim(coalesce(${normalizedPathColumn}, '')) = '' then ${rawPath} else ${normalizedPathColumn} end`;
|
|
32762
|
+
conditions.push(or6(sql12`${landingPath} = ${pathPrefix}`, sql12`${landingPath} like ${`${pathPrefix}/%`}`));
|
|
32289
32763
|
}
|
|
32290
32764
|
return conditions;
|
|
32291
32765
|
};
|
|
32292
32766
|
const acquisitionScope = scopedConditions(gaAcquisitionDaily.hostName, gaAcquisitionDaily.landingPageNormalized, gaAcquisitionDaily.landingPage);
|
|
32293
32767
|
const leadLandingScope = scopedConditions(gaLeadEventsDaily.hostName, gaLeadEventsDaily.landingPageNormalized, gaLeadEventsDaily.landingPage);
|
|
32294
|
-
const acquisitionAnchor = db.select({ date:
|
|
32295
|
-
const leadAnchor = db.select({ date:
|
|
32768
|
+
const acquisitionAnchor = db.select({ date: sql12`max(${gaAcquisitionDaily.date})` }).from(gaAcquisitionDaily).where(and30(eq34(gaAcquisitionDaily.projectId, project.id), ...acquisitionScope)).get()?.date ?? null;
|
|
32769
|
+
const leadAnchor = db.select({ date: sql12`max(${gaLeadEventsDaily.date})` }).from(gaLeadEventsDaily).where(and30(eq34(gaLeadEventsDaily.projectId, project.id), or6(
|
|
32296
32770
|
eq34(gaLeadEventsDaily.attributionScope, "channel"),
|
|
32297
32771
|
and30(eq34(gaLeadEventsDaily.attributionScope, "landing-page"), ...leadLandingScope)
|
|
32298
32772
|
))).get()?.date ?? null;
|
|
@@ -32301,7 +32775,7 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32301
32775
|
const gaStartDate = gaPeriods[0]?.startDate;
|
|
32302
32776
|
const acquisitionRows = gaStartDate ? db.select().from(gaAcquisitionDaily).where(and30(eq34(gaAcquisitionDaily.projectId, project.id), gte9(gaAcquisitionDaily.date, gaStartDate), lte7(gaAcquisitionDaily.date, gaAnchor))).all() : [];
|
|
32303
32777
|
const leadRows = gaStartDate ? db.select().from(gaLeadEventsDaily).where(and30(eq34(gaLeadEventsDaily.projectId, project.id), gte9(gaLeadEventsDaily.date, gaStartDate), lte7(gaLeadEventsDaily.date, gaAnchor))).all() : [];
|
|
32304
|
-
const engagementAnchor = db.select({ date:
|
|
32778
|
+
const engagementAnchor = db.select({ date: sql12`max(${gaDailyTotals.date})` }).from(gaDailyTotals).where(eq34(gaDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32305
32779
|
const engagementPeriodWindow = engagementAnchor ? buildPeriods(engagementAnchor, days) : [];
|
|
32306
32780
|
const engagementStartDate = engagementPeriodWindow[0]?.startDate;
|
|
32307
32781
|
const engagementRows = engagementStartDate ? db.select().from(gaDailyTotals).where(and30(
|
|
@@ -32310,15 +32784,15 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32310
32784
|
lte7(gaDailyTotals.date, engagementAnchor)
|
|
32311
32785
|
)).all() : [];
|
|
32312
32786
|
const engagementAvailableFrom = db.select({
|
|
32313
|
-
date:
|
|
32787
|
+
date: sql12`min(${gaDailyTotals.date})`
|
|
32314
32788
|
}).from(gaDailyTotals).where(and30(
|
|
32315
32789
|
eq34(gaDailyTotals.projectId, project.id),
|
|
32316
|
-
|
|
32317
|
-
|
|
32318
|
-
|
|
32790
|
+
or6(
|
|
32791
|
+
sql12`${gaDailyTotals.engagementRate} is not null`,
|
|
32792
|
+
sql12`${gaDailyTotals.newUsers} is not null`
|
|
32319
32793
|
)
|
|
32320
32794
|
)).get()?.date ?? null;
|
|
32321
|
-
const gscAnchor = db.select({ date:
|
|
32795
|
+
const gscAnchor = db.select({ date: sql12`max(${gscDailyTotals.date})` }).from(gscDailyTotals).where(eq34(gscDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32322
32796
|
const gscPeriods = gscAnchor ? buildPeriods(gscAnchor, days) : [];
|
|
32323
32797
|
const gscStartDate = gscPeriods[0]?.startDate;
|
|
32324
32798
|
const propertyRows = gscStartDate ? db.select().from(gscDailyTotals).where(and30(eq34(gscDailyTotals.projectId, project.id), gte9(gscDailyTotals.date, gscStartDate), lte7(gscDailyTotals.date, gscAnchor))).all() : [];
|
|
@@ -32645,22 +33119,22 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
32645
33119
|
const nativeStartDate = nativeAcquisition.periods[0]?.startDate;
|
|
32646
33120
|
const nativeEndDate = nativeAcquisition.periods.at(-1)?.endDate;
|
|
32647
33121
|
const gscCoverage = aggregateCoverage(db.select({
|
|
32648
|
-
startDate:
|
|
32649
|
-
endDate:
|
|
32650
|
-
observedDays:
|
|
33122
|
+
startDate: sql13`min(${gscDailyTotals.date})`,
|
|
33123
|
+
endDate: sql13`max(${gscDailyTotals.date})`,
|
|
33124
|
+
observedDays: sql13`count(distinct ${gscDailyTotals.date})`
|
|
32651
33125
|
}).from(gscDailyTotals).where(eq35(gscDailyTotals.projectId, project.id)).get());
|
|
32652
33126
|
const legacyGaCoverage = aggregateCoverage(db.select({
|
|
32653
|
-
startDate:
|
|
32654
|
-
endDate:
|
|
32655
|
-
observedDays:
|
|
33127
|
+
startDate: sql13`min(${gaTrafficSnapshots.date})`,
|
|
33128
|
+
endDate: sql13`max(${gaTrafficSnapshots.date})`,
|
|
33129
|
+
observedDays: sql13`count(distinct ${gaTrafficSnapshots.date})`
|
|
32656
33130
|
}).from(gaTrafficSnapshots).where(eq35(gaTrafficSnapshots.projectId, project.id)).get());
|
|
32657
33131
|
const nativeCoverage = nativeAcquisitionActive && normalizedMarketingHosts.length ? aggregateCoverage(db.select({
|
|
32658
|
-
startDate:
|
|
32659
|
-
endDate:
|
|
32660
|
-
observedDays:
|
|
33132
|
+
startDate: sql13`min(${gaAcquisitionDaily.date})`,
|
|
33133
|
+
endDate: sql13`max(${gaAcquisitionDaily.date})`,
|
|
33134
|
+
observedDays: sql13`count(distinct ${gaAcquisitionDaily.date})`
|
|
32661
33135
|
}).from(gaAcquisitionDaily).where(and31(
|
|
32662
33136
|
eq35(gaAcquisitionDaily.projectId, project.id),
|
|
32663
|
-
|
|
33137
|
+
or7(...normalizedMarketingHosts.map((candidate) => sql13`lower(trim(${gaAcquisitionDaily.hostName})) = ${candidate} or lower(trim(${gaAcquisitionDaily.hostName})) like ${`%.${candidate}`}`))
|
|
32664
33138
|
)).get()) : null;
|
|
32665
33139
|
const gaCoverage = nativeAcquisitionActive ? nativeCoverage : legacyGaCoverage;
|
|
32666
33140
|
const asOfDate = gscCoverage?.endDate ?? gaCoverage?.endDate ?? null;
|
|
@@ -32790,7 +33264,7 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
32790
33264
|
const allCrawlers = db.select().from(crawlerEventsHourly).where(and31(eq35(crawlerEventsHourly.projectId, project.id), gte10(crawlerEventsHourly.tsHour, serverStart), lte8(crawlerEventsHourly.tsHour, serverEnd))).all();
|
|
32791
33265
|
const allFetches = db.select().from(aiUserFetchEventsHourly).where(and31(eq35(aiUserFetchEventsHourly.projectId, project.id), gte10(aiUserFetchEventsHourly.tsHour, serverStart), lte8(aiUserFetchEventsHourly.tsHour, serverEnd))).all();
|
|
32792
33266
|
const allReferrals = db.select().from(aiReferralEventsHourly).where(and31(eq35(aiReferralEventsHourly.projectId, project.id), countableReferralCondition(), gte10(aiReferralEventsHourly.tsHour, serverStart), lte8(aiReferralEventsHourly.tsHour, serverEnd))).all();
|
|
32793
|
-
const [serverCoverageRow] = db.all(
|
|
33267
|
+
const [serverCoverageRow] = db.all(sql13`
|
|
32794
33268
|
select
|
|
32795
33269
|
min(day) as startDate,
|
|
32796
33270
|
max(day) as endDate,
|
|
@@ -33889,15 +34363,15 @@ function renderResultsExportCsv(report) {
|
|
|
33889
34363
|
].map(csvCell).join(","));
|
|
33890
34364
|
return [CSV_COLUMNS.join(","), ...rows].join("\r\n") + "\r\n";
|
|
33891
34365
|
}
|
|
33892
|
-
function filenameFor(projectName,
|
|
34366
|
+
function filenameFor(projectName, format2) {
|
|
33893
34367
|
const safeName = projectName.replace(/[^\w.-]+/gi, "_") || "project";
|
|
33894
34368
|
const date = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
33895
|
-
return `canonry-results-${safeName}-${date}.${
|
|
34369
|
+
return `canonry-results-${safeName}-${date}.${format2}`;
|
|
33896
34370
|
}
|
|
33897
34371
|
async function resultsExportRoutes(app) {
|
|
33898
34372
|
app.get("/projects/:name/results/export", async (request, reply) => {
|
|
33899
34373
|
const project = resolveProject(app.db, request.params.name);
|
|
33900
|
-
const
|
|
34374
|
+
const format2 = parseFormat(request.query.format);
|
|
33901
34375
|
const includeProbes = parseBoolean(request.query.includeProbes, "includeProbes");
|
|
33902
34376
|
const { sinceIso, untilIso } = parseDateBounds(request.query.since, request.query.until);
|
|
33903
34377
|
const conditions = [
|
|
@@ -33989,10 +34463,10 @@ async function resultsExportRoutes(app) {
|
|
|
33989
34463
|
recordCount: records.length,
|
|
33990
34464
|
records
|
|
33991
34465
|
};
|
|
33992
|
-
reply.header("Content-Disposition", `attachment; filename="${filenameFor(project.name,
|
|
34466
|
+
reply.header("Content-Disposition", `attachment; filename="${filenameFor(project.name, format2)}"`);
|
|
33993
34467
|
reply.header("Cache-Control", "private, no-store");
|
|
33994
34468
|
reply.header("X-Content-Type-Options", "nosniff");
|
|
33995
|
-
if (
|
|
34469
|
+
if (format2 === "csv") {
|
|
33996
34470
|
reply.type("text/csv; charset=utf-8");
|
|
33997
34471
|
return reply.send(renderResultsExportCsv(dto));
|
|
33998
34472
|
}
|
|
@@ -34002,7 +34476,7 @@ async function resultsExportRoutes(app) {
|
|
|
34002
34476
|
}
|
|
34003
34477
|
|
|
34004
34478
|
// ../api-routes/src/composites.ts
|
|
34005
|
-
import { eq as eq39, and as and35, desc as desc17, sql as
|
|
34479
|
+
import { eq as eq39, and as and35, desc as desc17, sql as sql14, like as like3, or as or8, inArray as inArray18 } from "drizzle-orm";
|
|
34006
34480
|
var TOP_INSIGHT_LIMIT = 5;
|
|
34007
34481
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
34008
34482
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -34199,23 +34673,23 @@ async function compositeRoutes(app) {
|
|
|
34199
34673
|
}).from(querySnapshots).innerJoin(queries, eq39(querySnapshots.queryId, queries.id)).where(
|
|
34200
34674
|
and35(
|
|
34201
34675
|
eq39(queries.projectId, project.id),
|
|
34202
|
-
|
|
34203
|
-
|
|
34204
|
-
|
|
34205
|
-
|
|
34206
|
-
|
|
34207
|
-
|
|
34676
|
+
or8(
|
|
34677
|
+
sql14`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
34678
|
+
sql14`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
34679
|
+
sql14`${querySnapshots.citedUrls} LIKE ${pattern} ESCAPE '\\'`,
|
|
34680
|
+
sql14`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
34681
|
+
like3(queries.query, pattern)
|
|
34208
34682
|
)
|
|
34209
34683
|
)
|
|
34210
34684
|
).orderBy(desc17(querySnapshots.createdAt)).limit(limit + 1).all());
|
|
34211
34685
|
const insightMatches = app.db.select().from(insights).where(
|
|
34212
34686
|
and35(
|
|
34213
34687
|
eq39(insights.projectId, project.id),
|
|
34214
|
-
|
|
34215
|
-
|
|
34216
|
-
|
|
34217
|
-
|
|
34218
|
-
|
|
34688
|
+
or8(
|
|
34689
|
+
like3(insights.title, pattern),
|
|
34690
|
+
like3(insights.query, pattern),
|
|
34691
|
+
sql14`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
34692
|
+
sql14`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
34219
34693
|
)
|
|
34220
34694
|
)
|
|
34221
34695
|
).orderBy(desc17(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -34438,20 +34912,20 @@ function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
|
|
|
34438
34912
|
const dimensionedRows = app.db.select({
|
|
34439
34913
|
date: gscSearchData.date,
|
|
34440
34914
|
query: gscSearchData.query,
|
|
34441
|
-
impressions:
|
|
34442
|
-
clicks:
|
|
34915
|
+
impressions: sql14`COALESCE(SUM(${gscSearchData.impressions}), 0)`,
|
|
34916
|
+
clicks: sql14`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
34443
34917
|
// Weighted average: SUM(position * impressions) / SUM(impressions).
|
|
34444
34918
|
// NULLIF guards the degenerate impressions=0 case (SQLite returns NULL,
|
|
34445
34919
|
// which the JS coerces to 0 — caught by the impression floor anyway).
|
|
34446
|
-
avgPosition:
|
|
34920
|
+
avgPosition: sql14`COALESCE(SUM(${gscSearchData.position} * ${gscSearchData.impressions}) * 1.0 / NULLIF(SUM(${gscSearchData.impressions}), 0), 0)`
|
|
34447
34921
|
}).from(gscSearchData).where(and35(
|
|
34448
34922
|
eq39(gscSearchData.projectId, projectId),
|
|
34449
|
-
|
|
34923
|
+
sql14`${gscSearchData.date} >= ${cutoff}`,
|
|
34450
34924
|
// Same upper bound as the accurate source below. The merge picks a source
|
|
34451
34925
|
// PER DAY, so an asymmetric range would let a dimensioned-only day past
|
|
34452
34926
|
// the window's end into a basket the other source cannot balance.
|
|
34453
|
-
|
|
34454
|
-
|
|
34927
|
+
sql14`${gscSearchData.date} <= ${suggestionWindow.endDate ?? "9999-12-31"}`,
|
|
34928
|
+
sql14`${gscSearchData.impressions} > 0`
|
|
34455
34929
|
)).groupBy(gscSearchData.date, gscSearchData.query).all();
|
|
34456
34930
|
const merged = mergeGscQueryTotalsWithFallback(
|
|
34457
34931
|
readGscQueryDailyRows(app.db, projectId, cutoff, suggestionWindow.endDate ?? "9999-12-31"),
|
|
@@ -34889,6 +35363,7 @@ var SCHEMA_TABLE = {
|
|
|
34889
35363
|
GtmWorkspaceListResponse: gtmWorkspaceListResponseSchema,
|
|
34890
35364
|
HealthSnapshotDto: healthSnapshotDtoSchema,
|
|
34891
35365
|
IndexingRequestResponseDto: indexingRequestResponseDtoSchema,
|
|
35366
|
+
IntegrationSettingsSummaryDto: integrationSettingsSummaryDtoSchema,
|
|
34892
35367
|
KeywordDto: keywordDtoSchema,
|
|
34893
35368
|
LatestProjectRunDto: latestProjectRunDtoSchema,
|
|
34894
35369
|
LocationContext: locationContextSchema,
|
|
@@ -34969,6 +35444,7 @@ var SCHEMA_TABLE = {
|
|
|
34969
35444
|
ProjectOverviewDto: projectOverviewDtoSchema,
|
|
34970
35445
|
ProjectReportDto: projectReportDtoSchema,
|
|
34971
35446
|
ProjectSearchResponseDto: projectSearchResponseSchema,
|
|
35447
|
+
ProviderSummaryEntryDto: providerSummaryEntryDtoSchema,
|
|
34972
35448
|
QueryDto: queryDtoSchema,
|
|
34973
35449
|
ResultsExportDto: resultsExportDtoSchema,
|
|
34974
35450
|
RunDetailDto: runDetailDtoSchema,
|
|
@@ -34998,8 +35474,12 @@ var SCHEMA_TABLE = {
|
|
|
34998
35474
|
SnapshotDiffResponse: snapshotDiffResponseSchema,
|
|
34999
35475
|
SnapshotListResponse: snapshotListResponseSchema,
|
|
35000
35476
|
SnapshotReportDto: snapshotReportSchema,
|
|
35477
|
+
// The API accepts legacy phrases, but the public contract advertises queries.
|
|
35478
|
+
SnapshotRequest: z.object(snapshotRequestInputSchema.shape).omit({ phrases: true }),
|
|
35001
35479
|
SourceBreakdownDto: sourceBreakdownDtoSchema,
|
|
35002
35480
|
TelemetryEventAcceptedDto: telemetryEventAcceptedDtoSchema,
|
|
35481
|
+
TelemetryStatusDto: telemetryStatusDtoSchema,
|
|
35482
|
+
OperationalLogListDto: operationalLogListDtoSchema,
|
|
35003
35483
|
CloudflareWorkerIngestRequest: cloudflareWorkerIngestRequestSchema,
|
|
35004
35484
|
CloudflareWorkerIngestResponse: cloudflareWorkerIngestResponseSchema,
|
|
35005
35485
|
TrafficBackfillResponse: trafficBackfillResponseSchema,
|
|
@@ -37424,6 +37904,7 @@ var routeCatalog = [
|
|
|
37424
37904
|
method: "put",
|
|
37425
37905
|
path: "/api/v1/settings/providers/{name}",
|
|
37426
37906
|
summary: "Update provider settings",
|
|
37907
|
+
description: "Requires settings.write. Already configured providers accept model or quota changes without resubmitting credentials. An unconfigured provider still requires its connection credentials.",
|
|
37427
37908
|
tags: ["settings"],
|
|
37428
37909
|
parameters: [providerNameParameter],
|
|
37429
37910
|
requestBody: {
|
|
@@ -37443,9 +37924,9 @@ var routeCatalog = [
|
|
|
37443
37924
|
}
|
|
37444
37925
|
},
|
|
37445
37926
|
responses: {
|
|
37446
|
-
|
|
37447
|
-
200: rawJsonResponse("Provider updated.", looseObjectSchema),
|
|
37927
|
+
200: jsonResponse("Provider updated.", "ProviderSummaryEntryDto"),
|
|
37448
37928
|
400: errorResponse("Invalid provider settings."),
|
|
37929
|
+
403: errorResponse("The credential lacks settings.write."),
|
|
37449
37930
|
501: errorResponse("Provider updates are not supported.")
|
|
37450
37931
|
}
|
|
37451
37932
|
},
|
|
@@ -37470,8 +37951,7 @@ var routeCatalog = [
|
|
|
37470
37951
|
}
|
|
37471
37952
|
},
|
|
37472
37953
|
responses: {
|
|
37473
|
-
|
|
37474
|
-
200: rawJsonResponse("Google settings updated.", looseObjectSchema),
|
|
37954
|
+
200: jsonResponse("Google settings updated.", "IntegrationSettingsSummaryDto"),
|
|
37475
37955
|
400: errorResponse("Invalid Google settings."),
|
|
37476
37956
|
501: errorResponse("Google settings updates are not supported.")
|
|
37477
37957
|
}
|
|
@@ -37650,21 +38130,13 @@ var routeCatalog = [
|
|
|
37650
38130
|
method: "post",
|
|
37651
38131
|
path: "/api/v1/snapshot",
|
|
37652
38132
|
summary: "Generate a one-shot AI perception snapshot",
|
|
38133
|
+
description: "Runs configured providers without creating a project. Optional providers and providerMode constrain both answer queries and analysis; omitted selection uses all configured providers. Browser-only selection requires manual queries. Spends provider quota.",
|
|
37653
38134
|
tags: ["snapshot"],
|
|
37654
38135
|
requestBody: {
|
|
37655
38136
|
required: true,
|
|
37656
38137
|
content: {
|
|
37657
38138
|
"application/json": {
|
|
37658
|
-
schema: {
|
|
37659
|
-
type: "object",
|
|
37660
|
-
required: ["companyName", "domain"],
|
|
37661
|
-
properties: {
|
|
37662
|
-
companyName: stringSchema,
|
|
37663
|
-
domain: stringSchema,
|
|
37664
|
-
queries: stringArraySchema,
|
|
37665
|
-
competitors: stringArraySchema
|
|
37666
|
-
}
|
|
37667
|
-
}
|
|
38139
|
+
schema: { $ref: "#/components/schemas/SnapshotRequest" }
|
|
37668
38140
|
}
|
|
37669
38141
|
}
|
|
37670
38142
|
},
|
|
@@ -37870,14 +38342,34 @@ var routeCatalog = [
|
|
|
37870
38342
|
502: errorResponse("Notification delivery failed.")
|
|
37871
38343
|
}
|
|
37872
38344
|
},
|
|
38345
|
+
{
|
|
38346
|
+
method: "get",
|
|
38347
|
+
path: "/api/v1/operations/logs",
|
|
38348
|
+
summary: "Read bounded, redacted runtime logs",
|
|
38349
|
+
description: "Requires logs.read (or wildcard), an admin role for user sessions, and an instance-wide credential. Includes application and HTTP diagnostics, separate from audit history. File-backed hosts retain logs across restarts; inspect retention, retentionPolicy, dropped, and captureErrors. Filters apply before pagination and must remain unchanged when resuming a cursor. Cursors expire on retention eviction.",
|
|
38350
|
+
tags: ["operations"],
|
|
38351
|
+
parameters: [
|
|
38352
|
+
{ name: "limit", in: "query", required: false, description: "Maximum entries (default 100).", schema: { type: "integer", minimum: 1, maximum: 200, default: 100 } },
|
|
38353
|
+
{ name: "cursor", in: "query", required: false, description: "Opaque cursor bound to the log store and query filters.", schema: { type: "string", minLength: 1, maxLength: 512 } },
|
|
38354
|
+
{ name: "level", in: "query", required: false, description: "Exact log level.", schema: { type: "string", enum: ["trace", "debug", "info", "warn", "error", "fatal"] } },
|
|
38355
|
+
...["module", "runId", "projectId", "actor", "requestId"].map((name) => ({ name, in: "query", required: false, description: `Exact ${name} filter (does not grant project-scoped log access).`, schema: { type: "string", minLength: 1, maxLength: name === "actor" ? 512 : 256 } })),
|
|
38356
|
+
...["since", "until"].map((name) => ({ name, in: "query", required: false, description: `Inclusive ${name} event timestamp.`, schema: { type: "string", format: "date-time" } }))
|
|
38357
|
+
],
|
|
38358
|
+
responses: {
|
|
38359
|
+
200: jsonResponse("Runtime log page with retention and capture-loss metadata.", "OperationalLogListDto"),
|
|
38360
|
+
400: errorResponse("Invalid filters or expired cursor."),
|
|
38361
|
+
401: errorResponse("Authentication required."),
|
|
38362
|
+
403: errorResponse("Instance logs.read permission required."),
|
|
38363
|
+
501: errorResponse("This deployment does not provide operational logs.")
|
|
38364
|
+
}
|
|
38365
|
+
},
|
|
37873
38366
|
{
|
|
37874
38367
|
method: "get",
|
|
37875
38368
|
path: "/api/v1/telemetry",
|
|
37876
38369
|
summary: "Get telemetry status",
|
|
37877
38370
|
tags: ["telemetry"],
|
|
37878
38371
|
responses: {
|
|
37879
|
-
|
|
37880
|
-
200: rawJsonResponse("Telemetry status returned.", looseObjectSchema),
|
|
38372
|
+
200: jsonResponse("Telemetry status returned.", "TelemetryStatusDto"),
|
|
37881
38373
|
501: errorResponse("Telemetry status is not available.")
|
|
37882
38374
|
}
|
|
37883
38375
|
},
|
|
@@ -37885,6 +38377,7 @@ var routeCatalog = [
|
|
|
37885
38377
|
method: "put",
|
|
37886
38378
|
path: "/api/v1/telemetry",
|
|
37887
38379
|
summary: "Update telemetry status",
|
|
38380
|
+
description: "Requires settings.write. The response reports the configured preference and effective state; deployment/environment opt-outs take precedence.",
|
|
37888
38381
|
tags: ["telemetry"],
|
|
37889
38382
|
requestBody: {
|
|
37890
38383
|
required: true,
|
|
@@ -37901,9 +38394,9 @@ var routeCatalog = [
|
|
|
37901
38394
|
}
|
|
37902
38395
|
},
|
|
37903
38396
|
responses: {
|
|
37904
|
-
|
|
37905
|
-
200: rawJsonResponse("Telemetry updated.", looseObjectSchema),
|
|
38397
|
+
200: jsonResponse("Telemetry updated.", "TelemetryStatusDto"),
|
|
37906
38398
|
400: errorResponse("Invalid telemetry request."),
|
|
38399
|
+
403: errorResponse("The credential lacks settings.write."),
|
|
37907
38400
|
501: errorResponse("Telemetry configuration is not available.")
|
|
37908
38401
|
}
|
|
37909
38402
|
},
|
|
@@ -42509,6 +43002,7 @@ async function snapshotRoutes(app, opts) {
|
|
|
42509
43002
|
try {
|
|
42510
43003
|
return await opts.onSnapshotRequested(input);
|
|
42511
43004
|
} catch (err) {
|
|
43005
|
+
if (err instanceof AppError) throw err;
|
|
42512
43006
|
request.log.error({ err }, "Snapshot report generation failed");
|
|
42513
43007
|
throw internalError(err instanceof Error ? err.message : "Failed to generate snapshot report");
|
|
42514
43008
|
}
|
|
@@ -42521,13 +43015,10 @@ async function telemetryRoutes(app, opts) {
|
|
|
42521
43015
|
if (!opts.getTelemetryStatus) {
|
|
42522
43016
|
throw notImplemented("Telemetry status is not available in this deployment");
|
|
42523
43017
|
}
|
|
42524
|
-
|
|
42525
|
-
return {
|
|
42526
|
-
enabled: status.enabled,
|
|
42527
|
-
anonymousId: status.anonymousId ? status.anonymousId.slice(0, 8) + "..." : void 0
|
|
42528
|
-
};
|
|
43018
|
+
return normalizeTelemetryStatus(opts.getTelemetryStatus());
|
|
42529
43019
|
});
|
|
42530
43020
|
app.put("/telemetry", async (request) => {
|
|
43021
|
+
requireScope(request, SETTINGS_WRITE_SCOPE);
|
|
42531
43022
|
if (!opts.setTelemetryEnabled) {
|
|
42532
43023
|
throw notImplemented("Telemetry configuration is not available in this deployment");
|
|
42533
43024
|
}
|
|
@@ -42537,10 +43028,24 @@ async function telemetryRoutes(app, opts) {
|
|
|
42537
43028
|
}
|
|
42538
43029
|
opts.setTelemetryEnabled(enabled);
|
|
42539
43030
|
const status = opts.getTelemetryStatus?.();
|
|
42540
|
-
|
|
42541
|
-
|
|
42542
|
-
|
|
42543
|
-
|
|
43031
|
+
const effective = normalizeTelemetryStatus(status ?? { enabled });
|
|
43032
|
+
if (app.hasDecorator("db")) {
|
|
43033
|
+
try {
|
|
43034
|
+
writeAuditLog(app.db, auditFromRequest(request, {
|
|
43035
|
+
actor: "api",
|
|
43036
|
+
action: "telemetry.updated",
|
|
43037
|
+
entityType: "telemetry",
|
|
43038
|
+
diff: {
|
|
43039
|
+
target: effective.target,
|
|
43040
|
+
configuredEnabled: effective.configuredEnabled,
|
|
43041
|
+
reason: effective.reason
|
|
43042
|
+
}
|
|
43043
|
+
}));
|
|
43044
|
+
} catch {
|
|
43045
|
+
app.log.warn({ action: "telemetry.updated" }, "Telemetry setting audit write failed");
|
|
43046
|
+
}
|
|
43047
|
+
}
|
|
43048
|
+
return effective;
|
|
42544
43049
|
});
|
|
42545
43050
|
app.post("/telemetry/onboarding", async (request, reply) => {
|
|
42546
43051
|
const parsed = onboardingTelemetryEventSchema.safeParse(request.body);
|
|
@@ -42557,6 +43062,25 @@ async function telemetryRoutes(app, opts) {
|
|
|
42557
43062
|
});
|
|
42558
43063
|
}
|
|
42559
43064
|
|
|
43065
|
+
// ../api-routes/src/operational-logs.ts
|
|
43066
|
+
async function operationalLogsRoutes(app, opts = {}) {
|
|
43067
|
+
app.get("/operations/logs", async (request) => {
|
|
43068
|
+
requireAdminSession(request);
|
|
43069
|
+
requireScope(request, "logs.read");
|
|
43070
|
+
if (request.apiKey?.projectId) {
|
|
43071
|
+
throw forbidden("This API key is scoped to one project and cannot read instance operational diagnostics.");
|
|
43072
|
+
}
|
|
43073
|
+
const parsed = logQuerySchema.safeParse(request.query);
|
|
43074
|
+
if (!parsed.success) {
|
|
43075
|
+
throw validationError("Invalid operational logs query.", { issues: parsed.error.issues });
|
|
43076
|
+
}
|
|
43077
|
+
if (!opts.listOperationalLogs) {
|
|
43078
|
+
throw notImplemented("Operational logs are not available in this deployment.");
|
|
43079
|
+
}
|
|
43080
|
+
return operationalLogListDtoSchema.parse(await opts.listOperationalLogs(parsed.data));
|
|
43081
|
+
});
|
|
43082
|
+
}
|
|
43083
|
+
|
|
42560
43084
|
// ../api-routes/src/schedules.ts
|
|
42561
43085
|
import crypto26 from "crypto";
|
|
42562
43086
|
import { and as and36, asc as asc7, eq as eq42 } from "drizzle-orm";
|
|
@@ -43099,7 +43623,7 @@ function formatNotification(row) {
|
|
|
43099
43623
|
|
|
43100
43624
|
// ../api-routes/src/google.ts
|
|
43101
43625
|
import crypto32 from "crypto";
|
|
43102
|
-
import { eq as eq44, and as and37, desc as desc19, sql as
|
|
43626
|
+
import { eq as eq44, and as and37, desc as desc19, sql as sql15, inArray as inArray19 } from "drizzle-orm";
|
|
43103
43627
|
|
|
43104
43628
|
// ../api-routes/src/gsc-period-comparison.ts
|
|
43105
43629
|
var DAY_MS = 864e5;
|
|
@@ -45528,11 +46052,11 @@ async function googleRoutes(app, opts) {
|
|
|
45528
46052
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
45529
46053
|
const effectiveEndDate = endDate ?? (daysParam === null ? void 0 : resolvedWindow.endDate ?? void 0);
|
|
45530
46054
|
const conditions = [eq44(gscSearchData.projectId, project.id)];
|
|
45531
|
-
if (startDate) conditions.push(
|
|
45532
|
-
else if (cutoffDate) conditions.push(
|
|
45533
|
-
if (effectiveEndDate) conditions.push(
|
|
45534
|
-
if (query) conditions.push(
|
|
45535
|
-
if (page3) conditions.push(
|
|
46055
|
+
if (startDate) conditions.push(sql15`${gscSearchData.date} >= ${startDate}`);
|
|
46056
|
+
else if (cutoffDate) conditions.push(sql15`${gscSearchData.date} >= ${cutoffDate}`);
|
|
46057
|
+
if (effectiveEndDate) conditions.push(sql15`${gscSearchData.date} <= ${effectiveEndDate}`);
|
|
46058
|
+
if (query) conditions.push(sql15`${gscSearchData.query} LIKE ${"%" + escapeLikePattern(query) + "%"} ESCAPE '\\'`);
|
|
46059
|
+
if (page3) conditions.push(sql15`${gscSearchData.page} LIKE ${"%" + escapeLikePattern(page3) + "%"} ESCAPE '\\'`);
|
|
45536
46060
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
45537
46061
|
const offsetVal = Math.max(parseInt(offset ?? "0", 10) || 0, 0);
|
|
45538
46062
|
const orderColumns = {
|
|
@@ -45541,8 +46065,8 @@ async function googleRoutes(app, opts) {
|
|
|
45541
46065
|
date: [desc19(gscSearchData.date), desc19(gscSearchData.clicks), gscSearchData.query]
|
|
45542
46066
|
}[orderBy];
|
|
45543
46067
|
const rows = app.db.select().from(gscSearchData).where(and37(...conditions)).orderBy(...orderColumns).limit(limitVal).offset(offsetVal).all();
|
|
45544
|
-
const totalMatching = app.db.select({ total:
|
|
45545
|
-
const latestAvailableDate = app.db.select({ latest:
|
|
46068
|
+
const totalMatching = app.db.select({ total: sql15`COUNT(*)` }).from(gscSearchData).where(and37(...conditions)).get()?.total ?? 0;
|
|
46069
|
+
const latestAvailableDate = app.db.select({ latest: sql15`MAX(${gscSearchData.date})` }).from(gscSearchData).where(eq44(gscSearchData.projectId, project.id)).get()?.latest ?? null;
|
|
45546
46070
|
return {
|
|
45547
46071
|
rows: rows.map((r) => ({
|
|
45548
46072
|
date: r.date,
|
|
@@ -45586,12 +46110,12 @@ async function googleRoutes(app, opts) {
|
|
|
45586
46110
|
const fetchStart = priorWindowIsSynced ? priorPeriodStart : windowStart;
|
|
45587
46111
|
const dailyTotals = readGscDailyTotals(app.db, project.id, fetchStart, windowEnd);
|
|
45588
46112
|
const conditions = [eq44(gscSearchData.projectId, project.id)];
|
|
45589
|
-
if (fetchStart) conditions.push(
|
|
45590
|
-
if (endDate) conditions.push(
|
|
46113
|
+
if (fetchStart) conditions.push(sql15`${gscSearchData.date} >= ${fetchStart}`);
|
|
46114
|
+
if (endDate) conditions.push(sql15`${gscSearchData.date} <= ${endDate}`);
|
|
45591
46115
|
const dimensionedRows = app.db.select({
|
|
45592
46116
|
date: gscSearchData.date,
|
|
45593
|
-
clicks:
|
|
45594
|
-
impressions:
|
|
46117
|
+
clicks: sql15`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
46118
|
+
impressions: sql15`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
45595
46119
|
}).from(gscSearchData).where(and37(...conditions)).groupBy(gscSearchData.date).orderBy(gscSearchData.date).all();
|
|
45596
46120
|
const propertyDates = new Set(dailyTotals.map((d) => d.date));
|
|
45597
46121
|
const comparisonDaily = mergeGscDailyTotalsWithFallback(
|
|
@@ -45689,23 +46213,23 @@ async function googleRoutes(app, opts) {
|
|
|
45689
46213
|
);
|
|
45690
46214
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
45691
46215
|
const conditions = [eq44(gscSearchData.projectId, project.id)];
|
|
45692
|
-
if (startDate) conditions.push(
|
|
45693
|
-
else if (cutoffDate) conditions.push(
|
|
45694
|
-
if (endDate) conditions.push(
|
|
46216
|
+
if (startDate) conditions.push(sql15`${gscSearchData.date} >= ${startDate}`);
|
|
46217
|
+
else if (cutoffDate) conditions.push(sql15`${gscSearchData.date} >= ${cutoffDate}`);
|
|
46218
|
+
if (endDate) conditions.push(sql15`${gscSearchData.date} <= ${endDate}`);
|
|
45695
46219
|
const limitVal = Math.max(parseInt(limit ?? "50", 10) || 0, 1);
|
|
45696
46220
|
const rows = app.db.select({
|
|
45697
46221
|
page: gscSearchData.page,
|
|
45698
|
-
clicks:
|
|
45699
|
-
impressions:
|
|
45700
|
-
}).from(gscSearchData).where(and37(...conditions)).groupBy(gscSearchData.page).orderBy(desc19(
|
|
46222
|
+
clicks: sql15`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
46223
|
+
impressions: sql15`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
46224
|
+
}).from(gscSearchData).where(and37(...conditions)).groupBy(gscSearchData.page).orderBy(desc19(sql15`SUM(${gscSearchData.clicks})`), desc19(sql15`SUM(${gscSearchData.impressions})`)).limit(limitVal).all();
|
|
45701
46225
|
const windowStart = startDate ?? cutoffDate ?? "";
|
|
45702
46226
|
const windowEnd = endDate ?? resolvedWindow.endDate ?? "9999-12-31";
|
|
45703
46227
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
45704
46228
|
const totalClicks = dailyTotals.reduce((sum, d) => sum + d.clicks, 0);
|
|
45705
46229
|
const totalImpressions = dailyTotals.reduce((sum, d) => sum + d.impressions, 0);
|
|
45706
46230
|
const rankedSpan = app.db.select({
|
|
45707
|
-
first:
|
|
45708
|
-
last:
|
|
46231
|
+
first: sql15`MIN(${gscSearchData.date})`,
|
|
46232
|
+
last: sql15`MAX(${gscSearchData.date})`
|
|
45709
46233
|
}).from(gscSearchData).where(and37(...conditions)).get();
|
|
45710
46234
|
const coveredFrom = dailyTotals.length > 0 ? dailyTotals[0].date : null;
|
|
45711
46235
|
const coveredThrough = dailyTotals.length > 0 ? dailyTotals[dailyTotals.length - 1].date : null;
|
|
@@ -46674,7 +47198,7 @@ async function googleRoutes(app, opts) {
|
|
|
46674
47198
|
|
|
46675
47199
|
// ../api-routes/src/google-marketing.ts
|
|
46676
47200
|
import crypto33 from "crypto";
|
|
46677
|
-
import { and as and38, desc as desc20, eq as eq45, isNull as isNull6, lt as lt6, ne as ne6, or as
|
|
47201
|
+
import { and as and38, desc as desc20, eq as eq45, isNull as isNull6, lt as lt6, ne as ne6, or as or9, sql as sql16 } from "drizzle-orm";
|
|
46678
47202
|
import { z as z4 } from "zod";
|
|
46679
47203
|
var snapshotPageQuerySchema = z4.object({
|
|
46680
47204
|
limit: z4.coerce.number().int().min(1).max(GOOGLE_MARKETING_STORED_SNAPSHOT_PAGE_MAX).optional(),
|
|
@@ -47625,7 +48149,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47625
48149
|
selectedCustomerCurrencyCode: null,
|
|
47626
48150
|
selectedCustomerTimeZone: null,
|
|
47627
48151
|
selectedCustomerStatus: null,
|
|
47628
|
-
selectionGeneration:
|
|
48152
|
+
selectionGeneration: sql16`${googleAdsConnections.selectionGeneration} + 1`,
|
|
47629
48153
|
scopes: tokens.scopes,
|
|
47630
48154
|
lastValidatedAt: now,
|
|
47631
48155
|
lastCustomerSnapshotId: null,
|
|
@@ -47656,7 +48180,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47656
48180
|
selectedContainerPublicId: null,
|
|
47657
48181
|
selectedWorkspaceId: null,
|
|
47658
48182
|
selectedWorkspaceName: null,
|
|
47659
|
-
selectionGeneration:
|
|
48183
|
+
selectionGeneration: sql16`${gtmConnections.selectionGeneration} + 1`,
|
|
47660
48184
|
scopes: tokens.scopes,
|
|
47661
48185
|
lastValidatedAt: now,
|
|
47662
48186
|
lastSnapshotAt: null,
|
|
@@ -47807,7 +48331,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47807
48331
|
selectedCustomerStatus: customerChanged ? null : existing.selectedCustomerStatus,
|
|
47808
48332
|
// A same-value write still starts a new selection generation. Do not
|
|
47809
48333
|
// let timestamp equality reuse append-only evidence from before it.
|
|
47810
|
-
selectionGeneration:
|
|
48334
|
+
selectionGeneration: sql16`${googleAdsConnections.selectionGeneration} + 1`,
|
|
47811
48335
|
lastValidatedAt: now,
|
|
47812
48336
|
lastCustomerSnapshotId: null,
|
|
47813
48337
|
lastInventorySnapshotAt: null,
|
|
@@ -47851,7 +48375,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47851
48375
|
selectedContainerPublicId: containerChanged ? null : existing.selectedContainerPublicId,
|
|
47852
48376
|
selectedWorkspaceId,
|
|
47853
48377
|
selectedWorkspaceName: workspaceChanged ? null : existing.selectedWorkspaceName,
|
|
47854
|
-
selectionGeneration:
|
|
48378
|
+
selectionGeneration: sql16`${gtmConnections.selectionGeneration} + 1`,
|
|
47855
48379
|
lastValidatedAt: now,
|
|
47856
48380
|
// Every selection write starts a new evidence generation, including a
|
|
47857
48381
|
// same-value reselection. The timestamp alone cannot order a prior
|
|
@@ -47889,7 +48413,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47889
48413
|
const existing = tx.select().from(runs).where(and38(
|
|
47890
48414
|
eq45(runs.projectId, project.id),
|
|
47891
48415
|
eq45(runs.kind, RunKinds["google-ads-sync"]),
|
|
47892
|
-
|
|
48416
|
+
or9(eq45(runs.status, RunStatuses.queued), eq45(runs.status, RunStatuses.running))
|
|
47893
48417
|
)).orderBy(desc20(runs.createdAt)).get();
|
|
47894
48418
|
if (existing) {
|
|
47895
48419
|
run = existing;
|
|
@@ -47925,7 +48449,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47925
48449
|
const existing = tx.select().from(runs).where(and38(
|
|
47926
48450
|
eq45(runs.projectId, project.id),
|
|
47927
48451
|
eq45(runs.kind, RunKinds["gtm-sync"]),
|
|
47928
|
-
|
|
48452
|
+
or9(eq45(runs.status, RunStatuses.queued), eq45(runs.status, RunStatuses.running))
|
|
47929
48453
|
)).orderBy(desc20(runs.createdAt)).get();
|
|
47930
48454
|
if (existing) {
|
|
47931
48455
|
run = existing;
|
|
@@ -47955,7 +48479,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47955
48479
|
const cursor = decodeSnapshotCursor(query.cursor);
|
|
47956
48480
|
const conditions = [eq45(googleAdsRawSnapshots.projectId, project.id)];
|
|
47957
48481
|
if (cursor) {
|
|
47958
|
-
conditions.push(
|
|
48482
|
+
conditions.push(or9(
|
|
47959
48483
|
lt6(googleAdsRawSnapshots.capturedAt, cursor.capturedAt),
|
|
47960
48484
|
and38(eq45(googleAdsRawSnapshots.capturedAt, cursor.capturedAt), lt6(googleAdsRawSnapshots.id, cursor.id))
|
|
47961
48485
|
));
|
|
@@ -47963,7 +48487,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47963
48487
|
const limit = query.limit ?? 50;
|
|
47964
48488
|
const rows = app.db.select().from(googleAdsRawSnapshots).where(and38(...conditions)).orderBy(desc20(googleAdsRawSnapshots.capturedAt), desc20(googleAdsRawSnapshots.id)).limit(limit + 1).all();
|
|
47965
48489
|
const page3 = rows.slice(0, limit);
|
|
47966
|
-
const total = app.db.select({ total:
|
|
48490
|
+
const total = app.db.select({ total: sql16`COUNT(*)`.mapWith(Number).as("total") }).from(googleAdsRawSnapshots).where(eq45(googleAdsRawSnapshots.projectId, project.id)).get();
|
|
47967
48491
|
return {
|
|
47968
48492
|
snapshots: page3.map(toGoogleAdsSnapshotMetadata),
|
|
47969
48493
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -47987,7 +48511,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47987
48511
|
const cursor = decodeSnapshotCursor(query.cursor);
|
|
47988
48512
|
const conditions = [eq45(gtmRawSnapshots.projectId, project.id)];
|
|
47989
48513
|
if (cursor) {
|
|
47990
|
-
conditions.push(
|
|
48514
|
+
conditions.push(or9(
|
|
47991
48515
|
lt6(gtmRawSnapshots.capturedAt, cursor.capturedAt),
|
|
47992
48516
|
and38(eq45(gtmRawSnapshots.capturedAt, cursor.capturedAt), lt6(gtmRawSnapshots.id, cursor.id))
|
|
47993
48517
|
));
|
|
@@ -47995,7 +48519,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47995
48519
|
const limit = query.limit ?? 50;
|
|
47996
48520
|
const rows = app.db.select().from(gtmRawSnapshots).where(and38(...conditions)).orderBy(desc20(gtmRawSnapshots.capturedAt), desc20(gtmRawSnapshots.id)).limit(limit + 1).all();
|
|
47997
48521
|
const page3 = rows.slice(0, limit);
|
|
47998
|
-
const total = app.db.select({ total:
|
|
48522
|
+
const total = app.db.select({ total: sql16`COUNT(*)`.mapWith(Number).as("total") }).from(gtmRawSnapshots).where(eq45(gtmRawSnapshots.projectId, project.id)).get();
|
|
47999
48523
|
return {
|
|
48000
48524
|
snapshots: page3.map(toGtmSnapshotMetadata),
|
|
48001
48525
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -48030,7 +48554,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
48030
48554
|
selectedCustomerCurrencyCode: null,
|
|
48031
48555
|
selectedCustomerTimeZone: null,
|
|
48032
48556
|
selectedCustomerStatus: null,
|
|
48033
|
-
selectionGeneration:
|
|
48557
|
+
selectionGeneration: sql16`${googleAdsConnections.selectionGeneration} + 1`,
|
|
48034
48558
|
scopes: [],
|
|
48035
48559
|
lastValidatedAt: null,
|
|
48036
48560
|
lastCustomerSnapshotId: null,
|
|
@@ -48069,7 +48593,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
48069
48593
|
selectedContainerPublicId: null,
|
|
48070
48594
|
selectedWorkspaceId: null,
|
|
48071
48595
|
selectedWorkspaceName: null,
|
|
48072
|
-
selectionGeneration:
|
|
48596
|
+
selectionGeneration: sql16`${gtmConnections.selectionGeneration} + 1`,
|
|
48073
48597
|
scopes: [],
|
|
48074
48598
|
lastValidatedAt: null,
|
|
48075
48599
|
lastSnapshotAt: null,
|
|
@@ -48264,7 +48788,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
48264
48788
|
|
|
48265
48789
|
// ../api-routes/src/ads.ts
|
|
48266
48790
|
import crypto36 from "crypto";
|
|
48267
|
-
import { eq as eq47, and as and40, asc as asc9, desc as desc21, gt as gt3, gte as gte12, lt as lt7, lte as lte11, ne as ne7, inArray as inArray20, or as
|
|
48791
|
+
import { eq as eq47, and as and40, asc as asc9, desc as desc21, gt as gt3, gte as gte12, lt as lt7, lte as lte11, ne as ne7, inArray as inArray20, or as or11, isNull as isNull8, isNotNull as isNotNull4, sql as sql18 } from "drizzle-orm";
|
|
48268
48792
|
|
|
48269
48793
|
// ../api-routes/src/ads-activation.ts
|
|
48270
48794
|
import crypto34 from "crypto";
|
|
@@ -49650,7 +50174,7 @@ async function executeApprovedAdsActivation(dependencies, request) {
|
|
|
49650
50174
|
|
|
49651
50175
|
// ../api-routes/src/ads-activation-routes.ts
|
|
49652
50176
|
import crypto35 from "crypto";
|
|
49653
|
-
import { and as and39, asc as asc8, eq as eq46, gt as gt2, isNotNull as isNotNull3, isNull as isNull7, lte as lte10, or as
|
|
50177
|
+
import { and as and39, asc as asc8, eq as eq46, gt as gt2, isNotNull as isNotNull3, isNull as isNull7, lte as lte10, or as or10, sql as sql17 } from "drizzle-orm";
|
|
49654
50178
|
var MAX_ACTIVATION_GRANT_LIFETIME_MS = 24 * 60 * 60 * 1e3;
|
|
49655
50179
|
var ACTIVATION_LEASE_MS = 5 * 6e4;
|
|
49656
50180
|
var ACTIVATION_REVOCATION_SETTLEMENT_MS = ACTIVATION_LEASE_MS;
|
|
@@ -49776,7 +50300,7 @@ function createActivationStore(app, request) {
|
|
|
49776
50300
|
}).where(and39(
|
|
49777
50301
|
eq46(adsOperations.id, existing.id),
|
|
49778
50302
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
49779
|
-
|
|
50303
|
+
or10(
|
|
49780
50304
|
isNull7(adsOperations.leaseOwner),
|
|
49781
50305
|
eq46(adsOperations.leaseOwner, input.leaseOwner),
|
|
49782
50306
|
isNull7(adsOperations.leaseExpiresAt),
|
|
@@ -50194,35 +50718,35 @@ async function executeActivationGrant(app, opts, request, project, executorApiKe
|
|
|
50194
50718
|
return activationResponse(opts, result.grant, result.operation);
|
|
50195
50719
|
}
|
|
50196
50720
|
function selectWatchdogOperations(app, nowIso, batchSize, allowLiveRevocationLease) {
|
|
50197
|
-
const leaseAvailable =
|
|
50721
|
+
const leaseAvailable = or10(
|
|
50198
50722
|
isNull7(adsOperations.leaseOwner),
|
|
50199
50723
|
isNull7(adsOperations.leaseExpiresAt),
|
|
50200
50724
|
lte10(adsOperations.leaseExpiresAt, nowIso)
|
|
50201
50725
|
);
|
|
50202
|
-
const campaignHasMutationEvidence =
|
|
50726
|
+
const campaignHasMutationEvidence = or10(
|
|
50203
50727
|
eq46(adsOperationSteps.state, AdsOperationStepStates.executing),
|
|
50204
50728
|
eq46(adsOperationSteps.state, AdsOperationStepStates.active),
|
|
50205
50729
|
eq46(adsOperationSteps.state, AdsOperationStepStates.rollback_executing),
|
|
50206
50730
|
eq46(adsOperationSteps.state, AdsOperationStepStates.rollback_failed),
|
|
50207
50731
|
eq46(adsOperationSteps.state, AdsOperationStepStates.unknown)
|
|
50208
50732
|
);
|
|
50209
|
-
const eligibleState = allowLiveRevocationLease ?
|
|
50733
|
+
const eligibleState = allowLiveRevocationLease ? or10(
|
|
50210
50734
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50211
50735
|
and39(
|
|
50212
50736
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
50213
|
-
|
|
50737
|
+
or10(
|
|
50214
50738
|
isNotNull3(adsActivationGrants.revocationRequestedAt),
|
|
50215
50739
|
and39(leaseAvailable, campaignHasMutationEvidence)
|
|
50216
50740
|
)
|
|
50217
50741
|
)
|
|
50218
50742
|
) : and39(
|
|
50219
|
-
|
|
50743
|
+
or10(
|
|
50220
50744
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50221
50745
|
eq46(adsOperations.state, AdsOperationStates.pending)
|
|
50222
50746
|
),
|
|
50223
50747
|
leaseAvailable
|
|
50224
50748
|
);
|
|
50225
|
-
const freshRevocationPriority =
|
|
50749
|
+
const freshRevocationPriority = sql17`CASE
|
|
50226
50750
|
WHEN ${adsActivationGrants.revocationRequestedAt} IS NOT NULL
|
|
50227
50751
|
AND (
|
|
50228
50752
|
${adsOperations.lastReconciledAt} IS NULL
|
|
@@ -50360,7 +50884,7 @@ async function containUrgentAdsActivationCampaignsOnce(app, opts) {
|
|
|
50360
50884
|
lastReconciledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
50361
50885
|
}).where(and39(
|
|
50362
50886
|
eq46(adsOperations.id, operation.id),
|
|
50363
|
-
|
|
50887
|
+
or10(
|
|
50364
50888
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50365
50889
|
eq46(adsOperations.state, AdsOperationStates.pending)
|
|
50366
50890
|
)
|
|
@@ -50386,11 +50910,11 @@ async function sweepStaleAdsActivationsOnce(app, opts) {
|
|
|
50386
50910
|
updatedAt: nowIso
|
|
50387
50911
|
}).where(and39(
|
|
50388
50912
|
eq46(adsOperations.id, operation.id),
|
|
50389
|
-
|
|
50913
|
+
or10(
|
|
50390
50914
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50391
50915
|
and39(
|
|
50392
50916
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
50393
|
-
|
|
50917
|
+
or10(
|
|
50394
50918
|
isNull7(adsOperations.leaseOwner),
|
|
50395
50919
|
isNull7(adsOperations.leaseExpiresAt),
|
|
50396
50920
|
lte10(adsOperations.leaseExpiresAt, nowIso)
|
|
@@ -50423,11 +50947,11 @@ async function sweepStaleAdsActivationsOnce(app, opts) {
|
|
|
50423
50947
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
50424
50948
|
app.db.update(adsOperations).set({ updatedAt: failedAt }).where(and39(
|
|
50425
50949
|
eq46(adsOperations.id, operation.id),
|
|
50426
|
-
|
|
50950
|
+
or10(
|
|
50427
50951
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50428
50952
|
and39(
|
|
50429
50953
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
50430
|
-
|
|
50954
|
+
or10(
|
|
50431
50955
|
isNull7(adsOperations.leaseOwner),
|
|
50432
50956
|
isNull7(adsOperations.leaseExpiresAt),
|
|
50433
50957
|
lte10(adsOperations.leaseExpiresAt, failedAt)
|
|
@@ -51613,13 +52137,13 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
51613
52137
|
enforceBackoff ? lte11(adsOperations.updatedAt, unknownCutoff) : void 0
|
|
51614
52138
|
) : and40(
|
|
51615
52139
|
eq47(adsOperations.state, AdsOperationStates.reconciling),
|
|
51616
|
-
|
|
52140
|
+
or11(isNull8(adsOperations.leaseExpiresAt), lte11(adsOperations.leaseExpiresAt, nowIso))
|
|
51617
52141
|
);
|
|
51618
52142
|
const claimed = app.db.update(adsOperations).set({
|
|
51619
52143
|
state: AdsOperationStates.reconciling,
|
|
51620
52144
|
leaseOwner,
|
|
51621
52145
|
leaseExpiresAt: leaseExpiresAt2,
|
|
51622
|
-
reconcileAttempts:
|
|
52146
|
+
reconcileAttempts: sql18`${adsOperations.reconcileAttempts} + 1`,
|
|
51623
52147
|
updatedAt: nowIso
|
|
51624
52148
|
}).where(and40(
|
|
51625
52149
|
eq47(adsOperations.id, row.id),
|
|
@@ -51935,28 +52459,28 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
51935
52459
|
);
|
|
51936
52460
|
}
|
|
51937
52461
|
);
|
|
51938
|
-
const retryableStateWhere =
|
|
52462
|
+
const retryableStateWhere = or11(
|
|
51939
52463
|
and40(
|
|
51940
52464
|
eq47(adsOperations.state, AdsOperationStates.pending),
|
|
51941
52465
|
lte11(adsOperations.updatedAt, pendingCutoff)
|
|
51942
52466
|
),
|
|
51943
52467
|
and40(
|
|
51944
52468
|
eq47(adsOperations.state, AdsOperationStates.unknown),
|
|
51945
|
-
|
|
52469
|
+
or11(...unknownRetryWindows)
|
|
51946
52470
|
),
|
|
51947
52471
|
and40(
|
|
51948
52472
|
eq47(adsOperations.state, AdsOperationStates.reconciling),
|
|
51949
|
-
|
|
52473
|
+
or11(isNull8(adsOperations.leaseExpiresAt), lte11(adsOperations.leaseExpiresAt, nowIso))
|
|
51950
52474
|
)
|
|
51951
52475
|
);
|
|
51952
52476
|
const staleReconcileWhere = and40(
|
|
51953
52477
|
lt7(adsOperations.reconcileAttempts, config.policy.maxAttempts),
|
|
51954
52478
|
ne7(adsOperations.kind, AdsOperationKinds.campaign_tree_activate),
|
|
51955
|
-
|
|
52479
|
+
or11(
|
|
51956
52480
|
eq47(adsOperations.reconcileStrategy, AdsReconcileStrategies.known_entity),
|
|
51957
52481
|
and40(
|
|
51958
52482
|
eq47(adsOperations.reconcileStrategy, AdsReconcileStrategies.create_fingerprint),
|
|
51959
|
-
|
|
52483
|
+
or11(
|
|
51960
52484
|
isNotNull4(adsOperations.entityId),
|
|
51961
52485
|
inArray20(adsOperations.state, [
|
|
51962
52486
|
AdsOperationStates.pending,
|
|
@@ -51970,7 +52494,7 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
51970
52494
|
const exhaustedRows = app.db.select().from(adsOperations).where(and40(
|
|
51971
52495
|
ne7(adsOperations.kind, AdsOperationKinds.campaign_tree_activate),
|
|
51972
52496
|
gte12(adsOperations.reconcileAttempts, config.policy.maxAttempts),
|
|
51973
|
-
|
|
52497
|
+
or11(
|
|
51974
52498
|
and40(
|
|
51975
52499
|
eq47(adsOperations.state, AdsOperationStates.pending),
|
|
51976
52500
|
lte11(adsOperations.updatedAt, pendingCutoff)
|
|
@@ -51978,10 +52502,10 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
51978
52502
|
eq47(adsOperations.state, AdsOperationStates.unknown),
|
|
51979
52503
|
and40(
|
|
51980
52504
|
eq47(adsOperations.state, AdsOperationStates.reconciling),
|
|
51981
|
-
|
|
52505
|
+
or11(isNull8(adsOperations.leaseExpiresAt), lte11(adsOperations.leaseExpiresAt, nowIso))
|
|
51982
52506
|
)
|
|
51983
52507
|
),
|
|
51984
|
-
|
|
52508
|
+
or11(
|
|
51985
52509
|
isNull8(adsOperations.errorCode),
|
|
51986
52510
|
ne7(adsOperations.errorCode, ADS_RECONCILIATION_QUARANTINED)
|
|
51987
52511
|
)
|
|
@@ -52329,7 +52853,7 @@ async function adsRoutes(app, opts) {
|
|
|
52329
52853
|
const rows = app.db.select().from(adsOperations).where(and40(
|
|
52330
52854
|
eq47(adsOperations.projectId, project.id),
|
|
52331
52855
|
inArray20(adsOperations.state, parsed.data.state),
|
|
52332
|
-
cursor ?
|
|
52856
|
+
cursor ? or11(
|
|
52333
52857
|
gt3(adsOperations.createdAt, cursor.createdAt),
|
|
52334
52858
|
and40(
|
|
52335
52859
|
eq47(adsOperations.createdAt, cursor.createdAt),
|
|
@@ -54218,7 +54742,7 @@ async function cdpRoutes(app, opts) {
|
|
|
54218
54742
|
|
|
54219
54743
|
// ../api-routes/src/ga.ts
|
|
54220
54744
|
import crypto39 from "crypto";
|
|
54221
|
-
import { eq as eq50, desc as desc23, and as and43, sql as
|
|
54745
|
+
import { eq as eq50, desc as desc23, and as and43, sql as sql19 } from "drizzle-orm";
|
|
54222
54746
|
|
|
54223
54747
|
// ../api-routes/src/ga-session-history.ts
|
|
54224
54748
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -54251,8 +54775,8 @@ function formatSharePct(numerator, total) {
|
|
|
54251
54775
|
}
|
|
54252
54776
|
function dateRangeConditions(column, range) {
|
|
54253
54777
|
const conditions = [];
|
|
54254
|
-
if (range.startDate) conditions.push(
|
|
54255
|
-
if (range.endDate) conditions.push(
|
|
54778
|
+
if (range.startDate) conditions.push(sql19`${column} >= ${range.startDate}`);
|
|
54779
|
+
if (range.endDate) conditions.push(sql19`${column} <= ${range.endDate}`);
|
|
54256
54780
|
return conditions;
|
|
54257
54781
|
}
|
|
54258
54782
|
var SOCIAL_CHANNEL_GROUPS2 = /* @__PURE__ */ new Set(["Organic Social", "Paid Social"]);
|
|
@@ -54312,8 +54836,8 @@ function persistAcquisitionMeasurement(db, input) {
|
|
|
54312
54836
|
db.transaction((tx) => {
|
|
54313
54837
|
tx.delete(gaAcquisitionDaily).where(and43(
|
|
54314
54838
|
eq50(gaAcquisitionDaily.projectId, input.projectId),
|
|
54315
|
-
|
|
54316
|
-
|
|
54839
|
+
sql19`${gaAcquisitionDaily.date} >= ${startDate}`,
|
|
54840
|
+
sql19`${gaAcquisitionDaily.date} <= ${endDate}`
|
|
54317
54841
|
)).run();
|
|
54318
54842
|
for (const row of input.report.rows) {
|
|
54319
54843
|
tx.insert(gaAcquisitionDaily).values({
|
|
@@ -54354,8 +54878,8 @@ function persistLeadMeasurement(db, input) {
|
|
|
54354
54878
|
db.transaction((tx) => {
|
|
54355
54879
|
tx.delete(gaLeadEventsDaily).where(and43(
|
|
54356
54880
|
eq50(gaLeadEventsDaily.projectId, input.projectId),
|
|
54357
|
-
|
|
54358
|
-
|
|
54881
|
+
sql19`${gaLeadEventsDaily.date} >= ${startDate}`,
|
|
54882
|
+
sql19`${gaLeadEventsDaily.date} <= ${endDate}`
|
|
54359
54883
|
)).run();
|
|
54360
54884
|
for (const row of input.report.rows) {
|
|
54361
54885
|
tx.insert(gaLeadEventsDaily).values({
|
|
@@ -54730,8 +55254,8 @@ async function ga4Routes(app, opts) {
|
|
|
54730
55254
|
tx.delete(gaTrafficSnapshots).where(
|
|
54731
55255
|
and43(
|
|
54732
55256
|
eq50(gaTrafficSnapshots.projectId, project.id),
|
|
54733
|
-
|
|
54734
|
-
|
|
55257
|
+
sql19`${gaTrafficSnapshots.date} >= ${summary2.periodStart}`,
|
|
55258
|
+
sql19`${gaTrafficSnapshots.date} <= ${summary2.periodEnd}`
|
|
54735
55259
|
)
|
|
54736
55260
|
).run();
|
|
54737
55261
|
for (const row of rows) {
|
|
@@ -54753,8 +55277,8 @@ async function ga4Routes(app, opts) {
|
|
|
54753
55277
|
tx.delete(gaDailyTotals).where(
|
|
54754
55278
|
and43(
|
|
54755
55279
|
eq50(gaDailyTotals.projectId, project.id),
|
|
54756
|
-
|
|
54757
|
-
|
|
55280
|
+
sql19`${gaDailyTotals.date} >= ${summary2.periodStart}`,
|
|
55281
|
+
sql19`${gaDailyTotals.date} <= ${summary2.periodEnd}`
|
|
54758
55282
|
)
|
|
54759
55283
|
).run();
|
|
54760
55284
|
for (const row of dailyTotals) {
|
|
@@ -54779,8 +55303,8 @@ async function ga4Routes(app, opts) {
|
|
|
54779
55303
|
tx.delete(gaAiReferrals).where(
|
|
54780
55304
|
and43(
|
|
54781
55305
|
eq50(gaAiReferrals.projectId, project.id),
|
|
54782
|
-
|
|
54783
|
-
|
|
55306
|
+
sql19`${gaAiReferrals.date} >= ${summary2.periodStart}`,
|
|
55307
|
+
sql19`${gaAiReferrals.date} <= ${summary2.periodEnd}`
|
|
54784
55308
|
)
|
|
54785
55309
|
).run();
|
|
54786
55310
|
for (const row of aiReferrals) {
|
|
@@ -54811,8 +55335,8 @@ async function ga4Routes(app, opts) {
|
|
|
54811
55335
|
tx.delete(gaSocialReferrals).where(
|
|
54812
55336
|
and43(
|
|
54813
55337
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
54814
|
-
|
|
54815
|
-
|
|
55338
|
+
sql19`${gaSocialReferrals.date} >= ${summary2.periodStart}`,
|
|
55339
|
+
sql19`${gaSocialReferrals.date} <= ${summary2.periodEnd}`
|
|
54816
55340
|
)
|
|
54817
55341
|
).run();
|
|
54818
55342
|
for (const row of socialReferrals) {
|
|
@@ -54994,9 +55518,9 @@ async function ga4Routes(app, opts) {
|
|
|
54994
55518
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
54995
55519
|
}).from(gaTrafficSummaries).where(eq50(gaTrafficSummaries.projectId, project.id)).get();
|
|
54996
55520
|
const retainedBounds = [
|
|
54997
|
-
app.db.select({ min:
|
|
54998
|
-
app.db.select({ min:
|
|
54999
|
-
app.db.select({ min:
|
|
55521
|
+
app.db.select({ min: sql19`MIN(${gaTrafficSnapshots.date})`, max: sql19`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq50(gaTrafficSnapshots.projectId, project.id)).get(),
|
|
55522
|
+
app.db.select({ min: sql19`MIN(${gaAiReferrals.date})`, max: sql19`MAX(${gaAiReferrals.date})` }).from(gaAiReferrals).where(eq50(gaAiReferrals.projectId, project.id)).get(),
|
|
55523
|
+
app.db.select({ min: sql19`MIN(${gaSocialReferrals.date})`, max: sql19`MAX(${gaSocialReferrals.date})` }).from(gaSocialReferrals).where(eq50(gaSocialReferrals.projectId, project.id)).get()
|
|
55000
55524
|
];
|
|
55001
55525
|
const retainedDates = retainedBounds.flatMap((bound) => [bound?.min, bound?.max]).filter((date) => Boolean(date));
|
|
55002
55526
|
const projectSummaryCoversAll = Boolean(
|
|
@@ -55010,9 +55534,9 @@ async function ga4Routes(app, opts) {
|
|
|
55010
55534
|
const socialConditions = [eq50(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, measuredRange)];
|
|
55011
55535
|
const snapshotTotalsRow = !denominatorRow ? (() => {
|
|
55012
55536
|
const summed = app.db.select({
|
|
55013
|
-
totalSessions:
|
|
55014
|
-
totalOrganicSessions:
|
|
55015
|
-
totalUsers:
|
|
55537
|
+
totalSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
55538
|
+
totalOrganicSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
55539
|
+
totalUsers: sql19`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
55016
55540
|
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).get();
|
|
55017
55541
|
return {
|
|
55018
55542
|
totalSessions: summed?.totalSessions ?? 0,
|
|
@@ -55025,35 +55549,35 @@ async function ga4Routes(app, opts) {
|
|
|
55025
55549
|
})() : null;
|
|
55026
55550
|
const summaryRow = denominatorRow ?? snapshotTotalsRow;
|
|
55027
55551
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
55028
|
-
totalDirectSessions:
|
|
55552
|
+
totalDirectSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
55029
55553
|
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).get();
|
|
55030
55554
|
const rows = app.db.select({
|
|
55031
|
-
landingPage:
|
|
55032
|
-
sessions:
|
|
55033
|
-
organicSessions:
|
|
55034
|
-
directSessions:
|
|
55035
|
-
users:
|
|
55036
|
-
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).groupBy(
|
|
55555
|
+
landingPage: sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
55556
|
+
sessions: sql19`SUM(${gaTrafficSnapshots.sessions})`,
|
|
55557
|
+
organicSessions: sql19`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
55558
|
+
directSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
55559
|
+
users: sql19`SUM(${gaTrafficSnapshots.users})`
|
|
55560
|
+
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).groupBy(sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql19`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
55037
55561
|
const aiReferralRows = app.db.select({
|
|
55038
55562
|
source: gaAiReferrals.source,
|
|
55039
55563
|
medium: gaAiReferrals.medium,
|
|
55040
55564
|
trafficClass: gaAiReferrals.trafficClass,
|
|
55041
55565
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
55042
|
-
sessions:
|
|
55566
|
+
sessions: sql19`SUM(${gaAiReferrals.sessions})`
|
|
55043
55567
|
}).from(gaAiReferrals).where(and43(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
55044
55568
|
const aiReferralLandingPageRows = app.db.select({
|
|
55045
55569
|
source: gaAiReferrals.source,
|
|
55046
55570
|
medium: gaAiReferrals.medium,
|
|
55047
55571
|
trafficClass: gaAiReferrals.trafficClass,
|
|
55048
55572
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
55049
|
-
landingPage:
|
|
55050
|
-
sessions:
|
|
55573
|
+
landingPage: sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
55574
|
+
sessions: sql19`SUM(${gaAiReferrals.sessions})`
|
|
55051
55575
|
}).from(gaAiReferrals).where(and43(...aiConditions)).groupBy(
|
|
55052
55576
|
gaAiReferrals.source,
|
|
55053
55577
|
gaAiReferrals.medium,
|
|
55054
55578
|
gaAiReferrals.trafficClass,
|
|
55055
55579
|
gaAiReferrals.sourceDimension,
|
|
55056
|
-
|
|
55580
|
+
sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
55057
55581
|
).all();
|
|
55058
55582
|
const aiRowsForTotals = app.db.select({
|
|
55059
55583
|
date: gaAiReferrals.date,
|
|
@@ -55079,10 +55603,10 @@ async function ga4Routes(app, opts) {
|
|
|
55079
55603
|
// is GA's COUNT DISTINCT at that grain, so summing over the window's
|
|
55080
55604
|
// dates counts a returning visitor once per day they returned. Same
|
|
55081
55605
|
// conclusion the AI-referral surfaces reached in 4.135.0.
|
|
55082
|
-
sessions:
|
|
55083
|
-
}).from(gaSocialReferrals).where(and43(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
55606
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55607
|
+
}).from(gaSocialReferrals).where(and43(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql19`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
55084
55608
|
const socialTotals = app.db.select({
|
|
55085
|
-
sessions:
|
|
55609
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55086
55610
|
}).from(gaSocialReferrals).where(and43(...socialConditions)).get();
|
|
55087
55611
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq50(gaTrafficSummaries.projectId, project.id)).orderBy(desc23(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
55088
55612
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -55197,20 +55721,20 @@ async function ga4Routes(app, opts) {
|
|
|
55197
55721
|
source: gaAiReferrals.source,
|
|
55198
55722
|
medium: gaAiReferrals.medium,
|
|
55199
55723
|
trafficClass: gaAiReferrals.trafficClass,
|
|
55200
|
-
landingPage:
|
|
55724
|
+
landingPage: sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
55201
55725
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
55202
55726
|
// No `users`: ga-ai-referral-aggregation states the rule for this
|
|
55203
55727
|
// table, that there is no grain at which the stored users column may be
|
|
55204
55728
|
// added up. ga4AiReferralDtoSchema.users was deprecated for this in
|
|
55205
55729
|
// 4.135.0; the history endpoint kept emitting one.
|
|
55206
|
-
sessions:
|
|
55730
|
+
sessions: sql19`SUM(${gaAiReferrals.sessions})`
|
|
55207
55731
|
}).from(gaAiReferrals).where(and43(...conditions)).groupBy(
|
|
55208
55732
|
gaAiReferrals.date,
|
|
55209
55733
|
gaAiReferrals.source,
|
|
55210
55734
|
gaAiReferrals.medium,
|
|
55211
55735
|
gaAiReferrals.trafficClass,
|
|
55212
55736
|
gaAiReferrals.sourceDimension,
|
|
55213
|
-
|
|
55737
|
+
sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
55214
55738
|
).orderBy(gaAiReferrals.date).all();
|
|
55215
55739
|
return rows;
|
|
55216
55740
|
});
|
|
@@ -55255,10 +55779,10 @@ async function ga4Routes(app, opts) {
|
|
|
55255
55779
|
d.setDate(d.getDate() - n);
|
|
55256
55780
|
return fmt(d);
|
|
55257
55781
|
};
|
|
55258
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
55782
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and43(
|
|
55259
55783
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
55260
|
-
|
|
55261
|
-
|
|
55784
|
+
sql19`${gaSocialReferrals.date} >= ${from}`,
|
|
55785
|
+
sql19`${gaSocialReferrals.date} < ${to}`
|
|
55262
55786
|
)).get();
|
|
55263
55787
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
55264
55788
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -55266,19 +55790,19 @@ async function ga4Routes(app, opts) {
|
|
|
55266
55790
|
const prev30d = sumSocial(daysAgo(60), daysAgo(30));
|
|
55267
55791
|
const sourceCurrent = app.db.select({
|
|
55268
55792
|
source: gaSocialReferrals.source,
|
|
55269
|
-
sessions:
|
|
55793
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55270
55794
|
}).from(gaSocialReferrals).where(and43(
|
|
55271
55795
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
55272
|
-
|
|
55273
|
-
|
|
55796
|
+
sql19`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
55797
|
+
sql19`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
55274
55798
|
)).groupBy(gaSocialReferrals.source).all();
|
|
55275
55799
|
const sourcePrev = app.db.select({
|
|
55276
55800
|
source: gaSocialReferrals.source,
|
|
55277
|
-
sessions:
|
|
55801
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55278
55802
|
}).from(gaSocialReferrals).where(and43(
|
|
55279
55803
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
55280
|
-
|
|
55281
|
-
|
|
55804
|
+
sql19`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
55805
|
+
sql19`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
55282
55806
|
)).groupBy(gaSocialReferrals.source).all();
|
|
55283
55807
|
const prevMap = new Map(sourcePrev.map((r) => [r.source, r.sessions]));
|
|
55284
55808
|
let biggestMover = null;
|
|
@@ -55316,16 +55840,16 @@ async function ga4Routes(app, opts) {
|
|
|
55316
55840
|
d.setDate(d.getDate() - n);
|
|
55317
55841
|
return fmt(d);
|
|
55318
55842
|
};
|
|
55319
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
55320
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
55321
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
55322
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
55843
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and43(eq50(gaTrafficSnapshots.projectId, project.id), sql19`${gaTrafficSnapshots.date} >= ${from}`, sql19`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
55844
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and43(eq50(gaTrafficSnapshots.projectId, project.id), sql19`${gaTrafficSnapshots.date} >= ${from}`, sql19`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
55845
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and43(eq50(gaTrafficSnapshots.projectId, project.id), sql19`${gaTrafficSnapshots.date} >= ${from}`, sql19`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
55846
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and43(
|
|
55323
55847
|
eq50(gaAiReferrals.projectId, project.id),
|
|
55324
|
-
|
|
55325
|
-
|
|
55848
|
+
sql19`${gaAiReferrals.date} >= ${from}`,
|
|
55849
|
+
sql19`${gaAiReferrals.date} < ${to}`,
|
|
55326
55850
|
eq50(gaAiReferrals.sourceDimension, "session")
|
|
55327
55851
|
)).get();
|
|
55328
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
55852
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and43(eq50(gaSocialReferrals.projectId, project.id), sql19`${gaSocialReferrals.date} >= ${from}`, sql19`${gaSocialReferrals.date} < ${to}`)).get();
|
|
55329
55853
|
const todayStr = fmt(today);
|
|
55330
55854
|
const buildTrend = (sum) => {
|
|
55331
55855
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -55334,16 +55858,16 @@ async function ga4Routes(app, opts) {
|
|
|
55334
55858
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
55335
55859
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: deltaPercent(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: deltaPercent(c30, p30) };
|
|
55336
55860
|
};
|
|
55337
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
55861
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql19`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and43(
|
|
55338
55862
|
eq50(gaAiReferrals.projectId, project.id),
|
|
55339
|
-
|
|
55340
|
-
|
|
55863
|
+
sql19`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
55864
|
+
sql19`${gaAiReferrals.date} < ${todayStr}`,
|
|
55341
55865
|
eq50(gaAiReferrals.sourceDimension, "session")
|
|
55342
55866
|
)).groupBy(gaAiReferrals.source).all();
|
|
55343
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
55867
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql19`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and43(
|
|
55344
55868
|
eq50(gaAiReferrals.projectId, project.id),
|
|
55345
|
-
|
|
55346
|
-
|
|
55869
|
+
sql19`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
55870
|
+
sql19`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
55347
55871
|
eq50(gaAiReferrals.sourceDimension, "session")
|
|
55348
55872
|
)).groupBy(gaAiReferrals.source).all();
|
|
55349
55873
|
const findBiggestMover = (current, prev) => {
|
|
@@ -55360,8 +55884,8 @@ async function ga4Routes(app, opts) {
|
|
|
55360
55884
|
}
|
|
55361
55885
|
return mover;
|
|
55362
55886
|
};
|
|
55363
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
55364
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
55887
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql19`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and43(eq50(gaSocialReferrals.projectId, project.id), sql19`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql19`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
55888
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql19`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and43(eq50(gaSocialReferrals.projectId, project.id), sql19`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql19`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
55365
55889
|
return {
|
|
55366
55890
|
total: buildTrend(sumTotal),
|
|
55367
55891
|
organic: buildTrend(sumOrganic),
|
|
@@ -55379,9 +55903,9 @@ async function ga4Routes(app, opts) {
|
|
|
55379
55903
|
const conditions = [eq50(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, range)];
|
|
55380
55904
|
const rows = app.db.select({
|
|
55381
55905
|
date: gaTrafficSnapshots.date,
|
|
55382
|
-
sessions:
|
|
55383
|
-
organicSessions:
|
|
55384
|
-
users:
|
|
55906
|
+
sessions: sql19`SUM(${gaTrafficSnapshots.sessions})`,
|
|
55907
|
+
organicSessions: sql19`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
55908
|
+
users: sql19`SUM(${gaTrafficSnapshots.users})`
|
|
55385
55909
|
}).from(gaTrafficSnapshots).where(and43(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
55386
55910
|
const totalConditions = [eq50(gaDailyTotals.projectId, project.id), ...dateRangeConditions(gaDailyTotals.date, range)];
|
|
55387
55911
|
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and43(...totalConditions)).all();
|
|
@@ -55399,11 +55923,11 @@ async function ga4Routes(app, opts) {
|
|
|
55399
55923
|
const project = resolveProject(app.db, request.params.name);
|
|
55400
55924
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
55401
55925
|
const trafficPages = app.db.select({
|
|
55402
|
-
landingPage:
|
|
55403
|
-
sessions:
|
|
55404
|
-
organicSessions:
|
|
55405
|
-
users:
|
|
55406
|
-
}).from(gaTrafficSnapshots).where(eq50(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
55926
|
+
landingPage: sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
55927
|
+
sessions: sql19`SUM(${gaTrafficSnapshots.sessions})`,
|
|
55928
|
+
organicSessions: sql19`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
55929
|
+
users: sql19`SUM(${gaTrafficSnapshots.users})`
|
|
55930
|
+
}).from(gaTrafficSnapshots).where(eq50(gaTrafficSnapshots.projectId, project.id)).groupBy(sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql19`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
55407
55931
|
return {
|
|
55408
55932
|
pages: trafficPages.map((r) => ({
|
|
55409
55933
|
landingPage: r.landingPage,
|
|
@@ -57058,7 +57582,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
57058
57582
|
|
|
57059
57583
|
// ../api-routes/src/backlinks.ts
|
|
57060
57584
|
import crypto41 from "crypto";
|
|
57061
|
-
import { and as and45, asc as asc10, desc as desc24, eq as eq51, sql as
|
|
57585
|
+
import { and as and45, asc as asc10, desc as desc24, eq as eq51, sql as sql20 } from "drizzle-orm";
|
|
57062
57586
|
|
|
57063
57587
|
// ../integration-commoncrawl/src/constants.ts
|
|
57064
57588
|
import os2 from "os";
|
|
@@ -57155,7 +57679,7 @@ function parseContentLength(value) {
|
|
|
57155
57679
|
}
|
|
57156
57680
|
|
|
57157
57681
|
// ../integration-commoncrawl/src/downloader.ts
|
|
57158
|
-
import { createHash as
|
|
57682
|
+
import { createHash as createHash6 } from "crypto";
|
|
57159
57683
|
import { createWriteStream } from "fs";
|
|
57160
57684
|
import fs2 from "fs/promises";
|
|
57161
57685
|
import path3 from "path";
|
|
@@ -57181,7 +57705,7 @@ async function downloadFile(opts) {
|
|
|
57181
57705
|
throw new Error(`HTTP ${res.status} ${res.statusText} for ${opts.url}`);
|
|
57182
57706
|
}
|
|
57183
57707
|
const total = parseContentLength2(res.headers.get("content-length"));
|
|
57184
|
-
const hasher =
|
|
57708
|
+
const hasher = createHash6("sha256");
|
|
57185
57709
|
let bytes = 0;
|
|
57186
57710
|
const hashAndCount = new Transform({
|
|
57187
57711
|
transform(chunk, _enc, cb) {
|
|
@@ -57202,7 +57726,7 @@ async function downloadFile(opts) {
|
|
|
57202
57726
|
return { bytes, sha256: sha2564, cached: false, elapsedMs: Date.now() - start };
|
|
57203
57727
|
}
|
|
57204
57728
|
async function hashFile(filePath) {
|
|
57205
|
-
const hasher =
|
|
57729
|
+
const hasher = createHash6("sha256");
|
|
57206
57730
|
const handle = await fs2.open(filePath, "r");
|
|
57207
57731
|
try {
|
|
57208
57732
|
const stream = handle.createReadStream();
|
|
@@ -57348,7 +57872,7 @@ async function queryBacklinks(opts) {
|
|
|
57348
57872
|
const reversed = opts.targets.map(reverseDomain);
|
|
57349
57873
|
const targetList = reversed.map(quote).join(", ");
|
|
57350
57874
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
57351
|
-
const
|
|
57875
|
+
const sql28 = `
|
|
57352
57876
|
WITH vertices AS (
|
|
57353
57877
|
SELECT * FROM read_csv(
|
|
57354
57878
|
${quote(opts.vertexPath)},
|
|
@@ -57384,7 +57908,7 @@ async function queryBacklinks(opts) {
|
|
|
57384
57908
|
const conn = await instance.connect();
|
|
57385
57909
|
let rows;
|
|
57386
57910
|
try {
|
|
57387
|
-
const reader = await conn.runAndReadAll(
|
|
57911
|
+
const reader = await conn.runAndReadAll(sql28);
|
|
57388
57912
|
rows = reader.getRowObjects();
|
|
57389
57913
|
} finally {
|
|
57390
57914
|
conn.disconnectSync?.();
|
|
@@ -57574,12 +58098,12 @@ function computeFilteredSummary(db, base) {
|
|
|
57574
58098
|
);
|
|
57575
58099
|
const filteredCondition = and45(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
57576
58100
|
const unfilteredAgg = db.select({
|
|
57577
|
-
count:
|
|
57578
|
-
total:
|
|
58101
|
+
count: sql20`count(*)`,
|
|
58102
|
+
total: sql20`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
57579
58103
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
57580
58104
|
const filteredAgg = db.select({
|
|
57581
|
-
count:
|
|
57582
|
-
total:
|
|
58105
|
+
count: sql20`count(*)`,
|
|
58106
|
+
total: sql20`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
57583
58107
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
57584
58108
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc24(backlinkDomains.numHosts)).limit(10).all();
|
|
57585
58109
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -57605,7 +58129,7 @@ function buildSourceAvailability(db, projectId, source, connected, excludeCrawle
|
|
|
57605
58129
|
const summary2 = db.select().from(backlinkSummaries).where(and45(eq51(backlinkSummaries.projectId, projectId), eq51(backlinkSummaries.source, source))).orderBy(desc24(backlinkSummaries.queriedAt)).limit(1).get();
|
|
57606
58130
|
let totalLinkingDomains = summary2?.totalLinkingDomains ?? 0;
|
|
57607
58131
|
if (summary2 && excludeCrawlers) {
|
|
57608
|
-
const filtered = db.select({ count:
|
|
58132
|
+
const filtered = db.select({ count: sql20`count(*)` }).from(backlinkDomains).where(and45(
|
|
57609
58133
|
eq51(backlinkDomains.projectId, projectId),
|
|
57610
58134
|
eq51(backlinkDomains.source, source),
|
|
57611
58135
|
eq51(backlinkDomains.release, summary2.release),
|
|
@@ -57793,7 +58317,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
57793
58317
|
eq51(backlinkDomains.release, targetRelease)
|
|
57794
58318
|
);
|
|
57795
58319
|
const domainCondition = excludeCrawlers ? and45(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
57796
|
-
const totalRow = app.db.select({ count:
|
|
58320
|
+
const totalRow = app.db.select({ count: sql20`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
57797
58321
|
const rows = app.db.select({
|
|
57798
58322
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
57799
58323
|
numHosts: backlinkDomains.numHosts,
|
|
@@ -57844,7 +58368,7 @@ import crypto44 from "crypto";
|
|
|
57844
58368
|
import { isIP } from "net";
|
|
57845
58369
|
import { isDeepStrictEqual } from "util";
|
|
57846
58370
|
import { Agent as UndiciAgent } from "undici";
|
|
57847
|
-
import { and as and49, desc as desc25, eq as eq55, gte as gte14, inArray as inArray21, lte as lte14, sql as
|
|
58371
|
+
import { and as and49, desc as desc25, eq as eq55, gte as gte14, inArray as inArray21, lte as lte14, sql as sql23 } from "drizzle-orm";
|
|
57848
58372
|
|
|
57849
58373
|
// ../api-routes/src/cloudflare-worker-version.ts
|
|
57850
58374
|
var CURRENT_CLOUDFLARE_WORKER_VERSION = "1.0.1";
|
|
@@ -71862,8 +72386,8 @@ var perplexitybot_default = {
|
|
|
71862
72386
|
};
|
|
71863
72387
|
|
|
71864
72388
|
// ../integration-traffic/src/ip-range-manifest.ts
|
|
71865
|
-
import { createHash as
|
|
71866
|
-
function
|
|
72389
|
+
import { createHash as createHash7 } from "crypto";
|
|
72390
|
+
function isRecord2(value) {
|
|
71867
72391
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
71868
72392
|
}
|
|
71869
72393
|
function stableJson2(value, seen2 = /* @__PURE__ */ new Set()) {
|
|
@@ -71932,7 +72456,7 @@ function parseCidr(cidr) {
|
|
|
71932
72456
|
};
|
|
71933
72457
|
}
|
|
71934
72458
|
function canonicalPrefixKey(entry) {
|
|
71935
|
-
if (
|
|
72459
|
+
if (isRecord2(entry)) {
|
|
71936
72460
|
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
71937
72461
|
const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
|
|
71938
72462
|
if (hasIpv4 !== hasIpv6) {
|
|
@@ -71952,12 +72476,12 @@ function canonicalPrefixKey(entry) {
|
|
|
71952
72476
|
return `invalid:${stableJson2(entry)}`;
|
|
71953
72477
|
}
|
|
71954
72478
|
function ipRangeManifestContentHash(value) {
|
|
71955
|
-
const prefixes =
|
|
72479
|
+
const prefixes = isRecord2(value) ? value.prefixes : void 0;
|
|
71956
72480
|
const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
|
|
71957
|
-
return
|
|
72481
|
+
return createHash7("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
71958
72482
|
}
|
|
71959
72483
|
function validateIpRangeManifestPayload(value) {
|
|
71960
|
-
if (!
|
|
72484
|
+
if (!isRecord2(value)) return { ok: false, error: "response must be a JSON object" };
|
|
71961
72485
|
const creationTime = value.creationTime;
|
|
71962
72486
|
if (typeof creationTime !== "string" || creationTime.trim().length === 0) {
|
|
71963
72487
|
return { ok: false, error: "response missing `creationTime` version" };
|
|
@@ -71967,7 +72491,7 @@ function validateIpRangeManifestPayload(value) {
|
|
|
71967
72491
|
}
|
|
71968
72492
|
const prefixes = [];
|
|
71969
72493
|
for (const [index2, entry] of value.prefixes.entries()) {
|
|
71970
|
-
if (!
|
|
72494
|
+
if (!isRecord2(entry)) {
|
|
71971
72495
|
return { ok: false, error: `prefixes[${index2}] must be an object` };
|
|
71972
72496
|
}
|
|
71973
72497
|
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
@@ -72955,7 +73479,7 @@ function incrementBucket(map, key, fields) {
|
|
|
72955
73479
|
}
|
|
72956
73480
|
|
|
72957
73481
|
// ../integration-wordpress-traffic/src/client.ts
|
|
72958
|
-
import { randomUUID as
|
|
73482
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
72959
73483
|
|
|
72960
73484
|
// ../integration-wordpress-traffic/src/normalize.ts
|
|
72961
73485
|
function trimOrNull(value) {
|
|
@@ -73053,7 +73577,7 @@ async function readErrorBody2(response) {
|
|
|
73053
73577
|
if (!text2) return void 0;
|
|
73054
73578
|
return text2.length <= 500 ? text2 : `${text2.slice(0, 500)}... [truncated]`;
|
|
73055
73579
|
}
|
|
73056
|
-
function
|
|
73580
|
+
function isRecord3(value) {
|
|
73057
73581
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
73058
73582
|
}
|
|
73059
73583
|
function parseRequestedBoundary(name, value) {
|
|
@@ -73073,7 +73597,7 @@ function parseRequestedWindow(options) {
|
|
|
73073
73597
|
return { since, until };
|
|
73074
73598
|
}
|
|
73075
73599
|
function parseSiteMetadata(value) {
|
|
73076
|
-
if (!
|
|
73600
|
+
if (!isRecord3(value)) return void 0;
|
|
73077
73601
|
return {
|
|
73078
73602
|
url: typeof value.url === "string" ? value.url : void 0,
|
|
73079
73603
|
anonymous_id: typeof value.anonymous_id === "string" ? value.anonymous_id : void 0,
|
|
@@ -73088,7 +73612,7 @@ function invalidResponse(message) {
|
|
|
73088
73612
|
);
|
|
73089
73613
|
}
|
|
73090
73614
|
function parseResponseBody(value) {
|
|
73091
|
-
if (!
|
|
73615
|
+
if (!isRecord3(value)) {
|
|
73092
73616
|
throw invalidResponse("expected an object body");
|
|
73093
73617
|
}
|
|
73094
73618
|
if (!Array.isArray(value.events)) {
|
|
@@ -73121,7 +73645,7 @@ async function readResponseJson(response) {
|
|
|
73121
73645
|
}
|
|
73122
73646
|
function assertEntryIsWithinRequestedWindow(entry, window) {
|
|
73123
73647
|
if (!window.since && !window.until) return;
|
|
73124
|
-
const observedAt =
|
|
73648
|
+
const observedAt = isRecord3(entry) && typeof entry.observed_at === "string" ? entry.observed_at : void 0;
|
|
73125
73649
|
const observedAtMs = observedAt === void 0 ? Number.NaN : Date.parse(observedAt);
|
|
73126
73650
|
const outsideWindow = !Number.isFinite(observedAtMs) || window.since !== void 0 && observedAtMs < window.since.timestampMs || window.until !== void 0 && observedAtMs >= window.until.timestampMs;
|
|
73127
73651
|
if (!outsideWindow) return;
|
|
@@ -73157,7 +73681,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
73157
73681
|
if (options.until !== void 0 && options.until !== "") {
|
|
73158
73682
|
url.searchParams.set("until", options.until);
|
|
73159
73683
|
}
|
|
73160
|
-
url.searchParams.set("_cb",
|
|
73684
|
+
url.searchParams.set("_cb", randomUUID3());
|
|
73161
73685
|
const fetchInit = {
|
|
73162
73686
|
method: "GET",
|
|
73163
73687
|
headers: {
|
|
@@ -73184,7 +73708,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
73184
73708
|
rawEntryCount += entries.length;
|
|
73185
73709
|
for (const entry of entries) {
|
|
73186
73710
|
assertEntryIsWithinRequestedWindow(entry, requestedWindow);
|
|
73187
|
-
if (!
|
|
73711
|
+
if (!isRecord3(entry)) {
|
|
73188
73712
|
skippedEntryCount += 1;
|
|
73189
73713
|
continue;
|
|
73190
73714
|
}
|
|
@@ -74296,7 +74820,7 @@ function asRecord(value, label) {
|
|
|
74296
74820
|
}
|
|
74297
74821
|
return value;
|
|
74298
74822
|
}
|
|
74299
|
-
function
|
|
74823
|
+
function requiredString2(record, key, label) {
|
|
74300
74824
|
const value = record[key];
|
|
74301
74825
|
if (typeof value !== "string" || !value.trim()) {
|
|
74302
74826
|
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
@@ -74370,12 +74894,12 @@ function safeMessageBase(record, leaseId) {
|
|
|
74370
74894
|
};
|
|
74371
74895
|
}
|
|
74372
74896
|
function decodeMessage(record) {
|
|
74373
|
-
const leaseId =
|
|
74897
|
+
const leaseId = requiredString2(record, "lease_id", "message");
|
|
74374
74898
|
let base;
|
|
74375
74899
|
let rawBody;
|
|
74376
74900
|
try {
|
|
74377
74901
|
base = {
|
|
74378
|
-
id:
|
|
74902
|
+
id: requiredString2(record, "id", "message"),
|
|
74379
74903
|
leaseId,
|
|
74380
74904
|
timestampMs: nonNegativeInteger(record.timestamp_ms, "message timestamp_ms"),
|
|
74381
74905
|
attempts: nonNegativeInteger(record.attempts, "message attempts"),
|
|
@@ -74515,10 +75039,10 @@ async function ackCloudflareQueueMessages(client, options) {
|
|
|
74515
75039
|
|
|
74516
75040
|
// ../api-routes/src/traffic-event-ingest.ts
|
|
74517
75041
|
import crypto43 from "crypto";
|
|
74518
|
-
import { and as and47, eq as eq53, gte as gte13, lt as lt9, lte as lte12, sql as
|
|
75042
|
+
import { and as and47, eq as eq53, gte as gte13, lt as lt9, lte as lte12, sql as sql22 } from "drizzle-orm";
|
|
74519
75043
|
|
|
74520
75044
|
// ../api-routes/src/raw-event-sample-retention.ts
|
|
74521
|
-
import { and as and46, eq as eq52, lt as lt8, sql as
|
|
75045
|
+
import { and as and46, eq as eq52, lt as lt8, sql as sql21 } from "drizzle-orm";
|
|
74522
75046
|
var RAW_EVENT_SAMPLE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
74523
75047
|
var RAW_EVENT_SAMPLE_RETENTION_SWEEP_INTERVAL_MS = 24 * 60 * 6e4;
|
|
74524
75048
|
var CANONICAL_UTC_TIMESTAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9]Z";
|
|
@@ -74529,16 +75053,16 @@ function rawEventSampleRetentionCutoff(referenceAt) {
|
|
|
74529
75053
|
}
|
|
74530
75054
|
function deleteExpiredRawEventSamples(db, cutoff, sourceId) {
|
|
74531
75055
|
const sourceScope = sourceId === void 0 ? void 0 : eq52(rawEventSamples.sourceId, sourceId);
|
|
74532
|
-
db.delete(rawEventSamples).where(and46(sourceScope,
|
|
75056
|
+
db.delete(rawEventSamples).where(and46(sourceScope, sql21`julianday(${rawEventSamples.ts}) IS NULL`)).run();
|
|
74533
75057
|
db.delete(rawEventSamples).where(and46(
|
|
74534
75058
|
sourceScope,
|
|
74535
|
-
|
|
75059
|
+
sql21`${rawEventSamples.ts} GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
74536
75060
|
lt8(rawEventSamples.ts, cutoff)
|
|
74537
75061
|
)).run();
|
|
74538
75062
|
db.delete(rawEventSamples).where(and46(
|
|
74539
75063
|
sourceScope,
|
|
74540
|
-
|
|
74541
|
-
|
|
75064
|
+
sql21`${rawEventSamples.ts} NOT GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
75065
|
+
sql21`julianday(${rawEventSamples.ts}) < julianday(${cutoff})`
|
|
74542
75066
|
)).run();
|
|
74543
75067
|
}
|
|
74544
75068
|
function enforceRawEventSampleRetention(db, sourceId, referenceAt) {
|
|
@@ -74584,7 +75108,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74584
75108
|
let aiReferralHits = 0;
|
|
74585
75109
|
let unknownHits = 0;
|
|
74586
75110
|
opts.db.transaction((tx) => {
|
|
74587
|
-
tx.update(trafficSources).set({ updatedAt:
|
|
75111
|
+
tx.update(trafficSources).set({ updatedAt: sql22`${trafficSources.updatedAt}` }).where(eq53(trafficSources.id, opts.sourceId)).run();
|
|
74588
75112
|
const source = tx.select().from(trafficSources).where(eq53(trafficSources.id, opts.sourceId)).get();
|
|
74589
75113
|
opts.validateSource(source);
|
|
74590
75114
|
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
@@ -74641,7 +75165,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74641
75165
|
crawlerEventsHourly.status
|
|
74642
75166
|
],
|
|
74643
75167
|
set: {
|
|
74644
|
-
hits:
|
|
75168
|
+
hits: sql22`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
74645
75169
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
74646
75170
|
updatedAt: opts.receivedAt
|
|
74647
75171
|
}
|
|
@@ -74672,7 +75196,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74672
75196
|
crawlerVerificationManifestsHourly.manifestId
|
|
74673
75197
|
],
|
|
74674
75198
|
set: {
|
|
74675
|
-
hits:
|
|
75199
|
+
hits: sql22`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
74676
75200
|
manifestJson: bucket.verificationManifest,
|
|
74677
75201
|
updatedAt: opts.receivedAt
|
|
74678
75202
|
}
|
|
@@ -74706,7 +75230,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74706
75230
|
aiUserFetchEventsHourly.status
|
|
74707
75231
|
],
|
|
74708
75232
|
set: {
|
|
74709
|
-
hits:
|
|
75233
|
+
hits: sql22`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
74710
75234
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
74711
75235
|
updatedAt: opts.receivedAt
|
|
74712
75236
|
}
|
|
@@ -74737,7 +75261,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74737
75261
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
74738
75262
|
],
|
|
74739
75263
|
set: {
|
|
74740
|
-
hits:
|
|
75264
|
+
hits: sql22`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
74741
75265
|
manifestJson: bucket.verificationManifest,
|
|
74742
75266
|
updatedAt: opts.receivedAt
|
|
74743
75267
|
}
|
|
@@ -74775,9 +75299,9 @@ function writeTrafficEventBatch(opts) {
|
|
|
74775
75299
|
aiReferralEventsHourly.status
|
|
74776
75300
|
],
|
|
74777
75301
|
set: {
|
|
74778
|
-
sessionsOrHits:
|
|
74779
|
-
paidSessionsOrHits:
|
|
74780
|
-
organicSessionsOrHits:
|
|
75302
|
+
sessionsOrHits: sql22`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
75303
|
+
paidSessionsOrHits: sql22`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
75304
|
+
organicSessionsOrHits: sql22`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
74781
75305
|
updatedAt: opts.receivedAt
|
|
74782
75306
|
}
|
|
74783
75307
|
}).run();
|
|
@@ -74792,7 +75316,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74792
75316
|
if (!hour || hourlySampleLimit === 0) continue;
|
|
74793
75317
|
let remaining = remainingSamplesByHour.get(hour.start);
|
|
74794
75318
|
if (remaining === void 0) {
|
|
74795
|
-
const existing = tx.select({ total:
|
|
75319
|
+
const existing = tx.select({ total: sql22`COUNT(*)` }).from(rawEventSamples).where(and47(
|
|
74796
75320
|
eq53(rawEventSamples.sourceId, opts.sourceId),
|
|
74797
75321
|
gte13(rawEventSamples.ts, hour.start),
|
|
74798
75322
|
lt9(rawEventSamples.ts, hour.end)
|
|
@@ -74852,7 +75376,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74852
75376
|
}
|
|
74853
75377
|
|
|
74854
75378
|
// ../api-routes/src/traffic-sync-lease.ts
|
|
74855
|
-
import { and as and48, eq as eq54, isNull as isNull9, lte as lte13, or as
|
|
75379
|
+
import { and as and48, eq as eq54, isNull as isNull9, lte as lte13, or as or12 } from "drizzle-orm";
|
|
74856
75380
|
function leaseExpiresAt(now, ttlMs) {
|
|
74857
75381
|
const nowMs = Date.parse(now);
|
|
74858
75382
|
if (!Number.isFinite(nowMs)) throw new RangeError("Traffic sync lease now must be an ISO instant");
|
|
@@ -74867,7 +75391,7 @@ function tryClaimTrafficSyncLease(input) {
|
|
|
74867
75391
|
syncLeaseExpiresAt: expiresAt
|
|
74868
75392
|
}).where(and48(
|
|
74869
75393
|
eq54(trafficSources.id, input.sourceId),
|
|
74870
|
-
|
|
75394
|
+
or12(
|
|
74871
75395
|
isNull9(trafficSources.syncLeaseOwner),
|
|
74872
75396
|
lte13(trafficSources.syncLeaseExpiresAt, input.now),
|
|
74873
75397
|
eq54(trafficSources.syncLeaseOwner, input.owner)
|
|
@@ -75118,7 +75642,7 @@ function crawlerVerificationProvenance(db, projectId, events) {
|
|
|
75118
75642
|
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
75119
75643
|
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
75120
75644
|
hits: crawlerVerificationManifestsHourly.hits
|
|
75121
|
-
}).from(crawlerVerificationManifestsHourly).where(
|
|
75645
|
+
}).from(crawlerVerificationManifestsHourly).where(sql23`(
|
|
75122
75646
|
${crawlerVerificationManifestsHourly.projectId},
|
|
75123
75647
|
${crawlerVerificationManifestsHourly.sourceId},
|
|
75124
75648
|
${crawlerVerificationManifestsHourly.tsHour},
|
|
@@ -75152,7 +75676,7 @@ function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
|
75152
75676
|
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
75153
75677
|
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
75154
75678
|
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
75155
|
-
}).from(aiUserFetchVerificationManifestsHourly).where(
|
|
75679
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql23`(
|
|
75156
75680
|
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
75157
75681
|
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
75158
75682
|
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
@@ -75416,7 +75940,7 @@ async function runBackfillTask(options) {
|
|
|
75416
75940
|
crawlerEventsHourly.status
|
|
75417
75941
|
],
|
|
75418
75942
|
set: {
|
|
75419
|
-
hits:
|
|
75943
|
+
hits: sql23`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
75420
75944
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
75421
75945
|
updatedAt: finishedAt
|
|
75422
75946
|
}
|
|
@@ -75447,7 +75971,7 @@ async function runBackfillTask(options) {
|
|
|
75447
75971
|
crawlerVerificationManifestsHourly.manifestId
|
|
75448
75972
|
],
|
|
75449
75973
|
set: {
|
|
75450
|
-
hits:
|
|
75974
|
+
hits: sql23`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
75451
75975
|
manifestJson: bucket.verificationManifest,
|
|
75452
75976
|
updatedAt: finishedAt
|
|
75453
75977
|
}
|
|
@@ -75480,7 +76004,7 @@ async function runBackfillTask(options) {
|
|
|
75480
76004
|
aiUserFetchEventsHourly.status
|
|
75481
76005
|
],
|
|
75482
76006
|
set: {
|
|
75483
|
-
hits:
|
|
76007
|
+
hits: sql23`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
75484
76008
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
75485
76009
|
updatedAt: finishedAt
|
|
75486
76010
|
}
|
|
@@ -75511,7 +76035,7 @@ async function runBackfillTask(options) {
|
|
|
75511
76035
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
75512
76036
|
],
|
|
75513
76037
|
set: {
|
|
75514
|
-
hits:
|
|
76038
|
+
hits: sql23`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
75515
76039
|
manifestJson: bucket.verificationManifest,
|
|
75516
76040
|
updatedAt: finishedAt
|
|
75517
76041
|
}
|
|
@@ -77115,7 +77639,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77115
77639
|
crawlerEventsHourly.status
|
|
77116
77640
|
],
|
|
77117
77641
|
set: {
|
|
77118
|
-
hits:
|
|
77642
|
+
hits: sql23`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
77119
77643
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77120
77644
|
updatedAt: finishedAt
|
|
77121
77645
|
}
|
|
@@ -77146,7 +77670,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77146
77670
|
crawlerVerificationManifestsHourly.manifestId
|
|
77147
77671
|
],
|
|
77148
77672
|
set: {
|
|
77149
|
-
hits:
|
|
77673
|
+
hits: sql23`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77150
77674
|
manifestJson: bucket.verificationManifest,
|
|
77151
77675
|
updatedAt: finishedAt
|
|
77152
77676
|
}
|
|
@@ -77180,7 +77704,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77180
77704
|
aiUserFetchEventsHourly.status
|
|
77181
77705
|
],
|
|
77182
77706
|
set: {
|
|
77183
|
-
hits:
|
|
77707
|
+
hits: sql23`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
77184
77708
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77185
77709
|
updatedAt: finishedAt
|
|
77186
77710
|
}
|
|
@@ -77211,7 +77735,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77211
77735
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
77212
77736
|
],
|
|
77213
77737
|
set: {
|
|
77214
|
-
hits:
|
|
77738
|
+
hits: sql23`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77215
77739
|
manifestJson: bucket.verificationManifest,
|
|
77216
77740
|
updatedAt: finishedAt
|
|
77217
77741
|
}
|
|
@@ -77252,9 +77776,9 @@ async function trafficRoutes(app, opts) {
|
|
|
77252
77776
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
77253
77777
|
// already there rather than overwriting a single label.
|
|
77254
77778
|
set: {
|
|
77255
|
-
sessionsOrHits:
|
|
77256
|
-
paidSessionsOrHits:
|
|
77257
|
-
organicSessionsOrHits:
|
|
77779
|
+
sessionsOrHits: sql23`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
77780
|
+
paidSessionsOrHits: sql23`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
77781
|
+
organicSessionsOrHits: sql23`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
77258
77782
|
updatedAt: finishedAt
|
|
77259
77783
|
}
|
|
77260
77784
|
}).run();
|
|
@@ -77547,7 +78071,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77547
78071
|
function buildSourceDetail(projectId, row, since) {
|
|
77548
78072
|
const crawlerPathRows = app.db.select({
|
|
77549
78073
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
77550
|
-
hits:
|
|
78074
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
77551
78075
|
}).from(crawlerEventsHourly).where(
|
|
77552
78076
|
and49(
|
|
77553
78077
|
eq55(crawlerEventsHourly.sourceId, row.id),
|
|
@@ -77558,23 +78082,23 @@ async function trafficRoutes(app, opts) {
|
|
|
77558
78082
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
77559
78083
|
);
|
|
77560
78084
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
77561
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
78085
|
+
const aiUserFetchTotals = app.db.select({ total: sql23`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
77562
78086
|
and49(
|
|
77563
78087
|
eq55(aiUserFetchEventsHourly.sourceId, row.id),
|
|
77564
78088
|
gte14(aiUserFetchEventsHourly.tsHour, since)
|
|
77565
78089
|
)
|
|
77566
78090
|
).get();
|
|
77567
78091
|
const aiTotals = app.db.select({
|
|
77568
|
-
total:
|
|
77569
|
-
nonhop:
|
|
77570
|
-
landed:
|
|
78092
|
+
total: sql23`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
78093
|
+
nonhop: sql23`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
78094
|
+
landed: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
77571
78095
|
}).from(aiReferralEventsHourly).where(
|
|
77572
78096
|
and49(
|
|
77573
78097
|
eq55(aiReferralEventsHourly.sourceId, row.id),
|
|
77574
78098
|
gte14(aiReferralEventsHourly.tsHour, since)
|
|
77575
78099
|
)
|
|
77576
78100
|
).get();
|
|
77577
|
-
const sampleTotals = app.db.select({ total:
|
|
78101
|
+
const sampleTotals = app.db.select({ total: sql23`COUNT(*)` }).from(rawEventSamples).where(
|
|
77578
78102
|
and49(
|
|
77579
78103
|
eq55(rawEventSamples.sourceId, row.id),
|
|
77580
78104
|
gte14(rawEventSamples.ts, since)
|
|
@@ -77763,15 +78287,15 @@ async function trafficRoutes(app, opts) {
|
|
|
77763
78287
|
const crawlerWhere = and49(...crawlerFilters);
|
|
77764
78288
|
const pathTotals = app.db.select({
|
|
77765
78289
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
77766
|
-
hits:
|
|
77767
|
-
rows:
|
|
78290
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
78291
|
+
rows: sql23`COUNT(*)`
|
|
77768
78292
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
77769
78293
|
crawlerSegments = segmentCrawlerHits(
|
|
77770
78294
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
77771
78295
|
);
|
|
77772
78296
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
77773
78297
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
77774
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
78298
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql23`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
77775
78299
|
const distinctPaths = pathTotals.map((r) => ({ pathNormalized: r.pathNormalized }));
|
|
77776
78300
|
const contentPaths = [];
|
|
77777
78301
|
const infraPaths = [];
|
|
@@ -77783,12 +78307,12 @@ async function trafficRoutes(app, opts) {
|
|
|
77783
78307
|
const listed = useContentList ? contentPaths : infraPaths;
|
|
77784
78308
|
const IN_LIST_CAP = 2e4;
|
|
77785
78309
|
const listFitsInSql = listed.length <= IN_LIST_CAP;
|
|
77786
|
-
const contentExpr = listed.length === 0 ? useContentList ?
|
|
78310
|
+
const contentExpr = listed.length === 0 ? useContentList ? sql23`0` : sql23`${crawlerEventsHourly.hits}` : useContentList ? sql23`CASE WHEN ${inArray21(crawlerEventsHourly.pathNormalized, listed)} THEN ${crawlerEventsHourly.hits} ELSE 0 END` : sql23`CASE WHEN ${inArray21(crawlerEventsHourly.pathNormalized, listed)} THEN 0 ELSE ${crawlerEventsHourly.hits} END`;
|
|
77787
78311
|
if (listFitsInSql) {
|
|
77788
78312
|
const crawlerSeries = app.db.select({
|
|
77789
78313
|
bucket: crawlerSeriesBucket,
|
|
77790
|
-
hits:
|
|
77791
|
-
content:
|
|
78314
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
78315
|
+
content: sql23`COALESCE(SUM(${contentExpr}), 0)`
|
|
77792
78316
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
77793
78317
|
for (const row of crawlerSeries) {
|
|
77794
78318
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -77800,7 +78324,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77800
78324
|
const grouped = app.db.select({
|
|
77801
78325
|
bucket: crawlerSeriesBucket,
|
|
77802
78326
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
77803
|
-
hits:
|
|
78327
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
77804
78328
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket, crawlerEventsHourly.pathNormalized).all();
|
|
77805
78329
|
for (const row of grouped) {
|
|
77806
78330
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -77837,15 +78361,15 @@ async function trafficRoutes(app, opts) {
|
|
|
77837
78361
|
if (sourceIdParam) userFetchFilters.push(eq55(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
77838
78362
|
const userFetchWhere = and49(...userFetchFilters);
|
|
77839
78363
|
const total = app.db.select({
|
|
77840
|
-
total:
|
|
77841
|
-
rows:
|
|
78364
|
+
total: sql23`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
78365
|
+
rows: sql23`COUNT(*)`
|
|
77842
78366
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
77843
78367
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
77844
78368
|
totalEventRows += Number(total?.rows ?? 0);
|
|
77845
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
78369
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql23`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
77846
78370
|
const userFetchSeries = app.db.select({
|
|
77847
78371
|
bucket: userFetchSeriesBucket,
|
|
77848
|
-
hits:
|
|
78372
|
+
hits: sql23`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
77849
78373
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
77850
78374
|
for (const row of userFetchSeries) {
|
|
77851
78375
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -77876,12 +78400,12 @@ async function trafficRoutes(app, opts) {
|
|
|
77876
78400
|
if (sourceIdParam) aiFilters.push(eq55(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
77877
78401
|
const aiWhere = and49(...aiFilters);
|
|
77878
78402
|
const total = app.db.select({
|
|
77879
|
-
total:
|
|
77880
|
-
nonhop:
|
|
77881
|
-
landed:
|
|
77882
|
-
paid:
|
|
77883
|
-
organic:
|
|
77884
|
-
rows:
|
|
78403
|
+
total: sql23`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
78404
|
+
nonhop: sql23`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
78405
|
+
landed: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
78406
|
+
paid: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.paidSessionsOrHits} ELSE 0 END), 0)`,
|
|
78407
|
+
organic: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.organicSessionsOrHits} ELSE 0 END), 0)`,
|
|
78408
|
+
rows: sql23`COUNT(*)`
|
|
77885
78409
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
77886
78410
|
aiReferralTotalHits = Number(total?.total ?? 0);
|
|
77887
78411
|
aiReferralNonHopHits = Number(total?.nonhop ?? 0);
|
|
@@ -77892,11 +78416,11 @@ async function trafficRoutes(app, opts) {
|
|
|
77892
78416
|
Number(total?.organic ?? 0)
|
|
77893
78417
|
);
|
|
77894
78418
|
totalEventRows += Number(total?.rows ?? 0);
|
|
77895
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
78419
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql23`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
77896
78420
|
const referralSeries = app.db.select({
|
|
77897
78421
|
bucket: referralSeriesBucket,
|
|
77898
|
-
hits:
|
|
77899
|
-
landed:
|
|
78422
|
+
hits: sql23`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
78423
|
+
landed: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
77900
78424
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
77901
78425
|
for (const row of referralSeries) {
|
|
77902
78426
|
const point = trafficSeriesPoint(seriesByBucket, row.bucket);
|
|
@@ -77951,9 +78475,9 @@ async function trafficRoutes(app, opts) {
|
|
|
77951
78475
|
const points = completeTrafficSeries(since, until, granularity, seriesByBucket);
|
|
77952
78476
|
const coverageScope = (col, srcCol) => sourceIdParam ? and49(eq55(col, project.id), eq55(srcCol, sourceIdParam)) : eq55(col, project.id);
|
|
77953
78477
|
const firsts = [
|
|
77954
|
-
app.db.select({ v:
|
|
77955
|
-
app.db.select({ v:
|
|
77956
|
-
app.db.select({ v:
|
|
78478
|
+
app.db.select({ v: sql23`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(coverageScope(crawlerEventsHourly.projectId, crawlerEventsHourly.sourceId)).get()?.v,
|
|
78479
|
+
app.db.select({ v: sql23`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(coverageScope(aiUserFetchEventsHourly.projectId, aiUserFetchEventsHourly.sourceId)).get()?.v,
|
|
78480
|
+
app.db.select({ v: sql23`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(coverageScope(aiReferralEventsHourly.projectId, aiReferralEventsHourly.sourceId)).get()?.v
|
|
77957
78481
|
].filter((v) => typeof v === "string" && v.length > 0);
|
|
77958
78482
|
const coverageStart = firsts.length ? firsts.slice().sort()[0] : null;
|
|
77959
78483
|
const bucketStartMs = (bucket) => Date.parse(
|
|
@@ -79759,7 +80283,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
79759
80283
|
];
|
|
79760
80284
|
|
|
79761
80285
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
79762
|
-
import { and as and52, eq as eq61, gte as gte15, inArray as inArray22, ne as ne9, sql as
|
|
80286
|
+
import { and as and52, eq as eq61, gte as gte15, inArray as inArray22, ne as ne9, sql as sql24 } from "drizzle-orm";
|
|
79763
80287
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
79764
80288
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
79765
80289
|
function isCloudflareDirectPush(source) {
|
|
@@ -79904,7 +80428,7 @@ var recentDataCheck = {
|
|
|
79904
80428
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
79905
80429
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
79906
80430
|
const recentCrawlers = Number(
|
|
79907
|
-
ctx.db.select({ total:
|
|
80431
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
79908
80432
|
and52(
|
|
79909
80433
|
eq61(crawlerEventsHourly.projectId, ctx.project.id),
|
|
79910
80434
|
inArray22(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79913,7 +80437,7 @@ var recentDataCheck = {
|
|
|
79913
80437
|
).get()?.total ?? 0
|
|
79914
80438
|
);
|
|
79915
80439
|
const recentReferrals = Number(
|
|
79916
|
-
ctx.db.select({ total:
|
|
80440
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
79917
80441
|
and52(
|
|
79918
80442
|
eq61(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
79919
80443
|
inArray22(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79922,7 +80446,7 @@ var recentDataCheck = {
|
|
|
79922
80446
|
).get()?.total ?? 0
|
|
79923
80447
|
);
|
|
79924
80448
|
const recentUserFetches = Number(
|
|
79925
|
-
ctx.db.select({ total:
|
|
80449
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
79926
80450
|
and52(
|
|
79927
80451
|
eq61(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
79928
80452
|
inArray22(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79948,7 +80472,7 @@ var recentDataCheck = {
|
|
|
79948
80472
|
};
|
|
79949
80473
|
}
|
|
79950
80474
|
const olderCrawlers = Number(
|
|
79951
|
-
ctx.db.select({ total:
|
|
80475
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
79952
80476
|
and52(
|
|
79953
80477
|
eq61(crawlerEventsHourly.projectId, ctx.project.id),
|
|
79954
80478
|
inArray22(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79957,7 +80481,7 @@ var recentDataCheck = {
|
|
|
79957
80481
|
).get()?.total ?? 0
|
|
79958
80482
|
);
|
|
79959
80483
|
const olderReferrals = Number(
|
|
79960
|
-
ctx.db.select({ total:
|
|
80484
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
79961
80485
|
and52(
|
|
79962
80486
|
eq61(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
79963
80487
|
inArray22(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79966,7 +80490,7 @@ var recentDataCheck = {
|
|
|
79966
80490
|
).get()?.total ?? 0
|
|
79967
80491
|
);
|
|
79968
80492
|
const olderUserFetches = Number(
|
|
79969
|
-
ctx.db.select({ total:
|
|
80493
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
79970
80494
|
and52(
|
|
79971
80495
|
eq61(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
79972
80496
|
inArray22(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -80885,7 +81409,7 @@ async function doctorRoutes(app, opts) {
|
|
|
80885
81409
|
|
|
80886
81410
|
// ../api-routes/src/discovery/routes.ts
|
|
80887
81411
|
import crypto46 from "crypto";
|
|
80888
|
-
import { and as and53, desc as desc26, eq as eq62, gte as gte16, inArray as inArray23, isNull as isNull10, or as
|
|
81412
|
+
import { and as and53, desc as desc26, eq as eq62, gte as gte16, inArray as inArray23, isNull as isNull10, or as or13 } from "drizzle-orm";
|
|
80889
81413
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
80890
81414
|
async function discoveryRoutes(app, opts) {
|
|
80891
81415
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -80935,7 +81459,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
80935
81459
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
80936
81460
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
80937
81461
|
// upgrade, never a wrong reuse.
|
|
80938
|
-
locations.length === 0 ?
|
|
81462
|
+
locations.length === 0 ? or13(isNull10(discoverySessions.locations), eq62(discoverySessions.locations, locations)) : eq62(discoverySessions.locations, locations),
|
|
80939
81463
|
// Seed provider set is identity: a different phrasing distribution
|
|
80940
81464
|
// must never reuse another set's session. Null (= Gemini-only
|
|
80941
81465
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
@@ -81565,7 +82089,7 @@ function dedupeStrings(input) {
|
|
|
81565
82089
|
|
|
81566
82090
|
// ../api-routes/src/technical-aeo.ts
|
|
81567
82091
|
import crypto48 from "crypto";
|
|
81568
|
-
import { and as and54, asc as asc11, count as count2, desc as desc27, eq as eq64, inArray as inArray24, isNotNull as isNotNull5, isNull as isNull11, lt as lt10, or as
|
|
82092
|
+
import { and as and54, asc as asc11, count as count2, desc as desc27, eq as eq64, inArray as inArray24, isNotNull as isNotNull5, isNull as isNull11, lt as lt10, or as or14, sql as sql25 } from "drizzle-orm";
|
|
81569
82093
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
81570
82094
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
81571
82095
|
...SiteCrawlFetchedStates,
|
|
@@ -81784,7 +82308,7 @@ function linkKindFilter(linkKind) {
|
|
|
81784
82308
|
}
|
|
81785
82309
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
81786
82310
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
81787
|
-
var PAGE_CHANGE_PREDICATE =
|
|
82311
|
+
var PAGE_CHANGE_PREDICATE = sql25`(
|
|
81788
82312
|
current.url IS NOT previous.url
|
|
81789
82313
|
OR current.final_url IS NOT previous.final_url
|
|
81790
82314
|
OR current.path IS NOT previous.path
|
|
@@ -81806,7 +82330,7 @@ var PAGE_CHANGE_PREDICATE = sql24`(
|
|
|
81806
82330
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
81807
82331
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
81808
82332
|
)`;
|
|
81809
|
-
var LINK_CHANGE_PREDICATE =
|
|
82333
|
+
var LINK_CHANGE_PREDICATE = sql25`(
|
|
81810
82334
|
current.source_node_key IS NOT previous.source_node_key
|
|
81811
82335
|
OR current.source_url IS NOT previous.source_url
|
|
81812
82336
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -82335,7 +82859,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82335
82859
|
eq64(siteCrawlEdges.attemptId, scope.attemptId),
|
|
82336
82860
|
eq64(siteCrawlEdges.internal, true),
|
|
82337
82861
|
isNotNull5(siteCrawlEdges.targetNodeKey),
|
|
82338
|
-
|
|
82862
|
+
or14(
|
|
82339
82863
|
inArray24(siteCrawlEdges.sourceNodeKey, frontier),
|
|
82340
82864
|
inArray24(siteCrawlEdges.targetNodeKey, frontier)
|
|
82341
82865
|
)
|
|
@@ -82582,7 +83106,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82582
83106
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
82583
83107
|
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq64(siteCrawlSnapshots.runId, runs.id)).where(and54(
|
|
82584
83108
|
...completeSnapshotFilters,
|
|
82585
|
-
|
|
83109
|
+
or14(
|
|
82586
83110
|
lt10(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
82587
83111
|
and54(
|
|
82588
83112
|
eq64(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
@@ -82663,9 +83187,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82663
83187
|
const summary3 = { pages: zero2(), links: zero2() };
|
|
82664
83188
|
if (scopeFilter !== "links") {
|
|
82665
83189
|
if (includeAdded || includeChanged) {
|
|
82666
|
-
const addedExpression = includeAdded ?
|
|
82667
|
-
const changedExpression = includeChanged ?
|
|
82668
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
83190
|
+
const addedExpression = includeAdded ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83191
|
+
const changedExpression = includeChanged ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83192
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql25`
|
|
82669
83193
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
82670
83194
|
FROM site_crawl_pages AS current
|
|
82671
83195
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -82681,7 +83205,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82681
83205
|
summary3.pages.changed = Number(counts2.changed);
|
|
82682
83206
|
}
|
|
82683
83207
|
if (includeRemoved) {
|
|
82684
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
83208
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql25`
|
|
82685
83209
|
SELECT COUNT(*) AS removed
|
|
82686
83210
|
FROM site_crawl_pages AS previous
|
|
82687
83211
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -82699,9 +83223,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82699
83223
|
}
|
|
82700
83224
|
if (scopeFilter !== "pages") {
|
|
82701
83225
|
if (includeAdded || includeChanged) {
|
|
82702
|
-
const addedExpression = includeAdded ?
|
|
82703
|
-
const changedExpression = includeChanged ?
|
|
82704
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
83226
|
+
const addedExpression = includeAdded ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83227
|
+
const changedExpression = includeChanged ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83228
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql25`
|
|
82705
83229
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
82706
83230
|
FROM site_crawl_edges AS current
|
|
82707
83231
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -82719,7 +83243,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82719
83243
|
summary3.links.changed = Number(counts2.changed);
|
|
82720
83244
|
}
|
|
82721
83245
|
if (includeRemoved) {
|
|
82722
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
83246
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql25`
|
|
82723
83247
|
SELECT COUNT(*) AS removed
|
|
82724
83248
|
FROM site_crawl_edges AS previous
|
|
82725
83249
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -82749,48 +83273,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82749
83273
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
82750
83274
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
82751
83275
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
82752
|
-
const internalCurrent = entity === "link" ?
|
|
82753
|
-
const internalPreviousJoin = entity === "link" ?
|
|
82754
|
-
const internalPrevious = entity === "link" ?
|
|
82755
|
-
const internalCurrentJoin = entity === "link" ?
|
|
83276
|
+
const internalCurrent = entity === "link" ? sql25`AND current.internal = 1` : sql25``;
|
|
83277
|
+
const internalPreviousJoin = entity === "link" ? sql25`AND previous.internal = 1` : sql25``;
|
|
83278
|
+
const internalPrevious = entity === "link" ? sql25`AND previous.internal = 1` : sql25``;
|
|
83279
|
+
const internalCurrentJoin = entity === "link" ? sql25`AND current.internal = 1` : sql25``;
|
|
82756
83280
|
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
82757
|
-
const afterKind = changeFilter === "added" ?
|
|
82758
|
-
const afterRows = includeAfter ? app.db.all(
|
|
83281
|
+
const afterKind = changeFilter === "added" ? sql25`previous.id IS NULL` : changeFilter === "changed" ? sql25`previous.id IS NOT NULL AND ${difference2}` : sql25`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
83282
|
+
const afterRows = includeAfter ? app.db.all(sql25`
|
|
82759
83283
|
SELECT ${entity} AS entity,
|
|
82760
83284
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
82761
|
-
current.${
|
|
82762
|
-
FROM ${
|
|
82763
|
-
LEFT JOIN ${
|
|
83285
|
+
current.${sql25.raw(keyColumn)} AS key
|
|
83286
|
+
FROM ${sql25.raw(table)} AS current
|
|
83287
|
+
LEFT JOIN ${sql25.raw(table)} AS previous
|
|
82764
83288
|
ON previous.project_id = ${beforeScope.projectId}
|
|
82765
83289
|
AND previous.run_id = ${beforeScope.runId}
|
|
82766
83290
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
82767
83291
|
${internalPreviousJoin}
|
|
82768
|
-
AND previous.${
|
|
83292
|
+
AND previous.${sql25.raw(keyColumn)} = current.${sql25.raw(keyColumn)}
|
|
82769
83293
|
WHERE current.project_id = ${afterScope.projectId}
|
|
82770
83294
|
AND current.run_id = ${afterScope.runId}
|
|
82771
83295
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
82772
83296
|
${internalCurrent}
|
|
82773
|
-
AND current.${
|
|
83297
|
+
AND current.${sql25.raw(keyColumn)} > ${afterKey}
|
|
82774
83298
|
AND (${afterKind})
|
|
82775
|
-
ORDER BY current.${
|
|
83299
|
+
ORDER BY current.${sql25.raw(keyColumn)} ASC
|
|
82776
83300
|
LIMIT ${wanted2}
|
|
82777
83301
|
`) : [];
|
|
82778
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
82779
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
82780
|
-
FROM ${
|
|
82781
|
-
LEFT JOIN ${
|
|
83302
|
+
const removedRows = includeRemoved ? app.db.all(sql25`
|
|
83303
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql25.raw(keyColumn)} AS key
|
|
83304
|
+
FROM ${sql25.raw(table)} AS previous
|
|
83305
|
+
LEFT JOIN ${sql25.raw(table)} AS current
|
|
82782
83306
|
ON current.project_id = ${afterScope.projectId}
|
|
82783
83307
|
AND current.run_id = ${afterScope.runId}
|
|
82784
83308
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
82785
83309
|
${internalCurrentJoin}
|
|
82786
|
-
AND current.${
|
|
83310
|
+
AND current.${sql25.raw(keyColumn)} = previous.${sql25.raw(keyColumn)}
|
|
82787
83311
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
82788
83312
|
AND previous.run_id = ${beforeScope.runId}
|
|
82789
83313
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
82790
83314
|
${internalPrevious}
|
|
82791
|
-
AND previous.${
|
|
83315
|
+
AND previous.${sql25.raw(keyColumn)} > ${afterKey}
|
|
82792
83316
|
AND current.id IS NULL
|
|
82793
|
-
ORDER BY previous.${
|
|
83317
|
+
ORDER BY previous.${sql25.raw(keyColumn)} ASC
|
|
82794
83318
|
LIMIT ${wanted2}
|
|
82795
83319
|
`) : [];
|
|
82796
83320
|
const merged = [];
|
|
@@ -83179,8 +83703,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
83179
83703
|
eq64(siteCrawlEdges.internal, true),
|
|
83180
83704
|
linkKindFilter(linkKind)
|
|
83181
83705
|
];
|
|
83182
|
-
const inboundMatch = request.query.nodeKey && request.query.url ?
|
|
83183
|
-
const outboundMatch = request.query.nodeKey && request.query.url ?
|
|
83706
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or14(eq64(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq64(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq64(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq64(siteCrawlEdges.targetUrl, request.query.url);
|
|
83707
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or14(eq64(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq64(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq64(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq64(siteCrawlEdges.sourceUrl, request.query.url);
|
|
83184
83708
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
83185
83709
|
const inboundRows = app.db.select().from(siteCrawlEdges).where(and54(...scope, inboundMatch)).orderBy(asc11(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
83186
83710
|
const outboundRows = app.db.select().from(siteCrawlEdges).where(and54(...scope, outboundMatch)).orderBy(asc11(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
@@ -83230,7 +83754,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
83230
83754
|
// a fabricated broken link shown to a client is not worth the saved
|
|
83231
83755
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
83232
83756
|
// promise a page the list cannot fill.
|
|
83233
|
-
isNotNull5(
|
|
83757
|
+
isNotNull5(sql25`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
83234
83758
|
);
|
|
83235
83759
|
const total = app.db.select({ value: count2() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
83236
83760
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -83475,7 +83999,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
83475
83999
|
// ../api-routes/src/research.ts
|
|
83476
84000
|
import crypto49 from "crypto";
|
|
83477
84001
|
import { z as z5 } from "zod";
|
|
83478
|
-
import { and as and55, count as count3, desc as desc28, eq as eq65, gte as gte17, lt as lt11, or as
|
|
84002
|
+
import { and as and55, count as count3, desc as desc28, eq as eq65, gte as gte17, lt as lt11, or as or15, sql as sql26 } from "drizzle-orm";
|
|
83479
84003
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
83480
84004
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
83481
84005
|
async function researchRoutes(app, opts) {
|
|
@@ -83537,7 +84061,7 @@ async function researchRoutes(app, opts) {
|
|
|
83537
84061
|
eq65(researchRuns.projectId, project.id),
|
|
83538
84062
|
gte17(researchRuns.createdAt, start),
|
|
83539
84063
|
lt11(researchRuns.createdAt, end),
|
|
83540
|
-
|
|
84064
|
+
sql26`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
83541
84065
|
)).get()?.value ?? 0;
|
|
83542
84066
|
if (used >= viewerDailyRunLimit) {
|
|
83543
84067
|
throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
@@ -83546,7 +84070,7 @@ async function researchRoutes(app, opts) {
|
|
|
83546
84070
|
const id = crypto49.randomUUID();
|
|
83547
84071
|
tx.insert(researchRuns).values({ id, projectId: project.id, status: ResearchRunStatuses.queued, provider: providerName, requestedModel: requestedModel2, resolvedModel, location: location ?? null, totalQueries: input.queries.length, scope, template, idempotencyKey: input.idempotencyKey ?? null, requestHash: input.idempotencyKey ? requestHash2 : null, initiatedBy, createdAt: now }).run();
|
|
83548
84072
|
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id: crypto49.randomUUID(), researchRunId: id, position, queryText: query, queryClass: queryClasses[position] ?? null, status: ResearchQueryStatuses.queued, requestedModel: requestedModel2, resolvedModel, groundingSources: [], citedDomains: [], searchQueries: [], createdAt: now }).run();
|
|
83549
|
-
writeAuditLog(tx, { projectId: project.id, actor:
|
|
84073
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
83550
84074
|
return { reused: false, id, shouldDispatch: true };
|
|
83551
84075
|
});
|
|
83552
84076
|
const result = getDetail(app, project.id, decision.id);
|
|
@@ -83592,7 +84116,7 @@ async function researchRoutes(app, opts) {
|
|
|
83592
84116
|
eq65(researchRuns.projectId, project.id),
|
|
83593
84117
|
gte17(researchRuns.createdAt, start),
|
|
83594
84118
|
lt11(researchRuns.createdAt, end),
|
|
83595
|
-
|
|
84119
|
+
sql26`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
83596
84120
|
)).get()?.value ?? 0;
|
|
83597
84121
|
if (used + prepared.length > viewerDailyRunLimit) throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
83598
84122
|
}
|
|
@@ -83630,7 +84154,7 @@ async function researchRoutes(app, opts) {
|
|
|
83630
84154
|
searchQueries: [],
|
|
83631
84155
|
createdAt: now
|
|
83632
84156
|
}).run();
|
|
83633
|
-
writeAuditLog(tx, { projectId: project.id, actor:
|
|
84157
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
83634
84158
|
}
|
|
83635
84159
|
return { ids };
|
|
83636
84160
|
});
|
|
@@ -83645,7 +84169,7 @@ async function researchRoutes(app, opts) {
|
|
|
83645
84169
|
const cursor = request.query.cursor ? parseResearchCursor(request.query.cursor, project.id) : null;
|
|
83646
84170
|
const rows = app.db.select().from(researchRuns).where(and55(
|
|
83647
84171
|
eq65(researchRuns.projectId, project.id),
|
|
83648
|
-
cursor ?
|
|
84172
|
+
cursor ? or15(lt11(researchRuns.createdAt, cursor.createdAt), and55(eq65(researchRuns.createdAt, cursor.createdAt), lt11(researchRuns.id, cursor.id))) : void 0
|
|
83649
84173
|
)).orderBy(desc28(researchRuns.createdAt), desc28(researchRuns.id)).limit(limit + 1).all();
|
|
83650
84174
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
83651
84175
|
const last = runs2.at(-1);
|
|
@@ -83933,6 +84457,7 @@ async function apiRoutes(app, opts) {
|
|
|
83933
84457
|
});
|
|
83934
84458
|
}
|
|
83935
84459
|
await app.register(async (api) => {
|
|
84460
|
+
registerRequestContext(api);
|
|
83936
84461
|
await api.register(rateLimit, {
|
|
83937
84462
|
global: false,
|
|
83938
84463
|
hook: "preHandler",
|
|
@@ -84054,6 +84579,7 @@ async function apiRoutes(app, opts) {
|
|
|
84054
84579
|
await api.register(notificationRoutes, {
|
|
84055
84580
|
allowLoopbackWebhooks: opts.allowLoopbackWebhooks
|
|
84056
84581
|
});
|
|
84582
|
+
await api.register(operationalLogsRoutes, { listOperationalLogs: opts.listOperationalLogs });
|
|
84057
84583
|
await api.register(telemetryRoutes, {
|
|
84058
84584
|
getTelemetryStatus: opts.getTelemetryStatus,
|
|
84059
84585
|
setTelemetryEnabled: opts.setTelemetryEnabled,
|
|
@@ -84411,42 +84937,260 @@ function buildTrafficSourceValidators(opts) {
|
|
|
84411
84937
|
// src/intelligence-service.ts
|
|
84412
84938
|
import crypto51 from "crypto";
|
|
84413
84939
|
|
|
84414
|
-
// src/logger.ts
|
|
84940
|
+
// ../api-routes/src/runtime-logger.ts
|
|
84941
|
+
import { format } from "util";
|
|
84415
84942
|
var IS_TTY = process.stdout.isTTY === true;
|
|
84416
|
-
|
|
84417
|
-
|
|
84418
|
-
|
|
84419
|
-
|
|
84420
|
-
|
|
84421
|
-
|
|
84422
|
-
|
|
84423
|
-
|
|
84424
|
-
|
|
84425
|
-
|
|
84426
|
-
|
|
84427
|
-
|
|
84428
|
-
|
|
84429
|
-
|
|
84430
|
-
|
|
84431
|
-
|
|
84943
|
+
var MAX_IDENTITY_LENGTH = 512;
|
|
84944
|
+
var RESERVED_FIELDS = /* @__PURE__ */ new Set([
|
|
84945
|
+
"ts",
|
|
84946
|
+
"level",
|
|
84947
|
+
"module",
|
|
84948
|
+
"action",
|
|
84949
|
+
"msg",
|
|
84950
|
+
"requestId",
|
|
84951
|
+
"actor",
|
|
84952
|
+
"credentialId",
|
|
84953
|
+
"userAgent",
|
|
84954
|
+
"actorSession",
|
|
84955
|
+
"__runtimeRequestId"
|
|
84956
|
+
]);
|
|
84957
|
+
var listeners = /* @__PURE__ */ new Set();
|
|
84958
|
+
function addLogListener(listener) {
|
|
84959
|
+
listeners.add(listener);
|
|
84960
|
+
return () => listeners.delete(listener);
|
|
84432
84961
|
}
|
|
84433
84962
|
function createLogger(module) {
|
|
84434
|
-
|
|
84435
|
-
const
|
|
84436
|
-
|
|
84437
|
-
|
|
84438
|
-
|
|
84439
|
-
|
|
84440
|
-
|
|
84441
|
-
|
|
84442
|
-
|
|
84963
|
+
const emit = (level, action, context) => {
|
|
84964
|
+
const msg = context ? safeProperty(context, "msg") : void 0;
|
|
84965
|
+
emitRuntimeLog({ level, module, action, context, ...typeof msg === "string" ? { msg } : {} });
|
|
84966
|
+
};
|
|
84967
|
+
return {
|
|
84968
|
+
info: (action, ctx) => emit("info", action, ctx),
|
|
84969
|
+
warn: (action, ctx) => emit("warn", action, ctx),
|
|
84970
|
+
error: (action, ctx) => emit("error", action, ctx)
|
|
84971
|
+
};
|
|
84972
|
+
}
|
|
84973
|
+
function createFastifyLogger(options = {}) {
|
|
84974
|
+
return buildFastifyLogger(options.module ?? "HTTP", options.enabled !== false, {});
|
|
84975
|
+
}
|
|
84976
|
+
var LEVELS = { trace: 10, debug: 20, info: 30, warn: 40, error: 50, fatal: 60, silent: Infinity };
|
|
84977
|
+
function buildFastifyLogger(module, enabled, bindings, inheritedLevel = "info") {
|
|
84978
|
+
let level = inheritedLevel;
|
|
84979
|
+
const emit = (severity, args) => {
|
|
84980
|
+
if (LEVELS[severity] >= LEVELS[level]) logFastify(severity, module, enabled, bindings, args);
|
|
84981
|
+
};
|
|
84982
|
+
const logger = {
|
|
84983
|
+
get level() {
|
|
84984
|
+
return level;
|
|
84985
|
+
},
|
|
84986
|
+
set level(value) {
|
|
84987
|
+
if (Object.hasOwn(LEVELS, value)) level = value;
|
|
84988
|
+
},
|
|
84989
|
+
trace: (...args) => emit("trace", args),
|
|
84990
|
+
debug: (...args) => emit("debug", args),
|
|
84991
|
+
info: (...args) => emit("info", args),
|
|
84992
|
+
warn: (...args) => emit("warn", args),
|
|
84993
|
+
error: (...args) => emit("error", args),
|
|
84994
|
+
fatal: (...args) => emit("fatal", args),
|
|
84995
|
+
silent: () => void 0,
|
|
84996
|
+
child: (childBindings) => buildFastifyLogger(module, enabled, mergeBindings(bindings, childBindings), level)
|
|
84997
|
+
};
|
|
84998
|
+
return logger;
|
|
84999
|
+
}
|
|
85000
|
+
function logFastify(level, module, enabled, bindings, args) {
|
|
85001
|
+
const { context, msg } = normalizePinoArgs(args);
|
|
85002
|
+
emitRuntimeLog({
|
|
85003
|
+
level,
|
|
85004
|
+
module,
|
|
85005
|
+
action: `fastify.${level}`,
|
|
85006
|
+
msg,
|
|
85007
|
+
context: { ...bindings, ...context },
|
|
85008
|
+
trustedRequestId: asBoundString(bindings.__runtimeRequestId),
|
|
85009
|
+
enabled
|
|
85010
|
+
});
|
|
85011
|
+
}
|
|
85012
|
+
function normalizePinoArgs(args) {
|
|
85013
|
+
const first = args[0];
|
|
85014
|
+
const second = args[1];
|
|
85015
|
+
if (typeof first === "string") return { context: {}, msg: safeFormat(first, args.slice(1)) };
|
|
85016
|
+
if (first instanceof Error) {
|
|
85017
|
+
return { context: { err: first }, msg: typeof second === "string" ? safeFormat(second, args.slice(2)) : void 0 };
|
|
85018
|
+
}
|
|
85019
|
+
if (isObject(first)) {
|
|
85020
|
+
return { context: copyOwnData(first), msg: typeof second === "string" ? safeFormat(second, args.slice(2)) : void 0 };
|
|
85021
|
+
}
|
|
85022
|
+
return { context: {}, msg: typeof second === "string" ? safeFormat(second, args.slice(2)) : void 0 };
|
|
85023
|
+
}
|
|
85024
|
+
function emitRuntimeLog({
|
|
85025
|
+
level,
|
|
85026
|
+
module,
|
|
85027
|
+
action,
|
|
85028
|
+
msg,
|
|
85029
|
+
context,
|
|
85030
|
+
trustedRequestId,
|
|
85031
|
+
enabled = true
|
|
85032
|
+
}) {
|
|
85033
|
+
const safeContext = sanitizeContext(context);
|
|
85034
|
+
const error = safeContext.err ?? safeContext.error;
|
|
85035
|
+
const errorMessage = typeof error === "string" ? error : isRecord4(error) && typeof error.message === "string" ? error.message : void 0;
|
|
85036
|
+
const message = errorMessage && errorMessage !== msg ? [msg, errorMessage].filter(Boolean).join(": ") : msg;
|
|
85037
|
+
const entry = {
|
|
85038
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
85039
|
+
level,
|
|
85040
|
+
module: boundText(module, "Runtime"),
|
|
85041
|
+
action: boundText(action, "log"),
|
|
85042
|
+
...message === void 0 ? {} : { msg: redactLogString(message) },
|
|
85043
|
+
...safeContext,
|
|
85044
|
+
...trustedAttribution(getRequestContext(trustedRequestId), trustedRequestId)
|
|
85045
|
+
};
|
|
85046
|
+
if (enabled) writeEntry(entry);
|
|
85047
|
+
for (const listener of listeners) {
|
|
85048
|
+
try {
|
|
85049
|
+
listener(entry);
|
|
85050
|
+
} catch {
|
|
85051
|
+
}
|
|
84443
85052
|
}
|
|
85053
|
+
}
|
|
85054
|
+
function sanitizeContext(context) {
|
|
85055
|
+
if (!context) return {};
|
|
85056
|
+
const output = {};
|
|
85057
|
+
for (const key of safeKeys(context).slice(0, 100)) {
|
|
85058
|
+
if (RESERVED_FIELDS.has(key)) continue;
|
|
85059
|
+
const value = safeProperty(context, key);
|
|
85060
|
+
if (key === "req" || key === "request") {
|
|
85061
|
+
Object.assign(output, normalizeRequest(value));
|
|
85062
|
+
continue;
|
|
85063
|
+
}
|
|
85064
|
+
if (key === "res" || key === "reply") {
|
|
85065
|
+
Object.assign(output, normalizeReply(value));
|
|
85066
|
+
continue;
|
|
85067
|
+
}
|
|
85068
|
+
if (key === "responseTime" || key === "duration") {
|
|
85069
|
+
if (typeof value === "number" && Number.isFinite(value)) output.durationMs = value;
|
|
85070
|
+
continue;
|
|
85071
|
+
}
|
|
85072
|
+
output[key] = value;
|
|
85073
|
+
}
|
|
85074
|
+
delete output.__runtimeRequestId;
|
|
85075
|
+
const redacted = redactLogValue(output);
|
|
85076
|
+
return isRecord4(redacted) ? redacted : {};
|
|
85077
|
+
}
|
|
85078
|
+
function trustedAttribution(context, trustedRequestId) {
|
|
85079
|
+
const requestId = boundOptional(context?.requestId) ?? trustedRequestId;
|
|
85080
|
+
const actor = diagnosticIdentity("actor", context?.actor);
|
|
85081
|
+
const credentialId = diagnosticIdentity("credentialId", context?.credentialId);
|
|
85082
|
+
const userAgent = boundOptional(context?.userAgent);
|
|
85083
|
+
const actorSession = boundOptional(context?.actorSession);
|
|
85084
|
+
const method = boundOptional(context?.method);
|
|
85085
|
+
const route = boundOptional(context?.route);
|
|
85086
|
+
return {
|
|
85087
|
+
...requestId ? { requestId } : {},
|
|
85088
|
+
...actor ? { actor } : {},
|
|
85089
|
+
...credentialId ? { credentialId } : {},
|
|
85090
|
+
...userAgent ? { userAgent } : {},
|
|
85091
|
+
...actorSession ? { actorSession } : {},
|
|
85092
|
+
...method ? { method } : {},
|
|
85093
|
+
...route ? { route } : {},
|
|
85094
|
+
...context?.statusCode === void 0 ? {} : { statusCode: context.statusCode }
|
|
85095
|
+
};
|
|
85096
|
+
}
|
|
85097
|
+
function mergeBindings(parent, child) {
|
|
85098
|
+
const merged = { ...parent };
|
|
85099
|
+
if (!isObject(child)) return merged;
|
|
85100
|
+
for (const key of safeKeys(child).slice(0, 100)) {
|
|
85101
|
+
const value = safeProperty(child, key);
|
|
85102
|
+
if (key === "req" || key === "request") Object.assign(merged, normalizeRequest(value));
|
|
85103
|
+
else if (key === "res" || key === "reply") Object.assign(merged, normalizeReply(value));
|
|
85104
|
+
else if (key === "reqId") {
|
|
85105
|
+
merged.reqId = value;
|
|
85106
|
+
merged.__runtimeRequestId = value;
|
|
85107
|
+
} else if (!RESERVED_FIELDS.has(key)) merged[key] = value;
|
|
85108
|
+
}
|
|
85109
|
+
return merged;
|
|
85110
|
+
}
|
|
85111
|
+
function normalizeRequest(value) {
|
|
85112
|
+
if (!isObject(value)) return {};
|
|
85113
|
+
const id = asBoundString(safeProperty(value, "id"));
|
|
85114
|
+
const method = asBoundString(safeProperty(value, "method"));
|
|
85115
|
+
const routeOptions = safeProperty(value, "routeOptions");
|
|
85116
|
+
const route = isObject(routeOptions) ? asBoundString(safeProperty(routeOptions, "url")) : void 0;
|
|
84444
85117
|
return {
|
|
84445
|
-
|
|
84446
|
-
|
|
84447
|
-
|
|
85118
|
+
...id ? { reqId: id, __runtimeRequestId: id } : {},
|
|
85119
|
+
...method ? { method } : {},
|
|
85120
|
+
...route ? { route } : {}
|
|
84448
85121
|
};
|
|
84449
85122
|
}
|
|
85123
|
+
function normalizeReply(value) {
|
|
85124
|
+
if (!isObject(value)) return {};
|
|
85125
|
+
const statusCode = safeProperty(value, "statusCode");
|
|
85126
|
+
return typeof statusCode === "number" && Number.isFinite(statusCode) ? { statusCode } : {};
|
|
85127
|
+
}
|
|
85128
|
+
function writeEntry(entry) {
|
|
85129
|
+
const stream = entry.level === "error" || entry.level === "fatal" ? process.stderr : process.stdout;
|
|
85130
|
+
const line = IS_TTY ? formatTTY(entry) : JSON.stringify(entry);
|
|
85131
|
+
try {
|
|
85132
|
+
stream.write(`${line}
|
|
85133
|
+
`);
|
|
85134
|
+
} catch {
|
|
85135
|
+
}
|
|
85136
|
+
}
|
|
85137
|
+
function formatTTY(entry) {
|
|
85138
|
+
const { ts, level, module, action, msg, ...context } = entry;
|
|
85139
|
+
const levelTag = level === "fatal" || level === "error" ? "ERR" : level === "warn" ? "WRN" : "INF";
|
|
85140
|
+
const parts = safeKeys(context).map((key) => `${key}=${safeDisplay(safeProperty(context, key))}`);
|
|
85141
|
+
return `${ts.slice(11, 19)} ${levelTag} [${module}] ${action}${msg ? ` ${msg}` : ""}${parts.length ? ` ${parts.join(" ")}` : ""}`;
|
|
85142
|
+
}
|
|
85143
|
+
function safeDisplay(value) {
|
|
85144
|
+
try {
|
|
85145
|
+
return typeof value === "string" ? value : JSON.stringify(value);
|
|
85146
|
+
} catch {
|
|
85147
|
+
return "[UNSERIALIZABLE]";
|
|
85148
|
+
}
|
|
85149
|
+
}
|
|
85150
|
+
function safeFormat(message, args) {
|
|
85151
|
+
try {
|
|
85152
|
+
return args.length ? format(message, ...args) : message;
|
|
85153
|
+
} catch {
|
|
85154
|
+
return message;
|
|
85155
|
+
}
|
|
85156
|
+
}
|
|
85157
|
+
function isObject(value) {
|
|
85158
|
+
return value !== null && typeof value === "object";
|
|
85159
|
+
}
|
|
85160
|
+
function isRecord4(value) {
|
|
85161
|
+
return isObject(value) && !Array.isArray(value);
|
|
85162
|
+
}
|
|
85163
|
+
function safeKeys(value) {
|
|
85164
|
+
try {
|
|
85165
|
+
return Object.keys(value);
|
|
85166
|
+
} catch {
|
|
85167
|
+
return [];
|
|
85168
|
+
}
|
|
85169
|
+
}
|
|
85170
|
+
function safeProperty(value, key) {
|
|
85171
|
+
try {
|
|
85172
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
85173
|
+
return descriptor && "value" in descriptor ? descriptor.value : void 0;
|
|
85174
|
+
} catch {
|
|
85175
|
+
return void 0;
|
|
85176
|
+
}
|
|
85177
|
+
}
|
|
85178
|
+
function copyOwnData(value) {
|
|
85179
|
+
const output = {};
|
|
85180
|
+
for (const key of safeKeys(value)) output[key] = safeProperty(value, key);
|
|
85181
|
+
return output;
|
|
85182
|
+
}
|
|
85183
|
+
function boundText(value, fallback) {
|
|
85184
|
+
return boundOptional(value) ?? fallback;
|
|
85185
|
+
}
|
|
85186
|
+
function asBoundString(value) {
|
|
85187
|
+
return typeof value === "string" ? boundOptional(value) : void 0;
|
|
85188
|
+
}
|
|
85189
|
+
function boundOptional(value) {
|
|
85190
|
+
if (typeof value !== "string") return void 0;
|
|
85191
|
+
const redacted = redactLogString(value).replace(/[\r\n]+/g, " ").trim().slice(0, MAX_IDENTITY_LENGTH);
|
|
85192
|
+
return redacted || void 0;
|
|
85193
|
+
}
|
|
84450
85194
|
|
|
84451
85195
|
// src/citation-utils.ts
|
|
84452
85196
|
function pickProjectCitedDomain(citedDomains, projectDomains) {
|
|
@@ -84594,13 +85338,13 @@ function matchesBrandAlias(candidate, aliases) {
|
|
|
84594
85338
|
var RECURRENCE_LOOKBACK_RUNS = 5;
|
|
84595
85339
|
var HISTORY_WINDOW_RUNS = Math.max(PERSISTENT_GAP_THRESHOLD, 5);
|
|
84596
85340
|
var log = createLogger("IntelligenceService");
|
|
84597
|
-
var runChronologyKey =
|
|
85341
|
+
var runChronologyKey = sql27`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
84598
85342
|
function analyzableRunPredicates(projectId) {
|
|
84599
85343
|
return [
|
|
84600
85344
|
eq67(runs.projectId, projectId),
|
|
84601
85345
|
// Only this kind writes query_snapshots.
|
|
84602
85346
|
eq67(runs.kind, RunKinds["answer-visibility"]),
|
|
84603
|
-
|
|
85347
|
+
or16(eq67(runs.status, RunStatuses.completed), eq67(runs.status, RunStatuses.partial)),
|
|
84604
85348
|
// Defensive: RunCoordinator already skips probes before analyzeAndPersist
|
|
84605
85349
|
// is called, but a future call site invoking it directly for a probe must
|
|
84606
85350
|
// still not pollute the intelligence window.
|
|
@@ -84662,7 +85406,7 @@ var IntelligenceService = class {
|
|
|
84662
85406
|
// MEASUREMENT: only runs that actually wrote snapshots — see above.
|
|
84663
85407
|
// Index seek on idx_snapshots_run, short-circuited on the first row.
|
|
84664
85408
|
exists(
|
|
84665
|
-
this.db.select({ one:
|
|
85409
|
+
this.db.select({ one: sql27`1` }).from(querySnapshots).where(eq67(querySnapshots.runId, runs.id))
|
|
84666
85410
|
)
|
|
84667
85411
|
)
|
|
84668
85412
|
).orderBy(desc29(runs.finishedAt), desc29(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
@@ -85190,7 +85934,7 @@ var IntelligenceService = class {
|
|
|
85190
85934
|
and57(
|
|
85191
85935
|
eq67(runs.projectId, projectId),
|
|
85192
85936
|
eq67(runs.kind, RunKinds["answer-visibility"]),
|
|
85193
|
-
|
|
85937
|
+
or16(eq67(runs.status, "completed"), eq67(runs.status, "partial")),
|
|
85194
85938
|
// Defensive — see top of file.
|
|
85195
85939
|
ne10(runs.trigger, RunTriggers.probe)
|
|
85196
85940
|
)
|
|
@@ -85373,6 +86117,7 @@ export {
|
|
|
85373
86117
|
extractLegacyCredentials,
|
|
85374
86118
|
dropLegacyCredentialColumns,
|
|
85375
86119
|
migrate,
|
|
86120
|
+
OperationalLogStore,
|
|
85376
86121
|
groupRunsByCreatedAt,
|
|
85377
86122
|
classifyCrawler,
|
|
85378
86123
|
classifyAiUserFetch,
|
|
@@ -85454,6 +86199,8 @@ export {
|
|
|
85454
86199
|
markSessionFailed,
|
|
85455
86200
|
captureSimpleMeasurementDefinition,
|
|
85456
86201
|
apiRoutes,
|
|
86202
|
+
addLogListener,
|
|
85457
86203
|
createLogger,
|
|
86204
|
+
createFastifyLogger,
|
|
85458
86205
|
IntelligenceService
|
|
85459
86206
|
};
|