@ainyc/canonry 4.194.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -105
- package/README.md +4 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +71 -1
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +30 -0
- package/assets/assets/{AuditHistoryPanel-C8hUWnaD.js → AuditHistoryPanel-vWUtVXOI.js} +1 -1
- package/assets/assets/{BacklinksPage-BcIdghvA.js → BacklinksPage-CSzuMEuv.js} +1 -1
- package/assets/assets/{HistoryPage-DF7PLFi1.js → HistoryPage-DtCTQ7CQ.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BTlXQbYt.js → MeasurementPropertyPage-B7gTMWNs.js} +1 -1
- package/assets/assets/{ProjectPage-YiZ_i5eX.js → ProjectPage-Bsm0_jUY.js} +1 -1
- package/assets/assets/{RunRow-CfzuyTSM.js → RunRow-D5k4Rdo4.js} +1 -1
- package/assets/assets/{RunsPage-Diu_-YWl.js → RunsPage-DIXBB2Om.js} +1 -1
- package/assets/assets/{SettingsPage-g93aqkg2.js → SettingsPage-wYhBm6Az.js} +1 -1
- package/assets/assets/{SiteHealthSection-DroMQJVF.js → SiteHealthSection-YebzsgDh.js} +3 -3
- package/assets/assets/{TrafficPage-BhHjy5Ee.js → TrafficPage-DtnSRFO8.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-DI3aZIHo.js → TrafficSourceDetailPage-DYQ6mpYq.js} +1 -1
- package/assets/assets/{extract-error-message-DjXzzSl3.js → extract-error-message-BdChrkmo.js} +1 -1
- package/assets/assets/{index-33ZEL6A5.js → index-C_E2NRYY.js} +26 -26
- package/assets/assets/{react-sigma_core.esm.min-BlfsZb5G.js → react-sigma_core.esm.min-sKWUk_tG.js} +1 -1
- package/assets/assets/{v2-overview-adapter-C6cROMod.js → v2-overview-adapter-CkPt-sRK.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-X5AE3GNC.js → chunk-3WODA4JF.js} +276 -5
- package/dist/{chunk-ZA2QVBQO.js → chunk-67NX4KVE.js} +73 -36
- package/dist/{chunk-7XXXNNE2.js → chunk-B2XM6AQ6.js} +280 -26
- package/dist/{chunk-AO5QJJE6.js → chunk-LJKAB4CX.js} +2 -2
- package/dist/{chunk-QVYGI6CL.js → chunk-UMU3IBXM.js} +1225 -474
- package/dist/cli.js +233 -154
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-M4V3YPFZ.js → intelligence-service-MGQRJW4V.js} +2 -2
- package/dist/mcp.js +4 -4
- package/package.json +12 -11
|
@@ -280,6 +280,7 @@ import {
|
|
|
280
280
|
deriveWinnabilityClass,
|
|
281
281
|
describeError,
|
|
282
282
|
determineAnswerMentioned,
|
|
283
|
+
diagnosticIdentity,
|
|
283
284
|
discoveryBucketSchema,
|
|
284
285
|
discoveryHarvestDtoSchema,
|
|
285
286
|
discoveryPromotePreviewSchema,
|
|
@@ -382,6 +383,7 @@ import {
|
|
|
382
383
|
hostOf,
|
|
383
384
|
inclusiveDayCount,
|
|
384
385
|
indexingRequestResponseDtoSchema,
|
|
386
|
+
integrationSettingsSummaryDtoSchema,
|
|
385
387
|
internalError,
|
|
386
388
|
intersectScopes,
|
|
387
389
|
isBrowserProvider,
|
|
@@ -397,6 +399,7 @@ import {
|
|
|
397
399
|
latestProjectRunDtoSchema,
|
|
398
400
|
linearTrend,
|
|
399
401
|
locationContextSchema,
|
|
402
|
+
logQuerySchema,
|
|
400
403
|
loginRequestSchema,
|
|
401
404
|
mapWithConcurrency,
|
|
402
405
|
matchedAliasKeys,
|
|
@@ -512,6 +515,7 @@ import {
|
|
|
512
515
|
normalizeProjectName,
|
|
513
516
|
normalizeQueryText,
|
|
514
517
|
normalizeSiteAuditRunRequest,
|
|
518
|
+
normalizeTelemetryStatus,
|
|
515
519
|
normalizeUrlPath,
|
|
516
520
|
normalizeUserName,
|
|
517
521
|
notFound,
|
|
@@ -520,6 +524,8 @@ import {
|
|
|
520
524
|
observeTemplateLinkEdges,
|
|
521
525
|
onboardingTelemetryEventSchema,
|
|
522
526
|
operationInProgress,
|
|
527
|
+
operationalLogEntryDtoSchema,
|
|
528
|
+
operationalLogListDtoSchema,
|
|
523
529
|
orderLocationsDefaultFirst,
|
|
524
530
|
organicEvidenceDtoSchema,
|
|
525
531
|
organicEvidencePeriodSchema,
|
|
@@ -542,6 +548,7 @@ import {
|
|
|
542
548
|
projectUpsertRequestSchema,
|
|
543
549
|
providerAuthError,
|
|
544
550
|
providerError,
|
|
551
|
+
providerSummaryEntryDtoSchema,
|
|
545
552
|
queryDtoSchema,
|
|
546
553
|
queryGenerateRequestSchema,
|
|
547
554
|
queryTrackingCommitRequestSchema,
|
|
@@ -555,6 +562,8 @@ import {
|
|
|
555
562
|
recommendationBriefDtoSchema,
|
|
556
563
|
recommendationExplainRequestSchema,
|
|
557
564
|
recommendationExplanationDtoSchema,
|
|
565
|
+
redactLogString,
|
|
566
|
+
redactLogValue,
|
|
558
567
|
registrableDomain,
|
|
559
568
|
relativeChangeRatio,
|
|
560
569
|
reportActionCategoryLabel,
|
|
@@ -648,6 +657,7 @@ import {
|
|
|
648
657
|
surfaceClassFromCompetitorType,
|
|
649
658
|
surfaceClassLabel,
|
|
650
659
|
telemetryEventAcceptedDtoSchema,
|
|
660
|
+
telemetryStatusDtoSchema,
|
|
651
661
|
templateLinkRatio,
|
|
652
662
|
templateLinkSource,
|
|
653
663
|
templateLinkUbiquityAvailable,
|
|
@@ -692,10 +702,10 @@ import {
|
|
|
692
702
|
wordpressSchemaDeployResultDtoSchema,
|
|
693
703
|
wordpressSchemaStatusResultDtoSchema,
|
|
694
704
|
wordpressStatusDtoSchema
|
|
695
|
-
} from "./chunk-
|
|
705
|
+
} from "./chunk-3WODA4JF.js";
|
|
696
706
|
|
|
697
707
|
// src/intelligence-service.ts
|
|
698
|
-
import { eq as eq67, desc as desc29, asc as asc12, and as and57, ne as ne10, or as
|
|
708
|
+
import { eq as eq67, desc as desc29, asc as asc12, and as and57, ne as ne10, or as or16, inArray as inArray25, gte as gte18, lte as lte15, isNull as isNull12, sql as sql27, exists } from "drizzle-orm";
|
|
699
709
|
|
|
700
710
|
// ../db/src/client.ts
|
|
701
711
|
import { mkdirSync } from "fs";
|
|
@@ -795,6 +805,8 @@ __export(schema_exports, {
|
|
|
795
805
|
researchRunQueries: () => researchRunQueries,
|
|
796
806
|
researchRuns: () => researchRuns,
|
|
797
807
|
runs: () => runs,
|
|
808
|
+
runtimeLogMetadata: () => runtimeLogMetadata,
|
|
809
|
+
runtimeLogs: () => runtimeLogs,
|
|
798
810
|
schedules: () => schedules,
|
|
799
811
|
simpleMeasurementDefinitions: () => simpleMeasurementDefinitions,
|
|
800
812
|
siteAuditPages: () => siteAuditPages,
|
|
@@ -1158,10 +1170,9 @@ var auditLog = sqliteTable("audit_log", {
|
|
|
1158
1170
|
// reader could see it (the deletion would erase the only evidence it
|
|
1159
1171
|
// happened). Detached rows surface in audit queries with project_id=NULL.
|
|
1160
1172
|
projectId: text("project_id").references(() => projects.id, { onDelete: "set null" }),
|
|
1161
|
-
//
|
|
1162
|
-
//
|
|
1163
|
-
//
|
|
1164
|
-
// `userAgent` and `actorSession`.
|
|
1173
|
+
// Authenticated user:<id> / api-key:<id> for HTTP callers; explicit business
|
|
1174
|
+
// actors (scheduler, system, etc.) remain unchanged. Client hints below
|
|
1175
|
+
// are untrusted correlation metadata, never authorization identities.
|
|
1165
1176
|
actor: text("actor").notNull(),
|
|
1166
1177
|
action: text("action").notNull(),
|
|
1167
1178
|
entityType: text("entity_type").notNull(),
|
|
@@ -1178,11 +1189,46 @@ var auditLog = sqliteTable("audit_log", {
|
|
|
1178
1189
|
// with its session id so post-mortems can group a related sequence of
|
|
1179
1190
|
// mutations. NULL when the caller didn't provide one.
|
|
1180
1191
|
actorSession: text("actor_session"),
|
|
1192
|
+
// Stable credential id, intentionally not an FK: audit evidence must remain
|
|
1193
|
+
// intelligible after a key is revoked or deleted.
|
|
1194
|
+
credentialId: text("credential_id"),
|
|
1195
|
+
// Server-issued request correlation id, separate from caller actorSession.
|
|
1196
|
+
requestId: text("request_id"),
|
|
1181
1197
|
createdAt: text("created_at").notNull()
|
|
1182
1198
|
}, (table) => [
|
|
1183
1199
|
index("idx_audit_log_project").on(table.projectId),
|
|
1184
|
-
index("idx_audit_log_created").on(table.createdAt)
|
|
1200
|
+
index("idx_audit_log_created").on(table.createdAt),
|
|
1201
|
+
index("idx_audit_log_credential_created").on(table.credentialId, table.createdAt),
|
|
1202
|
+
index("idx_audit_log_request_created").on(table.requestId, table.createdAt)
|
|
1185
1203
|
]);
|
|
1204
|
+
var runtimeLogs = sqliteTable("runtime_logs", {
|
|
1205
|
+
sequence: integer("sequence").primaryKey(),
|
|
1206
|
+
ts: text("ts").notNull(),
|
|
1207
|
+
level: text("level").notNull(),
|
|
1208
|
+
module: text("module").notNull(),
|
|
1209
|
+
action: text("action").notNull(),
|
|
1210
|
+
msg: text("msg"),
|
|
1211
|
+
projectId: text("project_id"),
|
|
1212
|
+
runId: text("run_id"),
|
|
1213
|
+
actor: text("actor"),
|
|
1214
|
+
requestId: text("request_id"),
|
|
1215
|
+
context: text("context", { mode: "json" }).$type().notNull().default({}),
|
|
1216
|
+
entryBytes: integer("entry_bytes").notNull()
|
|
1217
|
+
}, (table) => [
|
|
1218
|
+
index("idx_runtime_logs_project_ts").on(table.projectId, table.ts, table.sequence),
|
|
1219
|
+
index("idx_runtime_logs_run_ts").on(table.runId, table.ts, table.sequence),
|
|
1220
|
+
index("idx_runtime_logs_actor_ts").on(table.actor, table.ts, table.sequence),
|
|
1221
|
+
index("idx_runtime_logs_request_ts").on(table.requestId, table.ts, table.sequence),
|
|
1222
|
+
index("idx_runtime_logs_module_level_ts").on(table.module, table.level, table.ts, table.sequence),
|
|
1223
|
+
index("idx_runtime_logs_ts").on(table.ts, table.sequence)
|
|
1224
|
+
]);
|
|
1225
|
+
var runtimeLogMetadata = sqliteTable("runtime_log_metadata", {
|
|
1226
|
+
id: text("id").primaryKey(),
|
|
1227
|
+
cursorNamespace: text("cursor_namespace").notNull(),
|
|
1228
|
+
nextSequence: integer("next_sequence").notNull(),
|
|
1229
|
+
dropped: integer("dropped").notNull().default(0),
|
|
1230
|
+
captureErrors: integer("capture_errors").notNull().default(0)
|
|
1231
|
+
});
|
|
1186
1232
|
var apiKeys = sqliteTable("api_keys", {
|
|
1187
1233
|
id: text("id").primaryKey(),
|
|
1188
1234
|
name: text("name").notNull(),
|
|
@@ -7202,6 +7248,45 @@ var MIGRATION_VERSIONS = [
|
|
|
7202
7248
|
statements: [
|
|
7203
7249
|
`ALTER TABLE api_keys ADD COLUMN delegated_user_id TEXT REFERENCES users(id) ON DELETE CASCADE`
|
|
7204
7250
|
]
|
|
7251
|
+
},
|
|
7252
|
+
{
|
|
7253
|
+
version: 155,
|
|
7254
|
+
name: "durable-operational-logs",
|
|
7255
|
+
statements: [
|
|
7256
|
+
// Audit attribution is intentionally a plain nullable identity string:
|
|
7257
|
+
// deleting/revoking a credential must not delete its historical trail.
|
|
7258
|
+
`ALTER TABLE audit_log ADD COLUMN credential_id TEXT`,
|
|
7259
|
+
`ALTER TABLE audit_log ADD COLUMN request_id TEXT`,
|
|
7260
|
+
`CREATE INDEX IF NOT EXISTS idx_audit_log_credential_created ON audit_log(credential_id, created_at)`,
|
|
7261
|
+
`CREATE INDEX IF NOT EXISTS idx_audit_log_request_created ON audit_log(request_id, created_at)`,
|
|
7262
|
+
`CREATE TABLE IF NOT EXISTS runtime_logs (
|
|
7263
|
+
sequence INTEGER PRIMARY KEY,
|
|
7264
|
+
ts TEXT NOT NULL,
|
|
7265
|
+
level TEXT NOT NULL,
|
|
7266
|
+
module TEXT NOT NULL,
|
|
7267
|
+
action TEXT NOT NULL,
|
|
7268
|
+
msg TEXT,
|
|
7269
|
+
project_id TEXT,
|
|
7270
|
+
run_id TEXT,
|
|
7271
|
+
actor TEXT,
|
|
7272
|
+
request_id TEXT,
|
|
7273
|
+
context TEXT NOT NULL DEFAULT '{}',
|
|
7274
|
+
entry_bytes INTEGER NOT NULL
|
|
7275
|
+
)`,
|
|
7276
|
+
`CREATE TABLE IF NOT EXISTS runtime_log_metadata (
|
|
7277
|
+
id TEXT PRIMARY KEY,
|
|
7278
|
+
cursor_namespace TEXT NOT NULL,
|
|
7279
|
+
next_sequence INTEGER NOT NULL,
|
|
7280
|
+
dropped INTEGER NOT NULL DEFAULT 0,
|
|
7281
|
+
capture_errors INTEGER NOT NULL DEFAULT 0
|
|
7282
|
+
)`,
|
|
7283
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_project_ts ON runtime_logs(project_id, ts, sequence)`,
|
|
7284
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_run_ts ON runtime_logs(run_id, ts, sequence)`,
|
|
7285
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_actor_ts ON runtime_logs(actor, ts, sequence)`,
|
|
7286
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_request_ts ON runtime_logs(request_id, ts, sequence)`,
|
|
7287
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_module_level_ts ON runtime_logs(module, level, ts, sequence)`,
|
|
7288
|
+
`CREATE INDEX IF NOT EXISTS idx_runtime_logs_ts ON runtime_logs(ts, sequence)`
|
|
7289
|
+
]
|
|
7205
7290
|
}
|
|
7206
7291
|
];
|
|
7207
7292
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -7476,6 +7561,293 @@ function migrate(db, versions = MIGRATION_VERSIONS) {
|
|
|
7476
7561
|
}
|
|
7477
7562
|
}
|
|
7478
7563
|
|
|
7564
|
+
// ../db/src/operational-logs.ts
|
|
7565
|
+
import { createHash, randomUUID } from "crypto";
|
|
7566
|
+
import { sql as sql3 } from "drizzle-orm";
|
|
7567
|
+
var METADATA_ID = "runtime-log-store";
|
|
7568
|
+
var DEFAULT_MAX_ENTRIES = 1e4;
|
|
7569
|
+
var MAX_MAX_ENTRIES = 1e5;
|
|
7570
|
+
var DEFAULT_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
7571
|
+
var MIN_MAX_AGE_MS = 1e3;
|
|
7572
|
+
var MAX_MAX_AGE_MS = 31 * 24 * 60 * 60 * 1e3;
|
|
7573
|
+
var MAX_ENTRY_BYTES = 8192;
|
|
7574
|
+
var OperationalLogStore = class {
|
|
7575
|
+
constructor(db, options = {}) {
|
|
7576
|
+
this.db = db;
|
|
7577
|
+
this.maxEntries = boundedInteger(options.maxEntries ?? DEFAULT_MAX_ENTRIES, 1, MAX_MAX_ENTRIES, "maxEntries");
|
|
7578
|
+
this.maxAgeMs = boundedInteger(options.maxAgeMs ?? DEFAULT_MAX_AGE_MS, MIN_MAX_AGE_MS, MAX_MAX_AGE_MS, "maxAgeMs");
|
|
7579
|
+
this.retention = options.retention ?? "durable";
|
|
7580
|
+
this.now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
7581
|
+
this.ensureMetadata();
|
|
7582
|
+
}
|
|
7583
|
+
db;
|
|
7584
|
+
maxEntries;
|
|
7585
|
+
maxAgeMs;
|
|
7586
|
+
retention;
|
|
7587
|
+
now;
|
|
7588
|
+
localCaptureErrors = 0;
|
|
7589
|
+
append(event) {
|
|
7590
|
+
try {
|
|
7591
|
+
const entry = projectEvent(event);
|
|
7592
|
+
if (Buffer.byteLength(JSON.stringify(entry), "utf8") > MAX_ENTRY_BYTES) {
|
|
7593
|
+
throw new RangeError(`Operational log entry exceeds ${MAX_ENTRY_BYTES} bytes.`);
|
|
7594
|
+
}
|
|
7595
|
+
this.withoutLockWait(() => this.db.transaction((tx) => {
|
|
7596
|
+
this.prune(tx);
|
|
7597
|
+
const metadata2 = readMetadata(tx);
|
|
7598
|
+
tx.run(sql3`
|
|
7599
|
+
INSERT INTO runtime_logs (
|
|
7600
|
+
sequence, ts, level, module, action, msg, project_id, run_id, actor, request_id, context, entry_bytes
|
|
7601
|
+
) VALUES (
|
|
7602
|
+
${metadata2.nextSequence}, ${entry.ts}, ${entry.level}, ${entry.module}, ${entry.action}, ${entry.message ?? null},
|
|
7603
|
+
${entry.projectId ?? null}, ${entry.runId ?? null}, ${entry.context.actor ?? null}, ${entry.context.requestId ?? null},
|
|
7604
|
+
${JSON.stringify(entry.context)}, ${Buffer.byteLength(JSON.stringify(entry), "utf8")}
|
|
7605
|
+
)
|
|
7606
|
+
`);
|
|
7607
|
+
tx.run(sql3`UPDATE runtime_log_metadata SET next_sequence = ${metadata2.nextSequence + 1} WHERE id = ${METADATA_ID}`);
|
|
7608
|
+
this.prune(tx);
|
|
7609
|
+
}));
|
|
7610
|
+
} catch {
|
|
7611
|
+
this.recordCaptureError();
|
|
7612
|
+
}
|
|
7613
|
+
}
|
|
7614
|
+
list(query) {
|
|
7615
|
+
const parsed = logQuerySchema.safeParse(query);
|
|
7616
|
+
if (!parsed.success) throw validationError("Invalid operational logs query.", { issues: parsed.error.issues });
|
|
7617
|
+
const filters = normalizeLogQuery(parsed.data);
|
|
7618
|
+
this.withoutLockWait(() => this.db.transaction((tx) => {
|
|
7619
|
+
this.prune(tx);
|
|
7620
|
+
}));
|
|
7621
|
+
const metadata2 = readMetadata(this.db);
|
|
7622
|
+
const after = filters.cursor === void 0 ? 0 : this.parseCursor(filters.cursor, metadata2, filters);
|
|
7623
|
+
if (filters.cursor !== void 0) this.assertCursorCurrent(after);
|
|
7624
|
+
const where = buildWhere(filters, after);
|
|
7625
|
+
const rows = this.db.all(sql3`
|
|
7626
|
+
SELECT sequence, ts, level, module, action, msg,
|
|
7627
|
+
project_id AS projectId, run_id AS runId, actor, request_id AS requestId, context
|
|
7628
|
+
FROM runtime_logs
|
|
7629
|
+
WHERE ${sql3.join(where, sql3` AND `)}
|
|
7630
|
+
ORDER BY sequence ASC
|
|
7631
|
+
LIMIT ${filters.limit}
|
|
7632
|
+
`);
|
|
7633
|
+
const countRows = this.db.all(sql3`
|
|
7634
|
+
SELECT COUNT(*) AS count FROM runtime_logs WHERE ${sql3.join(where, sql3` AND `)}
|
|
7635
|
+
`);
|
|
7636
|
+
const matching = countRows[0]?.count;
|
|
7637
|
+
if (!Number.isSafeInteger(matching) || matching < rows.length) throw new Error("Operational log storage returned an invalid count.");
|
|
7638
|
+
const truncated = matching - rows.length;
|
|
7639
|
+
return {
|
|
7640
|
+
entries: rows.map((row) => this.toDto(row, metadata2.cursorNamespace, filters)),
|
|
7641
|
+
nextCursor: truncated > 0 ? this.cursorFor(metadata2.cursorNamespace, rows.at(-1).sequence, filters) : null,
|
|
7642
|
+
truncated,
|
|
7643
|
+
dropped: metadata2.dropped,
|
|
7644
|
+
retention: this.retention,
|
|
7645
|
+
retentionPolicy: { maxEntries: this.maxEntries, maxAgeSeconds: Math.ceil(this.maxAgeMs / 1e3) },
|
|
7646
|
+
captureErrors: metadata2.captureErrors + this.localCaptureErrors,
|
|
7647
|
+
observedAt: this.now().toISOString()
|
|
7648
|
+
};
|
|
7649
|
+
}
|
|
7650
|
+
ensureMetadata() {
|
|
7651
|
+
const existing = this.db.all(sql3`SELECT id FROM runtime_log_metadata WHERE id = ${METADATA_ID}`);
|
|
7652
|
+
if (existing.length > 0) return;
|
|
7653
|
+
this.db.run(sql3`
|
|
7654
|
+
INSERT OR IGNORE INTO runtime_log_metadata (id, cursor_namespace, next_sequence, dropped, capture_errors)
|
|
7655
|
+
VALUES (${METADATA_ID}, ${randomUUID()}, 1, 0, 0)
|
|
7656
|
+
`);
|
|
7657
|
+
}
|
|
7658
|
+
prune(db) {
|
|
7659
|
+
const expiry = new Date(this.now().getTime() - this.maxAgeMs).toISOString();
|
|
7660
|
+
db.run(sql3`DELETE FROM runtime_logs WHERE ts < ${expiry}`);
|
|
7661
|
+
db.run(sql3`UPDATE runtime_log_metadata SET dropped = dropped + changes() WHERE id = ${METADATA_ID}`);
|
|
7662
|
+
const metadata2 = readMetadata(db);
|
|
7663
|
+
const firstRetainedSequence = Math.max(1, metadata2.nextSequence - this.maxEntries);
|
|
7664
|
+
db.run(sql3`DELETE FROM runtime_logs WHERE sequence < ${firstRetainedSequence}`);
|
|
7665
|
+
db.run(sql3`UPDATE runtime_log_metadata SET dropped = dropped + changes() WHERE id = ${METADATA_ID}`);
|
|
7666
|
+
}
|
|
7667
|
+
recordCaptureError() {
|
|
7668
|
+
try {
|
|
7669
|
+
this.withoutLockWait(() => this.db.run(sql3`UPDATE runtime_log_metadata SET capture_errors = capture_errors + 1 WHERE id = ${METADATA_ID}`));
|
|
7670
|
+
} catch {
|
|
7671
|
+
this.localCaptureErrors++;
|
|
7672
|
+
}
|
|
7673
|
+
}
|
|
7674
|
+
/**
|
|
7675
|
+
* Diagnostics must not spend the application's busy timeout waiting for a
|
|
7676
|
+
* writer. All work here is synchronous, so restore the connection's exact
|
|
7677
|
+
* policy before any other application work can run, including on failure.
|
|
7678
|
+
*/
|
|
7679
|
+
withoutLockWait(work) {
|
|
7680
|
+
const timeout = this.db.$client.pragma("busy_timeout", { simple: true });
|
|
7681
|
+
try {
|
|
7682
|
+
this.db.$client.pragma("busy_timeout = 0");
|
|
7683
|
+
return work();
|
|
7684
|
+
} finally {
|
|
7685
|
+
this.db.$client.pragma(`busy_timeout = ${timeout}`);
|
|
7686
|
+
}
|
|
7687
|
+
}
|
|
7688
|
+
parseCursor(cursor, metadata2, filters) {
|
|
7689
|
+
let decoded;
|
|
7690
|
+
try {
|
|
7691
|
+
decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
|
|
7692
|
+
} catch {
|
|
7693
|
+
throw validationError("The operational-log cursor is invalid.");
|
|
7694
|
+
}
|
|
7695
|
+
if (!isCursorPayload(decoded)) throw validationError("The operational-log cursor is invalid.");
|
|
7696
|
+
if (decoded.namespace !== metadata2.cursorNamespace) throw validationError("The operational-log cursor belongs to a different database.");
|
|
7697
|
+
if (decoded.filters !== filterHash(filters)) throw validationError("The operational-log cursor does not match these query filters.");
|
|
7698
|
+
if (decoded.sequence < 1 || decoded.sequence >= metadata2.nextSequence) throw validationError("The operational-log cursor is invalid.");
|
|
7699
|
+
return decoded.sequence;
|
|
7700
|
+
}
|
|
7701
|
+
assertCursorCurrent(sequence) {
|
|
7702
|
+
const rows = this.db.all(sql3`SELECT sequence FROM runtime_logs WHERE sequence = ${sequence}`);
|
|
7703
|
+
if (rows.length === 0) throw validationError("The operational-log cursor is stale because retention evicted its entry.");
|
|
7704
|
+
}
|
|
7705
|
+
cursorFor(namespace, sequence, filters) {
|
|
7706
|
+
return Buffer.from(JSON.stringify({ namespace, sequence, filters: filterHash(filters) }), "utf8").toString("base64url");
|
|
7707
|
+
}
|
|
7708
|
+
toDto(row, namespace, filters) {
|
|
7709
|
+
const context = parseContext(row.context);
|
|
7710
|
+
const entry = {
|
|
7711
|
+
cursor: this.cursorFor(namespace, row.sequence, filters),
|
|
7712
|
+
ts: redactLogString(row.ts),
|
|
7713
|
+
level: redactLogString(row.level),
|
|
7714
|
+
module: redactLogString(row.module),
|
|
7715
|
+
action: redactLogString(row.action),
|
|
7716
|
+
...row.msg === null ? {} : { message: clip(redactLogString(row.msg), 4096) },
|
|
7717
|
+
...row.runId === null ? {} : { runId: clip(redactLogString(row.runId), 256) },
|
|
7718
|
+
...row.projectId === null ? {} : { projectId: clip(redactLogString(row.projectId), 256) },
|
|
7719
|
+
context
|
|
7720
|
+
};
|
|
7721
|
+
return operationalLogEntryDtoSchema.parse(entry);
|
|
7722
|
+
}
|
|
7723
|
+
};
|
|
7724
|
+
function projectEvent(event) {
|
|
7725
|
+
const value = redactLogValue(event);
|
|
7726
|
+
if (!isRecord(value)) throw new TypeError("Operational log event must be an object.");
|
|
7727
|
+
const contextSource = isRecord(value.context) ? value.context : {};
|
|
7728
|
+
const context = projectContext(value, contextSource);
|
|
7729
|
+
const entry = {
|
|
7730
|
+
ts: new Date(requiredString(value.ts, 64, "timestamp")).toISOString(),
|
|
7731
|
+
level: requiredString(value.level, 16, "level"),
|
|
7732
|
+
module: requiredString(value.module, 256, "module"),
|
|
7733
|
+
action: requiredString(value.action, 256, "action"),
|
|
7734
|
+
...optionalString(value.msg, 4096) === void 0 ? {} : { message: optionalString(value.msg, 4096) },
|
|
7735
|
+
...context.runId === void 0 ? {} : { runId: context.runId },
|
|
7736
|
+
...context.projectId === void 0 ? {} : { projectId: context.projectId },
|
|
7737
|
+
context
|
|
7738
|
+
};
|
|
7739
|
+
return operationalLogEntryDtoSchema.omit({ cursor: true }).parse(entry);
|
|
7740
|
+
}
|
|
7741
|
+
function projectContext(event, nested) {
|
|
7742
|
+
const context = {};
|
|
7743
|
+
for (const [key, max2] of ID_FIELDS) {
|
|
7744
|
+
const raw = event[key] ?? nested[key];
|
|
7745
|
+
const value = diagnosticIdentity(key, raw) ?? optionalString(raw, max2);
|
|
7746
|
+
if (value !== void 0) context[key] = value;
|
|
7747
|
+
}
|
|
7748
|
+
for (const key of NUMBER_FIELDS) {
|
|
7749
|
+
const value = event[key] ?? nested[key];
|
|
7750
|
+
if (typeof value === "number" && Number.isFinite(value)) context[key] = value;
|
|
7751
|
+
}
|
|
7752
|
+
for (const key of BOOLEAN_FIELDS) {
|
|
7753
|
+
const value = event[key] ?? nested[key];
|
|
7754
|
+
if (typeof value === "boolean") context[key] = value;
|
|
7755
|
+
}
|
|
7756
|
+
const errorCode = optionalString(event.errorCode ?? nested.errorCode, 128);
|
|
7757
|
+
if (errorCode !== void 0) context.errorCode = errorCode;
|
|
7758
|
+
return context;
|
|
7759
|
+
}
|
|
7760
|
+
var ID_FIELDS = [
|
|
7761
|
+
["runId", 256],
|
|
7762
|
+
["projectId", 256],
|
|
7763
|
+
["requestId", 256],
|
|
7764
|
+
["actor", 512],
|
|
7765
|
+
["credentialId", 512],
|
|
7766
|
+
["userAgent", 512],
|
|
7767
|
+
["actorSession", 512],
|
|
7768
|
+
["operationId", 256],
|
|
7769
|
+
["jobId", 256],
|
|
7770
|
+
["taskId", 256],
|
|
7771
|
+
["traceId", 256],
|
|
7772
|
+
["method", 16],
|
|
7773
|
+
["route", 256]
|
|
7774
|
+
];
|
|
7775
|
+
var NUMBER_FIELDS = ["attempt", "count", "total", "progress", "httpStatus", "statusCode", "durationMs", "bytes", "retryAfterMs"];
|
|
7776
|
+
var BOOLEAN_FIELDS = ["retriable", "retrying", "cancelled", "success"];
|
|
7777
|
+
function buildWhere(query, after) {
|
|
7778
|
+
const where = [sql3`sequence > ${after}`];
|
|
7779
|
+
if (query.level !== void 0) where.push(sql3`level = ${query.level}`);
|
|
7780
|
+
if (query.module !== void 0) where.push(sql3`module = ${query.module}`);
|
|
7781
|
+
if (query.runId !== void 0) where.push(sql3`run_id = ${query.runId}`);
|
|
7782
|
+
if (query.projectId !== void 0) where.push(sql3`project_id = ${query.projectId}`);
|
|
7783
|
+
if (query.actor !== void 0) where.push(sql3`actor = ${query.actor}`);
|
|
7784
|
+
if (query.requestId !== void 0) where.push(sql3`request_id = ${query.requestId}`);
|
|
7785
|
+
if (query.since !== void 0) where.push(sql3`ts >= ${query.since}`);
|
|
7786
|
+
if (query.until !== void 0) where.push(sql3`ts <= ${query.until}`);
|
|
7787
|
+
return where;
|
|
7788
|
+
}
|
|
7789
|
+
function readMetadata(db) {
|
|
7790
|
+
const rows = db.all(sql3`
|
|
7791
|
+
SELECT cursor_namespace AS cursorNamespace, next_sequence AS nextSequence, dropped, capture_errors AS captureErrors
|
|
7792
|
+
FROM runtime_log_metadata WHERE id = ${METADATA_ID}
|
|
7793
|
+
`);
|
|
7794
|
+
const metadata2 = rows[0];
|
|
7795
|
+
if (metadata2 === void 0 || !Number.isSafeInteger(metadata2.nextSequence) || metadata2.nextSequence < 1) {
|
|
7796
|
+
throw new Error("Operational log metadata is missing or corrupt.");
|
|
7797
|
+
}
|
|
7798
|
+
return metadata2;
|
|
7799
|
+
}
|
|
7800
|
+
function parseContext(value) {
|
|
7801
|
+
const parsed = typeof value === "string" ? parseJsonColumn(value, void 0) : value;
|
|
7802
|
+
const redacted = redactLogValue(parsed);
|
|
7803
|
+
if (!isRecord(redacted)) throw new Error("Operational log context is corrupt.");
|
|
7804
|
+
return redacted;
|
|
7805
|
+
}
|
|
7806
|
+
function filterHash(query) {
|
|
7807
|
+
const stable = JSON.stringify({
|
|
7808
|
+
level: query.level ?? null,
|
|
7809
|
+
module: query.module ?? null,
|
|
7810
|
+
runId: query.runId ?? null,
|
|
7811
|
+
projectId: query.projectId ?? null,
|
|
7812
|
+
actor: query.actor ?? null,
|
|
7813
|
+
requestId: query.requestId ?? null,
|
|
7814
|
+
since: query.since ?? null,
|
|
7815
|
+
until: query.until ?? null
|
|
7816
|
+
});
|
|
7817
|
+
return createHash("sha256").update(stable).digest("base64url");
|
|
7818
|
+
}
|
|
7819
|
+
function normalizeLogQuery(query) {
|
|
7820
|
+
return {
|
|
7821
|
+
...query,
|
|
7822
|
+
...query.since === void 0 ? {} : { since: new Date(query.since).toISOString() },
|
|
7823
|
+
...query.until === void 0 ? {} : { until: new Date(query.until).toISOString() }
|
|
7824
|
+
};
|
|
7825
|
+
}
|
|
7826
|
+
function isCursorPayload(value) {
|
|
7827
|
+
return isRecord(value) && typeof value.namespace === "string" && Number.isSafeInteger(value.sequence) && typeof value.filters === "string";
|
|
7828
|
+
}
|
|
7829
|
+
function isRecord(value) {
|
|
7830
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
7831
|
+
}
|
|
7832
|
+
function requiredString(value, max2, label) {
|
|
7833
|
+
const string = optionalString(value, max2);
|
|
7834
|
+
if (string === void 0) throw new TypeError(`Operational log ${label} must be a non-empty string.`);
|
|
7835
|
+
return string;
|
|
7836
|
+
}
|
|
7837
|
+
function optionalString(value, max2) {
|
|
7838
|
+
if (typeof value !== "string" || value.length === 0) return void 0;
|
|
7839
|
+
return clip(redactLogString(value), max2);
|
|
7840
|
+
}
|
|
7841
|
+
function clip(value, max2) {
|
|
7842
|
+
return value.length <= max2 ? value : value.slice(0, max2);
|
|
7843
|
+
}
|
|
7844
|
+
function boundedInteger(value, min2, max2, label) {
|
|
7845
|
+
if (!Number.isInteger(value) || value < min2 || value > max2) {
|
|
7846
|
+
throw new RangeError(`${label} must be an integer from ${min2} to ${max2}.`);
|
|
7847
|
+
}
|
|
7848
|
+
return value;
|
|
7849
|
+
}
|
|
7850
|
+
|
|
7479
7851
|
// ../db/src/run-helpers.ts
|
|
7480
7852
|
function groupRunsByCreatedAt(rows) {
|
|
7481
7853
|
const groups = [];
|
|
@@ -7664,12 +8036,12 @@ function analyzeCause(regression, currentSnapshots) {
|
|
|
7664
8036
|
}
|
|
7665
8037
|
|
|
7666
8038
|
// ../intelligence/src/insights.ts
|
|
7667
|
-
import { randomUUID } from "crypto";
|
|
8039
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
7668
8040
|
var QUERY_LEVEL_PROVIDER = "all";
|
|
7669
8041
|
function generateInsights(input) {
|
|
7670
8042
|
const insights2 = [];
|
|
7671
8043
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7672
|
-
const id = () => `ins_${
|
|
8044
|
+
const id = () => `ins_${randomUUID2().slice(0, 8)}`;
|
|
7673
8045
|
for (const reg of input.regressions) {
|
|
7674
8046
|
const cause = input.causes.get(`${reg.query}:${reg.provider}`);
|
|
7675
8047
|
insights2.push({
|
|
@@ -11148,6 +11520,70 @@ async function authPlugin(app, opts = {}) {
|
|
|
11148
11520
|
});
|
|
11149
11521
|
}
|
|
11150
11522
|
|
|
11523
|
+
// ../api-routes/src/request-context.ts
|
|
11524
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
11525
|
+
var requestContext = new AsyncLocalStorage();
|
|
11526
|
+
var MAX_REQUEST_CONTEXT_LENGTH = 512;
|
|
11527
|
+
function getRequestContext(boundRequestId) {
|
|
11528
|
+
const context = requestContext.getStore();
|
|
11529
|
+
if (!context || context.completed && context.requestId !== boundRequestId) return void 0;
|
|
11530
|
+
const { completed: _completed, ...safeContext } = context;
|
|
11531
|
+
return safeContext;
|
|
11532
|
+
}
|
|
11533
|
+
function requestAuditIdentity(request) {
|
|
11534
|
+
const principal = request.principal;
|
|
11535
|
+
if (!principal) return {};
|
|
11536
|
+
if (principal.delegatedUser) {
|
|
11537
|
+
return {
|
|
11538
|
+
actor: `user:${principal.delegatedUser.id}`,
|
|
11539
|
+
credentialId: request.apiKey?.id ?? principal.id
|
|
11540
|
+
};
|
|
11541
|
+
}
|
|
11542
|
+
if (principal.kind === "user") return { actor: `user:${principal.id}` };
|
|
11543
|
+
return {
|
|
11544
|
+
actor: `api-key:${principal.id}`,
|
|
11545
|
+
credentialId: request.apiKey?.id ?? principal.id
|
|
11546
|
+
};
|
|
11547
|
+
}
|
|
11548
|
+
function registerRequestContext(app) {
|
|
11549
|
+
app.addHook("onRequest", (request, reply, done) => {
|
|
11550
|
+
reply.header("x-request-id", request.id);
|
|
11551
|
+
requestContext.run({
|
|
11552
|
+
requestId: sanitizeRequestContext(request.id) ?? void 0,
|
|
11553
|
+
userAgent: sanitizeRequestContext(request.headers["user-agent"]) ?? void 0,
|
|
11554
|
+
actorSession: sanitizeRequestContext(request.headers["x-canonry-actor-session"]) ?? void 0,
|
|
11555
|
+
method: request.method,
|
|
11556
|
+
// Route template only, never a raw URL, query string, or route parameter.
|
|
11557
|
+
route: request.routeOptions.url
|
|
11558
|
+
}, done);
|
|
11559
|
+
});
|
|
11560
|
+
app.addHook("preHandler", (request, _reply, done) => {
|
|
11561
|
+
populateIdentity(request);
|
|
11562
|
+
done();
|
|
11563
|
+
});
|
|
11564
|
+
app.addHook("onError", (request, _reply, _error, done) => {
|
|
11565
|
+
populateIdentity(request);
|
|
11566
|
+
done();
|
|
11567
|
+
});
|
|
11568
|
+
app.addHook("onResponse", (request, reply, done) => {
|
|
11569
|
+
populateIdentity(request);
|
|
11570
|
+
const context = requestContext.getStore();
|
|
11571
|
+
if (context) context.statusCode = reply.statusCode;
|
|
11572
|
+
done();
|
|
11573
|
+
if (context) context.completed = true;
|
|
11574
|
+
});
|
|
11575
|
+
}
|
|
11576
|
+
function populateIdentity(request) {
|
|
11577
|
+
const context = requestContext.getStore();
|
|
11578
|
+
if (context) Object.assign(context, requestAuditIdentity(request));
|
|
11579
|
+
}
|
|
11580
|
+
function sanitizeRequestContext(value) {
|
|
11581
|
+
const joined = Array.isArray(value) ? value.join(", ") : value;
|
|
11582
|
+
if (!joined) return null;
|
|
11583
|
+
const sanitized = redactLogString(joined).replace(/[\r\n]+/g, " ").trim().slice(0, MAX_REQUEST_CONTEXT_LENGTH);
|
|
11584
|
+
return sanitized || null;
|
|
11585
|
+
}
|
|
11586
|
+
|
|
11151
11587
|
// ../api-routes/src/oauth.ts
|
|
11152
11588
|
import crypto5 from "crypto";
|
|
11153
11589
|
import { and, eq as eq3, isNull, lt } from "drizzle-orm";
|
|
@@ -11609,11 +12045,11 @@ function registerOAuthAdminRoutes(app, opts) {
|
|
|
11609
12045
|
|
|
11610
12046
|
// ../api-routes/src/projects.ts
|
|
11611
12047
|
import crypto7 from "crypto";
|
|
11612
|
-
import { and as and2, eq as eq5, sql as
|
|
12048
|
+
import { and as and2, eq as eq5, sql as sql5 } from "drizzle-orm";
|
|
11613
12049
|
|
|
11614
12050
|
// ../api-routes/src/helpers.ts
|
|
11615
12051
|
import crypto6 from "crypto";
|
|
11616
|
-
import { eq as eq4, ne, sql as
|
|
12052
|
+
import { eq as eq4, ne, sql as sql4 } from "drizzle-orm";
|
|
11617
12053
|
function notProbeRun() {
|
|
11618
12054
|
return ne(runs.trigger, RunTriggers.probe);
|
|
11619
12055
|
}
|
|
@@ -11625,29 +12061,37 @@ function resolveProject(db, name) {
|
|
|
11625
12061
|
return project;
|
|
11626
12062
|
}
|
|
11627
12063
|
function writeAuditLog(db, entry) {
|
|
12064
|
+
const context = getRequestContext();
|
|
12065
|
+
const actor = entry.actor === "api" ? context?.actor ?? entry.actor : entry.actor;
|
|
11628
12066
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
11629
12067
|
db.insert(auditLog).values({
|
|
11630
12068
|
id: crypto6.randomUUID(),
|
|
11631
12069
|
projectId: entry.projectId ?? null,
|
|
11632
|
-
actor
|
|
12070
|
+
actor,
|
|
11633
12071
|
action: entry.action,
|
|
11634
12072
|
entityType: entry.entityType,
|
|
11635
12073
|
entityId: entry.entityId ?? null,
|
|
11636
12074
|
diff: entry.diff != null ? JSON.stringify(entry.diff) : null,
|
|
11637
|
-
userAgent: entry.userAgent ??
|
|
11638
|
-
actorSession: entry.actorSession ??
|
|
12075
|
+
userAgent: sanitizeRequestContext(entry.userAgent ?? context?.userAgent),
|
|
12076
|
+
actorSession: sanitizeRequestContext(entry.actorSession ?? context?.actorSession),
|
|
12077
|
+
requestId: entry.requestId ?? context?.requestId ?? null,
|
|
12078
|
+
credentialId: entry.credentialId ?? context?.credentialId ?? null,
|
|
11639
12079
|
createdAt: now
|
|
11640
12080
|
}).run();
|
|
11641
12081
|
}
|
|
11642
12082
|
function auditFromRequest(request, entry) {
|
|
11643
|
-
const
|
|
11644
|
-
const
|
|
11645
|
-
const
|
|
11646
|
-
const
|
|
12083
|
+
const userAgent = sanitizeRequestContext(request.headers["user-agent"]);
|
|
12084
|
+
const actorSession = sanitizeRequestContext(request.headers["x-canonry-actor-session"]);
|
|
12085
|
+
const identity = requestAuditIdentity(request);
|
|
12086
|
+
const context = getRequestContext();
|
|
12087
|
+
const actor = entry.actor === "api" ? identity.actor ?? entry.actor : entry.actor;
|
|
11647
12088
|
return {
|
|
11648
12089
|
...entry,
|
|
11649
|
-
|
|
11650
|
-
|
|
12090
|
+
actor,
|
|
12091
|
+
credentialId: entry.credentialId ?? identity.credentialId ?? context?.credentialId ?? null,
|
|
12092
|
+
userAgent: sanitizeRequestContext(entry.userAgent ?? userAgent),
|
|
12093
|
+
actorSession: sanitizeRequestContext(entry.actorSession ?? actorSession),
|
|
12094
|
+
requestId: entry.requestId ?? context?.requestId ?? null
|
|
11651
12095
|
};
|
|
11652
12096
|
}
|
|
11653
12097
|
function resolveSnapshotMentionResult(snapshot, project) {
|
|
@@ -11727,18 +12171,32 @@ async function settingsRoutes(app, opts) {
|
|
|
11727
12171
|
validProviders: validNames
|
|
11728
12172
|
});
|
|
11729
12173
|
}
|
|
11730
|
-
if (
|
|
11731
|
-
|
|
12174
|
+
if (apiKey !== void 0 && (typeof apiKey !== "string" || !apiKey.trim())) {
|
|
12175
|
+
throw validationError("apiKey must be a non-empty string when provided");
|
|
12176
|
+
}
|
|
12177
|
+
if (baseUrl !== void 0 && (typeof baseUrl !== "string" || !baseUrl.trim())) {
|
|
12178
|
+
throw validationError("baseUrl must be a non-empty string when provided");
|
|
12179
|
+
}
|
|
12180
|
+
if (model !== void 0 && (typeof model !== "string" || !model.trim())) {
|
|
12181
|
+
throw validationError("model must be a non-empty string when provided");
|
|
12182
|
+
}
|
|
12183
|
+
if (apiKey === void 0 && baseUrl === void 0 && model === void 0 && quota === void 0) {
|
|
12184
|
+
throw validationError("at least one provider setting must be supplied");
|
|
12185
|
+
}
|
|
12186
|
+
const existing = (opts.providerSummary ?? []).find((provider) => provider.name === name);
|
|
12187
|
+
const configured = Boolean(existing?.configured);
|
|
12188
|
+
if (!configured && name === "local") {
|
|
12189
|
+
if (!baseUrl) {
|
|
11732
12190
|
throw validationError("baseUrl is required for local provider");
|
|
11733
12191
|
}
|
|
11734
|
-
} else if (name === "gemini" && !apiKey) {
|
|
12192
|
+
} else if (!configured && name === "gemini" && !apiKey) {
|
|
11735
12193
|
const geminiSummary = (opts.providerSummary ?? []).find((p) => p.name === "gemini");
|
|
11736
12194
|
if (!geminiSummary?.vertexConfigured) {
|
|
11737
12195
|
throw validationError(
|
|
11738
12196
|
"apiKey is required for Gemini unless Vertex AI is configured (set GEMINI_VERTEX_PROJECT env var or vertexProject in config file)"
|
|
11739
12197
|
);
|
|
11740
12198
|
}
|
|
11741
|
-
} else {
|
|
12199
|
+
} else if (!configured) {
|
|
11742
12200
|
if (!apiKey || typeof apiKey !== "string") {
|
|
11743
12201
|
throw validationError("apiKey is required");
|
|
11744
12202
|
}
|
|
@@ -11757,6 +12215,9 @@ async function settingsRoutes(app, opts) {
|
|
|
11757
12215
|
if (typeof quota !== "object" || quota === null) {
|
|
11758
12216
|
throw validationError("quota must be an object");
|
|
11759
12217
|
}
|
|
12218
|
+
if (Object.keys(quota).length === 0) {
|
|
12219
|
+
throw validationError("quota must include at least one field");
|
|
12220
|
+
}
|
|
11760
12221
|
for (const [key, val] of Object.entries(quota)) {
|
|
11761
12222
|
if (!["maxConcurrency", "maxRequestsPerMinute", "maxRequestsPerDay"].includes(key)) {
|
|
11762
12223
|
throw validationError(`Unknown quota field: ${key}`);
|
|
@@ -11766,7 +12227,12 @@ async function settingsRoutes(app, opts) {
|
|
|
11766
12227
|
}
|
|
11767
12228
|
}
|
|
11768
12229
|
}
|
|
11769
|
-
const
|
|
12230
|
+
const { actor, userAgent, actorSession, requestId, credentialId } = auditFromRequest(request, {
|
|
12231
|
+
actor: "api",
|
|
12232
|
+
action: "provider.updated",
|
|
12233
|
+
entityType: "provider"
|
|
12234
|
+
});
|
|
12235
|
+
const result = opts.onProviderUpdate(name, apiKey ?? "", model, baseUrl, quota, { actor, userAgent, actorSession, requestId, credentialId });
|
|
11770
12236
|
if (!result) {
|
|
11771
12237
|
throw internalError("Failed to update provider configuration");
|
|
11772
12238
|
}
|
|
@@ -12092,12 +12558,12 @@ async function projectRoutes(app, opts) {
|
|
|
12092
12558
|
const project = resolveProject(app.db, request.params.name);
|
|
12093
12559
|
const pid = project.id;
|
|
12094
12560
|
const count4 = (n) => n ?? 0;
|
|
12095
|
-
const queryCount = app.db.select({ n:
|
|
12096
|
-
const competitorCount = app.db.select({ n:
|
|
12097
|
-
const runCount = app.db.select({ n:
|
|
12098
|
-
const snapshotCount = app.db.select({ n:
|
|
12099
|
-
const insightCount = app.db.select({ n:
|
|
12100
|
-
const auditLogCount = app.db.select({ n:
|
|
12561
|
+
const queryCount = app.db.select({ n: sql5`count(*)` }).from(queries).where(eq5(queries.projectId, pid)).get();
|
|
12562
|
+
const competitorCount = app.db.select({ n: sql5`count(*)` }).from(competitors).where(eq5(competitors.projectId, pid)).get();
|
|
12563
|
+
const runCount = app.db.select({ n: sql5`count(*)` }).from(runs).where(eq5(runs.projectId, pid)).get();
|
|
12564
|
+
const snapshotCount = app.db.select({ n: sql5`count(*)` }).from(querySnapshots).innerJoin(runs, eq5(querySnapshots.runId, runs.id)).where(eq5(runs.projectId, pid)).get();
|
|
12565
|
+
const insightCount = app.db.select({ n: sql5`count(*)` }).from(insights).where(eq5(insights.projectId, pid)).get();
|
|
12566
|
+
const auditLogCount = app.db.select({ n: sql5`count(*)` }).from(auditLog).where(eq5(auditLog.projectId, pid)).get();
|
|
12101
12567
|
return reply.send({
|
|
12102
12568
|
project: { id: project.id, name: project.name },
|
|
12103
12569
|
cascadeRows: {
|
|
@@ -12312,7 +12778,7 @@ function aliasArraysEqual(a, b) {
|
|
|
12312
12778
|
|
|
12313
12779
|
// ../api-routes/src/queries.ts
|
|
12314
12780
|
import crypto10 from "crypto";
|
|
12315
|
-
import { and as and5, eq as eq8, inArray as inArray2, sql as
|
|
12781
|
+
import { and as and5, eq as eq8, inArray as inArray2, sql as sql6 } from "drizzle-orm";
|
|
12316
12782
|
|
|
12317
12783
|
// ../api-routes/src/query-replace.ts
|
|
12318
12784
|
import crypto9 from "crypto";
|
|
@@ -12675,9 +13141,9 @@ async function queryRoutes(app, opts) {
|
|
|
12675
13141
|
let snapshotsDetached = 0;
|
|
12676
13142
|
let affectedQueries = 0;
|
|
12677
13143
|
if (removedIds.length > 0) {
|
|
12678
|
-
const snapshotCount = app.db.select({ n:
|
|
13144
|
+
const snapshotCount = app.db.select({ n: sql6`count(*)` }).from(querySnapshots).where(inArray2(querySnapshots.queryId, removedIds)).get();
|
|
12679
13145
|
snapshotsDetached = snapshotCount?.n ?? 0;
|
|
12680
|
-
const distinctAffected = app.db.select({ n:
|
|
13146
|
+
const distinctAffected = app.db.select({ n: sql6`count(distinct ${querySnapshots.queryId})` }).from(querySnapshots).where(inArray2(querySnapshots.queryId, removedIds)).get();
|
|
12681
13147
|
affectedQueries = distinctAffected?.n ?? 0;
|
|
12682
13148
|
}
|
|
12683
13149
|
return reply.send({
|
|
@@ -13199,7 +13665,7 @@ function shareOfVoiceFromLandscape(landscape, queryClass) {
|
|
|
13199
13665
|
}
|
|
13200
13666
|
|
|
13201
13667
|
// ../api-routes/src/measurement-overview.ts
|
|
13202
|
-
import { createHash } from "crypto";
|
|
13668
|
+
import { createHash as createHash2 } from "crypto";
|
|
13203
13669
|
import { and as and8, eq as eq13 } from "drizzle-orm";
|
|
13204
13670
|
|
|
13205
13671
|
// ../api-routes/src/measurement-report.ts
|
|
@@ -14586,7 +15052,7 @@ function overviewFilterFingerprint(query) {
|
|
|
14586
15052
|
to: query.to ?? null,
|
|
14587
15053
|
search: query.search === void 0 ? null : normalizedText(query.search)
|
|
14588
15054
|
};
|
|
14589
|
-
return
|
|
15055
|
+
return createHash2("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
14590
15056
|
}
|
|
14591
15057
|
function overviewAggregateFingerprint(query) {
|
|
14592
15058
|
const filters = {
|
|
@@ -14599,11 +15065,11 @@ function overviewAggregateFingerprint(query) {
|
|
|
14599
15065
|
from: query.from ?? null,
|
|
14600
15066
|
to: query.to ?? null
|
|
14601
15067
|
};
|
|
14602
|
-
return
|
|
15068
|
+
return createHash2("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
14603
15069
|
}
|
|
14604
15070
|
function overviewEvidenceFingerprint(snapshots) {
|
|
14605
15071
|
const canonical = [...snapshots].sort((left, right) => left.id.localeCompare(right.id)).map((snapshot) => JSON.stringify(snapshot)).join("\n");
|
|
14606
|
-
return
|
|
15072
|
+
return createHash2("sha256").update(canonical).digest("base64url");
|
|
14607
15073
|
}
|
|
14608
15074
|
function compareLabels(left, right) {
|
|
14609
15075
|
const leftLabel = normalizedText(left.label);
|
|
@@ -15455,7 +15921,7 @@ function mergePins(...collections) {
|
|
|
15455
15921
|
|
|
15456
15922
|
// ../api-routes/src/runs.ts
|
|
15457
15923
|
import crypto14 from "crypto";
|
|
15458
|
-
import { and as and12, eq as eq17, asc, desc as desc3, inArray as inArray5, or as or2, sql as
|
|
15924
|
+
import { and as and12, eq as eq17, asc, desc as desc3, inArray as inArray5, or as or2, sql as sql7 } from "drizzle-orm";
|
|
15459
15925
|
import { gte as gte3 } from "drizzle-orm";
|
|
15460
15926
|
|
|
15461
15927
|
// ../api-routes/src/run-queue.ts
|
|
@@ -16031,7 +16497,7 @@ async function runRoutes(app, opts) {
|
|
|
16031
16497
|
});
|
|
16032
16498
|
app.get("/projects/:name/runs/latest", async (request, reply) => {
|
|
16033
16499
|
const project = resolveProject(app.db, request.params.name);
|
|
16034
|
-
const countRow = app.db.select({ count:
|
|
16500
|
+
const countRow = app.db.select({ count: sql7`count(*)` }).from(runs).where(and12(eq17(runs.projectId, project.id), notProbeRun())).get();
|
|
16035
16501
|
const totalRuns = countRow?.count ?? 0;
|
|
16036
16502
|
const latestRun = app.db.select().from(runs).where(and12(eq17(runs.projectId, project.id), notProbeRun())).orderBy(desc3(runs.createdAt), desc3(runs.id)).limit(1).get();
|
|
16037
16503
|
if (!latestRun) {
|
|
@@ -20962,7 +21428,7 @@ async function measurementDiscoveryV2Routes(app) {
|
|
|
20962
21428
|
}
|
|
20963
21429
|
|
|
20964
21430
|
// ../api-routes/src/measurement-property-evidence.ts
|
|
20965
|
-
import { createHash as
|
|
21431
|
+
import { createHash as createHash3 } from "crypto";
|
|
20966
21432
|
import { and as and16, eq as eq21 } from "drizzle-orm";
|
|
20967
21433
|
function parseEvidenceQuery(raw) {
|
|
20968
21434
|
const candidate = { ...raw, ...raw.limit === void 0 ? {} : { limit: Number(raw.limit) } };
|
|
@@ -20982,11 +21448,11 @@ function filterFingerprint(query) {
|
|
|
20982
21448
|
provider: query.provider === void 0 ? null : normalizedText2(query.provider),
|
|
20983
21449
|
location: query.location === void 0 ? null : normalizedText2(query.location)
|
|
20984
21450
|
};
|
|
20985
|
-
return
|
|
21451
|
+
return createHash3("sha256").update(JSON.stringify(filters)).digest("base64url");
|
|
20986
21452
|
}
|
|
20987
21453
|
function evidenceFingerprintOf(snapshots) {
|
|
20988
21454
|
const canonical = [...snapshots].sort((left, right) => left.id.localeCompare(right.id)).map((snapshot) => JSON.stringify(snapshot)).join("\n");
|
|
20989
|
-
return
|
|
21455
|
+
return createHash3("sha256").update(canonical).digest("base64url");
|
|
20990
21456
|
}
|
|
20991
21457
|
function sourceRowKey(row) {
|
|
20992
21458
|
return [row.expectedSlotId, row.usageEdgeId, row.sourceUrl].join("\0");
|
|
@@ -21153,7 +21619,7 @@ async function measurementPropertyEvidenceRoutes(app) {
|
|
|
21153
21619
|
import { and as and18, desc as desc6, eq as eq23, inArray as inArray8, lt as lt3, or as or3 } from "drizzle-orm";
|
|
21154
21620
|
|
|
21155
21621
|
// ../api-routes/src/measurement-question-reads.ts
|
|
21156
|
-
import { and as and17, eq as eq22, getTableColumns, inArray as inArray7, sql as
|
|
21622
|
+
import { and as and17, eq as eq22, getTableColumns, inArray as inArray7, sql as sql8 } from "drizzle-orm";
|
|
21157
21623
|
var DEFAULT_LIMIT2 = 50;
|
|
21158
21624
|
var TERMINAL_STATUSES = /* @__PURE__ */ new Set([
|
|
21159
21625
|
RunStatuses.completed,
|
|
@@ -21272,7 +21738,7 @@ function selectedQuestionSnapshots(db, runId, manifest, scope) {
|
|
|
21272
21738
|
conditions.push(inArray7(querySnapshots.measurementExecutionId, executionIds));
|
|
21273
21739
|
}
|
|
21274
21740
|
if (scope?.provider !== void 0) {
|
|
21275
|
-
conditions.push(
|
|
21741
|
+
conditions.push(sql8`lower(trim(${querySnapshots.provider})) = ${normalizeText(scope.provider)}`);
|
|
21276
21742
|
}
|
|
21277
21743
|
return db.select(measurementQuestionSnapshotColumns).from(querySnapshots).where(and17(...conditions)).all();
|
|
21278
21744
|
}
|
|
@@ -22227,11 +22693,11 @@ async function measurementPortfolioReadRoutes(app) {
|
|
|
22227
22693
|
}
|
|
22228
22694
|
|
|
22229
22695
|
// ../api-routes/src/visibility-report.ts
|
|
22230
|
-
import { createHash as
|
|
22696
|
+
import { createHash as createHash5 } from "crypto";
|
|
22231
22697
|
import { and as and19, desc as desc7, eq as eq24, gte as gte4, inArray as inArray9, isNotNull, isNull as isNull4, lte as lte4, ne as ne3 } from "drizzle-orm";
|
|
22232
22698
|
|
|
22233
22699
|
// ../api-routes/src/visibility-report-reader.ts
|
|
22234
|
-
import { createHash as
|
|
22700
|
+
import { createHash as createHash4 } from "crypto";
|
|
22235
22701
|
var VisibilityReportCursorError = class extends Error {
|
|
22236
22702
|
constructor(message) {
|
|
22237
22703
|
super(message);
|
|
@@ -22606,7 +23072,7 @@ function breakdown(candidates, definition, populationTargetKeys = selectedTarget
|
|
|
22606
23072
|
return { properties, groups };
|
|
22607
23073
|
}
|
|
22608
23074
|
function fingerprint(value) {
|
|
22609
|
-
return
|
|
23075
|
+
return createHash4("sha256").update(JSON.stringify(value)).digest("base64url");
|
|
22610
23076
|
}
|
|
22611
23077
|
function cursorOf2(cursor) {
|
|
22612
23078
|
return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url");
|
|
@@ -23099,11 +23565,11 @@ function stableSimpleDefinitionId(definition) {
|
|
|
23099
23565
|
// not freeze a competitor set, whereas a newly captured empty list did.
|
|
23100
23566
|
competitors: definition.competitors === void 0 ? null : [...definition.competitors].map((competitor) => ({ domain: competitor.domain, label: competitor.label, aliases: [...competitor.aliases].sort() })).sort((left, right) => left.domain.localeCompare(right.domain, "en"))
|
|
23101
23567
|
};
|
|
23102
|
-
return `simple:${
|
|
23568
|
+
return `simple:${createHash5("sha256").update(JSON.stringify(source)).digest("hex")}`;
|
|
23103
23569
|
}
|
|
23104
23570
|
function simpleModelFingerprint(definition) {
|
|
23105
23571
|
const engines = [...definition.engines].map((engine) => ({ provider: normalizedText3(engine.provider), requestedModel: engine.requestedModel })).sort((left, right) => left.provider.localeCompare(right.provider, "en"));
|
|
23106
|
-
return `simple-model:${
|
|
23572
|
+
return `simple-model:${createHash5("sha256").update(JSON.stringify(engines)).digest("hex")}`;
|
|
23107
23573
|
}
|
|
23108
23574
|
function frozenSimpleRun(run, definition, snapshots, includeEvidence = true) {
|
|
23109
23575
|
const matcher = compileBrandAliases(effectiveBrandNames({
|
|
@@ -25626,7 +26092,7 @@ function normalizeCompetitorList2(domains) {
|
|
|
25626
26092
|
}
|
|
25627
26093
|
|
|
25628
26094
|
// ../api-routes/src/history.ts
|
|
25629
|
-
import { and as and22, asc as asc3, count, desc as desc9, eq as eq27, gte as gte5, inArray as inArray11 } from "drizzle-orm";
|
|
26095
|
+
import { and as and22, asc as asc3, count, desc as desc9, eq as eq27, gte as gte5, inArray as inArray11, like, or as or4 } from "drizzle-orm";
|
|
25630
26096
|
|
|
25631
26097
|
// ../api-routes/src/notification-redaction.ts
|
|
25632
26098
|
var REDACTED_URL = {
|
|
@@ -25697,7 +26163,7 @@ async function historyRoutes(app) {
|
|
|
25697
26163
|
if (input.confirm) {
|
|
25698
26164
|
tx.delete(researchRuns).where(and22(eq27(researchRuns.projectId, project.id), inArray11(researchRuns.id, input.researchRunIds))).run();
|
|
25699
26165
|
tx.delete(runs).where(and22(eq27(runs.projectId, project.id), inArray11(runs.id, input.runIds))).run();
|
|
25700
|
-
writeAuditLog(tx, { projectId: project.id, actor:
|
|
26166
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "results.cleared", entityType: "project", entityId: project.id, diff: result });
|
|
25701
26167
|
}
|
|
25702
26168
|
return result;
|
|
25703
26169
|
});
|
|
@@ -25706,14 +26172,14 @@ async function historyRoutes(app) {
|
|
|
25706
26172
|
const project = resolveProject(app.db, request.params.name);
|
|
25707
26173
|
const filters = [eq27(auditLog.projectId, project.id)];
|
|
25708
26174
|
addAuditHistoryFilters(filters, request.query);
|
|
25709
|
-
const rows = app.db.select().from(auditLog).where(and22(...filters)).orderBy(desc9(auditLog.createdAt)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
26175
|
+
const rows = app.db.select().from(auditLog).where(and22(...filters)).orderBy(desc9(auditLog.createdAt), desc9(auditLog.id)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
25710
26176
|
return reply.send(rows.map(formatAuditEntry));
|
|
25711
26177
|
});
|
|
25712
26178
|
app.get("/history", async (request, reply) => {
|
|
25713
26179
|
const scopedProjectId = request.apiKey?.projectId;
|
|
25714
26180
|
const filters = scopedProjectId ? [eq27(auditLog.projectId, scopedProjectId)] : [];
|
|
25715
26181
|
addAuditHistoryFilters(filters, request.query);
|
|
25716
|
-
const rows = app.db.select().from(auditLog).where(filters.length > 0 ? and22(...filters) : void 0).orderBy(desc9(auditLog.createdAt)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
26182
|
+
const rows = app.db.select().from(auditLog).where(filters.length > 0 ? and22(...filters) : void 0).orderBy(desc9(auditLog.createdAt), desc9(auditLog.id)).limit(parseBoundedInt(request.query.limit, 100, 500)).offset(parseBoundedInt(request.query.offset, 0, Number.MAX_SAFE_INTEGER)).all();
|
|
25717
26183
|
return reply.send(rows.map(formatAuditEntry));
|
|
25718
26184
|
});
|
|
25719
26185
|
app.get("/projects/:name/snapshots", async (request, reply) => {
|
|
@@ -26004,13 +26470,20 @@ function formatAuditEntry(row) {
|
|
|
26004
26470
|
diff: row.diff ? row.entityType === "notification" ? redactNotificationDiff(parseJsonColumn(row.diff, null)) : parseJsonColumn(row.diff, null) : null,
|
|
26005
26471
|
userAgent: row.userAgent,
|
|
26006
26472
|
actorSession: row.actorSession,
|
|
26473
|
+
requestId: row.requestId,
|
|
26474
|
+
credentialId: row.credentialId,
|
|
26007
26475
|
createdAt: row.createdAt
|
|
26008
26476
|
};
|
|
26009
26477
|
}
|
|
26010
26478
|
function addAuditHistoryFilters(filters, query) {
|
|
26011
26479
|
if (query.since && !Number.isNaN(Date.parse(query.since))) filters.push(gte5(auditLog.createdAt, query.since));
|
|
26012
26480
|
if (query.action) filters.push(eq27(auditLog.action, query.action));
|
|
26013
|
-
if (query.actor) filters.push(
|
|
26481
|
+
if (query.actor === "api") filters.push(or4(
|
|
26482
|
+
eq27(auditLog.actor, "api"),
|
|
26483
|
+
like(auditLog.actor, "api-key:%"),
|
|
26484
|
+
like(auditLog.actor, "user:%")
|
|
26485
|
+
));
|
|
26486
|
+
else if (query.actor) filters.push(eq27(auditLog.actor, query.actor));
|
|
26014
26487
|
if (query.entityType) filters.push(eq27(auditLog.entityType, query.entityType));
|
|
26015
26488
|
}
|
|
26016
26489
|
function parseBoundedInt(value, fallback, max2) {
|
|
@@ -26819,7 +27292,7 @@ function buildCategoryCounts(counts2) {
|
|
|
26819
27292
|
}
|
|
26820
27293
|
|
|
26821
27294
|
// ../api-routes/src/intelligence.ts
|
|
26822
|
-
import { eq as eq29, desc as desc11, and as and24, inArray as inArray13, like } from "drizzle-orm";
|
|
27295
|
+
import { eq as eq29, desc as desc11, and as and24, inArray as inArray13, like as like2 } from "drizzle-orm";
|
|
26823
27296
|
var SEVERITY_RANK = { low: 0, medium: 1, high: 2, critical: 3 };
|
|
26824
27297
|
function severitiesAtOrAbove(min2) {
|
|
26825
27298
|
const floor = SEVERITY_RANK[min2];
|
|
@@ -26944,7 +27417,7 @@ async function intelligenceRoutes(app) {
|
|
|
26944
27417
|
const typeFilter = request.query.type?.trim();
|
|
26945
27418
|
if (typeFilter) {
|
|
26946
27419
|
conditions.push(
|
|
26947
|
-
typeFilter.endsWith("*") ?
|
|
27420
|
+
typeFilter.endsWith("*") ? like2(insights.type, `${typeFilter.slice(0, -1)}%`) : eq29(insights.type, typeFilter)
|
|
26948
27421
|
);
|
|
26949
27422
|
}
|
|
26950
27423
|
const severityFilter = request.query.severity?.trim();
|
|
@@ -27019,16 +27492,16 @@ async function intelligenceRoutes(app) {
|
|
|
27019
27492
|
}
|
|
27020
27493
|
|
|
27021
27494
|
// ../api-routes/src/ai-referral-status.ts
|
|
27022
|
-
import { and as and25, sql as
|
|
27023
|
-
var REDIRECT_LIST =
|
|
27495
|
+
import { and as and25, sql as sql9 } from "drizzle-orm";
|
|
27496
|
+
var REDIRECT_LIST = sql9.join([...LOCATION_REDIRECT_STATUSES].map((s) => sql9`${s}`), sql9`, `);
|
|
27024
27497
|
function referralLandedCondition() {
|
|
27025
|
-
return
|
|
27498
|
+
return sql9`${aiReferralEventsHourly.status} NOT IN (${REDIRECT_LIST})`;
|
|
27026
27499
|
}
|
|
27027
27500
|
function countableReferralCondition() {
|
|
27028
27501
|
return and25(nonSubresourceReferralPathCondition(), referralLandedCondition());
|
|
27029
27502
|
}
|
|
27030
27503
|
function nonSubresourceReferralPathCondition() {
|
|
27031
|
-
return
|
|
27504
|
+
return sql9`
|
|
27032
27505
|
LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/_next/static/%'
|
|
27033
27506
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/assets/%'
|
|
27034
27507
|
AND LOWER(${aiReferralEventsHourly.landingPathNormalized}) NOT LIKE '/static/%'
|
|
@@ -27052,7 +27525,7 @@ function nonSubresourceReferralPathCondition() {
|
|
|
27052
27525
|
}
|
|
27053
27526
|
|
|
27054
27527
|
// ../api-routes/src/report.ts
|
|
27055
|
-
import { and as and29, desc as desc14, eq as eq33, gte as gte8, inArray as inArray15, lt as lt5, lte as lte6, ne as ne5, or as
|
|
27528
|
+
import { and as and29, desc as desc14, eq as eq33, gte as gte8, inArray as inArray15, lt as lt5, lte as lte6, ne as ne5, or as or5, sql as sql11 } from "drizzle-orm";
|
|
27056
27529
|
|
|
27057
27530
|
// ../api-routes/src/content.ts
|
|
27058
27531
|
import crypto23 from "crypto";
|
|
@@ -27062,7 +27535,7 @@ import { and as and28, desc as desc13, eq as eq32 } from "drizzle-orm";
|
|
|
27062
27535
|
import { and as and27, eq as eq31, desc as desc12, gte as gte7, inArray as inArray14, lte as lte5 } from "drizzle-orm";
|
|
27063
27536
|
|
|
27064
27537
|
// ../api-routes/src/gsc-totals.ts
|
|
27065
|
-
import { and as and26, asc as asc4, eq as eq30, sql as
|
|
27538
|
+
import { and as and26, asc as asc4, eq as eq30, sql as sql10, max, min } from "drizzle-orm";
|
|
27066
27539
|
var WINDOW_DAYS = { "7d": 7, "30d": 30, "90d": 90 };
|
|
27067
27540
|
function resolveGscWindowRange(window, latestDataDate, today) {
|
|
27068
27541
|
if (window === "all") {
|
|
@@ -27117,8 +27590,8 @@ function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
|
27117
27590
|
}).from(gscDailyTotals).where(
|
|
27118
27591
|
and26(
|
|
27119
27592
|
eq30(gscDailyTotals.projectId, projectId),
|
|
27120
|
-
|
|
27121
|
-
|
|
27593
|
+
sql10`${gscDailyTotals.date} >= ${startDate}`,
|
|
27594
|
+
sql10`${gscDailyTotals.date} <= ${endDate}`
|
|
27122
27595
|
)
|
|
27123
27596
|
).orderBy(asc4(gscDailyTotals.date)).all();
|
|
27124
27597
|
return rows.map((r) => {
|
|
@@ -27147,8 +27620,8 @@ function readGscQueryDailyRows(db, projectId, startDate, endDate) {
|
|
|
27147
27620
|
}).from(gscQueryDailyTotals).where(
|
|
27148
27621
|
and26(
|
|
27149
27622
|
eq30(gscQueryDailyTotals.projectId, projectId),
|
|
27150
|
-
|
|
27151
|
-
|
|
27623
|
+
sql10`${gscQueryDailyTotals.date} >= ${startDate}`,
|
|
27624
|
+
sql10`${gscQueryDailyTotals.date} <= ${endDate}`
|
|
27152
27625
|
)
|
|
27153
27626
|
).all();
|
|
27154
27627
|
return rows.map((r) => {
|
|
@@ -30509,8 +30982,8 @@ function loadQueryLookup(db, projectId) {
|
|
|
30509
30982
|
return { byId };
|
|
30510
30983
|
}
|
|
30511
30984
|
function buildGscSection(db, projectId, projectBrandNames, canonicalDomain, trackedQueries, windowDays2) {
|
|
30512
|
-
const maxSearch = db.select({ m:
|
|
30513
|
-
const maxDaily = db.select({ m:
|
|
30985
|
+
const maxSearch = db.select({ m: sql11`MAX(${gscSearchData.date})` }).from(gscSearchData).where(eq33(gscSearchData.projectId, projectId)).get()?.m ?? "";
|
|
30986
|
+
const maxDaily = db.select({ m: sql11`MAX(${gscDailyTotals.date})` }).from(gscDailyTotals).where(eq33(gscDailyTotals.projectId, projectId)).get()?.m ?? "";
|
|
30514
30987
|
const maxDate = maxSearch > maxDaily ? maxSearch : maxDaily;
|
|
30515
30988
|
if (!maxDate) return null;
|
|
30516
30989
|
const startDate = windowStartDate(maxDate, windowDays2);
|
|
@@ -30621,7 +31094,7 @@ function buildGaSection(db, projectId, windowDays2) {
|
|
|
30621
31094
|
).limit(1).get() : void 0;
|
|
30622
31095
|
const fallbackSummary = windowSummary ? null : db.select().from(gaTrafficSummaries).where(eq33(gaTrafficSummaries.projectId, projectId)).orderBy(desc14(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
30623
31096
|
const matchingFallbackSummary = fallbackSummary && inclusiveDayCount(fallbackSummary.periodStart, fallbackSummary.periodEnd) === windowDays2 ? fallbackSummary : null;
|
|
30624
|
-
const snapshotMaxDate = db.select({ m:
|
|
31097
|
+
const snapshotMaxDate = db.select({ m: sql11`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq33(gaTrafficSnapshots.projectId, projectId)).get()?.m ?? "";
|
|
30625
31098
|
if (!windowSummary && !matchingFallbackSummary && !snapshotMaxDate) return null;
|
|
30626
31099
|
const summaryPeriod = windowSummary ?? matchingFallbackSummary;
|
|
30627
31100
|
const snapshotMaxInWindow = summaryPeriod?.periodEnd || snapshotMaxDate;
|
|
@@ -30663,7 +31136,7 @@ function buildGaSection(db, projectId, windowDays2) {
|
|
|
30663
31136
|
const aiSessionRows = db.select({
|
|
30664
31137
|
trafficClass: gaAiReferrals.trafficClass,
|
|
30665
31138
|
channelGroup: gaAiReferrals.channelGroup,
|
|
30666
|
-
sessions:
|
|
31139
|
+
sessions: sql11`COALESCE(SUM(${gaAiReferrals.sessions}), 0)`
|
|
30667
31140
|
}).from(gaAiReferrals).where(and29(...aiConditions)).groupBy(gaAiReferrals.trafficClass, gaAiReferrals.channelGroup).all();
|
|
30668
31141
|
let paidAiSessions = 0;
|
|
30669
31142
|
let organicAiSessions = 0;
|
|
@@ -30762,8 +31235,8 @@ function buildAiReferrals(db, projectId, windowDays2) {
|
|
|
30762
31235
|
const window = resolveAiReferralWindow(db, projectId, windowDays2);
|
|
30763
31236
|
const rows = db.select().from(gaAiReferrals).where(and29(
|
|
30764
31237
|
eq33(gaAiReferrals.projectId, projectId),
|
|
30765
|
-
|
|
30766
|
-
|
|
31238
|
+
sql11`${gaAiReferrals.date} >= ${window.start}`,
|
|
31239
|
+
sql11`${gaAiReferrals.date} <= ${window.end}`
|
|
30767
31240
|
)).all();
|
|
30768
31241
|
if (rows.length === 0) return null;
|
|
30769
31242
|
const dimSessionsByTuple = /* @__PURE__ */ new Map();
|
|
@@ -30848,7 +31321,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30848
31321
|
const priorStart = new Date(priorStartMs).toISOString();
|
|
30849
31322
|
const trendStart = new Date(trendStartMs).toISOString();
|
|
30850
31323
|
const sumVerifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
30851
|
-
db.select({ total:
|
|
31324
|
+
db.select({ total: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
30852
31325
|
and29(
|
|
30853
31326
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
30854
31327
|
eq33(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -30858,7 +31331,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30858
31331
|
).get()?.total ?? 0
|
|
30859
31332
|
);
|
|
30860
31333
|
const sumUnverifiedCrawlers = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
30861
|
-
db.select({ total:
|
|
31334
|
+
db.select({ total: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
30862
31335
|
and29(
|
|
30863
31336
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
30864
31337
|
ne5(crawlerEventsHourly.verificationStatus, VerificationStatuses.verified),
|
|
@@ -30869,9 +31342,9 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30869
31342
|
);
|
|
30870
31343
|
const sumReferrals = (windowStartIso, windowEndIso, exclusiveEnd = false) => {
|
|
30871
31344
|
const row = db.select({
|
|
30872
|
-
total:
|
|
30873
|
-
paid:
|
|
30874
|
-
organic:
|
|
31345
|
+
total: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
31346
|
+
paid: sql11`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
31347
|
+
organic: sql11`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`
|
|
30875
31348
|
}).from(aiReferralEventsHourly).where(
|
|
30876
31349
|
and29(
|
|
30877
31350
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -30883,7 +31356,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30883
31356
|
return aiReferralClassCounts(Number(row?.total ?? 0), Number(row?.paid ?? 0), Number(row?.organic ?? 0));
|
|
30884
31357
|
};
|
|
30885
31358
|
const sumUserFetches = (windowStartIso, windowEndIso, exclusiveEnd = false) => Number(
|
|
30886
|
-
db.select({ total:
|
|
31359
|
+
db.select({ total: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
30887
31360
|
and29(
|
|
30888
31361
|
eq33(aiUserFetchEventsHourly.projectId, projectId),
|
|
30889
31362
|
gte8(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
@@ -30899,7 +31372,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30899
31372
|
const userFetchPrior = sumUserFetches(priorStart, headlineStart, true);
|
|
30900
31373
|
const referralCurrent = sumReferrals(headlineStart, headlineEnd);
|
|
30901
31374
|
const referralAllCurrent = Number(
|
|
30902
|
-
db.select({ total:
|
|
31375
|
+
db.select({ total: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
30903
31376
|
and29(
|
|
30904
31377
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
30905
31378
|
nonSubresourceReferralPathCondition(),
|
|
@@ -30913,7 +31386,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30913
31386
|
const crawlerByOperatorRows = db.select({
|
|
30914
31387
|
operator: crawlerEventsHourly.operator,
|
|
30915
31388
|
verificationStatus: crawlerEventsHourly.verificationStatus,
|
|
30916
|
-
hits:
|
|
31389
|
+
hits: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
30917
31390
|
}).from(crawlerEventsHourly).where(
|
|
30918
31391
|
and29(
|
|
30919
31392
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
@@ -30923,7 +31396,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30923
31396
|
).groupBy(crawlerEventsHourly.operator, crawlerEventsHourly.verificationStatus).all();
|
|
30924
31397
|
const crawlerByOperatorPriorRows = db.select({
|
|
30925
31398
|
operator: crawlerEventsHourly.operator,
|
|
30926
|
-
hits:
|
|
31399
|
+
hits: sql11`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
30927
31400
|
}).from(crawlerEventsHourly).where(
|
|
30928
31401
|
and29(
|
|
30929
31402
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
@@ -30933,7 +31406,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30933
31406
|
).groupBy(crawlerEventsHourly.operator).all();
|
|
30934
31407
|
const referralByOperatorRows = db.select({
|
|
30935
31408
|
operator: aiReferralEventsHourly.operator,
|
|
30936
|
-
hits:
|
|
31409
|
+
hits: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
30937
31410
|
}).from(aiReferralEventsHourly).where(
|
|
30938
31411
|
and29(
|
|
30939
31412
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -30944,7 +31417,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30944
31417
|
).groupBy(aiReferralEventsHourly.operator).all();
|
|
30945
31418
|
const userFetchByOperatorRows = db.select({
|
|
30946
31419
|
operator: aiUserFetchEventsHourly.operator,
|
|
30947
|
-
hits:
|
|
31420
|
+
hits: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
30948
31421
|
}).from(aiUserFetchEventsHourly).where(
|
|
30949
31422
|
and29(
|
|
30950
31423
|
eq33(aiUserFetchEventsHourly.projectId, projectId),
|
|
@@ -30987,16 +31460,16 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
30987
31460
|
);
|
|
30988
31461
|
const topPathsRows = db.select({
|
|
30989
31462
|
path: crawlerEventsHourly.pathNormalized,
|
|
30990
|
-
verifiedHits:
|
|
30991
|
-
unverifiedHits:
|
|
30992
|
-
operators:
|
|
31463
|
+
verifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
31464
|
+
unverifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
31465
|
+
operators: sql11`COUNT(DISTINCT ${crawlerEventsHourly.operator})`
|
|
30993
31466
|
}).from(crawlerEventsHourly).where(
|
|
30994
31467
|
and29(
|
|
30995
31468
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
30996
31469
|
gte8(crawlerEventsHourly.tsHour, headlineStart),
|
|
30997
31470
|
lte6(crawlerEventsHourly.tsHour, headlineEnd)
|
|
30998
31471
|
)
|
|
30999
|
-
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc14(
|
|
31472
|
+
).groupBy(crawlerEventsHourly.pathNormalized).orderBy(desc14(sql11`SUM(${crawlerEventsHourly.hits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
31000
31473
|
const topCrawledPaths = topPathsRows.map((r) => ({
|
|
31001
31474
|
path: r.path,
|
|
31002
31475
|
verifiedHits: Number(r.verifiedHits),
|
|
@@ -31005,8 +31478,8 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31005
31478
|
}));
|
|
31006
31479
|
const referralProductsRows = db.select({
|
|
31007
31480
|
product: aiReferralEventsHourly.product,
|
|
31008
|
-
arrivals:
|
|
31009
|
-
landingPaths:
|
|
31481
|
+
arrivals: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
31482
|
+
landingPaths: sql11`COUNT(DISTINCT ${aiReferralEventsHourly.landingPathNormalized})`
|
|
31010
31483
|
}).from(aiReferralEventsHourly).where(
|
|
31011
31484
|
and29(
|
|
31012
31485
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -31014,7 +31487,7 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31014
31487
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
31015
31488
|
lte6(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
31016
31489
|
)
|
|
31017
|
-
).groupBy(aiReferralEventsHourly.product).orderBy(desc14(
|
|
31490
|
+
).groupBy(aiReferralEventsHourly.product).orderBy(desc14(sql11`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).all();
|
|
31018
31491
|
const referralProducts = referralProductsRows.map((r) => ({
|
|
31019
31492
|
product: r.product,
|
|
31020
31493
|
arrivals: Number(r.arrivals),
|
|
@@ -31022,8 +31495,8 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31022
31495
|
}));
|
|
31023
31496
|
const topReferralRows = db.select({
|
|
31024
31497
|
path: aiReferralEventsHourly.landingPathNormalized,
|
|
31025
|
-
arrivals:
|
|
31026
|
-
products:
|
|
31498
|
+
arrivals: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
31499
|
+
products: sql11`COUNT(DISTINCT ${aiReferralEventsHourly.product})`
|
|
31027
31500
|
}).from(aiReferralEventsHourly).where(
|
|
31028
31501
|
and29(
|
|
31029
31502
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -31031,26 +31504,26 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31031
31504
|
gte8(aiReferralEventsHourly.tsHour, headlineStart),
|
|
31032
31505
|
lte6(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
31033
31506
|
)
|
|
31034
|
-
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc14(
|
|
31507
|
+
).groupBy(aiReferralEventsHourly.landingPathNormalized).orderBy(desc14(sql11`SUM(${aiReferralEventsHourly.sessionsOrHits})`)).limit(SERVER_ACTIVITY_TOP_PATHS_LIMIT).all();
|
|
31035
31508
|
const topReferralLandingPaths = topReferralRows.map((r) => ({
|
|
31036
31509
|
path: r.path,
|
|
31037
31510
|
arrivals: Number(r.arrivals),
|
|
31038
31511
|
distinctProducts: Number(r.products)
|
|
31039
31512
|
}));
|
|
31040
31513
|
const crawlerTrendRows = db.select({
|
|
31041
|
-
date:
|
|
31042
|
-
verifiedHits:
|
|
31043
|
-
unverifiedHits:
|
|
31514
|
+
date: sql11`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`,
|
|
31515
|
+
verifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} = ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`,
|
|
31516
|
+
unverifiedHits: sql11`COALESCE(SUM(CASE WHEN ${crawlerEventsHourly.verificationStatus} <> ${VerificationStatuses.verified} THEN ${crawlerEventsHourly.hits} ELSE 0 END), 0)`
|
|
31044
31517
|
}).from(crawlerEventsHourly).where(
|
|
31045
31518
|
and29(
|
|
31046
31519
|
eq33(crawlerEventsHourly.projectId, projectId),
|
|
31047
31520
|
gte8(crawlerEventsHourly.tsHour, trendStart),
|
|
31048
31521
|
lte6(crawlerEventsHourly.tsHour, headlineEnd)
|
|
31049
31522
|
)
|
|
31050
|
-
).groupBy(
|
|
31523
|
+
).groupBy(sql11`SUBSTR(${crawlerEventsHourly.tsHour}, 1, 10)`).all();
|
|
31051
31524
|
const referralTrendRows = db.select({
|
|
31052
|
-
date:
|
|
31053
|
-
hits:
|
|
31525
|
+
date: sql11`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`,
|
|
31526
|
+
hits: sql11`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
31054
31527
|
}).from(aiReferralEventsHourly).where(
|
|
31055
31528
|
and29(
|
|
31056
31529
|
eq33(aiReferralEventsHourly.projectId, projectId),
|
|
@@ -31058,17 +31531,17 @@ function buildServerActivity(db, projectId, windowDays2) {
|
|
|
31058
31531
|
gte8(aiReferralEventsHourly.tsHour, trendStart),
|
|
31059
31532
|
lte6(aiReferralEventsHourly.tsHour, headlineEnd)
|
|
31060
31533
|
)
|
|
31061
|
-
).groupBy(
|
|
31534
|
+
).groupBy(sql11`SUBSTR(${aiReferralEventsHourly.tsHour}, 1, 10)`).all();
|
|
31062
31535
|
const userFetchTrendRows = db.select({
|
|
31063
|
-
date:
|
|
31064
|
-
hits:
|
|
31536
|
+
date: sql11`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`,
|
|
31537
|
+
hits: sql11`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
31065
31538
|
}).from(aiUserFetchEventsHourly).where(
|
|
31066
31539
|
and29(
|
|
31067
31540
|
eq33(aiUserFetchEventsHourly.projectId, projectId),
|
|
31068
31541
|
gte8(aiUserFetchEventsHourly.tsHour, trendStart),
|
|
31069
31542
|
lte6(aiUserFetchEventsHourly.tsHour, headlineEnd)
|
|
31070
31543
|
)
|
|
31071
|
-
).groupBy(
|
|
31544
|
+
).groupBy(sql11`SUBSTR(${aiUserFetchEventsHourly.tsHour}, 1, 10)`).all();
|
|
31072
31545
|
const emptyTrendEntry = () => ({ verifiedCrawlerHits: 0, unverifiedCrawlerHits: 0, userFetchHits: 0, referralArrivals: 0 });
|
|
31073
31546
|
const dailyTrendMap = /* @__PURE__ */ new Map();
|
|
31074
31547
|
for (const r of crawlerTrendRows) {
|
|
@@ -31219,7 +31692,7 @@ function buildInsightList(db, projectId, locationFilter) {
|
|
|
31219
31692
|
and29(
|
|
31220
31693
|
eq33(runs.projectId, projectId),
|
|
31221
31694
|
eq33(runs.kind, RunKinds["answer-visibility"]),
|
|
31222
|
-
|
|
31695
|
+
or5(eq33(runs.status, RunStatuses.completed), eq33(runs.status, RunStatuses.partial)),
|
|
31223
31696
|
notProbeRun()
|
|
31224
31697
|
)
|
|
31225
31698
|
).orderBy(desc14(runs.createdAt)).all().filter((r) => locationFilter === void 0 || (r.location ?? null) === locationFilter).slice(0, INSIGHT_LOOKBACK_RUNS).map((r) => r.id);
|
|
@@ -32101,10 +32574,10 @@ async function reportRoutes(app) {
|
|
|
32101
32574
|
}
|
|
32102
32575
|
|
|
32103
32576
|
// ../api-routes/src/organic-evidence.ts
|
|
32104
|
-
import { and as and31, desc as desc15, eq as eq35, gte as gte10, lte as lte8, or as
|
|
32577
|
+
import { and as and31, desc as desc15, eq as eq35, gte as gte10, lte as lte8, or as or7, sql as sql13 } from "drizzle-orm";
|
|
32105
32578
|
|
|
32106
32579
|
// ../api-routes/src/ga-measurement-analysis.ts
|
|
32107
|
-
import { and as and30, eq as eq34, gte as gte9, lte as lte7, or as
|
|
32580
|
+
import { and as and30, eq as eq34, gte as gte9, lte as lte7, or as or6, sql as sql12 } from "drizzle-orm";
|
|
32108
32581
|
var UNKNOWN_LANDING_PAGE = "(not set)";
|
|
32109
32582
|
function addDays(date, offset) {
|
|
32110
32583
|
const value = /* @__PURE__ */ new Date(`${date}T00:00:00.000Z`);
|
|
@@ -32276,24 +32749,24 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32276
32749
|
const scopedConditions = (hostColumn, normalizedPathColumn, rawPathColumn) => {
|
|
32277
32750
|
const conditions = [];
|
|
32278
32751
|
if (parsedHostScope.data === "marketing") {
|
|
32279
|
-
const lowerHost =
|
|
32280
|
-
const normalizedHost2 =
|
|
32281
|
-
conditions.push(
|
|
32282
|
-
|
|
32283
|
-
|
|
32752
|
+
const lowerHost = sql12`lower(${hostColumn})`;
|
|
32753
|
+
const normalizedHost2 = sql12`case when ${lowerHost} like 'www.%' then substr(${lowerHost}, 5) else ${lowerHost} end`;
|
|
32754
|
+
conditions.push(or6(...marketingHosts.flatMap((host) => [
|
|
32755
|
+
sql12`${normalizedHost2} = ${host}`,
|
|
32756
|
+
sql12`${normalizedHost2} like ${`%.${host}`}`
|
|
32284
32757
|
])));
|
|
32285
32758
|
}
|
|
32286
32759
|
if (pathPrefix && pathPrefix !== "/") {
|
|
32287
|
-
const rawPath =
|
|
32288
|
-
const landingPath =
|
|
32289
|
-
conditions.push(
|
|
32760
|
+
const rawPath = sql12`case when instr(${rawPathColumn}, '?') = 0 then ${rawPathColumn} else substr(${rawPathColumn}, 1, instr(${rawPathColumn}, '?') - 1) end`;
|
|
32761
|
+
const landingPath = sql12`case when trim(coalesce(${normalizedPathColumn}, '')) = '' then ${rawPath} else ${normalizedPathColumn} end`;
|
|
32762
|
+
conditions.push(or6(sql12`${landingPath} = ${pathPrefix}`, sql12`${landingPath} like ${`${pathPrefix}/%`}`));
|
|
32290
32763
|
}
|
|
32291
32764
|
return conditions;
|
|
32292
32765
|
};
|
|
32293
32766
|
const acquisitionScope = scopedConditions(gaAcquisitionDaily.hostName, gaAcquisitionDaily.landingPageNormalized, gaAcquisitionDaily.landingPage);
|
|
32294
32767
|
const leadLandingScope = scopedConditions(gaLeadEventsDaily.hostName, gaLeadEventsDaily.landingPageNormalized, gaLeadEventsDaily.landingPage);
|
|
32295
|
-
const acquisitionAnchor = db.select({ date:
|
|
32296
|
-
const leadAnchor = db.select({ date:
|
|
32768
|
+
const acquisitionAnchor = db.select({ date: sql12`max(${gaAcquisitionDaily.date})` }).from(gaAcquisitionDaily).where(and30(eq34(gaAcquisitionDaily.projectId, project.id), ...acquisitionScope)).get()?.date ?? null;
|
|
32769
|
+
const leadAnchor = db.select({ date: sql12`max(${gaLeadEventsDaily.date})` }).from(gaLeadEventsDaily).where(and30(eq34(gaLeadEventsDaily.projectId, project.id), or6(
|
|
32297
32770
|
eq34(gaLeadEventsDaily.attributionScope, "channel"),
|
|
32298
32771
|
and30(eq34(gaLeadEventsDaily.attributionScope, "landing-page"), ...leadLandingScope)
|
|
32299
32772
|
))).get()?.date ?? null;
|
|
@@ -32302,7 +32775,7 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32302
32775
|
const gaStartDate = gaPeriods[0]?.startDate;
|
|
32303
32776
|
const acquisitionRows = gaStartDate ? db.select().from(gaAcquisitionDaily).where(and30(eq34(gaAcquisitionDaily.projectId, project.id), gte9(gaAcquisitionDaily.date, gaStartDate), lte7(gaAcquisitionDaily.date, gaAnchor))).all() : [];
|
|
32304
32777
|
const leadRows = gaStartDate ? db.select().from(gaLeadEventsDaily).where(and30(eq34(gaLeadEventsDaily.projectId, project.id), gte9(gaLeadEventsDaily.date, gaStartDate), lte7(gaLeadEventsDaily.date, gaAnchor))).all() : [];
|
|
32305
|
-
const engagementAnchor = db.select({ date:
|
|
32778
|
+
const engagementAnchor = db.select({ date: sql12`max(${gaDailyTotals.date})` }).from(gaDailyTotals).where(eq34(gaDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32306
32779
|
const engagementPeriodWindow = engagementAnchor ? buildPeriods(engagementAnchor, days) : [];
|
|
32307
32780
|
const engagementStartDate = engagementPeriodWindow[0]?.startDate;
|
|
32308
32781
|
const engagementRows = engagementStartDate ? db.select().from(gaDailyTotals).where(and30(
|
|
@@ -32311,15 +32784,15 @@ function buildGaMeasurementAnalysis(db, projectName, options = {}) {
|
|
|
32311
32784
|
lte7(gaDailyTotals.date, engagementAnchor)
|
|
32312
32785
|
)).all() : [];
|
|
32313
32786
|
const engagementAvailableFrom = db.select({
|
|
32314
|
-
date:
|
|
32787
|
+
date: sql12`min(${gaDailyTotals.date})`
|
|
32315
32788
|
}).from(gaDailyTotals).where(and30(
|
|
32316
32789
|
eq34(gaDailyTotals.projectId, project.id),
|
|
32317
|
-
|
|
32318
|
-
|
|
32319
|
-
|
|
32790
|
+
or6(
|
|
32791
|
+
sql12`${gaDailyTotals.engagementRate} is not null`,
|
|
32792
|
+
sql12`${gaDailyTotals.newUsers} is not null`
|
|
32320
32793
|
)
|
|
32321
32794
|
)).get()?.date ?? null;
|
|
32322
|
-
const gscAnchor = db.select({ date:
|
|
32795
|
+
const gscAnchor = db.select({ date: sql12`max(${gscDailyTotals.date})` }).from(gscDailyTotals).where(eq34(gscDailyTotals.projectId, project.id)).get()?.date ?? null;
|
|
32323
32796
|
const gscPeriods = gscAnchor ? buildPeriods(gscAnchor, days) : [];
|
|
32324
32797
|
const gscStartDate = gscPeriods[0]?.startDate;
|
|
32325
32798
|
const propertyRows = gscStartDate ? db.select().from(gscDailyTotals).where(and30(eq34(gscDailyTotals.projectId, project.id), gte9(gscDailyTotals.date, gscStartDate), lte7(gscDailyTotals.date, gscAnchor))).all() : [];
|
|
@@ -32646,22 +33119,22 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
32646
33119
|
const nativeStartDate = nativeAcquisition.periods[0]?.startDate;
|
|
32647
33120
|
const nativeEndDate = nativeAcquisition.periods.at(-1)?.endDate;
|
|
32648
33121
|
const gscCoverage = aggregateCoverage(db.select({
|
|
32649
|
-
startDate:
|
|
32650
|
-
endDate:
|
|
32651
|
-
observedDays:
|
|
33122
|
+
startDate: sql13`min(${gscDailyTotals.date})`,
|
|
33123
|
+
endDate: sql13`max(${gscDailyTotals.date})`,
|
|
33124
|
+
observedDays: sql13`count(distinct ${gscDailyTotals.date})`
|
|
32652
33125
|
}).from(gscDailyTotals).where(eq35(gscDailyTotals.projectId, project.id)).get());
|
|
32653
33126
|
const legacyGaCoverage = aggregateCoverage(db.select({
|
|
32654
|
-
startDate:
|
|
32655
|
-
endDate:
|
|
32656
|
-
observedDays:
|
|
33127
|
+
startDate: sql13`min(${gaTrafficSnapshots.date})`,
|
|
33128
|
+
endDate: sql13`max(${gaTrafficSnapshots.date})`,
|
|
33129
|
+
observedDays: sql13`count(distinct ${gaTrafficSnapshots.date})`
|
|
32657
33130
|
}).from(gaTrafficSnapshots).where(eq35(gaTrafficSnapshots.projectId, project.id)).get());
|
|
32658
33131
|
const nativeCoverage = nativeAcquisitionActive && normalizedMarketingHosts.length ? aggregateCoverage(db.select({
|
|
32659
|
-
startDate:
|
|
32660
|
-
endDate:
|
|
32661
|
-
observedDays:
|
|
33132
|
+
startDate: sql13`min(${gaAcquisitionDaily.date})`,
|
|
33133
|
+
endDate: sql13`max(${gaAcquisitionDaily.date})`,
|
|
33134
|
+
observedDays: sql13`count(distinct ${gaAcquisitionDaily.date})`
|
|
32662
33135
|
}).from(gaAcquisitionDaily).where(and31(
|
|
32663
33136
|
eq35(gaAcquisitionDaily.projectId, project.id),
|
|
32664
|
-
|
|
33137
|
+
or7(...normalizedMarketingHosts.map((candidate) => sql13`lower(trim(${gaAcquisitionDaily.hostName})) = ${candidate} or lower(trim(${gaAcquisitionDaily.hostName})) like ${`%.${candidate}`}`))
|
|
32665
33138
|
)).get()) : null;
|
|
32666
33139
|
const gaCoverage = nativeAcquisitionActive ? nativeCoverage : legacyGaCoverage;
|
|
32667
33140
|
const asOfDate = gscCoverage?.endDate ?? gaCoverage?.endDate ?? null;
|
|
@@ -32791,7 +33264,7 @@ function buildOrganicEvidence(db, projectName, periodDays) {
|
|
|
32791
33264
|
const allCrawlers = db.select().from(crawlerEventsHourly).where(and31(eq35(crawlerEventsHourly.projectId, project.id), gte10(crawlerEventsHourly.tsHour, serverStart), lte8(crawlerEventsHourly.tsHour, serverEnd))).all();
|
|
32792
33265
|
const allFetches = db.select().from(aiUserFetchEventsHourly).where(and31(eq35(aiUserFetchEventsHourly.projectId, project.id), gte10(aiUserFetchEventsHourly.tsHour, serverStart), lte8(aiUserFetchEventsHourly.tsHour, serverEnd))).all();
|
|
32793
33266
|
const allReferrals = db.select().from(aiReferralEventsHourly).where(and31(eq35(aiReferralEventsHourly.projectId, project.id), countableReferralCondition(), gte10(aiReferralEventsHourly.tsHour, serverStart), lte8(aiReferralEventsHourly.tsHour, serverEnd))).all();
|
|
32794
|
-
const [serverCoverageRow] = db.all(
|
|
33267
|
+
const [serverCoverageRow] = db.all(sql13`
|
|
32795
33268
|
select
|
|
32796
33269
|
min(day) as startDate,
|
|
32797
33270
|
max(day) as endDate,
|
|
@@ -33890,15 +34363,15 @@ function renderResultsExportCsv(report) {
|
|
|
33890
34363
|
].map(csvCell).join(","));
|
|
33891
34364
|
return [CSV_COLUMNS.join(","), ...rows].join("\r\n") + "\r\n";
|
|
33892
34365
|
}
|
|
33893
|
-
function filenameFor(projectName,
|
|
34366
|
+
function filenameFor(projectName, format2) {
|
|
33894
34367
|
const safeName = projectName.replace(/[^\w.-]+/gi, "_") || "project";
|
|
33895
34368
|
const date = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
33896
|
-
return `canonry-results-${safeName}-${date}.${
|
|
34369
|
+
return `canonry-results-${safeName}-${date}.${format2}`;
|
|
33897
34370
|
}
|
|
33898
34371
|
async function resultsExportRoutes(app) {
|
|
33899
34372
|
app.get("/projects/:name/results/export", async (request, reply) => {
|
|
33900
34373
|
const project = resolveProject(app.db, request.params.name);
|
|
33901
|
-
const
|
|
34374
|
+
const format2 = parseFormat(request.query.format);
|
|
33902
34375
|
const includeProbes = parseBoolean(request.query.includeProbes, "includeProbes");
|
|
33903
34376
|
const { sinceIso, untilIso } = parseDateBounds(request.query.since, request.query.until);
|
|
33904
34377
|
const conditions = [
|
|
@@ -33990,10 +34463,10 @@ async function resultsExportRoutes(app) {
|
|
|
33990
34463
|
recordCount: records.length,
|
|
33991
34464
|
records
|
|
33992
34465
|
};
|
|
33993
|
-
reply.header("Content-Disposition", `attachment; filename="${filenameFor(project.name,
|
|
34466
|
+
reply.header("Content-Disposition", `attachment; filename="${filenameFor(project.name, format2)}"`);
|
|
33994
34467
|
reply.header("Cache-Control", "private, no-store");
|
|
33995
34468
|
reply.header("X-Content-Type-Options", "nosniff");
|
|
33996
|
-
if (
|
|
34469
|
+
if (format2 === "csv") {
|
|
33997
34470
|
reply.type("text/csv; charset=utf-8");
|
|
33998
34471
|
return reply.send(renderResultsExportCsv(dto));
|
|
33999
34472
|
}
|
|
@@ -34003,7 +34476,7 @@ async function resultsExportRoutes(app) {
|
|
|
34003
34476
|
}
|
|
34004
34477
|
|
|
34005
34478
|
// ../api-routes/src/composites.ts
|
|
34006
|
-
import { eq as eq39, and as and35, desc as desc17, sql as
|
|
34479
|
+
import { eq as eq39, and as and35, desc as desc17, sql as sql14, like as like3, or as or8, inArray as inArray18 } from "drizzle-orm";
|
|
34007
34480
|
var TOP_INSIGHT_LIMIT = 5;
|
|
34008
34481
|
var SEARCH_HIT_HARD_LIMIT = 50;
|
|
34009
34482
|
var SEARCH_SNIPPET_RADIUS = 80;
|
|
@@ -34200,23 +34673,23 @@ async function compositeRoutes(app) {
|
|
|
34200
34673
|
}).from(querySnapshots).innerJoin(queries, eq39(querySnapshots.queryId, queries.id)).where(
|
|
34201
34674
|
and35(
|
|
34202
34675
|
eq39(queries.projectId, project.id),
|
|
34203
|
-
|
|
34204
|
-
|
|
34205
|
-
|
|
34206
|
-
|
|
34207
|
-
|
|
34208
|
-
|
|
34676
|
+
or8(
|
|
34677
|
+
sql14`${querySnapshots.answerText} LIKE ${pattern} ESCAPE '\\'`,
|
|
34678
|
+
sql14`${querySnapshots.citedDomains} LIKE ${pattern} ESCAPE '\\'`,
|
|
34679
|
+
sql14`${querySnapshots.citedUrls} LIKE ${pattern} ESCAPE '\\'`,
|
|
34680
|
+
sql14`${querySnapshots.rawResponse} LIKE ${pattern} ESCAPE '\\'`,
|
|
34681
|
+
like3(queries.query, pattern)
|
|
34209
34682
|
)
|
|
34210
34683
|
)
|
|
34211
34684
|
).orderBy(desc17(querySnapshots.createdAt)).limit(limit + 1).all());
|
|
34212
34685
|
const insightMatches = app.db.select().from(insights).where(
|
|
34213
34686
|
and35(
|
|
34214
34687
|
eq39(insights.projectId, project.id),
|
|
34215
|
-
|
|
34216
|
-
|
|
34217
|
-
|
|
34218
|
-
|
|
34219
|
-
|
|
34688
|
+
or8(
|
|
34689
|
+
like3(insights.title, pattern),
|
|
34690
|
+
like3(insights.query, pattern),
|
|
34691
|
+
sql14`${insights.recommendation} LIKE ${pattern} ESCAPE '\\'`,
|
|
34692
|
+
sql14`${insights.cause} LIKE ${pattern} ESCAPE '\\'`
|
|
34220
34693
|
)
|
|
34221
34694
|
)
|
|
34222
34695
|
).orderBy(desc17(insights.createdAt)).limit(limit + 1).all();
|
|
@@ -34439,20 +34912,20 @@ function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
|
|
|
34439
34912
|
const dimensionedRows = app.db.select({
|
|
34440
34913
|
date: gscSearchData.date,
|
|
34441
34914
|
query: gscSearchData.query,
|
|
34442
|
-
impressions:
|
|
34443
|
-
clicks:
|
|
34915
|
+
impressions: sql14`COALESCE(SUM(${gscSearchData.impressions}), 0)`,
|
|
34916
|
+
clicks: sql14`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
34444
34917
|
// Weighted average: SUM(position * impressions) / SUM(impressions).
|
|
34445
34918
|
// NULLIF guards the degenerate impressions=0 case (SQLite returns NULL,
|
|
34446
34919
|
// which the JS coerces to 0 — caught by the impression floor anyway).
|
|
34447
|
-
avgPosition:
|
|
34920
|
+
avgPosition: sql14`COALESCE(SUM(${gscSearchData.position} * ${gscSearchData.impressions}) * 1.0 / NULLIF(SUM(${gscSearchData.impressions}), 0), 0)`
|
|
34448
34921
|
}).from(gscSearchData).where(and35(
|
|
34449
34922
|
eq39(gscSearchData.projectId, projectId),
|
|
34450
|
-
|
|
34923
|
+
sql14`${gscSearchData.date} >= ${cutoff}`,
|
|
34451
34924
|
// Same upper bound as the accurate source below. The merge picks a source
|
|
34452
34925
|
// PER DAY, so an asymmetric range would let a dimensioned-only day past
|
|
34453
34926
|
// the window's end into a basket the other source cannot balance.
|
|
34454
|
-
|
|
34455
|
-
|
|
34927
|
+
sql14`${gscSearchData.date} <= ${suggestionWindow.endDate ?? "9999-12-31"}`,
|
|
34928
|
+
sql14`${gscSearchData.impressions} > 0`
|
|
34456
34929
|
)).groupBy(gscSearchData.date, gscSearchData.query).all();
|
|
34457
34930
|
const merged = mergeGscQueryTotalsWithFallback(
|
|
34458
34931
|
readGscQueryDailyRows(app.db, projectId, cutoff, suggestionWindow.endDate ?? "9999-12-31"),
|
|
@@ -34890,6 +35363,7 @@ var SCHEMA_TABLE = {
|
|
|
34890
35363
|
GtmWorkspaceListResponse: gtmWorkspaceListResponseSchema,
|
|
34891
35364
|
HealthSnapshotDto: healthSnapshotDtoSchema,
|
|
34892
35365
|
IndexingRequestResponseDto: indexingRequestResponseDtoSchema,
|
|
35366
|
+
IntegrationSettingsSummaryDto: integrationSettingsSummaryDtoSchema,
|
|
34893
35367
|
KeywordDto: keywordDtoSchema,
|
|
34894
35368
|
LatestProjectRunDto: latestProjectRunDtoSchema,
|
|
34895
35369
|
LocationContext: locationContextSchema,
|
|
@@ -34970,6 +35444,7 @@ var SCHEMA_TABLE = {
|
|
|
34970
35444
|
ProjectOverviewDto: projectOverviewDtoSchema,
|
|
34971
35445
|
ProjectReportDto: projectReportDtoSchema,
|
|
34972
35446
|
ProjectSearchResponseDto: projectSearchResponseSchema,
|
|
35447
|
+
ProviderSummaryEntryDto: providerSummaryEntryDtoSchema,
|
|
34973
35448
|
QueryDto: queryDtoSchema,
|
|
34974
35449
|
ResultsExportDto: resultsExportDtoSchema,
|
|
34975
35450
|
RunDetailDto: runDetailDtoSchema,
|
|
@@ -35003,6 +35478,8 @@ var SCHEMA_TABLE = {
|
|
|
35003
35478
|
SnapshotRequest: z.object(snapshotRequestInputSchema.shape).omit({ phrases: true }),
|
|
35004
35479
|
SourceBreakdownDto: sourceBreakdownDtoSchema,
|
|
35005
35480
|
TelemetryEventAcceptedDto: telemetryEventAcceptedDtoSchema,
|
|
35481
|
+
TelemetryStatusDto: telemetryStatusDtoSchema,
|
|
35482
|
+
OperationalLogListDto: operationalLogListDtoSchema,
|
|
35006
35483
|
CloudflareWorkerIngestRequest: cloudflareWorkerIngestRequestSchema,
|
|
35007
35484
|
CloudflareWorkerIngestResponse: cloudflareWorkerIngestResponseSchema,
|
|
35008
35485
|
TrafficBackfillResponse: trafficBackfillResponseSchema,
|
|
@@ -37427,6 +37904,7 @@ var routeCatalog = [
|
|
|
37427
37904
|
method: "put",
|
|
37428
37905
|
path: "/api/v1/settings/providers/{name}",
|
|
37429
37906
|
summary: "Update provider settings",
|
|
37907
|
+
description: "Requires settings.write. Already configured providers accept model or quota changes without resubmitting credentials. An unconfigured provider still requires its connection credentials.",
|
|
37430
37908
|
tags: ["settings"],
|
|
37431
37909
|
parameters: [providerNameParameter],
|
|
37432
37910
|
requestBody: {
|
|
@@ -37446,9 +37924,9 @@ var routeCatalog = [
|
|
|
37446
37924
|
}
|
|
37447
37925
|
},
|
|
37448
37926
|
responses: {
|
|
37449
|
-
|
|
37450
|
-
200: rawJsonResponse("Provider updated.", looseObjectSchema),
|
|
37927
|
+
200: jsonResponse("Provider updated.", "ProviderSummaryEntryDto"),
|
|
37451
37928
|
400: errorResponse("Invalid provider settings."),
|
|
37929
|
+
403: errorResponse("The credential lacks settings.write."),
|
|
37452
37930
|
501: errorResponse("Provider updates are not supported.")
|
|
37453
37931
|
}
|
|
37454
37932
|
},
|
|
@@ -37473,8 +37951,7 @@ var routeCatalog = [
|
|
|
37473
37951
|
}
|
|
37474
37952
|
},
|
|
37475
37953
|
responses: {
|
|
37476
|
-
|
|
37477
|
-
200: rawJsonResponse("Google settings updated.", looseObjectSchema),
|
|
37954
|
+
200: jsonResponse("Google settings updated.", "IntegrationSettingsSummaryDto"),
|
|
37478
37955
|
400: errorResponse("Invalid Google settings."),
|
|
37479
37956
|
501: errorResponse("Google settings updates are not supported.")
|
|
37480
37957
|
}
|
|
@@ -37865,14 +38342,34 @@ var routeCatalog = [
|
|
|
37865
38342
|
502: errorResponse("Notification delivery failed.")
|
|
37866
38343
|
}
|
|
37867
38344
|
},
|
|
38345
|
+
{
|
|
38346
|
+
method: "get",
|
|
38347
|
+
path: "/api/v1/operations/logs",
|
|
38348
|
+
summary: "Read bounded, redacted runtime logs",
|
|
38349
|
+
description: "Requires logs.read (or wildcard), an admin role for user sessions, and an instance-wide credential. Includes application and HTTP diagnostics, separate from audit history. File-backed hosts retain logs across restarts; inspect retention, retentionPolicy, dropped, and captureErrors. Filters apply before pagination and must remain unchanged when resuming a cursor. Cursors expire on retention eviction.",
|
|
38350
|
+
tags: ["operations"],
|
|
38351
|
+
parameters: [
|
|
38352
|
+
{ name: "limit", in: "query", required: false, description: "Maximum entries (default 100).", schema: { type: "integer", minimum: 1, maximum: 200, default: 100 } },
|
|
38353
|
+
{ name: "cursor", in: "query", required: false, description: "Opaque cursor bound to the log store and query filters.", schema: { type: "string", minLength: 1, maxLength: 512 } },
|
|
38354
|
+
{ name: "level", in: "query", required: false, description: "Exact log level.", schema: { type: "string", enum: ["trace", "debug", "info", "warn", "error", "fatal"] } },
|
|
38355
|
+
...["module", "runId", "projectId", "actor", "requestId"].map((name) => ({ name, in: "query", required: false, description: `Exact ${name} filter (does not grant project-scoped log access).`, schema: { type: "string", minLength: 1, maxLength: name === "actor" ? 512 : 256 } })),
|
|
38356
|
+
...["since", "until"].map((name) => ({ name, in: "query", required: false, description: `Inclusive ${name} event timestamp.`, schema: { type: "string", format: "date-time" } }))
|
|
38357
|
+
],
|
|
38358
|
+
responses: {
|
|
38359
|
+
200: jsonResponse("Runtime log page with retention and capture-loss metadata.", "OperationalLogListDto"),
|
|
38360
|
+
400: errorResponse("Invalid filters or expired cursor."),
|
|
38361
|
+
401: errorResponse("Authentication required."),
|
|
38362
|
+
403: errorResponse("Instance logs.read permission required."),
|
|
38363
|
+
501: errorResponse("This deployment does not provide operational logs.")
|
|
38364
|
+
}
|
|
38365
|
+
},
|
|
37868
38366
|
{
|
|
37869
38367
|
method: "get",
|
|
37870
38368
|
path: "/api/v1/telemetry",
|
|
37871
38369
|
summary: "Get telemetry status",
|
|
37872
38370
|
tags: ["telemetry"],
|
|
37873
38371
|
responses: {
|
|
37874
|
-
|
|
37875
|
-
200: rawJsonResponse("Telemetry status returned.", looseObjectSchema),
|
|
38372
|
+
200: jsonResponse("Telemetry status returned.", "TelemetryStatusDto"),
|
|
37876
38373
|
501: errorResponse("Telemetry status is not available.")
|
|
37877
38374
|
}
|
|
37878
38375
|
},
|
|
@@ -37880,6 +38377,7 @@ var routeCatalog = [
|
|
|
37880
38377
|
method: "put",
|
|
37881
38378
|
path: "/api/v1/telemetry",
|
|
37882
38379
|
summary: "Update telemetry status",
|
|
38380
|
+
description: "Requires settings.write. The response reports the configured preference and effective state; deployment/environment opt-outs take precedence.",
|
|
37883
38381
|
tags: ["telemetry"],
|
|
37884
38382
|
requestBody: {
|
|
37885
38383
|
required: true,
|
|
@@ -37896,9 +38394,9 @@ var routeCatalog = [
|
|
|
37896
38394
|
}
|
|
37897
38395
|
},
|
|
37898
38396
|
responses: {
|
|
37899
|
-
|
|
37900
|
-
200: rawJsonResponse("Telemetry updated.", looseObjectSchema),
|
|
38397
|
+
200: jsonResponse("Telemetry updated.", "TelemetryStatusDto"),
|
|
37901
38398
|
400: errorResponse("Invalid telemetry request."),
|
|
38399
|
+
403: errorResponse("The credential lacks settings.write."),
|
|
37902
38400
|
501: errorResponse("Telemetry configuration is not available.")
|
|
37903
38401
|
}
|
|
37904
38402
|
},
|
|
@@ -42517,13 +43015,10 @@ async function telemetryRoutes(app, opts) {
|
|
|
42517
43015
|
if (!opts.getTelemetryStatus) {
|
|
42518
43016
|
throw notImplemented("Telemetry status is not available in this deployment");
|
|
42519
43017
|
}
|
|
42520
|
-
|
|
42521
|
-
return {
|
|
42522
|
-
enabled: status.enabled,
|
|
42523
|
-
anonymousId: status.anonymousId ? status.anonymousId.slice(0, 8) + "..." : void 0
|
|
42524
|
-
};
|
|
43018
|
+
return normalizeTelemetryStatus(opts.getTelemetryStatus());
|
|
42525
43019
|
});
|
|
42526
43020
|
app.put("/telemetry", async (request) => {
|
|
43021
|
+
requireScope(request, SETTINGS_WRITE_SCOPE);
|
|
42527
43022
|
if (!opts.setTelemetryEnabled) {
|
|
42528
43023
|
throw notImplemented("Telemetry configuration is not available in this deployment");
|
|
42529
43024
|
}
|
|
@@ -42533,10 +43028,24 @@ async function telemetryRoutes(app, opts) {
|
|
|
42533
43028
|
}
|
|
42534
43029
|
opts.setTelemetryEnabled(enabled);
|
|
42535
43030
|
const status = opts.getTelemetryStatus?.();
|
|
42536
|
-
|
|
42537
|
-
|
|
42538
|
-
|
|
42539
|
-
|
|
43031
|
+
const effective = normalizeTelemetryStatus(status ?? { enabled });
|
|
43032
|
+
if (app.hasDecorator("db")) {
|
|
43033
|
+
try {
|
|
43034
|
+
writeAuditLog(app.db, auditFromRequest(request, {
|
|
43035
|
+
actor: "api",
|
|
43036
|
+
action: "telemetry.updated",
|
|
43037
|
+
entityType: "telemetry",
|
|
43038
|
+
diff: {
|
|
43039
|
+
target: effective.target,
|
|
43040
|
+
configuredEnabled: effective.configuredEnabled,
|
|
43041
|
+
reason: effective.reason
|
|
43042
|
+
}
|
|
43043
|
+
}));
|
|
43044
|
+
} catch {
|
|
43045
|
+
app.log.warn({ action: "telemetry.updated" }, "Telemetry setting audit write failed");
|
|
43046
|
+
}
|
|
43047
|
+
}
|
|
43048
|
+
return effective;
|
|
42540
43049
|
});
|
|
42541
43050
|
app.post("/telemetry/onboarding", async (request, reply) => {
|
|
42542
43051
|
const parsed = onboardingTelemetryEventSchema.safeParse(request.body);
|
|
@@ -42553,6 +43062,25 @@ async function telemetryRoutes(app, opts) {
|
|
|
42553
43062
|
});
|
|
42554
43063
|
}
|
|
42555
43064
|
|
|
43065
|
+
// ../api-routes/src/operational-logs.ts
|
|
43066
|
+
async function operationalLogsRoutes(app, opts = {}) {
|
|
43067
|
+
app.get("/operations/logs", async (request) => {
|
|
43068
|
+
requireAdminSession(request);
|
|
43069
|
+
requireScope(request, "logs.read");
|
|
43070
|
+
if (request.apiKey?.projectId) {
|
|
43071
|
+
throw forbidden("This API key is scoped to one project and cannot read instance operational diagnostics.");
|
|
43072
|
+
}
|
|
43073
|
+
const parsed = logQuerySchema.safeParse(request.query);
|
|
43074
|
+
if (!parsed.success) {
|
|
43075
|
+
throw validationError("Invalid operational logs query.", { issues: parsed.error.issues });
|
|
43076
|
+
}
|
|
43077
|
+
if (!opts.listOperationalLogs) {
|
|
43078
|
+
throw notImplemented("Operational logs are not available in this deployment.");
|
|
43079
|
+
}
|
|
43080
|
+
return operationalLogListDtoSchema.parse(await opts.listOperationalLogs(parsed.data));
|
|
43081
|
+
});
|
|
43082
|
+
}
|
|
43083
|
+
|
|
42556
43084
|
// ../api-routes/src/schedules.ts
|
|
42557
43085
|
import crypto26 from "crypto";
|
|
42558
43086
|
import { and as and36, asc as asc7, eq as eq42 } from "drizzle-orm";
|
|
@@ -43095,7 +43623,7 @@ function formatNotification(row) {
|
|
|
43095
43623
|
|
|
43096
43624
|
// ../api-routes/src/google.ts
|
|
43097
43625
|
import crypto32 from "crypto";
|
|
43098
|
-
import { eq as eq44, and as and37, desc as desc19, sql as
|
|
43626
|
+
import { eq as eq44, and as and37, desc as desc19, sql as sql15, inArray as inArray19 } from "drizzle-orm";
|
|
43099
43627
|
|
|
43100
43628
|
// ../api-routes/src/gsc-period-comparison.ts
|
|
43101
43629
|
var DAY_MS = 864e5;
|
|
@@ -45524,11 +46052,11 @@ async function googleRoutes(app, opts) {
|
|
|
45524
46052
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
45525
46053
|
const effectiveEndDate = endDate ?? (daysParam === null ? void 0 : resolvedWindow.endDate ?? void 0);
|
|
45526
46054
|
const conditions = [eq44(gscSearchData.projectId, project.id)];
|
|
45527
|
-
if (startDate) conditions.push(
|
|
45528
|
-
else if (cutoffDate) conditions.push(
|
|
45529
|
-
if (effectiveEndDate) conditions.push(
|
|
45530
|
-
if (query) conditions.push(
|
|
45531
|
-
if (page3) conditions.push(
|
|
46055
|
+
if (startDate) conditions.push(sql15`${gscSearchData.date} >= ${startDate}`);
|
|
46056
|
+
else if (cutoffDate) conditions.push(sql15`${gscSearchData.date} >= ${cutoffDate}`);
|
|
46057
|
+
if (effectiveEndDate) conditions.push(sql15`${gscSearchData.date} <= ${effectiveEndDate}`);
|
|
46058
|
+
if (query) conditions.push(sql15`${gscSearchData.query} LIKE ${"%" + escapeLikePattern(query) + "%"} ESCAPE '\\'`);
|
|
46059
|
+
if (page3) conditions.push(sql15`${gscSearchData.page} LIKE ${"%" + escapeLikePattern(page3) + "%"} ESCAPE '\\'`);
|
|
45532
46060
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
45533
46061
|
const offsetVal = Math.max(parseInt(offset ?? "0", 10) || 0, 0);
|
|
45534
46062
|
const orderColumns = {
|
|
@@ -45537,8 +46065,8 @@ async function googleRoutes(app, opts) {
|
|
|
45537
46065
|
date: [desc19(gscSearchData.date), desc19(gscSearchData.clicks), gscSearchData.query]
|
|
45538
46066
|
}[orderBy];
|
|
45539
46067
|
const rows = app.db.select().from(gscSearchData).where(and37(...conditions)).orderBy(...orderColumns).limit(limitVal).offset(offsetVal).all();
|
|
45540
|
-
const totalMatching = app.db.select({ total:
|
|
45541
|
-
const latestAvailableDate = app.db.select({ latest:
|
|
46068
|
+
const totalMatching = app.db.select({ total: sql15`COUNT(*)` }).from(gscSearchData).where(and37(...conditions)).get()?.total ?? 0;
|
|
46069
|
+
const latestAvailableDate = app.db.select({ latest: sql15`MAX(${gscSearchData.date})` }).from(gscSearchData).where(eq44(gscSearchData.projectId, project.id)).get()?.latest ?? null;
|
|
45542
46070
|
return {
|
|
45543
46071
|
rows: rows.map((r) => ({
|
|
45544
46072
|
date: r.date,
|
|
@@ -45582,12 +46110,12 @@ async function googleRoutes(app, opts) {
|
|
|
45582
46110
|
const fetchStart = priorWindowIsSynced ? priorPeriodStart : windowStart;
|
|
45583
46111
|
const dailyTotals = readGscDailyTotals(app.db, project.id, fetchStart, windowEnd);
|
|
45584
46112
|
const conditions = [eq44(gscSearchData.projectId, project.id)];
|
|
45585
|
-
if (fetchStart) conditions.push(
|
|
45586
|
-
if (endDate) conditions.push(
|
|
46113
|
+
if (fetchStart) conditions.push(sql15`${gscSearchData.date} >= ${fetchStart}`);
|
|
46114
|
+
if (endDate) conditions.push(sql15`${gscSearchData.date} <= ${endDate}`);
|
|
45587
46115
|
const dimensionedRows = app.db.select({
|
|
45588
46116
|
date: gscSearchData.date,
|
|
45589
|
-
clicks:
|
|
45590
|
-
impressions:
|
|
46117
|
+
clicks: sql15`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
46118
|
+
impressions: sql15`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
45591
46119
|
}).from(gscSearchData).where(and37(...conditions)).groupBy(gscSearchData.date).orderBy(gscSearchData.date).all();
|
|
45592
46120
|
const propertyDates = new Set(dailyTotals.map((d) => d.date));
|
|
45593
46121
|
const comparisonDaily = mergeGscDailyTotalsWithFallback(
|
|
@@ -45685,23 +46213,23 @@ async function googleRoutes(app, opts) {
|
|
|
45685
46213
|
);
|
|
45686
46214
|
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
45687
46215
|
const conditions = [eq44(gscSearchData.projectId, project.id)];
|
|
45688
|
-
if (startDate) conditions.push(
|
|
45689
|
-
else if (cutoffDate) conditions.push(
|
|
45690
|
-
if (endDate) conditions.push(
|
|
46216
|
+
if (startDate) conditions.push(sql15`${gscSearchData.date} >= ${startDate}`);
|
|
46217
|
+
else if (cutoffDate) conditions.push(sql15`${gscSearchData.date} >= ${cutoffDate}`);
|
|
46218
|
+
if (endDate) conditions.push(sql15`${gscSearchData.date} <= ${endDate}`);
|
|
45691
46219
|
const limitVal = Math.max(parseInt(limit ?? "50", 10) || 0, 1);
|
|
45692
46220
|
const rows = app.db.select({
|
|
45693
46221
|
page: gscSearchData.page,
|
|
45694
|
-
clicks:
|
|
45695
|
-
impressions:
|
|
45696
|
-
}).from(gscSearchData).where(and37(...conditions)).groupBy(gscSearchData.page).orderBy(desc19(
|
|
46222
|
+
clicks: sql15`COALESCE(SUM(${gscSearchData.clicks}), 0)`,
|
|
46223
|
+
impressions: sql15`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
46224
|
+
}).from(gscSearchData).where(and37(...conditions)).groupBy(gscSearchData.page).orderBy(desc19(sql15`SUM(${gscSearchData.clicks})`), desc19(sql15`SUM(${gscSearchData.impressions})`)).limit(limitVal).all();
|
|
45697
46225
|
const windowStart = startDate ?? cutoffDate ?? "";
|
|
45698
46226
|
const windowEnd = endDate ?? resolvedWindow.endDate ?? "9999-12-31";
|
|
45699
46227
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
45700
46228
|
const totalClicks = dailyTotals.reduce((sum, d) => sum + d.clicks, 0);
|
|
45701
46229
|
const totalImpressions = dailyTotals.reduce((sum, d) => sum + d.impressions, 0);
|
|
45702
46230
|
const rankedSpan = app.db.select({
|
|
45703
|
-
first:
|
|
45704
|
-
last:
|
|
46231
|
+
first: sql15`MIN(${gscSearchData.date})`,
|
|
46232
|
+
last: sql15`MAX(${gscSearchData.date})`
|
|
45705
46233
|
}).from(gscSearchData).where(and37(...conditions)).get();
|
|
45706
46234
|
const coveredFrom = dailyTotals.length > 0 ? dailyTotals[0].date : null;
|
|
45707
46235
|
const coveredThrough = dailyTotals.length > 0 ? dailyTotals[dailyTotals.length - 1].date : null;
|
|
@@ -46670,7 +47198,7 @@ async function googleRoutes(app, opts) {
|
|
|
46670
47198
|
|
|
46671
47199
|
// ../api-routes/src/google-marketing.ts
|
|
46672
47200
|
import crypto33 from "crypto";
|
|
46673
|
-
import { and as and38, desc as desc20, eq as eq45, isNull as isNull6, lt as lt6, ne as ne6, or as
|
|
47201
|
+
import { and as and38, desc as desc20, eq as eq45, isNull as isNull6, lt as lt6, ne as ne6, or as or9, sql as sql16 } from "drizzle-orm";
|
|
46674
47202
|
import { z as z4 } from "zod";
|
|
46675
47203
|
var snapshotPageQuerySchema = z4.object({
|
|
46676
47204
|
limit: z4.coerce.number().int().min(1).max(GOOGLE_MARKETING_STORED_SNAPSHOT_PAGE_MAX).optional(),
|
|
@@ -47621,7 +48149,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47621
48149
|
selectedCustomerCurrencyCode: null,
|
|
47622
48150
|
selectedCustomerTimeZone: null,
|
|
47623
48151
|
selectedCustomerStatus: null,
|
|
47624
|
-
selectionGeneration:
|
|
48152
|
+
selectionGeneration: sql16`${googleAdsConnections.selectionGeneration} + 1`,
|
|
47625
48153
|
scopes: tokens.scopes,
|
|
47626
48154
|
lastValidatedAt: now,
|
|
47627
48155
|
lastCustomerSnapshotId: null,
|
|
@@ -47652,7 +48180,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47652
48180
|
selectedContainerPublicId: null,
|
|
47653
48181
|
selectedWorkspaceId: null,
|
|
47654
48182
|
selectedWorkspaceName: null,
|
|
47655
|
-
selectionGeneration:
|
|
48183
|
+
selectionGeneration: sql16`${gtmConnections.selectionGeneration} + 1`,
|
|
47656
48184
|
scopes: tokens.scopes,
|
|
47657
48185
|
lastValidatedAt: now,
|
|
47658
48186
|
lastSnapshotAt: null,
|
|
@@ -47803,7 +48331,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47803
48331
|
selectedCustomerStatus: customerChanged ? null : existing.selectedCustomerStatus,
|
|
47804
48332
|
// A same-value write still starts a new selection generation. Do not
|
|
47805
48333
|
// let timestamp equality reuse append-only evidence from before it.
|
|
47806
|
-
selectionGeneration:
|
|
48334
|
+
selectionGeneration: sql16`${googleAdsConnections.selectionGeneration} + 1`,
|
|
47807
48335
|
lastValidatedAt: now,
|
|
47808
48336
|
lastCustomerSnapshotId: null,
|
|
47809
48337
|
lastInventorySnapshotAt: null,
|
|
@@ -47847,7 +48375,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47847
48375
|
selectedContainerPublicId: containerChanged ? null : existing.selectedContainerPublicId,
|
|
47848
48376
|
selectedWorkspaceId,
|
|
47849
48377
|
selectedWorkspaceName: workspaceChanged ? null : existing.selectedWorkspaceName,
|
|
47850
|
-
selectionGeneration:
|
|
48378
|
+
selectionGeneration: sql16`${gtmConnections.selectionGeneration} + 1`,
|
|
47851
48379
|
lastValidatedAt: now,
|
|
47852
48380
|
// Every selection write starts a new evidence generation, including a
|
|
47853
48381
|
// same-value reselection. The timestamp alone cannot order a prior
|
|
@@ -47885,7 +48413,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47885
48413
|
const existing = tx.select().from(runs).where(and38(
|
|
47886
48414
|
eq45(runs.projectId, project.id),
|
|
47887
48415
|
eq45(runs.kind, RunKinds["google-ads-sync"]),
|
|
47888
|
-
|
|
48416
|
+
or9(eq45(runs.status, RunStatuses.queued), eq45(runs.status, RunStatuses.running))
|
|
47889
48417
|
)).orderBy(desc20(runs.createdAt)).get();
|
|
47890
48418
|
if (existing) {
|
|
47891
48419
|
run = existing;
|
|
@@ -47921,7 +48449,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47921
48449
|
const existing = tx.select().from(runs).where(and38(
|
|
47922
48450
|
eq45(runs.projectId, project.id),
|
|
47923
48451
|
eq45(runs.kind, RunKinds["gtm-sync"]),
|
|
47924
|
-
|
|
48452
|
+
or9(eq45(runs.status, RunStatuses.queued), eq45(runs.status, RunStatuses.running))
|
|
47925
48453
|
)).orderBy(desc20(runs.createdAt)).get();
|
|
47926
48454
|
if (existing) {
|
|
47927
48455
|
run = existing;
|
|
@@ -47951,7 +48479,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47951
48479
|
const cursor = decodeSnapshotCursor(query.cursor);
|
|
47952
48480
|
const conditions = [eq45(googleAdsRawSnapshots.projectId, project.id)];
|
|
47953
48481
|
if (cursor) {
|
|
47954
|
-
conditions.push(
|
|
48482
|
+
conditions.push(or9(
|
|
47955
48483
|
lt6(googleAdsRawSnapshots.capturedAt, cursor.capturedAt),
|
|
47956
48484
|
and38(eq45(googleAdsRawSnapshots.capturedAt, cursor.capturedAt), lt6(googleAdsRawSnapshots.id, cursor.id))
|
|
47957
48485
|
));
|
|
@@ -47959,7 +48487,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47959
48487
|
const limit = query.limit ?? 50;
|
|
47960
48488
|
const rows = app.db.select().from(googleAdsRawSnapshots).where(and38(...conditions)).orderBy(desc20(googleAdsRawSnapshots.capturedAt), desc20(googleAdsRawSnapshots.id)).limit(limit + 1).all();
|
|
47961
48489
|
const page3 = rows.slice(0, limit);
|
|
47962
|
-
const total = app.db.select({ total:
|
|
48490
|
+
const total = app.db.select({ total: sql16`COUNT(*)`.mapWith(Number).as("total") }).from(googleAdsRawSnapshots).where(eq45(googleAdsRawSnapshots.projectId, project.id)).get();
|
|
47963
48491
|
return {
|
|
47964
48492
|
snapshots: page3.map(toGoogleAdsSnapshotMetadata),
|
|
47965
48493
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -47983,7 +48511,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47983
48511
|
const cursor = decodeSnapshotCursor(query.cursor);
|
|
47984
48512
|
const conditions = [eq45(gtmRawSnapshots.projectId, project.id)];
|
|
47985
48513
|
if (cursor) {
|
|
47986
|
-
conditions.push(
|
|
48514
|
+
conditions.push(or9(
|
|
47987
48515
|
lt6(gtmRawSnapshots.capturedAt, cursor.capturedAt),
|
|
47988
48516
|
and38(eq45(gtmRawSnapshots.capturedAt, cursor.capturedAt), lt6(gtmRawSnapshots.id, cursor.id))
|
|
47989
48517
|
));
|
|
@@ -47991,7 +48519,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
47991
48519
|
const limit = query.limit ?? 50;
|
|
47992
48520
|
const rows = app.db.select().from(gtmRawSnapshots).where(and38(...conditions)).orderBy(desc20(gtmRawSnapshots.capturedAt), desc20(gtmRawSnapshots.id)).limit(limit + 1).all();
|
|
47993
48521
|
const page3 = rows.slice(0, limit);
|
|
47994
|
-
const total = app.db.select({ total:
|
|
48522
|
+
const total = app.db.select({ total: sql16`COUNT(*)`.mapWith(Number).as("total") }).from(gtmRawSnapshots).where(eq45(gtmRawSnapshots.projectId, project.id)).get();
|
|
47995
48523
|
return {
|
|
47996
48524
|
snapshots: page3.map(toGtmSnapshotMetadata),
|
|
47997
48525
|
nextCursor: rows.length > limit && page3.length > 0 ? encodeSnapshotCursor({ capturedAt: page3.at(-1).capturedAt, id: page3.at(-1).id }) : null,
|
|
@@ -48026,7 +48554,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
48026
48554
|
selectedCustomerCurrencyCode: null,
|
|
48027
48555
|
selectedCustomerTimeZone: null,
|
|
48028
48556
|
selectedCustomerStatus: null,
|
|
48029
|
-
selectionGeneration:
|
|
48557
|
+
selectionGeneration: sql16`${googleAdsConnections.selectionGeneration} + 1`,
|
|
48030
48558
|
scopes: [],
|
|
48031
48559
|
lastValidatedAt: null,
|
|
48032
48560
|
lastCustomerSnapshotId: null,
|
|
@@ -48065,7 +48593,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
48065
48593
|
selectedContainerPublicId: null,
|
|
48066
48594
|
selectedWorkspaceId: null,
|
|
48067
48595
|
selectedWorkspaceName: null,
|
|
48068
|
-
selectionGeneration:
|
|
48596
|
+
selectionGeneration: sql16`${gtmConnections.selectionGeneration} + 1`,
|
|
48069
48597
|
scopes: [],
|
|
48070
48598
|
lastValidatedAt: null,
|
|
48071
48599
|
lastSnapshotAt: null,
|
|
@@ -48260,7 +48788,7 @@ async function googleMarketingRoutes(app, opts) {
|
|
|
48260
48788
|
|
|
48261
48789
|
// ../api-routes/src/ads.ts
|
|
48262
48790
|
import crypto36 from "crypto";
|
|
48263
|
-
import { eq as eq47, and as and40, asc as asc9, desc as desc21, gt as gt3, gte as gte12, lt as lt7, lte as lte11, ne as ne7, inArray as inArray20, or as
|
|
48791
|
+
import { eq as eq47, and as and40, asc as asc9, desc as desc21, gt as gt3, gte as gte12, lt as lt7, lte as lte11, ne as ne7, inArray as inArray20, or as or11, isNull as isNull8, isNotNull as isNotNull4, sql as sql18 } from "drizzle-orm";
|
|
48264
48792
|
|
|
48265
48793
|
// ../api-routes/src/ads-activation.ts
|
|
48266
48794
|
import crypto34 from "crypto";
|
|
@@ -49646,7 +50174,7 @@ async function executeApprovedAdsActivation(dependencies, request) {
|
|
|
49646
50174
|
|
|
49647
50175
|
// ../api-routes/src/ads-activation-routes.ts
|
|
49648
50176
|
import crypto35 from "crypto";
|
|
49649
|
-
import { and as and39, asc as asc8, eq as eq46, gt as gt2, isNotNull as isNotNull3, isNull as isNull7, lte as lte10, or as
|
|
50177
|
+
import { and as and39, asc as asc8, eq as eq46, gt as gt2, isNotNull as isNotNull3, isNull as isNull7, lte as lte10, or as or10, sql as sql17 } from "drizzle-orm";
|
|
49650
50178
|
var MAX_ACTIVATION_GRANT_LIFETIME_MS = 24 * 60 * 60 * 1e3;
|
|
49651
50179
|
var ACTIVATION_LEASE_MS = 5 * 6e4;
|
|
49652
50180
|
var ACTIVATION_REVOCATION_SETTLEMENT_MS = ACTIVATION_LEASE_MS;
|
|
@@ -49772,7 +50300,7 @@ function createActivationStore(app, request) {
|
|
|
49772
50300
|
}).where(and39(
|
|
49773
50301
|
eq46(adsOperations.id, existing.id),
|
|
49774
50302
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
49775
|
-
|
|
50303
|
+
or10(
|
|
49776
50304
|
isNull7(adsOperations.leaseOwner),
|
|
49777
50305
|
eq46(adsOperations.leaseOwner, input.leaseOwner),
|
|
49778
50306
|
isNull7(adsOperations.leaseExpiresAt),
|
|
@@ -50190,35 +50718,35 @@ async function executeActivationGrant(app, opts, request, project, executorApiKe
|
|
|
50190
50718
|
return activationResponse(opts, result.grant, result.operation);
|
|
50191
50719
|
}
|
|
50192
50720
|
function selectWatchdogOperations(app, nowIso, batchSize, allowLiveRevocationLease) {
|
|
50193
|
-
const leaseAvailable =
|
|
50721
|
+
const leaseAvailable = or10(
|
|
50194
50722
|
isNull7(adsOperations.leaseOwner),
|
|
50195
50723
|
isNull7(adsOperations.leaseExpiresAt),
|
|
50196
50724
|
lte10(adsOperations.leaseExpiresAt, nowIso)
|
|
50197
50725
|
);
|
|
50198
|
-
const campaignHasMutationEvidence =
|
|
50726
|
+
const campaignHasMutationEvidence = or10(
|
|
50199
50727
|
eq46(adsOperationSteps.state, AdsOperationStepStates.executing),
|
|
50200
50728
|
eq46(adsOperationSteps.state, AdsOperationStepStates.active),
|
|
50201
50729
|
eq46(adsOperationSteps.state, AdsOperationStepStates.rollback_executing),
|
|
50202
50730
|
eq46(adsOperationSteps.state, AdsOperationStepStates.rollback_failed),
|
|
50203
50731
|
eq46(adsOperationSteps.state, AdsOperationStepStates.unknown)
|
|
50204
50732
|
);
|
|
50205
|
-
const eligibleState = allowLiveRevocationLease ?
|
|
50733
|
+
const eligibleState = allowLiveRevocationLease ? or10(
|
|
50206
50734
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50207
50735
|
and39(
|
|
50208
50736
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
50209
|
-
|
|
50737
|
+
or10(
|
|
50210
50738
|
isNotNull3(adsActivationGrants.revocationRequestedAt),
|
|
50211
50739
|
and39(leaseAvailable, campaignHasMutationEvidence)
|
|
50212
50740
|
)
|
|
50213
50741
|
)
|
|
50214
50742
|
) : and39(
|
|
50215
|
-
|
|
50743
|
+
or10(
|
|
50216
50744
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50217
50745
|
eq46(adsOperations.state, AdsOperationStates.pending)
|
|
50218
50746
|
),
|
|
50219
50747
|
leaseAvailable
|
|
50220
50748
|
);
|
|
50221
|
-
const freshRevocationPriority =
|
|
50749
|
+
const freshRevocationPriority = sql17`CASE
|
|
50222
50750
|
WHEN ${adsActivationGrants.revocationRequestedAt} IS NOT NULL
|
|
50223
50751
|
AND (
|
|
50224
50752
|
${adsOperations.lastReconciledAt} IS NULL
|
|
@@ -50356,7 +50884,7 @@ async function containUrgentAdsActivationCampaignsOnce(app, opts) {
|
|
|
50356
50884
|
lastReconciledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
50357
50885
|
}).where(and39(
|
|
50358
50886
|
eq46(adsOperations.id, operation.id),
|
|
50359
|
-
|
|
50887
|
+
or10(
|
|
50360
50888
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50361
50889
|
eq46(adsOperations.state, AdsOperationStates.pending)
|
|
50362
50890
|
)
|
|
@@ -50382,11 +50910,11 @@ async function sweepStaleAdsActivationsOnce(app, opts) {
|
|
|
50382
50910
|
updatedAt: nowIso
|
|
50383
50911
|
}).where(and39(
|
|
50384
50912
|
eq46(adsOperations.id, operation.id),
|
|
50385
|
-
|
|
50913
|
+
or10(
|
|
50386
50914
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50387
50915
|
and39(
|
|
50388
50916
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
50389
|
-
|
|
50917
|
+
or10(
|
|
50390
50918
|
isNull7(adsOperations.leaseOwner),
|
|
50391
50919
|
isNull7(adsOperations.leaseExpiresAt),
|
|
50392
50920
|
lte10(adsOperations.leaseExpiresAt, nowIso)
|
|
@@ -50419,11 +50947,11 @@ async function sweepStaleAdsActivationsOnce(app, opts) {
|
|
|
50419
50947
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
50420
50948
|
app.db.update(adsOperations).set({ updatedAt: failedAt }).where(and39(
|
|
50421
50949
|
eq46(adsOperations.id, operation.id),
|
|
50422
|
-
|
|
50950
|
+
or10(
|
|
50423
50951
|
eq46(adsOperations.state, AdsOperationStates.unknown),
|
|
50424
50952
|
and39(
|
|
50425
50953
|
eq46(adsOperations.state, AdsOperationStates.pending),
|
|
50426
|
-
|
|
50954
|
+
or10(
|
|
50427
50955
|
isNull7(adsOperations.leaseOwner),
|
|
50428
50956
|
isNull7(adsOperations.leaseExpiresAt),
|
|
50429
50957
|
lte10(adsOperations.leaseExpiresAt, failedAt)
|
|
@@ -51609,13 +52137,13 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
51609
52137
|
enforceBackoff ? lte11(adsOperations.updatedAt, unknownCutoff) : void 0
|
|
51610
52138
|
) : and40(
|
|
51611
52139
|
eq47(adsOperations.state, AdsOperationStates.reconciling),
|
|
51612
|
-
|
|
52140
|
+
or11(isNull8(adsOperations.leaseExpiresAt), lte11(adsOperations.leaseExpiresAt, nowIso))
|
|
51613
52141
|
);
|
|
51614
52142
|
const claimed = app.db.update(adsOperations).set({
|
|
51615
52143
|
state: AdsOperationStates.reconciling,
|
|
51616
52144
|
leaseOwner,
|
|
51617
52145
|
leaseExpiresAt: leaseExpiresAt2,
|
|
51618
|
-
reconcileAttempts:
|
|
52146
|
+
reconcileAttempts: sql18`${adsOperations.reconcileAttempts} + 1`,
|
|
51619
52147
|
updatedAt: nowIso
|
|
51620
52148
|
}).where(and40(
|
|
51621
52149
|
eq47(adsOperations.id, row.id),
|
|
@@ -51931,28 +52459,28 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
51931
52459
|
);
|
|
51932
52460
|
}
|
|
51933
52461
|
);
|
|
51934
|
-
const retryableStateWhere =
|
|
52462
|
+
const retryableStateWhere = or11(
|
|
51935
52463
|
and40(
|
|
51936
52464
|
eq47(adsOperations.state, AdsOperationStates.pending),
|
|
51937
52465
|
lte11(adsOperations.updatedAt, pendingCutoff)
|
|
51938
52466
|
),
|
|
51939
52467
|
and40(
|
|
51940
52468
|
eq47(adsOperations.state, AdsOperationStates.unknown),
|
|
51941
|
-
|
|
52469
|
+
or11(...unknownRetryWindows)
|
|
51942
52470
|
),
|
|
51943
52471
|
and40(
|
|
51944
52472
|
eq47(adsOperations.state, AdsOperationStates.reconciling),
|
|
51945
|
-
|
|
52473
|
+
or11(isNull8(adsOperations.leaseExpiresAt), lte11(adsOperations.leaseExpiresAt, nowIso))
|
|
51946
52474
|
)
|
|
51947
52475
|
);
|
|
51948
52476
|
const staleReconcileWhere = and40(
|
|
51949
52477
|
lt7(adsOperations.reconcileAttempts, config.policy.maxAttempts),
|
|
51950
52478
|
ne7(adsOperations.kind, AdsOperationKinds.campaign_tree_activate),
|
|
51951
|
-
|
|
52479
|
+
or11(
|
|
51952
52480
|
eq47(adsOperations.reconcileStrategy, AdsReconcileStrategies.known_entity),
|
|
51953
52481
|
and40(
|
|
51954
52482
|
eq47(adsOperations.reconcileStrategy, AdsReconcileStrategies.create_fingerprint),
|
|
51955
|
-
|
|
52483
|
+
or11(
|
|
51956
52484
|
isNotNull4(adsOperations.entityId),
|
|
51957
52485
|
inArray20(adsOperations.state, [
|
|
51958
52486
|
AdsOperationStates.pending,
|
|
@@ -51966,7 +52494,7 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
51966
52494
|
const exhaustedRows = app.db.select().from(adsOperations).where(and40(
|
|
51967
52495
|
ne7(adsOperations.kind, AdsOperationKinds.campaign_tree_activate),
|
|
51968
52496
|
gte12(adsOperations.reconcileAttempts, config.policy.maxAttempts),
|
|
51969
|
-
|
|
52497
|
+
or11(
|
|
51970
52498
|
and40(
|
|
51971
52499
|
eq47(adsOperations.state, AdsOperationStates.pending),
|
|
51972
52500
|
lte11(adsOperations.updatedAt, pendingCutoff)
|
|
@@ -51974,10 +52502,10 @@ async function sweepAdsOperationsOnce(app, opts, config) {
|
|
|
51974
52502
|
eq47(adsOperations.state, AdsOperationStates.unknown),
|
|
51975
52503
|
and40(
|
|
51976
52504
|
eq47(adsOperations.state, AdsOperationStates.reconciling),
|
|
51977
|
-
|
|
52505
|
+
or11(isNull8(adsOperations.leaseExpiresAt), lte11(adsOperations.leaseExpiresAt, nowIso))
|
|
51978
52506
|
)
|
|
51979
52507
|
),
|
|
51980
|
-
|
|
52508
|
+
or11(
|
|
51981
52509
|
isNull8(adsOperations.errorCode),
|
|
51982
52510
|
ne7(adsOperations.errorCode, ADS_RECONCILIATION_QUARANTINED)
|
|
51983
52511
|
)
|
|
@@ -52325,7 +52853,7 @@ async function adsRoutes(app, opts) {
|
|
|
52325
52853
|
const rows = app.db.select().from(adsOperations).where(and40(
|
|
52326
52854
|
eq47(adsOperations.projectId, project.id),
|
|
52327
52855
|
inArray20(adsOperations.state, parsed.data.state),
|
|
52328
|
-
cursor ?
|
|
52856
|
+
cursor ? or11(
|
|
52329
52857
|
gt3(adsOperations.createdAt, cursor.createdAt),
|
|
52330
52858
|
and40(
|
|
52331
52859
|
eq47(adsOperations.createdAt, cursor.createdAt),
|
|
@@ -54214,7 +54742,7 @@ async function cdpRoutes(app, opts) {
|
|
|
54214
54742
|
|
|
54215
54743
|
// ../api-routes/src/ga.ts
|
|
54216
54744
|
import crypto39 from "crypto";
|
|
54217
|
-
import { eq as eq50, desc as desc23, and as and43, sql as
|
|
54745
|
+
import { eq as eq50, desc as desc23, and as and43, sql as sql19 } from "drizzle-orm";
|
|
54218
54746
|
|
|
54219
54747
|
// ../api-routes/src/ga-session-history.ts
|
|
54220
54748
|
function buildSessionHistory(landingPageDays, dailyTotals) {
|
|
@@ -54247,8 +54775,8 @@ function formatSharePct(numerator, total) {
|
|
|
54247
54775
|
}
|
|
54248
54776
|
function dateRangeConditions(column, range) {
|
|
54249
54777
|
const conditions = [];
|
|
54250
|
-
if (range.startDate) conditions.push(
|
|
54251
|
-
if (range.endDate) conditions.push(
|
|
54778
|
+
if (range.startDate) conditions.push(sql19`${column} >= ${range.startDate}`);
|
|
54779
|
+
if (range.endDate) conditions.push(sql19`${column} <= ${range.endDate}`);
|
|
54252
54780
|
return conditions;
|
|
54253
54781
|
}
|
|
54254
54782
|
var SOCIAL_CHANNEL_GROUPS2 = /* @__PURE__ */ new Set(["Organic Social", "Paid Social"]);
|
|
@@ -54308,8 +54836,8 @@ function persistAcquisitionMeasurement(db, input) {
|
|
|
54308
54836
|
db.transaction((tx) => {
|
|
54309
54837
|
tx.delete(gaAcquisitionDaily).where(and43(
|
|
54310
54838
|
eq50(gaAcquisitionDaily.projectId, input.projectId),
|
|
54311
|
-
|
|
54312
|
-
|
|
54839
|
+
sql19`${gaAcquisitionDaily.date} >= ${startDate}`,
|
|
54840
|
+
sql19`${gaAcquisitionDaily.date} <= ${endDate}`
|
|
54313
54841
|
)).run();
|
|
54314
54842
|
for (const row of input.report.rows) {
|
|
54315
54843
|
tx.insert(gaAcquisitionDaily).values({
|
|
@@ -54350,8 +54878,8 @@ function persistLeadMeasurement(db, input) {
|
|
|
54350
54878
|
db.transaction((tx) => {
|
|
54351
54879
|
tx.delete(gaLeadEventsDaily).where(and43(
|
|
54352
54880
|
eq50(gaLeadEventsDaily.projectId, input.projectId),
|
|
54353
|
-
|
|
54354
|
-
|
|
54881
|
+
sql19`${gaLeadEventsDaily.date} >= ${startDate}`,
|
|
54882
|
+
sql19`${gaLeadEventsDaily.date} <= ${endDate}`
|
|
54355
54883
|
)).run();
|
|
54356
54884
|
for (const row of input.report.rows) {
|
|
54357
54885
|
tx.insert(gaLeadEventsDaily).values({
|
|
@@ -54726,8 +55254,8 @@ async function ga4Routes(app, opts) {
|
|
|
54726
55254
|
tx.delete(gaTrafficSnapshots).where(
|
|
54727
55255
|
and43(
|
|
54728
55256
|
eq50(gaTrafficSnapshots.projectId, project.id),
|
|
54729
|
-
|
|
54730
|
-
|
|
55257
|
+
sql19`${gaTrafficSnapshots.date} >= ${summary2.periodStart}`,
|
|
55258
|
+
sql19`${gaTrafficSnapshots.date} <= ${summary2.periodEnd}`
|
|
54731
55259
|
)
|
|
54732
55260
|
).run();
|
|
54733
55261
|
for (const row of rows) {
|
|
@@ -54749,8 +55277,8 @@ async function ga4Routes(app, opts) {
|
|
|
54749
55277
|
tx.delete(gaDailyTotals).where(
|
|
54750
55278
|
and43(
|
|
54751
55279
|
eq50(gaDailyTotals.projectId, project.id),
|
|
54752
|
-
|
|
54753
|
-
|
|
55280
|
+
sql19`${gaDailyTotals.date} >= ${summary2.periodStart}`,
|
|
55281
|
+
sql19`${gaDailyTotals.date} <= ${summary2.periodEnd}`
|
|
54754
55282
|
)
|
|
54755
55283
|
).run();
|
|
54756
55284
|
for (const row of dailyTotals) {
|
|
@@ -54775,8 +55303,8 @@ async function ga4Routes(app, opts) {
|
|
|
54775
55303
|
tx.delete(gaAiReferrals).where(
|
|
54776
55304
|
and43(
|
|
54777
55305
|
eq50(gaAiReferrals.projectId, project.id),
|
|
54778
|
-
|
|
54779
|
-
|
|
55306
|
+
sql19`${gaAiReferrals.date} >= ${summary2.periodStart}`,
|
|
55307
|
+
sql19`${gaAiReferrals.date} <= ${summary2.periodEnd}`
|
|
54780
55308
|
)
|
|
54781
55309
|
).run();
|
|
54782
55310
|
for (const row of aiReferrals) {
|
|
@@ -54807,8 +55335,8 @@ async function ga4Routes(app, opts) {
|
|
|
54807
55335
|
tx.delete(gaSocialReferrals).where(
|
|
54808
55336
|
and43(
|
|
54809
55337
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
54810
|
-
|
|
54811
|
-
|
|
55338
|
+
sql19`${gaSocialReferrals.date} >= ${summary2.periodStart}`,
|
|
55339
|
+
sql19`${gaSocialReferrals.date} <= ${summary2.periodEnd}`
|
|
54812
55340
|
)
|
|
54813
55341
|
).run();
|
|
54814
55342
|
for (const row of socialReferrals) {
|
|
@@ -54990,9 +55518,9 @@ async function ga4Routes(app, opts) {
|
|
|
54990
55518
|
totalUsers: gaTrafficSummaries.totalUsers
|
|
54991
55519
|
}).from(gaTrafficSummaries).where(eq50(gaTrafficSummaries.projectId, project.id)).get();
|
|
54992
55520
|
const retainedBounds = [
|
|
54993
|
-
app.db.select({ min:
|
|
54994
|
-
app.db.select({ min:
|
|
54995
|
-
app.db.select({ min:
|
|
55521
|
+
app.db.select({ min: sql19`MIN(${gaTrafficSnapshots.date})`, max: sql19`MAX(${gaTrafficSnapshots.date})` }).from(gaTrafficSnapshots).where(eq50(gaTrafficSnapshots.projectId, project.id)).get(),
|
|
55522
|
+
app.db.select({ min: sql19`MIN(${gaAiReferrals.date})`, max: sql19`MAX(${gaAiReferrals.date})` }).from(gaAiReferrals).where(eq50(gaAiReferrals.projectId, project.id)).get(),
|
|
55523
|
+
app.db.select({ min: sql19`MIN(${gaSocialReferrals.date})`, max: sql19`MAX(${gaSocialReferrals.date})` }).from(gaSocialReferrals).where(eq50(gaSocialReferrals.projectId, project.id)).get()
|
|
54996
55524
|
];
|
|
54997
55525
|
const retainedDates = retainedBounds.flatMap((bound) => [bound?.min, bound?.max]).filter((date) => Boolean(date));
|
|
54998
55526
|
const projectSummaryCoversAll = Boolean(
|
|
@@ -55006,9 +55534,9 @@ async function ga4Routes(app, opts) {
|
|
|
55006
55534
|
const socialConditions = [eq50(gaSocialReferrals.projectId, project.id), ...dateRangeConditions(gaSocialReferrals.date, measuredRange)];
|
|
55007
55535
|
const snapshotTotalsRow = !denominatorRow ? (() => {
|
|
55008
55536
|
const summed = app.db.select({
|
|
55009
|
-
totalSessions:
|
|
55010
|
-
totalOrganicSessions:
|
|
55011
|
-
totalUsers:
|
|
55537
|
+
totalSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)`,
|
|
55538
|
+
totalOrganicSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)`,
|
|
55539
|
+
totalUsers: sql19`COALESCE(SUM(${gaTrafficSnapshots.users}), 0)`
|
|
55012
55540
|
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).get();
|
|
55013
55541
|
return {
|
|
55014
55542
|
totalSessions: summed?.totalSessions ?? 0,
|
|
@@ -55021,35 +55549,35 @@ async function ga4Routes(app, opts) {
|
|
|
55021
55549
|
})() : null;
|
|
55022
55550
|
const summaryRow = denominatorRow ?? snapshotTotalsRow;
|
|
55023
55551
|
const directTotalRow = windowSummaryRow ? { totalDirectSessions: windowSummaryRow.totalDirectSessions } : app.db.select({
|
|
55024
|
-
totalDirectSessions:
|
|
55552
|
+
totalDirectSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`
|
|
55025
55553
|
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).get();
|
|
55026
55554
|
const rows = app.db.select({
|
|
55027
|
-
landingPage:
|
|
55028
|
-
sessions:
|
|
55029
|
-
organicSessions:
|
|
55030
|
-
directSessions:
|
|
55031
|
-
users:
|
|
55032
|
-
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).groupBy(
|
|
55555
|
+
landingPage: sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
55556
|
+
sessions: sql19`SUM(${gaTrafficSnapshots.sessions})`,
|
|
55557
|
+
organicSessions: sql19`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
55558
|
+
directSessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)`,
|
|
55559
|
+
users: sql19`SUM(${gaTrafficSnapshots.users})`
|
|
55560
|
+
}).from(gaTrafficSnapshots).where(and43(...snapshotConditions)).groupBy(sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql19`SUM(${gaTrafficSnapshots.sessions}) DESC`).limit(limit).all();
|
|
55033
55561
|
const aiReferralRows = app.db.select({
|
|
55034
55562
|
source: gaAiReferrals.source,
|
|
55035
55563
|
medium: gaAiReferrals.medium,
|
|
55036
55564
|
trafficClass: gaAiReferrals.trafficClass,
|
|
55037
55565
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
55038
|
-
sessions:
|
|
55566
|
+
sessions: sql19`SUM(${gaAiReferrals.sessions})`
|
|
55039
55567
|
}).from(gaAiReferrals).where(and43(...aiConditions)).groupBy(gaAiReferrals.source, gaAiReferrals.medium, gaAiReferrals.trafficClass, gaAiReferrals.sourceDimension).all();
|
|
55040
55568
|
const aiReferralLandingPageRows = app.db.select({
|
|
55041
55569
|
source: gaAiReferrals.source,
|
|
55042
55570
|
medium: gaAiReferrals.medium,
|
|
55043
55571
|
trafficClass: gaAiReferrals.trafficClass,
|
|
55044
55572
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
55045
|
-
landingPage:
|
|
55046
|
-
sessions:
|
|
55573
|
+
landingPage: sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
55574
|
+
sessions: sql19`SUM(${gaAiReferrals.sessions})`
|
|
55047
55575
|
}).from(gaAiReferrals).where(and43(...aiConditions)).groupBy(
|
|
55048
55576
|
gaAiReferrals.source,
|
|
55049
55577
|
gaAiReferrals.medium,
|
|
55050
55578
|
gaAiReferrals.trafficClass,
|
|
55051
55579
|
gaAiReferrals.sourceDimension,
|
|
55052
|
-
|
|
55580
|
+
sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
55053
55581
|
).all();
|
|
55054
55582
|
const aiRowsForTotals = app.db.select({
|
|
55055
55583
|
date: gaAiReferrals.date,
|
|
@@ -55075,10 +55603,10 @@ async function ga4Routes(app, opts) {
|
|
|
55075
55603
|
// is GA's COUNT DISTINCT at that grain, so summing over the window's
|
|
55076
55604
|
// dates counts a returning visitor once per day they returned. Same
|
|
55077
55605
|
// conclusion the AI-referral surfaces reached in 4.135.0.
|
|
55078
|
-
sessions:
|
|
55079
|
-
}).from(gaSocialReferrals).where(and43(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(
|
|
55606
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55607
|
+
}).from(gaSocialReferrals).where(and43(...socialConditions)).groupBy(gaSocialReferrals.source, gaSocialReferrals.medium, gaSocialReferrals.channelGroup).orderBy(sql19`SUM(${gaSocialReferrals.sessions}) DESC`).all();
|
|
55080
55608
|
const socialTotals = app.db.select({
|
|
55081
|
-
sessions:
|
|
55609
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55082
55610
|
}).from(gaSocialReferrals).where(and43(...socialConditions)).get();
|
|
55083
55611
|
const latestSync = app.db.select({ syncedAt: gaTrafficSummaries.syncedAt }).from(gaTrafficSummaries).where(eq50(gaTrafficSummaries.projectId, project.id)).orderBy(desc23(gaTrafficSummaries.syncedAt)).limit(1).get();
|
|
55084
55612
|
const total = summaryRow?.totalSessions ?? 0;
|
|
@@ -55193,20 +55721,20 @@ async function ga4Routes(app, opts) {
|
|
|
55193
55721
|
source: gaAiReferrals.source,
|
|
55194
55722
|
medium: gaAiReferrals.medium,
|
|
55195
55723
|
trafficClass: gaAiReferrals.trafficClass,
|
|
55196
|
-
landingPage:
|
|
55724
|
+
landingPage: sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`,
|
|
55197
55725
|
sourceDimension: gaAiReferrals.sourceDimension,
|
|
55198
55726
|
// No `users`: ga-ai-referral-aggregation states the rule for this
|
|
55199
55727
|
// table, that there is no grain at which the stored users column may be
|
|
55200
55728
|
// added up. ga4AiReferralDtoSchema.users was deprecated for this in
|
|
55201
55729
|
// 4.135.0; the history endpoint kept emitting one.
|
|
55202
|
-
sessions:
|
|
55730
|
+
sessions: sql19`SUM(${gaAiReferrals.sessions})`
|
|
55203
55731
|
}).from(gaAiReferrals).where(and43(...conditions)).groupBy(
|
|
55204
55732
|
gaAiReferrals.date,
|
|
55205
55733
|
gaAiReferrals.source,
|
|
55206
55734
|
gaAiReferrals.medium,
|
|
55207
55735
|
gaAiReferrals.trafficClass,
|
|
55208
55736
|
gaAiReferrals.sourceDimension,
|
|
55209
|
-
|
|
55737
|
+
sql19`COALESCE(${gaAiReferrals.landingPageNormalized}, ${gaAiReferrals.landingPage})`
|
|
55210
55738
|
).orderBy(gaAiReferrals.date).all();
|
|
55211
55739
|
return rows;
|
|
55212
55740
|
});
|
|
@@ -55251,10 +55779,10 @@ async function ga4Routes(app, opts) {
|
|
|
55251
55779
|
d.setDate(d.getDate() - n);
|
|
55252
55780
|
return fmt(d);
|
|
55253
55781
|
};
|
|
55254
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
55782
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and43(
|
|
55255
55783
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
55256
|
-
|
|
55257
|
-
|
|
55784
|
+
sql19`${gaSocialReferrals.date} >= ${from}`,
|
|
55785
|
+
sql19`${gaSocialReferrals.date} < ${to}`
|
|
55258
55786
|
)).get();
|
|
55259
55787
|
const current7d = sumSocial(daysAgo(7), fmt(today));
|
|
55260
55788
|
const prev7d = sumSocial(daysAgo(14), daysAgo(7));
|
|
@@ -55262,19 +55790,19 @@ async function ga4Routes(app, opts) {
|
|
|
55262
55790
|
const prev30d = sumSocial(daysAgo(60), daysAgo(30));
|
|
55263
55791
|
const sourceCurrent = app.db.select({
|
|
55264
55792
|
source: gaSocialReferrals.source,
|
|
55265
|
-
sessions:
|
|
55793
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55266
55794
|
}).from(gaSocialReferrals).where(and43(
|
|
55267
55795
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
55268
|
-
|
|
55269
|
-
|
|
55796
|
+
sql19`${gaSocialReferrals.date} >= ${daysAgo(7)}`,
|
|
55797
|
+
sql19`${gaSocialReferrals.date} < ${fmt(today)}`
|
|
55270
55798
|
)).groupBy(gaSocialReferrals.source).all();
|
|
55271
55799
|
const sourcePrev = app.db.select({
|
|
55272
55800
|
source: gaSocialReferrals.source,
|
|
55273
|
-
sessions:
|
|
55801
|
+
sessions: sql19`SUM(${gaSocialReferrals.sessions})`
|
|
55274
55802
|
}).from(gaSocialReferrals).where(and43(
|
|
55275
55803
|
eq50(gaSocialReferrals.projectId, project.id),
|
|
55276
|
-
|
|
55277
|
-
|
|
55804
|
+
sql19`${gaSocialReferrals.date} >= ${daysAgo(14)}`,
|
|
55805
|
+
sql19`${gaSocialReferrals.date} < ${daysAgo(7)}`
|
|
55278
55806
|
)).groupBy(gaSocialReferrals.source).all();
|
|
55279
55807
|
const prevMap = new Map(sourcePrev.map((r) => [r.source, r.sessions]));
|
|
55280
55808
|
let biggestMover = null;
|
|
@@ -55312,16 +55840,16 @@ async function ga4Routes(app, opts) {
|
|
|
55312
55840
|
d.setDate(d.getDate() - n);
|
|
55313
55841
|
return fmt(d);
|
|
55314
55842
|
};
|
|
55315
|
-
const sumTotal = (from, to) => app.db.select({ sessions:
|
|
55316
|
-
const sumOrganic = (from, to) => app.db.select({ sessions:
|
|
55317
|
-
const sumDirect = (from, to) => app.db.select({ sessions:
|
|
55318
|
-
const sumAi = (from, to) => app.db.select({ sessions:
|
|
55843
|
+
const sumTotal = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.sessions}), 0)` }).from(gaTrafficSnapshots).where(and43(eq50(gaTrafficSnapshots.projectId, project.id), sql19`${gaTrafficSnapshots.date} >= ${from}`, sql19`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
55844
|
+
const sumOrganic = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.organicSessions}), 0)` }).from(gaTrafficSnapshots).where(and43(eq50(gaTrafficSnapshots.projectId, project.id), sql19`${gaTrafficSnapshots.date} >= ${from}`, sql19`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
55845
|
+
const sumDirect = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaTrafficSnapshots.directSessions}), 0)` }).from(gaTrafficSnapshots).where(and43(eq50(gaTrafficSnapshots.projectId, project.id), sql19`${gaTrafficSnapshots.date} >= ${from}`, sql19`${gaTrafficSnapshots.date} < ${to}`)).get();
|
|
55846
|
+
const sumAi = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and43(
|
|
55319
55847
|
eq50(gaAiReferrals.projectId, project.id),
|
|
55320
|
-
|
|
55321
|
-
|
|
55848
|
+
sql19`${gaAiReferrals.date} >= ${from}`,
|
|
55849
|
+
sql19`${gaAiReferrals.date} < ${to}`,
|
|
55322
55850
|
eq50(gaAiReferrals.sourceDimension, "session")
|
|
55323
55851
|
)).get();
|
|
55324
|
-
const sumSocial = (from, to) => app.db.select({ sessions:
|
|
55852
|
+
const sumSocial = (from, to) => app.db.select({ sessions: sql19`COALESCE(SUM(${gaSocialReferrals.sessions}), 0)` }).from(gaSocialReferrals).where(and43(eq50(gaSocialReferrals.projectId, project.id), sql19`${gaSocialReferrals.date} >= ${from}`, sql19`${gaSocialReferrals.date} < ${to}`)).get();
|
|
55325
55853
|
const todayStr = fmt(today);
|
|
55326
55854
|
const buildTrend = (sum) => {
|
|
55327
55855
|
const c7 = sum(daysAgo(7), todayStr)?.sessions ?? 0;
|
|
@@ -55330,16 +55858,16 @@ async function ga4Routes(app, opts) {
|
|
|
55330
55858
|
const p30 = sum(daysAgo(60), daysAgo(30))?.sessions ?? 0;
|
|
55331
55859
|
return { sessions7d: c7, sessionsPrev7d: p7, trend7dPct: deltaPercent(c7, p7), sessions30d: c30, sessionsPrev30d: p30, trend30dPct: deltaPercent(c30, p30) };
|
|
55332
55860
|
};
|
|
55333
|
-
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
55861
|
+
const aiSourceCurrent = app.db.select({ source: gaAiReferrals.source, sessions: sql19`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and43(
|
|
55334
55862
|
eq50(gaAiReferrals.projectId, project.id),
|
|
55335
|
-
|
|
55336
|
-
|
|
55863
|
+
sql19`${gaAiReferrals.date} >= ${daysAgo(7)}`,
|
|
55864
|
+
sql19`${gaAiReferrals.date} < ${todayStr}`,
|
|
55337
55865
|
eq50(gaAiReferrals.sourceDimension, "session")
|
|
55338
55866
|
)).groupBy(gaAiReferrals.source).all();
|
|
55339
|
-
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions:
|
|
55867
|
+
const aiSourcePrev = app.db.select({ source: gaAiReferrals.source, sessions: sql19`COALESCE(SUM(${gaAiReferrals.sessions}), 0)` }).from(gaAiReferrals).where(and43(
|
|
55340
55868
|
eq50(gaAiReferrals.projectId, project.id),
|
|
55341
|
-
|
|
55342
|
-
|
|
55869
|
+
sql19`${gaAiReferrals.date} >= ${daysAgo(14)}`,
|
|
55870
|
+
sql19`${gaAiReferrals.date} < ${daysAgo(7)}`,
|
|
55343
55871
|
eq50(gaAiReferrals.sourceDimension, "session")
|
|
55344
55872
|
)).groupBy(gaAiReferrals.source).all();
|
|
55345
55873
|
const findBiggestMover = (current, prev) => {
|
|
@@ -55356,8 +55884,8 @@ async function ga4Routes(app, opts) {
|
|
|
55356
55884
|
}
|
|
55357
55885
|
return mover;
|
|
55358
55886
|
};
|
|
55359
|
-
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
55360
|
-
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions:
|
|
55887
|
+
const socialSourceCurrent = app.db.select({ source: gaSocialReferrals.source, sessions: sql19`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and43(eq50(gaSocialReferrals.projectId, project.id), sql19`${gaSocialReferrals.date} >= ${daysAgo(7)}`, sql19`${gaSocialReferrals.date} < ${todayStr}`)).groupBy(gaSocialReferrals.source).all();
|
|
55888
|
+
const socialSourcePrev = app.db.select({ source: gaSocialReferrals.source, sessions: sql19`SUM(${gaSocialReferrals.sessions})` }).from(gaSocialReferrals).where(and43(eq50(gaSocialReferrals.projectId, project.id), sql19`${gaSocialReferrals.date} >= ${daysAgo(14)}`, sql19`${gaSocialReferrals.date} < ${daysAgo(7)}`)).groupBy(gaSocialReferrals.source).all();
|
|
55361
55889
|
return {
|
|
55362
55890
|
total: buildTrend(sumTotal),
|
|
55363
55891
|
organic: buildTrend(sumOrganic),
|
|
@@ -55375,9 +55903,9 @@ async function ga4Routes(app, opts) {
|
|
|
55375
55903
|
const conditions = [eq50(gaTrafficSnapshots.projectId, project.id), ...dateRangeConditions(gaTrafficSnapshots.date, range)];
|
|
55376
55904
|
const rows = app.db.select({
|
|
55377
55905
|
date: gaTrafficSnapshots.date,
|
|
55378
|
-
sessions:
|
|
55379
|
-
organicSessions:
|
|
55380
|
-
users:
|
|
55906
|
+
sessions: sql19`SUM(${gaTrafficSnapshots.sessions})`,
|
|
55907
|
+
organicSessions: sql19`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
55908
|
+
users: sql19`SUM(${gaTrafficSnapshots.users})`
|
|
55381
55909
|
}).from(gaTrafficSnapshots).where(and43(...conditions)).groupBy(gaTrafficSnapshots.date).orderBy(gaTrafficSnapshots.date).all();
|
|
55382
55910
|
const totalConditions = [eq50(gaDailyTotals.projectId, project.id), ...dateRangeConditions(gaDailyTotals.date, range)];
|
|
55383
55911
|
const totals = app.db.select({ date: gaDailyTotals.date, users: gaDailyTotals.users }).from(gaDailyTotals).where(and43(...totalConditions)).all();
|
|
@@ -55395,11 +55923,11 @@ async function ga4Routes(app, opts) {
|
|
|
55395
55923
|
const project = resolveProject(app.db, request.params.name);
|
|
55396
55924
|
requireGa4Connection(opts, project.name, project.canonicalDomain);
|
|
55397
55925
|
const trafficPages = app.db.select({
|
|
55398
|
-
landingPage:
|
|
55399
|
-
sessions:
|
|
55400
|
-
organicSessions:
|
|
55401
|
-
users:
|
|
55402
|
-
}).from(gaTrafficSnapshots).where(eq50(gaTrafficSnapshots.projectId, project.id)).groupBy(
|
|
55926
|
+
landingPage: sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`,
|
|
55927
|
+
sessions: sql19`SUM(${gaTrafficSnapshots.sessions})`,
|
|
55928
|
+
organicSessions: sql19`SUM(${gaTrafficSnapshots.organicSessions})`,
|
|
55929
|
+
users: sql19`SUM(${gaTrafficSnapshots.users})`
|
|
55930
|
+
}).from(gaTrafficSnapshots).where(eq50(gaTrafficSnapshots.projectId, project.id)).groupBy(sql19`COALESCE(${gaTrafficSnapshots.landingPageNormalized}, ${gaTrafficSnapshots.landingPage})`).orderBy(sql19`SUM(${gaTrafficSnapshots.sessions}) DESC`).all();
|
|
55403
55931
|
return {
|
|
55404
55932
|
pages: trafficPages.map((r) => ({
|
|
55405
55933
|
landingPage: r.landingPage,
|
|
@@ -57054,7 +57582,7 @@ async function wordpressRoutes(app, opts) {
|
|
|
57054
57582
|
|
|
57055
57583
|
// ../api-routes/src/backlinks.ts
|
|
57056
57584
|
import crypto41 from "crypto";
|
|
57057
|
-
import { and as and45, asc as asc10, desc as desc24, eq as eq51, sql as
|
|
57585
|
+
import { and as and45, asc as asc10, desc as desc24, eq as eq51, sql as sql20 } from "drizzle-orm";
|
|
57058
57586
|
|
|
57059
57587
|
// ../integration-commoncrawl/src/constants.ts
|
|
57060
57588
|
import os2 from "os";
|
|
@@ -57151,7 +57679,7 @@ function parseContentLength(value) {
|
|
|
57151
57679
|
}
|
|
57152
57680
|
|
|
57153
57681
|
// ../integration-commoncrawl/src/downloader.ts
|
|
57154
|
-
import { createHash as
|
|
57682
|
+
import { createHash as createHash6 } from "crypto";
|
|
57155
57683
|
import { createWriteStream } from "fs";
|
|
57156
57684
|
import fs2 from "fs/promises";
|
|
57157
57685
|
import path3 from "path";
|
|
@@ -57177,7 +57705,7 @@ async function downloadFile(opts) {
|
|
|
57177
57705
|
throw new Error(`HTTP ${res.status} ${res.statusText} for ${opts.url}`);
|
|
57178
57706
|
}
|
|
57179
57707
|
const total = parseContentLength2(res.headers.get("content-length"));
|
|
57180
|
-
const hasher =
|
|
57708
|
+
const hasher = createHash6("sha256");
|
|
57181
57709
|
let bytes = 0;
|
|
57182
57710
|
const hashAndCount = new Transform({
|
|
57183
57711
|
transform(chunk, _enc, cb) {
|
|
@@ -57198,7 +57726,7 @@ async function downloadFile(opts) {
|
|
|
57198
57726
|
return { bytes, sha256: sha2564, cached: false, elapsedMs: Date.now() - start };
|
|
57199
57727
|
}
|
|
57200
57728
|
async function hashFile(filePath) {
|
|
57201
|
-
const hasher =
|
|
57729
|
+
const hasher = createHash6("sha256");
|
|
57202
57730
|
const handle = await fs2.open(filePath, "r");
|
|
57203
57731
|
try {
|
|
57204
57732
|
const stream = handle.createReadStream();
|
|
@@ -57344,7 +57872,7 @@ async function queryBacklinks(opts) {
|
|
|
57344
57872
|
const reversed = opts.targets.map(reverseDomain);
|
|
57345
57873
|
const targetList = reversed.map(quote).join(", ");
|
|
57346
57874
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
57347
|
-
const
|
|
57875
|
+
const sql28 = `
|
|
57348
57876
|
WITH vertices AS (
|
|
57349
57877
|
SELECT * FROM read_csv(
|
|
57350
57878
|
${quote(opts.vertexPath)},
|
|
@@ -57380,7 +57908,7 @@ async function queryBacklinks(opts) {
|
|
|
57380
57908
|
const conn = await instance.connect();
|
|
57381
57909
|
let rows;
|
|
57382
57910
|
try {
|
|
57383
|
-
const reader = await conn.runAndReadAll(
|
|
57911
|
+
const reader = await conn.runAndReadAll(sql28);
|
|
57384
57912
|
rows = reader.getRowObjects();
|
|
57385
57913
|
} finally {
|
|
57386
57914
|
conn.disconnectSync?.();
|
|
@@ -57570,12 +58098,12 @@ function computeFilteredSummary(db, base) {
|
|
|
57570
58098
|
);
|
|
57571
58099
|
const filteredCondition = and45(baseDomainCondition, backlinkCrawlerExclusionClause());
|
|
57572
58100
|
const unfilteredAgg = db.select({
|
|
57573
|
-
count:
|
|
57574
|
-
total:
|
|
58101
|
+
count: sql20`count(*)`,
|
|
58102
|
+
total: sql20`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
57575
58103
|
}).from(backlinkDomains).where(baseDomainCondition).get();
|
|
57576
58104
|
const filteredAgg = db.select({
|
|
57577
|
-
count:
|
|
57578
|
-
total:
|
|
58105
|
+
count: sql20`count(*)`,
|
|
58106
|
+
total: sql20`coalesce(sum(${backlinkDomains.numHosts}), 0)`
|
|
57579
58107
|
}).from(backlinkDomains).where(filteredCondition).get();
|
|
57580
58108
|
const top10Rows = db.select({ numHosts: backlinkDomains.numHosts }).from(backlinkDomains).where(filteredCondition).orderBy(desc24(backlinkDomains.numHosts)).limit(10).all();
|
|
57581
58109
|
const totalLinkingDomains = Number(filteredAgg?.count ?? 0);
|
|
@@ -57601,7 +58129,7 @@ function buildSourceAvailability(db, projectId, source, connected, excludeCrawle
|
|
|
57601
58129
|
const summary2 = db.select().from(backlinkSummaries).where(and45(eq51(backlinkSummaries.projectId, projectId), eq51(backlinkSummaries.source, source))).orderBy(desc24(backlinkSummaries.queriedAt)).limit(1).get();
|
|
57602
58130
|
let totalLinkingDomains = summary2?.totalLinkingDomains ?? 0;
|
|
57603
58131
|
if (summary2 && excludeCrawlers) {
|
|
57604
|
-
const filtered = db.select({ count:
|
|
58132
|
+
const filtered = db.select({ count: sql20`count(*)` }).from(backlinkDomains).where(and45(
|
|
57605
58133
|
eq51(backlinkDomains.projectId, projectId),
|
|
57606
58134
|
eq51(backlinkDomains.source, source),
|
|
57607
58135
|
eq51(backlinkDomains.release, summary2.release),
|
|
@@ -57789,7 +58317,7 @@ async function backlinksRoutes(app, opts) {
|
|
|
57789
58317
|
eq51(backlinkDomains.release, targetRelease)
|
|
57790
58318
|
);
|
|
57791
58319
|
const domainCondition = excludeCrawlers ? and45(baseDomainCondition, backlinkCrawlerExclusionClause()) : baseDomainCondition;
|
|
57792
|
-
const totalRow = app.db.select({ count:
|
|
58320
|
+
const totalRow = app.db.select({ count: sql20`count(*)` }).from(backlinkDomains).where(domainCondition).get();
|
|
57793
58321
|
const rows = app.db.select({
|
|
57794
58322
|
linkingDomain: backlinkDomains.linkingDomain,
|
|
57795
58323
|
numHosts: backlinkDomains.numHosts,
|
|
@@ -57840,7 +58368,7 @@ import crypto44 from "crypto";
|
|
|
57840
58368
|
import { isIP } from "net";
|
|
57841
58369
|
import { isDeepStrictEqual } from "util";
|
|
57842
58370
|
import { Agent as UndiciAgent } from "undici";
|
|
57843
|
-
import { and as and49, desc as desc25, eq as eq55, gte as gte14, inArray as inArray21, lte as lte14, sql as
|
|
58371
|
+
import { and as and49, desc as desc25, eq as eq55, gte as gte14, inArray as inArray21, lte as lte14, sql as sql23 } from "drizzle-orm";
|
|
57844
58372
|
|
|
57845
58373
|
// ../api-routes/src/cloudflare-worker-version.ts
|
|
57846
58374
|
var CURRENT_CLOUDFLARE_WORKER_VERSION = "1.0.1";
|
|
@@ -71858,8 +72386,8 @@ var perplexitybot_default = {
|
|
|
71858
72386
|
};
|
|
71859
72387
|
|
|
71860
72388
|
// ../integration-traffic/src/ip-range-manifest.ts
|
|
71861
|
-
import { createHash as
|
|
71862
|
-
function
|
|
72389
|
+
import { createHash as createHash7 } from "crypto";
|
|
72390
|
+
function isRecord2(value) {
|
|
71863
72391
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
71864
72392
|
}
|
|
71865
72393
|
function stableJson2(value, seen2 = /* @__PURE__ */ new Set()) {
|
|
@@ -71928,7 +72456,7 @@ function parseCidr(cidr) {
|
|
|
71928
72456
|
};
|
|
71929
72457
|
}
|
|
71930
72458
|
function canonicalPrefixKey(entry) {
|
|
71931
|
-
if (
|
|
72459
|
+
if (isRecord2(entry)) {
|
|
71932
72460
|
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
71933
72461
|
const hasIpv6 = Object.hasOwn(entry, "ipv6Prefix");
|
|
71934
72462
|
if (hasIpv4 !== hasIpv6) {
|
|
@@ -71948,12 +72476,12 @@ function canonicalPrefixKey(entry) {
|
|
|
71948
72476
|
return `invalid:${stableJson2(entry)}`;
|
|
71949
72477
|
}
|
|
71950
72478
|
function ipRangeManifestContentHash(value) {
|
|
71951
|
-
const prefixes =
|
|
72479
|
+
const prefixes = isRecord2(value) ? value.prefixes : void 0;
|
|
71952
72480
|
const canonical = Array.isArray(prefixes) ? prefixes.map(canonicalPrefixKey).sort() : [`invalid-prefixes:${stableJson2(prefixes)}`];
|
|
71953
|
-
return
|
|
72481
|
+
return createHash7("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
71954
72482
|
}
|
|
71955
72483
|
function validateIpRangeManifestPayload(value) {
|
|
71956
|
-
if (!
|
|
72484
|
+
if (!isRecord2(value)) return { ok: false, error: "response must be a JSON object" };
|
|
71957
72485
|
const creationTime = value.creationTime;
|
|
71958
72486
|
if (typeof creationTime !== "string" || creationTime.trim().length === 0) {
|
|
71959
72487
|
return { ok: false, error: "response missing `creationTime` version" };
|
|
@@ -71963,7 +72491,7 @@ function validateIpRangeManifestPayload(value) {
|
|
|
71963
72491
|
}
|
|
71964
72492
|
const prefixes = [];
|
|
71965
72493
|
for (const [index2, entry] of value.prefixes.entries()) {
|
|
71966
|
-
if (!
|
|
72494
|
+
if (!isRecord2(entry)) {
|
|
71967
72495
|
return { ok: false, error: `prefixes[${index2}] must be an object` };
|
|
71968
72496
|
}
|
|
71969
72497
|
const hasIpv4 = Object.hasOwn(entry, "ipv4Prefix");
|
|
@@ -72951,7 +73479,7 @@ function incrementBucket(map, key, fields) {
|
|
|
72951
73479
|
}
|
|
72952
73480
|
|
|
72953
73481
|
// ../integration-wordpress-traffic/src/client.ts
|
|
72954
|
-
import { randomUUID as
|
|
73482
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
72955
73483
|
|
|
72956
73484
|
// ../integration-wordpress-traffic/src/normalize.ts
|
|
72957
73485
|
function trimOrNull(value) {
|
|
@@ -73049,7 +73577,7 @@ async function readErrorBody2(response) {
|
|
|
73049
73577
|
if (!text2) return void 0;
|
|
73050
73578
|
return text2.length <= 500 ? text2 : `${text2.slice(0, 500)}... [truncated]`;
|
|
73051
73579
|
}
|
|
73052
|
-
function
|
|
73580
|
+
function isRecord3(value) {
|
|
73053
73581
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
73054
73582
|
}
|
|
73055
73583
|
function parseRequestedBoundary(name, value) {
|
|
@@ -73069,7 +73597,7 @@ function parseRequestedWindow(options) {
|
|
|
73069
73597
|
return { since, until };
|
|
73070
73598
|
}
|
|
73071
73599
|
function parseSiteMetadata(value) {
|
|
73072
|
-
if (!
|
|
73600
|
+
if (!isRecord3(value)) return void 0;
|
|
73073
73601
|
return {
|
|
73074
73602
|
url: typeof value.url === "string" ? value.url : void 0,
|
|
73075
73603
|
anonymous_id: typeof value.anonymous_id === "string" ? value.anonymous_id : void 0,
|
|
@@ -73084,7 +73612,7 @@ function invalidResponse(message) {
|
|
|
73084
73612
|
);
|
|
73085
73613
|
}
|
|
73086
73614
|
function parseResponseBody(value) {
|
|
73087
|
-
if (!
|
|
73615
|
+
if (!isRecord3(value)) {
|
|
73088
73616
|
throw invalidResponse("expected an object body");
|
|
73089
73617
|
}
|
|
73090
73618
|
if (!Array.isArray(value.events)) {
|
|
@@ -73117,7 +73645,7 @@ async function readResponseJson(response) {
|
|
|
73117
73645
|
}
|
|
73118
73646
|
function assertEntryIsWithinRequestedWindow(entry, window) {
|
|
73119
73647
|
if (!window.since && !window.until) return;
|
|
73120
|
-
const observedAt =
|
|
73648
|
+
const observedAt = isRecord3(entry) && typeof entry.observed_at === "string" ? entry.observed_at : void 0;
|
|
73121
73649
|
const observedAtMs = observedAt === void 0 ? Number.NaN : Date.parse(observedAt);
|
|
73122
73650
|
const outsideWindow = !Number.isFinite(observedAtMs) || window.since !== void 0 && observedAtMs < window.since.timestampMs || window.until !== void 0 && observedAtMs >= window.until.timestampMs;
|
|
73123
73651
|
if (!outsideWindow) return;
|
|
@@ -73153,7 +73681,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
73153
73681
|
if (options.until !== void 0 && options.until !== "") {
|
|
73154
73682
|
url.searchParams.set("until", options.until);
|
|
73155
73683
|
}
|
|
73156
|
-
url.searchParams.set("_cb",
|
|
73684
|
+
url.searchParams.set("_cb", randomUUID3());
|
|
73157
73685
|
const fetchInit = {
|
|
73158
73686
|
method: "GET",
|
|
73159
73687
|
headers: {
|
|
@@ -73180,7 +73708,7 @@ async function listWordpressTrafficEvents(options) {
|
|
|
73180
73708
|
rawEntryCount += entries.length;
|
|
73181
73709
|
for (const entry of entries) {
|
|
73182
73710
|
assertEntryIsWithinRequestedWindow(entry, requestedWindow);
|
|
73183
|
-
if (!
|
|
73711
|
+
if (!isRecord3(entry)) {
|
|
73184
73712
|
skippedEntryCount += 1;
|
|
73185
73713
|
continue;
|
|
73186
73714
|
}
|
|
@@ -74292,7 +74820,7 @@ function asRecord(value, label) {
|
|
|
74292
74820
|
}
|
|
74293
74821
|
return value;
|
|
74294
74822
|
}
|
|
74295
|
-
function
|
|
74823
|
+
function requiredString2(record, key, label) {
|
|
74296
74824
|
const value = record[key];
|
|
74297
74825
|
if (typeof value !== "string" || !value.trim()) {
|
|
74298
74826
|
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
@@ -74366,12 +74894,12 @@ function safeMessageBase(record, leaseId) {
|
|
|
74366
74894
|
};
|
|
74367
74895
|
}
|
|
74368
74896
|
function decodeMessage(record) {
|
|
74369
|
-
const leaseId =
|
|
74897
|
+
const leaseId = requiredString2(record, "lease_id", "message");
|
|
74370
74898
|
let base;
|
|
74371
74899
|
let rawBody;
|
|
74372
74900
|
try {
|
|
74373
74901
|
base = {
|
|
74374
|
-
id:
|
|
74902
|
+
id: requiredString2(record, "id", "message"),
|
|
74375
74903
|
leaseId,
|
|
74376
74904
|
timestampMs: nonNegativeInteger(record.timestamp_ms, "message timestamp_ms"),
|
|
74377
74905
|
attempts: nonNegativeInteger(record.attempts, "message attempts"),
|
|
@@ -74511,10 +75039,10 @@ async function ackCloudflareQueueMessages(client, options) {
|
|
|
74511
75039
|
|
|
74512
75040
|
// ../api-routes/src/traffic-event-ingest.ts
|
|
74513
75041
|
import crypto43 from "crypto";
|
|
74514
|
-
import { and as and47, eq as eq53, gte as gte13, lt as lt9, lte as lte12, sql as
|
|
75042
|
+
import { and as and47, eq as eq53, gte as gte13, lt as lt9, lte as lte12, sql as sql22 } from "drizzle-orm";
|
|
74515
75043
|
|
|
74516
75044
|
// ../api-routes/src/raw-event-sample-retention.ts
|
|
74517
|
-
import { and as and46, eq as eq52, lt as lt8, sql as
|
|
75045
|
+
import { and as and46, eq as eq52, lt as lt8, sql as sql21 } from "drizzle-orm";
|
|
74518
75046
|
var RAW_EVENT_SAMPLE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
74519
75047
|
var RAW_EVENT_SAMPLE_RETENTION_SWEEP_INTERVAL_MS = 24 * 60 * 6e4;
|
|
74520
75048
|
var CANONICAL_UTC_TIMESTAMP_GLOB = "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9]Z";
|
|
@@ -74525,16 +75053,16 @@ function rawEventSampleRetentionCutoff(referenceAt) {
|
|
|
74525
75053
|
}
|
|
74526
75054
|
function deleteExpiredRawEventSamples(db, cutoff, sourceId) {
|
|
74527
75055
|
const sourceScope = sourceId === void 0 ? void 0 : eq52(rawEventSamples.sourceId, sourceId);
|
|
74528
|
-
db.delete(rawEventSamples).where(and46(sourceScope,
|
|
75056
|
+
db.delete(rawEventSamples).where(and46(sourceScope, sql21`julianday(${rawEventSamples.ts}) IS NULL`)).run();
|
|
74529
75057
|
db.delete(rawEventSamples).where(and46(
|
|
74530
75058
|
sourceScope,
|
|
74531
|
-
|
|
75059
|
+
sql21`${rawEventSamples.ts} GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
74532
75060
|
lt8(rawEventSamples.ts, cutoff)
|
|
74533
75061
|
)).run();
|
|
74534
75062
|
db.delete(rawEventSamples).where(and46(
|
|
74535
75063
|
sourceScope,
|
|
74536
|
-
|
|
74537
|
-
|
|
75064
|
+
sql21`${rawEventSamples.ts} NOT GLOB ${CANONICAL_UTC_TIMESTAMP_GLOB}`,
|
|
75065
|
+
sql21`julianday(${rawEventSamples.ts}) < julianday(${cutoff})`
|
|
74538
75066
|
)).run();
|
|
74539
75067
|
}
|
|
74540
75068
|
function enforceRawEventSampleRetention(db, sourceId, referenceAt) {
|
|
@@ -74580,7 +75108,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74580
75108
|
let aiReferralHits = 0;
|
|
74581
75109
|
let unknownHits = 0;
|
|
74582
75110
|
opts.db.transaction((tx) => {
|
|
74583
|
-
tx.update(trafficSources).set({ updatedAt:
|
|
75111
|
+
tx.update(trafficSources).set({ updatedAt: sql22`${trafficSources.updatedAt}` }).where(eq53(trafficSources.id, opts.sourceId)).run();
|
|
74584
75112
|
const source = tx.select().from(trafficSources).where(eq53(trafficSources.id, opts.sourceId)).get();
|
|
74585
75113
|
opts.validateSource(source);
|
|
74586
75114
|
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
@@ -74637,7 +75165,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74637
75165
|
crawlerEventsHourly.status
|
|
74638
75166
|
],
|
|
74639
75167
|
set: {
|
|
74640
|
-
hits:
|
|
75168
|
+
hits: sql22`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
74641
75169
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
74642
75170
|
updatedAt: opts.receivedAt
|
|
74643
75171
|
}
|
|
@@ -74668,7 +75196,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74668
75196
|
crawlerVerificationManifestsHourly.manifestId
|
|
74669
75197
|
],
|
|
74670
75198
|
set: {
|
|
74671
|
-
hits:
|
|
75199
|
+
hits: sql22`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
74672
75200
|
manifestJson: bucket.verificationManifest,
|
|
74673
75201
|
updatedAt: opts.receivedAt
|
|
74674
75202
|
}
|
|
@@ -74702,7 +75230,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74702
75230
|
aiUserFetchEventsHourly.status
|
|
74703
75231
|
],
|
|
74704
75232
|
set: {
|
|
74705
|
-
hits:
|
|
75233
|
+
hits: sql22`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
74706
75234
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
74707
75235
|
updatedAt: opts.receivedAt
|
|
74708
75236
|
}
|
|
@@ -74733,7 +75261,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74733
75261
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
74734
75262
|
],
|
|
74735
75263
|
set: {
|
|
74736
|
-
hits:
|
|
75264
|
+
hits: sql22`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
74737
75265
|
manifestJson: bucket.verificationManifest,
|
|
74738
75266
|
updatedAt: opts.receivedAt
|
|
74739
75267
|
}
|
|
@@ -74771,9 +75299,9 @@ function writeTrafficEventBatch(opts) {
|
|
|
74771
75299
|
aiReferralEventsHourly.status
|
|
74772
75300
|
],
|
|
74773
75301
|
set: {
|
|
74774
|
-
sessionsOrHits:
|
|
74775
|
-
paidSessionsOrHits:
|
|
74776
|
-
organicSessionsOrHits:
|
|
75302
|
+
sessionsOrHits: sql22`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
75303
|
+
paidSessionsOrHits: sql22`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
75304
|
+
organicSessionsOrHits: sql22`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
74777
75305
|
updatedAt: opts.receivedAt
|
|
74778
75306
|
}
|
|
74779
75307
|
}).run();
|
|
@@ -74788,7 +75316,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74788
75316
|
if (!hour || hourlySampleLimit === 0) continue;
|
|
74789
75317
|
let remaining = remainingSamplesByHour.get(hour.start);
|
|
74790
75318
|
if (remaining === void 0) {
|
|
74791
|
-
const existing = tx.select({ total:
|
|
75319
|
+
const existing = tx.select({ total: sql22`COUNT(*)` }).from(rawEventSamples).where(and47(
|
|
74792
75320
|
eq53(rawEventSamples.sourceId, opts.sourceId),
|
|
74793
75321
|
gte13(rawEventSamples.ts, hour.start),
|
|
74794
75322
|
lt9(rawEventSamples.ts, hour.end)
|
|
@@ -74848,7 +75376,7 @@ function writeTrafficEventBatch(opts) {
|
|
|
74848
75376
|
}
|
|
74849
75377
|
|
|
74850
75378
|
// ../api-routes/src/traffic-sync-lease.ts
|
|
74851
|
-
import { and as and48, eq as eq54, isNull as isNull9, lte as lte13, or as
|
|
75379
|
+
import { and as and48, eq as eq54, isNull as isNull9, lte as lte13, or as or12 } from "drizzle-orm";
|
|
74852
75380
|
function leaseExpiresAt(now, ttlMs) {
|
|
74853
75381
|
const nowMs = Date.parse(now);
|
|
74854
75382
|
if (!Number.isFinite(nowMs)) throw new RangeError("Traffic sync lease now must be an ISO instant");
|
|
@@ -74863,7 +75391,7 @@ function tryClaimTrafficSyncLease(input) {
|
|
|
74863
75391
|
syncLeaseExpiresAt: expiresAt
|
|
74864
75392
|
}).where(and48(
|
|
74865
75393
|
eq54(trafficSources.id, input.sourceId),
|
|
74866
|
-
|
|
75394
|
+
or12(
|
|
74867
75395
|
isNull9(trafficSources.syncLeaseOwner),
|
|
74868
75396
|
lte13(trafficSources.syncLeaseExpiresAt, input.now),
|
|
74869
75397
|
eq54(trafficSources.syncLeaseOwner, input.owner)
|
|
@@ -75114,7 +75642,7 @@ function crawlerVerificationProvenance(db, projectId, events) {
|
|
|
75114
75642
|
manifestId: crawlerVerificationManifestsHourly.manifestId,
|
|
75115
75643
|
manifest: crawlerVerificationManifestsHourly.manifestJson,
|
|
75116
75644
|
hits: crawlerVerificationManifestsHourly.hits
|
|
75117
|
-
}).from(crawlerVerificationManifestsHourly).where(
|
|
75645
|
+
}).from(crawlerVerificationManifestsHourly).where(sql23`(
|
|
75118
75646
|
${crawlerVerificationManifestsHourly.projectId},
|
|
75119
75647
|
${crawlerVerificationManifestsHourly.sourceId},
|
|
75120
75648
|
${crawlerVerificationManifestsHourly.tsHour},
|
|
@@ -75148,7 +75676,7 @@ function aiUserFetchVerificationProvenance(db, projectId, events) {
|
|
|
75148
75676
|
manifestId: aiUserFetchVerificationManifestsHourly.manifestId,
|
|
75149
75677
|
manifest: aiUserFetchVerificationManifestsHourly.manifestJson,
|
|
75150
75678
|
hits: aiUserFetchVerificationManifestsHourly.hits
|
|
75151
|
-
}).from(aiUserFetchVerificationManifestsHourly).where(
|
|
75679
|
+
}).from(aiUserFetchVerificationManifestsHourly).where(sql23`(
|
|
75152
75680
|
${aiUserFetchVerificationManifestsHourly.projectId},
|
|
75153
75681
|
${aiUserFetchVerificationManifestsHourly.sourceId},
|
|
75154
75682
|
${aiUserFetchVerificationManifestsHourly.tsHour},
|
|
@@ -75412,7 +75940,7 @@ async function runBackfillTask(options) {
|
|
|
75412
75940
|
crawlerEventsHourly.status
|
|
75413
75941
|
],
|
|
75414
75942
|
set: {
|
|
75415
|
-
hits:
|
|
75943
|
+
hits: sql23`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
75416
75944
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
75417
75945
|
updatedAt: finishedAt
|
|
75418
75946
|
}
|
|
@@ -75443,7 +75971,7 @@ async function runBackfillTask(options) {
|
|
|
75443
75971
|
crawlerVerificationManifestsHourly.manifestId
|
|
75444
75972
|
],
|
|
75445
75973
|
set: {
|
|
75446
|
-
hits:
|
|
75974
|
+
hits: sql23`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
75447
75975
|
manifestJson: bucket.verificationManifest,
|
|
75448
75976
|
updatedAt: finishedAt
|
|
75449
75977
|
}
|
|
@@ -75476,7 +76004,7 @@ async function runBackfillTask(options) {
|
|
|
75476
76004
|
aiUserFetchEventsHourly.status
|
|
75477
76005
|
],
|
|
75478
76006
|
set: {
|
|
75479
|
-
hits:
|
|
76007
|
+
hits: sql23`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
75480
76008
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
75481
76009
|
updatedAt: finishedAt
|
|
75482
76010
|
}
|
|
@@ -75507,7 +76035,7 @@ async function runBackfillTask(options) {
|
|
|
75507
76035
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
75508
76036
|
],
|
|
75509
76037
|
set: {
|
|
75510
|
-
hits:
|
|
76038
|
+
hits: sql23`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
75511
76039
|
manifestJson: bucket.verificationManifest,
|
|
75512
76040
|
updatedAt: finishedAt
|
|
75513
76041
|
}
|
|
@@ -77111,7 +77639,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77111
77639
|
crawlerEventsHourly.status
|
|
77112
77640
|
],
|
|
77113
77641
|
set: {
|
|
77114
|
-
hits:
|
|
77642
|
+
hits: sql23`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
77115
77643
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77116
77644
|
updatedAt: finishedAt
|
|
77117
77645
|
}
|
|
@@ -77142,7 +77670,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77142
77670
|
crawlerVerificationManifestsHourly.manifestId
|
|
77143
77671
|
],
|
|
77144
77672
|
set: {
|
|
77145
|
-
hits:
|
|
77673
|
+
hits: sql23`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77146
77674
|
manifestJson: bucket.verificationManifest,
|
|
77147
77675
|
updatedAt: finishedAt
|
|
77148
77676
|
}
|
|
@@ -77176,7 +77704,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77176
77704
|
aiUserFetchEventsHourly.status
|
|
77177
77705
|
],
|
|
77178
77706
|
set: {
|
|
77179
|
-
hits:
|
|
77707
|
+
hits: sql23`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
77180
77708
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
77181
77709
|
updatedAt: finishedAt
|
|
77182
77710
|
}
|
|
@@ -77207,7 +77735,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77207
77735
|
aiUserFetchVerificationManifestsHourly.manifestId
|
|
77208
77736
|
],
|
|
77209
77737
|
set: {
|
|
77210
|
-
hits:
|
|
77738
|
+
hits: sql23`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
77211
77739
|
manifestJson: bucket.verificationManifest,
|
|
77212
77740
|
updatedAt: finishedAt
|
|
77213
77741
|
}
|
|
@@ -77248,9 +77776,9 @@ async function trafficRoutes(app, opts) {
|
|
|
77248
77776
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
77249
77777
|
// already there rather than overwriting a single label.
|
|
77250
77778
|
set: {
|
|
77251
|
-
sessionsOrHits:
|
|
77252
|
-
paidSessionsOrHits:
|
|
77253
|
-
organicSessionsOrHits:
|
|
77779
|
+
sessionsOrHits: sql23`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
77780
|
+
paidSessionsOrHits: sql23`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
77781
|
+
organicSessionsOrHits: sql23`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
77254
77782
|
updatedAt: finishedAt
|
|
77255
77783
|
}
|
|
77256
77784
|
}).run();
|
|
@@ -77543,7 +78071,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77543
78071
|
function buildSourceDetail(projectId, row, since) {
|
|
77544
78072
|
const crawlerPathRows = app.db.select({
|
|
77545
78073
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
77546
|
-
hits:
|
|
78074
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
77547
78075
|
}).from(crawlerEventsHourly).where(
|
|
77548
78076
|
and49(
|
|
77549
78077
|
eq55(crawlerEventsHourly.sourceId, row.id),
|
|
@@ -77554,23 +78082,23 @@ async function trafficRoutes(app, opts) {
|
|
|
77554
78082
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
77555
78083
|
);
|
|
77556
78084
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
77557
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
78085
|
+
const aiUserFetchTotals = app.db.select({ total: sql23`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
77558
78086
|
and49(
|
|
77559
78087
|
eq55(aiUserFetchEventsHourly.sourceId, row.id),
|
|
77560
78088
|
gte14(aiUserFetchEventsHourly.tsHour, since)
|
|
77561
78089
|
)
|
|
77562
78090
|
).get();
|
|
77563
78091
|
const aiTotals = app.db.select({
|
|
77564
|
-
total:
|
|
77565
|
-
nonhop:
|
|
77566
|
-
landed:
|
|
78092
|
+
total: sql23`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
78093
|
+
nonhop: sql23`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
78094
|
+
landed: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
77567
78095
|
}).from(aiReferralEventsHourly).where(
|
|
77568
78096
|
and49(
|
|
77569
78097
|
eq55(aiReferralEventsHourly.sourceId, row.id),
|
|
77570
78098
|
gte14(aiReferralEventsHourly.tsHour, since)
|
|
77571
78099
|
)
|
|
77572
78100
|
).get();
|
|
77573
|
-
const sampleTotals = app.db.select({ total:
|
|
78101
|
+
const sampleTotals = app.db.select({ total: sql23`COUNT(*)` }).from(rawEventSamples).where(
|
|
77574
78102
|
and49(
|
|
77575
78103
|
eq55(rawEventSamples.sourceId, row.id),
|
|
77576
78104
|
gte14(rawEventSamples.ts, since)
|
|
@@ -77759,15 +78287,15 @@ async function trafficRoutes(app, opts) {
|
|
|
77759
78287
|
const crawlerWhere = and49(...crawlerFilters);
|
|
77760
78288
|
const pathTotals = app.db.select({
|
|
77761
78289
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
77762
|
-
hits:
|
|
77763
|
-
rows:
|
|
78290
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
78291
|
+
rows: sql23`COUNT(*)`
|
|
77764
78292
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
77765
78293
|
crawlerSegments = segmentCrawlerHits(
|
|
77766
78294
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
77767
78295
|
);
|
|
77768
78296
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
77769
78297
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
77770
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
78298
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql23`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
77771
78299
|
const distinctPaths = pathTotals.map((r) => ({ pathNormalized: r.pathNormalized }));
|
|
77772
78300
|
const contentPaths = [];
|
|
77773
78301
|
const infraPaths = [];
|
|
@@ -77779,12 +78307,12 @@ async function trafficRoutes(app, opts) {
|
|
|
77779
78307
|
const listed = useContentList ? contentPaths : infraPaths;
|
|
77780
78308
|
const IN_LIST_CAP = 2e4;
|
|
77781
78309
|
const listFitsInSql = listed.length <= IN_LIST_CAP;
|
|
77782
|
-
const contentExpr = listed.length === 0 ? useContentList ?
|
|
78310
|
+
const contentExpr = listed.length === 0 ? useContentList ? sql23`0` : sql23`${crawlerEventsHourly.hits}` : useContentList ? sql23`CASE WHEN ${inArray21(crawlerEventsHourly.pathNormalized, listed)} THEN ${crawlerEventsHourly.hits} ELSE 0 END` : sql23`CASE WHEN ${inArray21(crawlerEventsHourly.pathNormalized, listed)} THEN 0 ELSE ${crawlerEventsHourly.hits} END`;
|
|
77783
78311
|
if (listFitsInSql) {
|
|
77784
78312
|
const crawlerSeries = app.db.select({
|
|
77785
78313
|
bucket: crawlerSeriesBucket,
|
|
77786
|
-
hits:
|
|
77787
|
-
content:
|
|
78314
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
78315
|
+
content: sql23`COALESCE(SUM(${contentExpr}), 0)`
|
|
77788
78316
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
77789
78317
|
for (const row of crawlerSeries) {
|
|
77790
78318
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -77796,7 +78324,7 @@ async function trafficRoutes(app, opts) {
|
|
|
77796
78324
|
const grouped = app.db.select({
|
|
77797
78325
|
bucket: crawlerSeriesBucket,
|
|
77798
78326
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
77799
|
-
hits:
|
|
78327
|
+
hits: sql23`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
77800
78328
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket, crawlerEventsHourly.pathNormalized).all();
|
|
77801
78329
|
for (const row of grouped) {
|
|
77802
78330
|
const point = trafficSeriesPoint(seriesByBucket, String(row.bucket));
|
|
@@ -77833,15 +78361,15 @@ async function trafficRoutes(app, opts) {
|
|
|
77833
78361
|
if (sourceIdParam) userFetchFilters.push(eq55(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
77834
78362
|
const userFetchWhere = and49(...userFetchFilters);
|
|
77835
78363
|
const total = app.db.select({
|
|
77836
|
-
total:
|
|
77837
|
-
rows:
|
|
78364
|
+
total: sql23`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
78365
|
+
rows: sql23`COUNT(*)`
|
|
77838
78366
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
77839
78367
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
77840
78368
|
totalEventRows += Number(total?.rows ?? 0);
|
|
77841
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
78369
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql23`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
77842
78370
|
const userFetchSeries = app.db.select({
|
|
77843
78371
|
bucket: userFetchSeriesBucket,
|
|
77844
|
-
hits:
|
|
78372
|
+
hits: sql23`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
77845
78373
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
77846
78374
|
for (const row of userFetchSeries) {
|
|
77847
78375
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -77872,12 +78400,12 @@ async function trafficRoutes(app, opts) {
|
|
|
77872
78400
|
if (sourceIdParam) aiFilters.push(eq55(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
77873
78401
|
const aiWhere = and49(...aiFilters);
|
|
77874
78402
|
const total = app.db.select({
|
|
77875
|
-
total:
|
|
77876
|
-
nonhop:
|
|
77877
|
-
landed:
|
|
77878
|
-
paid:
|
|
77879
|
-
organic:
|
|
77880
|
-
rows:
|
|
78403
|
+
total: sql23`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
78404
|
+
nonhop: sql23`COALESCE(SUM(CASE WHEN ${referralLandedCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
78405
|
+
landed: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`,
|
|
78406
|
+
paid: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.paidSessionsOrHits} ELSE 0 END), 0)`,
|
|
78407
|
+
organic: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.organicSessionsOrHits} ELSE 0 END), 0)`,
|
|
78408
|
+
rows: sql23`COUNT(*)`
|
|
77881
78409
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
77882
78410
|
aiReferralTotalHits = Number(total?.total ?? 0);
|
|
77883
78411
|
aiReferralNonHopHits = Number(total?.nonhop ?? 0);
|
|
@@ -77888,11 +78416,11 @@ async function trafficRoutes(app, opts) {
|
|
|
77888
78416
|
Number(total?.organic ?? 0)
|
|
77889
78417
|
);
|
|
77890
78418
|
totalEventRows += Number(total?.rows ?? 0);
|
|
77891
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
78419
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql23`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
77892
78420
|
const referralSeries = app.db.select({
|
|
77893
78421
|
bucket: referralSeriesBucket,
|
|
77894
|
-
hits:
|
|
77895
|
-
landed:
|
|
78422
|
+
hits: sql23`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
78423
|
+
landed: sql23`COALESCE(SUM(CASE WHEN ${countableReferralCondition()} THEN ${aiReferralEventsHourly.sessionsOrHits} ELSE 0 END), 0)`
|
|
77896
78424
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
77897
78425
|
for (const row of referralSeries) {
|
|
77898
78426
|
const point = trafficSeriesPoint(seriesByBucket, row.bucket);
|
|
@@ -77947,9 +78475,9 @@ async function trafficRoutes(app, opts) {
|
|
|
77947
78475
|
const points = completeTrafficSeries(since, until, granularity, seriesByBucket);
|
|
77948
78476
|
const coverageScope = (col, srcCol) => sourceIdParam ? and49(eq55(col, project.id), eq55(srcCol, sourceIdParam)) : eq55(col, project.id);
|
|
77949
78477
|
const firsts = [
|
|
77950
|
-
app.db.select({ v:
|
|
77951
|
-
app.db.select({ v:
|
|
77952
|
-
app.db.select({ v:
|
|
78478
|
+
app.db.select({ v: sql23`MIN(${crawlerEventsHourly.tsHour})` }).from(crawlerEventsHourly).where(coverageScope(crawlerEventsHourly.projectId, crawlerEventsHourly.sourceId)).get()?.v,
|
|
78479
|
+
app.db.select({ v: sql23`MIN(${aiUserFetchEventsHourly.tsHour})` }).from(aiUserFetchEventsHourly).where(coverageScope(aiUserFetchEventsHourly.projectId, aiUserFetchEventsHourly.sourceId)).get()?.v,
|
|
78480
|
+
app.db.select({ v: sql23`MIN(${aiReferralEventsHourly.tsHour})` }).from(aiReferralEventsHourly).where(coverageScope(aiReferralEventsHourly.projectId, aiReferralEventsHourly.sourceId)).get()?.v
|
|
77953
78481
|
].filter((v) => typeof v === "string" && v.length > 0);
|
|
77954
78482
|
const coverageStart = firsts.length ? firsts.slice().sort()[0] : null;
|
|
77955
78483
|
const bucketStartMs = (bucket) => Date.parse(
|
|
@@ -79755,7 +80283,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
79755
80283
|
];
|
|
79756
80284
|
|
|
79757
80285
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
79758
|
-
import { and as and52, eq as eq61, gte as gte15, inArray as inArray22, ne as ne9, sql as
|
|
80286
|
+
import { and as and52, eq as eq61, gte as gte15, inArray as inArray22, ne as ne9, sql as sql24 } from "drizzle-orm";
|
|
79759
80287
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
79760
80288
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
79761
80289
|
function isCloudflareDirectPush(source) {
|
|
@@ -79900,7 +80428,7 @@ var recentDataCheck = {
|
|
|
79900
80428
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
79901
80429
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
79902
80430
|
const recentCrawlers = Number(
|
|
79903
|
-
ctx.db.select({ total:
|
|
80431
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
79904
80432
|
and52(
|
|
79905
80433
|
eq61(crawlerEventsHourly.projectId, ctx.project.id),
|
|
79906
80434
|
inArray22(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79909,7 +80437,7 @@ var recentDataCheck = {
|
|
|
79909
80437
|
).get()?.total ?? 0
|
|
79910
80438
|
);
|
|
79911
80439
|
const recentReferrals = Number(
|
|
79912
|
-
ctx.db.select({ total:
|
|
80440
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
79913
80441
|
and52(
|
|
79914
80442
|
eq61(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
79915
80443
|
inArray22(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79918,7 +80446,7 @@ var recentDataCheck = {
|
|
|
79918
80446
|
).get()?.total ?? 0
|
|
79919
80447
|
);
|
|
79920
80448
|
const recentUserFetches = Number(
|
|
79921
|
-
ctx.db.select({ total:
|
|
80449
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
79922
80450
|
and52(
|
|
79923
80451
|
eq61(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
79924
80452
|
inArray22(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79944,7 +80472,7 @@ var recentDataCheck = {
|
|
|
79944
80472
|
};
|
|
79945
80473
|
}
|
|
79946
80474
|
const olderCrawlers = Number(
|
|
79947
|
-
ctx.db.select({ total:
|
|
80475
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
79948
80476
|
and52(
|
|
79949
80477
|
eq61(crawlerEventsHourly.projectId, ctx.project.id),
|
|
79950
80478
|
inArray22(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79953,7 +80481,7 @@ var recentDataCheck = {
|
|
|
79953
80481
|
).get()?.total ?? 0
|
|
79954
80482
|
);
|
|
79955
80483
|
const olderReferrals = Number(
|
|
79956
|
-
ctx.db.select({ total:
|
|
80484
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
79957
80485
|
and52(
|
|
79958
80486
|
eq61(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
79959
80487
|
inArray22(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
@@ -79962,7 +80490,7 @@ var recentDataCheck = {
|
|
|
79962
80490
|
).get()?.total ?? 0
|
|
79963
80491
|
);
|
|
79964
80492
|
const olderUserFetches = Number(
|
|
79965
|
-
ctx.db.select({ total:
|
|
80493
|
+
ctx.db.select({ total: sql24`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
79966
80494
|
and52(
|
|
79967
80495
|
eq61(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
79968
80496
|
inArray22(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
@@ -80881,7 +81409,7 @@ async function doctorRoutes(app, opts) {
|
|
|
80881
81409
|
|
|
80882
81410
|
// ../api-routes/src/discovery/routes.ts
|
|
80883
81411
|
import crypto46 from "crypto";
|
|
80884
|
-
import { and as and53, desc as desc26, eq as eq62, gte as gte16, inArray as inArray23, isNull as isNull10, or as
|
|
81412
|
+
import { and as and53, desc as desc26, eq as eq62, gte as gte16, inArray as inArray23, isNull as isNull10, or as or13 } from "drizzle-orm";
|
|
80885
81413
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
80886
81414
|
async function discoveryRoutes(app, opts) {
|
|
80887
81415
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -80931,7 +81459,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
80931
81459
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
80932
81460
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
80933
81461
|
// upgrade, never a wrong reuse.
|
|
80934
|
-
locations.length === 0 ?
|
|
81462
|
+
locations.length === 0 ? or13(isNull10(discoverySessions.locations), eq62(discoverySessions.locations, locations)) : eq62(discoverySessions.locations, locations),
|
|
80935
81463
|
// Seed provider set is identity: a different phrasing distribution
|
|
80936
81464
|
// must never reuse another set's session. Null (= Gemini-only
|
|
80937
81465
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
@@ -81561,7 +82089,7 @@ function dedupeStrings(input) {
|
|
|
81561
82089
|
|
|
81562
82090
|
// ../api-routes/src/technical-aeo.ts
|
|
81563
82091
|
import crypto48 from "crypto";
|
|
81564
|
-
import { and as and54, asc as asc11, count as count2, desc as desc27, eq as eq64, inArray as inArray24, isNotNull as isNotNull5, isNull as isNull11, lt as lt10, or as
|
|
82092
|
+
import { and as and54, asc as asc11, count as count2, desc as desc27, eq as eq64, inArray as inArray24, isNotNull as isNotNull5, isNull as isNull11, lt as lt10, or as or14, sql as sql25 } from "drizzle-orm";
|
|
81565
82093
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
81566
82094
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
81567
82095
|
...SiteCrawlFetchedStates,
|
|
@@ -81780,7 +82308,7 @@ function linkKindFilter(linkKind) {
|
|
|
81780
82308
|
}
|
|
81781
82309
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
81782
82310
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
81783
|
-
var PAGE_CHANGE_PREDICATE =
|
|
82311
|
+
var PAGE_CHANGE_PREDICATE = sql25`(
|
|
81784
82312
|
current.url IS NOT previous.url
|
|
81785
82313
|
OR current.final_url IS NOT previous.final_url
|
|
81786
82314
|
OR current.path IS NOT previous.path
|
|
@@ -81802,7 +82330,7 @@ var PAGE_CHANGE_PREDICATE = sql24`(
|
|
|
81802
82330
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
81803
82331
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
81804
82332
|
)`;
|
|
81805
|
-
var LINK_CHANGE_PREDICATE =
|
|
82333
|
+
var LINK_CHANGE_PREDICATE = sql25`(
|
|
81806
82334
|
current.source_node_key IS NOT previous.source_node_key
|
|
81807
82335
|
OR current.source_url IS NOT previous.source_url
|
|
81808
82336
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -82331,7 +82859,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82331
82859
|
eq64(siteCrawlEdges.attemptId, scope.attemptId),
|
|
82332
82860
|
eq64(siteCrawlEdges.internal, true),
|
|
82333
82861
|
isNotNull5(siteCrawlEdges.targetNodeKey),
|
|
82334
|
-
|
|
82862
|
+
or14(
|
|
82335
82863
|
inArray24(siteCrawlEdges.sourceNodeKey, frontier),
|
|
82336
82864
|
inArray24(siteCrawlEdges.targetNodeKey, frontier)
|
|
82337
82865
|
)
|
|
@@ -82578,7 +83106,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82578
83106
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
82579
83107
|
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq64(siteCrawlSnapshots.runId, runs.id)).where(and54(
|
|
82580
83108
|
...completeSnapshotFilters,
|
|
82581
|
-
|
|
83109
|
+
or14(
|
|
82582
83110
|
lt10(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
82583
83111
|
and54(
|
|
82584
83112
|
eq64(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
@@ -82659,9 +83187,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82659
83187
|
const summary3 = { pages: zero2(), links: zero2() };
|
|
82660
83188
|
if (scopeFilter !== "links") {
|
|
82661
83189
|
if (includeAdded || includeChanged) {
|
|
82662
|
-
const addedExpression = includeAdded ?
|
|
82663
|
-
const changedExpression = includeChanged ?
|
|
82664
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
83190
|
+
const addedExpression = includeAdded ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83191
|
+
const changedExpression = includeChanged ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83192
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql25`
|
|
82665
83193
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
82666
83194
|
FROM site_crawl_pages AS current
|
|
82667
83195
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -82677,7 +83205,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82677
83205
|
summary3.pages.changed = Number(counts2.changed);
|
|
82678
83206
|
}
|
|
82679
83207
|
if (includeRemoved) {
|
|
82680
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
83208
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql25`
|
|
82681
83209
|
SELECT COUNT(*) AS removed
|
|
82682
83210
|
FROM site_crawl_pages AS previous
|
|
82683
83211
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -82695,9 +83223,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82695
83223
|
}
|
|
82696
83224
|
if (scopeFilter !== "pages") {
|
|
82697
83225
|
if (includeAdded || includeChanged) {
|
|
82698
|
-
const addedExpression = includeAdded ?
|
|
82699
|
-
const changedExpression = includeChanged ?
|
|
82700
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
83226
|
+
const addedExpression = includeAdded ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83227
|
+
const changedExpression = includeChanged ? sql25`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql25`0`;
|
|
83228
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql25`
|
|
82701
83229
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
82702
83230
|
FROM site_crawl_edges AS current
|
|
82703
83231
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -82715,7 +83243,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82715
83243
|
summary3.links.changed = Number(counts2.changed);
|
|
82716
83244
|
}
|
|
82717
83245
|
if (includeRemoved) {
|
|
82718
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
83246
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql25`
|
|
82719
83247
|
SELECT COUNT(*) AS removed
|
|
82720
83248
|
FROM site_crawl_edges AS previous
|
|
82721
83249
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -82745,48 +83273,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
82745
83273
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
82746
83274
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
82747
83275
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
82748
|
-
const internalCurrent = entity === "link" ?
|
|
82749
|
-
const internalPreviousJoin = entity === "link" ?
|
|
82750
|
-
const internalPrevious = entity === "link" ?
|
|
82751
|
-
const internalCurrentJoin = entity === "link" ?
|
|
83276
|
+
const internalCurrent = entity === "link" ? sql25`AND current.internal = 1` : sql25``;
|
|
83277
|
+
const internalPreviousJoin = entity === "link" ? sql25`AND previous.internal = 1` : sql25``;
|
|
83278
|
+
const internalPrevious = entity === "link" ? sql25`AND previous.internal = 1` : sql25``;
|
|
83279
|
+
const internalCurrentJoin = entity === "link" ? sql25`AND current.internal = 1` : sql25``;
|
|
82752
83280
|
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
82753
|
-
const afterKind = changeFilter === "added" ?
|
|
82754
|
-
const afterRows = includeAfter ? app.db.all(
|
|
83281
|
+
const afterKind = changeFilter === "added" ? sql25`previous.id IS NULL` : changeFilter === "changed" ? sql25`previous.id IS NOT NULL AND ${difference2}` : sql25`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
83282
|
+
const afterRows = includeAfter ? app.db.all(sql25`
|
|
82755
83283
|
SELECT ${entity} AS entity,
|
|
82756
83284
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
82757
|
-
current.${
|
|
82758
|
-
FROM ${
|
|
82759
|
-
LEFT JOIN ${
|
|
83285
|
+
current.${sql25.raw(keyColumn)} AS key
|
|
83286
|
+
FROM ${sql25.raw(table)} AS current
|
|
83287
|
+
LEFT JOIN ${sql25.raw(table)} AS previous
|
|
82760
83288
|
ON previous.project_id = ${beforeScope.projectId}
|
|
82761
83289
|
AND previous.run_id = ${beforeScope.runId}
|
|
82762
83290
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
82763
83291
|
${internalPreviousJoin}
|
|
82764
|
-
AND previous.${
|
|
83292
|
+
AND previous.${sql25.raw(keyColumn)} = current.${sql25.raw(keyColumn)}
|
|
82765
83293
|
WHERE current.project_id = ${afterScope.projectId}
|
|
82766
83294
|
AND current.run_id = ${afterScope.runId}
|
|
82767
83295
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
82768
83296
|
${internalCurrent}
|
|
82769
|
-
AND current.${
|
|
83297
|
+
AND current.${sql25.raw(keyColumn)} > ${afterKey}
|
|
82770
83298
|
AND (${afterKind})
|
|
82771
|
-
ORDER BY current.${
|
|
83299
|
+
ORDER BY current.${sql25.raw(keyColumn)} ASC
|
|
82772
83300
|
LIMIT ${wanted2}
|
|
82773
83301
|
`) : [];
|
|
82774
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
82775
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
82776
|
-
FROM ${
|
|
82777
|
-
LEFT JOIN ${
|
|
83302
|
+
const removedRows = includeRemoved ? app.db.all(sql25`
|
|
83303
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql25.raw(keyColumn)} AS key
|
|
83304
|
+
FROM ${sql25.raw(table)} AS previous
|
|
83305
|
+
LEFT JOIN ${sql25.raw(table)} AS current
|
|
82778
83306
|
ON current.project_id = ${afterScope.projectId}
|
|
82779
83307
|
AND current.run_id = ${afterScope.runId}
|
|
82780
83308
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
82781
83309
|
${internalCurrentJoin}
|
|
82782
|
-
AND current.${
|
|
83310
|
+
AND current.${sql25.raw(keyColumn)} = previous.${sql25.raw(keyColumn)}
|
|
82783
83311
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
82784
83312
|
AND previous.run_id = ${beforeScope.runId}
|
|
82785
83313
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
82786
83314
|
${internalPrevious}
|
|
82787
|
-
AND previous.${
|
|
83315
|
+
AND previous.${sql25.raw(keyColumn)} > ${afterKey}
|
|
82788
83316
|
AND current.id IS NULL
|
|
82789
|
-
ORDER BY previous.${
|
|
83317
|
+
ORDER BY previous.${sql25.raw(keyColumn)} ASC
|
|
82790
83318
|
LIMIT ${wanted2}
|
|
82791
83319
|
`) : [];
|
|
82792
83320
|
const merged = [];
|
|
@@ -83175,8 +83703,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
83175
83703
|
eq64(siteCrawlEdges.internal, true),
|
|
83176
83704
|
linkKindFilter(linkKind)
|
|
83177
83705
|
];
|
|
83178
|
-
const inboundMatch = request.query.nodeKey && request.query.url ?
|
|
83179
|
-
const outboundMatch = request.query.nodeKey && request.query.url ?
|
|
83706
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or14(eq64(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq64(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq64(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq64(siteCrawlEdges.targetUrl, request.query.url);
|
|
83707
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or14(eq64(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq64(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq64(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq64(siteCrawlEdges.sourceUrl, request.query.url);
|
|
83180
83708
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
83181
83709
|
const inboundRows = app.db.select().from(siteCrawlEdges).where(and54(...scope, inboundMatch)).orderBy(asc11(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
83182
83710
|
const outboundRows = app.db.select().from(siteCrawlEdges).where(and54(...scope, outboundMatch)).orderBy(asc11(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
@@ -83226,7 +83754,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
83226
83754
|
// a fabricated broken link shown to a client is not worth the saved
|
|
83227
83755
|
// predicate. Applied to `total` as well as the rows, or the count would
|
|
83228
83756
|
// promise a page the list cannot fill.
|
|
83229
|
-
isNotNull5(
|
|
83757
|
+
isNotNull5(sql25`json_extract(${siteCrawlFindings.evidence}, '$.statusCode')`)
|
|
83230
83758
|
);
|
|
83231
83759
|
const total = app.db.select({ value: count2() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
83232
83760
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -83471,7 +83999,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
83471
83999
|
// ../api-routes/src/research.ts
|
|
83472
84000
|
import crypto49 from "crypto";
|
|
83473
84001
|
import { z as z5 } from "zod";
|
|
83474
|
-
import { and as and55, count as count3, desc as desc28, eq as eq65, gte as gte17, lt as lt11, or as
|
|
84002
|
+
import { and as and55, count as count3, desc as desc28, eq as eq65, gte as gte17, lt as lt11, or as or15, sql as sql26 } from "drizzle-orm";
|
|
83475
84003
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
83476
84004
|
var BATCH_RECEIPT_PREFIX = "__canonry_research_batch__:";
|
|
83477
84005
|
async function researchRoutes(app, opts) {
|
|
@@ -83533,7 +84061,7 @@ async function researchRoutes(app, opts) {
|
|
|
83533
84061
|
eq65(researchRuns.projectId, project.id),
|
|
83534
84062
|
gte17(researchRuns.createdAt, start),
|
|
83535
84063
|
lt11(researchRuns.createdAt, end),
|
|
83536
|
-
|
|
84064
|
+
sql26`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
83537
84065
|
)).get()?.value ?? 0;
|
|
83538
84066
|
if (used >= viewerDailyRunLimit) {
|
|
83539
84067
|
throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
@@ -83542,7 +84070,7 @@ async function researchRoutes(app, opts) {
|
|
|
83542
84070
|
const id = crypto49.randomUUID();
|
|
83543
84071
|
tx.insert(researchRuns).values({ id, projectId: project.id, status: ResearchRunStatuses.queued, provider: providerName, requestedModel: requestedModel2, resolvedModel, location: location ?? null, totalQueries: input.queries.length, scope, template, idempotencyKey: input.idempotencyKey ?? null, requestHash: input.idempotencyKey ? requestHash2 : null, initiatedBy, createdAt: now }).run();
|
|
83544
84072
|
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id: crypto49.randomUUID(), researchRunId: id, position, queryText: query, queryClass: queryClasses[position] ?? null, status: ResearchQueryStatuses.queued, requestedModel: requestedModel2, resolvedModel, groundingSources: [], citedDomains: [], searchQueries: [], createdAt: now }).run();
|
|
83545
|
-
writeAuditLog(tx, { projectId: project.id, actor:
|
|
84073
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
83546
84074
|
return { reused: false, id, shouldDispatch: true };
|
|
83547
84075
|
});
|
|
83548
84076
|
const result = getDetail(app, project.id, decision.id);
|
|
@@ -83588,7 +84116,7 @@ async function researchRoutes(app, opts) {
|
|
|
83588
84116
|
eq65(researchRuns.projectId, project.id),
|
|
83589
84117
|
gte17(researchRuns.createdAt, start),
|
|
83590
84118
|
lt11(researchRuns.createdAt, end),
|
|
83591
|
-
|
|
84119
|
+
sql26`(json_extract(${researchRuns.initiatedBy}, '$.role') = ${UserRoles.viewer} OR json_extract(${researchRuns.initiatedBy}, '$.limited') = 1)`
|
|
83592
84120
|
)).get()?.value ?? 0;
|
|
83593
84121
|
if (used + prepared.length > viewerDailyRunLimit) throw researchDailyLimitExceeded(project.name, viewerDailyRunLimit, date);
|
|
83594
84122
|
}
|
|
@@ -83626,7 +84154,7 @@ async function researchRoutes(app, opts) {
|
|
|
83626
84154
|
searchQueries: [],
|
|
83627
84155
|
createdAt: now
|
|
83628
84156
|
}).run();
|
|
83629
|
-
writeAuditLog(tx, { projectId: project.id, actor:
|
|
84157
|
+
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
83630
84158
|
}
|
|
83631
84159
|
return { ids };
|
|
83632
84160
|
});
|
|
@@ -83641,7 +84169,7 @@ async function researchRoutes(app, opts) {
|
|
|
83641
84169
|
const cursor = request.query.cursor ? parseResearchCursor(request.query.cursor, project.id) : null;
|
|
83642
84170
|
const rows = app.db.select().from(researchRuns).where(and55(
|
|
83643
84171
|
eq65(researchRuns.projectId, project.id),
|
|
83644
|
-
cursor ?
|
|
84172
|
+
cursor ? or15(lt11(researchRuns.createdAt, cursor.createdAt), and55(eq65(researchRuns.createdAt, cursor.createdAt), lt11(researchRuns.id, cursor.id))) : void 0
|
|
83645
84173
|
)).orderBy(desc28(researchRuns.createdAt), desc28(researchRuns.id)).limit(limit + 1).all();
|
|
83646
84174
|
const runs2 = rows.slice(0, limit).map(serializeRun);
|
|
83647
84175
|
const last = runs2.at(-1);
|
|
@@ -83929,6 +84457,7 @@ async function apiRoutes(app, opts) {
|
|
|
83929
84457
|
});
|
|
83930
84458
|
}
|
|
83931
84459
|
await app.register(async (api) => {
|
|
84460
|
+
registerRequestContext(api);
|
|
83932
84461
|
await api.register(rateLimit, {
|
|
83933
84462
|
global: false,
|
|
83934
84463
|
hook: "preHandler",
|
|
@@ -84050,6 +84579,7 @@ async function apiRoutes(app, opts) {
|
|
|
84050
84579
|
await api.register(notificationRoutes, {
|
|
84051
84580
|
allowLoopbackWebhooks: opts.allowLoopbackWebhooks
|
|
84052
84581
|
});
|
|
84582
|
+
await api.register(operationalLogsRoutes, { listOperationalLogs: opts.listOperationalLogs });
|
|
84053
84583
|
await api.register(telemetryRoutes, {
|
|
84054
84584
|
getTelemetryStatus: opts.getTelemetryStatus,
|
|
84055
84585
|
setTelemetryEnabled: opts.setTelemetryEnabled,
|
|
@@ -84407,42 +84937,260 @@ function buildTrafficSourceValidators(opts) {
|
|
|
84407
84937
|
// src/intelligence-service.ts
|
|
84408
84938
|
import crypto51 from "crypto";
|
|
84409
84939
|
|
|
84410
|
-
// src/logger.ts
|
|
84940
|
+
// ../api-routes/src/runtime-logger.ts
|
|
84941
|
+
import { format } from "util";
|
|
84411
84942
|
var IS_TTY = process.stdout.isTTY === true;
|
|
84412
|
-
|
|
84413
|
-
|
|
84414
|
-
|
|
84415
|
-
|
|
84416
|
-
|
|
84417
|
-
|
|
84418
|
-
|
|
84419
|
-
|
|
84420
|
-
|
|
84421
|
-
|
|
84422
|
-
|
|
84423
|
-
|
|
84424
|
-
|
|
84425
|
-
|
|
84426
|
-
|
|
84427
|
-
|
|
84943
|
+
var MAX_IDENTITY_LENGTH = 512;
|
|
84944
|
+
var RESERVED_FIELDS = /* @__PURE__ */ new Set([
|
|
84945
|
+
"ts",
|
|
84946
|
+
"level",
|
|
84947
|
+
"module",
|
|
84948
|
+
"action",
|
|
84949
|
+
"msg",
|
|
84950
|
+
"requestId",
|
|
84951
|
+
"actor",
|
|
84952
|
+
"credentialId",
|
|
84953
|
+
"userAgent",
|
|
84954
|
+
"actorSession",
|
|
84955
|
+
"__runtimeRequestId"
|
|
84956
|
+
]);
|
|
84957
|
+
var listeners = /* @__PURE__ */ new Set();
|
|
84958
|
+
function addLogListener(listener) {
|
|
84959
|
+
listeners.add(listener);
|
|
84960
|
+
return () => listeners.delete(listener);
|
|
84428
84961
|
}
|
|
84429
84962
|
function createLogger(module) {
|
|
84430
|
-
|
|
84431
|
-
const
|
|
84432
|
-
|
|
84433
|
-
|
|
84434
|
-
|
|
84435
|
-
|
|
84436
|
-
|
|
84437
|
-
|
|
84438
|
-
|
|
84963
|
+
const emit = (level, action, context) => {
|
|
84964
|
+
const msg = context ? safeProperty(context, "msg") : void 0;
|
|
84965
|
+
emitRuntimeLog({ level, module, action, context, ...typeof msg === "string" ? { msg } : {} });
|
|
84966
|
+
};
|
|
84967
|
+
return {
|
|
84968
|
+
info: (action, ctx) => emit("info", action, ctx),
|
|
84969
|
+
warn: (action, ctx) => emit("warn", action, ctx),
|
|
84970
|
+
error: (action, ctx) => emit("error", action, ctx)
|
|
84971
|
+
};
|
|
84972
|
+
}
|
|
84973
|
+
function createFastifyLogger(options = {}) {
|
|
84974
|
+
return buildFastifyLogger(options.module ?? "HTTP", options.enabled !== false, {});
|
|
84975
|
+
}
|
|
84976
|
+
var LEVELS = { trace: 10, debug: 20, info: 30, warn: 40, error: 50, fatal: 60, silent: Infinity };
|
|
84977
|
+
function buildFastifyLogger(module, enabled, bindings, inheritedLevel = "info") {
|
|
84978
|
+
let level = inheritedLevel;
|
|
84979
|
+
const emit = (severity, args) => {
|
|
84980
|
+
if (LEVELS[severity] >= LEVELS[level]) logFastify(severity, module, enabled, bindings, args);
|
|
84981
|
+
};
|
|
84982
|
+
const logger = {
|
|
84983
|
+
get level() {
|
|
84984
|
+
return level;
|
|
84985
|
+
},
|
|
84986
|
+
set level(value) {
|
|
84987
|
+
if (Object.hasOwn(LEVELS, value)) level = value;
|
|
84988
|
+
},
|
|
84989
|
+
trace: (...args) => emit("trace", args),
|
|
84990
|
+
debug: (...args) => emit("debug", args),
|
|
84991
|
+
info: (...args) => emit("info", args),
|
|
84992
|
+
warn: (...args) => emit("warn", args),
|
|
84993
|
+
error: (...args) => emit("error", args),
|
|
84994
|
+
fatal: (...args) => emit("fatal", args),
|
|
84995
|
+
silent: () => void 0,
|
|
84996
|
+
child: (childBindings) => buildFastifyLogger(module, enabled, mergeBindings(bindings, childBindings), level)
|
|
84997
|
+
};
|
|
84998
|
+
return logger;
|
|
84999
|
+
}
|
|
85000
|
+
function logFastify(level, module, enabled, bindings, args) {
|
|
85001
|
+
const { context, msg } = normalizePinoArgs(args);
|
|
85002
|
+
emitRuntimeLog({
|
|
85003
|
+
level,
|
|
85004
|
+
module,
|
|
85005
|
+
action: `fastify.${level}`,
|
|
85006
|
+
msg,
|
|
85007
|
+
context: { ...bindings, ...context },
|
|
85008
|
+
trustedRequestId: asBoundString(bindings.__runtimeRequestId),
|
|
85009
|
+
enabled
|
|
85010
|
+
});
|
|
85011
|
+
}
|
|
85012
|
+
function normalizePinoArgs(args) {
|
|
85013
|
+
const first = args[0];
|
|
85014
|
+
const second = args[1];
|
|
85015
|
+
if (typeof first === "string") return { context: {}, msg: safeFormat(first, args.slice(1)) };
|
|
85016
|
+
if (first instanceof Error) {
|
|
85017
|
+
return { context: { err: first }, msg: typeof second === "string" ? safeFormat(second, args.slice(2)) : void 0 };
|
|
85018
|
+
}
|
|
85019
|
+
if (isObject(first)) {
|
|
85020
|
+
return { context: copyOwnData(first), msg: typeof second === "string" ? safeFormat(second, args.slice(2)) : void 0 };
|
|
85021
|
+
}
|
|
85022
|
+
return { context: {}, msg: typeof second === "string" ? safeFormat(second, args.slice(2)) : void 0 };
|
|
85023
|
+
}
|
|
85024
|
+
function emitRuntimeLog({
|
|
85025
|
+
level,
|
|
85026
|
+
module,
|
|
85027
|
+
action,
|
|
85028
|
+
msg,
|
|
85029
|
+
context,
|
|
85030
|
+
trustedRequestId,
|
|
85031
|
+
enabled = true
|
|
85032
|
+
}) {
|
|
85033
|
+
const safeContext = sanitizeContext(context);
|
|
85034
|
+
const error = safeContext.err ?? safeContext.error;
|
|
85035
|
+
const errorMessage = typeof error === "string" ? error : isRecord4(error) && typeof error.message === "string" ? error.message : void 0;
|
|
85036
|
+
const message = errorMessage && errorMessage !== msg ? [msg, errorMessage].filter(Boolean).join(": ") : msg;
|
|
85037
|
+
const entry = {
|
|
85038
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
85039
|
+
level,
|
|
85040
|
+
module: boundText(module, "Runtime"),
|
|
85041
|
+
action: boundText(action, "log"),
|
|
85042
|
+
...message === void 0 ? {} : { msg: redactLogString(message) },
|
|
85043
|
+
...safeContext,
|
|
85044
|
+
...trustedAttribution(getRequestContext(trustedRequestId), trustedRequestId)
|
|
85045
|
+
};
|
|
85046
|
+
if (enabled) writeEntry(entry);
|
|
85047
|
+
for (const listener of listeners) {
|
|
85048
|
+
try {
|
|
85049
|
+
listener(entry);
|
|
85050
|
+
} catch {
|
|
85051
|
+
}
|
|
85052
|
+
}
|
|
85053
|
+
}
|
|
85054
|
+
function sanitizeContext(context) {
|
|
85055
|
+
if (!context) return {};
|
|
85056
|
+
const output = {};
|
|
85057
|
+
for (const key of safeKeys(context).slice(0, 100)) {
|
|
85058
|
+
if (RESERVED_FIELDS.has(key)) continue;
|
|
85059
|
+
const value = safeProperty(context, key);
|
|
85060
|
+
if (key === "req" || key === "request") {
|
|
85061
|
+
Object.assign(output, normalizeRequest(value));
|
|
85062
|
+
continue;
|
|
85063
|
+
}
|
|
85064
|
+
if (key === "res" || key === "reply") {
|
|
85065
|
+
Object.assign(output, normalizeReply(value));
|
|
85066
|
+
continue;
|
|
85067
|
+
}
|
|
85068
|
+
if (key === "responseTime" || key === "duration") {
|
|
85069
|
+
if (typeof value === "number" && Number.isFinite(value)) output.durationMs = value;
|
|
85070
|
+
continue;
|
|
85071
|
+
}
|
|
85072
|
+
output[key] = value;
|
|
84439
85073
|
}
|
|
85074
|
+
delete output.__runtimeRequestId;
|
|
85075
|
+
const redacted = redactLogValue(output);
|
|
85076
|
+
return isRecord4(redacted) ? redacted : {};
|
|
85077
|
+
}
|
|
85078
|
+
function trustedAttribution(context, trustedRequestId) {
|
|
85079
|
+
const requestId = boundOptional(context?.requestId) ?? trustedRequestId;
|
|
85080
|
+
const actor = diagnosticIdentity("actor", context?.actor);
|
|
85081
|
+
const credentialId = diagnosticIdentity("credentialId", context?.credentialId);
|
|
85082
|
+
const userAgent = boundOptional(context?.userAgent);
|
|
85083
|
+
const actorSession = boundOptional(context?.actorSession);
|
|
85084
|
+
const method = boundOptional(context?.method);
|
|
85085
|
+
const route = boundOptional(context?.route);
|
|
84440
85086
|
return {
|
|
84441
|
-
|
|
84442
|
-
|
|
84443
|
-
|
|
85087
|
+
...requestId ? { requestId } : {},
|
|
85088
|
+
...actor ? { actor } : {},
|
|
85089
|
+
...credentialId ? { credentialId } : {},
|
|
85090
|
+
...userAgent ? { userAgent } : {},
|
|
85091
|
+
...actorSession ? { actorSession } : {},
|
|
85092
|
+
...method ? { method } : {},
|
|
85093
|
+
...route ? { route } : {},
|
|
85094
|
+
...context?.statusCode === void 0 ? {} : { statusCode: context.statusCode }
|
|
84444
85095
|
};
|
|
84445
85096
|
}
|
|
85097
|
+
function mergeBindings(parent, child) {
|
|
85098
|
+
const merged = { ...parent };
|
|
85099
|
+
if (!isObject(child)) return merged;
|
|
85100
|
+
for (const key of safeKeys(child).slice(0, 100)) {
|
|
85101
|
+
const value = safeProperty(child, key);
|
|
85102
|
+
if (key === "req" || key === "request") Object.assign(merged, normalizeRequest(value));
|
|
85103
|
+
else if (key === "res" || key === "reply") Object.assign(merged, normalizeReply(value));
|
|
85104
|
+
else if (key === "reqId") {
|
|
85105
|
+
merged.reqId = value;
|
|
85106
|
+
merged.__runtimeRequestId = value;
|
|
85107
|
+
} else if (!RESERVED_FIELDS.has(key)) merged[key] = value;
|
|
85108
|
+
}
|
|
85109
|
+
return merged;
|
|
85110
|
+
}
|
|
85111
|
+
function normalizeRequest(value) {
|
|
85112
|
+
if (!isObject(value)) return {};
|
|
85113
|
+
const id = asBoundString(safeProperty(value, "id"));
|
|
85114
|
+
const method = asBoundString(safeProperty(value, "method"));
|
|
85115
|
+
const routeOptions = safeProperty(value, "routeOptions");
|
|
85116
|
+
const route = isObject(routeOptions) ? asBoundString(safeProperty(routeOptions, "url")) : void 0;
|
|
85117
|
+
return {
|
|
85118
|
+
...id ? { reqId: id, __runtimeRequestId: id } : {},
|
|
85119
|
+
...method ? { method } : {},
|
|
85120
|
+
...route ? { route } : {}
|
|
85121
|
+
};
|
|
85122
|
+
}
|
|
85123
|
+
function normalizeReply(value) {
|
|
85124
|
+
if (!isObject(value)) return {};
|
|
85125
|
+
const statusCode = safeProperty(value, "statusCode");
|
|
85126
|
+
return typeof statusCode === "number" && Number.isFinite(statusCode) ? { statusCode } : {};
|
|
85127
|
+
}
|
|
85128
|
+
function writeEntry(entry) {
|
|
85129
|
+
const stream = entry.level === "error" || entry.level === "fatal" ? process.stderr : process.stdout;
|
|
85130
|
+
const line = IS_TTY ? formatTTY(entry) : JSON.stringify(entry);
|
|
85131
|
+
try {
|
|
85132
|
+
stream.write(`${line}
|
|
85133
|
+
`);
|
|
85134
|
+
} catch {
|
|
85135
|
+
}
|
|
85136
|
+
}
|
|
85137
|
+
function formatTTY(entry) {
|
|
85138
|
+
const { ts, level, module, action, msg, ...context } = entry;
|
|
85139
|
+
const levelTag = level === "fatal" || level === "error" ? "ERR" : level === "warn" ? "WRN" : "INF";
|
|
85140
|
+
const parts = safeKeys(context).map((key) => `${key}=${safeDisplay(safeProperty(context, key))}`);
|
|
85141
|
+
return `${ts.slice(11, 19)} ${levelTag} [${module}] ${action}${msg ? ` ${msg}` : ""}${parts.length ? ` ${parts.join(" ")}` : ""}`;
|
|
85142
|
+
}
|
|
85143
|
+
function safeDisplay(value) {
|
|
85144
|
+
try {
|
|
85145
|
+
return typeof value === "string" ? value : JSON.stringify(value);
|
|
85146
|
+
} catch {
|
|
85147
|
+
return "[UNSERIALIZABLE]";
|
|
85148
|
+
}
|
|
85149
|
+
}
|
|
85150
|
+
function safeFormat(message, args) {
|
|
85151
|
+
try {
|
|
85152
|
+
return args.length ? format(message, ...args) : message;
|
|
85153
|
+
} catch {
|
|
85154
|
+
return message;
|
|
85155
|
+
}
|
|
85156
|
+
}
|
|
85157
|
+
function isObject(value) {
|
|
85158
|
+
return value !== null && typeof value === "object";
|
|
85159
|
+
}
|
|
85160
|
+
function isRecord4(value) {
|
|
85161
|
+
return isObject(value) && !Array.isArray(value);
|
|
85162
|
+
}
|
|
85163
|
+
function safeKeys(value) {
|
|
85164
|
+
try {
|
|
85165
|
+
return Object.keys(value);
|
|
85166
|
+
} catch {
|
|
85167
|
+
return [];
|
|
85168
|
+
}
|
|
85169
|
+
}
|
|
85170
|
+
function safeProperty(value, key) {
|
|
85171
|
+
try {
|
|
85172
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
85173
|
+
return descriptor && "value" in descriptor ? descriptor.value : void 0;
|
|
85174
|
+
} catch {
|
|
85175
|
+
return void 0;
|
|
85176
|
+
}
|
|
85177
|
+
}
|
|
85178
|
+
function copyOwnData(value) {
|
|
85179
|
+
const output = {};
|
|
85180
|
+
for (const key of safeKeys(value)) output[key] = safeProperty(value, key);
|
|
85181
|
+
return output;
|
|
85182
|
+
}
|
|
85183
|
+
function boundText(value, fallback) {
|
|
85184
|
+
return boundOptional(value) ?? fallback;
|
|
85185
|
+
}
|
|
85186
|
+
function asBoundString(value) {
|
|
85187
|
+
return typeof value === "string" ? boundOptional(value) : void 0;
|
|
85188
|
+
}
|
|
85189
|
+
function boundOptional(value) {
|
|
85190
|
+
if (typeof value !== "string") return void 0;
|
|
85191
|
+
const redacted = redactLogString(value).replace(/[\r\n]+/g, " ").trim().slice(0, MAX_IDENTITY_LENGTH);
|
|
85192
|
+
return redacted || void 0;
|
|
85193
|
+
}
|
|
84446
85194
|
|
|
84447
85195
|
// src/citation-utils.ts
|
|
84448
85196
|
function pickProjectCitedDomain(citedDomains, projectDomains) {
|
|
@@ -84590,13 +85338,13 @@ function matchesBrandAlias(candidate, aliases) {
|
|
|
84590
85338
|
var RECURRENCE_LOOKBACK_RUNS = 5;
|
|
84591
85339
|
var HISTORY_WINDOW_RUNS = Math.max(PERSISTENT_GAP_THRESHOLD, 5);
|
|
84592
85340
|
var log = createLogger("IntelligenceService");
|
|
84593
|
-
var runChronologyKey =
|
|
85341
|
+
var runChronologyKey = sql27`coalesce(${runs.finishedAt}, ${runs.createdAt})`;
|
|
84594
85342
|
function analyzableRunPredicates(projectId) {
|
|
84595
85343
|
return [
|
|
84596
85344
|
eq67(runs.projectId, projectId),
|
|
84597
85345
|
// Only this kind writes query_snapshots.
|
|
84598
85346
|
eq67(runs.kind, RunKinds["answer-visibility"]),
|
|
84599
|
-
|
|
85347
|
+
or16(eq67(runs.status, RunStatuses.completed), eq67(runs.status, RunStatuses.partial)),
|
|
84600
85348
|
// Defensive: RunCoordinator already skips probes before analyzeAndPersist
|
|
84601
85349
|
// is called, but a future call site invoking it directly for a probe must
|
|
84602
85350
|
// still not pollute the intelligence window.
|
|
@@ -84658,7 +85406,7 @@ var IntelligenceService = class {
|
|
|
84658
85406
|
// MEASUREMENT: only runs that actually wrote snapshots — see above.
|
|
84659
85407
|
// Index seek on idx_snapshots_run, short-circuited on the first row.
|
|
84660
85408
|
exists(
|
|
84661
|
-
this.db.select({ one:
|
|
85409
|
+
this.db.select({ one: sql27`1` }).from(querySnapshots).where(eq67(querySnapshots.runId, runs.id))
|
|
84662
85410
|
)
|
|
84663
85411
|
)
|
|
84664
85412
|
).orderBy(desc29(runs.finishedAt), desc29(runs.createdAt)).limit(HISTORY_WINDOW_RUNS).all();
|
|
@@ -85186,7 +85934,7 @@ var IntelligenceService = class {
|
|
|
85186
85934
|
and57(
|
|
85187
85935
|
eq67(runs.projectId, projectId),
|
|
85188
85936
|
eq67(runs.kind, RunKinds["answer-visibility"]),
|
|
85189
|
-
|
|
85937
|
+
or16(eq67(runs.status, "completed"), eq67(runs.status, "partial")),
|
|
85190
85938
|
// Defensive — see top of file.
|
|
85191
85939
|
ne10(runs.trigger, RunTriggers.probe)
|
|
85192
85940
|
)
|
|
@@ -85369,6 +86117,7 @@ export {
|
|
|
85369
86117
|
extractLegacyCredentials,
|
|
85370
86118
|
dropLegacyCredentialColumns,
|
|
85371
86119
|
migrate,
|
|
86120
|
+
OperationalLogStore,
|
|
85372
86121
|
groupRunsByCreatedAt,
|
|
85373
86122
|
classifyCrawler,
|
|
85374
86123
|
classifyAiUserFetch,
|
|
@@ -85450,6 +86199,8 @@ export {
|
|
|
85450
86199
|
markSessionFailed,
|
|
85451
86200
|
captureSimpleMeasurementDefinition,
|
|
85452
86201
|
apiRoutes,
|
|
86202
|
+
addLogListener,
|
|
85453
86203
|
createLogger,
|
|
86204
|
+
createFastifyLogger,
|
|
85454
86205
|
IntelligenceService
|
|
85455
86206
|
};
|