@canonry/canonry 4.159.0 → 4.161.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/aero/SKILL.md +16 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +17 -5
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +66 -0
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +231 -62
- package/assets/assets/{AuditHistoryPanel-BiloGcQA.js → AuditHistoryPanel-C_1twF00.js} +1 -1
- package/assets/assets/{BacklinksPage-CPM1O2bC.js → BacklinksPage-CfAuMCaN.js} +1 -1
- package/assets/assets/{HistoryPage-1YOjtdJy.js → HistoryPage-DrRxOL3j.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-CVDTY_gC.js → MeasurementPropertyPage-BFtxx5Jw.js} +1 -1
- package/assets/assets/{ProjectPage-DnGXMx1n.js → ProjectPage-Djvhx4HD.js} +9 -9
- package/assets/assets/{RunRow-fIaadH4h.js → RunRow-DVZXibR2.js} +1 -1
- package/assets/assets/{RunsPage-DfifdM5z.js → RunsPage-CAUMy7By.js} +1 -1
- package/assets/assets/{SettingsPage-efA2EEmS.js → SettingsPage-CcSOmQAl.js} +1 -1
- package/assets/assets/{SiteHealthSection-BROtQGfu.js → SiteHealthSection-DYjEWd1I.js} +3 -3
- package/assets/assets/{TrafficPage-Wk43Bm_q.js → TrafficPage-BWM33jnJ.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BhBvoLWu.js → TrafficSourceDetailPage-DvEZ_JGY.js} +1 -1
- package/assets/assets/{extract-error-message-YGzZFvWs.js → extract-error-message-CnKOJkYB.js} +1 -1
- package/assets/assets/{index-DmsIZQAm.js → index-CQGAqmDx.js} +22 -22
- package/assets/assets/{react-sigma_core.esm.min-DM9XptbM.js → react-sigma_core.esm.min-CHCAIMBW.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-ZL3VY435.js → chunk-EXDO7EPL.js} +104 -9
- package/dist/{chunk-ZEG43TY2.js → chunk-HCGTKQCG.js} +21 -1
- package/dist/{chunk-5AFWHKKI.js → chunk-J27J2DF6.js} +2095 -703
- package/dist/{chunk-BZPBVOAM.js → chunk-W6OWEPZG.js} +70 -30
- package/dist/cli.js +308 -124
- package/dist/index.d.ts +28 -14
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-DHY5CYWX.js → intelligence-service-HKDIVTZB.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +8 -7
|
@@ -499,6 +499,7 @@ import {
|
|
|
499
499
|
segmentCrawlerHits,
|
|
500
500
|
serializeRunError,
|
|
501
501
|
settingsDtoSchema,
|
|
502
|
+
shiftIsoCalendarDate,
|
|
502
503
|
siteAuditLivePageHealthSchema,
|
|
503
504
|
siteAuditPageFactorSchema,
|
|
504
505
|
siteAuditPagesResponseSchema,
|
|
@@ -577,10 +578,10 @@ import {
|
|
|
577
578
|
wordpressSchemaDeployResultDtoSchema,
|
|
578
579
|
wordpressSchemaStatusResultDtoSchema,
|
|
579
580
|
wordpressStatusDtoSchema
|
|
580
|
-
} from "./chunk-
|
|
581
|
+
} from "./chunk-EXDO7EPL.js";
|
|
581
582
|
|
|
582
583
|
// src/intelligence-service.ts
|
|
583
|
-
import { eq as
|
|
584
|
+
import { eq as eq59, desc as desc26, asc as asc11, and as and47, ne as ne7, or as or13, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
|
|
584
585
|
|
|
585
586
|
// ../db/src/client.ts
|
|
586
587
|
import { mkdirSync } from "fs";
|
|
@@ -641,6 +642,7 @@ __export(schema_exports, {
|
|
|
641
642
|
googleConnections: () => googleConnections,
|
|
642
643
|
gscCoverageSnapshots: () => gscCoverageSnapshots,
|
|
643
644
|
gscDailyTotals: () => gscDailyTotals,
|
|
645
|
+
gscDataWatermarks: () => gscDataWatermarks,
|
|
644
646
|
gscQueryDailyTotals: () => gscQueryDailyTotals,
|
|
645
647
|
gscSearchData: () => gscSearchData,
|
|
646
648
|
gscUrlInspections: () => gscUrlInspections,
|
|
@@ -1141,6 +1143,12 @@ var gscSearchData = sqliteTable("gsc_search_data", {
|
|
|
1141
1143
|
index("idx_gsc_search_query").on(table.query),
|
|
1142
1144
|
index("idx_gsc_search_run").on(table.syncRunId)
|
|
1143
1145
|
]);
|
|
1146
|
+
var gscDataWatermarks = sqliteTable("gsc_data_watermarks", {
|
|
1147
|
+
projectId: text("project_id").primaryKey().references(() => projects.id, { onDelete: "cascade" }),
|
|
1148
|
+
dataThroughDate: text("data_through_date").notNull(),
|
|
1149
|
+
syncedThroughDate: text("synced_through_date"),
|
|
1150
|
+
updatedAt: text("updated_at").notNull()
|
|
1151
|
+
});
|
|
1144
1152
|
var gscDailyTotals = sqliteTable("gsc_daily_totals", {
|
|
1145
1153
|
id: text("id").primaryKey(),
|
|
1146
1154
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
@@ -2152,6 +2160,16 @@ var trafficSources = sqliteTable("traffic_sources", {
|
|
|
2152
2160
|
// `traffic.source.worker-version` doctor check. NULL until the first
|
|
2153
2161
|
// event arrives or for source types that don't forward versioned events.
|
|
2154
2162
|
lastWorkerVersion: text("last_worker_version"),
|
|
2163
|
+
// Pull adapters use this durable, owner-bound lease to prevent a manual
|
|
2164
|
+
// sync and the scheduler from consuming the same source concurrently.
|
|
2165
|
+
// Both fields are nullable so pre-lease sources remain immediately valid.
|
|
2166
|
+
syncLeaseOwner: text("sync_lease_owner"),
|
|
2167
|
+
syncLeaseExpiresAt: text("sync_lease_expires_at"),
|
|
2168
|
+
// Residual Queue depth returned by Cloudflare after the most recent bounded
|
|
2169
|
+
// pull. NULL means a queue-backed source has not observed backlog yet; zero
|
|
2170
|
+
// is an explicit observation that the Queue was drained at that instant.
|
|
2171
|
+
queueBacklogCount: integer("queue_backlog_count"),
|
|
2172
|
+
queueBacklogObservedAt: text("queue_backlog_observed_at"),
|
|
2155
2173
|
createdAt: text("created_at").notNull(),
|
|
2156
2174
|
updatedAt: text("updated_at").notNull()
|
|
2157
2175
|
}, (table) => [
|
|
@@ -6005,7 +6023,7 @@ var MIGRATION_VERSIONS = [
|
|
|
6005
6023
|
// Push-receive traffic sources (currently only `cloudflare`) need a
|
|
6006
6024
|
// per-source bearer for the Worker to authenticate against canonry's
|
|
6007
6025
|
// ingest endpoint, plus a place to remember the deployed Worker version.
|
|
6008
|
-
// Durable receipts are transport-neutral: direct push and
|
|
6026
|
+
// Durable receipts are transport-neutral: direct push and Queue pull
|
|
6009
6027
|
// pull consumer both claim an event in the same transaction as rollups.
|
|
6010
6028
|
// Cleartext credentials remain outside the database.
|
|
6011
6029
|
//
|
|
@@ -6145,6 +6163,47 @@ var MIGRATION_VERSIONS = [
|
|
|
6145
6163
|
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_live_preview
|
|
6146
6164
|
ON site_crawl_pages(project_id, run_id, attempt_id, audit_state, audit_score, node_key)`
|
|
6147
6165
|
]
|
|
6166
|
+
},
|
|
6167
|
+
{
|
|
6168
|
+
version: 135,
|
|
6169
|
+
name: "traffic-source-sync-lease",
|
|
6170
|
+
// A source-scoped lease serializes external pull consumers. Nullable
|
|
6171
|
+
// fields preserve every existing source and let an older binary continue
|
|
6172
|
+
// to insert source rows without knowing about leases.
|
|
6173
|
+
statements: [
|
|
6174
|
+
`ALTER TABLE traffic_sources ADD COLUMN sync_lease_owner TEXT`,
|
|
6175
|
+
`ALTER TABLE traffic_sources ADD COLUMN sync_lease_expires_at TEXT`
|
|
6176
|
+
]
|
|
6177
|
+
},
|
|
6178
|
+
{
|
|
6179
|
+
version: 136,
|
|
6180
|
+
name: "traffic-source-queue-backlog",
|
|
6181
|
+
// Persist Cloudflare's residual Queue depth so a bounded successful drain
|
|
6182
|
+
// cannot hide that work remains. NULL preserves every legacy source and
|
|
6183
|
+
// distinguishes "not observed" from an observed empty Queue.
|
|
6184
|
+
statements: [
|
|
6185
|
+
`ALTER TABLE traffic_sources ADD COLUMN queue_backlog_count INTEGER`,
|
|
6186
|
+
`ALTER TABLE traffic_sources ADD COLUMN queue_backlog_observed_at TEXT`
|
|
6187
|
+
]
|
|
6188
|
+
},
|
|
6189
|
+
{
|
|
6190
|
+
version: 137,
|
|
6191
|
+
name: "gsc-data-watermark",
|
|
6192
|
+
// The furthest GSC reporting date this project has EVER observed.
|
|
6193
|
+
//
|
|
6194
|
+
// `MAX(date)` over the stored rows is not a frontier: Search Analytics
|
|
6195
|
+
// returns no row for a day with no data, so a quiet tail makes the observed
|
|
6196
|
+
// max walk BACKWARD and drags every anchored window back with it. This
|
|
6197
|
+
// column is monotonic — a sync may only advance it — so a zero-traffic
|
|
6198
|
+
// stretch can never move the frontier the wrong way.
|
|
6199
|
+
statements: [
|
|
6200
|
+
`CREATE TABLE IF NOT EXISTS gsc_data_watermarks (
|
|
6201
|
+
project_id TEXT PRIMARY KEY REFERENCES projects(id) ON DELETE CASCADE,
|
|
6202
|
+
data_through_date TEXT NOT NULL,
|
|
6203
|
+
synced_through_date TEXT,
|
|
6204
|
+
updated_at TEXT NOT NULL
|
|
6205
|
+
)`
|
|
6206
|
+
]
|
|
6148
6207
|
}
|
|
6149
6208
|
];
|
|
6150
6209
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -12162,13 +12221,13 @@ function parseRunTriggerRequest(value) {
|
|
|
12162
12221
|
}))
|
|
12163
12222
|
});
|
|
12164
12223
|
}
|
|
12165
|
-
function parseListLimit(raw, defaultValue,
|
|
12224
|
+
function parseListLimit(raw, defaultValue, max2) {
|
|
12166
12225
|
if (raw === void 0) return defaultValue;
|
|
12167
12226
|
const parsed = Number(raw);
|
|
12168
12227
|
if (!Number.isFinite(parsed) || parsed < 1 || !Number.isInteger(parsed)) {
|
|
12169
12228
|
throw validationError('"limit" must be a positive integer');
|
|
12170
12229
|
}
|
|
12171
|
-
return Math.min(parsed,
|
|
12230
|
+
return Math.min(parsed, max2);
|
|
12172
12231
|
}
|
|
12173
12232
|
function parseListKind(raw) {
|
|
12174
12233
|
if (raw === void 0 || raw === "") return null;
|
|
@@ -14587,9 +14646,9 @@ function parseHeader(records) {
|
|
|
14587
14646
|
}
|
|
14588
14647
|
indices.set(name, index2);
|
|
14589
14648
|
}
|
|
14590
|
-
for (const
|
|
14591
|
-
if (!indices.has(
|
|
14592
|
-
csvError("csv-header-missing", `CSV is missing the required "${
|
|
14649
|
+
for (const required2 of ["property", "group"]) {
|
|
14650
|
+
if (!indices.has(required2)) {
|
|
14651
|
+
csvError("csv-header-missing", `CSV is missing the required "${required2}" header.`, 400, { header: required2 });
|
|
14593
14652
|
}
|
|
14594
14653
|
}
|
|
14595
14654
|
return { indices, length: header.fields.length };
|
|
@@ -18544,7 +18603,7 @@ function validateCron(expr) {
|
|
|
18544
18603
|
}
|
|
18545
18604
|
return true;
|
|
18546
18605
|
}
|
|
18547
|
-
function validateCronField(field, min,
|
|
18606
|
+
function validateCronField(field, min, max2) {
|
|
18548
18607
|
if (field === "*") return true;
|
|
18549
18608
|
const segments = field.split(",");
|
|
18550
18609
|
for (const segment of segments) {
|
|
@@ -18560,7 +18619,7 @@ function validateCronField(field, min, max) {
|
|
|
18560
18619
|
if (rangeParts.length > 2) return false;
|
|
18561
18620
|
for (const part of rangeParts) {
|
|
18562
18621
|
const num = parseInt(part, 10);
|
|
18563
|
-
if (isNaN(num) || num < min || num >
|
|
18622
|
+
if (isNaN(num) || num < min || num > max2) return false;
|
|
18564
18623
|
}
|
|
18565
18624
|
}
|
|
18566
18625
|
return true;
|
|
@@ -19380,16 +19439,16 @@ function addAuditHistoryFilters(filters, query) {
|
|
|
19380
19439
|
if (query.actor) filters.push(eq22(auditLog.actor, query.actor));
|
|
19381
19440
|
if (query.entityType) filters.push(eq22(auditLog.entityType, query.entityType));
|
|
19382
19441
|
}
|
|
19383
|
-
function parseBoundedInt(value, fallback,
|
|
19442
|
+
function parseBoundedInt(value, fallback, max2) {
|
|
19384
19443
|
const parsed = Number.parseInt(value ?? "", 10);
|
|
19385
19444
|
if (!Number.isFinite(parsed) || parsed < 0) return fallback;
|
|
19386
|
-
return Math.min(parsed,
|
|
19445
|
+
return Math.min(parsed, max2);
|
|
19387
19446
|
}
|
|
19388
|
-
function parseOptionalPositiveInt(value,
|
|
19447
|
+
function parseOptionalPositiveInt(value, max2) {
|
|
19389
19448
|
if (value == null) return void 0;
|
|
19390
19449
|
const parsed = Number.parseInt(value, 10);
|
|
19391
19450
|
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
19392
|
-
return Math.min(parsed,
|
|
19451
|
+
return Math.min(parsed, max2);
|
|
19393
19452
|
}
|
|
19394
19453
|
|
|
19395
19454
|
// ../api-routes/src/analytics.ts
|
|
@@ -21136,7 +21195,47 @@ function winnabilityClassRank(winnabilityClass) {
|
|
|
21136
21195
|
}
|
|
21137
21196
|
|
|
21138
21197
|
// ../api-routes/src/gsc-totals.ts
|
|
21139
|
-
import { and as and21, asc as asc4, eq as eq27, sql as sql8 } from "drizzle-orm";
|
|
21198
|
+
import { and as and21, asc as asc4, eq as eq27, sql as sql8, max } from "drizzle-orm";
|
|
21199
|
+
var WINDOW_DAYS = { "7d": 7, "30d": 30, "90d": 90 };
|
|
21200
|
+
function resolveGscWindowRange(window, latestDataDate, today) {
|
|
21201
|
+
if (window === "all") {
|
|
21202
|
+
return {
|
|
21203
|
+
startDate: null,
|
|
21204
|
+
endDate: latestDataDate,
|
|
21205
|
+
latestDataDate,
|
|
21206
|
+
daysSinceLatestData: gscDaysSinceLatestData(latestDataDate, today)
|
|
21207
|
+
};
|
|
21208
|
+
}
|
|
21209
|
+
return resolveGscWindowDays(WINDOW_DAYS[window], latestDataDate, today);
|
|
21210
|
+
}
|
|
21211
|
+
function resolveGscWindowDays(days, latestDataDate, today) {
|
|
21212
|
+
if (latestDataDate === null) {
|
|
21213
|
+
return {
|
|
21214
|
+
startDate: shiftIsoCalendarDate(today, -days),
|
|
21215
|
+
endDate: null,
|
|
21216
|
+
latestDataDate: null,
|
|
21217
|
+
daysSinceLatestData: null
|
|
21218
|
+
};
|
|
21219
|
+
}
|
|
21220
|
+
return {
|
|
21221
|
+
startDate: shiftIsoCalendarDate(latestDataDate, -(days - 1)),
|
|
21222
|
+
endDate: latestDataDate,
|
|
21223
|
+
latestDataDate,
|
|
21224
|
+
daysSinceLatestData: gscDaysSinceLatestData(latestDataDate, today)
|
|
21225
|
+
};
|
|
21226
|
+
}
|
|
21227
|
+
function gscDaysSinceLatestData(latestDataDate, today) {
|
|
21228
|
+
if (latestDataDate === null) return null;
|
|
21229
|
+
return Math.max(0, Math.round(
|
|
21230
|
+
(Date.parse(`${today}T00:00:00Z`) - Date.parse(`${latestDataDate}T00:00:00Z`)) / 864e5
|
|
21231
|
+
));
|
|
21232
|
+
}
|
|
21233
|
+
function readLatestGscDataDate(db, projectId) {
|
|
21234
|
+
const watermark = db.select({ through: gscDataWatermarks.dataThroughDate }).from(gscDataWatermarks).where(eq27(gscDataWatermarks.projectId, projectId)).get()?.through ?? null;
|
|
21235
|
+
const property = db.select({ latest: max(gscDailyTotals.date) }).from(gscDailyTotals).where(eq27(gscDailyTotals.projectId, projectId)).get()?.latest ?? null;
|
|
21236
|
+
const dimensioned = db.select({ latest: max(gscSearchData.date) }).from(gscSearchData).where(eq27(gscSearchData.projectId, projectId)).get()?.latest ?? null;
|
|
21237
|
+
return [watermark, property, dimensioned].filter((d) => d !== null).reduce((maxDate, d) => maxDate === null || d > maxDate ? d : maxDate, null);
|
|
21238
|
+
}
|
|
21140
21239
|
function readGscDailyTotals(db, projectId, startDate, endDate) {
|
|
21141
21240
|
const rows = db.select({
|
|
21142
21241
|
date: gscDailyTotals.date,
|
|
@@ -22395,7 +22494,7 @@ function renderWhatsChanged(report, audience) {
|
|
|
22395
22494
|
}
|
|
22396
22495
|
function renderProviderBars(rates) {
|
|
22397
22496
|
if (rates.length === 0) return "";
|
|
22398
|
-
const
|
|
22497
|
+
const max2 = Math.max(...rates.map((r) => r.citationRate), 100);
|
|
22399
22498
|
const width = 600;
|
|
22400
22499
|
const height = Math.max(rates.length * 32 + 24, 80);
|
|
22401
22500
|
const labelWidth = 80;
|
|
@@ -22404,7 +22503,7 @@ function renderProviderBars(rates) {
|
|
|
22404
22503
|
const bars = rates.map((r, i) => {
|
|
22405
22504
|
const y = i * 32 + padding;
|
|
22406
22505
|
const barHeight = 22;
|
|
22407
|
-
const w =
|
|
22506
|
+
const w = max2 > 0 ? r.citationRate / max2 * barWidth : 0;
|
|
22408
22507
|
const color = COLORS.series[i % COLORS.series.length];
|
|
22409
22508
|
return `
|
|
22410
22509
|
<text x="${labelWidth - 8}" y="${y + 16}" fill="${COLORS.textMuted}" font-size="11" text-anchor="end">${escapeHtml(r.provider)}</text>
|
|
@@ -22454,14 +22553,14 @@ function renderCitationScorecard(report) {
|
|
|
22454
22553
|
}
|
|
22455
22554
|
function renderLandscapeBars(data, heading, ariaLabel) {
|
|
22456
22555
|
if (data.length <= 1) return "";
|
|
22457
|
-
const
|
|
22556
|
+
const max2 = Math.max(...data.map((d) => d.count), 1);
|
|
22458
22557
|
const width = 600;
|
|
22459
22558
|
const height = data.length * 28 + 16;
|
|
22460
22559
|
const labelWidth = 160;
|
|
22461
22560
|
const bars = data.map((d, i) => {
|
|
22462
22561
|
const y = i * 28 + 8;
|
|
22463
22562
|
const barHeight = 18;
|
|
22464
|
-
const w = d.count /
|
|
22563
|
+
const w = d.count / max2 * (width - labelWidth - 60);
|
|
22465
22564
|
const color = d.isProject ? COLORS.accent : COLORS.series[(i + 1) % COLORS.series.length];
|
|
22466
22565
|
return `
|
|
22467
22566
|
<text x="${labelWidth - 8}" y="${y + 13}" fill="${COLORS.textMuted}" font-size="11" text-anchor="end">${escapeHtml(d.label)}</text>
|
|
@@ -22552,9 +22651,9 @@ function renderCategoryBars(buckets) {
|
|
|
22552
22651
|
if (buckets.length === 0) return "";
|
|
22553
22652
|
const total = buckets.reduce((s, b) => s + b.count, 0);
|
|
22554
22653
|
if (total === 0) return "";
|
|
22555
|
-
const
|
|
22654
|
+
const max2 = Math.max(...buckets.map((b) => b.count), 1);
|
|
22556
22655
|
const rows = buckets.map((b) => {
|
|
22557
|
-
const pct = b.count /
|
|
22656
|
+
const pct = b.count / max2 * 100;
|
|
22558
22657
|
const tone = SOURCE_CATEGORY_TONE[b.category] ?? "neutral";
|
|
22559
22658
|
const color = tone === "negative" ? COLORS.negative : tone === "caution" ? COLORS.caution : COLORS.accent;
|
|
22560
22659
|
return `
|
|
@@ -22630,11 +22729,11 @@ function renderLineChart(points, color, title, height = 200) {
|
|
|
22630
22729
|
const padY = 24;
|
|
22631
22730
|
const usableW = width - padX * 2;
|
|
22632
22731
|
const usableH = height - padY * 2;
|
|
22633
|
-
const
|
|
22732
|
+
const max2 = Math.max(...points.map((p) => p.y), 1);
|
|
22634
22733
|
const stepX = points.length > 1 ? usableW / (points.length - 1) : 0;
|
|
22635
22734
|
const xy = points.map((p, i) => ({
|
|
22636
22735
|
x: padX + i * stepX,
|
|
22637
|
-
y: padY + usableH - p.y /
|
|
22736
|
+
y: padY + usableH - p.y / max2 * usableH,
|
|
22638
22737
|
raw: p
|
|
22639
22738
|
}));
|
|
22640
22739
|
const path8 = xy.map((p, i) => `${i === 0 ? "M" : "L"} ${p.x.toFixed(1)} ${p.y.toFixed(1)}`).join(" ");
|
|
@@ -22647,7 +22746,7 @@ function renderLineChart(points, color, title, height = 200) {
|
|
|
22647
22746
|
<h3>${escapeHtml(title)}</h3>
|
|
22648
22747
|
<svg viewBox="0 0 ${width} ${height}" width="100%" preserveAspectRatio="xMinYMin meet" role="img" aria-label="${escapeHtml(title)} line chart">
|
|
22649
22748
|
<line x1="${padX}" y1="${padY + usableH}" x2="${padX + usableW}" y2="${padY + usableH}" stroke="${COLORS.border}" stroke-width="1" />
|
|
22650
|
-
<text x="${padX - 6}" y="${(padY + 4).toFixed(1)}" fill="${COLORS.textFaint}" font-size="9" text-anchor="end">${formatNumber(
|
|
22749
|
+
<text x="${padX - 6}" y="${(padY + 4).toFixed(1)}" fill="${COLORS.textFaint}" font-size="9" text-anchor="end">${formatNumber(max2)}</text>
|
|
22651
22750
|
<text x="${padX - 6}" y="${(padY + usableH).toFixed(1)}" fill="${COLORS.textFaint}" font-size="9" text-anchor="end">0</text>
|
|
22652
22751
|
<path d="${path8}" stroke="${color}" stroke-width="2" fill="none" />
|
|
22653
22752
|
${dots}
|
|
@@ -27339,7 +27438,12 @@ function summarizeTransitionsFromSnapshots(latest, previous, since) {
|
|
|
27339
27438
|
return { since, gained, lost, emerging };
|
|
27340
27439
|
}
|
|
27341
27440
|
function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
|
|
27342
|
-
const
|
|
27441
|
+
const suggestionWindow = resolveGscWindowDays(
|
|
27442
|
+
28,
|
|
27443
|
+
readLatestGscDataDate(app.db, projectId),
|
|
27444
|
+
(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)
|
|
27445
|
+
);
|
|
27446
|
+
const cutoff = suggestionWindow.startDate ?? "";
|
|
27343
27447
|
const dimensionedRows = app.db.select({
|
|
27344
27448
|
date: gscSearchData.date,
|
|
27345
27449
|
query: gscSearchData.query,
|
|
@@ -27352,11 +27456,14 @@ function buildSuggestedQueriesFromGsc(app, projectId, trackedQueries) {
|
|
|
27352
27456
|
}).from(gscSearchData).where(and28(
|
|
27353
27457
|
eq34(gscSearchData.projectId, projectId),
|
|
27354
27458
|
sql12`${gscSearchData.date} >= ${cutoff}`,
|
|
27459
|
+
// Same upper bound as the accurate source below. The merge picks a source
|
|
27460
|
+
// PER DAY, so an asymmetric range would let a dimensioned-only day past
|
|
27461
|
+
// the window's end into a basket the other source cannot balance.
|
|
27462
|
+
sql12`${gscSearchData.date} <= ${suggestionWindow.endDate ?? "9999-12-31"}`,
|
|
27355
27463
|
sql12`${gscSearchData.impressions} > 0`
|
|
27356
27464
|
)).groupBy(gscSearchData.date, gscSearchData.query).all();
|
|
27357
|
-
const todayIso = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
27358
27465
|
const merged = mergeGscQueryTotalsWithFallback(
|
|
27359
|
-
readGscQueryDailyRows(app.db, projectId, cutoff,
|
|
27466
|
+
readGscQueryDailyRows(app.db, projectId, cutoff, suggestionWindow.endDate ?? "9999-12-31"),
|
|
27360
27467
|
dimensionedRows.map((r) => ({
|
|
27361
27468
|
date: r.date,
|
|
27362
27469
|
query: r.query,
|
|
@@ -30818,6 +30925,12 @@ var routeCatalog = [
|
|
|
30818
30925
|
nameParameter,
|
|
30819
30926
|
{ name: "startDate", in: "query", description: "Filter by start date.", schema: stringSchema },
|
|
30820
30927
|
{ name: "endDate", in: "query", description: "Filter by end date.", schema: stringSchema },
|
|
30928
|
+
{
|
|
30929
|
+
name: "days",
|
|
30930
|
+
in: "query",
|
|
30931
|
+
description: "Relative span in days, resolved server-side against the last published GSC date. Prefer this over client-computed start/end dates: those bypass the published-day anchoring and are pinned to the caller's clock rather than Google's Pacific calendar.",
|
|
30932
|
+
schema: stringSchema
|
|
30933
|
+
},
|
|
30821
30934
|
{ name: "query", in: "query", description: "Filter by search query.", schema: stringSchema },
|
|
30822
30935
|
{ name: "page", in: "query", description: "Filter by page URL.", schema: stringSchema },
|
|
30823
30936
|
{
|
|
@@ -33469,7 +33582,7 @@ var routeCatalog = [
|
|
|
33469
33582
|
method: "post",
|
|
33470
33583
|
path: "/api/v1/projects/{name}/traffic/connect/cloudflare",
|
|
33471
33584
|
summary: "Connect a Cloudflare Worker traffic source",
|
|
33472
|
-
description: "Creates or updates a Cloudflare `direct-push` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration.
|
|
33585
|
+
description: "Creates or updates a Cloudflare `direct-push` or `queue-pull` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration. Direct bearer/HMAC credentials and the Queue API token remain in Canonry's local credential store; none appears in source, TOML, DB config, response, or MCP output. Reconnect is idempotent by source mode. A different mode is staged paused when another project traffic source is active, and requires the explicit activation operation after deployment. Wrangler deploys the Worker without a route; the operator attaches the exact site route manually with Request limit failure mode set to Fail open.",
|
|
33473
33586
|
tags: ["traffic"],
|
|
33474
33587
|
parameters: [nameParameter],
|
|
33475
33588
|
requestBody: {
|
|
@@ -33513,11 +33626,27 @@ var routeCatalog = [
|
|
|
33513
33626
|
429: errorResponse("The per-IP or authenticated-source ingest request budget was exceeded.")
|
|
33514
33627
|
}
|
|
33515
33628
|
},
|
|
33629
|
+
{
|
|
33630
|
+
method: "post",
|
|
33631
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}/activate",
|
|
33632
|
+
summary: "Activate a staged traffic source",
|
|
33633
|
+
description: "Explicit single-team cutover. Atomically pauses every sibling project traffic source and connects the selected Cloudflare, Cloud Run, WordPress, or Vercel source after validating its local credential. Pull delivery creates or repoints the one project traffic-sync schedule; Cloudflare direct push removes it. Deployment and provider-side routing remain separate operator actions.",
|
|
33634
|
+
tags: ["traffic"],
|
|
33635
|
+
parameters: [
|
|
33636
|
+
nameParameter,
|
|
33637
|
+
{ name: "id", in: "path", required: true, description: "Staged traffic source ID.", schema: stringSchema }
|
|
33638
|
+
],
|
|
33639
|
+
responses: {
|
|
33640
|
+
200: jsonResponse("Activated traffic source DTO returned.", "TrafficSourceDto"),
|
|
33641
|
+
400: errorResponse("Source is archived, unsupported, or cannot be activated."),
|
|
33642
|
+
404: errorResponse("Project or traffic source not found.")
|
|
33643
|
+
}
|
|
33644
|
+
},
|
|
33516
33645
|
{
|
|
33517
33646
|
method: "post",
|
|
33518
33647
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
33519
33648
|
summary: "Trigger a sync run for a traffic source",
|
|
33520
|
-
description: "Pulls
|
|
33649
|
+
description: "Pulls from the selected Cloud Run, WordPress, Vercel, or Cloudflare Queue source, classifies crawler hits / user fetches / AI-referral sessions, and commits hourly buckets plus a bounded sample tail. Queue pull uses a durable source lease and acknowledges each Cloudflare message only after its event receipts and rollups commit.",
|
|
33521
33650
|
tags: ["traffic"],
|
|
33522
33651
|
parameters: [
|
|
33523
33652
|
nameParameter,
|
|
@@ -33530,7 +33659,7 @@ var routeCatalog = [
|
|
|
33530
33659
|
schema: {
|
|
33531
33660
|
type: "object",
|
|
33532
33661
|
properties: {
|
|
33533
|
-
sinceMinutes: { ...integerSchema, description: "
|
|
33662
|
+
sinceMinutes: { ...integerSchema, description: "Optional lookback for time-window sources; Cloudflare Queue pull ignores it." }
|
|
33534
33663
|
}
|
|
33535
33664
|
}
|
|
33536
33665
|
}
|
|
@@ -33539,8 +33668,9 @@ var routeCatalog = [
|
|
|
33539
33668
|
responses: {
|
|
33540
33669
|
200: jsonResponse("Sync summary returned.", "TrafficSyncResponse"),
|
|
33541
33670
|
400: errorResponse("Invalid sync request or missing credentials."),
|
|
33671
|
+
409: errorResponse("Another Queue sync currently owns the source lease."),
|
|
33542
33672
|
404: errorResponse("Project or traffic source not found."),
|
|
33543
|
-
502: errorResponse("Upstream
|
|
33673
|
+
502: errorResponse("Upstream pull, acknowledgement, or credential resolution failed.")
|
|
33544
33674
|
}
|
|
33545
33675
|
},
|
|
33546
33676
|
{
|
|
@@ -34966,8 +35096,8 @@ var SEVERITY_ICONS = {
|
|
|
34966
35096
|
[AlertSeverities.success]: "\u2705",
|
|
34967
35097
|
[AlertSeverities.info]: "\u{1F4CA}"
|
|
34968
35098
|
};
|
|
34969
|
-
function clamp(value,
|
|
34970
|
-
return value.length <=
|
|
35099
|
+
function clamp(value, max2) {
|
|
35100
|
+
return value.length <= max2 ? value : `${value.slice(0, max2 - 1)}\u2026`;
|
|
34971
35101
|
}
|
|
34972
35102
|
function absoluteLink(url) {
|
|
34973
35103
|
if (!url) return void 0;
|
|
@@ -35356,6 +35486,7 @@ var INDEXING_SCOPE = "https://www.googleapis.com/auth/indexing";
|
|
|
35356
35486
|
var GSC_API_BASE = "https://www.googleapis.com/webmasters/v3";
|
|
35357
35487
|
var URL_INSPECTION_API = "https://searchconsole.googleapis.com/v1/urlInspection/index:inspect";
|
|
35358
35488
|
var GSC_MAX_ROWS_PER_REQUEST = 25e3;
|
|
35489
|
+
var GSC_REPORTING_TIME_ZONE = "America/Los_Angeles";
|
|
35359
35490
|
var GSC_DATA_LAG_DAYS = 3;
|
|
35360
35491
|
var INDEXING_API_BASE = "https://indexing.googleapis.com/v3";
|
|
35361
35492
|
var INDEXING_API_DAILY_LIMIT = 200;
|
|
@@ -36399,11 +36530,11 @@ async function fetchDailyTotals(accessToken, propertyId, days) {
|
|
|
36399
36530
|
ga4Log("info", "fetch-daily-totals.done", { propertyId, days: syncDays, rows: rows.length });
|
|
36400
36531
|
return rows;
|
|
36401
36532
|
}
|
|
36402
|
-
var
|
|
36533
|
+
var WINDOW_DAYS2 = { "7d": 7, "30d": 30, "90d": 90 };
|
|
36403
36534
|
async function fetchWindowSummary(accessToken, propertyId, windowKey) {
|
|
36404
36535
|
validateAccessToken2(accessToken);
|
|
36405
36536
|
validatePropertyId(propertyId);
|
|
36406
|
-
const days =
|
|
36537
|
+
const days = WINDOW_DAYS2[windowKey];
|
|
36407
36538
|
if (!days) {
|
|
36408
36539
|
throw new GA4ApiError(`Unsupported windowKey "${windowKey}" \u2014 must be 7d, 30d, or 90d`, 400);
|
|
36409
36540
|
}
|
|
@@ -36978,6 +37109,26 @@ function stableStringify3(value) {
|
|
|
36978
37109
|
}
|
|
36979
37110
|
|
|
36980
37111
|
// ../api-routes/src/google.ts
|
|
37112
|
+
function resolveReportedWindow(resolved, startDate, endDate) {
|
|
37113
|
+
const start = startDate ?? resolved.startDate;
|
|
37114
|
+
const end = endDate ?? resolved.endDate;
|
|
37115
|
+
if (start === null || end === null || start <= end) {
|
|
37116
|
+
return { ...resolved, startDate: start, endDate: end };
|
|
37117
|
+
}
|
|
37118
|
+
return {
|
|
37119
|
+
...resolved,
|
|
37120
|
+
startDate: startDate ? start : null,
|
|
37121
|
+
endDate: endDate ? end : null
|
|
37122
|
+
};
|
|
37123
|
+
}
|
|
37124
|
+
function assertForwardRange(startDate, endDate) {
|
|
37125
|
+
if (startDate && endDate && startDate > endDate) {
|
|
37126
|
+
throw validationError(`startDate "${startDate}" is after endDate "${endDate}".`);
|
|
37127
|
+
}
|
|
37128
|
+
}
|
|
37129
|
+
function gscToday() {
|
|
37130
|
+
return formatIsoDateInTimeZone((/* @__PURE__ */ new Date()).toISOString(), GSC_REPORTING_TIME_ZONE);
|
|
37131
|
+
}
|
|
36981
37132
|
var GOOGLE_OAUTH_COMPLETE_MESSAGE = "canonry:google-oauth-complete";
|
|
36982
37133
|
function googleOAuthSuccessHtml(type) {
|
|
36983
37134
|
const message = JSON.stringify({
|
|
@@ -37408,11 +37559,19 @@ async function googleRoutes(app, opts) {
|
|
|
37408
37559
|
);
|
|
37409
37560
|
}
|
|
37410
37561
|
const orderBy = parsedOrderBy.data;
|
|
37411
|
-
|
|
37562
|
+
assertForwardRange(startDate, endDate);
|
|
37563
|
+
const daysParam = request.query.days === void 0 ? null : Number(request.query.days);
|
|
37564
|
+
if (daysParam !== null && (!Number.isInteger(daysParam) || daysParam < 1)) {
|
|
37565
|
+
throw validationError('"days" must be a positive integer');
|
|
37566
|
+
}
|
|
37567
|
+
const latestDataDate = readLatestGscDataDate(app.db, project.id);
|
|
37568
|
+
const resolvedWindow = daysParam === null ? resolveGscWindowRange(parseWindow(request.query.window), latestDataDate, gscToday()) : resolveGscWindowDays(daysParam, latestDataDate, gscToday());
|
|
37569
|
+
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
37570
|
+
const effectiveEndDate = endDate ?? (daysParam === null ? void 0 : resolvedWindow.endDate ?? void 0);
|
|
37412
37571
|
const conditions = [eq39(gscSearchData.projectId, project.id)];
|
|
37413
37572
|
if (startDate) conditions.push(sql13`${gscSearchData.date} >= ${startDate}`);
|
|
37414
37573
|
else if (cutoffDate) conditions.push(sql13`${gscSearchData.date} >= ${cutoffDate}`);
|
|
37415
|
-
if (
|
|
37574
|
+
if (effectiveEndDate) conditions.push(sql13`${gscSearchData.date} <= ${effectiveEndDate}`);
|
|
37416
37575
|
if (query) conditions.push(sql13`${gscSearchData.query} LIKE ${"%" + escapeLikePattern(query) + "%"} ESCAPE '\\'`);
|
|
37417
37576
|
if (page) conditions.push(sql13`${gscSearchData.page} LIKE ${"%" + escapeLikePattern(page) + "%"} ESCAPE '\\'`);
|
|
37418
37577
|
const limitVal = Math.max(parseInt(limit ?? "500", 10) || 0, 1);
|
|
@@ -37449,9 +37608,15 @@ async function googleRoutes(app, opts) {
|
|
|
37449
37608
|
app.get("/projects/:name/google/gsc/performance/daily", async (request) => {
|
|
37450
37609
|
const project = resolveProject(app.db, request.params.name);
|
|
37451
37610
|
const { startDate, endDate } = request.query;
|
|
37452
|
-
|
|
37611
|
+
assertForwardRange(startDate, endDate);
|
|
37612
|
+
const resolvedWindow = resolveGscWindowRange(
|
|
37613
|
+
parseWindow(request.query.window),
|
|
37614
|
+
readLatestGscDataDate(app.db, project.id),
|
|
37615
|
+
gscToday()
|
|
37616
|
+
);
|
|
37617
|
+
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
37453
37618
|
const windowStart = startDate ?? cutoffDate ?? "";
|
|
37454
|
-
const windowEnd = endDate ?? "9999-12-31";
|
|
37619
|
+
const windowEnd = endDate ?? resolvedWindow.endDate ?? "9999-12-31";
|
|
37455
37620
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
37456
37621
|
const conditions = [eq39(gscSearchData.projectId, project.id)];
|
|
37457
37622
|
if (startDate) conditions.push(sql13`${gscSearchData.date} >= ${startDate}`);
|
|
@@ -37485,13 +37650,30 @@ async function googleRoutes(app, opts) {
|
|
|
37485
37650
|
ctr: totalImpressions > 0 ? totalClicks / totalImpressions : 0,
|
|
37486
37651
|
days: daily.length
|
|
37487
37652
|
},
|
|
37488
|
-
daily
|
|
37653
|
+
daily,
|
|
37654
|
+
// The period actually returned. An explicit start/end wins over the
|
|
37655
|
+
// label, so echo what was used rather than what the window would have
|
|
37656
|
+
// chosen — a caller must be able to label the data it got.
|
|
37657
|
+
//
|
|
37658
|
+
// Mixing one explicit bound with the computed opposite one can invert the
|
|
37659
|
+
// range (an explicit `startDate` of 2030-01-01 against a computed
|
|
37660
|
+
// `endDate` of 2026-01-06), which would describe a period that cannot
|
|
37661
|
+
// contain the rows beside it. When only one side is given, the other is
|
|
37662
|
+
// dropped rather than reported reversed: an absent bound is honest about
|
|
37663
|
+
// being unspecified, a reversed pair is not.
|
|
37664
|
+
window: resolveReportedWindow(resolvedWindow, startDate, endDate)
|
|
37489
37665
|
};
|
|
37490
37666
|
});
|
|
37491
37667
|
app.get("/projects/:name/google/gsc/top-pages", async (request) => {
|
|
37492
37668
|
const project = resolveProject(app.db, request.params.name);
|
|
37493
37669
|
const { startDate, endDate, limit } = request.query;
|
|
37494
|
-
|
|
37670
|
+
assertForwardRange(startDate, endDate);
|
|
37671
|
+
const resolvedWindow = resolveGscWindowRange(
|
|
37672
|
+
parseWindow(request.query.window),
|
|
37673
|
+
readLatestGscDataDate(app.db, project.id),
|
|
37674
|
+
gscToday()
|
|
37675
|
+
);
|
|
37676
|
+
const cutoffDate = startDate ? null : resolvedWindow.startDate;
|
|
37495
37677
|
const conditions = [eq39(gscSearchData.projectId, project.id)];
|
|
37496
37678
|
if (startDate) conditions.push(sql13`${gscSearchData.date} >= ${startDate}`);
|
|
37497
37679
|
else if (cutoffDate) conditions.push(sql13`${gscSearchData.date} >= ${cutoffDate}`);
|
|
@@ -37503,7 +37685,7 @@ async function googleRoutes(app, opts) {
|
|
|
37503
37685
|
impressions: sql13`COALESCE(SUM(${gscSearchData.impressions}), 0)`
|
|
37504
37686
|
}).from(gscSearchData).where(and30(...conditions)).groupBy(gscSearchData.page).orderBy(desc17(sql13`SUM(${gscSearchData.clicks})`), desc17(sql13`SUM(${gscSearchData.impressions})`)).limit(limitVal).all();
|
|
37505
37687
|
const windowStart = startDate ?? cutoffDate ?? "";
|
|
37506
|
-
const windowEnd = endDate ?? "9999-12-31";
|
|
37688
|
+
const windowEnd = endDate ?? resolvedWindow.endDate ?? "9999-12-31";
|
|
37507
37689
|
const dailyTotals = readGscDailyTotals(app.db, project.id, windowStart, windowEnd);
|
|
37508
37690
|
const totalClicks = dailyTotals.reduce((sum, d) => sum + d.clicks, 0);
|
|
37509
37691
|
const totalImpressions = dailyTotals.reduce((sum, d) => sum + d.impressions, 0);
|
|
@@ -41728,7 +41910,7 @@ function assertGenericReconciliationKind(row) {
|
|
|
41728
41910
|
}
|
|
41729
41911
|
function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, policy, enforceBackoff) {
|
|
41730
41912
|
const nowIso = now.toISOString();
|
|
41731
|
-
const
|
|
41913
|
+
const leaseExpiresAt2 = new Date(now.getTime() + leaseMs).toISOString();
|
|
41732
41914
|
const pendingCutoff = new Date(now.getTime() - policy.pendingMinIdleMs).toISOString();
|
|
41733
41915
|
const backoffMs = policy.backoffBaseMs * 2 ** Math.max(0, row.reconcileAttempts - 1);
|
|
41734
41916
|
const unknownCutoff = new Date(now.getTime() - backoffMs).toISOString();
|
|
@@ -41745,7 +41927,7 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
41745
41927
|
const claimed = app.db.update(adsOperations).set({
|
|
41746
41928
|
state: AdsOperationStates.reconciling,
|
|
41747
41929
|
leaseOwner,
|
|
41748
|
-
leaseExpiresAt,
|
|
41930
|
+
leaseExpiresAt: leaseExpiresAt2,
|
|
41749
41931
|
reconcileAttempts: sql15`${adsOperations.reconcileAttempts} + 1`,
|
|
41750
41932
|
updatedAt: nowIso
|
|
41751
41933
|
}).where(and32(
|
|
@@ -47936,12 +48118,16 @@ async function backlinksRoutes(app, opts) {
|
|
|
47936
48118
|
// ../api-routes/src/traffic.ts
|
|
47937
48119
|
import crypto40 from "crypto";
|
|
47938
48120
|
import { isIP } from "net";
|
|
48121
|
+
import { isDeepStrictEqual } from "util";
|
|
47939
48122
|
import { Agent as UndiciAgent } from "undici";
|
|
47940
|
-
import { and as
|
|
48123
|
+
import { and as and40, desc as desc22, eq as eq48, gte as gte11, lte as lte11, sql as sql19 } from "drizzle-orm";
|
|
47941
48124
|
|
|
47942
48125
|
// ../api-routes/src/traffic-limits.ts
|
|
47943
48126
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
47944
48127
|
var DEFAULT_VERCEL_SYNC_DEADLINE_MS = 4 * 6e4;
|
|
48128
|
+
var CLOUDFLARE_QUEUE_BATCH_SIZE = 100;
|
|
48129
|
+
var DEFAULT_CLOUDFLARE_QUEUE_MAX_BATCHES = 10;
|
|
48130
|
+
var DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET = CLOUDFLARE_QUEUE_BATCH_SIZE * DEFAULT_CLOUDFLARE_QUEUE_MAX_BATCHES;
|
|
47945
48131
|
var MIN_VERCEL_SYNC_DEADLINE_MS = 3e4;
|
|
47946
48132
|
var MAX_VERCEL_SYNC_DEADLINE_MS = 15 * 6e4;
|
|
47947
48133
|
var TRAFFIC_SOURCE_MAX_CATCHUP_MS = {
|
|
@@ -51869,7 +52055,8 @@ var CLOUDFLARE_WORKER_BINDINGS = {
|
|
|
51869
52055
|
ingestUrl: "CANONRY_INGEST_URL",
|
|
51870
52056
|
workerVersion: "CANONRY_WORKER_VERSION",
|
|
51871
52057
|
bearerToken: "CANONRY_BEARER_TOKEN",
|
|
51872
|
-
hmacSecret: "CANONRY_HMAC_SECRET"
|
|
52058
|
+
hmacSecret: "CANONRY_HMAC_SECRET",
|
|
52059
|
+
trafficQueue: "CANONRY_TRAFFIC_QUEUE"
|
|
51873
52060
|
};
|
|
51874
52061
|
var CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS = [
|
|
51875
52062
|
CLOUDFLARE_WORKER_BINDINGS.bearerToken,
|
|
@@ -51910,6 +52097,110 @@ function jsArray(values) {
|
|
|
51910
52097
|
function generateWorkerScript(opts) {
|
|
51911
52098
|
const botScoreMax = opts.botScoreMaxForward ?? DEFAULT_BOT_SCORE_MAX_FORWARD;
|
|
51912
52099
|
const canonicalJsonFunction = canonicalizeCloudflareJson.toString();
|
|
52100
|
+
const directPushDelivery = opts.deliveryMode === "direct-push" ? `
|
|
52101
|
+
function toHex(buffer) {
|
|
52102
|
+
const bytes = new Uint8Array(buffer)
|
|
52103
|
+
let out = ''
|
|
52104
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
52105
|
+
out += bytes[i].toString(16).padStart(2, '0')
|
|
52106
|
+
}
|
|
52107
|
+
return out
|
|
52108
|
+
}
|
|
52109
|
+
|
|
52110
|
+
async function signBody(secret, timestamp, body) {
|
|
52111
|
+
const key = await crypto.subtle.importKey(
|
|
52112
|
+
'raw',
|
|
52113
|
+
new TextEncoder().encode(secret),
|
|
52114
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
52115
|
+
false,
|
|
52116
|
+
['sign'],
|
|
52117
|
+
)
|
|
52118
|
+
const sig = await crypto.subtle.sign(
|
|
52119
|
+
'HMAC',
|
|
52120
|
+
key,
|
|
52121
|
+
new TextEncoder().encode(timestamp + '.' + body),
|
|
52122
|
+
)
|
|
52123
|
+
return toHex(sig)
|
|
52124
|
+
}
|
|
52125
|
+
|
|
52126
|
+
function isRetryableStatus(status) {
|
|
52127
|
+
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
52128
|
+
}
|
|
52129
|
+
|
|
52130
|
+
function sleep(ms) {
|
|
52131
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
52132
|
+
}
|
|
52133
|
+
|
|
52134
|
+
async function cancelUnusedResponseBody(response) {
|
|
52135
|
+
try {
|
|
52136
|
+
if (response.body) await response.body.cancel()
|
|
52137
|
+
} catch (_) {
|
|
52138
|
+
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
52139
|
+
}
|
|
52140
|
+
}
|
|
52141
|
+
|
|
52142
|
+
async function withRetry(operation) {
|
|
52143
|
+
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
52144
|
+
let response
|
|
52145
|
+
try {
|
|
52146
|
+
response = await operation()
|
|
52147
|
+
} catch (error) {
|
|
52148
|
+
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
52149
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52150
|
+
continue
|
|
52151
|
+
}
|
|
52152
|
+
|
|
52153
|
+
const status = response.status
|
|
52154
|
+
const ok = response.ok
|
|
52155
|
+
await cancelUnusedResponseBody(response)
|
|
52156
|
+
if (ok) return
|
|
52157
|
+
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
52158
|
+
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
52159
|
+
}
|
|
52160
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52161
|
+
}
|
|
52162
|
+
}
|
|
52163
|
+
` : "";
|
|
52164
|
+
const directPushDeliveryAdapter = opts.deliveryMode === "direct-push" ? `
|
|
52165
|
+
async function deliverViaDirectPush(env, batch) {
|
|
52166
|
+
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
52167
|
+
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
52168
|
+
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
52169
|
+
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
52170
|
+
const body = canonicalizeJson(batch)
|
|
52171
|
+
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
52172
|
+
const signature = await signBody(hmacSecret, timestamp, body)
|
|
52173
|
+
|
|
52174
|
+
await withRetry(() => fetch(ingestUrl, {
|
|
52175
|
+
method: 'POST',
|
|
52176
|
+
headers: {
|
|
52177
|
+
'content-type': 'application/json',
|
|
52178
|
+
'Authorization': 'Bearer ' + bearerToken,
|
|
52179
|
+
'X-Canonry-Timestamp': timestamp,
|
|
52180
|
+
'X-Canonry-Signature': signature,
|
|
52181
|
+
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
52182
|
+
'X-Canonry-Source-Id': sourceId,
|
|
52183
|
+
},
|
|
52184
|
+
body,
|
|
52185
|
+
}))
|
|
52186
|
+
}
|
|
52187
|
+
` : "";
|
|
52188
|
+
const queuePullDeliveryAdapter = opts.deliveryMode === "queue-pull" ? `
|
|
52189
|
+
function requireQueueBinding(value, name) {
|
|
52190
|
+
if (value && typeof value.send === 'function') return value
|
|
52191
|
+
throw new Error('Missing required Worker binding: ' + name)
|
|
52192
|
+
}
|
|
52193
|
+
|
|
52194
|
+
async function deliverViaQueue(env, batch) {
|
|
52195
|
+
const queue = requireQueueBinding(env.${CLOUDFLARE_WORKER_BINDINGS.trafficQueue}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.trafficQueue)})
|
|
52196
|
+
await queue.send(batch, { contentType: 'json' })
|
|
52197
|
+
}
|
|
52198
|
+
` : "";
|
|
52199
|
+
const deliveryModeBranch = opts.deliveryMode === "direct-push" ? `if (deliveryMode === 'direct-push') {
|
|
52200
|
+
return deliverViaDirectPush(env, batch)
|
|
52201
|
+
}` : `if (deliveryMode === 'queue-pull') {
|
|
52202
|
+
return deliverViaQueue(env, batch)
|
|
52203
|
+
}`;
|
|
51913
52204
|
return `${CLOUDFLARE_WORKER_GENERATED_MARKER}
|
|
51914
52205
|
// worker version: ${opts.workerVersion}
|
|
51915
52206
|
// bot-list version: ${opts.botList.version}
|
|
@@ -51999,68 +52290,7 @@ function shouldForward(request) {
|
|
|
51999
52290
|
return botSignals(request.cf)
|
|
52000
52291
|
}
|
|
52001
52292
|
|
|
52002
|
-
|
|
52003
|
-
const bytes = new Uint8Array(buffer)
|
|
52004
|
-
let out = ''
|
|
52005
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
52006
|
-
out += bytes[i].toString(16).padStart(2, '0')
|
|
52007
|
-
}
|
|
52008
|
-
return out
|
|
52009
|
-
}
|
|
52010
|
-
|
|
52011
|
-
async function signBody(secret, timestamp, body) {
|
|
52012
|
-
const key = await crypto.subtle.importKey(
|
|
52013
|
-
'raw',
|
|
52014
|
-
new TextEncoder().encode(secret),
|
|
52015
|
-
{ name: 'HMAC', hash: 'SHA-256' },
|
|
52016
|
-
false,
|
|
52017
|
-
['sign'],
|
|
52018
|
-
)
|
|
52019
|
-
const sig = await crypto.subtle.sign(
|
|
52020
|
-
'HMAC',
|
|
52021
|
-
key,
|
|
52022
|
-
new TextEncoder().encode(timestamp + '.' + body),
|
|
52023
|
-
)
|
|
52024
|
-
return toHex(sig)
|
|
52025
|
-
}
|
|
52026
|
-
|
|
52027
|
-
function isRetryableStatus(status) {
|
|
52028
|
-
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
52029
|
-
}
|
|
52030
|
-
|
|
52031
|
-
function sleep(ms) {
|
|
52032
|
-
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
52033
|
-
}
|
|
52034
|
-
|
|
52035
|
-
async function cancelUnusedResponseBody(response) {
|
|
52036
|
-
try {
|
|
52037
|
-
if (response.body) await response.body.cancel()
|
|
52038
|
-
} catch (_) {
|
|
52039
|
-
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
52040
|
-
}
|
|
52041
|
-
}
|
|
52042
|
-
|
|
52043
|
-
async function withRetry(operation) {
|
|
52044
|
-
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
52045
|
-
let response
|
|
52046
|
-
try {
|
|
52047
|
-
response = await operation()
|
|
52048
|
-
} catch (error) {
|
|
52049
|
-
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
52050
|
-
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52051
|
-
continue
|
|
52052
|
-
}
|
|
52053
|
-
|
|
52054
|
-
const status = response.status
|
|
52055
|
-
const ok = response.ok
|
|
52056
|
-
await cancelUnusedResponseBody(response)
|
|
52057
|
-
if (ok) return
|
|
52058
|
-
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
52059
|
-
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
52060
|
-
}
|
|
52061
|
-
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52062
|
-
}
|
|
52063
|
-
}
|
|
52293
|
+
${directPushDelivery}
|
|
52064
52294
|
|
|
52065
52295
|
function pickCf(cf) {
|
|
52066
52296
|
if (!cf) return null
|
|
@@ -52099,34 +52329,11 @@ function buildEdgeEventBatch(env, request, status, observedAt) {
|
|
|
52099
52329
|
}
|
|
52100
52330
|
}
|
|
52101
52331
|
|
|
52102
|
-
|
|
52103
|
-
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
52104
|
-
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
52105
|
-
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
52106
|
-
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
52107
|
-
const body = canonicalizeJson(batch)
|
|
52108
|
-
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
52109
|
-
const signature = await signBody(hmacSecret, timestamp, body)
|
|
52110
|
-
|
|
52111
|
-
await withRetry(() => fetch(ingestUrl, {
|
|
52112
|
-
method: 'POST',
|
|
52113
|
-
headers: {
|
|
52114
|
-
'content-type': 'application/json',
|
|
52115
|
-
'Authorization': 'Bearer ' + bearerToken,
|
|
52116
|
-
'X-Canonry-Timestamp': timestamp,
|
|
52117
|
-
'X-Canonry-Signature': signature,
|
|
52118
|
-
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
52119
|
-
'X-Canonry-Source-Id': sourceId,
|
|
52120
|
-
},
|
|
52121
|
-
body,
|
|
52122
|
-
}))
|
|
52123
|
-
}
|
|
52332
|
+
${directPushDeliveryAdapter}${queuePullDeliveryAdapter}
|
|
52124
52333
|
|
|
52125
52334
|
async function deliverEdgeEventBatch(env, batch) {
|
|
52126
52335
|
const deliveryMode = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.deliveryMode}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.deliveryMode)})
|
|
52127
|
-
|
|
52128
|
-
return deliverViaDirectPush(env, batch)
|
|
52129
|
-
}
|
|
52336
|
+
${deliveryModeBranch}
|
|
52130
52337
|
throw new Error('Unsupported Canonry delivery mode: ' + deliveryMode)
|
|
52131
52338
|
}
|
|
52132
52339
|
|
|
@@ -52200,6 +52407,31 @@ function generateWranglerToml(opts) {
|
|
|
52200
52407
|
` : "";
|
|
52201
52408
|
const zoneHint = opts.zoneId ? `# Target zone id: ${jsString(opts.zoneId)}
|
|
52202
52409
|
` : "# Target zone id was not provided. Select the canonical site zone.\n";
|
|
52410
|
+
if (opts.deliveryMode === "queue-pull") {
|
|
52411
|
+
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
52412
|
+
name = "canonry-traffic-${opts.sourceId}"
|
|
52413
|
+
${accountConfig}main = "worker.js"
|
|
52414
|
+
compatibility_date = "${WORKER_COMPATIBILITY_DATE}"
|
|
52415
|
+
workers_dev = false
|
|
52416
|
+
|
|
52417
|
+
[vars]
|
|
52418
|
+
${CLOUDFLARE_WORKER_BINDINGS.deliveryMode} = ${jsString(opts.deliveryMode)}
|
|
52419
|
+
${CLOUDFLARE_WORKER_BINDINGS.sourceId} = ${jsString(opts.sourceId)}
|
|
52420
|
+
${CLOUDFLARE_WORKER_BINDINGS.workerVersion} = ${jsString(opts.workerVersion)}
|
|
52421
|
+
|
|
52422
|
+
[[queues.producers]]
|
|
52423
|
+
queue = ${jsString(opts.queueName)}
|
|
52424
|
+
binding = ${jsString(CLOUDFLARE_WORKER_BINDINGS.trafficQueue)}
|
|
52425
|
+
|
|
52426
|
+
# Deploy this Worker via:
|
|
52427
|
+
# wrangler deploy
|
|
52428
|
+
# Canonry intentionally does not declare a route in this file.
|
|
52429
|
+
# After deploy, attach this exact route in the Cloudflare dashboard:
|
|
52430
|
+
# ${hostname}/*
|
|
52431
|
+
${zoneHint}# Set the route Request limit failure mode to Fail open before activation.
|
|
52432
|
+
# Wrangler cannot configure this route toggle.
|
|
52433
|
+
`;
|
|
52434
|
+
}
|
|
52203
52435
|
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
52204
52436
|
name = "canonry-traffic-${opts.sourceId}"
|
|
52205
52437
|
${accountConfig}main = "worker.js"
|
|
@@ -52277,6 +52509,365 @@ function verifyRequestSignature(opts) {
|
|
|
52277
52509
|
return { ok: false, reason: "signature_mismatch" };
|
|
52278
52510
|
}
|
|
52279
52511
|
|
|
52512
|
+
// ../integration-cloudflare-queue/src/client.ts
|
|
52513
|
+
var DEFAULT_API_BASE_URL = "https://api.cloudflare.com/client/v4";
|
|
52514
|
+
var DEFAULT_TIMEOUT_MS4 = 3e4;
|
|
52515
|
+
var DEFAULT_MAX_RETRIES2 = 3;
|
|
52516
|
+
var DEFAULT_RETRY_BASE_DELAY_MS = 1e3;
|
|
52517
|
+
var MAX_RETRY_DELAY_MS = 3e4;
|
|
52518
|
+
var MAX_PULL_BATCH_SIZE = 100;
|
|
52519
|
+
var MAX_VISIBILITY_TIMEOUT_MS = 12 * 60 * 6e4;
|
|
52520
|
+
var CloudflareQueueApiError = class extends Error {
|
|
52521
|
+
constructor(message, status, retryAfter) {
|
|
52522
|
+
super(message);
|
|
52523
|
+
this.status = status;
|
|
52524
|
+
this.retryAfter = retryAfter;
|
|
52525
|
+
this.name = "CloudflareQueueApiError";
|
|
52526
|
+
}
|
|
52527
|
+
status;
|
|
52528
|
+
retryAfter;
|
|
52529
|
+
};
|
|
52530
|
+
function required(value, label) {
|
|
52531
|
+
const trimmed = value.trim();
|
|
52532
|
+
if (!trimmed) throw new CloudflareQueueApiError(`${label} is required`, 400);
|
|
52533
|
+
return trimmed;
|
|
52534
|
+
}
|
|
52535
|
+
function positiveInteger(value, label, maximum) {
|
|
52536
|
+
if (!Number.isInteger(value) || value < 1 || maximum !== void 0 && value > maximum) {
|
|
52537
|
+
const range = maximum === void 0 ? "a positive integer" : `an integer from 1 to ${maximum}`;
|
|
52538
|
+
throw new CloudflareQueueApiError(`${label} must be ${range}`, 400);
|
|
52539
|
+
}
|
|
52540
|
+
return value;
|
|
52541
|
+
}
|
|
52542
|
+
function nonNegativeInteger(value, label) {
|
|
52543
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
52544
|
+
throw new CloudflareQueueApiError(`${label} must be a non-negative integer`, 502);
|
|
52545
|
+
}
|
|
52546
|
+
return value;
|
|
52547
|
+
}
|
|
52548
|
+
function resolveOptions(options) {
|
|
52549
|
+
const apiBaseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
|
|
52550
|
+
try {
|
|
52551
|
+
new URL(apiBaseUrl);
|
|
52552
|
+
} catch {
|
|
52553
|
+
throw new CloudflareQueueApiError("apiBaseUrl must be a valid URL", 400);
|
|
52554
|
+
}
|
|
52555
|
+
return {
|
|
52556
|
+
accountId: required(options.accountId, "accountId"),
|
|
52557
|
+
queueId: required(options.queueId, "queueId"),
|
|
52558
|
+
apiToken: required(options.apiToken, "apiToken"),
|
|
52559
|
+
fetchImpl: options.fetchImpl ?? fetch,
|
|
52560
|
+
apiBaseUrl,
|
|
52561
|
+
timeoutMs: positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS4, "timeoutMs"),
|
|
52562
|
+
maxRetries: nonNegativeInteger(options.maxRetries ?? DEFAULT_MAX_RETRIES2, "maxRetries"),
|
|
52563
|
+
retryBaseDelayMs: positiveInteger(options.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS, "retryBaseDelayMs"),
|
|
52564
|
+
sleep: options.sleep
|
|
52565
|
+
};
|
|
52566
|
+
}
|
|
52567
|
+
function queueMessagesUrl(options, action) {
|
|
52568
|
+
return `${options.apiBaseUrl}/accounts/${encodeURIComponent(options.accountId)}/queues/${encodeURIComponent(options.queueId)}/messages/${action}`;
|
|
52569
|
+
}
|
|
52570
|
+
function requestSignal(timeoutMs, signal) {
|
|
52571
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
52572
|
+
return signal ? AbortSignal.any([timeout, signal]) : timeout;
|
|
52573
|
+
}
|
|
52574
|
+
function isAbortError(error) {
|
|
52575
|
+
return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "TimeoutError";
|
|
52576
|
+
}
|
|
52577
|
+
function isRetryableQueueError(error) {
|
|
52578
|
+
if (isAbortError(error)) return false;
|
|
52579
|
+
if (error instanceof CloudflareQueueApiError) {
|
|
52580
|
+
return error.status === 429 || error.status >= 500;
|
|
52581
|
+
}
|
|
52582
|
+
return true;
|
|
52583
|
+
}
|
|
52584
|
+
function retryAfterDelay(value) {
|
|
52585
|
+
if (!value) return void 0;
|
|
52586
|
+
const trimmed = value.trim();
|
|
52587
|
+
if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1e3;
|
|
52588
|
+
const at = Date.parse(trimmed);
|
|
52589
|
+
return Number.isNaN(at) ? void 0 : Math.max(0, at - Date.now());
|
|
52590
|
+
}
|
|
52591
|
+
async function withQueueRetry(attempt, options) {
|
|
52592
|
+
let lastError;
|
|
52593
|
+
for (let attemptNumber = 0; attemptNumber <= options.maxRetries; attemptNumber += 1) {
|
|
52594
|
+
try {
|
|
52595
|
+
return await attempt();
|
|
52596
|
+
} catch (error) {
|
|
52597
|
+
lastError = error;
|
|
52598
|
+
if (attemptNumber >= options.maxRetries || !isRetryableQueueError(error)) throw error;
|
|
52599
|
+
const defaultDelayMs = options.retryBaseDelayMs * Math.pow(2, attemptNumber);
|
|
52600
|
+
const requestedDelayMs = error instanceof CloudflareQueueApiError ? retryAfterDelay(error.retryAfter) ?? defaultDelayMs : defaultDelayMs;
|
|
52601
|
+
const delayMs = Math.min(requestedDelayMs, MAX_RETRY_DELAY_MS);
|
|
52602
|
+
if (options.sleep) {
|
|
52603
|
+
await options.sleep(delayMs);
|
|
52604
|
+
} else if (delayMs > 0) {
|
|
52605
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
52606
|
+
}
|
|
52607
|
+
}
|
|
52608
|
+
}
|
|
52609
|
+
throw lastError;
|
|
52610
|
+
}
|
|
52611
|
+
async function postJson(options, action, body, signal) {
|
|
52612
|
+
return withQueueRetry(async () => {
|
|
52613
|
+
let response;
|
|
52614
|
+
try {
|
|
52615
|
+
response = await options.fetchImpl(queueMessagesUrl(options, action), {
|
|
52616
|
+
method: "POST",
|
|
52617
|
+
headers: {
|
|
52618
|
+
Authorization: `Bearer ${options.apiToken}`,
|
|
52619
|
+
"Content-Type": "application/json"
|
|
52620
|
+
},
|
|
52621
|
+
body: JSON.stringify(body),
|
|
52622
|
+
signal: requestSignal(options.timeoutMs, signal)
|
|
52623
|
+
});
|
|
52624
|
+
} catch (error) {
|
|
52625
|
+
if (isAbortError(error)) {
|
|
52626
|
+
throw new CloudflareQueueApiError("Cloudflare Queue request timed out or was aborted", 408);
|
|
52627
|
+
}
|
|
52628
|
+
throw new CloudflareQueueApiError("Cloudflare Queue request failed", 503);
|
|
52629
|
+
}
|
|
52630
|
+
if (!response.ok) {
|
|
52631
|
+
throw new CloudflareQueueApiError(
|
|
52632
|
+
`Cloudflare Queue ${action} failed with HTTP ${response.status}`,
|
|
52633
|
+
response.status,
|
|
52634
|
+
response.headers.get("retry-after") ?? void 0
|
|
52635
|
+
);
|
|
52636
|
+
}
|
|
52637
|
+
try {
|
|
52638
|
+
return await response.json();
|
|
52639
|
+
} catch {
|
|
52640
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${action} returned invalid JSON`, 502);
|
|
52641
|
+
}
|
|
52642
|
+
}, {
|
|
52643
|
+
maxRetries: options.maxRetries,
|
|
52644
|
+
retryBaseDelayMs: options.retryBaseDelayMs,
|
|
52645
|
+
sleep: options.sleep
|
|
52646
|
+
});
|
|
52647
|
+
}
|
|
52648
|
+
function asRecord(value, label) {
|
|
52649
|
+
if (value == null || typeof value !== "object" || Array.isArray(value)) {
|
|
52650
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
52651
|
+
}
|
|
52652
|
+
return value;
|
|
52653
|
+
}
|
|
52654
|
+
function requiredString(record, key, label) {
|
|
52655
|
+
const value = record[key];
|
|
52656
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
52657
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
52658
|
+
}
|
|
52659
|
+
return value;
|
|
52660
|
+
}
|
|
52661
|
+
function stringField(record, key, label) {
|
|
52662
|
+
const value = record[key];
|
|
52663
|
+
if (typeof value !== "string") {
|
|
52664
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
52665
|
+
}
|
|
52666
|
+
return value;
|
|
52667
|
+
}
|
|
52668
|
+
function metadata(record) {
|
|
52669
|
+
const value = asRecord(record.metadata, "message metadata");
|
|
52670
|
+
const parsed = {};
|
|
52671
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
52672
|
+
if (typeof entry !== "string") {
|
|
52673
|
+
throw new CloudflareQueueApiError("Cloudflare Queue message metadata is malformed", 502);
|
|
52674
|
+
}
|
|
52675
|
+
parsed[key] = entry;
|
|
52676
|
+
}
|
|
52677
|
+
return parsed;
|
|
52678
|
+
}
|
|
52679
|
+
function decodeBase64(value) {
|
|
52680
|
+
if (!/^(?:[a-z0-9+/]{4})*(?:[a-z0-9+/]{2}==|[a-z0-9+/]{3}=)?$/i.test(value)) {
|
|
52681
|
+
throw new CloudflareQueueApiError("Cloudflare Queue message body is malformed", 502);
|
|
52682
|
+
}
|
|
52683
|
+
return new Uint8Array(Buffer.from(value, "base64"));
|
|
52684
|
+
}
|
|
52685
|
+
function parseJson(value) {
|
|
52686
|
+
try {
|
|
52687
|
+
return JSON.parse(value);
|
|
52688
|
+
} catch {
|
|
52689
|
+
throw new CloudflareQueueApiError("Cloudflare Queue JSON message body is malformed", 502);
|
|
52690
|
+
}
|
|
52691
|
+
}
|
|
52692
|
+
function decodeUtf8(bytes) {
|
|
52693
|
+
try {
|
|
52694
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
52695
|
+
} catch {
|
|
52696
|
+
throw new CloudflareQueueApiError("Cloudflare Queue JSON message body is malformed", 502);
|
|
52697
|
+
}
|
|
52698
|
+
}
|
|
52699
|
+
function decodeJson(value) {
|
|
52700
|
+
try {
|
|
52701
|
+
return JSON.parse(value);
|
|
52702
|
+
} catch {
|
|
52703
|
+
return parseJson(decodeUtf8(decodeBase64(value)));
|
|
52704
|
+
}
|
|
52705
|
+
}
|
|
52706
|
+
function safeMessageBase(record, leaseId) {
|
|
52707
|
+
const rawMetadata = record.metadata;
|
|
52708
|
+
const safeMetadata = {};
|
|
52709
|
+
let metadataIsSafe = rawMetadata != null && typeof rawMetadata === "object" && !Array.isArray(rawMetadata);
|
|
52710
|
+
if (metadataIsSafe) {
|
|
52711
|
+
for (const [key, value] of Object.entries(rawMetadata)) {
|
|
52712
|
+
if (typeof value !== "string") {
|
|
52713
|
+
metadataIsSafe = false;
|
|
52714
|
+
break;
|
|
52715
|
+
}
|
|
52716
|
+
safeMetadata[key] = value;
|
|
52717
|
+
}
|
|
52718
|
+
}
|
|
52719
|
+
return {
|
|
52720
|
+
id: typeof record.id === "string" && record.id.trim() ? record.id : "<malformed>",
|
|
52721
|
+
leaseId,
|
|
52722
|
+
timestampMs: Number.isInteger(record.timestamp_ms) && record.timestamp_ms >= 0 ? record.timestamp_ms : 0,
|
|
52723
|
+
attempts: Number.isInteger(record.attempts) && record.attempts >= 0 ? record.attempts : 0,
|
|
52724
|
+
metadata: metadataIsSafe ? safeMetadata : {}
|
|
52725
|
+
};
|
|
52726
|
+
}
|
|
52727
|
+
function decodeMessage(record) {
|
|
52728
|
+
const leaseId = requiredString(record, "lease_id", "message");
|
|
52729
|
+
let base;
|
|
52730
|
+
let rawBody;
|
|
52731
|
+
try {
|
|
52732
|
+
base = {
|
|
52733
|
+
id: requiredString(record, "id", "message"),
|
|
52734
|
+
leaseId,
|
|
52735
|
+
timestampMs: nonNegativeInteger(record.timestamp_ms, "message timestamp_ms"),
|
|
52736
|
+
attempts: nonNegativeInteger(record.attempts, "message attempts"),
|
|
52737
|
+
metadata: metadata(record)
|
|
52738
|
+
};
|
|
52739
|
+
rawBody = stringField(record, "body", "message");
|
|
52740
|
+
} catch (error) {
|
|
52741
|
+
if (!(error instanceof CloudflareQueueApiError)) throw error;
|
|
52742
|
+
return {
|
|
52743
|
+
...safeMessageBase(record, leaseId),
|
|
52744
|
+
contentType: "poison",
|
|
52745
|
+
reason: "malformed-envelope"
|
|
52746
|
+
};
|
|
52747
|
+
}
|
|
52748
|
+
const contentType = base.metadata["CF-Content-Type"] ?? "json";
|
|
52749
|
+
const poison = (reason) => ({
|
|
52750
|
+
...base,
|
|
52751
|
+
contentType: "poison",
|
|
52752
|
+
reason
|
|
52753
|
+
});
|
|
52754
|
+
if (contentType === "bytes") {
|
|
52755
|
+
try {
|
|
52756
|
+
return { ...base, contentType, body: decodeBase64(rawBody) };
|
|
52757
|
+
} catch {
|
|
52758
|
+
return poison("malformed-body");
|
|
52759
|
+
}
|
|
52760
|
+
}
|
|
52761
|
+
if (contentType === "json") {
|
|
52762
|
+
try {
|
|
52763
|
+
return { ...base, contentType, body: decodeJson(rawBody) };
|
|
52764
|
+
} catch {
|
|
52765
|
+
return poison("malformed-body");
|
|
52766
|
+
}
|
|
52767
|
+
}
|
|
52768
|
+
if (contentType === "text") {
|
|
52769
|
+
return { ...base, contentType, body: rawBody };
|
|
52770
|
+
}
|
|
52771
|
+
return poison("unsupported-content-type");
|
|
52772
|
+
}
|
|
52773
|
+
function parsePullEnvelope(value) {
|
|
52774
|
+
const envelope = asRecord(value, "pull response");
|
|
52775
|
+
if (envelope.success !== true) {
|
|
52776
|
+
throw new CloudflareQueueApiError("Cloudflare Queue pull was not successful", 502);
|
|
52777
|
+
}
|
|
52778
|
+
const result = asRecord(envelope.result, "pull result");
|
|
52779
|
+
const messageBacklogCount = nonNegativeInteger(result.message_backlog_count, "message_backlog_count");
|
|
52780
|
+
if (!Array.isArray(result.messages)) {
|
|
52781
|
+
throw new CloudflareQueueApiError("Cloudflare Queue messages is malformed", 502);
|
|
52782
|
+
}
|
|
52783
|
+
if (result.messages.length > MAX_PULL_BATCH_SIZE) {
|
|
52784
|
+
throw new CloudflareQueueApiError("Cloudflare Queue returned too many messages", 502);
|
|
52785
|
+
}
|
|
52786
|
+
const messages = [];
|
|
52787
|
+
let skippedUnleasedMessageCount = 0;
|
|
52788
|
+
for (const rawMessage of result.messages) {
|
|
52789
|
+
if (rawMessage == null || typeof rawMessage !== "object" || Array.isArray(rawMessage)) {
|
|
52790
|
+
skippedUnleasedMessageCount += 1;
|
|
52791
|
+
continue;
|
|
52792
|
+
}
|
|
52793
|
+
const message = rawMessage;
|
|
52794
|
+
if (typeof message.lease_id !== "string" || !message.lease_id.trim()) {
|
|
52795
|
+
skippedUnleasedMessageCount += 1;
|
|
52796
|
+
continue;
|
|
52797
|
+
}
|
|
52798
|
+
messages.push(decodeMessage(message));
|
|
52799
|
+
}
|
|
52800
|
+
return { messageBacklogCount, messages, skippedUnleasedMessageCount };
|
|
52801
|
+
}
|
|
52802
|
+
function optionalAckCount(record, key) {
|
|
52803
|
+
if (!(key in record)) return void 0;
|
|
52804
|
+
return nonNegativeInteger(record[key], `ack response ${key}`);
|
|
52805
|
+
}
|
|
52806
|
+
function ackWarningCount(value) {
|
|
52807
|
+
if (value == null) return 0;
|
|
52808
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
52809
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement warnings are malformed", 502);
|
|
52810
|
+
}
|
|
52811
|
+
const warnings = value;
|
|
52812
|
+
if (Object.values(warnings).some((warning) => typeof warning !== "string")) {
|
|
52813
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement warnings are malformed", 502);
|
|
52814
|
+
}
|
|
52815
|
+
return Object.keys(warnings).length;
|
|
52816
|
+
}
|
|
52817
|
+
function parseAckEnvelope(value, expectedAckCount, expectedRetryCount) {
|
|
52818
|
+
const envelope = asRecord(value, "ack response");
|
|
52819
|
+
if (envelope.success !== true) {
|
|
52820
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement was not successful", 502);
|
|
52821
|
+
}
|
|
52822
|
+
if (envelope.result == null) return 0;
|
|
52823
|
+
const result = asRecord(envelope.result, "ack result");
|
|
52824
|
+
const ackCount = optionalAckCount(result, "ackCount");
|
|
52825
|
+
const retryCount = optionalAckCount(result, "retryCount");
|
|
52826
|
+
if (ackCount !== void 0 && ackCount !== expectedAckCount || retryCount !== void 0 && retryCount !== expectedRetryCount) {
|
|
52827
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement was incomplete", 502);
|
|
52828
|
+
}
|
|
52829
|
+
return ackWarningCount(result.warnings);
|
|
52830
|
+
}
|
|
52831
|
+
function normalizeRetries(retries) {
|
|
52832
|
+
return (retries ?? []).map((retry) => {
|
|
52833
|
+
const leaseId = required(retry.leaseId, "retry leaseId");
|
|
52834
|
+
if (retry.delaySeconds !== void 0) {
|
|
52835
|
+
nonNegativeInteger(retry.delaySeconds, "retry delaySeconds");
|
|
52836
|
+
}
|
|
52837
|
+
return { leaseId, ...retry.delaySeconds === void 0 ? {} : { delaySeconds: retry.delaySeconds } };
|
|
52838
|
+
});
|
|
52839
|
+
}
|
|
52840
|
+
async function pullCloudflareQueueMessages(client, options = {}) {
|
|
52841
|
+
const resolved = resolveOptions(client);
|
|
52842
|
+
const body = {};
|
|
52843
|
+
if (options.batchSize !== void 0) body.batch_size = positiveInteger(options.batchSize, "batchSize", MAX_PULL_BATCH_SIZE);
|
|
52844
|
+
if (options.visibilityTimeoutMs !== void 0) {
|
|
52845
|
+
body.visibility_timeout_ms = positiveInteger(
|
|
52846
|
+
options.visibilityTimeoutMs,
|
|
52847
|
+
"visibilityTimeoutMs",
|
|
52848
|
+
MAX_VISIBILITY_TIMEOUT_MS
|
|
52849
|
+
);
|
|
52850
|
+
}
|
|
52851
|
+
return parsePullEnvelope(await postJson(resolved, "pull", body, options.signal));
|
|
52852
|
+
}
|
|
52853
|
+
async function ackCloudflareQueueMessages(client, options) {
|
|
52854
|
+
const resolved = resolveOptions(client);
|
|
52855
|
+
const acknowledgedLeaseIds = (options.acks ?? []).map((leaseId) => required(leaseId, "ack leaseId"));
|
|
52856
|
+
const retriedLeaseIds = normalizeRetries(options.retries);
|
|
52857
|
+
if (acknowledgedLeaseIds.length === 0 && retriedLeaseIds.length === 0) {
|
|
52858
|
+
throw new CloudflareQueueApiError("at least one acknowledgement or retry is required", 400);
|
|
52859
|
+
}
|
|
52860
|
+
const envelope = await postJson(resolved, "ack", {
|
|
52861
|
+
acks: acknowledgedLeaseIds.map((lease_id) => ({ lease_id })),
|
|
52862
|
+
retries: retriedLeaseIds.map(({ leaseId, delaySeconds }) => ({
|
|
52863
|
+
lease_id: leaseId,
|
|
52864
|
+
...delaySeconds === void 0 ? {} : { delay_seconds: delaySeconds }
|
|
52865
|
+
}))
|
|
52866
|
+
}, options.signal);
|
|
52867
|
+
const warningCount = parseAckEnvelope(envelope, acknowledgedLeaseIds.length, retriedLeaseIds.length);
|
|
52868
|
+
return { acknowledgedLeaseIds, retriedLeaseIds, warningCount };
|
|
52869
|
+
}
|
|
52870
|
+
|
|
52280
52871
|
// ../api-routes/src/traffic-event-ingest.ts
|
|
52281
52872
|
import crypto39 from "crypto";
|
|
52282
52873
|
import { and as and38, eq as eq46, gte as gte10, lt as lt7, lte as lte9, sql as sql18 } from "drizzle-orm";
|
|
@@ -52297,6 +52888,11 @@ function writeTrafficEventBatch(opts) {
|
|
|
52297
52888
|
let aiUserFetchBucketRows = 0;
|
|
52298
52889
|
let aiReferralBucketRows = 0;
|
|
52299
52890
|
let sampleRows = 0;
|
|
52891
|
+
let selfTrafficExcluded = 0;
|
|
52892
|
+
let crawlerHits = 0;
|
|
52893
|
+
let aiUserFetchHits = 0;
|
|
52894
|
+
let aiReferralHits = 0;
|
|
52895
|
+
let unknownHits = 0;
|
|
52300
52896
|
opts.db.transaction((tx) => {
|
|
52301
52897
|
tx.update(trafficSources).set({ updatedAt: sql18`${trafficSources.updatedAt}` }).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
52302
52898
|
const source = tx.select().from(trafficSources).where(eq46(trafficSources.id, opts.sourceId)).get();
|
|
@@ -52319,6 +52915,11 @@ function writeTrafficEventBatch(opts) {
|
|
|
52319
52915
|
acceptedEvents = claimedEvents.length;
|
|
52320
52916
|
if (claimedEvents.length > 0) {
|
|
52321
52917
|
const report = buildTrafficProbeReport(claimedEvents, { sampleLimit: claimedEvents.length });
|
|
52918
|
+
selfTrafficExcluded = report.totals.selfTrafficExcluded;
|
|
52919
|
+
crawlerHits = report.totals.crawlerHits;
|
|
52920
|
+
aiUserFetchHits = report.totals.aiUserFetchHits;
|
|
52921
|
+
aiReferralHits = report.totals.aiReferralHits;
|
|
52922
|
+
unknownHits = report.totals.unknownHits;
|
|
52322
52923
|
for (const bucket of report.crawlerEventsHourly) {
|
|
52323
52924
|
const status = bucket.status ?? 0;
|
|
52324
52925
|
tx.insert(crawlerEventsHourly).values({
|
|
@@ -52477,6 +53078,11 @@ function writeTrafficEventBatch(opts) {
|
|
|
52477
53078
|
return {
|
|
52478
53079
|
acceptedEvents,
|
|
52479
53080
|
duplicateEvents,
|
|
53081
|
+
selfTrafficExcluded,
|
|
53082
|
+
crawlerHits,
|
|
53083
|
+
aiUserFetchHits,
|
|
53084
|
+
aiReferralHits,
|
|
53085
|
+
unknownHits,
|
|
52480
53086
|
crawlerBucketRows,
|
|
52481
53087
|
aiUserFetchBucketRows,
|
|
52482
53088
|
aiReferralBucketRows,
|
|
@@ -52484,6 +53090,46 @@ function writeTrafficEventBatch(opts) {
|
|
|
52484
53090
|
};
|
|
52485
53091
|
}
|
|
52486
53092
|
|
|
53093
|
+
// ../api-routes/src/traffic-sync-lease.ts
|
|
53094
|
+
import { and as and39, eq as eq47, isNull as isNull4, lte as lte10, or as or10 } from "drizzle-orm";
|
|
53095
|
+
function leaseExpiresAt(now, ttlMs) {
|
|
53096
|
+
const nowMs = Date.parse(now);
|
|
53097
|
+
if (!Number.isFinite(nowMs)) throw new RangeError("Traffic sync lease now must be an ISO instant");
|
|
53098
|
+
if (!Number.isFinite(ttlMs) || ttlMs <= 0) throw new RangeError("Traffic sync lease ttlMs must be positive");
|
|
53099
|
+
return new Date(nowMs + ttlMs).toISOString();
|
|
53100
|
+
}
|
|
53101
|
+
function tryClaimTrafficSyncLease(input) {
|
|
53102
|
+
const expiresAt = leaseExpiresAt(input.now, input.ttlMs);
|
|
53103
|
+
return input.db.transaction((tx) => {
|
|
53104
|
+
const changed = tx.update(trafficSources).set({
|
|
53105
|
+
syncLeaseOwner: input.owner,
|
|
53106
|
+
syncLeaseExpiresAt: expiresAt,
|
|
53107
|
+
updatedAt: input.now
|
|
53108
|
+
}).where(and39(
|
|
53109
|
+
eq47(trafficSources.id, input.sourceId),
|
|
53110
|
+
or10(
|
|
53111
|
+
isNull4(trafficSources.syncLeaseOwner),
|
|
53112
|
+
lte10(trafficSources.syncLeaseExpiresAt, input.now),
|
|
53113
|
+
eq47(trafficSources.syncLeaseOwner, input.owner)
|
|
53114
|
+
)
|
|
53115
|
+
)).run();
|
|
53116
|
+
return changed.changes === 1;
|
|
53117
|
+
}, { behavior: "immediate" });
|
|
53118
|
+
}
|
|
53119
|
+
function releaseTrafficSyncLease(input) {
|
|
53120
|
+
return input.db.transaction((tx) => {
|
|
53121
|
+
const changed = tx.update(trafficSources).set({
|
|
53122
|
+
syncLeaseOwner: null,
|
|
53123
|
+
syncLeaseExpiresAt: null,
|
|
53124
|
+
updatedAt: input.now
|
|
53125
|
+
}).where(and39(
|
|
53126
|
+
eq47(trafficSources.id, input.sourceId),
|
|
53127
|
+
eq47(trafficSources.syncLeaseOwner, input.owner)
|
|
53128
|
+
)).run();
|
|
53129
|
+
return changed.changes === 1;
|
|
53130
|
+
}, { behavior: "immediate" });
|
|
53131
|
+
}
|
|
53132
|
+
|
|
52487
53133
|
// ../api-routes/src/traffic.ts
|
|
52488
53134
|
var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
52489
53135
|
var DEFAULT_PAGE_SIZE3 = 1e3;
|
|
@@ -52502,6 +53148,9 @@ var BACKFILL_MAX_PAGES = 1e3;
|
|
|
52502
53148
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
52503
53149
|
var CLOUDFLARE_WORKER_VERSION = "1.0.0";
|
|
52504
53150
|
var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
|
|
53151
|
+
var CLOUDFLARE_QUEUE_RECEIPT_TTL_MS = 14 * 24 * 60 * 6e4 + 10 * 6e4;
|
|
53152
|
+
var CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS = 5 * 6e4;
|
|
53153
|
+
var CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS = 5 * 6e4;
|
|
52505
53154
|
var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
|
|
52506
53155
|
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
|
|
52507
53156
|
function sha256Hex3(value) {
|
|
@@ -52519,7 +53168,11 @@ function timingSafeEqualHex(a, b) {
|
|
|
52519
53168
|
}
|
|
52520
53169
|
function parseDirectPushCloudflareSourceConfig(config) {
|
|
52521
53170
|
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
52522
|
-
return parsed.success ? parsed.data : null;
|
|
53171
|
+
return parsed.success && parsed.data.deliveryMode === CloudflareTrafficDeliveryModes["direct-push"] ? parsed.data : null;
|
|
53172
|
+
}
|
|
53173
|
+
function parseQueuePullCloudflareSourceConfig(config) {
|
|
53174
|
+
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
53175
|
+
return parsed.success && parsed.data.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"] ? parsed.data : null;
|
|
52523
53176
|
}
|
|
52524
53177
|
function isDirectPushCloudflareDeliveryMode(value) {
|
|
52525
53178
|
return value === CloudflareTrafficDeliveryModes["direct-push"];
|
|
@@ -52536,7 +53189,7 @@ function authenticateCloudflareIngest(request, store) {
|
|
|
52536
53189
|
const signature = typeof signatureHeader === "string" ? signatureHeader : "";
|
|
52537
53190
|
if (!bearerToken || !sourceId || !timestamp || !signature) return null;
|
|
52538
53191
|
const credential = store.getConnectionBySourceId(sourceId);
|
|
52539
|
-
if (!credential ||
|
|
53192
|
+
if (!credential || credential.deliveryMode !== "direct-push") return null;
|
|
52540
53193
|
const bearerHash = sha256Hex3(bearerToken);
|
|
52541
53194
|
if (!timingSafeEqualHex(bearerHash, sha256Hex3(credential.bearerToken))) return null;
|
|
52542
53195
|
if (!verifyRequestSignature({
|
|
@@ -52551,8 +53204,8 @@ function cloudflareIngestRateLimitKey(request, store, db) {
|
|
|
52551
53204
|
const authenticated = authenticateCloudflareIngest(request, store);
|
|
52552
53205
|
const params = request.params;
|
|
52553
53206
|
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
52554
|
-
const source = db.select().from(trafficSources).where(
|
|
52555
|
-
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status
|
|
53207
|
+
const source = db.select().from(trafficSources).where(eq48(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
53208
|
+
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status === TrafficSourceStatuses.connected && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
52556
53209
|
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
52557
53210
|
}
|
|
52558
53211
|
}
|
|
@@ -52663,6 +53316,8 @@ function rowToDto(row) {
|
|
|
52663
53316
|
lastCursor: row.lastCursor ?? null,
|
|
52664
53317
|
lastError: row.lastError ?? null,
|
|
52665
53318
|
skippedThroughAt: row.skippedThroughAt ?? null,
|
|
53319
|
+
queueBacklogCount: row.queueBacklogCount ?? null,
|
|
53320
|
+
queueBacklogObservedAt: row.queueBacklogObservedAt ?? null,
|
|
52666
53321
|
archivedAt: row.archivedAt ?? null,
|
|
52667
53322
|
config: parseSourceConfig(row),
|
|
52668
53323
|
createdAt: row.createdAt,
|
|
@@ -52680,6 +53335,44 @@ async function defaultResolveAccessToken(record) {
|
|
|
52680
53335
|
"OAuth-mode Cloud Run sync is not yet supported in v1. Provide a service-account key file."
|
|
52681
53336
|
);
|
|
52682
53337
|
}
|
|
53338
|
+
function hasConnectedTrafficSourceSibling(db, projectId, sourceId) {
|
|
53339
|
+
return db.select().from(trafficSources).where(eq48(trafficSources.projectId, projectId)).all().some((row) => row.id !== sourceId && row.status === TrafficSourceStatuses.connected);
|
|
53340
|
+
}
|
|
53341
|
+
function isAuthoritativeTrafficSource(db, source) {
|
|
53342
|
+
return (source.status === TrafficSourceStatuses.connected || source.status === TrafficSourceStatuses.error) && !hasConnectedTrafficSourceSibling(db, source.projectId, source.id);
|
|
53343
|
+
}
|
|
53344
|
+
function trafficConnectStatus(tx, projectId, existingSource) {
|
|
53345
|
+
return existingSource?.status === TrafficSourceStatuses.paused || hasConnectedTrafficSourceSibling(tx, projectId, existingSource?.id ?? "") ? TrafficSourceStatuses.paused : TrafficSourceStatuses.connected;
|
|
53346
|
+
}
|
|
53347
|
+
function isSameTrafficSourceGeneration(current, started) {
|
|
53348
|
+
return isDeepStrictEqual(current.configJson, started.configJson) && current.updatedAt === started.updatedAt && current.lastSyncedAt === started.lastSyncedAt;
|
|
53349
|
+
}
|
|
53350
|
+
function bindTrafficSyncSchedule(tx, projectId, sourceId, now, createIfMissing = true) {
|
|
53351
|
+
const schedule = tx.select().from(schedules).where(and40(
|
|
53352
|
+
eq48(schedules.projectId, projectId),
|
|
53353
|
+
eq48(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
53354
|
+
)).get();
|
|
53355
|
+
if (schedule) {
|
|
53356
|
+
if (schedule.sourceId === sourceId) return { changed: false, created: false };
|
|
53357
|
+
tx.update(schedules).set({ sourceId, updatedAt: now }).where(eq48(schedules.id, schedule.id)).run();
|
|
53358
|
+
return { changed: true, created: false };
|
|
53359
|
+
}
|
|
53360
|
+
if (!createIfMissing) return { changed: false, created: false };
|
|
53361
|
+
tx.insert(schedules).values({
|
|
53362
|
+
id: crypto40.randomUUID(),
|
|
53363
|
+
projectId,
|
|
53364
|
+
kind: SchedulableRunKinds["traffic-sync"],
|
|
53365
|
+
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
53366
|
+
preset: null,
|
|
53367
|
+
timezone: "UTC",
|
|
53368
|
+
enabled: true,
|
|
53369
|
+
providers: [],
|
|
53370
|
+
sourceId,
|
|
53371
|
+
createdAt: now,
|
|
53372
|
+
updatedAt: now
|
|
53373
|
+
}).run();
|
|
53374
|
+
return { changed: true, created: true };
|
|
53375
|
+
}
|
|
52683
53376
|
function vercelRetentionClampError(requestedStartMs, effectiveStartMs) {
|
|
52684
53377
|
return new Error(
|
|
52685
53378
|
`Vercel request-logs retention starts at ${new Date(effectiveStartMs).toISOString()}, after requested start ${new Date(requestedStartMs).toISOString()}; refusing to advance because historical traffic would be skipped`
|
|
@@ -52700,8 +53393,11 @@ async function runBackfillTask(options) {
|
|
|
52700
53393
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
52701
53394
|
try {
|
|
52702
53395
|
app.db.transaction((tx) => {
|
|
52703
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
52704
|
-
tx.
|
|
53396
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq48(runs.id, runId)).run();
|
|
53397
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
53398
|
+
if (latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
53399
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
53400
|
+
}
|
|
52705
53401
|
});
|
|
52706
53402
|
} catch {
|
|
52707
53403
|
}
|
|
@@ -52716,7 +53412,15 @@ async function runBackfillTask(options) {
|
|
|
52716
53412
|
if (allEvents.length === 0) {
|
|
52717
53413
|
const finishedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
52718
53414
|
try {
|
|
52719
|
-
app.db.
|
|
53415
|
+
app.db.transaction((tx) => {
|
|
53416
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
53417
|
+
const stillAuthoritative = latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameTrafficSourceGeneration(latestSource, sourceRow2);
|
|
53418
|
+
tx.update(runs).set(stillAuthoritative ? { status: RunStatuses.completed, finishedAt: finishedAt2 } : {
|
|
53419
|
+
status: RunStatuses.failed,
|
|
53420
|
+
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
53421
|
+
finishedAt: finishedAt2
|
|
53422
|
+
}).where(eq48(runs.id, runId)).run();
|
|
53423
|
+
});
|
|
52720
53424
|
} catch {
|
|
52721
53425
|
}
|
|
52722
53426
|
return;
|
|
@@ -52736,33 +53440,42 @@ async function runBackfillTask(options) {
|
|
|
52736
53440
|
const currentLastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
52737
53441
|
const nextLastSyncedAt = Math.max(currentLastSyncedMs, windowEnd.getTime()) === windowEnd.getTime() ? windowEndIso : sourceRow2.lastSyncedAt;
|
|
52738
53442
|
try {
|
|
52739
|
-
app.db.transaction((tx) => {
|
|
53443
|
+
const commitOutcome = app.db.transaction((tx) => {
|
|
53444
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
53445
|
+
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
53446
|
+
tx.update(runs).set({
|
|
53447
|
+
status: RunStatuses.failed,
|
|
53448
|
+
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
53449
|
+
finishedAt
|
|
53450
|
+
}).where(eq48(runs.id, runId)).run();
|
|
53451
|
+
return "source-inactive";
|
|
53452
|
+
}
|
|
52740
53453
|
tx.delete(crawlerEventsHourly).where(
|
|
52741
|
-
|
|
52742
|
-
|
|
53454
|
+
and40(
|
|
53455
|
+
eq48(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
52743
53456
|
gte11(crawlerEventsHourly.tsHour, windowStartIso),
|
|
52744
|
-
|
|
53457
|
+
lte11(crawlerEventsHourly.tsHour, windowEndIso)
|
|
52745
53458
|
)
|
|
52746
53459
|
).run();
|
|
52747
53460
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
52748
|
-
|
|
52749
|
-
|
|
53461
|
+
and40(
|
|
53462
|
+
eq48(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
52750
53463
|
gte11(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
52751
|
-
|
|
53464
|
+
lte11(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
52752
53465
|
)
|
|
52753
53466
|
).run();
|
|
52754
53467
|
tx.delete(aiReferralEventsHourly).where(
|
|
52755
|
-
|
|
52756
|
-
|
|
53468
|
+
and40(
|
|
53469
|
+
eq48(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
52757
53470
|
gte11(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
52758
|
-
|
|
53471
|
+
lte11(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
52759
53472
|
)
|
|
52760
53473
|
).run();
|
|
52761
53474
|
tx.delete(rawEventSamples).where(
|
|
52762
|
-
|
|
52763
|
-
|
|
53475
|
+
and40(
|
|
53476
|
+
eq48(rawEventSamples.sourceId, sourceRow2.id),
|
|
52764
53477
|
gte11(rawEventSamples.ts, windowStartIso),
|
|
52765
|
-
|
|
53478
|
+
lte11(rawEventSamples.ts, windowEndIso)
|
|
52766
53479
|
)
|
|
52767
53480
|
).run();
|
|
52768
53481
|
for (const bucket of report.crawlerEventsHourly) {
|
|
@@ -52854,9 +53567,11 @@ async function runBackfillTask(options) {
|
|
|
52854
53567
|
lastEventIds: newRingBuffer,
|
|
52855
53568
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
52856
53569
|
updatedAt: finishedAt
|
|
52857
|
-
}).where(
|
|
52858
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
53570
|
+
}).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
53571
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq48(runs.id, runId)).run();
|
|
53572
|
+
return "committed";
|
|
52859
53573
|
});
|
|
53574
|
+
if (commitOutcome === "source-inactive") return;
|
|
52860
53575
|
} catch (e) {
|
|
52861
53576
|
markFailed(`Backfill rollup write failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
52862
53577
|
}
|
|
@@ -52866,6 +53581,12 @@ async function trafficRoutes(app, opts) {
|
|
|
52866
53581
|
const resolveAccessToken2 = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
52867
53582
|
const pullWordpressEvents = opts.pullWordpressTrafficEvents ?? listWordpressTrafficEvents;
|
|
52868
53583
|
const pullVercelEvents = opts.pullVercelTrafficEvents ?? listVercelTrafficEvents;
|
|
53584
|
+
const pullQueueMessages = opts.pullCloudflareQueueMessages ?? pullCloudflareQueueMessages;
|
|
53585
|
+
const ackQueueMessages = opts.ackCloudflareQueueMessages ?? ackCloudflareQueueMessages;
|
|
53586
|
+
const cloudflareQueueMaxBatches = opts.cloudflareQueueMaxBatches ?? DEFAULT_CLOUDFLARE_QUEUE_MAX_BATCHES;
|
|
53587
|
+
if (!Number.isInteger(cloudflareQueueMaxBatches) || cloudflareQueueMaxBatches < 1 || cloudflareQueueMaxBatches > 50) {
|
|
53588
|
+
throw new RangeError("cloudflareQueueMaxBatches must be an integer from 1 to 50");
|
|
53589
|
+
}
|
|
52869
53590
|
const allowLoopback = opts.allowLoopbackWebhooks === true;
|
|
52870
53591
|
async function assertWordpressTargetAllowed(baseUrl) {
|
|
52871
53592
|
const check2 = await resolveWebhookTarget(baseUrl, { allowLoopback });
|
|
@@ -52905,6 +53626,45 @@ async function trafficRoutes(app, opts) {
|
|
|
52905
53626
|
timeWindow: "1 minute",
|
|
52906
53627
|
keyGenerator: (request) => cloudflareIngestRateLimitKey(request, opts.cloudflareTrafficCredentialStore, app.db)
|
|
52907
53628
|
});
|
|
53629
|
+
function validateTrafficSourceCredential(source, projectName) {
|
|
53630
|
+
if (source.sourceType === TrafficSourceTypes.cloudflare) {
|
|
53631
|
+
const config = cloudflareTrafficSourceConfigSchema.safeParse(source.configJson);
|
|
53632
|
+
if (!config.success) throw validationError("Cloudflare source configuration is invalid");
|
|
53633
|
+
const credential = opts.cloudflareTrafficCredentialStore?.getConnectionBySourceId(source.id);
|
|
53634
|
+
if (config.data.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"]) {
|
|
53635
|
+
if (!credential || credential.deliveryMode !== "queue-pull" || credential.projectName !== projectName || typeof credential.apiToken !== "string" || credential.apiToken.trim().length === 0 || credential.accountId !== config.data.accountId || credential.queueId !== config.data.queueId || credential.queueName !== config.data.queueName || credential.retentionSeconds !== config.data.retentionSeconds) {
|
|
53636
|
+
throw validationError("Cloudflare Queue credential is not configured for this source");
|
|
53637
|
+
}
|
|
53638
|
+
return { usesPullSchedule: true };
|
|
53639
|
+
}
|
|
53640
|
+
if (!credential || credential.deliveryMode !== "direct-push" || credential.projectName !== projectName || !credential.bearerToken || !credential.hmacSecret || !source.ingestTokenHash || !timingSafeEqualHex(sha256Hex3(credential.bearerToken), source.ingestTokenHash)) {
|
|
53641
|
+
throw validationError("Cloudflare direct-push credential is not configured for this source");
|
|
53642
|
+
}
|
|
53643
|
+
return { usesPullSchedule: false };
|
|
53644
|
+
}
|
|
53645
|
+
if (source.sourceType === TrafficSourceTypes["cloud-run"]) {
|
|
53646
|
+
const credential = opts.cloudRunCredentialStore?.getConnection(projectName);
|
|
53647
|
+
if (!credential || credential.authMode !== TrafficSourceAuthModes["service-account"] || !credential.clientEmail?.trim() || !credential.privateKey?.trim() || credential.gcpProjectId !== source.configJson.gcpProjectId || source.configJson.authMode !== TrafficSourceAuthModes["service-account"] || (credential.serviceName ?? null) !== (source.configJson.serviceName ?? null) || (credential.location ?? null) !== (source.configJson.location ?? null)) {
|
|
53648
|
+
throw validationError("Cloud Run credential is not configured for this source");
|
|
53649
|
+
}
|
|
53650
|
+
return { usesPullSchedule: true };
|
|
53651
|
+
}
|
|
53652
|
+
if (source.sourceType === TrafficSourceTypes.wordpress) {
|
|
53653
|
+
const credential = opts.wordpressTrafficCredentialStore?.getConnection(projectName);
|
|
53654
|
+
if (!credential || typeof credential.applicationPassword !== "string" || !credential.applicationPassword.trim() || credential.baseUrl !== source.configJson.baseUrl || credential.username !== source.configJson.username) {
|
|
53655
|
+
throw validationError("WordPress traffic credential is not configured for this source");
|
|
53656
|
+
}
|
|
53657
|
+
return { usesPullSchedule: true };
|
|
53658
|
+
}
|
|
53659
|
+
if (source.sourceType === TrafficSourceTypes.vercel) {
|
|
53660
|
+
const credential = opts.vercelTrafficCredentialStore?.getConnection(projectName);
|
|
53661
|
+
if (!credential || typeof credential.token !== "string" || !credential.token.trim() || credential.projectId !== source.configJson.projectId || credential.teamId !== source.configJson.teamId || credential.environment !== source.configJson.environment) {
|
|
53662
|
+
throw validationError("Vercel traffic credential is not configured for this source");
|
|
53663
|
+
}
|
|
53664
|
+
return { usesPullSchedule: true };
|
|
53665
|
+
}
|
|
53666
|
+
throw validationError(`Traffic source type "${source.sourceType}" cannot be activated`);
|
|
53667
|
+
}
|
|
52908
53668
|
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
52909
53669
|
const project = resolveProject(app.db, request.params.name);
|
|
52910
53670
|
const body = request.body ?? {};
|
|
@@ -52942,7 +53702,6 @@ async function trafficRoutes(app, opts) {
|
|
|
52942
53702
|
createdAt: existing?.createdAt ?? now,
|
|
52943
53703
|
updatedAt: now
|
|
52944
53704
|
});
|
|
52945
|
-
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
52946
53705
|
const config = {
|
|
52947
53706
|
gcpProjectId,
|
|
52948
53707
|
serviceName: serviceName ?? null,
|
|
@@ -52950,41 +53709,50 @@ async function trafficRoutes(app, opts) {
|
|
|
52950
53709
|
authMode: TrafficSourceAuthModes["service-account"]
|
|
52951
53710
|
};
|
|
52952
53711
|
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
52953
|
-
|
|
52954
|
-
|
|
52955
|
-
|
|
52956
|
-
|
|
52957
|
-
|
|
52958
|
-
|
|
52959
|
-
|
|
52960
|
-
|
|
52961
|
-
|
|
52962
|
-
|
|
52963
|
-
|
|
52964
|
-
|
|
52965
|
-
|
|
52966
|
-
|
|
53712
|
+
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
53713
|
+
const activeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
53714
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
53715
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
53716
|
+
if (activeSource) {
|
|
53717
|
+
tx.update(trafficSources).set({
|
|
53718
|
+
displayName: fallbackName,
|
|
53719
|
+
status: sourceStatus,
|
|
53720
|
+
lastError: null,
|
|
53721
|
+
configJson: config,
|
|
53722
|
+
updatedAt: now
|
|
53723
|
+
}).where(eq48(trafficSources.id, sourceId)).run();
|
|
53724
|
+
} else {
|
|
53725
|
+
tx.insert(trafficSources).values({
|
|
53726
|
+
id: sourceId,
|
|
53727
|
+
projectId: project.id,
|
|
53728
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
53729
|
+
displayName: fallbackName,
|
|
53730
|
+
status: sourceStatus,
|
|
53731
|
+
lastSyncedAt: null,
|
|
53732
|
+
lastCursor: null,
|
|
53733
|
+
lastError: null,
|
|
53734
|
+
archivedAt: null,
|
|
53735
|
+
configJson: config,
|
|
53736
|
+
createdAt: now,
|
|
53737
|
+
updatedAt: now
|
|
53738
|
+
}).run();
|
|
53739
|
+
}
|
|
53740
|
+
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId, now, false) : { changed: false, created: false };
|
|
53741
|
+
writeAuditLog(tx, {
|
|
52967
53742
|
projectId: project.id,
|
|
52968
|
-
|
|
52969
|
-
|
|
52970
|
-
|
|
52971
|
-
|
|
52972
|
-
|
|
52973
|
-
|
|
52974
|
-
|
|
52975
|
-
|
|
52976
|
-
|
|
52977
|
-
|
|
52978
|
-
|
|
52979
|
-
|
|
53743
|
+
actor: "api",
|
|
53744
|
+
action: "traffic.cloud-run.connected",
|
|
53745
|
+
entityType: "traffic_source",
|
|
53746
|
+
entityId: sourceId
|
|
53747
|
+
});
|
|
53748
|
+
return {
|
|
53749
|
+
sourceRow: tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get(),
|
|
53750
|
+
scheduleChanged: scheduleBinding.changed
|
|
53751
|
+
};
|
|
53752
|
+
}, { behavior: "immediate" });
|
|
53753
|
+
if (scheduleChanged) {
|
|
53754
|
+
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
52980
53755
|
}
|
|
52981
|
-
writeAuditLog(app.db, {
|
|
52982
|
-
projectId: project.id,
|
|
52983
|
-
actor: "api",
|
|
52984
|
-
action: "traffic.cloud-run.connected",
|
|
52985
|
-
entityType: "traffic_source",
|
|
52986
|
-
entityId: sourceRow2.id
|
|
52987
|
-
});
|
|
52988
53756
|
return rowToDto(sourceRow2);
|
|
52989
53757
|
});
|
|
52990
53758
|
app.post("/projects/:name/traffic/connect/wordpress", async (request) => {
|
|
@@ -53028,44 +53796,52 @@ async function trafficRoutes(app, opts) {
|
|
|
53028
53796
|
createdAt: existing?.createdAt ?? now,
|
|
53029
53797
|
updatedAt: now
|
|
53030
53798
|
});
|
|
53031
|
-
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
53032
53799
|
const config = { baseUrl, username };
|
|
53033
53800
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
53034
|
-
|
|
53035
|
-
|
|
53036
|
-
|
|
53037
|
-
|
|
53038
|
-
|
|
53039
|
-
|
|
53040
|
-
|
|
53041
|
-
|
|
53042
|
-
|
|
53043
|
-
|
|
53044
|
-
|
|
53045
|
-
|
|
53046
|
-
|
|
53047
|
-
|
|
53801
|
+
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
53802
|
+
const activeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
53803
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
53804
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
53805
|
+
if (activeSource) {
|
|
53806
|
+
tx.update(trafficSources).set({
|
|
53807
|
+
displayName: fallbackName,
|
|
53808
|
+
status: sourceStatus,
|
|
53809
|
+
lastError: null,
|
|
53810
|
+
configJson: config,
|
|
53811
|
+
updatedAt: now
|
|
53812
|
+
}).where(eq48(trafficSources.id, sourceId)).run();
|
|
53813
|
+
} else {
|
|
53814
|
+
tx.insert(trafficSources).values({
|
|
53815
|
+
id: sourceId,
|
|
53816
|
+
projectId: project.id,
|
|
53817
|
+
sourceType: TrafficSourceTypes.wordpress,
|
|
53818
|
+
displayName: fallbackName,
|
|
53819
|
+
status: sourceStatus,
|
|
53820
|
+
lastSyncedAt: null,
|
|
53821
|
+
lastCursor: null,
|
|
53822
|
+
lastError: null,
|
|
53823
|
+
archivedAt: null,
|
|
53824
|
+
configJson: config,
|
|
53825
|
+
createdAt: now,
|
|
53826
|
+
updatedAt: now
|
|
53827
|
+
}).run();
|
|
53828
|
+
}
|
|
53829
|
+
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId, now, false) : { changed: false, created: false };
|
|
53830
|
+
writeAuditLog(tx, {
|
|
53048
53831
|
projectId: project.id,
|
|
53049
|
-
|
|
53050
|
-
|
|
53051
|
-
|
|
53052
|
-
|
|
53053
|
-
|
|
53054
|
-
|
|
53055
|
-
|
|
53056
|
-
|
|
53057
|
-
|
|
53058
|
-
|
|
53059
|
-
|
|
53060
|
-
|
|
53832
|
+
actor: "api",
|
|
53833
|
+
action: "traffic.wordpress.connected",
|
|
53834
|
+
entityType: "traffic_source",
|
|
53835
|
+
entityId: sourceId
|
|
53836
|
+
});
|
|
53837
|
+
return {
|
|
53838
|
+
sourceRow: tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get(),
|
|
53839
|
+
scheduleChanged: scheduleBinding.changed
|
|
53840
|
+
};
|
|
53841
|
+
}, { behavior: "immediate" });
|
|
53842
|
+
if (scheduleChanged) {
|
|
53843
|
+
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
53061
53844
|
}
|
|
53062
|
-
writeAuditLog(app.db, {
|
|
53063
|
-
projectId: project.id,
|
|
53064
|
-
actor: "api",
|
|
53065
|
-
action: "traffic.wordpress.connected",
|
|
53066
|
-
entityType: "traffic_source",
|
|
53067
|
-
entityId: sourceRow2.id
|
|
53068
|
-
});
|
|
53069
53845
|
return rowToDto(sourceRow2);
|
|
53070
53846
|
});
|
|
53071
53847
|
app.post("/projects/:name/traffic/connect/vercel", async (request) => {
|
|
@@ -53111,20 +53887,21 @@ async function trafficRoutes(app, opts) {
|
|
|
53111
53887
|
createdAt: existing?.createdAt ?? now,
|
|
53112
53888
|
updatedAt: now
|
|
53113
53889
|
});
|
|
53114
|
-
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.vercel && row.status !== TrafficSourceStatuses.archived);
|
|
53115
53890
|
const config = { projectId, teamId, environment };
|
|
53116
53891
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
53117
|
-
const { sourceRow: sourceRow2,
|
|
53892
|
+
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
53893
|
+
const activeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.vercel && row2.status !== TrafficSourceStatuses.archived);
|
|
53894
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
53118
53895
|
let row;
|
|
53119
53896
|
if (activeSource) {
|
|
53120
53897
|
tx.update(trafficSources).set({
|
|
53121
53898
|
displayName: fallbackName,
|
|
53122
|
-
status:
|
|
53899
|
+
status: sourceStatus,
|
|
53123
53900
|
lastError: null,
|
|
53124
53901
|
configJson: config,
|
|
53125
53902
|
updatedAt: now
|
|
53126
|
-
}).where(
|
|
53127
|
-
row = tx.select().from(trafficSources).where(
|
|
53903
|
+
}).where(eq48(trafficSources.id, activeSource.id)).run();
|
|
53904
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, activeSource.id)).get();
|
|
53128
53905
|
} else {
|
|
53129
53906
|
const newId = crypto40.randomUUID();
|
|
53130
53907
|
tx.insert(trafficSources).values({
|
|
@@ -53132,7 +53909,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53132
53909
|
projectId: project.id,
|
|
53133
53910
|
sourceType: TrafficSourceTypes.vercel,
|
|
53134
53911
|
displayName: fallbackName,
|
|
53135
|
-
status:
|
|
53912
|
+
status: sourceStatus,
|
|
53136
53913
|
// Seed lastSyncedAt to NOW so the first sync uses a tight window.
|
|
53137
53914
|
// Leaving this null would make the first sync fall back to
|
|
53138
53915
|
// DEFAULT_SYNC_WINDOW_MINUTES (30 days) — which exceeds Vercel's
|
|
@@ -53149,31 +53926,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53149
53926
|
createdAt: now,
|
|
53150
53927
|
updatedAt: now
|
|
53151
53928
|
}).run();
|
|
53152
|
-
row = tx.select().from(trafficSources).where(
|
|
53153
|
-
}
|
|
53154
|
-
const existingSchedule = tx.select().from(schedules).where(
|
|
53155
|
-
and39(
|
|
53156
|
-
eq47(schedules.projectId, project.id),
|
|
53157
|
-
eq47(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
53158
|
-
)
|
|
53159
|
-
).get();
|
|
53160
|
-
let created = false;
|
|
53161
|
-
if (!existingSchedule) {
|
|
53162
|
-
tx.insert(schedules).values({
|
|
53163
|
-
id: crypto40.randomUUID(),
|
|
53164
|
-
projectId: project.id,
|
|
53165
|
-
kind: SchedulableRunKinds["traffic-sync"],
|
|
53166
|
-
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
53167
|
-
preset: null,
|
|
53168
|
-
timezone: "UTC",
|
|
53169
|
-
enabled: true,
|
|
53170
|
-
providers: [],
|
|
53171
|
-
sourceId: row.id,
|
|
53172
|
-
createdAt: now,
|
|
53173
|
-
updatedAt: now
|
|
53174
|
-
}).run();
|
|
53175
|
-
created = true;
|
|
53929
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, newId)).get();
|
|
53176
53930
|
}
|
|
53931
|
+
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, row.id, now) : { changed: false, created: false };
|
|
53177
53932
|
writeAuditLog(tx, {
|
|
53178
53933
|
projectId: project.id,
|
|
53179
53934
|
actor: "api",
|
|
@@ -53181,7 +53936,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53181
53936
|
entityType: "traffic_source",
|
|
53182
53937
|
entityId: row.id
|
|
53183
53938
|
});
|
|
53184
|
-
if (created) {
|
|
53939
|
+
if (scheduleBinding.created) {
|
|
53185
53940
|
writeAuditLog(tx, {
|
|
53186
53941
|
projectId: project.id,
|
|
53187
53942
|
actor: "api",
|
|
@@ -53194,9 +53949,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53194
53949
|
}
|
|
53195
53950
|
});
|
|
53196
53951
|
}
|
|
53197
|
-
return { sourceRow: row,
|
|
53198
|
-
});
|
|
53199
|
-
if (
|
|
53952
|
+
return { sourceRow: row, scheduleChanged: scheduleBinding.changed };
|
|
53953
|
+
}, { behavior: "immediate" });
|
|
53954
|
+
if (scheduleChanged) {
|
|
53200
53955
|
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
53201
53956
|
}
|
|
53202
53957
|
return rowToDto(sourceRow2);
|
|
@@ -53206,27 +53961,159 @@ async function trafficRoutes(app, opts) {
|
|
|
53206
53961
|
if (!opts.cloudflareTrafficCredentialStore) {
|
|
53207
53962
|
throw validationError("Cloudflare traffic credential storage is not configured for this deployment");
|
|
53208
53963
|
}
|
|
53209
|
-
if (!opts.cloudflareTrafficIngestUrl) {
|
|
53210
|
-
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
53211
|
-
}
|
|
53212
53964
|
const credentialStore = opts.cloudflareTrafficCredentialStore;
|
|
53213
53965
|
const parsed = trafficConnectCloudflareRequestSchema.safeParse(request.body ?? {});
|
|
53214
53966
|
if (!parsed.success) {
|
|
53215
53967
|
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
53216
53968
|
}
|
|
53969
|
+
if (parsed.data.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"]) {
|
|
53970
|
+
const { displayName: displayName2, zoneId: zoneId2, accountId: accountId2, queueId, queueName, retentionSeconds, apiToken } = parsed.data;
|
|
53971
|
+
const workerRouteHost2 = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
53972
|
+
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
53973
|
+
const cloudflareSources = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().filter((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived);
|
|
53974
|
+
const sameModeSource = cloudflareSources.find((row) => parseQueuePullCloudflareSourceConfig(row.configJson));
|
|
53975
|
+
const sourceId2 = sameModeSource?.id ?? crypto40.randomUUID();
|
|
53976
|
+
const previousCredential2 = credentialStore.getConnectionBySourceId(sourceId2);
|
|
53977
|
+
const workerVersion2 = CLOUDFLARE_WORKER_VERSION;
|
|
53978
|
+
const workerScript2 = generateWorkerScript({
|
|
53979
|
+
deliveryMode: "queue-pull",
|
|
53980
|
+
workerVersion: workerVersion2,
|
|
53981
|
+
botList: DEFAULT_BOT_LIST
|
|
53982
|
+
});
|
|
53983
|
+
const wranglerToml2 = generateWranglerToml({
|
|
53984
|
+
deliveryMode: "queue-pull",
|
|
53985
|
+
sourceId: sourceId2,
|
|
53986
|
+
hostname: workerRouteHost2,
|
|
53987
|
+
workerVersion: workerVersion2,
|
|
53988
|
+
queueName,
|
|
53989
|
+
zoneId: zoneId2 ?? null,
|
|
53990
|
+
accountId: accountId2
|
|
53991
|
+
});
|
|
53992
|
+
const config2 = {
|
|
53993
|
+
schemaVersion: 1,
|
|
53994
|
+
deliveryMode: "queue-pull",
|
|
53995
|
+
workerVersion: workerVersion2,
|
|
53996
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
53997
|
+
zoneId: zoneId2 ?? null,
|
|
53998
|
+
accountId: accountId2,
|
|
53999
|
+
queueId,
|
|
54000
|
+
queueName,
|
|
54001
|
+
retentionSeconds
|
|
54002
|
+
};
|
|
54003
|
+
const nextCredential = {
|
|
54004
|
+
projectName: project.name,
|
|
54005
|
+
sourceId: sourceId2,
|
|
54006
|
+
deliveryMode: "queue-pull",
|
|
54007
|
+
apiToken,
|
|
54008
|
+
accountId: accountId2,
|
|
54009
|
+
queueId,
|
|
54010
|
+
queueName,
|
|
54011
|
+
retentionSeconds,
|
|
54012
|
+
workerVersion: workerVersion2,
|
|
54013
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
54014
|
+
zoneId: zoneId2 ?? null,
|
|
54015
|
+
createdAt: previousCredential2?.createdAt ?? now2,
|
|
54016
|
+
updatedAt: now2
|
|
54017
|
+
};
|
|
54018
|
+
credentialStore.upsertConnection(nextCredential);
|
|
54019
|
+
let sourceRow3;
|
|
54020
|
+
let scheduleChanged = false;
|
|
54021
|
+
try {
|
|
54022
|
+
const result = app.db.transaction((tx) => {
|
|
54023
|
+
const currentSameModeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseQueuePullCloudflareSourceConfig(row.configJson) !== null);
|
|
54024
|
+
if ((currentSameModeSource?.id ?? null) !== (sameModeSource?.id ?? null)) {
|
|
54025
|
+
throw operationInProgress("Cloudflare Queue source changed during connect; retry");
|
|
54026
|
+
}
|
|
54027
|
+
const status = trafficConnectStatus(tx, project.id, currentSameModeSource);
|
|
54028
|
+
if (currentSameModeSource) {
|
|
54029
|
+
const queueConfigChanged = !isDeepStrictEqual(currentSameModeSource.configJson, config2);
|
|
54030
|
+
tx.update(trafficSources).set({
|
|
54031
|
+
displayName: displayName2 ?? currentSameModeSource.displayName,
|
|
54032
|
+
status,
|
|
54033
|
+
lastError: null,
|
|
54034
|
+
configJson: config2,
|
|
54035
|
+
ingestTokenHash: null,
|
|
54036
|
+
...queueConfigChanged ? {
|
|
54037
|
+
queueBacklogCount: null,
|
|
54038
|
+
queueBacklogObservedAt: null
|
|
54039
|
+
} : {},
|
|
54040
|
+
updatedAt: now2
|
|
54041
|
+
}).where(eq48(trafficSources.id, sourceId2)).run();
|
|
54042
|
+
} else {
|
|
54043
|
+
tx.insert(trafficSources).values({
|
|
54044
|
+
id: sourceId2,
|
|
54045
|
+
projectId: project.id,
|
|
54046
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
54047
|
+
displayName: displayName2 ?? `Cloudflare Queue \xB7 ${queueName}`,
|
|
54048
|
+
status,
|
|
54049
|
+
lastSyncedAt: null,
|
|
54050
|
+
lastCursor: null,
|
|
54051
|
+
lastError: null,
|
|
54052
|
+
archivedAt: null,
|
|
54053
|
+
configJson: config2,
|
|
54054
|
+
ingestTokenHash: null,
|
|
54055
|
+
createdAt: now2,
|
|
54056
|
+
updatedAt: now2
|
|
54057
|
+
}).run();
|
|
54058
|
+
}
|
|
54059
|
+
const changed = status === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId2, now2).changed : false;
|
|
54060
|
+
return {
|
|
54061
|
+
row: tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId2)).get(),
|
|
54062
|
+
scheduleChanged: changed
|
|
54063
|
+
};
|
|
54064
|
+
}, { behavior: "immediate" });
|
|
54065
|
+
sourceRow3 = result.row;
|
|
54066
|
+
scheduleChanged = result.scheduleChanged;
|
|
54067
|
+
} catch (error) {
|
|
54068
|
+
if (previousCredential2) credentialStore.upsertConnection(previousCredential2);
|
|
54069
|
+
else credentialStore.deleteConnectionBySourceId?.(sourceId2);
|
|
54070
|
+
throw error;
|
|
54071
|
+
}
|
|
54072
|
+
writeAuditLog(app.db, {
|
|
54073
|
+
projectId: project.id,
|
|
54074
|
+
actor: "api",
|
|
54075
|
+
action: "traffic.cloudflare.queue-connected",
|
|
54076
|
+
entityType: "traffic_source",
|
|
54077
|
+
entityId: sourceId2
|
|
54078
|
+
});
|
|
54079
|
+
if (scheduleChanged) {
|
|
54080
|
+
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
54081
|
+
}
|
|
54082
|
+
return {
|
|
54083
|
+
sourceId: sourceId2,
|
|
54084
|
+
deliveryMode: "queue-pull",
|
|
54085
|
+
activationRequired: sourceRow3.status !== TrafficSourceStatuses.connected,
|
|
54086
|
+
accountId: accountId2,
|
|
54087
|
+
queueId,
|
|
54088
|
+
queueName,
|
|
54089
|
+
retentionSeconds,
|
|
54090
|
+
workerScript: workerScript2,
|
|
54091
|
+
wranglerToml: wranglerToml2,
|
|
54092
|
+
workerVersion: workerVersion2,
|
|
54093
|
+
instructions: [
|
|
54094
|
+
"Deploy this Worker to your Cloudflare zone:",
|
|
54095
|
+
" 1. Save worker.js and wrangler.toml; neither file contains Queue credentials",
|
|
54096
|
+
` 2. Enable the Queue HTTP pull consumer: wrangler queues consumer http add ${queueName}`,
|
|
54097
|
+
` 3. Attach ${workerRouteHost2}/* in the Cloudflare Dashboard with Fail open`,
|
|
54098
|
+
` 4. Canonry pulls Queue ${queueName}; activate this source after the route is live.`,
|
|
54099
|
+
`Source id: ${sourceRow3.id}`
|
|
54100
|
+
].join("\n")
|
|
54101
|
+
};
|
|
54102
|
+
}
|
|
54103
|
+
if (!opts.cloudflareTrafficIngestUrl) {
|
|
54104
|
+
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
54105
|
+
}
|
|
53217
54106
|
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
53218
54107
|
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
53219
54108
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
53220
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
53221
|
-
(row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived
|
|
53222
|
-
);
|
|
54109
|
+
const activeSource = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row.configJson) !== null);
|
|
53223
54110
|
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
53224
54111
|
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
53225
54112
|
if (activeSource && !parsedActiveConfig) {
|
|
53226
54113
|
throw validationError("Existing Cloudflare source is not configured for direct-push delivery");
|
|
53227
54114
|
}
|
|
53228
|
-
const previousCredential = credentialStore.
|
|
53229
|
-
if (previousCredential &&
|
|
54115
|
+
const previousCredential = activeSource ? credentialStore.getConnectionBySourceId(activeSource.id) : void 0;
|
|
54116
|
+
if (previousCredential && previousCredential.deliveryMode !== "direct-push") {
|
|
53230
54117
|
throw validationError(
|
|
53231
54118
|
`Cloudflare credential delivery mode "${previousCredential.deliveryMode}" does not match "${deliveryMode}"`
|
|
53232
54119
|
);
|
|
@@ -53280,44 +54167,67 @@ async function trafficRoutes(app, opts) {
|
|
|
53280
54167
|
updatedAt: now
|
|
53281
54168
|
});
|
|
53282
54169
|
let sourceRow2;
|
|
54170
|
+
let scheduleRemoved = false;
|
|
53283
54171
|
try {
|
|
53284
|
-
|
|
53285
|
-
|
|
53286
|
-
|
|
53287
|
-
|
|
53288
|
-
|
|
53289
|
-
|
|
53290
|
-
|
|
53291
|
-
|
|
53292
|
-
|
|
53293
|
-
|
|
53294
|
-
|
|
53295
|
-
|
|
53296
|
-
|
|
53297
|
-
|
|
53298
|
-
|
|
53299
|
-
|
|
53300
|
-
|
|
53301
|
-
|
|
53302
|
-
|
|
53303
|
-
|
|
53304
|
-
|
|
53305
|
-
|
|
53306
|
-
|
|
53307
|
-
|
|
53308
|
-
|
|
53309
|
-
|
|
53310
|
-
|
|
53311
|
-
|
|
53312
|
-
|
|
53313
|
-
|
|
53314
|
-
|
|
54172
|
+
const result = app.db.transaction((tx) => {
|
|
54173
|
+
const currentActiveSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.cloudflare && row2.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row2.configJson) !== null);
|
|
54174
|
+
if ((currentActiveSource?.id ?? null) !== (activeSource?.id ?? null)) {
|
|
54175
|
+
throw operationInProgress("Cloudflare direct-push source changed during connect; retry");
|
|
54176
|
+
}
|
|
54177
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, currentActiveSource);
|
|
54178
|
+
let row;
|
|
54179
|
+
if (currentActiveSource) {
|
|
54180
|
+
tx.update(trafficSources).set({
|
|
54181
|
+
displayName: fallbackName,
|
|
54182
|
+
status: sourceStatus,
|
|
54183
|
+
lastError: null,
|
|
54184
|
+
configJson: config,
|
|
54185
|
+
ingestTokenHash,
|
|
54186
|
+
updatedAt: now
|
|
54187
|
+
}).where(eq48(trafficSources.id, currentActiveSource.id)).run();
|
|
54188
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, currentActiveSource.id)).get();
|
|
54189
|
+
} else {
|
|
54190
|
+
tx.insert(trafficSources).values({
|
|
54191
|
+
id: sourceId,
|
|
54192
|
+
projectId: project.id,
|
|
54193
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
54194
|
+
displayName: fallbackName,
|
|
54195
|
+
status: sourceStatus,
|
|
54196
|
+
// Seed `lastSyncedAt` to NOW so the `traffic.source.recent-data`
|
|
54197
|
+
// doctor check has a non-null baseline. Successful ingest advances
|
|
54198
|
+
// it as the receiver's last-activity timestamp.
|
|
54199
|
+
lastSyncedAt: now,
|
|
54200
|
+
lastCursor: null,
|
|
54201
|
+
lastError: null,
|
|
54202
|
+
archivedAt: null,
|
|
54203
|
+
configJson: config,
|
|
54204
|
+
ingestTokenHash,
|
|
54205
|
+
createdAt: now,
|
|
54206
|
+
updatedAt: now
|
|
54207
|
+
}).run();
|
|
54208
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get();
|
|
54209
|
+
}
|
|
54210
|
+
let removed = false;
|
|
54211
|
+
if (row.status === TrafficSourceStatuses.connected) {
|
|
54212
|
+
const schedule = tx.select().from(schedules).where(and40(
|
|
54213
|
+
eq48(schedules.projectId, project.id),
|
|
54214
|
+
eq48(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
54215
|
+
)).get();
|
|
54216
|
+
if (schedule) {
|
|
54217
|
+
tx.delete(schedules).where(eq48(schedules.id, schedule.id)).run();
|
|
54218
|
+
removed = true;
|
|
54219
|
+
}
|
|
54220
|
+
}
|
|
54221
|
+
return { row, scheduleRemoved: removed };
|
|
54222
|
+
}, { behavior: "immediate" });
|
|
54223
|
+
sourceRow2 = result.row;
|
|
54224
|
+
scheduleRemoved = result.scheduleRemoved;
|
|
53315
54225
|
} catch (err) {
|
|
53316
54226
|
try {
|
|
53317
54227
|
if (previousCredential) {
|
|
53318
54228
|
credentialStore.upsertConnection(previousCredential);
|
|
53319
54229
|
} else {
|
|
53320
|
-
credentialStore.
|
|
54230
|
+
credentialStore.deleteConnectionBySourceId?.(sourceId);
|
|
53321
54231
|
}
|
|
53322
54232
|
} catch {
|
|
53323
54233
|
}
|
|
@@ -53330,6 +54240,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53330
54240
|
entityType: "traffic_source",
|
|
53331
54241
|
entityId: sourceRow2.id
|
|
53332
54242
|
});
|
|
54243
|
+
if (scheduleRemoved) {
|
|
54244
|
+
opts.onScheduleUpdated?.("delete", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
54245
|
+
}
|
|
53333
54246
|
const routeInstruction = effectiveZoneId ? ` 4. Deploy with \`wrangler deploy\`; wrangler.toml records zone ${effectiveZoneId} but does not claim a route` : " 4. Deploy with `wrangler deploy`; wrangler.toml intentionally does not claim a route";
|
|
53334
54247
|
const instructions = [
|
|
53335
54248
|
"Deploy this Worker to your Cloudflare zone:",
|
|
@@ -53347,12 +54260,65 @@ async function trafficRoutes(app, opts) {
|
|
|
53347
54260
|
return {
|
|
53348
54261
|
sourceId: sourceRow2.id,
|
|
53349
54262
|
deliveryMode,
|
|
54263
|
+
activationRequired: sourceRow2.status !== TrafficSourceStatuses.connected,
|
|
53350
54264
|
workerScript,
|
|
53351
54265
|
wranglerToml,
|
|
53352
54266
|
workerVersion,
|
|
53353
54267
|
instructions
|
|
53354
54268
|
};
|
|
53355
54269
|
});
|
|
54270
|
+
app.post("/projects/:name/traffic/sources/:id/activate", async (request) => {
|
|
54271
|
+
const project = resolveProject(app.db, request.params.name);
|
|
54272
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54273
|
+
const result = app.db.transaction((tx) => {
|
|
54274
|
+
const target = tx.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
54275
|
+
if (!target || target.projectId !== project.id || target.status === TrafficSourceStatuses.archived) {
|
|
54276
|
+
throw notFound("Traffic source", request.params.id);
|
|
54277
|
+
}
|
|
54278
|
+
const { usesPullSchedule } = validateTrafficSourceCredential(target, project.name);
|
|
54279
|
+
const siblings = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all();
|
|
54280
|
+
for (const sibling of siblings) {
|
|
54281
|
+
if (sibling.id === target.id || sibling.status === TrafficSourceStatuses.archived) continue;
|
|
54282
|
+
tx.update(trafficSources).set({
|
|
54283
|
+
status: TrafficSourceStatuses.paused,
|
|
54284
|
+
updatedAt: now
|
|
54285
|
+
}).where(eq48(trafficSources.id, sibling.id)).run();
|
|
54286
|
+
}
|
|
54287
|
+
tx.update(trafficSources).set({
|
|
54288
|
+
status: TrafficSourceStatuses.connected,
|
|
54289
|
+
lastError: null,
|
|
54290
|
+
archivedAt: null,
|
|
54291
|
+
updatedAt: now
|
|
54292
|
+
}).where(eq48(trafficSources.id, target.id)).run();
|
|
54293
|
+
let scheduleAction = null;
|
|
54294
|
+
if (usesPullSchedule) {
|
|
54295
|
+
bindTrafficSyncSchedule(tx, project.id, target.id, now);
|
|
54296
|
+
scheduleAction = "upsert";
|
|
54297
|
+
} else {
|
|
54298
|
+
const schedule = tx.select().from(schedules).where(and40(
|
|
54299
|
+
eq48(schedules.projectId, project.id),
|
|
54300
|
+
eq48(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
54301
|
+
)).get();
|
|
54302
|
+
if (schedule) {
|
|
54303
|
+
tx.delete(schedules).where(eq48(schedules.id, schedule.id)).run();
|
|
54304
|
+
scheduleAction = "delete";
|
|
54305
|
+
}
|
|
54306
|
+
}
|
|
54307
|
+
const source = tx.select().from(trafficSources).where(eq48(trafficSources.id, target.id)).get();
|
|
54308
|
+
writeAuditLog(tx, {
|
|
54309
|
+
projectId: project.id,
|
|
54310
|
+
actor: "api",
|
|
54311
|
+
action: "traffic.source.activated",
|
|
54312
|
+
entityType: "traffic_source",
|
|
54313
|
+
entityId: target.id
|
|
54314
|
+
});
|
|
54315
|
+
return { source, scheduleAction };
|
|
54316
|
+
});
|
|
54317
|
+
if (result.scheduleAction) {
|
|
54318
|
+
opts.onScheduleUpdated?.(result.scheduleAction, project.id, SchedulableRunKinds["traffic-sync"]);
|
|
54319
|
+
}
|
|
54320
|
+
return rowToDto(result.source);
|
|
54321
|
+
});
|
|
53356
54322
|
app.post("/projects/:name/traffic/cloudflare/ingest", {
|
|
53357
54323
|
bodyLimit: CLOUDFLARE_INGEST_BODY_LIMIT,
|
|
53358
54324
|
preHandler: async (request, reply) => {
|
|
@@ -53390,8 +54356,8 @@ async function trafficRoutes(app, opts) {
|
|
|
53390
54356
|
}
|
|
53391
54357
|
const { bearerHash, credential } = authenticated;
|
|
53392
54358
|
const sourceId = credential.sourceId;
|
|
53393
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53394
|
-
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status
|
|
54359
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get();
|
|
54360
|
+
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status !== TrafficSourceStatuses.connected || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
53395
54361
|
throw authRequired();
|
|
53396
54362
|
}
|
|
53397
54363
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -53427,7 +54393,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53427
54393
|
receiptTtlMs: DIRECT_PUSH_RECEIPT_TTL_MS,
|
|
53428
54394
|
sampleLimit,
|
|
53429
54395
|
validateSource: (latestRow) => {
|
|
53430
|
-
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status
|
|
54396
|
+
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status !== TrafficSourceStatuses.connected || !parseDirectPushCloudflareSourceConfig(latestRow.configJson) || !timingSafeEqualHex(bearerHash, latestRow.ingestTokenHash)) {
|
|
53431
54397
|
throw authRequired();
|
|
53432
54398
|
}
|
|
53433
54399
|
},
|
|
@@ -53450,15 +54416,259 @@ async function trafficRoutes(app, opts) {
|
|
|
53450
54416
|
});
|
|
53451
54417
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
53452
54418
|
const project = resolveProject(app.db, request.params.name);
|
|
53453
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
54419
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
53454
54420
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
53455
54421
|
throw notFound("Traffic source", request.params.id);
|
|
53456
54422
|
}
|
|
54423
|
+
const queueConfig = sourceRow2.sourceType === TrafficSourceTypes.cloudflare ? parseQueuePullCloudflareSourceConfig(sourceRow2.configJson) : null;
|
|
54424
|
+
if (queueConfig) {
|
|
54425
|
+
if (sourceRow2.status !== TrafficSourceStatuses.connected) {
|
|
54426
|
+
throw validationError("Cloudflare Queue source must be active before it can sync");
|
|
54427
|
+
}
|
|
54428
|
+
const credential = opts.cloudflareTrafficCredentialStore?.getConnectionBySourceId(sourceRow2.id);
|
|
54429
|
+
if (!credential || credential.deliveryMode !== "queue-pull" || credential.projectName !== project.name || typeof credential.apiToken !== "string" || credential.apiToken.trim().length === 0 || credential.accountId !== queueConfig.accountId || credential.queueId !== queueConfig.queueId || credential.queueName !== queueConfig.queueName || credential.retentionSeconds !== queueConfig.retentionSeconds) {
|
|
54430
|
+
throw validationError("Cloudflare Queue credential is not configured for this source");
|
|
54431
|
+
}
|
|
54432
|
+
const queueClient = {
|
|
54433
|
+
accountId: credential.accountId,
|
|
54434
|
+
queueId: credential.queueId,
|
|
54435
|
+
apiToken: credential.apiToken
|
|
54436
|
+
};
|
|
54437
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54438
|
+
const leaseOwner = crypto40.randomUUID();
|
|
54439
|
+
if (!tryClaimTrafficSyncLease({
|
|
54440
|
+
db: app.db,
|
|
54441
|
+
sourceId: sourceRow2.id,
|
|
54442
|
+
owner: leaseOwner,
|
|
54443
|
+
now,
|
|
54444
|
+
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
54445
|
+
})) {
|
|
54446
|
+
throw operationInProgress("Cloudflare Queue source sync is already in progress", { sourceId: sourceRow2.id });
|
|
54447
|
+
}
|
|
54448
|
+
const runId2 = crypto40.randomUUID();
|
|
54449
|
+
const startedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
54450
|
+
const startedMs = Date.now();
|
|
54451
|
+
try {
|
|
54452
|
+
app.db.insert(runs).values({
|
|
54453
|
+
id: runId2,
|
|
54454
|
+
projectId: project.id,
|
|
54455
|
+
kind: RunKinds["traffic-sync"],
|
|
54456
|
+
status: RunStatuses.running,
|
|
54457
|
+
trigger: RunTriggers.manual,
|
|
54458
|
+
sourceId: sourceRow2.id,
|
|
54459
|
+
startedAt: startedAt2,
|
|
54460
|
+
createdAt: startedAt2
|
|
54461
|
+
}).run();
|
|
54462
|
+
let acceptedEvents = 0;
|
|
54463
|
+
let selfTrafficExcluded = 0;
|
|
54464
|
+
let crawlerHits = 0;
|
|
54465
|
+
let aiUserFetchHits = 0;
|
|
54466
|
+
let aiReferralHits = 0;
|
|
54467
|
+
let unknownHits = 0;
|
|
54468
|
+
let crawlerBucketRows2 = 0;
|
|
54469
|
+
let aiUserFetchBucketRows2 = 0;
|
|
54470
|
+
let aiReferralBucketRows2 = 0;
|
|
54471
|
+
let sampleRows2 = 0;
|
|
54472
|
+
let committedAt = startedAt2;
|
|
54473
|
+
let remainingBacklogCount = 0;
|
|
54474
|
+
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
54475
|
+
for (let batch = 0; batch < cloudflareQueueMaxBatches; batch += 1) {
|
|
54476
|
+
if (!tryClaimTrafficSyncLease({
|
|
54477
|
+
db: app.db,
|
|
54478
|
+
sourceId: sourceRow2.id,
|
|
54479
|
+
owner: leaseOwner,
|
|
54480
|
+
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
54481
|
+
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
54482
|
+
})) throw new Error("Queue sync lease was lost");
|
|
54483
|
+
const pulled = await pullQueueMessages(queueClient, {
|
|
54484
|
+
batchSize: CLOUDFLARE_QUEUE_BATCH_SIZE,
|
|
54485
|
+
visibilityTimeoutMs: CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS
|
|
54486
|
+
});
|
|
54487
|
+
remainingBacklogCount = pulled.messageBacklogCount;
|
|
54488
|
+
if (pulled.skippedUnleasedMessageCount > 0) {
|
|
54489
|
+
request.log.warn({
|
|
54490
|
+
sourceId: sourceRow2.id,
|
|
54491
|
+
skippedUnleasedMessageCount: pulled.skippedUnleasedMessageCount
|
|
54492
|
+
}, "Skipped unacknowledgeable Cloudflare Queue messages; they remain eligible for redelivery");
|
|
54493
|
+
}
|
|
54494
|
+
const validLeases = [];
|
|
54495
|
+
const poisonLeases = [];
|
|
54496
|
+
const normalized = [];
|
|
54497
|
+
let workerVersion = null;
|
|
54498
|
+
for (const message of pulled.messages) {
|
|
54499
|
+
if (message.contentType === "poison") {
|
|
54500
|
+
poisonLeases.push(message.leaseId);
|
|
54501
|
+
request.log.warn(
|
|
54502
|
+
{ sourceId: sourceRow2.id, messageId: message.id, reason: message.reason },
|
|
54503
|
+
"Dropping malformed Cloudflare Queue message"
|
|
54504
|
+
);
|
|
54505
|
+
continue;
|
|
54506
|
+
}
|
|
54507
|
+
if (message.contentType !== "json") {
|
|
54508
|
+
poisonLeases.push(message.leaseId);
|
|
54509
|
+
request.log.warn({
|
|
54510
|
+
sourceId: sourceRow2.id,
|
|
54511
|
+
messageId: message.id,
|
|
54512
|
+
contentType: message.contentType,
|
|
54513
|
+
reason: "unsupported-content-type"
|
|
54514
|
+
}, "Dropping Cloudflare Queue message with an unsupported content type");
|
|
54515
|
+
continue;
|
|
54516
|
+
}
|
|
54517
|
+
const parsedBatch = cloudflareWorkerIngestRequestSchema.safeParse(message.body);
|
|
54518
|
+
if (!parsedBatch.success) {
|
|
54519
|
+
poisonLeases.push(message.leaseId);
|
|
54520
|
+
request.log.warn(
|
|
54521
|
+
{ sourceId: sourceRow2.id, messageId: message.id },
|
|
54522
|
+
"Dropping Cloudflare Queue message with an invalid Canonry batch"
|
|
54523
|
+
);
|
|
54524
|
+
continue;
|
|
54525
|
+
}
|
|
54526
|
+
if (parsedBatch.data.events.some((event) => normalizeCloudflareEventHost(event.host) !== canonicalHost)) {
|
|
54527
|
+
poisonLeases.push(message.leaseId);
|
|
54528
|
+
request.log.warn(
|
|
54529
|
+
{ sourceId: sourceRow2.id, messageId: message.id },
|
|
54530
|
+
"Dropping Cloudflare Queue message for another host"
|
|
54531
|
+
);
|
|
54532
|
+
continue;
|
|
54533
|
+
}
|
|
54534
|
+
workerVersion = parsedBatch.data.workerVersion;
|
|
54535
|
+
for (const event of parsedBatch.data.events) {
|
|
54536
|
+
const normalizedEvent = normalizeCloudflareWorkerEvent(event);
|
|
54537
|
+
if (normalizedEvent) normalized.push(normalizedEvent);
|
|
54538
|
+
}
|
|
54539
|
+
validLeases.push(message.leaseId);
|
|
54540
|
+
}
|
|
54541
|
+
committedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54542
|
+
const writeResult = writeTrafficEventBatch({
|
|
54543
|
+
db: app.db,
|
|
54544
|
+
projectId: project.id,
|
|
54545
|
+
sourceId: sourceRow2.id,
|
|
54546
|
+
events: normalized,
|
|
54547
|
+
receivedAt: committedAt,
|
|
54548
|
+
receiptTtlMs: CLOUDFLARE_QUEUE_RECEIPT_TTL_MS,
|
|
54549
|
+
sampleLimit,
|
|
54550
|
+
validateSource: (latest) => {
|
|
54551
|
+
if (!latest || latest.projectId !== project.id || latest.status !== TrafficSourceStatuses.connected || latest.syncLeaseOwner !== leaseOwner || !isDeepStrictEqual(latest.configJson, sourceRow2.configJson)) {
|
|
54552
|
+
throw validationError("Cloudflare Queue source is no longer active");
|
|
54553
|
+
}
|
|
54554
|
+
},
|
|
54555
|
+
sourceUpdate: {
|
|
54556
|
+
status: TrafficSourceStatuses.connected,
|
|
54557
|
+
lastSyncedAt: committedAt,
|
|
54558
|
+
lastError: null,
|
|
54559
|
+
queueBacklogCount: remainingBacklogCount,
|
|
54560
|
+
queueBacklogObservedAt: committedAt,
|
|
54561
|
+
...workerVersion ? { lastWorkerVersion: workerVersion } : {},
|
|
54562
|
+
updatedAt: committedAt
|
|
54563
|
+
}
|
|
54564
|
+
});
|
|
54565
|
+
if (validLeases.length > 0 || poisonLeases.length > 0) {
|
|
54566
|
+
if (!tryClaimTrafficSyncLease({
|
|
54567
|
+
db: app.db,
|
|
54568
|
+
sourceId: sourceRow2.id,
|
|
54569
|
+
owner: leaseOwner,
|
|
54570
|
+
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
54571
|
+
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
54572
|
+
})) throw new Error("Queue sync lease was lost before acknowledgement");
|
|
54573
|
+
const latestSource = app.db.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54574
|
+
if (!latestSource || latestSource.projectId !== project.id || latestSource.status !== TrafficSourceStatuses.connected || latestSource.syncLeaseOwner !== leaseOwner || !isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
54575
|
+
throw validationError("Cloudflare Queue source was reconfigured before acknowledgement");
|
|
54576
|
+
}
|
|
54577
|
+
const ackResult = await ackQueueMessages(queueClient, { acks: [...validLeases, ...poisonLeases] });
|
|
54578
|
+
if (ackResult.warningCount > 0) {
|
|
54579
|
+
request.log.warn({
|
|
54580
|
+
sourceId: sourceRow2.id,
|
|
54581
|
+
warningCount: ackResult.warningCount
|
|
54582
|
+
}, "Cloudflare Queue acknowledgement completed with warnings");
|
|
54583
|
+
}
|
|
54584
|
+
}
|
|
54585
|
+
acceptedEvents += writeResult.acceptedEvents;
|
|
54586
|
+
selfTrafficExcluded += writeResult.selfTrafficExcluded;
|
|
54587
|
+
crawlerHits += writeResult.crawlerHits;
|
|
54588
|
+
aiUserFetchHits += writeResult.aiUserFetchHits;
|
|
54589
|
+
aiReferralHits += writeResult.aiReferralHits;
|
|
54590
|
+
unknownHits += writeResult.unknownHits;
|
|
54591
|
+
crawlerBucketRows2 += writeResult.crawlerBucketRows;
|
|
54592
|
+
aiUserFetchBucketRows2 += writeResult.aiUserFetchBucketRows;
|
|
54593
|
+
aiReferralBucketRows2 += writeResult.aiReferralBucketRows;
|
|
54594
|
+
sampleRows2 += writeResult.sampleRows;
|
|
54595
|
+
const pulledEnvelopeCount = pulled.messages.length + pulled.skippedUnleasedMessageCount;
|
|
54596
|
+
if (pulledEnvelopeCount < CLOUDFLARE_QUEUE_BATCH_SIZE || remainingBacklogCount === 0) break;
|
|
54597
|
+
}
|
|
54598
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: committedAt }).where(eq48(runs.id, runId2)).run();
|
|
54599
|
+
writeAuditLog(app.db, {
|
|
54600
|
+
projectId: project.id,
|
|
54601
|
+
actor: "api",
|
|
54602
|
+
action: "traffic.cloudflare.queue-synced",
|
|
54603
|
+
entityType: "traffic_source",
|
|
54604
|
+
entityId: sourceRow2.id
|
|
54605
|
+
});
|
|
54606
|
+
try {
|
|
54607
|
+
opts.onTrafficSynced?.({
|
|
54608
|
+
status: "completed",
|
|
54609
|
+
sourceType: sourceRow2.sourceType,
|
|
54610
|
+
sourceId: sourceRow2.id,
|
|
54611
|
+
pulledEvents: acceptedEvents,
|
|
54612
|
+
selfTrafficExcluded,
|
|
54613
|
+
crawlerHits,
|
|
54614
|
+
aiUserFetchHits,
|
|
54615
|
+
aiReferralHits,
|
|
54616
|
+
durationMs: Date.now() - startedMs
|
|
54617
|
+
});
|
|
54618
|
+
} catch {
|
|
54619
|
+
}
|
|
54620
|
+
const response2 = {
|
|
54621
|
+
sourceId: sourceRow2.id,
|
|
54622
|
+
runId: runId2,
|
|
54623
|
+
syncedAt: committedAt,
|
|
54624
|
+
pulledEvents: acceptedEvents,
|
|
54625
|
+
selfTrafficExcluded,
|
|
54626
|
+
crawlerHits,
|
|
54627
|
+
aiUserFetchHits,
|
|
54628
|
+
aiReferralHits,
|
|
54629
|
+
unknownHits,
|
|
54630
|
+
crawlerBucketRows: crawlerBucketRows2,
|
|
54631
|
+
aiUserFetchBucketRows: aiUserFetchBucketRows2,
|
|
54632
|
+
aiReferralBucketRows: aiReferralBucketRows2,
|
|
54633
|
+
sampleRows: sampleRows2,
|
|
54634
|
+
remainingBacklogCount,
|
|
54635
|
+
windowStart: startedAt2,
|
|
54636
|
+
windowEnd: committedAt
|
|
54637
|
+
};
|
|
54638
|
+
return response2;
|
|
54639
|
+
} catch (error) {
|
|
54640
|
+
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54641
|
+
const safeError = error instanceof CloudflareQueueApiError ? error.message : "Cloudflare Queue sync failed; retry the active source.";
|
|
54642
|
+
app.db.transaction((tx) => {
|
|
54643
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: safeError, finishedAt: failedAt }).where(eq48(runs.id, runId2)).run();
|
|
54644
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54645
|
+
if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
54646
|
+
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and40(
|
|
54647
|
+
eq48(trafficSources.id, sourceRow2.id),
|
|
54648
|
+
eq48(trafficSources.status, TrafficSourceStatuses.connected),
|
|
54649
|
+
eq48(trafficSources.syncLeaseOwner, leaseOwner)
|
|
54650
|
+
)).run();
|
|
54651
|
+
}
|
|
54652
|
+
});
|
|
54653
|
+
throw providerError(safeError);
|
|
54654
|
+
} finally {
|
|
54655
|
+
releaseTrafficSyncLease({
|
|
54656
|
+
db: app.db,
|
|
54657
|
+
sourceId: sourceRow2.id,
|
|
54658
|
+
owner: leaseOwner,
|
|
54659
|
+
now: (/* @__PURE__ */ new Date()).toISOString()
|
|
54660
|
+
});
|
|
54661
|
+
}
|
|
54662
|
+
}
|
|
53457
54663
|
if (sourceRow2.sourceType !== TrafficSourceTypes["cloud-run"] && sourceRow2.sourceType !== TrafficSourceTypes.wordpress && sourceRow2.sourceType !== TrafficSourceTypes.vercel) {
|
|
53458
54664
|
throw validationError(
|
|
53459
54665
|
`Sync for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
|
|
53460
54666
|
);
|
|
53461
54667
|
}
|
|
54668
|
+
const hasConnectedSibling = sourceRow2.status === TrafficSourceStatuses.error && app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
54669
|
+
if (sourceRow2.status === TrafficSourceStatuses.paused || sourceRow2.status === TrafficSourceStatuses.archived || hasConnectedSibling) {
|
|
54670
|
+
throw validationError("Traffic source must be active before it can sync");
|
|
54671
|
+
}
|
|
53462
54672
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
53463
54673
|
const startedAt = windowEnd.toISOString();
|
|
53464
54674
|
const syncStartedAtMs = windowEnd.getTime();
|
|
@@ -53476,8 +54686,11 @@ async function trafficRoutes(app, opts) {
|
|
|
53476
54686
|
const markFailed = (msg, errorCode) => {
|
|
53477
54687
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
53478
54688
|
app.db.transaction((tx) => {
|
|
53479
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
53480
|
-
tx.
|
|
54689
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq48(runs.id, runId)).run();
|
|
54690
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54691
|
+
if (latestSource?.status === TrafficSourceStatuses.connected && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson) && latestSource.updatedAt === sourceRow2.updatedAt && latestSource.lastSyncedAt === sourceRow2.lastSyncedAt) {
|
|
54692
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
54693
|
+
}
|
|
53481
54694
|
});
|
|
53482
54695
|
try {
|
|
53483
54696
|
opts.onTrafficSynced?.({
|
|
@@ -53558,7 +54771,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53558
54771
|
}
|
|
53559
54772
|
const credential = credentialStore.getConnection(project.name);
|
|
53560
54773
|
if (!credential) {
|
|
53561
|
-
app.db.delete(runs).where(
|
|
54774
|
+
app.db.delete(runs).where(eq48(runs.id, runId)).run();
|
|
53562
54775
|
throw validationError(
|
|
53563
54776
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
53564
54777
|
);
|
|
@@ -53607,12 +54820,12 @@ async function trafficRoutes(app, opts) {
|
|
|
53607
54820
|
auditAction = "traffic.vercel.synced";
|
|
53608
54821
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
53609
54822
|
if (!credentialStore) {
|
|
53610
|
-
app.db.delete(runs).where(
|
|
54823
|
+
app.db.delete(runs).where(eq48(runs.id, runId)).run();
|
|
53611
54824
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
53612
54825
|
}
|
|
53613
54826
|
const credential = credentialStore.getConnection(project.name);
|
|
53614
54827
|
if (!credential) {
|
|
53615
|
-
app.db.delete(runs).where(
|
|
54828
|
+
app.db.delete(runs).where(eq48(runs.id, runId)).run();
|
|
53616
54829
|
throw validationError(
|
|
53617
54830
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
53618
54831
|
);
|
|
@@ -53630,7 +54843,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53630
54843
|
if (cappedStartMs > clampedStartMs) {
|
|
53631
54844
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
53632
54845
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
53633
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
54846
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
53634
54847
|
request.log.warn(
|
|
53635
54848
|
{
|
|
53636
54849
|
sourceId: sourceRow2.id,
|
|
@@ -53714,8 +54927,18 @@ async function trafficRoutes(app, opts) {
|
|
|
53714
54927
|
let aiUserFetchHitsCount = 0;
|
|
53715
54928
|
let aiReferralHitsCount = 0;
|
|
53716
54929
|
let unknownHitsCount = 0;
|
|
53717
|
-
app.db.transaction((tx) => {
|
|
53718
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
54930
|
+
const commitOutcome = app.db.transaction((tx) => {
|
|
54931
|
+
const latestRow = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54932
|
+
const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
54933
|
+
if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || !isDeepStrictEqual(latestRow.configJson, sourceRow2.configJson) || latestRow.updatedAt !== sourceRow2.updatedAt || latestRow.lastSyncedAt !== sourceRow2.lastSyncedAt) {
|
|
54934
|
+
const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54935
|
+
tx.update(runs).set({
|
|
54936
|
+
status: RunStatuses.failed,
|
|
54937
|
+
error: "Traffic source was deactivated or reconfigured during sync",
|
|
54938
|
+
finishedAt: abortedAt
|
|
54939
|
+
}).where(eq48(runs.id, runId)).run();
|
|
54940
|
+
return "source-inactive";
|
|
54941
|
+
}
|
|
53719
54942
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
53720
54943
|
const seenEventIds = new Set(previousIds);
|
|
53721
54944
|
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
@@ -53892,7 +55115,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53892
55115
|
// stuck if you first disabled the schedule and drained the in-flight run.
|
|
53893
55116
|
lastSyncedAt: new Date(
|
|
53894
55117
|
Math.max(
|
|
53895
|
-
|
|
55118
|
+
latestRow.lastSyncedAt ? new Date(latestRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY,
|
|
53896
55119
|
effectiveWindowEnd.getTime()
|
|
53897
55120
|
)
|
|
53898
55121
|
).toISOString(),
|
|
@@ -53903,9 +55126,13 @@ async function trafficRoutes(app, opts) {
|
|
|
53903
55126
|
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
53904
55127
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
53905
55128
|
}
|
|
53906
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
53907
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
55129
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
55130
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq48(runs.id, runId)).run();
|
|
55131
|
+
return "committed";
|
|
53908
55132
|
});
|
|
55133
|
+
if (commitOutcome === "source-inactive") {
|
|
55134
|
+
throw validationError("Traffic source is no longer active; discarded the in-flight sync");
|
|
55135
|
+
}
|
|
53909
55136
|
writeAuditLog(app.db, {
|
|
53910
55137
|
projectId: project.id,
|
|
53911
55138
|
actor: "api",
|
|
@@ -53956,7 +55183,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53956
55183
|
});
|
|
53957
55184
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
53958
55185
|
const project = resolveProject(app.db, request.params.name);
|
|
53959
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
55186
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
53960
55187
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
53961
55188
|
throw notFound("Traffic source", request.params.id);
|
|
53962
55189
|
}
|
|
@@ -53965,6 +55192,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53965
55192
|
`Backfill for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
|
|
53966
55193
|
);
|
|
53967
55194
|
}
|
|
55195
|
+
if (!isAuthoritativeTrafficSource(app.db, sourceRow2)) {
|
|
55196
|
+
throw validationError("Traffic source must be active before it can backfill");
|
|
55197
|
+
}
|
|
53968
55198
|
const requestedDays = request.body?.days ?? DEFAULT_BACKFILL_DAYS;
|
|
53969
55199
|
if (!Number.isInteger(requestedDays) || requestedDays <= 0) {
|
|
53970
55200
|
throw validationError('"days" must be a positive integer');
|
|
@@ -54107,16 +55337,22 @@ async function trafficRoutes(app, opts) {
|
|
|
54107
55337
|
}
|
|
54108
55338
|
const startedAt = windowEnd.toISOString();
|
|
54109
55339
|
const runId = crypto40.randomUUID();
|
|
54110
|
-
app.db.
|
|
54111
|
-
id
|
|
54112
|
-
|
|
54113
|
-
|
|
54114
|
-
|
|
54115
|
-
|
|
54116
|
-
|
|
54117
|
-
|
|
54118
|
-
|
|
54119
|
-
|
|
55340
|
+
app.db.transaction((tx) => {
|
|
55341
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
55342
|
+
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
55343
|
+
throw validationError("Traffic source must remain active and unchanged before it can backfill");
|
|
55344
|
+
}
|
|
55345
|
+
tx.insert(runs).values({
|
|
55346
|
+
id: runId,
|
|
55347
|
+
projectId: project.id,
|
|
55348
|
+
kind: RunKinds["traffic-sync"],
|
|
55349
|
+
status: RunStatuses.running,
|
|
55350
|
+
trigger: RunTriggers.backfill,
|
|
55351
|
+
sourceId: sourceRow2.id,
|
|
55352
|
+
startedAt,
|
|
55353
|
+
createdAt: startedAt
|
|
55354
|
+
}).run();
|
|
55355
|
+
});
|
|
54120
55356
|
void runBackfillTask({
|
|
54121
55357
|
app,
|
|
54122
55358
|
runId,
|
|
@@ -54144,8 +55380,8 @@ async function trafficRoutes(app, opts) {
|
|
|
54144
55380
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
54145
55381
|
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
54146
55382
|
}).from(crawlerEventsHourly).where(
|
|
54147
|
-
|
|
54148
|
-
|
|
55383
|
+
and40(
|
|
55384
|
+
eq48(crawlerEventsHourly.sourceId, row.id),
|
|
54149
55385
|
gte11(crawlerEventsHourly.tsHour, since)
|
|
54150
55386
|
)
|
|
54151
55387
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
@@ -54154,28 +55390,28 @@ async function trafficRoutes(app, opts) {
|
|
|
54154
55390
|
);
|
|
54155
55391
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
54156
55392
|
const aiUserFetchTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
54157
|
-
|
|
54158
|
-
|
|
55393
|
+
and40(
|
|
55394
|
+
eq48(aiUserFetchEventsHourly.sourceId, row.id),
|
|
54159
55395
|
gte11(aiUserFetchEventsHourly.tsHour, since)
|
|
54160
55396
|
)
|
|
54161
55397
|
).get();
|
|
54162
55398
|
const aiTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
54163
|
-
|
|
54164
|
-
|
|
55399
|
+
and40(
|
|
55400
|
+
eq48(aiReferralEventsHourly.sourceId, row.id),
|
|
54165
55401
|
gte11(aiReferralEventsHourly.tsHour, since)
|
|
54166
55402
|
)
|
|
54167
55403
|
).get();
|
|
54168
55404
|
const sampleTotals = app.db.select({ total: sql19`COUNT(*)` }).from(rawEventSamples).where(
|
|
54169
|
-
|
|
54170
|
-
|
|
55405
|
+
and40(
|
|
55406
|
+
eq48(rawEventSamples.sourceId, row.id),
|
|
54171
55407
|
gte11(rawEventSamples.ts, since)
|
|
54172
55408
|
)
|
|
54173
55409
|
).get();
|
|
54174
55410
|
const latestRun = app.db.select().from(runs).where(
|
|
54175
|
-
|
|
54176
|
-
|
|
54177
|
-
|
|
54178
|
-
|
|
55411
|
+
and40(
|
|
55412
|
+
eq48(runs.projectId, projectId),
|
|
55413
|
+
eq48(runs.kind, RunKinds["traffic-sync"]),
|
|
55414
|
+
eq48(runs.sourceId, row.id)
|
|
54179
55415
|
)
|
|
54180
55416
|
).orderBy(desc22(runs.startedAt)).limit(1).get();
|
|
54181
55417
|
return {
|
|
@@ -54206,24 +55442,36 @@ async function trafficRoutes(app, opts) {
|
|
|
54206
55442
|
"`advanceToNow` must be `true`. There is no implicit reset."
|
|
54207
55443
|
);
|
|
54208
55444
|
}
|
|
54209
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(and39(eq47(trafficSources.projectId, project.id), eq47(trafficSources.id, request.params.id))).get();
|
|
54210
|
-
if (!sourceRow2) {
|
|
54211
|
-
throw notFound("traffic source", request.params.id);
|
|
54212
|
-
}
|
|
54213
|
-
if (sourceRow2.status === TrafficSourceStatuses.archived) {
|
|
54214
|
-
throw validationError(
|
|
54215
|
-
`Traffic source "${sourceRow2.id}" is archived. Re-connect via "canonry traffic connect ..." to start tracking it again.`
|
|
54216
|
-
);
|
|
54217
|
-
}
|
|
54218
55445
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54219
55446
|
let updatedRow;
|
|
54220
55447
|
app.db.transaction((tx) => {
|
|
55448
|
+
const sourceRow2 = tx.select().from(trafficSources).where(and40(eq48(trafficSources.projectId, project.id), eq48(trafficSources.id, request.params.id))).get();
|
|
55449
|
+
if (!sourceRow2) {
|
|
55450
|
+
throw notFound("traffic source", request.params.id);
|
|
55451
|
+
}
|
|
55452
|
+
if (sourceRow2.status === TrafficSourceStatuses.archived) {
|
|
55453
|
+
throw validationError(
|
|
55454
|
+
`Traffic source "${sourceRow2.id}" is archived. Re-connect via "canonry traffic connect ..." to start tracking it again.`
|
|
55455
|
+
);
|
|
55456
|
+
}
|
|
55457
|
+
if (sourceRow2.status === TrafficSourceStatuses.paused) {
|
|
55458
|
+
throw validationError(
|
|
55459
|
+
`Traffic source "${sourceRow2.id}" is staged. Activate it explicitly instead of resetting it.`
|
|
55460
|
+
);
|
|
55461
|
+
}
|
|
55462
|
+
const hasConnectedSibling = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
55463
|
+
if (hasConnectedSibling) {
|
|
55464
|
+
throw validationError(
|
|
55465
|
+
`Traffic source "${sourceRow2.id}" is not authoritative. Activate it explicitly instead of resetting it.`
|
|
55466
|
+
);
|
|
55467
|
+
}
|
|
55468
|
+
validateTrafficSourceCredential(sourceRow2, project.name);
|
|
54221
55469
|
tx.update(trafficSources).set({
|
|
54222
55470
|
lastSyncedAt: now,
|
|
54223
55471
|
status: TrafficSourceStatuses.connected,
|
|
54224
55472
|
lastError: null,
|
|
54225
55473
|
updatedAt: now
|
|
54226
|
-
}).where(
|
|
55474
|
+
}).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
54227
55475
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
54228
55476
|
projectId: project.id,
|
|
54229
55477
|
actor: "api",
|
|
@@ -54231,20 +55479,20 @@ async function trafficRoutes(app, opts) {
|
|
|
54231
55479
|
entityType: "traffic_source",
|
|
54232
55480
|
entityId: sourceRow2.id
|
|
54233
55481
|
}));
|
|
54234
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
55482
|
+
updatedRow = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54235
55483
|
});
|
|
54236
55484
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
54237
55485
|
});
|
|
54238
55486
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
54239
55487
|
const project = resolveProject(app.db, request.params.name);
|
|
54240
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
55488
|
+
const rows = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
54241
55489
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
54242
55490
|
const response = { sources };
|
|
54243
55491
|
return response;
|
|
54244
55492
|
});
|
|
54245
55493
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
54246
55494
|
const project = resolveProject(app.db, request.params.name);
|
|
54247
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
55495
|
+
const rows = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
54248
55496
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
54249
55497
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
54250
55498
|
const response = { sources };
|
|
@@ -54254,7 +55502,7 @@ async function trafficRoutes(app, opts) {
|
|
|
54254
55502
|
"/projects/:name/traffic/sources/:id",
|
|
54255
55503
|
async (request) => {
|
|
54256
55504
|
const project = resolveProject(app.db, request.params.name);
|
|
54257
|
-
const row = app.db.select().from(trafficSources).where(
|
|
55505
|
+
const row = app.db.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
54258
55506
|
if (!row || row.projectId !== project.id) {
|
|
54259
55507
|
throw notFound("Traffic source", request.params.id);
|
|
54260
55508
|
}
|
|
@@ -54319,12 +55567,12 @@ async function trafficRoutes(app, opts) {
|
|
|
54319
55567
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
54320
55568
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
54321
55569
|
const crawlerFilters = [
|
|
54322
|
-
|
|
55570
|
+
eq48(crawlerEventsHourly.projectId, project.id),
|
|
54323
55571
|
gte11(crawlerEventsHourly.tsHour, sinceIso),
|
|
54324
|
-
|
|
55572
|
+
lte11(crawlerEventsHourly.tsHour, untilIso)
|
|
54325
55573
|
];
|
|
54326
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
54327
|
-
const crawlerWhere =
|
|
55574
|
+
if (sourceIdParam) crawlerFilters.push(eq48(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
55575
|
+
const crawlerWhere = and40(...crawlerFilters);
|
|
54328
55576
|
const pathTotals = app.db.select({
|
|
54329
55577
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
54330
55578
|
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
@@ -54361,12 +55609,12 @@ async function trafficRoutes(app, opts) {
|
|
|
54361
55609
|
}
|
|
54362
55610
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
54363
55611
|
const userFetchFilters = [
|
|
54364
|
-
|
|
55612
|
+
eq48(aiUserFetchEventsHourly.projectId, project.id),
|
|
54365
55613
|
gte11(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
54366
|
-
|
|
55614
|
+
lte11(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
54367
55615
|
];
|
|
54368
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
54369
|
-
const userFetchWhere =
|
|
55616
|
+
if (sourceIdParam) userFetchFilters.push(eq48(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
55617
|
+
const userFetchWhere = and40(...userFetchFilters);
|
|
54370
55618
|
const total = app.db.select({
|
|
54371
55619
|
total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
54372
55620
|
rows: sql19`COUNT(*)`
|
|
@@ -54398,12 +55646,12 @@ async function trafficRoutes(app, opts) {
|
|
|
54398
55646
|
}
|
|
54399
55647
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
54400
55648
|
const aiFilters = [
|
|
54401
|
-
|
|
55649
|
+
eq48(aiReferralEventsHourly.projectId, project.id),
|
|
54402
55650
|
gte11(aiReferralEventsHourly.tsHour, sinceIso),
|
|
54403
|
-
|
|
55651
|
+
lte11(aiReferralEventsHourly.tsHour, untilIso)
|
|
54404
55652
|
];
|
|
54405
|
-
if (sourceIdParam) aiFilters.push(
|
|
54406
|
-
const aiWhere =
|
|
55653
|
+
if (sourceIdParam) aiFilters.push(eq48(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
55654
|
+
const aiWhere = and40(...aiFilters);
|
|
54407
55655
|
const total = app.db.select({
|
|
54408
55656
|
total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
54409
55657
|
paid: sql19`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
@@ -54722,7 +55970,7 @@ function readInstalledManifest(skillDir) {
|
|
|
54722
55970
|
var AGENT_CHECKS = [skillsInstalledCheck, skillsCurrentCheck];
|
|
54723
55971
|
|
|
54724
55972
|
// ../api-routes/src/doctor/checks/backlinks.ts
|
|
54725
|
-
import { and as
|
|
55973
|
+
import { and as and41, eq as eq49 } from "drizzle-orm";
|
|
54726
55974
|
function skippedNoProject() {
|
|
54727
55975
|
return {
|
|
54728
55976
|
status: CheckStatuses.skipped,
|
|
@@ -54738,8 +55986,8 @@ var BACKLINKS_CHECKS = [
|
|
|
54738
55986
|
title: "Backlinks source connected",
|
|
54739
55987
|
run: (ctx) => {
|
|
54740
55988
|
if (!ctx.project) return skippedNoProject();
|
|
54741
|
-
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(
|
|
54742
|
-
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(
|
|
55989
|
+
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(eq49(projects.id, ctx.project.id)).get();
|
|
55990
|
+
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(eq49(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).limit(1).get();
|
|
54743
55991
|
const ccConnected = projectRow?.autoExtract === true && !!readySync;
|
|
54744
55992
|
const connected = [];
|
|
54745
55993
|
if (ccConnected) connected.push(BacklinkSources.commoncrawl);
|
|
@@ -54752,9 +56000,9 @@ var BACKLINKS_CHECKS = [
|
|
|
54752
56000
|
details: { commoncrawl: ccConnected }
|
|
54753
56001
|
};
|
|
54754
56002
|
}
|
|
54755
|
-
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
54756
|
-
|
|
54757
|
-
|
|
56003
|
+
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and41(
|
|
56004
|
+
eq49(backlinkSummaries.projectId, ctx.project.id),
|
|
56005
|
+
eq49(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
54758
56006
|
)).limit(1).get() : false;
|
|
54759
56007
|
return {
|
|
54760
56008
|
status: CheckStatuses.ok,
|
|
@@ -54914,7 +56162,7 @@ var BING_AUTH_CHECKS = [
|
|
|
54914
56162
|
];
|
|
54915
56163
|
|
|
54916
56164
|
// ../api-routes/src/doctor/checks/content.ts
|
|
54917
|
-
import { eq as
|
|
56165
|
+
import { eq as eq50 } from "drizzle-orm";
|
|
54918
56166
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
54919
56167
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
54920
56168
|
function skippedNoProject2() {
|
|
@@ -54927,7 +56175,7 @@ function skippedNoProject2() {
|
|
|
54927
56175
|
}
|
|
54928
56176
|
function loadProject(ctx) {
|
|
54929
56177
|
if (!ctx.project) return null;
|
|
54930
|
-
return ctx.db.select().from(projects).where(
|
|
56178
|
+
return ctx.db.select().from(projects).where(eq50(projects.id, ctx.project.id)).get() ?? null;
|
|
54931
56179
|
}
|
|
54932
56180
|
function percent(value) {
|
|
54933
56181
|
return Math.round(value * 100);
|
|
@@ -55019,7 +56267,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
55019
56267
|
);
|
|
55020
56268
|
|
|
55021
56269
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
55022
|
-
import { eq as
|
|
56270
|
+
import { eq as eq51 } from "drizzle-orm";
|
|
55023
56271
|
var RECENT_SYNC_WARN_DAYS = 7;
|
|
55024
56272
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
55025
56273
|
var adsConnectionCheck = {
|
|
@@ -55036,7 +56284,7 @@ var adsConnectionCheck = {
|
|
|
55036
56284
|
remediation: null
|
|
55037
56285
|
};
|
|
55038
56286
|
}
|
|
55039
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
56287
|
+
const row = ctx.db.select().from(adsConnections).where(eq51(adsConnections.projectId, ctx.project.id)).get();
|
|
55040
56288
|
if (!row) {
|
|
55041
56289
|
return {
|
|
55042
56290
|
status: CheckStatuses.skipped,
|
|
@@ -55086,7 +56334,7 @@ var adsRecentSyncCheck = {
|
|
|
55086
56334
|
remediation: null
|
|
55087
56335
|
};
|
|
55088
56336
|
}
|
|
55089
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
56337
|
+
const row = ctx.db.select().from(adsConnections).where(eq51(adsConnections.projectId, ctx.project.id)).get();
|
|
55090
56338
|
if (!row) {
|
|
55091
56339
|
return {
|
|
55092
56340
|
status: CheckStatuses.skipped,
|
|
@@ -55277,7 +56525,7 @@ var ga4ConnectionCheck = {
|
|
|
55277
56525
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
55278
56526
|
|
|
55279
56527
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
55280
|
-
import { and as
|
|
56528
|
+
import { and as and42, eq as eq52 } from "drizzle-orm";
|
|
55281
56529
|
var RECENT_SYNC_WARN_DAYS2 = 7;
|
|
55282
56530
|
var RECENT_SYNC_FAIL_DAYS2 = 30;
|
|
55283
56531
|
function skippedNoProject3() {
|
|
@@ -55510,7 +56758,7 @@ var recentSyncCheck = {
|
|
|
55510
56758
|
title: "GBP recent sync",
|
|
55511
56759
|
run: (ctx) => {
|
|
55512
56760
|
if (!ctx.project) return skippedNoProject3();
|
|
55513
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
56761
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and42(eq52(gbpLocations.projectId, ctx.project.id), eq52(gbpLocations.selected, true))).all();
|
|
55514
56762
|
if (selected.length === 0) {
|
|
55515
56763
|
return {
|
|
55516
56764
|
status: CheckStatuses.skipped,
|
|
@@ -55570,7 +56818,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
55570
56818
|
);
|
|
55571
56819
|
|
|
55572
56820
|
// ../api-routes/src/doctor/checks/places.ts
|
|
55573
|
-
import { eq as
|
|
56821
|
+
import { eq as eq53 } from "drizzle-orm";
|
|
55574
56822
|
var apiKeyCheck = {
|
|
55575
56823
|
id: "gbp.places.api-key",
|
|
55576
56824
|
category: CheckCategories.auth,
|
|
@@ -55615,7 +56863,7 @@ var apiKeyCheck = {
|
|
|
55615
56863
|
details: { tier: cfg.tier }
|
|
55616
56864
|
};
|
|
55617
56865
|
}
|
|
55618
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
56866
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq53(gbpLocations.projectId, ctx.project.id)).all();
|
|
55619
56867
|
const selected = rows.filter((r) => r.selected);
|
|
55620
56868
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
55621
56869
|
const details = {
|
|
@@ -56130,7 +57378,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
56130
57378
|
];
|
|
56131
57379
|
|
|
56132
57380
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
56133
|
-
import { and as
|
|
57381
|
+
import { and as and43, eq as eq54, gte as gte12, inArray as inArray18, ne as ne6, sql as sql20 } from "drizzle-orm";
|
|
56134
57382
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
56135
57383
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
56136
57384
|
function isCloudflareDirectPush(source) {
|
|
@@ -56138,6 +57386,12 @@ function isCloudflareDirectPush(source) {
|
|
|
56138
57386
|
const deliveryMode = source.configJson.deliveryMode;
|
|
56139
57387
|
return deliveryMode === void 0 || deliveryMode === CloudflareTrafficDeliveryModes["direct-push"];
|
|
56140
57388
|
}
|
|
57389
|
+
function isCloudflareQueuePull(source) {
|
|
57390
|
+
return source.sourceType === TrafficSourceTypes.cloudflare && source.configJson.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"];
|
|
57391
|
+
}
|
|
57392
|
+
function isActiveSource(source) {
|
|
57393
|
+
return source.status !== TrafficSourceStatuses.paused;
|
|
57394
|
+
}
|
|
56141
57395
|
function recentDataRemediation(sources, lastSyncedAt) {
|
|
56142
57396
|
const hasCloudflare = sources.some(isCloudflareDirectPush);
|
|
56143
57397
|
const hasPullSource = sources.some((source) => !isCloudflareDirectPush(source));
|
|
@@ -56160,8 +57414,8 @@ function skippedNoProject5() {
|
|
|
56160
57414
|
function loadProbes(ctx) {
|
|
56161
57415
|
if (!ctx.project) return [];
|
|
56162
57416
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
56163
|
-
|
|
56164
|
-
|
|
57417
|
+
and43(
|
|
57418
|
+
eq54(trafficSources.projectId, ctx.project.id),
|
|
56165
57419
|
ne6(trafficSources.status, TrafficSourceStatuses.archived)
|
|
56166
57420
|
)
|
|
56167
57421
|
).all();
|
|
@@ -56176,6 +57430,8 @@ function loadProbes(ctx) {
|
|
|
56176
57430
|
lastWorkerVersion: r.lastWorkerVersion,
|
|
56177
57431
|
ingestTokenHash: r.ingestTokenHash,
|
|
56178
57432
|
skippedThroughAt: r.skippedThroughAt,
|
|
57433
|
+
queueBacklogCount: r.queueBacklogCount,
|
|
57434
|
+
queueBacklogObservedAt: r.queueBacklogObservedAt,
|
|
56179
57435
|
lastError: r.lastError,
|
|
56180
57436
|
configJson: r.configJson
|
|
56181
57437
|
}));
|
|
@@ -56197,30 +57453,40 @@ var sourceConnectedCheck = {
|
|
|
56197
57453
|
details: { sourceCount: 0 }
|
|
56198
57454
|
};
|
|
56199
57455
|
}
|
|
56200
|
-
const
|
|
56201
|
-
if (
|
|
57456
|
+
const activeSources = sources.filter(isActiveSource);
|
|
57457
|
+
if (activeSources.length === 0) {
|
|
57458
|
+
return {
|
|
57459
|
+
status: CheckStatuses.skipped,
|
|
57460
|
+
code: "traffic.source.paused",
|
|
57461
|
+
summary: `${sources.length} traffic source(s) are paused or staged; none are actively ingesting.`,
|
|
57462
|
+
remediation: "Activate the intended traffic source after its deployment smoke test succeeds.",
|
|
57463
|
+
details: { sourceCount: sources.length, pausedIds: sources.map((s) => s.id) }
|
|
57464
|
+
};
|
|
57465
|
+
}
|
|
57466
|
+
const errored = activeSources.filter((s) => s.status === "error");
|
|
57467
|
+
if (errored.length > 0 && errored.length === activeSources.length) {
|
|
56202
57468
|
return {
|
|
56203
57469
|
status: CheckStatuses.fail,
|
|
56204
57470
|
code: "traffic.source.all-errored",
|
|
56205
|
-
summary: `All ${
|
|
57471
|
+
summary: `All ${activeSources.length} active traffic source(s) are in error state. No data is being ingested.`,
|
|
56206
57472
|
remediation: errored[0].lastError ? `Latest error: "${errored[0].lastError}". Re-connect the source or run \`canonry traffic sync <project> --source <id>\` to retry.` : "Run `canonry traffic sources <project>` to inspect the failing source(s) and re-connect.",
|
|
56207
|
-
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
57473
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length, erroredIds: errored.map((s) => s.id) }
|
|
56208
57474
|
};
|
|
56209
57475
|
}
|
|
56210
57476
|
if (errored.length > 0) {
|
|
56211
57477
|
return {
|
|
56212
57478
|
status: CheckStatuses.warn,
|
|
56213
57479
|
code: "traffic.source.partially-errored",
|
|
56214
|
-
summary: `${errored.length} of ${
|
|
57480
|
+
summary: `${errored.length} of ${activeSources.length} active traffic source(s) are in error state.`,
|
|
56215
57481
|
remediation: "Run `canonry traffic sources <project>` to inspect the failing sources individually.",
|
|
56216
|
-
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
57482
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length, erroredIds: errored.map((s) => s.id) }
|
|
56217
57483
|
};
|
|
56218
57484
|
}
|
|
56219
57485
|
return {
|
|
56220
57486
|
status: CheckStatuses.ok,
|
|
56221
57487
|
code: "traffic.source.connected",
|
|
56222
|
-
summary: `${
|
|
56223
|
-
details: { sourceCount: sources.length, sourceTypes: [...new Set(
|
|
57488
|
+
summary: `${activeSources.length} traffic source(s) connected: ${activeSources.map((s) => s.displayName).join(", ")}.`,
|
|
57489
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length, sourceTypes: [...new Set(activeSources.map((s) => s.sourceType))] }
|
|
56224
57490
|
};
|
|
56225
57491
|
}
|
|
56226
57492
|
};
|
|
@@ -56239,29 +57505,43 @@ var recentDataCheck = {
|
|
|
56239
57505
|
summary: "No traffic source connected \u2014 recent-data check skipped."
|
|
56240
57506
|
};
|
|
56241
57507
|
}
|
|
57508
|
+
const activeSources = sources.filter(isActiveSource);
|
|
57509
|
+
if (activeSources.length === 0) {
|
|
57510
|
+
return {
|
|
57511
|
+
status: CheckStatuses.skipped,
|
|
57512
|
+
code: "traffic.recent-data.no-active-source",
|
|
57513
|
+
summary: "Only paused or staged traffic sources are configured \u2014 recent-data check skipped.",
|
|
57514
|
+
remediation: "Activate the intended source after its deployment smoke test succeeds.",
|
|
57515
|
+
details: { sourceCount: sources.length, pausedIds: sources.map((source) => source.id) }
|
|
57516
|
+
};
|
|
57517
|
+
}
|
|
57518
|
+
const activeSourceIds = activeSources.map((source) => source.id);
|
|
56242
57519
|
const now = /* @__PURE__ */ new Date();
|
|
56243
57520
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
56244
57521
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
56245
57522
|
const recentCrawlers = Number(
|
|
56246
57523
|
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
56247
|
-
|
|
56248
|
-
|
|
57524
|
+
and43(
|
|
57525
|
+
eq54(crawlerEventsHourly.projectId, ctx.project.id),
|
|
57526
|
+
inArray18(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
56249
57527
|
gte12(crawlerEventsHourly.tsHour, warnCutoff)
|
|
56250
57528
|
)
|
|
56251
57529
|
).get()?.total ?? 0
|
|
56252
57530
|
);
|
|
56253
57531
|
const recentReferrals = Number(
|
|
56254
57532
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
56255
|
-
|
|
56256
|
-
|
|
57533
|
+
and43(
|
|
57534
|
+
eq54(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
57535
|
+
inArray18(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
56257
57536
|
gte12(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
56258
57537
|
)
|
|
56259
57538
|
).get()?.total ?? 0
|
|
56260
57539
|
);
|
|
56261
57540
|
const recentUserFetches = Number(
|
|
56262
57541
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
56263
|
-
|
|
56264
|
-
|
|
57542
|
+
and43(
|
|
57543
|
+
eq54(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
57544
|
+
inArray18(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
56265
57545
|
gte12(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
56266
57546
|
)
|
|
56267
57547
|
).get()?.total ?? 0
|
|
@@ -56281,37 +57561,40 @@ var recentDataCheck = {
|
|
|
56281
57561
|
}
|
|
56282
57562
|
const olderCrawlers = Number(
|
|
56283
57563
|
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
56284
|
-
|
|
56285
|
-
|
|
57564
|
+
and43(
|
|
57565
|
+
eq54(crawlerEventsHourly.projectId, ctx.project.id),
|
|
57566
|
+
inArray18(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
56286
57567
|
gte12(crawlerEventsHourly.tsHour, failCutoff)
|
|
56287
57568
|
)
|
|
56288
57569
|
).get()?.total ?? 0
|
|
56289
57570
|
);
|
|
56290
57571
|
const olderReferrals = Number(
|
|
56291
57572
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
56292
|
-
|
|
56293
|
-
|
|
57573
|
+
and43(
|
|
57574
|
+
eq54(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
57575
|
+
inArray18(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
56294
57576
|
gte12(aiReferralEventsHourly.tsHour, failCutoff)
|
|
56295
57577
|
)
|
|
56296
57578
|
).get()?.total ?? 0
|
|
56297
57579
|
);
|
|
56298
57580
|
const olderUserFetches = Number(
|
|
56299
57581
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
56300
|
-
|
|
56301
|
-
|
|
57582
|
+
and43(
|
|
57583
|
+
eq54(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
57584
|
+
inArray18(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
56302
57585
|
gte12(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
56303
57586
|
)
|
|
56304
57587
|
).get()?.total ?? 0
|
|
56305
57588
|
);
|
|
56306
|
-
const lastSyncedAt =
|
|
57589
|
+
const lastSyncedAt = activeSources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
56307
57590
|
const hasOlderData = olderCrawlers > 0 || olderUserFetches > 0 || olderReferrals > 0;
|
|
56308
57591
|
if (hasOlderData || lastSyncedAt) {
|
|
56309
57592
|
return {
|
|
56310
57593
|
status: CheckStatuses.warn,
|
|
56311
57594
|
code: "traffic.recent-data.stale",
|
|
56312
57595
|
summary: hasOlderData ? `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.` : `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
56313
|
-
remediation: recentDataRemediation(
|
|
56314
|
-
details: { lastSyncedAt, sourceCount: sources.length }
|
|
57596
|
+
remediation: recentDataRemediation(activeSources, lastSyncedAt),
|
|
57597
|
+
details: { lastSyncedAt, sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length }
|
|
56315
57598
|
};
|
|
56316
57599
|
}
|
|
56317
57600
|
return {
|
|
@@ -56319,7 +57602,7 @@ var recentDataCheck = {
|
|
|
56319
57602
|
code: "traffic.recent-data.empty",
|
|
56320
57603
|
summary: `No traffic data in the last ${RECENT_DATA_FAIL_DAYS} days. The source is connected but isn't ingesting.`,
|
|
56321
57604
|
remediation: "Verify the source's configuration with `canonry traffic sources <project>` and run a manual sync to confirm credentials + scopes are still valid.",
|
|
56322
|
-
details: { sourceCount: sources.length }
|
|
57605
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length }
|
|
56323
57606
|
};
|
|
56324
57607
|
}
|
|
56325
57608
|
};
|
|
@@ -56405,6 +57688,7 @@ function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
|
56405
57688
|
status: CheckStatuses.skipped,
|
|
56406
57689
|
code: `traffic.${fallbackId}.all-skipped`,
|
|
56407
57690
|
summary: `No source-type validator was available for any of the ${results.length} connected source(s).`,
|
|
57691
|
+
remediation: skipped.find((result) => result.output.remediation)?.output.remediation,
|
|
56408
57692
|
details: detail
|
|
56409
57693
|
};
|
|
56410
57694
|
}
|
|
@@ -56509,8 +57793,16 @@ var syncLagCheck = {
|
|
|
56509
57793
|
run: (ctx) => {
|
|
56510
57794
|
if (!ctx.project) return skippedNoProject5();
|
|
56511
57795
|
const allSources = loadProbes(ctx);
|
|
56512
|
-
const
|
|
56513
|
-
if (allSources.length > 0 &&
|
|
57796
|
+
const activeSources = allSources.filter(isActiveSource);
|
|
57797
|
+
if (allSources.length > 0 && activeSources.length === 0) {
|
|
57798
|
+
return {
|
|
57799
|
+
status: CheckStatuses.skipped,
|
|
57800
|
+
code: "traffic.sync-lag.no-active-source",
|
|
57801
|
+
summary: "Only paused or staged traffic sources are configured \u2014 pull sync lag does not apply."
|
|
57802
|
+
};
|
|
57803
|
+
}
|
|
57804
|
+
const sources = activeSources.filter((source) => !isCloudflareDirectPush(source));
|
|
57805
|
+
if (activeSources.length > 0 && sources.length === 0) {
|
|
56514
57806
|
return {
|
|
56515
57807
|
status: CheckStatuses.skipped,
|
|
56516
57808
|
code: "traffic.sync-lag.push-only",
|
|
@@ -56606,6 +57898,69 @@ var syncLagCheck = {
|
|
|
56606
57898
|
};
|
|
56607
57899
|
}
|
|
56608
57900
|
};
|
|
57901
|
+
var queueBacklogCheck = {
|
|
57902
|
+
id: "traffic.source.queue-backlog",
|
|
57903
|
+
category: CheckCategories.integrations,
|
|
57904
|
+
scope: CheckScopes.project,
|
|
57905
|
+
title: "Cloudflare Queue backlog",
|
|
57906
|
+
run: (ctx) => {
|
|
57907
|
+
if (!ctx.project) return skippedNoProject5();
|
|
57908
|
+
const sources = loadProbes(ctx).filter(
|
|
57909
|
+
(source) => isActiveSource(source) && isCloudflareQueuePull(source)
|
|
57910
|
+
);
|
|
57911
|
+
if (sources.length === 0) {
|
|
57912
|
+
return {
|
|
57913
|
+
status: CheckStatuses.skipped,
|
|
57914
|
+
code: "traffic.queue-backlog.not-applicable",
|
|
57915
|
+
summary: "No active Cloudflare Queue pull source is connected."
|
|
57916
|
+
};
|
|
57917
|
+
}
|
|
57918
|
+
const measured = sources.filter((source) => source.queueBacklogCount !== null && source.queueBacklogCount !== void 0);
|
|
57919
|
+
const details = {
|
|
57920
|
+
sources: sources.map((source) => ({
|
|
57921
|
+
id: source.id,
|
|
57922
|
+
displayName: source.displayName,
|
|
57923
|
+
queueBacklogCount: source.queueBacklogCount ?? null,
|
|
57924
|
+
queueBacklogObservedAt: source.queueBacklogObservedAt ?? null
|
|
57925
|
+
}))
|
|
57926
|
+
};
|
|
57927
|
+
if (measured.length === 0) {
|
|
57928
|
+
return {
|
|
57929
|
+
status: CheckStatuses.skipped,
|
|
57930
|
+
code: "traffic.queue-backlog.not-observed",
|
|
57931
|
+
summary: "Cloudflare Queue backlog has not been observed yet.",
|
|
57932
|
+
remediation: "Run `canonry traffic sync <project> --source <id>` to pull the Queue and record its residual depth.",
|
|
57933
|
+
details
|
|
57934
|
+
};
|
|
57935
|
+
}
|
|
57936
|
+
const remaining = measured.filter((source) => (source.queueBacklogCount ?? 0) > 0);
|
|
57937
|
+
const total = remaining.reduce((sum, source) => sum + source.queueBacklogCount, 0);
|
|
57938
|
+
if (total > DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET) {
|
|
57939
|
+
return {
|
|
57940
|
+
status: CheckStatuses.warn,
|
|
57941
|
+
code: "traffic.queue-backlog.remaining",
|
|
57942
|
+
summary: `${total.toLocaleString("en-US")} message(s) remained across ${remaining.length} Cloudflare Queue source(s) after the last bounded sync, exceeding the default ${DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET.toLocaleString("en-US")}-message drain budget.`,
|
|
57943
|
+
remediation: "Run `canonry traffic sync <project> --source <id>` again. If the backlog stays above 1,000 messages, shorten the traffic-sync schedule interval.",
|
|
57944
|
+
details
|
|
57945
|
+
};
|
|
57946
|
+
}
|
|
57947
|
+
if (total > 0) {
|
|
57948
|
+
return {
|
|
57949
|
+
status: CheckStatuses.ok,
|
|
57950
|
+
code: "traffic.queue-backlog.within-drain-budget",
|
|
57951
|
+
summary: `${total.toLocaleString("en-US")} message(s) remained after the last bounded sync, within the default ${DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET.toLocaleString("en-US")}-message drain budget.`,
|
|
57952
|
+
remediation: "If no new messages arrive, the next scheduled sync can drain this residual backlog. Run `canonry traffic sync <project> --source <id>` to accelerate it.",
|
|
57953
|
+
details
|
|
57954
|
+
};
|
|
57955
|
+
}
|
|
57956
|
+
return {
|
|
57957
|
+
status: CheckStatuses.ok,
|
|
57958
|
+
code: "traffic.queue-backlog.empty",
|
|
57959
|
+
summary: "The last observed Cloudflare Queue backlog was empty.",
|
|
57960
|
+
details
|
|
57961
|
+
};
|
|
57962
|
+
}
|
|
57963
|
+
};
|
|
56609
57964
|
var workerVersionCheck = {
|
|
56610
57965
|
id: "traffic.source.worker-version",
|
|
56611
57966
|
category: CheckCategories.integrations,
|
|
@@ -56613,7 +57968,7 @@ var workerVersionCheck = {
|
|
|
56613
57968
|
title: "Cloudflare Worker version",
|
|
56614
57969
|
run: (ctx) => {
|
|
56615
57970
|
if (!ctx.project) return skippedNoProject5();
|
|
56616
|
-
const sources = loadProbes(ctx).filter(isCloudflareDirectPush);
|
|
57971
|
+
const sources = loadProbes(ctx).filter((source) => isActiveSource(source) && isCloudflareDirectPush(source));
|
|
56617
57972
|
if (sources.length === 0) {
|
|
56618
57973
|
return {
|
|
56619
57974
|
status: CheckStatuses.skipped,
|
|
@@ -56667,6 +58022,7 @@ var TRAFFIC_SOURCE_CHECKS = [
|
|
|
56667
58022
|
sourceConnectedCheck,
|
|
56668
58023
|
recentDataCheck,
|
|
56669
58024
|
syncLagCheck,
|
|
58025
|
+
queueBacklogCheck,
|
|
56670
58026
|
workerVersionCheck,
|
|
56671
58027
|
credentialsCheck,
|
|
56672
58028
|
scopesCheck3,
|
|
@@ -56887,7 +58243,7 @@ async function doctorRoutes(app, opts) {
|
|
|
56887
58243
|
|
|
56888
58244
|
// ../api-routes/src/discovery/routes.ts
|
|
56889
58245
|
import crypto42 from "crypto";
|
|
56890
|
-
import { and as
|
|
58246
|
+
import { and as and44, desc as desc23, eq as eq55, gte as gte13, inArray as inArray19, isNull as isNull5, or as or11 } from "drizzle-orm";
|
|
56891
58247
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
56892
58248
|
async function discoveryRoutes(app, opts) {
|
|
56893
58249
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -56921,13 +58277,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
56921
58277
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
56922
58278
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
56923
58279
|
const decision = app.db.transaction((tx) => {
|
|
56924
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
56925
|
-
|
|
56926
|
-
|
|
58280
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and44(
|
|
58281
|
+
eq55(discoverySessions.projectId, project.id),
|
|
58282
|
+
eq55(discoverySessions.icpDescription, icpDescription),
|
|
56927
58283
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
56928
58284
|
// semantics, so a request with a different (or no) buyer must start
|
|
56929
58285
|
// its own session, never adopt another buyer's probes.
|
|
56930
|
-
parsed.data.buyerDescription == null ?
|
|
58286
|
+
parsed.data.buyerDescription == null ? isNull5(discoverySessions.buyerDescription) : eq55(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
56931
58287
|
// Locations are identity too: a different service-area subset seeds
|
|
56932
58288
|
// and probes a different geo, so it must never reuse another geo's
|
|
56933
58289
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -56937,12 +58293,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
56937
58293
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
56938
58294
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
56939
58295
|
// upgrade, never a wrong reuse.
|
|
56940
|
-
locations.length === 0 ?
|
|
58296
|
+
locations.length === 0 ? or11(isNull5(discoverySessions.locations), eq55(discoverySessions.locations, locations)) : eq55(discoverySessions.locations, locations),
|
|
56941
58297
|
// Seed provider set is identity: a different phrasing distribution
|
|
56942
58298
|
// must never reuse another set's session. Null (= Gemini-only
|
|
56943
58299
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
56944
|
-
seedProviders == null ?
|
|
56945
|
-
|
|
58300
|
+
seedProviders == null ? isNull5(discoverySessions.seedProviders) : eq55(discoverySessions.seedProviders, seedProviders),
|
|
58301
|
+
inArray19(discoverySessions.status, [
|
|
56946
58302
|
DiscoverySessionStatuses.queued,
|
|
56947
58303
|
DiscoverySessionStatuses.seeding,
|
|
56948
58304
|
DiscoverySessionStatuses.probing
|
|
@@ -57017,7 +58373,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57017
58373
|
const project = resolveProject(app.db, request.params.name);
|
|
57018
58374
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
57019
58375
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
57020
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
58376
|
+
const rows = app.db.select().from(discoverySessions).where(eq55(discoverySessions.projectId, project.id)).orderBy(desc23(discoverySessions.createdAt)).limit(limit).all();
|
|
57021
58377
|
return reply.send(rows.map(serializeSession));
|
|
57022
58378
|
}
|
|
57023
58379
|
);
|
|
@@ -57025,11 +58381,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
57025
58381
|
"/projects/:name/discover/sessions/:id",
|
|
57026
58382
|
async (request, reply) => {
|
|
57027
58383
|
const project = resolveProject(app.db, request.params.name);
|
|
57028
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58384
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57029
58385
|
if (!session || session.projectId !== project.id) {
|
|
57030
58386
|
throw notFound("Discovery session", request.params.id);
|
|
57031
58387
|
}
|
|
57032
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
58388
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
57033
58389
|
const detail = {
|
|
57034
58390
|
...serializeSession(session),
|
|
57035
58391
|
probes: probeRows.map(serializeProbe)
|
|
@@ -57041,7 +58397,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57041
58397
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
57042
58398
|
async (request, reply) => {
|
|
57043
58399
|
const project = resolveProject(app.db, request.params.name);
|
|
57044
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58400
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57045
58401
|
if (!session || session.projectId !== project.id) {
|
|
57046
58402
|
throw notFound("Discovery session", request.params.id);
|
|
57047
58403
|
}
|
|
@@ -57049,7 +58405,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57049
58405
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
57050
58406
|
const applyAnchor = request.query.anchor !== "false";
|
|
57051
58407
|
const provider = session.seedProvider ?? "gemini";
|
|
57052
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
58408
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
57053
58409
|
const extract = opts.harvestSearchQueries;
|
|
57054
58410
|
const probesWithQueries = probeRows.map((row) => {
|
|
57055
58411
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -57060,7 +58416,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57060
58416
|
return { searchQueries: [] };
|
|
57061
58417
|
}
|
|
57062
58418
|
});
|
|
57063
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
58419
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq55(queries.projectId, project.id)).all().map((r) => r.query);
|
|
57064
58420
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
57065
58421
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
57066
58422
|
effectiveDomains(project)
|
|
@@ -57107,12 +58463,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
57107
58463
|
"/projects/:name/discover/sessions/:id/promote",
|
|
57108
58464
|
async (request, reply) => {
|
|
57109
58465
|
const project = resolveProject(app.db, request.params.name);
|
|
57110
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58466
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57111
58467
|
if (!session || session.projectId !== project.id) {
|
|
57112
58468
|
throw notFound("Discovery session", request.params.id);
|
|
57113
58469
|
}
|
|
57114
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
57115
|
-
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
58470
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
58471
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq55(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
57116
58472
|
const seenCompetitors = new Set(existingCompetitors);
|
|
57117
58473
|
const cited = /* @__PURE__ */ new Set();
|
|
57118
58474
|
const aspirational = /* @__PURE__ */ new Set();
|
|
@@ -57141,7 +58497,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57141
58497
|
);
|
|
57142
58498
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
57143
58499
|
const project = resolveProject(app.db, request.params.name);
|
|
57144
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58500
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57145
58501
|
if (!session || session.projectId !== project.id) {
|
|
57146
58502
|
throw notFound("Discovery session", request.params.id);
|
|
57147
58503
|
}
|
|
@@ -57164,7 +58520,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57164
58520
|
const bucketSet = new Set(buckets);
|
|
57165
58521
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
57166
58522
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
57167
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
58523
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
57168
58524
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
57169
58525
|
for (const probe of probeRows) {
|
|
57170
58526
|
if (!probe.bucket) continue;
|
|
@@ -57172,7 +58528,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57172
58528
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
57173
58529
|
}
|
|
57174
58530
|
const existingQueries = new Set(
|
|
57175
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
58531
|
+
app.db.select({ query: queries.query }).from(queries).where(eq55(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
57176
58532
|
);
|
|
57177
58533
|
const promotedQueries = [];
|
|
57178
58534
|
const skippedQueries = [];
|
|
@@ -57188,7 +58544,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57188
58544
|
const skippedCompetitors = [];
|
|
57189
58545
|
if (includeCompetitors) {
|
|
57190
58546
|
const existingCompetitors = new Set(
|
|
57191
|
-
app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
58547
|
+
app.db.select({ domain: competitors.domain }).from(competitors).where(eq55(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase())
|
|
57192
58548
|
);
|
|
57193
58549
|
const competitorMap = parseCompetitorMap(session.competitorMap);
|
|
57194
58550
|
for (const entry of selectEligibleCompetitors(competitorMap, competitorTypes)) {
|
|
@@ -57307,7 +58663,7 @@ function selectEligibleCompetitors(competitorMap, competitorTypes) {
|
|
|
57307
58663
|
|
|
57308
58664
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
57309
58665
|
import crypto43 from "crypto";
|
|
57310
|
-
import { eq as
|
|
58666
|
+
import { eq as eq56 } from "drizzle-orm";
|
|
57311
58667
|
var DEFAULT_MAX_PROBES = 100;
|
|
57312
58668
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
57313
58669
|
function classifyProbeBucket(input) {
|
|
@@ -57394,7 +58750,7 @@ async function executeDiscovery(opts) {
|
|
|
57394
58750
|
status: DiscoverySessionStatuses.seeding,
|
|
57395
58751
|
dedupThreshold,
|
|
57396
58752
|
startedAt
|
|
57397
|
-
}).where(
|
|
58753
|
+
}).where(eq56(discoverySessions.id, opts.sessionId)).run();
|
|
57398
58754
|
const seedResult = await opts.deps.seed({
|
|
57399
58755
|
project: opts.project,
|
|
57400
58756
|
icpDescription: opts.icpDescription,
|
|
@@ -57455,7 +58811,7 @@ async function executeDiscovery(opts) {
|
|
|
57455
58811
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
57456
58812
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
57457
58813
|
warning
|
|
57458
|
-
}).where(
|
|
58814
|
+
}).where(eq56(discoverySessions.id, opts.sessionId)).run();
|
|
57459
58815
|
const probeLocation = opts.locations?.[0];
|
|
57460
58816
|
const probeResults = await mapWithConcurrency(
|
|
57461
58817
|
probedCanonicals,
|
|
@@ -57509,7 +58865,7 @@ async function executeDiscovery(opts) {
|
|
|
57509
58865
|
wastedCount: buckets["wasted-surface"],
|
|
57510
58866
|
competitorMap,
|
|
57511
58867
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
57512
|
-
}).where(
|
|
58868
|
+
}).where(eq56(discoverySessions.id, opts.sessionId)).run();
|
|
57513
58869
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
57514
58870
|
return {
|
|
57515
58871
|
buckets,
|
|
@@ -57549,7 +58905,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
57549
58905
|
status: DiscoverySessionStatuses.failed,
|
|
57550
58906
|
error,
|
|
57551
58907
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
57552
|
-
}).where(
|
|
58908
|
+
}).where(eq56(discoverySessions.id, sessionId)).run();
|
|
57553
58909
|
}
|
|
57554
58910
|
function dedupeStrings(input) {
|
|
57555
58911
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -57567,7 +58923,7 @@ function dedupeStrings(input) {
|
|
|
57567
58923
|
|
|
57568
58924
|
// ../api-routes/src/technical-aeo.ts
|
|
57569
58925
|
import crypto44 from "crypto";
|
|
57570
|
-
import { and as
|
|
58926
|
+
import { and as and45, asc as asc10, count, desc as desc24, eq as eq57, inArray as inArray20, isNotNull as isNotNull3, isNull as isNull6, lt as lt8, or as or12, sql as sql21 } from "drizzle-orm";
|
|
57571
58927
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
57572
58928
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
57573
58929
|
...SiteCrawlFetchedStates,
|
|
@@ -57615,13 +58971,13 @@ function emptyScore(projectName) {
|
|
|
57615
58971
|
prioritizedFixes: []
|
|
57616
58972
|
};
|
|
57617
58973
|
}
|
|
57618
|
-
function parsePositiveInt(value, fallback,
|
|
58974
|
+
function parsePositiveInt(value, fallback, max2) {
|
|
57619
58975
|
const n = typeof value === "string" ? Number.parseInt(value, 10) : typeof value === "number" ? value : NaN;
|
|
57620
58976
|
if (!Number.isFinite(n) || n < 0) return fallback;
|
|
57621
|
-
return Math.min(
|
|
58977
|
+
return Math.min(max2, Math.floor(n));
|
|
57622
58978
|
}
|
|
57623
|
-
function parseBoundedLimit(value, fallback,
|
|
57624
|
-
const parsed = parsePositiveInt(value, fallback,
|
|
58979
|
+
function parseBoundedLimit(value, fallback, max2) {
|
|
58980
|
+
const parsed = parsePositiveInt(value, fallback, max2);
|
|
57625
58981
|
return Math.max(1, parsed);
|
|
57626
58982
|
}
|
|
57627
58983
|
function parseSiteHealthState(value) {
|
|
@@ -57766,8 +59122,8 @@ function parseLinkKind(value) {
|
|
|
57766
59122
|
return allowed.data;
|
|
57767
59123
|
}
|
|
57768
59124
|
function linkKindFilter(linkKind) {
|
|
57769
|
-
if (linkKind === SiteHealthLinkKinds.content) return
|
|
57770
|
-
if (linkKind === SiteHealthLinkKinds.template) return
|
|
59125
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq57(siteCrawlEdges.isTemplate, false);
|
|
59126
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq57(siteCrawlEdges.isTemplate, true);
|
|
57771
59127
|
return void 0;
|
|
57772
59128
|
}
|
|
57773
59129
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
@@ -57872,50 +59228,50 @@ function changedFields(before, after, fields) {
|
|
|
57872
59228
|
async function technicalAeoRoutes(app, opts) {
|
|
57873
59229
|
const resolveCrawl = (projectId, runId) => {
|
|
57874
59230
|
const filters = [
|
|
57875
|
-
|
|
57876
|
-
|
|
57877
|
-
|
|
57878
|
-
|
|
59231
|
+
eq57(siteCrawlSnapshots.projectId, projectId),
|
|
59232
|
+
eq57(runs.projectId, projectId),
|
|
59233
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59234
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57879
59235
|
notProbeRun()
|
|
57880
59236
|
];
|
|
57881
59237
|
if (runId) {
|
|
57882
|
-
filters.push(
|
|
59238
|
+
filters.push(eq57(siteCrawlSnapshots.runId, runId));
|
|
57883
59239
|
} else {
|
|
57884
|
-
filters.push(
|
|
59240
|
+
filters.push(eq57(siteCrawlSnapshots.complete, true), eq57(runs.status, RunStatuses.completed));
|
|
57885
59241
|
}
|
|
57886
|
-
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59242
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
57887
59243
|
};
|
|
57888
59244
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
57889
59245
|
const pageInScope = (scope, selector) => {
|
|
57890
59246
|
if (!selector.nodeKey && !selector.url) return null;
|
|
57891
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
57892
|
-
|
|
57893
|
-
|
|
57894
|
-
|
|
57895
|
-
selector.nodeKey ?
|
|
59247
|
+
return app.db.select().from(siteCrawlPages).where(and45(
|
|
59248
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59249
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59250
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59251
|
+
selector.nodeKey ? eq57(siteCrawlPages.nodeKey, selector.nodeKey) : eq57(siteCrawlPages.url, selector.url)
|
|
57896
59252
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57897
59253
|
};
|
|
57898
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
57899
|
-
|
|
57900
|
-
|
|
57901
|
-
|
|
57902
|
-
|
|
59254
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and45(
|
|
59255
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59256
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59257
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59258
|
+
eq57(siteCrawlPages.depth, 0)
|
|
57903
59259
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57904
|
-
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(
|
|
57905
|
-
|
|
57906
|
-
|
|
57907
|
-
|
|
57908
|
-
|
|
59260
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and45(
|
|
59261
|
+
eq57(runs.id, runId),
|
|
59262
|
+
eq57(runs.projectId, projectId),
|
|
59263
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59264
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57909
59265
|
notProbeRun()
|
|
57910
59266
|
)).limit(1).get());
|
|
57911
59267
|
const assertKnownAuditRun = (projectId, runId) => {
|
|
57912
59268
|
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
57913
59269
|
};
|
|
57914
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57915
|
-
|
|
57916
|
-
|
|
57917
|
-
|
|
57918
|
-
|
|
59270
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(
|
|
59271
|
+
eq57(siteAuditSnapshots.projectId, projectId),
|
|
59272
|
+
eq57(runs.projectId, projectId),
|
|
59273
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59274
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57919
59275
|
notProbeRun()
|
|
57920
59276
|
)).limit(1).get());
|
|
57921
59277
|
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
@@ -57936,20 +59292,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57936
59292
|
app.get("/projects/:name/technical-aeo", async (request) => {
|
|
57937
59293
|
const project = resolveProject(app.db, request.params.name);
|
|
57938
59294
|
const baseFilters = [
|
|
57939
|
-
|
|
57940
|
-
|
|
57941
|
-
|
|
57942
|
-
|
|
59295
|
+
eq57(siteAuditSnapshots.projectId, project.id),
|
|
59296
|
+
eq57(runs.projectId, project.id),
|
|
59297
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59298
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57943
59299
|
notProbeRun()
|
|
57944
59300
|
];
|
|
57945
|
-
const targetFilters = request.query.runId ? [...baseFilters,
|
|
57946
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
59301
|
+
const targetFilters = request.query.runId ? [...baseFilters, eq57(siteAuditSnapshots.runId, request.query.runId)] : baseFilters;
|
|
59302
|
+
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
57947
59303
|
if (!latest) {
|
|
57948
59304
|
if (request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
57949
59305
|
return emptyScore(project.name);
|
|
57950
59306
|
}
|
|
57951
59307
|
const snap = latest.snap;
|
|
57952
|
-
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs,
|
|
59308
|
+
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(...baseFilters, lt8(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
57953
59309
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
57954
59310
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
57955
59311
|
return {
|
|
@@ -57976,14 +59332,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57976
59332
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
57977
59333
|
const project = resolveProject(app.db, request.params.name);
|
|
57978
59334
|
const targetFilters = [
|
|
57979
|
-
|
|
57980
|
-
|
|
57981
|
-
|
|
57982
|
-
|
|
59335
|
+
eq57(siteAuditSnapshots.projectId, project.id),
|
|
59336
|
+
eq57(runs.projectId, project.id),
|
|
59337
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59338
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57983
59339
|
notProbeRun()
|
|
57984
59340
|
];
|
|
57985
|
-
if (request.query.runId) targetFilters.push(
|
|
57986
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs,
|
|
59341
|
+
if (request.query.runId) targetFilters.push(eq57(siteAuditSnapshots.runId, request.query.runId));
|
|
59342
|
+
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
57987
59343
|
if (!latest && request.query.runId) {
|
|
57988
59344
|
throw notFound("Site audit run", request.query.runId);
|
|
57989
59345
|
}
|
|
@@ -57991,9 +59347,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57991
59347
|
return { project: project.name, runId: null, auditedAt: null, total: 0, pages: [] };
|
|
57992
59348
|
}
|
|
57993
59349
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
57994
|
-
const conds = [
|
|
57995
|
-
if (statusFilter) conds.push(
|
|
57996
|
-
const where =
|
|
59350
|
+
const conds = [eq57(siteAuditPages.projectId, project.id), eq57(siteAuditPages.runId, latest.runId)];
|
|
59351
|
+
if (statusFilter) conds.push(eq57(siteAuditPages.status, statusFilter));
|
|
59352
|
+
const where = and45(...conds);
|
|
57997
59353
|
const totalRow = app.db.select({ value: count() }).from(siteAuditPages).where(where).get();
|
|
57998
59354
|
const total = totalRow?.value ?? 0;
|
|
57999
59355
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
@@ -58017,11 +59373,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58017
59373
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
58018
59374
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
58019
59375
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
58020
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
58021
|
-
|
|
58022
|
-
|
|
58023
|
-
|
|
58024
|
-
|
|
59376
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(
|
|
59377
|
+
eq57(siteAuditSnapshots.projectId, project.id),
|
|
59378
|
+
eq57(runs.projectId, project.id),
|
|
59379
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59380
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
58025
59381
|
notProbeRun()
|
|
58026
59382
|
)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
58027
59383
|
return { project: project.name, points: rows.reverse() };
|
|
@@ -58115,10 +59471,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58115
59471
|
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
58116
59472
|
snapshot.rootUrl
|
|
58117
59473
|
)?.nodeKey ?? null;
|
|
58118
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
58119
|
-
|
|
58120
|
-
|
|
58121
|
-
|
|
59474
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and45(
|
|
59475
|
+
eq57(siteCrawlGraphLayouts.projectId, project.id),
|
|
59476
|
+
eq57(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
59477
|
+
eq57(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
58122
59478
|
)).limit(1).get();
|
|
58123
59479
|
if (!persistedLayout) {
|
|
58124
59480
|
return {
|
|
@@ -58169,9 +59525,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58169
59525
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
58170
59526
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
58171
59527
|
const graphScope = [
|
|
58172
|
-
|
|
58173
|
-
|
|
58174
|
-
|
|
59528
|
+
eq57(siteCrawlGraphNodes.projectId, project.id),
|
|
59529
|
+
eq57(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
59530
|
+
eq57(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
58175
59531
|
];
|
|
58176
59532
|
const nodeRows = app.db.select({
|
|
58177
59533
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -58190,17 +59546,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58190
59546
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
58191
59547
|
x: siteCrawlGraphNodes.x,
|
|
58192
59548
|
y: siteCrawlGraphNodes.y
|
|
58193
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
58194
|
-
|
|
58195
|
-
|
|
58196
|
-
|
|
58197
|
-
|
|
58198
|
-
)).where(
|
|
59549
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and45(
|
|
59550
|
+
eq57(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
59551
|
+
eq57(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
59552
|
+
eq57(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
59553
|
+
eq57(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
59554
|
+
)).where(and45(...graphScope, lt8(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
58199
59555
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
58200
59556
|
...row,
|
|
58201
59557
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
58202
59558
|
}));
|
|
58203
|
-
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 :
|
|
59559
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq57(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
58204
59560
|
const edges = app.db.select({
|
|
58205
59561
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
58206
59562
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
@@ -58208,22 +59564,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58208
59564
|
followable: siteCrawlGraphEdges.followable,
|
|
58209
59565
|
occurrences: siteCrawlGraphEdges.occurrences,
|
|
58210
59566
|
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
58211
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
58212
|
-
|
|
58213
|
-
|
|
58214
|
-
|
|
58215
|
-
|
|
59567
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and45(
|
|
59568
|
+
eq57(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
59569
|
+
eq57(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
59570
|
+
eq57(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
59571
|
+
eq57(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
58216
59572
|
lt8(graphSourceNode.sampleRank, maxNodes)
|
|
58217
|
-
)).innerJoin(graphTargetNode,
|
|
58218
|
-
|
|
58219
|
-
|
|
58220
|
-
|
|
58221
|
-
|
|
59573
|
+
)).innerJoin(graphTargetNode, and45(
|
|
59574
|
+
eq57(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
59575
|
+
eq57(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
59576
|
+
eq57(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
59577
|
+
eq57(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
58222
59578
|
lt8(graphTargetNode.sampleRank, maxNodes)
|
|
58223
|
-
)).where(
|
|
58224
|
-
|
|
58225
|
-
|
|
58226
|
-
|
|
59579
|
+
)).where(and45(
|
|
59580
|
+
eq57(siteCrawlGraphEdges.projectId, project.id),
|
|
59581
|
+
eq57(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
59582
|
+
eq57(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
58227
59583
|
lt8(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
58228
59584
|
graphLinkKindFilter
|
|
58229
59585
|
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
@@ -58317,15 +59673,15 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58317
59673
|
let frontier = [focus.nodeKey];
|
|
58318
59674
|
let queryTruncated = false;
|
|
58319
59675
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
58320
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
58321
|
-
|
|
58322
|
-
|
|
58323
|
-
|
|
58324
|
-
|
|
59676
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and45(
|
|
59677
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
59678
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
59679
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
59680
|
+
eq57(siteCrawlEdges.internal, true),
|
|
58325
59681
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
58326
|
-
|
|
58327
|
-
|
|
58328
|
-
|
|
59682
|
+
or12(
|
|
59683
|
+
inArray20(siteCrawlEdges.sourceNodeKey, frontier),
|
|
59684
|
+
inArray20(siteCrawlEdges.targetNodeKey, frontier)
|
|
58329
59685
|
)
|
|
58330
59686
|
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
58331
59687
|
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
@@ -58358,22 +59714,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58358
59714
|
frontier = [...next].sort();
|
|
58359
59715
|
}
|
|
58360
59716
|
const nodeKeys = [...distances.keys()];
|
|
58361
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
58362
|
-
|
|
58363
|
-
|
|
58364
|
-
|
|
58365
|
-
|
|
59717
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and45(
|
|
59718
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59719
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59720
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59721
|
+
inArray20(siteCrawlPages.nodeKey, nodeKeys)
|
|
58366
59722
|
)).all();
|
|
58367
59723
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
58368
59724
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
58369
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
58370
|
-
|
|
58371
|
-
|
|
58372
|
-
|
|
58373
|
-
|
|
59725
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and45(
|
|
59726
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
59727
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
59728
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
59729
|
+
eq57(siteCrawlEdges.internal, true),
|
|
58374
59730
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
58375
|
-
|
|
58376
|
-
|
|
59731
|
+
inArray20(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
59732
|
+
inArray20(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
58377
59733
|
)).limit(1).get();
|
|
58378
59734
|
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
58379
59735
|
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
@@ -58467,23 +59823,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58467
59823
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
58468
59824
|
let truncated = false;
|
|
58469
59825
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
58470
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
58471
|
-
|
|
58472
|
-
|
|
58473
|
-
|
|
58474
|
-
|
|
58475
|
-
|
|
58476
|
-
|
|
59826
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and45(
|
|
59827
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
59828
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
59829
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
59830
|
+
eq57(siteCrawlEdges.internal, true),
|
|
59831
|
+
eq57(siteCrawlEdges.followable, true),
|
|
59832
|
+
eq57(siteCrawlEdges.relation, "anchor"),
|
|
58477
59833
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
58478
|
-
|
|
59834
|
+
inArray20(siteCrawlEdges.sourceNodeKey, frontier)
|
|
58479
59835
|
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
58480
59836
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
58481
59837
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
58482
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
58483
|
-
|
|
58484
|
-
|
|
58485
|
-
|
|
58486
|
-
|
|
59838
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and45(
|
|
59839
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59840
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59841
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59842
|
+
inArray20(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
58487
59843
|
)).all().map((row) => row.nodeKey));
|
|
58488
59844
|
const next = /* @__PURE__ */ new Set();
|
|
58489
59845
|
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
@@ -58525,11 +59881,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58525
59881
|
pathEdges.unshift(step.edge);
|
|
58526
59882
|
pathKeys.unshift(step.nodeKey);
|
|
58527
59883
|
}
|
|
58528
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
58529
|
-
|
|
58530
|
-
|
|
58531
|
-
|
|
58532
|
-
|
|
59884
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and45(
|
|
59885
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59886
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59887
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59888
|
+
inArray20(siteCrawlPages.nodeKey, pathKeys)
|
|
58533
59889
|
)).all();
|
|
58534
59890
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
58535
59891
|
return {
|
|
@@ -58557,21 +59913,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58557
59913
|
throw validationError("change must be all, added, removed, or changed");
|
|
58558
59914
|
}
|
|
58559
59915
|
const completeSnapshotFilters = [
|
|
58560
|
-
|
|
58561
|
-
|
|
58562
|
-
|
|
58563
|
-
|
|
58564
|
-
|
|
59916
|
+
eq57(siteCrawlSnapshots.projectId, project.id),
|
|
59917
|
+
eq57(runs.projectId, project.id),
|
|
59918
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59919
|
+
eq57(runs.status, RunStatuses.completed),
|
|
59920
|
+
eq57(siteCrawlSnapshots.complete, true),
|
|
58565
59921
|
notProbeRun()
|
|
58566
59922
|
];
|
|
58567
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59923
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
58568
59924
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
58569
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59925
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(
|
|
58570
59926
|
...completeSnapshotFilters,
|
|
58571
|
-
|
|
59927
|
+
or12(
|
|
58572
59928
|
lt8(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
58573
|
-
|
|
58574
|
-
|
|
59929
|
+
and45(
|
|
59930
|
+
eq57(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
58575
59931
|
lt8(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
58576
59932
|
)
|
|
58577
59933
|
)
|
|
@@ -58806,18 +60162,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58806
60162
|
const visibleKeys = keyRows.slice(0, limit);
|
|
58807
60163
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
58808
60164
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
58809
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
58810
|
-
|
|
58811
|
-
|
|
58812
|
-
|
|
58813
|
-
|
|
60165
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and45(
|
|
60166
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
60167
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
60168
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
60169
|
+
inArray20(siteCrawlPages.nodeKey, pageKeys)
|
|
58814
60170
|
)).all();
|
|
58815
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
58816
|
-
|
|
58817
|
-
|
|
58818
|
-
|
|
58819
|
-
|
|
58820
|
-
|
|
60171
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and45(
|
|
60172
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
60173
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
60174
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
60175
|
+
eq57(siteCrawlEdges.internal, true),
|
|
60176
|
+
inArray20(siteCrawlEdges.edgeKey, linkKeys)
|
|
58821
60177
|
)).all();
|
|
58822
60178
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
58823
60179
|
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -58940,30 +60296,30 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58940
60296
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, healthStateFilter: null, pages: [] };
|
|
58941
60297
|
}
|
|
58942
60298
|
const filters = [
|
|
58943
|
-
|
|
58944
|
-
|
|
58945
|
-
|
|
60299
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60300
|
+
eq57(siteCrawlPages.runId, snapshot.runId),
|
|
60301
|
+
eq57(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
58946
60302
|
];
|
|
58947
|
-
if (request.query.nodeKey) filters.push(
|
|
60303
|
+
if (request.query.nodeKey) filters.push(eq57(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
58948
60304
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
58949
|
-
if (inventoryEligible != null) filters.push(
|
|
58950
|
-
if (request.query.fetchState) filters.push(
|
|
58951
|
-
if (request.query.indexabilityState) filters.push(
|
|
58952
|
-
if (request.query.auditState) filters.push(
|
|
60305
|
+
if (inventoryEligible != null) filters.push(eq57(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
60306
|
+
if (request.query.fetchState) filters.push(eq57(siteCrawlPages.fetchState, request.query.fetchState));
|
|
60307
|
+
if (request.query.indexabilityState) filters.push(eq57(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
60308
|
+
if (request.query.auditState) filters.push(eq57(siteCrawlPages.auditState, request.query.auditState));
|
|
58953
60309
|
const healthState = parseSiteHealthState(request.query.healthState);
|
|
58954
60310
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
58955
60311
|
const offset = decodeCursor(request.query.cursor);
|
|
58956
60312
|
const orderBy = request.query.sort === "score-desc" ? [desc24(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "score-asc" ? [asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "path" ? [asc10(siteCrawlPages.path), asc10(siteCrawlPages.nodeKey)] : [asc10(siteCrawlPages.url), asc10(siteCrawlPages.nodeKey)];
|
|
58957
60313
|
let healthStateFilter = null;
|
|
58958
60314
|
if (healthState) {
|
|
58959
|
-
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(
|
|
58960
|
-
|
|
58961
|
-
|
|
58962
|
-
|
|
58963
|
-
|
|
60315
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and45(
|
|
60316
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60317
|
+
eq57(siteCrawlPages.runId, snapshot.runId),
|
|
60318
|
+
eq57(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
60319
|
+
isNull6(siteCrawlPages.healthState)
|
|
58964
60320
|
)).limit(1).get();
|
|
58965
60321
|
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
58966
|
-
if (healthStateFilter === "applied") filters.push(
|
|
60322
|
+
if (healthStateFilter === "applied") filters.push(eq57(siteCrawlPages.healthState, healthState));
|
|
58967
60323
|
}
|
|
58968
60324
|
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
58969
60325
|
return {
|
|
@@ -58976,7 +60332,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58976
60332
|
pages: []
|
|
58977
60333
|
};
|
|
58978
60334
|
}
|
|
58979
|
-
const where =
|
|
60335
|
+
const where = and45(...filters);
|
|
58980
60336
|
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
58981
60337
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
58982
60338
|
const nextOffset = offset + rows.length;
|
|
@@ -59007,10 +60363,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59007
60363
|
url: siteCrawlPages.url,
|
|
59008
60364
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
59009
60365
|
fetchState: siteCrawlPages.fetchState
|
|
59010
|
-
}).from(siteCrawlPages).where(
|
|
59011
|
-
|
|
59012
|
-
|
|
59013
|
-
|
|
60366
|
+
}).from(siteCrawlPages).where(and45(
|
|
60367
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60368
|
+
eq57(siteCrawlPages.runId, snapshot.runId),
|
|
60369
|
+
eq57(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
59014
60370
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
59015
60371
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
59016
60372
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -59089,17 +60445,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59089
60445
|
};
|
|
59090
60446
|
}
|
|
59091
60447
|
const filters = [
|
|
59092
|
-
|
|
59093
|
-
|
|
59094
|
-
|
|
59095
|
-
|
|
60448
|
+
eq57(siteCrawlEdges.projectId, project.id),
|
|
60449
|
+
eq57(siteCrawlEdges.runId, snapshot.runId),
|
|
60450
|
+
eq57(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
60451
|
+
eq57(siteCrawlEdges.internal, true),
|
|
59096
60452
|
linkKindFilter(linkKind)
|
|
59097
60453
|
];
|
|
59098
|
-
if (request.query.sourceUrl) filters.push(
|
|
59099
|
-
if (request.query.targetUrl) filters.push(
|
|
60454
|
+
if (request.query.sourceUrl) filters.push(eq57(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
60455
|
+
if (request.query.targetUrl) filters.push(eq57(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
59100
60456
|
const followable = parseBoolean2(request.query.followable);
|
|
59101
|
-
if (followable != null) filters.push(
|
|
59102
|
-
const where =
|
|
60457
|
+
if (followable != null) filters.push(eq57(siteCrawlEdges.followable, followable));
|
|
60458
|
+
const where = and45(...filters);
|
|
59103
60459
|
const total = app.db.select({ value: count() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
59104
60460
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
59105
60461
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -59157,17 +60513,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59157
60513
|
};
|
|
59158
60514
|
}
|
|
59159
60515
|
const scope = [
|
|
59160
|
-
|
|
59161
|
-
|
|
59162
|
-
|
|
59163
|
-
|
|
60516
|
+
eq57(siteCrawlEdges.projectId, project.id),
|
|
60517
|
+
eq57(siteCrawlEdges.runId, snapshot.runId),
|
|
60518
|
+
eq57(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
60519
|
+
eq57(siteCrawlEdges.internal, true),
|
|
59164
60520
|
linkKindFilter(linkKind)
|
|
59165
60521
|
];
|
|
59166
|
-
const inboundMatch = request.query.nodeKey && request.query.url ?
|
|
59167
|
-
const outboundMatch = request.query.nodeKey && request.query.url ?
|
|
60522
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or12(eq57(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq57(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq57(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq57(siteCrawlEdges.targetUrl, request.query.url);
|
|
60523
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or12(eq57(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq57(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq57(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq57(siteCrawlEdges.sourceUrl, request.query.url);
|
|
59168
60524
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
59169
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
59170
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
60525
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and45(...scope, inboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
60526
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and45(...scope, outboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
59171
60527
|
return {
|
|
59172
60528
|
project: project.name,
|
|
59173
60529
|
hasCrawlData: true,
|
|
@@ -59200,11 +60556,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59200
60556
|
if (!snapshot.attemptId) {
|
|
59201
60557
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, total: 0, nextCursor: null, deadLinks: [] };
|
|
59202
60558
|
}
|
|
59203
|
-
const where =
|
|
59204
|
-
|
|
59205
|
-
|
|
59206
|
-
|
|
59207
|
-
|
|
60559
|
+
const where = and45(
|
|
60560
|
+
eq57(siteCrawlFindings.projectId, project.id),
|
|
60561
|
+
eq57(siteCrawlFindings.runId, snapshot.runId),
|
|
60562
|
+
eq57(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
60563
|
+
eq57(siteCrawlFindings.findingType, "dead-link")
|
|
59208
60564
|
);
|
|
59209
60565
|
const total = app.db.select({ value: count() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
59210
60566
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -59242,19 +60598,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59242
60598
|
createdAt: runs.createdAt,
|
|
59243
60599
|
startedAt: runs.startedAt,
|
|
59244
60600
|
finishedAt: runs.finishedAt
|
|
59245
|
-
}).from(runs).where(
|
|
59246
|
-
|
|
59247
|
-
|
|
60601
|
+
}).from(runs).where(and45(
|
|
60602
|
+
eq57(runs.projectId, project.id),
|
|
60603
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59248
60604
|
notProbeRun()
|
|
59249
60605
|
)).orderBy(desc24(runs.createdAt), desc24(runs.id)).limit(limit).all();
|
|
59250
60606
|
const runIds = rows.map((row) => row.runId);
|
|
59251
|
-
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59252
|
-
|
|
59253
|
-
|
|
59254
|
-
|
|
59255
|
-
|
|
60607
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(
|
|
60608
|
+
eq57(siteCrawlSnapshots.projectId, project.id),
|
|
60609
|
+
eq57(runs.projectId, project.id),
|
|
60610
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
60611
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
59256
60612
|
notProbeRun(),
|
|
59257
|
-
|
|
60613
|
+
inArray20(siteCrawlSnapshots.runId, runIds)
|
|
59258
60614
|
)).all().map((row) => row.runId));
|
|
59259
60615
|
return {
|
|
59260
60616
|
project: project.name,
|
|
@@ -59267,21 +60623,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59267
60623
|
});
|
|
59268
60624
|
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
59269
60625
|
const project = resolveProject(app.db, request.params.name);
|
|
59270
|
-
const run = app.db.select().from(runs).where(
|
|
59271
|
-
|
|
59272
|
-
|
|
59273
|
-
|
|
60626
|
+
const run = app.db.select().from(runs).where(and45(
|
|
60627
|
+
eq57(runs.id, request.params.runId),
|
|
60628
|
+
eq57(runs.projectId, project.id),
|
|
60629
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59274
60630
|
notProbeRun()
|
|
59275
60631
|
)).get();
|
|
59276
60632
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
59277
|
-
const attempt = app.db.select().from(siteCrawlAttempts).where(
|
|
59278
|
-
|
|
59279
|
-
|
|
60633
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and45(
|
|
60634
|
+
eq57(siteCrawlAttempts.projectId, project.id),
|
|
60635
|
+
eq57(siteCrawlAttempts.runId, run.id)
|
|
59280
60636
|
)).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
59281
|
-
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(
|
|
59282
|
-
|
|
59283
|
-
|
|
59284
|
-
|
|
60637
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and45(
|
|
60638
|
+
eq57(siteCrawlGraphLayouts.projectId, project.id),
|
|
60639
|
+
eq57(siteCrawlGraphLayouts.runId, run.id),
|
|
60640
|
+
eq57(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
59285
60641
|
)).get() : void 0;
|
|
59286
60642
|
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
59287
60643
|
return {
|
|
@@ -59312,18 +60668,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59312
60668
|
};
|
|
59313
60669
|
});
|
|
59314
60670
|
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
59315
|
-
const project = tx.select().from(projects).where(
|
|
60671
|
+
const project = tx.select().from(projects).where(eq57(projects.name, request.params.name)).get();
|
|
59316
60672
|
if (!project) throw notFound("Project", request.params.name);
|
|
59317
|
-
const run = tx.select().from(runs).where(
|
|
59318
|
-
|
|
59319
|
-
|
|
59320
|
-
|
|
60673
|
+
const run = tx.select().from(runs).where(and45(
|
|
60674
|
+
eq57(runs.id, request.params.runId),
|
|
60675
|
+
eq57(runs.projectId, project.id),
|
|
60676
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59321
60677
|
notProbeRun()
|
|
59322
60678
|
)).get();
|
|
59323
60679
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
59324
|
-
const attempt = tx.select().from(siteCrawlAttempts).where(
|
|
59325
|
-
|
|
59326
|
-
|
|
60680
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and45(
|
|
60681
|
+
eq57(siteCrawlAttempts.projectId, project.id),
|
|
60682
|
+
eq57(siteCrawlAttempts.runId, run.id)
|
|
59327
60683
|
)).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
59328
60684
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
59329
60685
|
const base = {
|
|
@@ -59337,11 +60693,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59337
60693
|
if (!attempt) {
|
|
59338
60694
|
return { ...base, pagesAudited: 0, examples: [] };
|
|
59339
60695
|
}
|
|
59340
|
-
const auditedWhere =
|
|
59341
|
-
|
|
59342
|
-
|
|
59343
|
-
|
|
59344
|
-
|
|
60696
|
+
const auditedWhere = and45(
|
|
60697
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60698
|
+
eq57(siteCrawlPages.runId, run.id),
|
|
60699
|
+
eq57(siteCrawlPages.attemptId, attempt.id),
|
|
60700
|
+
eq57(siteCrawlPages.auditState, "success")
|
|
59345
60701
|
);
|
|
59346
60702
|
const pagesAudited = tx.select({ value: count() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
59347
60703
|
if (state !== "collecting") {
|
|
@@ -59352,7 +60708,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59352
60708
|
url: siteCrawlPages.url,
|
|
59353
60709
|
auditScore: siteCrawlPages.auditScore,
|
|
59354
60710
|
auditFields: siteCrawlPages.auditFields
|
|
59355
|
-
}).from(siteCrawlPages).where(
|
|
60711
|
+
}).from(siteCrawlPages).where(and45(
|
|
59356
60712
|
auditedWhere,
|
|
59357
60713
|
isNotNull3(siteCrawlPages.auditScore),
|
|
59358
60714
|
lt8(siteCrawlPages.auditScore, 70)
|
|
@@ -59388,13 +60744,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59388
60744
|
status: runs.status,
|
|
59389
60745
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
59390
60746
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
59391
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
59392
|
-
|
|
59393
|
-
|
|
59394
|
-
)).where(
|
|
59395
|
-
|
|
59396
|
-
|
|
59397
|
-
|
|
60747
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and45(
|
|
60748
|
+
eq57(siteCrawlRunRequests.projectId, runs.projectId),
|
|
60749
|
+
eq57(siteCrawlRunRequests.runId, runs.id)
|
|
60750
|
+
)).where(and45(
|
|
60751
|
+
eq57(runs.projectId, project.id),
|
|
60752
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
60753
|
+
inArray20(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
59398
60754
|
)).get();
|
|
59399
60755
|
if (existing) {
|
|
59400
60756
|
if (existing.identityKey === identityKey) {
|
|
@@ -59445,7 +60801,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59445
60801
|
|
|
59446
60802
|
// ../api-routes/src/research.ts
|
|
59447
60803
|
import crypto45 from "crypto";
|
|
59448
|
-
import { and as
|
|
60804
|
+
import { and as and46, desc as desc25, eq as eq58 } from "drizzle-orm";
|
|
59449
60805
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
59450
60806
|
async function researchRoutes(app, opts) {
|
|
59451
60807
|
app.post("/projects/:name/research/runs", async (request, reply) => {
|
|
@@ -59475,7 +60831,7 @@ async function researchRoutes(app, opts) {
|
|
|
59475
60831
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
59476
60832
|
const decision = app.db.transaction((tx) => {
|
|
59477
60833
|
if (input.idempotencyKey) {
|
|
59478
|
-
const existing = tx.select().from(researchRuns).where(
|
|
60834
|
+
const existing = tx.select().from(researchRuns).where(and46(eq58(researchRuns.projectId, project.id), eq58(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
59479
60835
|
if (existing) {
|
|
59480
60836
|
if (existing.requestHash !== requestHash2) throw alreadyExists("Research idempotency key", input.idempotencyKey);
|
|
59481
60837
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
@@ -59496,7 +60852,7 @@ async function researchRoutes(app, opts) {
|
|
|
59496
60852
|
const project = resolveProject(app.db, request.params.name);
|
|
59497
60853
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
59498
60854
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
59499
|
-
const runs2 = app.db.select().from(researchRuns).where(
|
|
60855
|
+
const runs2 = app.db.select().from(researchRuns).where(eq58(researchRuns.projectId, project.id)).orderBy(desc25(researchRuns.createdAt)).limit(limit).all().map(serializeRun);
|
|
59500
60856
|
return { runs: runs2 };
|
|
59501
60857
|
});
|
|
59502
60858
|
app.get("/projects/:name/research/runs/:runId", async (request) => {
|
|
@@ -59505,9 +60861,9 @@ async function researchRoutes(app, opts) {
|
|
|
59505
60861
|
});
|
|
59506
60862
|
}
|
|
59507
60863
|
function getDetail(app, projectId, id) {
|
|
59508
|
-
const row = app.db.select().from(researchRuns).where(
|
|
60864
|
+
const row = app.db.select().from(researchRuns).where(and46(eq58(researchRuns.id, id), eq58(researchRuns.projectId, projectId))).get();
|
|
59509
60865
|
if (!row) throw notFound("Research run", id);
|
|
59510
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
60866
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq58(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
59511
60867
|
return { ...serializeRun(row), queries: queries2 };
|
|
59512
60868
|
}
|
|
59513
60869
|
function serializeRun(row) {
|
|
@@ -59718,6 +61074,9 @@ async function apiRoutes(app, opts) {
|
|
|
59718
61074
|
pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
|
|
59719
61075
|
vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,
|
|
59720
61076
|
cloudflareTrafficCredentialStore: opts.cloudflareTrafficCredentialStore,
|
|
61077
|
+
pullCloudflareQueueMessages: opts.pullCloudflareQueueMessages,
|
|
61078
|
+
ackCloudflareQueueMessages: opts.ackCloudflareQueueMessages,
|
|
61079
|
+
cloudflareQueueMaxBatches: opts.cloudflareQueueMaxBatches,
|
|
59721
61080
|
cloudflareTrafficIngestUrl: opts.cloudflareTrafficIngestUrl,
|
|
59722
61081
|
cloudflareIngestRateLimitMax: opts.cloudflareIngestRateLimitMax,
|
|
59723
61082
|
cloudflareIngestIpRateLimitMax: opts.cloudflareIngestIpRateLimitMax,
|
|
@@ -59907,22 +61266,31 @@ function buildTrafficSourceValidators(opts) {
|
|
|
59907
61266
|
validators[TrafficSourceTypes.cloudflare] = {
|
|
59908
61267
|
validateCredentials: (source) => {
|
|
59909
61268
|
const sourceMode = source.configJson.deliveryMode;
|
|
59910
|
-
|
|
59911
|
-
const record = store.
|
|
61269
|
+
const deliveryMode = sourceMode === "queue-pull" ? "queue-pull" : "direct-push";
|
|
61270
|
+
const record = store.getConnectionBySourceId(source.id);
|
|
59912
61271
|
if (!record) {
|
|
61272
|
+
const projectRecord = store.getConnection(source.projectName);
|
|
61273
|
+
if (projectRecord && projectRecord.sourceId !== source.id) {
|
|
61274
|
+
return {
|
|
61275
|
+
status: CheckStatuses.fail,
|
|
61276
|
+
code: "traffic.credentials.source-mismatch",
|
|
61277
|
+
summary: `The stored Cloudflare credential belongs to a different source than "${source.displayName}".`,
|
|
61278
|
+
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
61279
|
+
};
|
|
61280
|
+
}
|
|
59913
61281
|
return {
|
|
59914
61282
|
status: CheckStatuses.fail,
|
|
59915
61283
|
code: "traffic.credentials.missing",
|
|
59916
|
-
summary: `No Cloudflare
|
|
61284
|
+
summary: `No Cloudflare ${deliveryMode} credential is stored for source "${source.displayName}".`,
|
|
59917
61285
|
remediation: "Re-run `canonry traffic connect cloudflare <project> --zone-id <id>` from the credential-owning host."
|
|
59918
61286
|
};
|
|
59919
61287
|
}
|
|
59920
61288
|
const recordMode = record.deliveryMode;
|
|
59921
|
-
if (recordMode !==
|
|
61289
|
+
if (recordMode !== deliveryMode) {
|
|
59922
61290
|
return {
|
|
59923
61291
|
status: CheckStatuses.fail,
|
|
59924
61292
|
code: "traffic.credentials.mode-mismatch",
|
|
59925
|
-
summary: `The stored Cloudflare credential mode does not match
|
|
61293
|
+
summary: `The stored Cloudflare credential mode does not match ${deliveryMode} source "${source.displayName}".`,
|
|
59926
61294
|
remediation: "Reconnect the Cloudflare source from the credential-owning host."
|
|
59927
61295
|
};
|
|
59928
61296
|
}
|
|
@@ -59934,9 +61302,23 @@ function buildTrafficSourceValidators(opts) {
|
|
|
59934
61302
|
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
59935
61303
|
};
|
|
59936
61304
|
}
|
|
59937
|
-
|
|
59938
|
-
|
|
59939
|
-
|
|
61305
|
+
if (record.deliveryMode === "queue-pull") {
|
|
61306
|
+
const queueConfig = source.configJson;
|
|
61307
|
+
if (typeof record.apiToken !== "string" || record.apiToken.length === 0 || typeof record.accountId !== "string" || record.accountId.length === 0 || typeof record.queueId !== "string" || record.queueId.length === 0 || typeof record.queueName !== "string" || record.queueName.length === 0 || !Number.isInteger(record.retentionSeconds) || record.retentionSeconds < 60 || record.retentionSeconds > 1209600 || queueConfig.accountId !== record.accountId || queueConfig.queueId !== record.queueId || queueConfig.queueName !== record.queueName || queueConfig.retentionSeconds !== record.retentionSeconds) {
|
|
61308
|
+
return {
|
|
61309
|
+
status: CheckStatuses.fail,
|
|
61310
|
+
code: "traffic.credentials.queue-mismatch",
|
|
61311
|
+
summary: `The stored Cloudflare Queue credential does not match source "${source.displayName}".`,
|
|
61312
|
+
remediation: "Reconnect the Cloudflare Queue source from the credential-owning host."
|
|
61313
|
+
};
|
|
61314
|
+
}
|
|
61315
|
+
return {
|
|
61316
|
+
status: CheckStatuses.ok,
|
|
61317
|
+
code: "traffic.credentials.resolved",
|
|
61318
|
+
summary: `Cloudflare Queue credentials match source "${source.displayName}".`
|
|
61319
|
+
};
|
|
61320
|
+
}
|
|
61321
|
+
if (typeof record.bearerToken !== "string" || record.bearerToken.length === 0 || typeof record.hmacSecret !== "string" || record.hmacSecret.length === 0 || !source.ingestTokenHash || hashCloudflareBearerToken(record.bearerToken) !== source.ingestTokenHash) {
|
|
59940
61322
|
return {
|
|
59941
61323
|
status: CheckStatuses.fail,
|
|
59942
61324
|
code: "traffic.credentials.bearer-mismatch",
|
|
@@ -59950,7 +61332,15 @@ function buildTrafficSourceValidators(opts) {
|
|
|
59950
61332
|
summary: `Cloudflare direct-push credentials match source "${source.displayName}".`
|
|
59951
61333
|
};
|
|
59952
61334
|
},
|
|
59953
|
-
validateScopes: () =>
|
|
61335
|
+
validateScopes: (source) => {
|
|
61336
|
+
if (source.configJson.deliveryMode !== "queue-pull") return null;
|
|
61337
|
+
return {
|
|
61338
|
+
status: CheckStatuses.skipped,
|
|
61339
|
+
code: "traffic.scopes.queue-pull-static",
|
|
61340
|
+
summary: `Cloudflare Queue token scopes for "${source.displayName}" are not inspected by Doctor.`,
|
|
61341
|
+
remediation: `Verify the token has Account Queues Edit permission and run \`wrangler queues consumer http add ${source.configJson.queueName}\` for the configured Queue, then re-run the Queue smoke test.`
|
|
61342
|
+
};
|
|
61343
|
+
}
|
|
59954
61344
|
};
|
|
59955
61345
|
}
|
|
59956
61346
|
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
@@ -60165,9 +61555,9 @@ var IntelligenceService = class {
|
|
|
60165
61555
|
*/
|
|
60166
61556
|
analyzeAndPersist(runId, projectId) {
|
|
60167
61557
|
const recentRuns = this.db.select().from(runs).where(
|
|
60168
|
-
|
|
60169
|
-
|
|
60170
|
-
|
|
61558
|
+
and47(
|
|
61559
|
+
eq59(runs.projectId, projectId),
|
|
61560
|
+
or13(eq59(runs.status, "completed"), eq59(runs.status, "partial")),
|
|
60171
61561
|
// Defensive: RunCoordinator already skips probes before this is
|
|
60172
61562
|
// called, but if a future call site invokes analyzeAndPersist
|
|
60173
61563
|
// directly for a probe, probes still must not pollute the
|
|
@@ -60249,7 +61639,7 @@ var IntelligenceService = class {
|
|
|
60249
61639
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
60250
61640
|
*/
|
|
60251
61641
|
analyzeAndPersistGbp(runId, projectId) {
|
|
60252
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
61642
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq59(runs.id, runId)).get();
|
|
60253
61643
|
if (!runRow) {
|
|
60254
61644
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
60255
61645
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -60257,11 +61647,11 @@ var IntelligenceService = class {
|
|
|
60257
61647
|
}
|
|
60258
61648
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
60259
61649
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
60260
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
60261
|
-
|
|
60262
|
-
|
|
61650
|
+
const selected = this.db.select().from(gbpLocations).where(and47(
|
|
61651
|
+
eq59(gbpLocations.projectId, projectId),
|
|
61652
|
+
eq59(gbpLocations.selected, true),
|
|
60263
61653
|
gte14(gbpLocations.syncedAt, windowStart),
|
|
60264
|
-
|
|
61654
|
+
lte12(gbpLocations.syncedAt, windowEnd)
|
|
60265
61655
|
)).all();
|
|
60266
61656
|
if (selected.length === 0) {
|
|
60267
61657
|
log.info("gbp-intelligence.skip", { runId, reason: "no locations synced during run" });
|
|
@@ -60294,12 +61684,12 @@ var IntelligenceService = class {
|
|
|
60294
61684
|
}
|
|
60295
61685
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
60296
61686
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
60297
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
60298
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
60299
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
60300
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
61687
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and47(eq59(gbpDailyMetrics.projectId, projectId), eq59(gbpDailyMetrics.locationName, locationName))).all();
|
|
61688
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and47(eq59(gbpPlaceActions.projectId, projectId), eq59(gbpPlaceActions.locationName, locationName))).all();
|
|
61689
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and47(eq59(gbpLodgingSnapshots.projectId, projectId), eq59(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc26(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
61690
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and47(eq59(gbpLocations.projectId, projectId), eq59(gbpLocations.locationName, locationName))).get();
|
|
60301
61691
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
60302
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
61692
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and47(eq59(gbpPlaceDetails.projectId, projectId), eq59(gbpPlaceDetails.locationName, locationName))).orderBy(desc26(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
60303
61693
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
60304
61694
|
const summary = buildGbpSummary({
|
|
60305
61695
|
locationName,
|
|
@@ -60335,7 +61725,7 @@ var IntelligenceService = class {
|
|
|
60335
61725
|
/** Build the month-over-month keyword series for a location from the
|
|
60336
61726
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
60337
61727
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
60338
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
61728
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and47(eq59(gbpKeywordMonthly.projectId, projectId), eq59(gbpKeywordMonthly.locationName, locationName))).all();
|
|
60339
61729
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
60340
61730
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
60341
61731
|
const recentMonth = months[0] ?? null;
|
|
@@ -60366,7 +61756,7 @@ var IntelligenceService = class {
|
|
|
60366
61756
|
*/
|
|
60367
61757
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
60368
61758
|
const covered = new Set(coveredLocationNames);
|
|
60369
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
61759
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and47(eq59(insights.projectId, projectId), eq59(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
60370
61760
|
const staleIds = [];
|
|
60371
61761
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
60372
61762
|
for (const row of existing) {
|
|
@@ -60377,7 +61767,7 @@ var IntelligenceService = class {
|
|
|
60377
61767
|
}
|
|
60378
61768
|
this.db.transaction((tx) => {
|
|
60379
61769
|
for (const id of staleIds) {
|
|
60380
|
-
tx.delete(insights).where(
|
|
61770
|
+
tx.delete(insights).where(eq59(insights.id, id)).run();
|
|
60381
61771
|
}
|
|
60382
61772
|
for (const insight of gbpInsights) {
|
|
60383
61773
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -60455,7 +61845,7 @@ var IntelligenceService = class {
|
|
|
60455
61845
|
* create per run + aggregate). DB is left untouched.
|
|
60456
61846
|
*/
|
|
60457
61847
|
backfill(projectName, opts, onProgress) {
|
|
60458
|
-
const project = this.db.select().from(projects).where(
|
|
61848
|
+
const project = this.db.select().from(projects).where(eq59(projects.name, projectName)).get();
|
|
60459
61849
|
if (!project) {
|
|
60460
61850
|
throw new Error(`Project "${projectName}" not found`);
|
|
60461
61851
|
}
|
|
@@ -60468,9 +61858,9 @@ var IntelligenceService = class {
|
|
|
60468
61858
|
sinceTimestamp = parsed;
|
|
60469
61859
|
}
|
|
60470
61860
|
const allRuns = this.db.select().from(runs).where(
|
|
60471
|
-
|
|
60472
|
-
|
|
60473
|
-
|
|
61861
|
+
and47(
|
|
61862
|
+
eq59(runs.projectId, project.id),
|
|
61863
|
+
or13(eq59(runs.status, "completed"), eq59(runs.status, "partial")),
|
|
60474
61864
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
60475
61865
|
ne7(runs.trigger, RunTriggers.probe)
|
|
60476
61866
|
)
|
|
@@ -60503,7 +61893,7 @@ var IntelligenceService = class {
|
|
|
60503
61893
|
let wouldDeleteTotal = 0;
|
|
60504
61894
|
const existingByRunId = /* @__PURE__ */ new Map();
|
|
60505
61895
|
if (isDryRun && targetRuns.length > 0) {
|
|
60506
|
-
const rows = this.db.select({ runId: insights.runId }).from(insights).where(
|
|
61896
|
+
const rows = this.db.select({ runId: insights.runId }).from(insights).where(inArray21(insights.runId, targetRuns.map((r) => r.id))).all();
|
|
60507
61897
|
for (const r of rows) {
|
|
60508
61898
|
if (r.runId == null) continue;
|
|
60509
61899
|
existingByRunId.set(r.runId, (existingByRunId.get(r.runId) ?? 0) + 1);
|
|
@@ -60549,7 +61939,7 @@ var IntelligenceService = class {
|
|
|
60549
61939
|
return { processed, skipped, totalInsights };
|
|
60550
61940
|
}
|
|
60551
61941
|
loadTrackedCompetitors(projectId) {
|
|
60552
|
-
return this.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
61942
|
+
return this.db.select({ domain: competitors.domain }).from(competitors).where(eq59(competitors.projectId, projectId)).all().map((r) => r.domain);
|
|
60553
61943
|
}
|
|
60554
61944
|
/**
|
|
60555
61945
|
* Wipe transition signals from an analysis result while keeping health.
|
|
@@ -60570,15 +61960,15 @@ var IntelligenceService = class {
|
|
|
60570
61960
|
}
|
|
60571
61961
|
persistResult(result, runId, projectId) {
|
|
60572
61962
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
60573
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
61963
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq59(insights.runId, runId)).all();
|
|
60574
61964
|
for (const row of existingInsights) {
|
|
60575
61965
|
if (row.dismissed) {
|
|
60576
61966
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
60577
61967
|
}
|
|
60578
61968
|
}
|
|
60579
61969
|
this.db.transaction((tx) => {
|
|
60580
|
-
tx.delete(insights).where(
|
|
60581
|
-
tx.delete(healthSnapshots).where(
|
|
61970
|
+
tx.delete(insights).where(eq59(insights.runId, runId)).run();
|
|
61971
|
+
tx.delete(healthSnapshots).where(eq59(healthSnapshots.runId, runId)).run();
|
|
60582
61972
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
60583
61973
|
for (const insight of result.insights) {
|
|
60584
61974
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -60631,24 +62021,24 @@ var IntelligenceService = class {
|
|
|
60631
62021
|
applySeverityTiering(rawInsights, excludeRunId, projectId) {
|
|
60632
62022
|
const regressions = rawInsights.filter((i) => i.type === "regression");
|
|
60633
62023
|
if (regressions.length === 0) return rawInsights;
|
|
60634
|
-
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(
|
|
62024
|
+
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(eq59(gscSearchData.projectId, projectId)).all();
|
|
60635
62025
|
const gscConnected = gscRows.length > 0;
|
|
60636
62026
|
const gscImpressionsByQuery = /* @__PURE__ */ new Map();
|
|
60637
62027
|
for (const row of gscRows) {
|
|
60638
62028
|
const key = row.query.toLowerCase();
|
|
60639
62029
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
60640
62030
|
}
|
|
60641
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
62031
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq59(projects.id, projectId)).get();
|
|
60642
62032
|
const locationCount = Math.max(
|
|
60643
62033
|
1,
|
|
60644
62034
|
(projectRow?.locations ?? []).length
|
|
60645
62035
|
);
|
|
60646
62036
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
60647
62037
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
60648
|
-
|
|
60649
|
-
|
|
60650
|
-
|
|
60651
|
-
|
|
62038
|
+
and47(
|
|
62039
|
+
eq59(runs.projectId, projectId),
|
|
62040
|
+
eq59(runs.kind, RunKinds["answer-visibility"]),
|
|
62041
|
+
or13(eq59(runs.status, "completed"), eq59(runs.status, "partial")),
|
|
60652
62042
|
// Defensive — see top of file.
|
|
60653
62043
|
ne7(runs.trigger, RunTriggers.probe)
|
|
60654
62044
|
)
|
|
@@ -60668,7 +62058,7 @@ var IntelligenceService = class {
|
|
|
60668
62058
|
const haveHistory = recentRunIds.length > 0;
|
|
60669
62059
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
60670
62060
|
if (haveHistory) {
|
|
60671
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
62061
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and47(eq59(insights.type, "regression"), inArray21(insights.runId, recentRunIds))).all();
|
|
60672
62062
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
60673
62063
|
for (const row of priorRows) {
|
|
60674
62064
|
if (!row.runId) continue;
|
|
@@ -60697,7 +62087,7 @@ var IntelligenceService = class {
|
|
|
60697
62087
|
});
|
|
60698
62088
|
}
|
|
60699
62089
|
buildRunData(runId, projectId, completedAt, location = null) {
|
|
60700
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
62090
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq59(projects.id, projectId)).get();
|
|
60701
62091
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
60702
62092
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
60703
62093
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -60714,7 +62104,7 @@ var IntelligenceService = class {
|
|
|
60714
62104
|
citedDomains: querySnapshots.citedDomains,
|
|
60715
62105
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
60716
62106
|
snapshotLocation: querySnapshots.location
|
|
60717
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
62107
|
+
}).from(querySnapshots).leftJoin(queries, eq59(querySnapshots.queryId, queries.id)).where(eq59(querySnapshots.runId, runId)).all();
|
|
60718
62108
|
const snapshots = [];
|
|
60719
62109
|
let orphanCount = 0;
|
|
60720
62110
|
for (const r of rows) {
|
|
@@ -60768,6 +62158,7 @@ export {
|
|
|
60768
62158
|
doctorHealthState,
|
|
60769
62159
|
notifications,
|
|
60770
62160
|
gscSearchData,
|
|
62161
|
+
gscDataWatermarks,
|
|
60771
62162
|
gscDailyTotals,
|
|
60772
62163
|
gscQueryDailyTotals,
|
|
60773
62164
|
gscUrlInspections,
|
|
@@ -60844,6 +62235,7 @@ export {
|
|
|
60844
62235
|
renderReportHtml,
|
|
60845
62236
|
toAlertView,
|
|
60846
62237
|
resolveDestination,
|
|
62238
|
+
GSC_REPORTING_TIME_ZONE,
|
|
60847
62239
|
GSC_DATA_LAG_DAYS,
|
|
60848
62240
|
refreshAccessToken,
|
|
60849
62241
|
fetchSearchAnalytics,
|