@canonry/canonry 4.159.0 → 4.160.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/aero/SKILL.md +16 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +17 -5
- package/assets/agent-workspace/skills/canonry/references/canonry-cli.md +66 -0
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +231 -62
- package/assets/assets/{AuditHistoryPanel-BiloGcQA.js → AuditHistoryPanel-AtgoG2AH.js} +1 -1
- package/assets/assets/{BacklinksPage-CPM1O2bC.js → BacklinksPage-DrOSF7Mu.js} +1 -1
- package/assets/assets/{HistoryPage-1YOjtdJy.js → HistoryPage-CGQXleJc.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-CVDTY_gC.js → MeasurementPropertyPage-C7D2OgIQ.js} +1 -1
- package/assets/assets/{ProjectPage-DnGXMx1n.js → ProjectPage-Lp1VjNvL.js} +1 -1
- package/assets/assets/{RunRow-fIaadH4h.js → RunRow-BdzKKODo.js} +1 -1
- package/assets/assets/{RunsPage-DfifdM5z.js → RunsPage-BU96L07S.js} +1 -1
- package/assets/assets/{SettingsPage-efA2EEmS.js → SettingsPage-LEca0kDH.js} +1 -1
- package/assets/assets/{SiteHealthSection-BROtQGfu.js → SiteHealthSection-CKM4DUfE.js} +3 -3
- package/assets/assets/{TrafficPage-Wk43Bm_q.js → TrafficPage-tU9wl1F_.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-BhBvoLWu.js → TrafficSourceDetailPage-DZTpnU9f.js} +1 -1
- package/assets/assets/{extract-error-message-YGzZFvWs.js → extract-error-message-BAyN-ftk.js} +1 -1
- package/assets/assets/{index-DmsIZQAm.js → index-Wqv7kvs-.js} +22 -22
- package/assets/assets/{react-sigma_core.esm.min-DM9XptbM.js → react-sigma_core.esm.min-DlZOAc_w.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-ZL3VY435.js → chunk-7M7T5QXS.js} +77 -8
- package/dist/{chunk-ZEG43TY2.js → chunk-HQAVEK42.js} +21 -1
- package/dist/{chunk-5AFWHKKI.js → chunk-X7CYY5Q7.js} +1924 -667
- package/dist/{chunk-BZPBVOAM.js → chunk-XGESWHFP.js} +39 -16
- package/dist/cli.js +295 -117
- package/dist/index.d.ts +28 -14
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-DHY5CYWX.js → intelligence-service-6XQMM466.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +10 -9
|
@@ -577,10 +577,10 @@ import {
|
|
|
577
577
|
wordpressSchemaDeployResultDtoSchema,
|
|
578
578
|
wordpressSchemaStatusResultDtoSchema,
|
|
579
579
|
wordpressStatusDtoSchema
|
|
580
|
-
} from "./chunk-
|
|
580
|
+
} from "./chunk-7M7T5QXS.js";
|
|
581
581
|
|
|
582
582
|
// src/intelligence-service.ts
|
|
583
|
-
import { eq as
|
|
583
|
+
import { eq as eq59, desc as desc26, asc as asc11, and as and47, ne as ne7, or as or13, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
|
|
584
584
|
|
|
585
585
|
// ../db/src/client.ts
|
|
586
586
|
import { mkdirSync } from "fs";
|
|
@@ -2152,6 +2152,16 @@ var trafficSources = sqliteTable("traffic_sources", {
|
|
|
2152
2152
|
// `traffic.source.worker-version` doctor check. NULL until the first
|
|
2153
2153
|
// event arrives or for source types that don't forward versioned events.
|
|
2154
2154
|
lastWorkerVersion: text("last_worker_version"),
|
|
2155
|
+
// Pull adapters use this durable, owner-bound lease to prevent a manual
|
|
2156
|
+
// sync and the scheduler from consuming the same source concurrently.
|
|
2157
|
+
// Both fields are nullable so pre-lease sources remain immediately valid.
|
|
2158
|
+
syncLeaseOwner: text("sync_lease_owner"),
|
|
2159
|
+
syncLeaseExpiresAt: text("sync_lease_expires_at"),
|
|
2160
|
+
// Residual Queue depth returned by Cloudflare after the most recent bounded
|
|
2161
|
+
// pull. NULL means a queue-backed source has not observed backlog yet; zero
|
|
2162
|
+
// is an explicit observation that the Queue was drained at that instant.
|
|
2163
|
+
queueBacklogCount: integer("queue_backlog_count"),
|
|
2164
|
+
queueBacklogObservedAt: text("queue_backlog_observed_at"),
|
|
2155
2165
|
createdAt: text("created_at").notNull(),
|
|
2156
2166
|
updatedAt: text("updated_at").notNull()
|
|
2157
2167
|
}, (table) => [
|
|
@@ -6005,7 +6015,7 @@ var MIGRATION_VERSIONS = [
|
|
|
6005
6015
|
// Push-receive traffic sources (currently only `cloudflare`) need a
|
|
6006
6016
|
// per-source bearer for the Worker to authenticate against canonry's
|
|
6007
6017
|
// ingest endpoint, plus a place to remember the deployed Worker version.
|
|
6008
|
-
// Durable receipts are transport-neutral: direct push and
|
|
6018
|
+
// Durable receipts are transport-neutral: direct push and Queue pull
|
|
6009
6019
|
// pull consumer both claim an event in the same transaction as rollups.
|
|
6010
6020
|
// Cleartext credentials remain outside the database.
|
|
6011
6021
|
//
|
|
@@ -6145,6 +6155,28 @@ var MIGRATION_VERSIONS = [
|
|
|
6145
6155
|
`CREATE INDEX IF NOT EXISTS idx_site_crawl_pages_live_preview
|
|
6146
6156
|
ON site_crawl_pages(project_id, run_id, attempt_id, audit_state, audit_score, node_key)`
|
|
6147
6157
|
]
|
|
6158
|
+
},
|
|
6159
|
+
{
|
|
6160
|
+
version: 135,
|
|
6161
|
+
name: "traffic-source-sync-lease",
|
|
6162
|
+
// A source-scoped lease serializes external pull consumers. Nullable
|
|
6163
|
+
// fields preserve every existing source and let an older binary continue
|
|
6164
|
+
// to insert source rows without knowing about leases.
|
|
6165
|
+
statements: [
|
|
6166
|
+
`ALTER TABLE traffic_sources ADD COLUMN sync_lease_owner TEXT`,
|
|
6167
|
+
`ALTER TABLE traffic_sources ADD COLUMN sync_lease_expires_at TEXT`
|
|
6168
|
+
]
|
|
6169
|
+
},
|
|
6170
|
+
{
|
|
6171
|
+
version: 136,
|
|
6172
|
+
name: "traffic-source-queue-backlog",
|
|
6173
|
+
// Persist Cloudflare's residual Queue depth so a bounded successful drain
|
|
6174
|
+
// cannot hide that work remains. NULL preserves every legacy source and
|
|
6175
|
+
// distinguishes "not observed" from an observed empty Queue.
|
|
6176
|
+
statements: [
|
|
6177
|
+
`ALTER TABLE traffic_sources ADD COLUMN queue_backlog_count INTEGER`,
|
|
6178
|
+
`ALTER TABLE traffic_sources ADD COLUMN queue_backlog_observed_at TEXT`
|
|
6179
|
+
]
|
|
6148
6180
|
}
|
|
6149
6181
|
];
|
|
6150
6182
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -14587,9 +14619,9 @@ function parseHeader(records) {
|
|
|
14587
14619
|
}
|
|
14588
14620
|
indices.set(name, index2);
|
|
14589
14621
|
}
|
|
14590
|
-
for (const
|
|
14591
|
-
if (!indices.has(
|
|
14592
|
-
csvError("csv-header-missing", `CSV is missing the required "${
|
|
14622
|
+
for (const required2 of ["property", "group"]) {
|
|
14623
|
+
if (!indices.has(required2)) {
|
|
14624
|
+
csvError("csv-header-missing", `CSV is missing the required "${required2}" header.`, 400, { header: required2 });
|
|
14593
14625
|
}
|
|
14594
14626
|
}
|
|
14595
14627
|
return { indices, length: header.fields.length };
|
|
@@ -33469,7 +33501,7 @@ var routeCatalog = [
|
|
|
33469
33501
|
method: "post",
|
|
33470
33502
|
path: "/api/v1/projects/{name}/traffic/connect/cloudflare",
|
|
33471
33503
|
summary: "Connect a Cloudflare Worker traffic source",
|
|
33472
|
-
description: "Creates or updates a Cloudflare `direct-push` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration.
|
|
33504
|
+
description: "Creates or updates a Cloudflare `direct-push` or `queue-pull` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration. Direct bearer/HMAC credentials and the Queue API token remain in Canonry's local credential store; none appears in source, TOML, DB config, response, or MCP output. Reconnect is idempotent by source mode. A different mode is staged paused when another project traffic source is active, and requires the explicit activation operation after deployment. Wrangler deploys the Worker without a route; the operator attaches the exact site route manually with Request limit failure mode set to Fail open.",
|
|
33473
33505
|
tags: ["traffic"],
|
|
33474
33506
|
parameters: [nameParameter],
|
|
33475
33507
|
requestBody: {
|
|
@@ -33513,11 +33545,27 @@ var routeCatalog = [
|
|
|
33513
33545
|
429: errorResponse("The per-IP or authenticated-source ingest request budget was exceeded.")
|
|
33514
33546
|
}
|
|
33515
33547
|
},
|
|
33548
|
+
{
|
|
33549
|
+
method: "post",
|
|
33550
|
+
path: "/api/v1/projects/{name}/traffic/sources/{id}/activate",
|
|
33551
|
+
summary: "Activate a staged traffic source",
|
|
33552
|
+
description: "Explicit single-team cutover. Atomically pauses every sibling project traffic source and connects the selected Cloudflare, Cloud Run, WordPress, or Vercel source after validating its local credential. Pull delivery creates or repoints the one project traffic-sync schedule; Cloudflare direct push removes it. Deployment and provider-side routing remain separate operator actions.",
|
|
33553
|
+
tags: ["traffic"],
|
|
33554
|
+
parameters: [
|
|
33555
|
+
nameParameter,
|
|
33556
|
+
{ name: "id", in: "path", required: true, description: "Staged traffic source ID.", schema: stringSchema }
|
|
33557
|
+
],
|
|
33558
|
+
responses: {
|
|
33559
|
+
200: jsonResponse("Activated traffic source DTO returned.", "TrafficSourceDto"),
|
|
33560
|
+
400: errorResponse("Source is archived, unsupported, or cannot be activated."),
|
|
33561
|
+
404: errorResponse("Project or traffic source not found.")
|
|
33562
|
+
}
|
|
33563
|
+
},
|
|
33516
33564
|
{
|
|
33517
33565
|
method: "post",
|
|
33518
33566
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
33519
33567
|
summary: "Trigger a sync run for a traffic source",
|
|
33520
|
-
description: "Pulls
|
|
33568
|
+
description: "Pulls from the selected Cloud Run, WordPress, Vercel, or Cloudflare Queue source, classifies crawler hits / user fetches / AI-referral sessions, and commits hourly buckets plus a bounded sample tail. Queue pull uses a durable source lease and acknowledges each Cloudflare message only after its event receipts and rollups commit.",
|
|
33521
33569
|
tags: ["traffic"],
|
|
33522
33570
|
parameters: [
|
|
33523
33571
|
nameParameter,
|
|
@@ -33530,7 +33578,7 @@ var routeCatalog = [
|
|
|
33530
33578
|
schema: {
|
|
33531
33579
|
type: "object",
|
|
33532
33580
|
properties: {
|
|
33533
|
-
sinceMinutes: { ...integerSchema, description: "
|
|
33581
|
+
sinceMinutes: { ...integerSchema, description: "Optional lookback for time-window sources; Cloudflare Queue pull ignores it." }
|
|
33534
33582
|
}
|
|
33535
33583
|
}
|
|
33536
33584
|
}
|
|
@@ -33539,8 +33587,9 @@ var routeCatalog = [
|
|
|
33539
33587
|
responses: {
|
|
33540
33588
|
200: jsonResponse("Sync summary returned.", "TrafficSyncResponse"),
|
|
33541
33589
|
400: errorResponse("Invalid sync request or missing credentials."),
|
|
33590
|
+
409: errorResponse("Another Queue sync currently owns the source lease."),
|
|
33542
33591
|
404: errorResponse("Project or traffic source not found."),
|
|
33543
|
-
502: errorResponse("Upstream
|
|
33592
|
+
502: errorResponse("Upstream pull, acknowledgement, or credential resolution failed.")
|
|
33544
33593
|
}
|
|
33545
33594
|
},
|
|
33546
33595
|
{
|
|
@@ -41728,7 +41777,7 @@ function assertGenericReconciliationKind(row) {
|
|
|
41728
41777
|
}
|
|
41729
41778
|
function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, policy, enforceBackoff) {
|
|
41730
41779
|
const nowIso = now.toISOString();
|
|
41731
|
-
const
|
|
41780
|
+
const leaseExpiresAt2 = new Date(now.getTime() + leaseMs).toISOString();
|
|
41732
41781
|
const pendingCutoff = new Date(now.getTime() - policy.pendingMinIdleMs).toISOString();
|
|
41733
41782
|
const backoffMs = policy.backoffBaseMs * 2 ** Math.max(0, row.reconcileAttempts - 1);
|
|
41734
41783
|
const unknownCutoff = new Date(now.getTime() - backoffMs).toISOString();
|
|
@@ -41745,7 +41794,7 @@ function claimOperationForReconciliation(app, row, leaseOwner, now, leaseMs, pol
|
|
|
41745
41794
|
const claimed = app.db.update(adsOperations).set({
|
|
41746
41795
|
state: AdsOperationStates.reconciling,
|
|
41747
41796
|
leaseOwner,
|
|
41748
|
-
leaseExpiresAt,
|
|
41797
|
+
leaseExpiresAt: leaseExpiresAt2,
|
|
41749
41798
|
reconcileAttempts: sql15`${adsOperations.reconcileAttempts} + 1`,
|
|
41750
41799
|
updatedAt: nowIso
|
|
41751
41800
|
}).where(and32(
|
|
@@ -47936,12 +47985,16 @@ async function backlinksRoutes(app, opts) {
|
|
|
47936
47985
|
// ../api-routes/src/traffic.ts
|
|
47937
47986
|
import crypto40 from "crypto";
|
|
47938
47987
|
import { isIP } from "net";
|
|
47988
|
+
import { isDeepStrictEqual } from "util";
|
|
47939
47989
|
import { Agent as UndiciAgent } from "undici";
|
|
47940
|
-
import { and as
|
|
47990
|
+
import { and as and40, desc as desc22, eq as eq48, gte as gte11, lte as lte11, sql as sql19 } from "drizzle-orm";
|
|
47941
47991
|
|
|
47942
47992
|
// ../api-routes/src/traffic-limits.ts
|
|
47943
47993
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
47944
47994
|
var DEFAULT_VERCEL_SYNC_DEADLINE_MS = 4 * 6e4;
|
|
47995
|
+
var CLOUDFLARE_QUEUE_BATCH_SIZE = 100;
|
|
47996
|
+
var DEFAULT_CLOUDFLARE_QUEUE_MAX_BATCHES = 10;
|
|
47997
|
+
var DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET = CLOUDFLARE_QUEUE_BATCH_SIZE * DEFAULT_CLOUDFLARE_QUEUE_MAX_BATCHES;
|
|
47945
47998
|
var MIN_VERCEL_SYNC_DEADLINE_MS = 3e4;
|
|
47946
47999
|
var MAX_VERCEL_SYNC_DEADLINE_MS = 15 * 6e4;
|
|
47947
48000
|
var TRAFFIC_SOURCE_MAX_CATCHUP_MS = {
|
|
@@ -51869,7 +51922,8 @@ var CLOUDFLARE_WORKER_BINDINGS = {
|
|
|
51869
51922
|
ingestUrl: "CANONRY_INGEST_URL",
|
|
51870
51923
|
workerVersion: "CANONRY_WORKER_VERSION",
|
|
51871
51924
|
bearerToken: "CANONRY_BEARER_TOKEN",
|
|
51872
|
-
hmacSecret: "CANONRY_HMAC_SECRET"
|
|
51925
|
+
hmacSecret: "CANONRY_HMAC_SECRET",
|
|
51926
|
+
trafficQueue: "CANONRY_TRAFFIC_QUEUE"
|
|
51873
51927
|
};
|
|
51874
51928
|
var CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS = [
|
|
51875
51929
|
CLOUDFLARE_WORKER_BINDINGS.bearerToken,
|
|
@@ -51910,6 +51964,110 @@ function jsArray(values) {
|
|
|
51910
51964
|
function generateWorkerScript(opts) {
|
|
51911
51965
|
const botScoreMax = opts.botScoreMaxForward ?? DEFAULT_BOT_SCORE_MAX_FORWARD;
|
|
51912
51966
|
const canonicalJsonFunction = canonicalizeCloudflareJson.toString();
|
|
51967
|
+
const directPushDelivery = opts.deliveryMode === "direct-push" ? `
|
|
51968
|
+
function toHex(buffer) {
|
|
51969
|
+
const bytes = new Uint8Array(buffer)
|
|
51970
|
+
let out = ''
|
|
51971
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
51972
|
+
out += bytes[i].toString(16).padStart(2, '0')
|
|
51973
|
+
}
|
|
51974
|
+
return out
|
|
51975
|
+
}
|
|
51976
|
+
|
|
51977
|
+
async function signBody(secret, timestamp, body) {
|
|
51978
|
+
const key = await crypto.subtle.importKey(
|
|
51979
|
+
'raw',
|
|
51980
|
+
new TextEncoder().encode(secret),
|
|
51981
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
51982
|
+
false,
|
|
51983
|
+
['sign'],
|
|
51984
|
+
)
|
|
51985
|
+
const sig = await crypto.subtle.sign(
|
|
51986
|
+
'HMAC',
|
|
51987
|
+
key,
|
|
51988
|
+
new TextEncoder().encode(timestamp + '.' + body),
|
|
51989
|
+
)
|
|
51990
|
+
return toHex(sig)
|
|
51991
|
+
}
|
|
51992
|
+
|
|
51993
|
+
function isRetryableStatus(status) {
|
|
51994
|
+
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
51995
|
+
}
|
|
51996
|
+
|
|
51997
|
+
function sleep(ms) {
|
|
51998
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
51999
|
+
}
|
|
52000
|
+
|
|
52001
|
+
async function cancelUnusedResponseBody(response) {
|
|
52002
|
+
try {
|
|
52003
|
+
if (response.body) await response.body.cancel()
|
|
52004
|
+
} catch (_) {
|
|
52005
|
+
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
52006
|
+
}
|
|
52007
|
+
}
|
|
52008
|
+
|
|
52009
|
+
async function withRetry(operation) {
|
|
52010
|
+
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
52011
|
+
let response
|
|
52012
|
+
try {
|
|
52013
|
+
response = await operation()
|
|
52014
|
+
} catch (error) {
|
|
52015
|
+
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
52016
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52017
|
+
continue
|
|
52018
|
+
}
|
|
52019
|
+
|
|
52020
|
+
const status = response.status
|
|
52021
|
+
const ok = response.ok
|
|
52022
|
+
await cancelUnusedResponseBody(response)
|
|
52023
|
+
if (ok) return
|
|
52024
|
+
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
52025
|
+
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
52026
|
+
}
|
|
52027
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52028
|
+
}
|
|
52029
|
+
}
|
|
52030
|
+
` : "";
|
|
52031
|
+
const directPushDeliveryAdapter = opts.deliveryMode === "direct-push" ? `
|
|
52032
|
+
async function deliverViaDirectPush(env, batch) {
|
|
52033
|
+
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
52034
|
+
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
52035
|
+
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
52036
|
+
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
52037
|
+
const body = canonicalizeJson(batch)
|
|
52038
|
+
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
52039
|
+
const signature = await signBody(hmacSecret, timestamp, body)
|
|
52040
|
+
|
|
52041
|
+
await withRetry(() => fetch(ingestUrl, {
|
|
52042
|
+
method: 'POST',
|
|
52043
|
+
headers: {
|
|
52044
|
+
'content-type': 'application/json',
|
|
52045
|
+
'Authorization': 'Bearer ' + bearerToken,
|
|
52046
|
+
'X-Canonry-Timestamp': timestamp,
|
|
52047
|
+
'X-Canonry-Signature': signature,
|
|
52048
|
+
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
52049
|
+
'X-Canonry-Source-Id': sourceId,
|
|
52050
|
+
},
|
|
52051
|
+
body,
|
|
52052
|
+
}))
|
|
52053
|
+
}
|
|
52054
|
+
` : "";
|
|
52055
|
+
const queuePullDeliveryAdapter = opts.deliveryMode === "queue-pull" ? `
|
|
52056
|
+
function requireQueueBinding(value, name) {
|
|
52057
|
+
if (value && typeof value.send === 'function') return value
|
|
52058
|
+
throw new Error('Missing required Worker binding: ' + name)
|
|
52059
|
+
}
|
|
52060
|
+
|
|
52061
|
+
async function deliverViaQueue(env, batch) {
|
|
52062
|
+
const queue = requireQueueBinding(env.${CLOUDFLARE_WORKER_BINDINGS.trafficQueue}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.trafficQueue)})
|
|
52063
|
+
await queue.send(batch, { contentType: 'json' })
|
|
52064
|
+
}
|
|
52065
|
+
` : "";
|
|
52066
|
+
const deliveryModeBranch = opts.deliveryMode === "direct-push" ? `if (deliveryMode === 'direct-push') {
|
|
52067
|
+
return deliverViaDirectPush(env, batch)
|
|
52068
|
+
}` : `if (deliveryMode === 'queue-pull') {
|
|
52069
|
+
return deliverViaQueue(env, batch)
|
|
52070
|
+
}`;
|
|
51913
52071
|
return `${CLOUDFLARE_WORKER_GENERATED_MARKER}
|
|
51914
52072
|
// worker version: ${opts.workerVersion}
|
|
51915
52073
|
// bot-list version: ${opts.botList.version}
|
|
@@ -51999,68 +52157,7 @@ function shouldForward(request) {
|
|
|
51999
52157
|
return botSignals(request.cf)
|
|
52000
52158
|
}
|
|
52001
52159
|
|
|
52002
|
-
|
|
52003
|
-
const bytes = new Uint8Array(buffer)
|
|
52004
|
-
let out = ''
|
|
52005
|
-
for (let i = 0; i < bytes.length; i++) {
|
|
52006
|
-
out += bytes[i].toString(16).padStart(2, '0')
|
|
52007
|
-
}
|
|
52008
|
-
return out
|
|
52009
|
-
}
|
|
52010
|
-
|
|
52011
|
-
async function signBody(secret, timestamp, body) {
|
|
52012
|
-
const key = await crypto.subtle.importKey(
|
|
52013
|
-
'raw',
|
|
52014
|
-
new TextEncoder().encode(secret),
|
|
52015
|
-
{ name: 'HMAC', hash: 'SHA-256' },
|
|
52016
|
-
false,
|
|
52017
|
-
['sign'],
|
|
52018
|
-
)
|
|
52019
|
-
const sig = await crypto.subtle.sign(
|
|
52020
|
-
'HMAC',
|
|
52021
|
-
key,
|
|
52022
|
-
new TextEncoder().encode(timestamp + '.' + body),
|
|
52023
|
-
)
|
|
52024
|
-
return toHex(sig)
|
|
52025
|
-
}
|
|
52026
|
-
|
|
52027
|
-
function isRetryableStatus(status) {
|
|
52028
|
-
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
52029
|
-
}
|
|
52030
|
-
|
|
52031
|
-
function sleep(ms) {
|
|
52032
|
-
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
52033
|
-
}
|
|
52034
|
-
|
|
52035
|
-
async function cancelUnusedResponseBody(response) {
|
|
52036
|
-
try {
|
|
52037
|
-
if (response.body) await response.body.cancel()
|
|
52038
|
-
} catch (_) {
|
|
52039
|
-
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
52040
|
-
}
|
|
52041
|
-
}
|
|
52042
|
-
|
|
52043
|
-
async function withRetry(operation) {
|
|
52044
|
-
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
52045
|
-
let response
|
|
52046
|
-
try {
|
|
52047
|
-
response = await operation()
|
|
52048
|
-
} catch (error) {
|
|
52049
|
-
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
52050
|
-
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52051
|
-
continue
|
|
52052
|
-
}
|
|
52053
|
-
|
|
52054
|
-
const status = response.status
|
|
52055
|
-
const ok = response.ok
|
|
52056
|
-
await cancelUnusedResponseBody(response)
|
|
52057
|
-
if (ok) return
|
|
52058
|
-
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
52059
|
-
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
52060
|
-
}
|
|
52061
|
-
await sleep(RETRY_DELAYS_MS[attempt])
|
|
52062
|
-
}
|
|
52063
|
-
}
|
|
52160
|
+
${directPushDelivery}
|
|
52064
52161
|
|
|
52065
52162
|
function pickCf(cf) {
|
|
52066
52163
|
if (!cf) return null
|
|
@@ -52099,34 +52196,11 @@ function buildEdgeEventBatch(env, request, status, observedAt) {
|
|
|
52099
52196
|
}
|
|
52100
52197
|
}
|
|
52101
52198
|
|
|
52102
|
-
|
|
52103
|
-
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
52104
|
-
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
52105
|
-
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
52106
|
-
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
52107
|
-
const body = canonicalizeJson(batch)
|
|
52108
|
-
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
52109
|
-
const signature = await signBody(hmacSecret, timestamp, body)
|
|
52110
|
-
|
|
52111
|
-
await withRetry(() => fetch(ingestUrl, {
|
|
52112
|
-
method: 'POST',
|
|
52113
|
-
headers: {
|
|
52114
|
-
'content-type': 'application/json',
|
|
52115
|
-
'Authorization': 'Bearer ' + bearerToken,
|
|
52116
|
-
'X-Canonry-Timestamp': timestamp,
|
|
52117
|
-
'X-Canonry-Signature': signature,
|
|
52118
|
-
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
52119
|
-
'X-Canonry-Source-Id': sourceId,
|
|
52120
|
-
},
|
|
52121
|
-
body,
|
|
52122
|
-
}))
|
|
52123
|
-
}
|
|
52199
|
+
${directPushDeliveryAdapter}${queuePullDeliveryAdapter}
|
|
52124
52200
|
|
|
52125
52201
|
async function deliverEdgeEventBatch(env, batch) {
|
|
52126
52202
|
const deliveryMode = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.deliveryMode}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.deliveryMode)})
|
|
52127
|
-
|
|
52128
|
-
return deliverViaDirectPush(env, batch)
|
|
52129
|
-
}
|
|
52203
|
+
${deliveryModeBranch}
|
|
52130
52204
|
throw new Error('Unsupported Canonry delivery mode: ' + deliveryMode)
|
|
52131
52205
|
}
|
|
52132
52206
|
|
|
@@ -52200,6 +52274,31 @@ function generateWranglerToml(opts) {
|
|
|
52200
52274
|
` : "";
|
|
52201
52275
|
const zoneHint = opts.zoneId ? `# Target zone id: ${jsString(opts.zoneId)}
|
|
52202
52276
|
` : "# Target zone id was not provided. Select the canonical site zone.\n";
|
|
52277
|
+
if (opts.deliveryMode === "queue-pull") {
|
|
52278
|
+
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
52279
|
+
name = "canonry-traffic-${opts.sourceId}"
|
|
52280
|
+
${accountConfig}main = "worker.js"
|
|
52281
|
+
compatibility_date = "${WORKER_COMPATIBILITY_DATE}"
|
|
52282
|
+
workers_dev = false
|
|
52283
|
+
|
|
52284
|
+
[vars]
|
|
52285
|
+
${CLOUDFLARE_WORKER_BINDINGS.deliveryMode} = ${jsString(opts.deliveryMode)}
|
|
52286
|
+
${CLOUDFLARE_WORKER_BINDINGS.sourceId} = ${jsString(opts.sourceId)}
|
|
52287
|
+
${CLOUDFLARE_WORKER_BINDINGS.workerVersion} = ${jsString(opts.workerVersion)}
|
|
52288
|
+
|
|
52289
|
+
[[queues.producers]]
|
|
52290
|
+
queue = ${jsString(opts.queueName)}
|
|
52291
|
+
binding = ${jsString(CLOUDFLARE_WORKER_BINDINGS.trafficQueue)}
|
|
52292
|
+
|
|
52293
|
+
# Deploy this Worker via:
|
|
52294
|
+
# wrangler deploy
|
|
52295
|
+
# Canonry intentionally does not declare a route in this file.
|
|
52296
|
+
# After deploy, attach this exact route in the Cloudflare dashboard:
|
|
52297
|
+
# ${hostname}/*
|
|
52298
|
+
${zoneHint}# Set the route Request limit failure mode to Fail open before activation.
|
|
52299
|
+
# Wrangler cannot configure this route toggle.
|
|
52300
|
+
`;
|
|
52301
|
+
}
|
|
52203
52302
|
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
52204
52303
|
name = "canonry-traffic-${opts.sourceId}"
|
|
52205
52304
|
${accountConfig}main = "worker.js"
|
|
@@ -52277,6 +52376,365 @@ function verifyRequestSignature(opts) {
|
|
|
52277
52376
|
return { ok: false, reason: "signature_mismatch" };
|
|
52278
52377
|
}
|
|
52279
52378
|
|
|
52379
|
+
// ../integration-cloudflare-queue/src/client.ts
|
|
52380
|
+
var DEFAULT_API_BASE_URL = "https://api.cloudflare.com/client/v4";
|
|
52381
|
+
var DEFAULT_TIMEOUT_MS4 = 3e4;
|
|
52382
|
+
var DEFAULT_MAX_RETRIES2 = 3;
|
|
52383
|
+
var DEFAULT_RETRY_BASE_DELAY_MS = 1e3;
|
|
52384
|
+
var MAX_RETRY_DELAY_MS = 3e4;
|
|
52385
|
+
var MAX_PULL_BATCH_SIZE = 100;
|
|
52386
|
+
var MAX_VISIBILITY_TIMEOUT_MS = 12 * 60 * 6e4;
|
|
52387
|
+
var CloudflareQueueApiError = class extends Error {
|
|
52388
|
+
constructor(message, status, retryAfter) {
|
|
52389
|
+
super(message);
|
|
52390
|
+
this.status = status;
|
|
52391
|
+
this.retryAfter = retryAfter;
|
|
52392
|
+
this.name = "CloudflareQueueApiError";
|
|
52393
|
+
}
|
|
52394
|
+
status;
|
|
52395
|
+
retryAfter;
|
|
52396
|
+
};
|
|
52397
|
+
function required(value, label) {
|
|
52398
|
+
const trimmed = value.trim();
|
|
52399
|
+
if (!trimmed) throw new CloudflareQueueApiError(`${label} is required`, 400);
|
|
52400
|
+
return trimmed;
|
|
52401
|
+
}
|
|
52402
|
+
function positiveInteger(value, label, maximum) {
|
|
52403
|
+
if (!Number.isInteger(value) || value < 1 || maximum !== void 0 && value > maximum) {
|
|
52404
|
+
const range = maximum === void 0 ? "a positive integer" : `an integer from 1 to ${maximum}`;
|
|
52405
|
+
throw new CloudflareQueueApiError(`${label} must be ${range}`, 400);
|
|
52406
|
+
}
|
|
52407
|
+
return value;
|
|
52408
|
+
}
|
|
52409
|
+
function nonNegativeInteger(value, label) {
|
|
52410
|
+
if (!Number.isInteger(value) || value < 0) {
|
|
52411
|
+
throw new CloudflareQueueApiError(`${label} must be a non-negative integer`, 502);
|
|
52412
|
+
}
|
|
52413
|
+
return value;
|
|
52414
|
+
}
|
|
52415
|
+
function resolveOptions(options) {
|
|
52416
|
+
const apiBaseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
|
|
52417
|
+
try {
|
|
52418
|
+
new URL(apiBaseUrl);
|
|
52419
|
+
} catch {
|
|
52420
|
+
throw new CloudflareQueueApiError("apiBaseUrl must be a valid URL", 400);
|
|
52421
|
+
}
|
|
52422
|
+
return {
|
|
52423
|
+
accountId: required(options.accountId, "accountId"),
|
|
52424
|
+
queueId: required(options.queueId, "queueId"),
|
|
52425
|
+
apiToken: required(options.apiToken, "apiToken"),
|
|
52426
|
+
fetchImpl: options.fetchImpl ?? fetch,
|
|
52427
|
+
apiBaseUrl,
|
|
52428
|
+
timeoutMs: positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS4, "timeoutMs"),
|
|
52429
|
+
maxRetries: nonNegativeInteger(options.maxRetries ?? DEFAULT_MAX_RETRIES2, "maxRetries"),
|
|
52430
|
+
retryBaseDelayMs: positiveInteger(options.retryBaseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS, "retryBaseDelayMs"),
|
|
52431
|
+
sleep: options.sleep
|
|
52432
|
+
};
|
|
52433
|
+
}
|
|
52434
|
+
function queueMessagesUrl(options, action) {
|
|
52435
|
+
return `${options.apiBaseUrl}/accounts/${encodeURIComponent(options.accountId)}/queues/${encodeURIComponent(options.queueId)}/messages/${action}`;
|
|
52436
|
+
}
|
|
52437
|
+
function requestSignal(timeoutMs, signal) {
|
|
52438
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
52439
|
+
return signal ? AbortSignal.any([timeout, signal]) : timeout;
|
|
52440
|
+
}
|
|
52441
|
+
function isAbortError(error) {
|
|
52442
|
+
return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "TimeoutError";
|
|
52443
|
+
}
|
|
52444
|
+
function isRetryableQueueError(error) {
|
|
52445
|
+
if (isAbortError(error)) return false;
|
|
52446
|
+
if (error instanceof CloudflareQueueApiError) {
|
|
52447
|
+
return error.status === 429 || error.status >= 500;
|
|
52448
|
+
}
|
|
52449
|
+
return true;
|
|
52450
|
+
}
|
|
52451
|
+
function retryAfterDelay(value) {
|
|
52452
|
+
if (!value) return void 0;
|
|
52453
|
+
const trimmed = value.trim();
|
|
52454
|
+
if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1e3;
|
|
52455
|
+
const at = Date.parse(trimmed);
|
|
52456
|
+
return Number.isNaN(at) ? void 0 : Math.max(0, at - Date.now());
|
|
52457
|
+
}
|
|
52458
|
+
async function withQueueRetry(attempt, options) {
|
|
52459
|
+
let lastError;
|
|
52460
|
+
for (let attemptNumber = 0; attemptNumber <= options.maxRetries; attemptNumber += 1) {
|
|
52461
|
+
try {
|
|
52462
|
+
return await attempt();
|
|
52463
|
+
} catch (error) {
|
|
52464
|
+
lastError = error;
|
|
52465
|
+
if (attemptNumber >= options.maxRetries || !isRetryableQueueError(error)) throw error;
|
|
52466
|
+
const defaultDelayMs = options.retryBaseDelayMs * Math.pow(2, attemptNumber);
|
|
52467
|
+
const requestedDelayMs = error instanceof CloudflareQueueApiError ? retryAfterDelay(error.retryAfter) ?? defaultDelayMs : defaultDelayMs;
|
|
52468
|
+
const delayMs = Math.min(requestedDelayMs, MAX_RETRY_DELAY_MS);
|
|
52469
|
+
if (options.sleep) {
|
|
52470
|
+
await options.sleep(delayMs);
|
|
52471
|
+
} else if (delayMs > 0) {
|
|
52472
|
+
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
52473
|
+
}
|
|
52474
|
+
}
|
|
52475
|
+
}
|
|
52476
|
+
throw lastError;
|
|
52477
|
+
}
|
|
52478
|
+
async function postJson(options, action, body, signal) {
|
|
52479
|
+
return withQueueRetry(async () => {
|
|
52480
|
+
let response;
|
|
52481
|
+
try {
|
|
52482
|
+
response = await options.fetchImpl(queueMessagesUrl(options, action), {
|
|
52483
|
+
method: "POST",
|
|
52484
|
+
headers: {
|
|
52485
|
+
Authorization: `Bearer ${options.apiToken}`,
|
|
52486
|
+
"Content-Type": "application/json"
|
|
52487
|
+
},
|
|
52488
|
+
body: JSON.stringify(body),
|
|
52489
|
+
signal: requestSignal(options.timeoutMs, signal)
|
|
52490
|
+
});
|
|
52491
|
+
} catch (error) {
|
|
52492
|
+
if (isAbortError(error)) {
|
|
52493
|
+
throw new CloudflareQueueApiError("Cloudflare Queue request timed out or was aborted", 408);
|
|
52494
|
+
}
|
|
52495
|
+
throw new CloudflareQueueApiError("Cloudflare Queue request failed", 503);
|
|
52496
|
+
}
|
|
52497
|
+
if (!response.ok) {
|
|
52498
|
+
throw new CloudflareQueueApiError(
|
|
52499
|
+
`Cloudflare Queue ${action} failed with HTTP ${response.status}`,
|
|
52500
|
+
response.status,
|
|
52501
|
+
response.headers.get("retry-after") ?? void 0
|
|
52502
|
+
);
|
|
52503
|
+
}
|
|
52504
|
+
try {
|
|
52505
|
+
return await response.json();
|
|
52506
|
+
} catch {
|
|
52507
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${action} returned invalid JSON`, 502);
|
|
52508
|
+
}
|
|
52509
|
+
}, {
|
|
52510
|
+
maxRetries: options.maxRetries,
|
|
52511
|
+
retryBaseDelayMs: options.retryBaseDelayMs,
|
|
52512
|
+
sleep: options.sleep
|
|
52513
|
+
});
|
|
52514
|
+
}
|
|
52515
|
+
function asRecord(value, label) {
|
|
52516
|
+
if (value == null || typeof value !== "object" || Array.isArray(value)) {
|
|
52517
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
52518
|
+
}
|
|
52519
|
+
return value;
|
|
52520
|
+
}
|
|
52521
|
+
function requiredString(record, key, label) {
|
|
52522
|
+
const value = record[key];
|
|
52523
|
+
if (typeof value !== "string" || !value.trim()) {
|
|
52524
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
52525
|
+
}
|
|
52526
|
+
return value;
|
|
52527
|
+
}
|
|
52528
|
+
function stringField(record, key, label) {
|
|
52529
|
+
const value = record[key];
|
|
52530
|
+
if (typeof value !== "string") {
|
|
52531
|
+
throw new CloudflareQueueApiError(`Cloudflare Queue ${label} is malformed`, 502);
|
|
52532
|
+
}
|
|
52533
|
+
return value;
|
|
52534
|
+
}
|
|
52535
|
+
function metadata(record) {
|
|
52536
|
+
const value = asRecord(record.metadata, "message metadata");
|
|
52537
|
+
const parsed = {};
|
|
52538
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
52539
|
+
if (typeof entry !== "string") {
|
|
52540
|
+
throw new CloudflareQueueApiError("Cloudflare Queue message metadata is malformed", 502);
|
|
52541
|
+
}
|
|
52542
|
+
parsed[key] = entry;
|
|
52543
|
+
}
|
|
52544
|
+
return parsed;
|
|
52545
|
+
}
|
|
52546
|
+
function decodeBase64(value) {
|
|
52547
|
+
if (!/^(?:[a-z0-9+/]{4})*(?:[a-z0-9+/]{2}==|[a-z0-9+/]{3}=)?$/i.test(value)) {
|
|
52548
|
+
throw new CloudflareQueueApiError("Cloudflare Queue message body is malformed", 502);
|
|
52549
|
+
}
|
|
52550
|
+
return new Uint8Array(Buffer.from(value, "base64"));
|
|
52551
|
+
}
|
|
52552
|
+
function parseJson(value) {
|
|
52553
|
+
try {
|
|
52554
|
+
return JSON.parse(value);
|
|
52555
|
+
} catch {
|
|
52556
|
+
throw new CloudflareQueueApiError("Cloudflare Queue JSON message body is malformed", 502);
|
|
52557
|
+
}
|
|
52558
|
+
}
|
|
52559
|
+
function decodeUtf8(bytes) {
|
|
52560
|
+
try {
|
|
52561
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
52562
|
+
} catch {
|
|
52563
|
+
throw new CloudflareQueueApiError("Cloudflare Queue JSON message body is malformed", 502);
|
|
52564
|
+
}
|
|
52565
|
+
}
|
|
52566
|
+
function decodeJson(value) {
|
|
52567
|
+
try {
|
|
52568
|
+
return JSON.parse(value);
|
|
52569
|
+
} catch {
|
|
52570
|
+
return parseJson(decodeUtf8(decodeBase64(value)));
|
|
52571
|
+
}
|
|
52572
|
+
}
|
|
52573
|
+
function safeMessageBase(record, leaseId) {
|
|
52574
|
+
const rawMetadata = record.metadata;
|
|
52575
|
+
const safeMetadata = {};
|
|
52576
|
+
let metadataIsSafe = rawMetadata != null && typeof rawMetadata === "object" && !Array.isArray(rawMetadata);
|
|
52577
|
+
if (metadataIsSafe) {
|
|
52578
|
+
for (const [key, value] of Object.entries(rawMetadata)) {
|
|
52579
|
+
if (typeof value !== "string") {
|
|
52580
|
+
metadataIsSafe = false;
|
|
52581
|
+
break;
|
|
52582
|
+
}
|
|
52583
|
+
safeMetadata[key] = value;
|
|
52584
|
+
}
|
|
52585
|
+
}
|
|
52586
|
+
return {
|
|
52587
|
+
id: typeof record.id === "string" && record.id.trim() ? record.id : "<malformed>",
|
|
52588
|
+
leaseId,
|
|
52589
|
+
timestampMs: Number.isInteger(record.timestamp_ms) && record.timestamp_ms >= 0 ? record.timestamp_ms : 0,
|
|
52590
|
+
attempts: Number.isInteger(record.attempts) && record.attempts >= 0 ? record.attempts : 0,
|
|
52591
|
+
metadata: metadataIsSafe ? safeMetadata : {}
|
|
52592
|
+
};
|
|
52593
|
+
}
|
|
52594
|
+
function decodeMessage(record) {
|
|
52595
|
+
const leaseId = requiredString(record, "lease_id", "message");
|
|
52596
|
+
let base;
|
|
52597
|
+
let rawBody;
|
|
52598
|
+
try {
|
|
52599
|
+
base = {
|
|
52600
|
+
id: requiredString(record, "id", "message"),
|
|
52601
|
+
leaseId,
|
|
52602
|
+
timestampMs: nonNegativeInteger(record.timestamp_ms, "message timestamp_ms"),
|
|
52603
|
+
attempts: nonNegativeInteger(record.attempts, "message attempts"),
|
|
52604
|
+
metadata: metadata(record)
|
|
52605
|
+
};
|
|
52606
|
+
rawBody = stringField(record, "body", "message");
|
|
52607
|
+
} catch (error) {
|
|
52608
|
+
if (!(error instanceof CloudflareQueueApiError)) throw error;
|
|
52609
|
+
return {
|
|
52610
|
+
...safeMessageBase(record, leaseId),
|
|
52611
|
+
contentType: "poison",
|
|
52612
|
+
reason: "malformed-envelope"
|
|
52613
|
+
};
|
|
52614
|
+
}
|
|
52615
|
+
const contentType = base.metadata["CF-Content-Type"] ?? "json";
|
|
52616
|
+
const poison = (reason) => ({
|
|
52617
|
+
...base,
|
|
52618
|
+
contentType: "poison",
|
|
52619
|
+
reason
|
|
52620
|
+
});
|
|
52621
|
+
if (contentType === "bytes") {
|
|
52622
|
+
try {
|
|
52623
|
+
return { ...base, contentType, body: decodeBase64(rawBody) };
|
|
52624
|
+
} catch {
|
|
52625
|
+
return poison("malformed-body");
|
|
52626
|
+
}
|
|
52627
|
+
}
|
|
52628
|
+
if (contentType === "json") {
|
|
52629
|
+
try {
|
|
52630
|
+
return { ...base, contentType, body: decodeJson(rawBody) };
|
|
52631
|
+
} catch {
|
|
52632
|
+
return poison("malformed-body");
|
|
52633
|
+
}
|
|
52634
|
+
}
|
|
52635
|
+
if (contentType === "text") {
|
|
52636
|
+
return { ...base, contentType, body: rawBody };
|
|
52637
|
+
}
|
|
52638
|
+
return poison("unsupported-content-type");
|
|
52639
|
+
}
|
|
52640
|
+
function parsePullEnvelope(value) {
|
|
52641
|
+
const envelope = asRecord(value, "pull response");
|
|
52642
|
+
if (envelope.success !== true) {
|
|
52643
|
+
throw new CloudflareQueueApiError("Cloudflare Queue pull was not successful", 502);
|
|
52644
|
+
}
|
|
52645
|
+
const result = asRecord(envelope.result, "pull result");
|
|
52646
|
+
const messageBacklogCount = nonNegativeInteger(result.message_backlog_count, "message_backlog_count");
|
|
52647
|
+
if (!Array.isArray(result.messages)) {
|
|
52648
|
+
throw new CloudflareQueueApiError("Cloudflare Queue messages is malformed", 502);
|
|
52649
|
+
}
|
|
52650
|
+
if (result.messages.length > MAX_PULL_BATCH_SIZE) {
|
|
52651
|
+
throw new CloudflareQueueApiError("Cloudflare Queue returned too many messages", 502);
|
|
52652
|
+
}
|
|
52653
|
+
const messages = [];
|
|
52654
|
+
let skippedUnleasedMessageCount = 0;
|
|
52655
|
+
for (const rawMessage of result.messages) {
|
|
52656
|
+
if (rawMessage == null || typeof rawMessage !== "object" || Array.isArray(rawMessage)) {
|
|
52657
|
+
skippedUnleasedMessageCount += 1;
|
|
52658
|
+
continue;
|
|
52659
|
+
}
|
|
52660
|
+
const message = rawMessage;
|
|
52661
|
+
if (typeof message.lease_id !== "string" || !message.lease_id.trim()) {
|
|
52662
|
+
skippedUnleasedMessageCount += 1;
|
|
52663
|
+
continue;
|
|
52664
|
+
}
|
|
52665
|
+
messages.push(decodeMessage(message));
|
|
52666
|
+
}
|
|
52667
|
+
return { messageBacklogCount, messages, skippedUnleasedMessageCount };
|
|
52668
|
+
}
|
|
52669
|
+
function optionalAckCount(record, key) {
|
|
52670
|
+
if (!(key in record)) return void 0;
|
|
52671
|
+
return nonNegativeInteger(record[key], `ack response ${key}`);
|
|
52672
|
+
}
|
|
52673
|
+
function ackWarningCount(value) {
|
|
52674
|
+
if (value == null) return 0;
|
|
52675
|
+
if (typeof value !== "object" || Array.isArray(value)) {
|
|
52676
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement warnings are malformed", 502);
|
|
52677
|
+
}
|
|
52678
|
+
const warnings = value;
|
|
52679
|
+
if (Object.values(warnings).some((warning) => typeof warning !== "string")) {
|
|
52680
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement warnings are malformed", 502);
|
|
52681
|
+
}
|
|
52682
|
+
return Object.keys(warnings).length;
|
|
52683
|
+
}
|
|
52684
|
+
function parseAckEnvelope(value, expectedAckCount, expectedRetryCount) {
|
|
52685
|
+
const envelope = asRecord(value, "ack response");
|
|
52686
|
+
if (envelope.success !== true) {
|
|
52687
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement was not successful", 502);
|
|
52688
|
+
}
|
|
52689
|
+
if (envelope.result == null) return 0;
|
|
52690
|
+
const result = asRecord(envelope.result, "ack result");
|
|
52691
|
+
const ackCount = optionalAckCount(result, "ackCount");
|
|
52692
|
+
const retryCount = optionalAckCount(result, "retryCount");
|
|
52693
|
+
if (ackCount !== void 0 && ackCount !== expectedAckCount || retryCount !== void 0 && retryCount !== expectedRetryCount) {
|
|
52694
|
+
throw new CloudflareQueueApiError("Cloudflare Queue acknowledgement was incomplete", 502);
|
|
52695
|
+
}
|
|
52696
|
+
return ackWarningCount(result.warnings);
|
|
52697
|
+
}
|
|
52698
|
+
function normalizeRetries(retries) {
|
|
52699
|
+
return (retries ?? []).map((retry) => {
|
|
52700
|
+
const leaseId = required(retry.leaseId, "retry leaseId");
|
|
52701
|
+
if (retry.delaySeconds !== void 0) {
|
|
52702
|
+
nonNegativeInteger(retry.delaySeconds, "retry delaySeconds");
|
|
52703
|
+
}
|
|
52704
|
+
return { leaseId, ...retry.delaySeconds === void 0 ? {} : { delaySeconds: retry.delaySeconds } };
|
|
52705
|
+
});
|
|
52706
|
+
}
|
|
52707
|
+
async function pullCloudflareQueueMessages(client, options = {}) {
|
|
52708
|
+
const resolved = resolveOptions(client);
|
|
52709
|
+
const body = {};
|
|
52710
|
+
if (options.batchSize !== void 0) body.batch_size = positiveInteger(options.batchSize, "batchSize", MAX_PULL_BATCH_SIZE);
|
|
52711
|
+
if (options.visibilityTimeoutMs !== void 0) {
|
|
52712
|
+
body.visibility_timeout_ms = positiveInteger(
|
|
52713
|
+
options.visibilityTimeoutMs,
|
|
52714
|
+
"visibilityTimeoutMs",
|
|
52715
|
+
MAX_VISIBILITY_TIMEOUT_MS
|
|
52716
|
+
);
|
|
52717
|
+
}
|
|
52718
|
+
return parsePullEnvelope(await postJson(resolved, "pull", body, options.signal));
|
|
52719
|
+
}
|
|
52720
|
+
async function ackCloudflareQueueMessages(client, options) {
|
|
52721
|
+
const resolved = resolveOptions(client);
|
|
52722
|
+
const acknowledgedLeaseIds = (options.acks ?? []).map((leaseId) => required(leaseId, "ack leaseId"));
|
|
52723
|
+
const retriedLeaseIds = normalizeRetries(options.retries);
|
|
52724
|
+
if (acknowledgedLeaseIds.length === 0 && retriedLeaseIds.length === 0) {
|
|
52725
|
+
throw new CloudflareQueueApiError("at least one acknowledgement or retry is required", 400);
|
|
52726
|
+
}
|
|
52727
|
+
const envelope = await postJson(resolved, "ack", {
|
|
52728
|
+
acks: acknowledgedLeaseIds.map((lease_id) => ({ lease_id })),
|
|
52729
|
+
retries: retriedLeaseIds.map(({ leaseId, delaySeconds }) => ({
|
|
52730
|
+
lease_id: leaseId,
|
|
52731
|
+
...delaySeconds === void 0 ? {} : { delay_seconds: delaySeconds }
|
|
52732
|
+
}))
|
|
52733
|
+
}, options.signal);
|
|
52734
|
+
const warningCount = parseAckEnvelope(envelope, acknowledgedLeaseIds.length, retriedLeaseIds.length);
|
|
52735
|
+
return { acknowledgedLeaseIds, retriedLeaseIds, warningCount };
|
|
52736
|
+
}
|
|
52737
|
+
|
|
52280
52738
|
// ../api-routes/src/traffic-event-ingest.ts
|
|
52281
52739
|
import crypto39 from "crypto";
|
|
52282
52740
|
import { and as and38, eq as eq46, gte as gte10, lt as lt7, lte as lte9, sql as sql18 } from "drizzle-orm";
|
|
@@ -52297,6 +52755,11 @@ function writeTrafficEventBatch(opts) {
|
|
|
52297
52755
|
let aiUserFetchBucketRows = 0;
|
|
52298
52756
|
let aiReferralBucketRows = 0;
|
|
52299
52757
|
let sampleRows = 0;
|
|
52758
|
+
let selfTrafficExcluded = 0;
|
|
52759
|
+
let crawlerHits = 0;
|
|
52760
|
+
let aiUserFetchHits = 0;
|
|
52761
|
+
let aiReferralHits = 0;
|
|
52762
|
+
let unknownHits = 0;
|
|
52300
52763
|
opts.db.transaction((tx) => {
|
|
52301
52764
|
tx.update(trafficSources).set({ updatedAt: sql18`${trafficSources.updatedAt}` }).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
52302
52765
|
const source = tx.select().from(trafficSources).where(eq46(trafficSources.id, opts.sourceId)).get();
|
|
@@ -52319,6 +52782,11 @@ function writeTrafficEventBatch(opts) {
|
|
|
52319
52782
|
acceptedEvents = claimedEvents.length;
|
|
52320
52783
|
if (claimedEvents.length > 0) {
|
|
52321
52784
|
const report = buildTrafficProbeReport(claimedEvents, { sampleLimit: claimedEvents.length });
|
|
52785
|
+
selfTrafficExcluded = report.totals.selfTrafficExcluded;
|
|
52786
|
+
crawlerHits = report.totals.crawlerHits;
|
|
52787
|
+
aiUserFetchHits = report.totals.aiUserFetchHits;
|
|
52788
|
+
aiReferralHits = report.totals.aiReferralHits;
|
|
52789
|
+
unknownHits = report.totals.unknownHits;
|
|
52322
52790
|
for (const bucket of report.crawlerEventsHourly) {
|
|
52323
52791
|
const status = bucket.status ?? 0;
|
|
52324
52792
|
tx.insert(crawlerEventsHourly).values({
|
|
@@ -52477,6 +52945,11 @@ function writeTrafficEventBatch(opts) {
|
|
|
52477
52945
|
return {
|
|
52478
52946
|
acceptedEvents,
|
|
52479
52947
|
duplicateEvents,
|
|
52948
|
+
selfTrafficExcluded,
|
|
52949
|
+
crawlerHits,
|
|
52950
|
+
aiUserFetchHits,
|
|
52951
|
+
aiReferralHits,
|
|
52952
|
+
unknownHits,
|
|
52480
52953
|
crawlerBucketRows,
|
|
52481
52954
|
aiUserFetchBucketRows,
|
|
52482
52955
|
aiReferralBucketRows,
|
|
@@ -52484,6 +52957,46 @@ function writeTrafficEventBatch(opts) {
|
|
|
52484
52957
|
};
|
|
52485
52958
|
}
|
|
52486
52959
|
|
|
52960
|
+
// ../api-routes/src/traffic-sync-lease.ts
|
|
52961
|
+
import { and as and39, eq as eq47, isNull as isNull4, lte as lte10, or as or10 } from "drizzle-orm";
|
|
52962
|
+
function leaseExpiresAt(now, ttlMs) {
|
|
52963
|
+
const nowMs = Date.parse(now);
|
|
52964
|
+
if (!Number.isFinite(nowMs)) throw new RangeError("Traffic sync lease now must be an ISO instant");
|
|
52965
|
+
if (!Number.isFinite(ttlMs) || ttlMs <= 0) throw new RangeError("Traffic sync lease ttlMs must be positive");
|
|
52966
|
+
return new Date(nowMs + ttlMs).toISOString();
|
|
52967
|
+
}
|
|
52968
|
+
function tryClaimTrafficSyncLease(input) {
|
|
52969
|
+
const expiresAt = leaseExpiresAt(input.now, input.ttlMs);
|
|
52970
|
+
return input.db.transaction((tx) => {
|
|
52971
|
+
const changed = tx.update(trafficSources).set({
|
|
52972
|
+
syncLeaseOwner: input.owner,
|
|
52973
|
+
syncLeaseExpiresAt: expiresAt,
|
|
52974
|
+
updatedAt: input.now
|
|
52975
|
+
}).where(and39(
|
|
52976
|
+
eq47(trafficSources.id, input.sourceId),
|
|
52977
|
+
or10(
|
|
52978
|
+
isNull4(trafficSources.syncLeaseOwner),
|
|
52979
|
+
lte10(trafficSources.syncLeaseExpiresAt, input.now),
|
|
52980
|
+
eq47(trafficSources.syncLeaseOwner, input.owner)
|
|
52981
|
+
)
|
|
52982
|
+
)).run();
|
|
52983
|
+
return changed.changes === 1;
|
|
52984
|
+
}, { behavior: "immediate" });
|
|
52985
|
+
}
|
|
52986
|
+
function releaseTrafficSyncLease(input) {
|
|
52987
|
+
return input.db.transaction((tx) => {
|
|
52988
|
+
const changed = tx.update(trafficSources).set({
|
|
52989
|
+
syncLeaseOwner: null,
|
|
52990
|
+
syncLeaseExpiresAt: null,
|
|
52991
|
+
updatedAt: input.now
|
|
52992
|
+
}).where(and39(
|
|
52993
|
+
eq47(trafficSources.id, input.sourceId),
|
|
52994
|
+
eq47(trafficSources.syncLeaseOwner, input.owner)
|
|
52995
|
+
)).run();
|
|
52996
|
+
return changed.changes === 1;
|
|
52997
|
+
}, { behavior: "immediate" });
|
|
52998
|
+
}
|
|
52999
|
+
|
|
52487
53000
|
// ../api-routes/src/traffic.ts
|
|
52488
53001
|
var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
52489
53002
|
var DEFAULT_PAGE_SIZE3 = 1e3;
|
|
@@ -52502,6 +53015,9 @@ var BACKFILL_MAX_PAGES = 1e3;
|
|
|
52502
53015
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
52503
53016
|
var CLOUDFLARE_WORKER_VERSION = "1.0.0";
|
|
52504
53017
|
var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
|
|
53018
|
+
var CLOUDFLARE_QUEUE_RECEIPT_TTL_MS = 14 * 24 * 60 * 6e4 + 10 * 6e4;
|
|
53019
|
+
var CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS = 5 * 6e4;
|
|
53020
|
+
var CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS = 5 * 6e4;
|
|
52505
53021
|
var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
|
|
52506
53022
|
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
|
|
52507
53023
|
function sha256Hex3(value) {
|
|
@@ -52519,7 +53035,11 @@ function timingSafeEqualHex(a, b) {
|
|
|
52519
53035
|
}
|
|
52520
53036
|
function parseDirectPushCloudflareSourceConfig(config) {
|
|
52521
53037
|
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
52522
|
-
return parsed.success ? parsed.data : null;
|
|
53038
|
+
return parsed.success && parsed.data.deliveryMode === CloudflareTrafficDeliveryModes["direct-push"] ? parsed.data : null;
|
|
53039
|
+
}
|
|
53040
|
+
function parseQueuePullCloudflareSourceConfig(config) {
|
|
53041
|
+
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
53042
|
+
return parsed.success && parsed.data.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"] ? parsed.data : null;
|
|
52523
53043
|
}
|
|
52524
53044
|
function isDirectPushCloudflareDeliveryMode(value) {
|
|
52525
53045
|
return value === CloudflareTrafficDeliveryModes["direct-push"];
|
|
@@ -52536,7 +53056,7 @@ function authenticateCloudflareIngest(request, store) {
|
|
|
52536
53056
|
const signature = typeof signatureHeader === "string" ? signatureHeader : "";
|
|
52537
53057
|
if (!bearerToken || !sourceId || !timestamp || !signature) return null;
|
|
52538
53058
|
const credential = store.getConnectionBySourceId(sourceId);
|
|
52539
|
-
if (!credential ||
|
|
53059
|
+
if (!credential || credential.deliveryMode !== "direct-push") return null;
|
|
52540
53060
|
const bearerHash = sha256Hex3(bearerToken);
|
|
52541
53061
|
if (!timingSafeEqualHex(bearerHash, sha256Hex3(credential.bearerToken))) return null;
|
|
52542
53062
|
if (!verifyRequestSignature({
|
|
@@ -52551,8 +53071,8 @@ function cloudflareIngestRateLimitKey(request, store, db) {
|
|
|
52551
53071
|
const authenticated = authenticateCloudflareIngest(request, store);
|
|
52552
53072
|
const params = request.params;
|
|
52553
53073
|
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
52554
|
-
const source = db.select().from(trafficSources).where(
|
|
52555
|
-
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status
|
|
53074
|
+
const source = db.select().from(trafficSources).where(eq48(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
53075
|
+
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status === TrafficSourceStatuses.connected && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
52556
53076
|
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
52557
53077
|
}
|
|
52558
53078
|
}
|
|
@@ -52663,6 +53183,8 @@ function rowToDto(row) {
|
|
|
52663
53183
|
lastCursor: row.lastCursor ?? null,
|
|
52664
53184
|
lastError: row.lastError ?? null,
|
|
52665
53185
|
skippedThroughAt: row.skippedThroughAt ?? null,
|
|
53186
|
+
queueBacklogCount: row.queueBacklogCount ?? null,
|
|
53187
|
+
queueBacklogObservedAt: row.queueBacklogObservedAt ?? null,
|
|
52666
53188
|
archivedAt: row.archivedAt ?? null,
|
|
52667
53189
|
config: parseSourceConfig(row),
|
|
52668
53190
|
createdAt: row.createdAt,
|
|
@@ -52680,6 +53202,44 @@ async function defaultResolveAccessToken(record) {
|
|
|
52680
53202
|
"OAuth-mode Cloud Run sync is not yet supported in v1. Provide a service-account key file."
|
|
52681
53203
|
);
|
|
52682
53204
|
}
|
|
53205
|
+
function hasConnectedTrafficSourceSibling(db, projectId, sourceId) {
|
|
53206
|
+
return db.select().from(trafficSources).where(eq48(trafficSources.projectId, projectId)).all().some((row) => row.id !== sourceId && row.status === TrafficSourceStatuses.connected);
|
|
53207
|
+
}
|
|
53208
|
+
function isAuthoritativeTrafficSource(db, source) {
|
|
53209
|
+
return (source.status === TrafficSourceStatuses.connected || source.status === TrafficSourceStatuses.error) && !hasConnectedTrafficSourceSibling(db, source.projectId, source.id);
|
|
53210
|
+
}
|
|
53211
|
+
function trafficConnectStatus(tx, projectId, existingSource) {
|
|
53212
|
+
return existingSource?.status === TrafficSourceStatuses.paused || hasConnectedTrafficSourceSibling(tx, projectId, existingSource?.id ?? "") ? TrafficSourceStatuses.paused : TrafficSourceStatuses.connected;
|
|
53213
|
+
}
|
|
53214
|
+
function isSameTrafficSourceGeneration(current, started) {
|
|
53215
|
+
return isDeepStrictEqual(current.configJson, started.configJson) && current.updatedAt === started.updatedAt && current.lastSyncedAt === started.lastSyncedAt;
|
|
53216
|
+
}
|
|
53217
|
+
function bindTrafficSyncSchedule(tx, projectId, sourceId, now, createIfMissing = true) {
|
|
53218
|
+
const schedule = tx.select().from(schedules).where(and40(
|
|
53219
|
+
eq48(schedules.projectId, projectId),
|
|
53220
|
+
eq48(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
53221
|
+
)).get();
|
|
53222
|
+
if (schedule) {
|
|
53223
|
+
if (schedule.sourceId === sourceId) return { changed: false, created: false };
|
|
53224
|
+
tx.update(schedules).set({ sourceId, updatedAt: now }).where(eq48(schedules.id, schedule.id)).run();
|
|
53225
|
+
return { changed: true, created: false };
|
|
53226
|
+
}
|
|
53227
|
+
if (!createIfMissing) return { changed: false, created: false };
|
|
53228
|
+
tx.insert(schedules).values({
|
|
53229
|
+
id: crypto40.randomUUID(),
|
|
53230
|
+
projectId,
|
|
53231
|
+
kind: SchedulableRunKinds["traffic-sync"],
|
|
53232
|
+
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
53233
|
+
preset: null,
|
|
53234
|
+
timezone: "UTC",
|
|
53235
|
+
enabled: true,
|
|
53236
|
+
providers: [],
|
|
53237
|
+
sourceId,
|
|
53238
|
+
createdAt: now,
|
|
53239
|
+
updatedAt: now
|
|
53240
|
+
}).run();
|
|
53241
|
+
return { changed: true, created: true };
|
|
53242
|
+
}
|
|
52683
53243
|
function vercelRetentionClampError(requestedStartMs, effectiveStartMs) {
|
|
52684
53244
|
return new Error(
|
|
52685
53245
|
`Vercel request-logs retention starts at ${new Date(effectiveStartMs).toISOString()}, after requested start ${new Date(requestedStartMs).toISOString()}; refusing to advance because historical traffic would be skipped`
|
|
@@ -52700,8 +53260,11 @@ async function runBackfillTask(options) {
|
|
|
52700
53260
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
52701
53261
|
try {
|
|
52702
53262
|
app.db.transaction((tx) => {
|
|
52703
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
52704
|
-
tx.
|
|
53263
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq48(runs.id, runId)).run();
|
|
53264
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
53265
|
+
if (latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
53266
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
53267
|
+
}
|
|
52705
53268
|
});
|
|
52706
53269
|
} catch {
|
|
52707
53270
|
}
|
|
@@ -52716,7 +53279,15 @@ async function runBackfillTask(options) {
|
|
|
52716
53279
|
if (allEvents.length === 0) {
|
|
52717
53280
|
const finishedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
52718
53281
|
try {
|
|
52719
|
-
app.db.
|
|
53282
|
+
app.db.transaction((tx) => {
|
|
53283
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
53284
|
+
const stillAuthoritative = latestSource && isAuthoritativeTrafficSource(tx, latestSource) && isSameTrafficSourceGeneration(latestSource, sourceRow2);
|
|
53285
|
+
tx.update(runs).set(stillAuthoritative ? { status: RunStatuses.completed, finishedAt: finishedAt2 } : {
|
|
53286
|
+
status: RunStatuses.failed,
|
|
53287
|
+
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
53288
|
+
finishedAt: finishedAt2
|
|
53289
|
+
}).where(eq48(runs.id, runId)).run();
|
|
53290
|
+
});
|
|
52720
53291
|
} catch {
|
|
52721
53292
|
}
|
|
52722
53293
|
return;
|
|
@@ -52736,33 +53307,42 @@ async function runBackfillTask(options) {
|
|
|
52736
53307
|
const currentLastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
52737
53308
|
const nextLastSyncedAt = Math.max(currentLastSyncedMs, windowEnd.getTime()) === windowEnd.getTime() ? windowEndIso : sourceRow2.lastSyncedAt;
|
|
52738
53309
|
try {
|
|
52739
|
-
app.db.transaction((tx) => {
|
|
53310
|
+
const commitOutcome = app.db.transaction((tx) => {
|
|
53311
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
53312
|
+
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
53313
|
+
tx.update(runs).set({
|
|
53314
|
+
status: RunStatuses.failed,
|
|
53315
|
+
error: "Traffic source was deactivated or reconfigured during backfill",
|
|
53316
|
+
finishedAt
|
|
53317
|
+
}).where(eq48(runs.id, runId)).run();
|
|
53318
|
+
return "source-inactive";
|
|
53319
|
+
}
|
|
52740
53320
|
tx.delete(crawlerEventsHourly).where(
|
|
52741
|
-
|
|
52742
|
-
|
|
53321
|
+
and40(
|
|
53322
|
+
eq48(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
52743
53323
|
gte11(crawlerEventsHourly.tsHour, windowStartIso),
|
|
52744
|
-
|
|
53324
|
+
lte11(crawlerEventsHourly.tsHour, windowEndIso)
|
|
52745
53325
|
)
|
|
52746
53326
|
).run();
|
|
52747
53327
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
52748
|
-
|
|
52749
|
-
|
|
53328
|
+
and40(
|
|
53329
|
+
eq48(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
52750
53330
|
gte11(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
52751
|
-
|
|
53331
|
+
lte11(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
52752
53332
|
)
|
|
52753
53333
|
).run();
|
|
52754
53334
|
tx.delete(aiReferralEventsHourly).where(
|
|
52755
|
-
|
|
52756
|
-
|
|
53335
|
+
and40(
|
|
53336
|
+
eq48(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
52757
53337
|
gte11(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
52758
|
-
|
|
53338
|
+
lte11(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
52759
53339
|
)
|
|
52760
53340
|
).run();
|
|
52761
53341
|
tx.delete(rawEventSamples).where(
|
|
52762
|
-
|
|
52763
|
-
|
|
53342
|
+
and40(
|
|
53343
|
+
eq48(rawEventSamples.sourceId, sourceRow2.id),
|
|
52764
53344
|
gte11(rawEventSamples.ts, windowStartIso),
|
|
52765
|
-
|
|
53345
|
+
lte11(rawEventSamples.ts, windowEndIso)
|
|
52766
53346
|
)
|
|
52767
53347
|
).run();
|
|
52768
53348
|
for (const bucket of report.crawlerEventsHourly) {
|
|
@@ -52854,9 +53434,11 @@ async function runBackfillTask(options) {
|
|
|
52854
53434
|
lastEventIds: newRingBuffer,
|
|
52855
53435
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
52856
53436
|
updatedAt: finishedAt
|
|
52857
|
-
}).where(
|
|
52858
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
53437
|
+
}).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
53438
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq48(runs.id, runId)).run();
|
|
53439
|
+
return "committed";
|
|
52859
53440
|
});
|
|
53441
|
+
if (commitOutcome === "source-inactive") return;
|
|
52860
53442
|
} catch (e) {
|
|
52861
53443
|
markFailed(`Backfill rollup write failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
52862
53444
|
}
|
|
@@ -52866,6 +53448,12 @@ async function trafficRoutes(app, opts) {
|
|
|
52866
53448
|
const resolveAccessToken2 = opts.resolveCloudRunAccessToken ?? defaultResolveAccessToken;
|
|
52867
53449
|
const pullWordpressEvents = opts.pullWordpressTrafficEvents ?? listWordpressTrafficEvents;
|
|
52868
53450
|
const pullVercelEvents = opts.pullVercelTrafficEvents ?? listVercelTrafficEvents;
|
|
53451
|
+
const pullQueueMessages = opts.pullCloudflareQueueMessages ?? pullCloudflareQueueMessages;
|
|
53452
|
+
const ackQueueMessages = opts.ackCloudflareQueueMessages ?? ackCloudflareQueueMessages;
|
|
53453
|
+
const cloudflareQueueMaxBatches = opts.cloudflareQueueMaxBatches ?? DEFAULT_CLOUDFLARE_QUEUE_MAX_BATCHES;
|
|
53454
|
+
if (!Number.isInteger(cloudflareQueueMaxBatches) || cloudflareQueueMaxBatches < 1 || cloudflareQueueMaxBatches > 50) {
|
|
53455
|
+
throw new RangeError("cloudflareQueueMaxBatches must be an integer from 1 to 50");
|
|
53456
|
+
}
|
|
52869
53457
|
const allowLoopback = opts.allowLoopbackWebhooks === true;
|
|
52870
53458
|
async function assertWordpressTargetAllowed(baseUrl) {
|
|
52871
53459
|
const check2 = await resolveWebhookTarget(baseUrl, { allowLoopback });
|
|
@@ -52905,6 +53493,45 @@ async function trafficRoutes(app, opts) {
|
|
|
52905
53493
|
timeWindow: "1 minute",
|
|
52906
53494
|
keyGenerator: (request) => cloudflareIngestRateLimitKey(request, opts.cloudflareTrafficCredentialStore, app.db)
|
|
52907
53495
|
});
|
|
53496
|
+
function validateTrafficSourceCredential(source, projectName) {
|
|
53497
|
+
if (source.sourceType === TrafficSourceTypes.cloudflare) {
|
|
53498
|
+
const config = cloudflareTrafficSourceConfigSchema.safeParse(source.configJson);
|
|
53499
|
+
if (!config.success) throw validationError("Cloudflare source configuration is invalid");
|
|
53500
|
+
const credential = opts.cloudflareTrafficCredentialStore?.getConnectionBySourceId(source.id);
|
|
53501
|
+
if (config.data.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"]) {
|
|
53502
|
+
if (!credential || credential.deliveryMode !== "queue-pull" || credential.projectName !== projectName || typeof credential.apiToken !== "string" || credential.apiToken.trim().length === 0 || credential.accountId !== config.data.accountId || credential.queueId !== config.data.queueId || credential.queueName !== config.data.queueName || credential.retentionSeconds !== config.data.retentionSeconds) {
|
|
53503
|
+
throw validationError("Cloudflare Queue credential is not configured for this source");
|
|
53504
|
+
}
|
|
53505
|
+
return { usesPullSchedule: true };
|
|
53506
|
+
}
|
|
53507
|
+
if (!credential || credential.deliveryMode !== "direct-push" || credential.projectName !== projectName || !credential.bearerToken || !credential.hmacSecret || !source.ingestTokenHash || !timingSafeEqualHex(sha256Hex3(credential.bearerToken), source.ingestTokenHash)) {
|
|
53508
|
+
throw validationError("Cloudflare direct-push credential is not configured for this source");
|
|
53509
|
+
}
|
|
53510
|
+
return { usesPullSchedule: false };
|
|
53511
|
+
}
|
|
53512
|
+
if (source.sourceType === TrafficSourceTypes["cloud-run"]) {
|
|
53513
|
+
const credential = opts.cloudRunCredentialStore?.getConnection(projectName);
|
|
53514
|
+
if (!credential || credential.authMode !== TrafficSourceAuthModes["service-account"] || !credential.clientEmail?.trim() || !credential.privateKey?.trim() || credential.gcpProjectId !== source.configJson.gcpProjectId || source.configJson.authMode !== TrafficSourceAuthModes["service-account"] || (credential.serviceName ?? null) !== (source.configJson.serviceName ?? null) || (credential.location ?? null) !== (source.configJson.location ?? null)) {
|
|
53515
|
+
throw validationError("Cloud Run credential is not configured for this source");
|
|
53516
|
+
}
|
|
53517
|
+
return { usesPullSchedule: true };
|
|
53518
|
+
}
|
|
53519
|
+
if (source.sourceType === TrafficSourceTypes.wordpress) {
|
|
53520
|
+
const credential = opts.wordpressTrafficCredentialStore?.getConnection(projectName);
|
|
53521
|
+
if (!credential || typeof credential.applicationPassword !== "string" || !credential.applicationPassword.trim() || credential.baseUrl !== source.configJson.baseUrl || credential.username !== source.configJson.username) {
|
|
53522
|
+
throw validationError("WordPress traffic credential is not configured for this source");
|
|
53523
|
+
}
|
|
53524
|
+
return { usesPullSchedule: true };
|
|
53525
|
+
}
|
|
53526
|
+
if (source.sourceType === TrafficSourceTypes.vercel) {
|
|
53527
|
+
const credential = opts.vercelTrafficCredentialStore?.getConnection(projectName);
|
|
53528
|
+
if (!credential || typeof credential.token !== "string" || !credential.token.trim() || credential.projectId !== source.configJson.projectId || credential.teamId !== source.configJson.teamId || credential.environment !== source.configJson.environment) {
|
|
53529
|
+
throw validationError("Vercel traffic credential is not configured for this source");
|
|
53530
|
+
}
|
|
53531
|
+
return { usesPullSchedule: true };
|
|
53532
|
+
}
|
|
53533
|
+
throw validationError(`Traffic source type "${source.sourceType}" cannot be activated`);
|
|
53534
|
+
}
|
|
52908
53535
|
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
52909
53536
|
const project = resolveProject(app.db, request.params.name);
|
|
52910
53537
|
const body = request.body ?? {};
|
|
@@ -52942,7 +53569,6 @@ async function trafficRoutes(app, opts) {
|
|
|
52942
53569
|
createdAt: existing?.createdAt ?? now,
|
|
52943
53570
|
updatedAt: now
|
|
52944
53571
|
});
|
|
52945
|
-
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
52946
53572
|
const config = {
|
|
52947
53573
|
gcpProjectId,
|
|
52948
53574
|
serviceName: serviceName ?? null,
|
|
@@ -52950,41 +53576,50 @@ async function trafficRoutes(app, opts) {
|
|
|
52950
53576
|
authMode: TrafficSourceAuthModes["service-account"]
|
|
52951
53577
|
};
|
|
52952
53578
|
const fallbackName = displayName ?? `Cloud Run \xB7 ${gcpProjectId}${serviceName ? ` / ${serviceName}` : ""}`;
|
|
52953
|
-
|
|
52954
|
-
|
|
52955
|
-
|
|
52956
|
-
|
|
52957
|
-
|
|
52958
|
-
|
|
52959
|
-
|
|
52960
|
-
|
|
52961
|
-
|
|
52962
|
-
|
|
52963
|
-
|
|
52964
|
-
|
|
52965
|
-
|
|
52966
|
-
|
|
53579
|
+
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
53580
|
+
const activeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
53581
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
53582
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
53583
|
+
if (activeSource) {
|
|
53584
|
+
tx.update(trafficSources).set({
|
|
53585
|
+
displayName: fallbackName,
|
|
53586
|
+
status: sourceStatus,
|
|
53587
|
+
lastError: null,
|
|
53588
|
+
configJson: config,
|
|
53589
|
+
updatedAt: now
|
|
53590
|
+
}).where(eq48(trafficSources.id, sourceId)).run();
|
|
53591
|
+
} else {
|
|
53592
|
+
tx.insert(trafficSources).values({
|
|
53593
|
+
id: sourceId,
|
|
53594
|
+
projectId: project.id,
|
|
53595
|
+
sourceType: TrafficSourceTypes["cloud-run"],
|
|
53596
|
+
displayName: fallbackName,
|
|
53597
|
+
status: sourceStatus,
|
|
53598
|
+
lastSyncedAt: null,
|
|
53599
|
+
lastCursor: null,
|
|
53600
|
+
lastError: null,
|
|
53601
|
+
archivedAt: null,
|
|
53602
|
+
configJson: config,
|
|
53603
|
+
createdAt: now,
|
|
53604
|
+
updatedAt: now
|
|
53605
|
+
}).run();
|
|
53606
|
+
}
|
|
53607
|
+
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId, now, false) : { changed: false, created: false };
|
|
53608
|
+
writeAuditLog(tx, {
|
|
52967
53609
|
projectId: project.id,
|
|
52968
|
-
|
|
52969
|
-
|
|
52970
|
-
|
|
52971
|
-
|
|
52972
|
-
|
|
52973
|
-
|
|
52974
|
-
|
|
52975
|
-
|
|
52976
|
-
|
|
52977
|
-
|
|
52978
|
-
|
|
52979
|
-
|
|
53610
|
+
actor: "api",
|
|
53611
|
+
action: "traffic.cloud-run.connected",
|
|
53612
|
+
entityType: "traffic_source",
|
|
53613
|
+
entityId: sourceId
|
|
53614
|
+
});
|
|
53615
|
+
return {
|
|
53616
|
+
sourceRow: tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get(),
|
|
53617
|
+
scheduleChanged: scheduleBinding.changed
|
|
53618
|
+
};
|
|
53619
|
+
}, { behavior: "immediate" });
|
|
53620
|
+
if (scheduleChanged) {
|
|
53621
|
+
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
52980
53622
|
}
|
|
52981
|
-
writeAuditLog(app.db, {
|
|
52982
|
-
projectId: project.id,
|
|
52983
|
-
actor: "api",
|
|
52984
|
-
action: "traffic.cloud-run.connected",
|
|
52985
|
-
entityType: "traffic_source",
|
|
52986
|
-
entityId: sourceRow2.id
|
|
52987
|
-
});
|
|
52988
53623
|
return rowToDto(sourceRow2);
|
|
52989
53624
|
});
|
|
52990
53625
|
app.post("/projects/:name/traffic/connect/wordpress", async (request) => {
|
|
@@ -53028,44 +53663,52 @@ async function trafficRoutes(app, opts) {
|
|
|
53028
53663
|
createdAt: existing?.createdAt ?? now,
|
|
53029
53664
|
updatedAt: now
|
|
53030
53665
|
});
|
|
53031
|
-
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
53032
53666
|
const config = { baseUrl, username };
|
|
53033
53667
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
53034
|
-
|
|
53035
|
-
|
|
53036
|
-
|
|
53037
|
-
|
|
53038
|
-
|
|
53039
|
-
|
|
53040
|
-
|
|
53041
|
-
|
|
53042
|
-
|
|
53043
|
-
|
|
53044
|
-
|
|
53045
|
-
|
|
53046
|
-
|
|
53047
|
-
|
|
53668
|
+
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
53669
|
+
const activeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
53670
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
53671
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
53672
|
+
if (activeSource) {
|
|
53673
|
+
tx.update(trafficSources).set({
|
|
53674
|
+
displayName: fallbackName,
|
|
53675
|
+
status: sourceStatus,
|
|
53676
|
+
lastError: null,
|
|
53677
|
+
configJson: config,
|
|
53678
|
+
updatedAt: now
|
|
53679
|
+
}).where(eq48(trafficSources.id, sourceId)).run();
|
|
53680
|
+
} else {
|
|
53681
|
+
tx.insert(trafficSources).values({
|
|
53682
|
+
id: sourceId,
|
|
53683
|
+
projectId: project.id,
|
|
53684
|
+
sourceType: TrafficSourceTypes.wordpress,
|
|
53685
|
+
displayName: fallbackName,
|
|
53686
|
+
status: sourceStatus,
|
|
53687
|
+
lastSyncedAt: null,
|
|
53688
|
+
lastCursor: null,
|
|
53689
|
+
lastError: null,
|
|
53690
|
+
archivedAt: null,
|
|
53691
|
+
configJson: config,
|
|
53692
|
+
createdAt: now,
|
|
53693
|
+
updatedAt: now
|
|
53694
|
+
}).run();
|
|
53695
|
+
}
|
|
53696
|
+
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId, now, false) : { changed: false, created: false };
|
|
53697
|
+
writeAuditLog(tx, {
|
|
53048
53698
|
projectId: project.id,
|
|
53049
|
-
|
|
53050
|
-
|
|
53051
|
-
|
|
53052
|
-
|
|
53053
|
-
|
|
53054
|
-
|
|
53055
|
-
|
|
53056
|
-
|
|
53057
|
-
|
|
53058
|
-
|
|
53059
|
-
|
|
53060
|
-
|
|
53699
|
+
actor: "api",
|
|
53700
|
+
action: "traffic.wordpress.connected",
|
|
53701
|
+
entityType: "traffic_source",
|
|
53702
|
+
entityId: sourceId
|
|
53703
|
+
});
|
|
53704
|
+
return {
|
|
53705
|
+
sourceRow: tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get(),
|
|
53706
|
+
scheduleChanged: scheduleBinding.changed
|
|
53707
|
+
};
|
|
53708
|
+
}, { behavior: "immediate" });
|
|
53709
|
+
if (scheduleChanged) {
|
|
53710
|
+
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
53061
53711
|
}
|
|
53062
|
-
writeAuditLog(app.db, {
|
|
53063
|
-
projectId: project.id,
|
|
53064
|
-
actor: "api",
|
|
53065
|
-
action: "traffic.wordpress.connected",
|
|
53066
|
-
entityType: "traffic_source",
|
|
53067
|
-
entityId: sourceRow2.id
|
|
53068
|
-
});
|
|
53069
53712
|
return rowToDto(sourceRow2);
|
|
53070
53713
|
});
|
|
53071
53714
|
app.post("/projects/:name/traffic/connect/vercel", async (request) => {
|
|
@@ -53111,20 +53754,21 @@ async function trafficRoutes(app, opts) {
|
|
|
53111
53754
|
createdAt: existing?.createdAt ?? now,
|
|
53112
53755
|
updatedAt: now
|
|
53113
53756
|
});
|
|
53114
|
-
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.vercel && row.status !== TrafficSourceStatuses.archived);
|
|
53115
53757
|
const config = { projectId, teamId, environment };
|
|
53116
53758
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
53117
|
-
const { sourceRow: sourceRow2,
|
|
53759
|
+
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
53760
|
+
const activeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.vercel && row2.status !== TrafficSourceStatuses.archived);
|
|
53761
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
53118
53762
|
let row;
|
|
53119
53763
|
if (activeSource) {
|
|
53120
53764
|
tx.update(trafficSources).set({
|
|
53121
53765
|
displayName: fallbackName,
|
|
53122
|
-
status:
|
|
53766
|
+
status: sourceStatus,
|
|
53123
53767
|
lastError: null,
|
|
53124
53768
|
configJson: config,
|
|
53125
53769
|
updatedAt: now
|
|
53126
|
-
}).where(
|
|
53127
|
-
row = tx.select().from(trafficSources).where(
|
|
53770
|
+
}).where(eq48(trafficSources.id, activeSource.id)).run();
|
|
53771
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, activeSource.id)).get();
|
|
53128
53772
|
} else {
|
|
53129
53773
|
const newId = crypto40.randomUUID();
|
|
53130
53774
|
tx.insert(trafficSources).values({
|
|
@@ -53132,7 +53776,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53132
53776
|
projectId: project.id,
|
|
53133
53777
|
sourceType: TrafficSourceTypes.vercel,
|
|
53134
53778
|
displayName: fallbackName,
|
|
53135
|
-
status:
|
|
53779
|
+
status: sourceStatus,
|
|
53136
53780
|
// Seed lastSyncedAt to NOW so the first sync uses a tight window.
|
|
53137
53781
|
// Leaving this null would make the first sync fall back to
|
|
53138
53782
|
// DEFAULT_SYNC_WINDOW_MINUTES (30 days) — which exceeds Vercel's
|
|
@@ -53149,31 +53793,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53149
53793
|
createdAt: now,
|
|
53150
53794
|
updatedAt: now
|
|
53151
53795
|
}).run();
|
|
53152
|
-
row = tx.select().from(trafficSources).where(
|
|
53153
|
-
}
|
|
53154
|
-
const existingSchedule = tx.select().from(schedules).where(
|
|
53155
|
-
and39(
|
|
53156
|
-
eq47(schedules.projectId, project.id),
|
|
53157
|
-
eq47(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
53158
|
-
)
|
|
53159
|
-
).get();
|
|
53160
|
-
let created = false;
|
|
53161
|
-
if (!existingSchedule) {
|
|
53162
|
-
tx.insert(schedules).values({
|
|
53163
|
-
id: crypto40.randomUUID(),
|
|
53164
|
-
projectId: project.id,
|
|
53165
|
-
kind: SchedulableRunKinds["traffic-sync"],
|
|
53166
|
-
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
53167
|
-
preset: null,
|
|
53168
|
-
timezone: "UTC",
|
|
53169
|
-
enabled: true,
|
|
53170
|
-
providers: [],
|
|
53171
|
-
sourceId: row.id,
|
|
53172
|
-
createdAt: now,
|
|
53173
|
-
updatedAt: now
|
|
53174
|
-
}).run();
|
|
53175
|
-
created = true;
|
|
53796
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, newId)).get();
|
|
53176
53797
|
}
|
|
53798
|
+
const scheduleBinding = sourceStatus === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, row.id, now) : { changed: false, created: false };
|
|
53177
53799
|
writeAuditLog(tx, {
|
|
53178
53800
|
projectId: project.id,
|
|
53179
53801
|
actor: "api",
|
|
@@ -53181,7 +53803,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53181
53803
|
entityType: "traffic_source",
|
|
53182
53804
|
entityId: row.id
|
|
53183
53805
|
});
|
|
53184
|
-
if (created) {
|
|
53806
|
+
if (scheduleBinding.created) {
|
|
53185
53807
|
writeAuditLog(tx, {
|
|
53186
53808
|
projectId: project.id,
|
|
53187
53809
|
actor: "api",
|
|
@@ -53194,9 +53816,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53194
53816
|
}
|
|
53195
53817
|
});
|
|
53196
53818
|
}
|
|
53197
|
-
return { sourceRow: row,
|
|
53198
|
-
});
|
|
53199
|
-
if (
|
|
53819
|
+
return { sourceRow: row, scheduleChanged: scheduleBinding.changed };
|
|
53820
|
+
}, { behavior: "immediate" });
|
|
53821
|
+
if (scheduleChanged) {
|
|
53200
53822
|
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
53201
53823
|
}
|
|
53202
53824
|
return rowToDto(sourceRow2);
|
|
@@ -53206,27 +53828,159 @@ async function trafficRoutes(app, opts) {
|
|
|
53206
53828
|
if (!opts.cloudflareTrafficCredentialStore) {
|
|
53207
53829
|
throw validationError("Cloudflare traffic credential storage is not configured for this deployment");
|
|
53208
53830
|
}
|
|
53209
|
-
if (!opts.cloudflareTrafficIngestUrl) {
|
|
53210
|
-
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
53211
|
-
}
|
|
53212
53831
|
const credentialStore = opts.cloudflareTrafficCredentialStore;
|
|
53213
53832
|
const parsed = trafficConnectCloudflareRequestSchema.safeParse(request.body ?? {});
|
|
53214
53833
|
if (!parsed.success) {
|
|
53215
53834
|
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
53216
53835
|
}
|
|
53836
|
+
if (parsed.data.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"]) {
|
|
53837
|
+
const { displayName: displayName2, zoneId: zoneId2, accountId: accountId2, queueId, queueName, retentionSeconds, apiToken } = parsed.data;
|
|
53838
|
+
const workerRouteHost2 = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
53839
|
+
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
53840
|
+
const cloudflareSources = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().filter((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived);
|
|
53841
|
+
const sameModeSource = cloudflareSources.find((row) => parseQueuePullCloudflareSourceConfig(row.configJson));
|
|
53842
|
+
const sourceId2 = sameModeSource?.id ?? crypto40.randomUUID();
|
|
53843
|
+
const previousCredential2 = credentialStore.getConnectionBySourceId(sourceId2);
|
|
53844
|
+
const workerVersion2 = CLOUDFLARE_WORKER_VERSION;
|
|
53845
|
+
const workerScript2 = generateWorkerScript({
|
|
53846
|
+
deliveryMode: "queue-pull",
|
|
53847
|
+
workerVersion: workerVersion2,
|
|
53848
|
+
botList: DEFAULT_BOT_LIST
|
|
53849
|
+
});
|
|
53850
|
+
const wranglerToml2 = generateWranglerToml({
|
|
53851
|
+
deliveryMode: "queue-pull",
|
|
53852
|
+
sourceId: sourceId2,
|
|
53853
|
+
hostname: workerRouteHost2,
|
|
53854
|
+
workerVersion: workerVersion2,
|
|
53855
|
+
queueName,
|
|
53856
|
+
zoneId: zoneId2 ?? null,
|
|
53857
|
+
accountId: accountId2
|
|
53858
|
+
});
|
|
53859
|
+
const config2 = {
|
|
53860
|
+
schemaVersion: 1,
|
|
53861
|
+
deliveryMode: "queue-pull",
|
|
53862
|
+
workerVersion: workerVersion2,
|
|
53863
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
53864
|
+
zoneId: zoneId2 ?? null,
|
|
53865
|
+
accountId: accountId2,
|
|
53866
|
+
queueId,
|
|
53867
|
+
queueName,
|
|
53868
|
+
retentionSeconds
|
|
53869
|
+
};
|
|
53870
|
+
const nextCredential = {
|
|
53871
|
+
projectName: project.name,
|
|
53872
|
+
sourceId: sourceId2,
|
|
53873
|
+
deliveryMode: "queue-pull",
|
|
53874
|
+
apiToken,
|
|
53875
|
+
accountId: accountId2,
|
|
53876
|
+
queueId,
|
|
53877
|
+
queueName,
|
|
53878
|
+
retentionSeconds,
|
|
53879
|
+
workerVersion: workerVersion2,
|
|
53880
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
53881
|
+
zoneId: zoneId2 ?? null,
|
|
53882
|
+
createdAt: previousCredential2?.createdAt ?? now2,
|
|
53883
|
+
updatedAt: now2
|
|
53884
|
+
};
|
|
53885
|
+
credentialStore.upsertConnection(nextCredential);
|
|
53886
|
+
let sourceRow3;
|
|
53887
|
+
let scheduleChanged = false;
|
|
53888
|
+
try {
|
|
53889
|
+
const result = app.db.transaction((tx) => {
|
|
53890
|
+
const currentSameModeSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseQueuePullCloudflareSourceConfig(row.configJson) !== null);
|
|
53891
|
+
if ((currentSameModeSource?.id ?? null) !== (sameModeSource?.id ?? null)) {
|
|
53892
|
+
throw operationInProgress("Cloudflare Queue source changed during connect; retry");
|
|
53893
|
+
}
|
|
53894
|
+
const status = trafficConnectStatus(tx, project.id, currentSameModeSource);
|
|
53895
|
+
if (currentSameModeSource) {
|
|
53896
|
+
const queueConfigChanged = !isDeepStrictEqual(currentSameModeSource.configJson, config2);
|
|
53897
|
+
tx.update(trafficSources).set({
|
|
53898
|
+
displayName: displayName2 ?? currentSameModeSource.displayName,
|
|
53899
|
+
status,
|
|
53900
|
+
lastError: null,
|
|
53901
|
+
configJson: config2,
|
|
53902
|
+
ingestTokenHash: null,
|
|
53903
|
+
...queueConfigChanged ? {
|
|
53904
|
+
queueBacklogCount: null,
|
|
53905
|
+
queueBacklogObservedAt: null
|
|
53906
|
+
} : {},
|
|
53907
|
+
updatedAt: now2
|
|
53908
|
+
}).where(eq48(trafficSources.id, sourceId2)).run();
|
|
53909
|
+
} else {
|
|
53910
|
+
tx.insert(trafficSources).values({
|
|
53911
|
+
id: sourceId2,
|
|
53912
|
+
projectId: project.id,
|
|
53913
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
53914
|
+
displayName: displayName2 ?? `Cloudflare Queue \xB7 ${queueName}`,
|
|
53915
|
+
status,
|
|
53916
|
+
lastSyncedAt: null,
|
|
53917
|
+
lastCursor: null,
|
|
53918
|
+
lastError: null,
|
|
53919
|
+
archivedAt: null,
|
|
53920
|
+
configJson: config2,
|
|
53921
|
+
ingestTokenHash: null,
|
|
53922
|
+
createdAt: now2,
|
|
53923
|
+
updatedAt: now2
|
|
53924
|
+
}).run();
|
|
53925
|
+
}
|
|
53926
|
+
const changed = status === TrafficSourceStatuses.connected ? bindTrafficSyncSchedule(tx, project.id, sourceId2, now2).changed : false;
|
|
53927
|
+
return {
|
|
53928
|
+
row: tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId2)).get(),
|
|
53929
|
+
scheduleChanged: changed
|
|
53930
|
+
};
|
|
53931
|
+
}, { behavior: "immediate" });
|
|
53932
|
+
sourceRow3 = result.row;
|
|
53933
|
+
scheduleChanged = result.scheduleChanged;
|
|
53934
|
+
} catch (error) {
|
|
53935
|
+
if (previousCredential2) credentialStore.upsertConnection(previousCredential2);
|
|
53936
|
+
else credentialStore.deleteConnectionBySourceId?.(sourceId2);
|
|
53937
|
+
throw error;
|
|
53938
|
+
}
|
|
53939
|
+
writeAuditLog(app.db, {
|
|
53940
|
+
projectId: project.id,
|
|
53941
|
+
actor: "api",
|
|
53942
|
+
action: "traffic.cloudflare.queue-connected",
|
|
53943
|
+
entityType: "traffic_source",
|
|
53944
|
+
entityId: sourceId2
|
|
53945
|
+
});
|
|
53946
|
+
if (scheduleChanged) {
|
|
53947
|
+
opts.onScheduleUpdated?.("upsert", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
53948
|
+
}
|
|
53949
|
+
return {
|
|
53950
|
+
sourceId: sourceId2,
|
|
53951
|
+
deliveryMode: "queue-pull",
|
|
53952
|
+
activationRequired: sourceRow3.status !== TrafficSourceStatuses.connected,
|
|
53953
|
+
accountId: accountId2,
|
|
53954
|
+
queueId,
|
|
53955
|
+
queueName,
|
|
53956
|
+
retentionSeconds,
|
|
53957
|
+
workerScript: workerScript2,
|
|
53958
|
+
wranglerToml: wranglerToml2,
|
|
53959
|
+
workerVersion: workerVersion2,
|
|
53960
|
+
instructions: [
|
|
53961
|
+
"Deploy this Worker to your Cloudflare zone:",
|
|
53962
|
+
" 1. Save worker.js and wrangler.toml; neither file contains Queue credentials",
|
|
53963
|
+
` 2. Enable the Queue HTTP pull consumer: wrangler queues consumer http add ${queueName}`,
|
|
53964
|
+
` 3. Attach ${workerRouteHost2}/* in the Cloudflare Dashboard with Fail open`,
|
|
53965
|
+
` 4. Canonry pulls Queue ${queueName}; activate this source after the route is live.`,
|
|
53966
|
+
`Source id: ${sourceRow3.id}`
|
|
53967
|
+
].join("\n")
|
|
53968
|
+
};
|
|
53969
|
+
}
|
|
53970
|
+
if (!opts.cloudflareTrafficIngestUrl) {
|
|
53971
|
+
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
53972
|
+
}
|
|
53217
53973
|
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
53218
53974
|
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
53219
53975
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
53220
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
53221
|
-
(row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived
|
|
53222
|
-
);
|
|
53976
|
+
const activeSource = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row.configJson) !== null);
|
|
53223
53977
|
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
53224
53978
|
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
53225
53979
|
if (activeSource && !parsedActiveConfig) {
|
|
53226
53980
|
throw validationError("Existing Cloudflare source is not configured for direct-push delivery");
|
|
53227
53981
|
}
|
|
53228
|
-
const previousCredential = credentialStore.
|
|
53229
|
-
if (previousCredential &&
|
|
53982
|
+
const previousCredential = activeSource ? credentialStore.getConnectionBySourceId(activeSource.id) : void 0;
|
|
53983
|
+
if (previousCredential && previousCredential.deliveryMode !== "direct-push") {
|
|
53230
53984
|
throw validationError(
|
|
53231
53985
|
`Cloudflare credential delivery mode "${previousCredential.deliveryMode}" does not match "${deliveryMode}"`
|
|
53232
53986
|
);
|
|
@@ -53280,44 +54034,67 @@ async function trafficRoutes(app, opts) {
|
|
|
53280
54034
|
updatedAt: now
|
|
53281
54035
|
});
|
|
53282
54036
|
let sourceRow2;
|
|
54037
|
+
let scheduleRemoved = false;
|
|
53283
54038
|
try {
|
|
53284
|
-
|
|
53285
|
-
|
|
53286
|
-
|
|
53287
|
-
|
|
53288
|
-
|
|
53289
|
-
|
|
53290
|
-
|
|
53291
|
-
|
|
53292
|
-
|
|
53293
|
-
|
|
53294
|
-
|
|
53295
|
-
|
|
53296
|
-
|
|
53297
|
-
|
|
53298
|
-
|
|
53299
|
-
|
|
53300
|
-
|
|
53301
|
-
|
|
53302
|
-
|
|
53303
|
-
|
|
53304
|
-
|
|
53305
|
-
|
|
53306
|
-
|
|
53307
|
-
|
|
53308
|
-
|
|
53309
|
-
|
|
53310
|
-
|
|
53311
|
-
|
|
53312
|
-
|
|
53313
|
-
|
|
53314
|
-
|
|
54039
|
+
const result = app.db.transaction((tx) => {
|
|
54040
|
+
const currentActiveSource = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().find((row2) => row2.sourceType === TrafficSourceTypes.cloudflare && row2.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(row2.configJson) !== null);
|
|
54041
|
+
if ((currentActiveSource?.id ?? null) !== (activeSource?.id ?? null)) {
|
|
54042
|
+
throw operationInProgress("Cloudflare direct-push source changed during connect; retry");
|
|
54043
|
+
}
|
|
54044
|
+
const sourceStatus = trafficConnectStatus(tx, project.id, currentActiveSource);
|
|
54045
|
+
let row;
|
|
54046
|
+
if (currentActiveSource) {
|
|
54047
|
+
tx.update(trafficSources).set({
|
|
54048
|
+
displayName: fallbackName,
|
|
54049
|
+
status: sourceStatus,
|
|
54050
|
+
lastError: null,
|
|
54051
|
+
configJson: config,
|
|
54052
|
+
ingestTokenHash,
|
|
54053
|
+
updatedAt: now
|
|
54054
|
+
}).where(eq48(trafficSources.id, currentActiveSource.id)).run();
|
|
54055
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, currentActiveSource.id)).get();
|
|
54056
|
+
} else {
|
|
54057
|
+
tx.insert(trafficSources).values({
|
|
54058
|
+
id: sourceId,
|
|
54059
|
+
projectId: project.id,
|
|
54060
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
54061
|
+
displayName: fallbackName,
|
|
54062
|
+
status: sourceStatus,
|
|
54063
|
+
// Seed `lastSyncedAt` to NOW so the `traffic.source.recent-data`
|
|
54064
|
+
// doctor check has a non-null baseline. Successful ingest advances
|
|
54065
|
+
// it as the receiver's last-activity timestamp.
|
|
54066
|
+
lastSyncedAt: now,
|
|
54067
|
+
lastCursor: null,
|
|
54068
|
+
lastError: null,
|
|
54069
|
+
archivedAt: null,
|
|
54070
|
+
configJson: config,
|
|
54071
|
+
ingestTokenHash,
|
|
54072
|
+
createdAt: now,
|
|
54073
|
+
updatedAt: now
|
|
54074
|
+
}).run();
|
|
54075
|
+
row = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get();
|
|
54076
|
+
}
|
|
54077
|
+
let removed = false;
|
|
54078
|
+
if (row.status === TrafficSourceStatuses.connected) {
|
|
54079
|
+
const schedule = tx.select().from(schedules).where(and40(
|
|
54080
|
+
eq48(schedules.projectId, project.id),
|
|
54081
|
+
eq48(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
54082
|
+
)).get();
|
|
54083
|
+
if (schedule) {
|
|
54084
|
+
tx.delete(schedules).where(eq48(schedules.id, schedule.id)).run();
|
|
54085
|
+
removed = true;
|
|
54086
|
+
}
|
|
54087
|
+
}
|
|
54088
|
+
return { row, scheduleRemoved: removed };
|
|
54089
|
+
}, { behavior: "immediate" });
|
|
54090
|
+
sourceRow2 = result.row;
|
|
54091
|
+
scheduleRemoved = result.scheduleRemoved;
|
|
53315
54092
|
} catch (err) {
|
|
53316
54093
|
try {
|
|
53317
54094
|
if (previousCredential) {
|
|
53318
54095
|
credentialStore.upsertConnection(previousCredential);
|
|
53319
54096
|
} else {
|
|
53320
|
-
credentialStore.
|
|
54097
|
+
credentialStore.deleteConnectionBySourceId?.(sourceId);
|
|
53321
54098
|
}
|
|
53322
54099
|
} catch {
|
|
53323
54100
|
}
|
|
@@ -53330,6 +54107,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53330
54107
|
entityType: "traffic_source",
|
|
53331
54108
|
entityId: sourceRow2.id
|
|
53332
54109
|
});
|
|
54110
|
+
if (scheduleRemoved) {
|
|
54111
|
+
opts.onScheduleUpdated?.("delete", project.id, SchedulableRunKinds["traffic-sync"]);
|
|
54112
|
+
}
|
|
53333
54113
|
const routeInstruction = effectiveZoneId ? ` 4. Deploy with \`wrangler deploy\`; wrangler.toml records zone ${effectiveZoneId} but does not claim a route` : " 4. Deploy with `wrangler deploy`; wrangler.toml intentionally does not claim a route";
|
|
53334
54114
|
const instructions = [
|
|
53335
54115
|
"Deploy this Worker to your Cloudflare zone:",
|
|
@@ -53347,12 +54127,65 @@ async function trafficRoutes(app, opts) {
|
|
|
53347
54127
|
return {
|
|
53348
54128
|
sourceId: sourceRow2.id,
|
|
53349
54129
|
deliveryMode,
|
|
54130
|
+
activationRequired: sourceRow2.status !== TrafficSourceStatuses.connected,
|
|
53350
54131
|
workerScript,
|
|
53351
54132
|
wranglerToml,
|
|
53352
54133
|
workerVersion,
|
|
53353
54134
|
instructions
|
|
53354
54135
|
};
|
|
53355
54136
|
});
|
|
54137
|
+
app.post("/projects/:name/traffic/sources/:id/activate", async (request) => {
|
|
54138
|
+
const project = resolveProject(app.db, request.params.name);
|
|
54139
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54140
|
+
const result = app.db.transaction((tx) => {
|
|
54141
|
+
const target = tx.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
54142
|
+
if (!target || target.projectId !== project.id || target.status === TrafficSourceStatuses.archived) {
|
|
54143
|
+
throw notFound("Traffic source", request.params.id);
|
|
54144
|
+
}
|
|
54145
|
+
const { usesPullSchedule } = validateTrafficSourceCredential(target, project.name);
|
|
54146
|
+
const siblings = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all();
|
|
54147
|
+
for (const sibling of siblings) {
|
|
54148
|
+
if (sibling.id === target.id || sibling.status === TrafficSourceStatuses.archived) continue;
|
|
54149
|
+
tx.update(trafficSources).set({
|
|
54150
|
+
status: TrafficSourceStatuses.paused,
|
|
54151
|
+
updatedAt: now
|
|
54152
|
+
}).where(eq48(trafficSources.id, sibling.id)).run();
|
|
54153
|
+
}
|
|
54154
|
+
tx.update(trafficSources).set({
|
|
54155
|
+
status: TrafficSourceStatuses.connected,
|
|
54156
|
+
lastError: null,
|
|
54157
|
+
archivedAt: null,
|
|
54158
|
+
updatedAt: now
|
|
54159
|
+
}).where(eq48(trafficSources.id, target.id)).run();
|
|
54160
|
+
let scheduleAction = null;
|
|
54161
|
+
if (usesPullSchedule) {
|
|
54162
|
+
bindTrafficSyncSchedule(tx, project.id, target.id, now);
|
|
54163
|
+
scheduleAction = "upsert";
|
|
54164
|
+
} else {
|
|
54165
|
+
const schedule = tx.select().from(schedules).where(and40(
|
|
54166
|
+
eq48(schedules.projectId, project.id),
|
|
54167
|
+
eq48(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
54168
|
+
)).get();
|
|
54169
|
+
if (schedule) {
|
|
54170
|
+
tx.delete(schedules).where(eq48(schedules.id, schedule.id)).run();
|
|
54171
|
+
scheduleAction = "delete";
|
|
54172
|
+
}
|
|
54173
|
+
}
|
|
54174
|
+
const source = tx.select().from(trafficSources).where(eq48(trafficSources.id, target.id)).get();
|
|
54175
|
+
writeAuditLog(tx, {
|
|
54176
|
+
projectId: project.id,
|
|
54177
|
+
actor: "api",
|
|
54178
|
+
action: "traffic.source.activated",
|
|
54179
|
+
entityType: "traffic_source",
|
|
54180
|
+
entityId: target.id
|
|
54181
|
+
});
|
|
54182
|
+
return { source, scheduleAction };
|
|
54183
|
+
});
|
|
54184
|
+
if (result.scheduleAction) {
|
|
54185
|
+
opts.onScheduleUpdated?.(result.scheduleAction, project.id, SchedulableRunKinds["traffic-sync"]);
|
|
54186
|
+
}
|
|
54187
|
+
return rowToDto(result.source);
|
|
54188
|
+
});
|
|
53356
54189
|
app.post("/projects/:name/traffic/cloudflare/ingest", {
|
|
53357
54190
|
bodyLimit: CLOUDFLARE_INGEST_BODY_LIMIT,
|
|
53358
54191
|
preHandler: async (request, reply) => {
|
|
@@ -53390,8 +54223,8 @@ async function trafficRoutes(app, opts) {
|
|
|
53390
54223
|
}
|
|
53391
54224
|
const { bearerHash, credential } = authenticated;
|
|
53392
54225
|
const sourceId = credential.sourceId;
|
|
53393
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53394
|
-
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status
|
|
54226
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq48(trafficSources.id, sourceId)).get();
|
|
54227
|
+
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status !== TrafficSourceStatuses.connected || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
53395
54228
|
throw authRequired();
|
|
53396
54229
|
}
|
|
53397
54230
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -53427,7 +54260,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53427
54260
|
receiptTtlMs: DIRECT_PUSH_RECEIPT_TTL_MS,
|
|
53428
54261
|
sampleLimit,
|
|
53429
54262
|
validateSource: (latestRow) => {
|
|
53430
|
-
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status
|
|
54263
|
+
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status !== TrafficSourceStatuses.connected || !parseDirectPushCloudflareSourceConfig(latestRow.configJson) || !timingSafeEqualHex(bearerHash, latestRow.ingestTokenHash)) {
|
|
53431
54264
|
throw authRequired();
|
|
53432
54265
|
}
|
|
53433
54266
|
},
|
|
@@ -53450,15 +54283,259 @@ async function trafficRoutes(app, opts) {
|
|
|
53450
54283
|
});
|
|
53451
54284
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
53452
54285
|
const project = resolveProject(app.db, request.params.name);
|
|
53453
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
54286
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
53454
54287
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
53455
54288
|
throw notFound("Traffic source", request.params.id);
|
|
53456
54289
|
}
|
|
54290
|
+
const queueConfig = sourceRow2.sourceType === TrafficSourceTypes.cloudflare ? parseQueuePullCloudflareSourceConfig(sourceRow2.configJson) : null;
|
|
54291
|
+
if (queueConfig) {
|
|
54292
|
+
if (sourceRow2.status !== TrafficSourceStatuses.connected) {
|
|
54293
|
+
throw validationError("Cloudflare Queue source must be active before it can sync");
|
|
54294
|
+
}
|
|
54295
|
+
const credential = opts.cloudflareTrafficCredentialStore?.getConnectionBySourceId(sourceRow2.id);
|
|
54296
|
+
if (!credential || credential.deliveryMode !== "queue-pull" || credential.projectName !== project.name || typeof credential.apiToken !== "string" || credential.apiToken.trim().length === 0 || credential.accountId !== queueConfig.accountId || credential.queueId !== queueConfig.queueId || credential.queueName !== queueConfig.queueName || credential.retentionSeconds !== queueConfig.retentionSeconds) {
|
|
54297
|
+
throw validationError("Cloudflare Queue credential is not configured for this source");
|
|
54298
|
+
}
|
|
54299
|
+
const queueClient = {
|
|
54300
|
+
accountId: credential.accountId,
|
|
54301
|
+
queueId: credential.queueId,
|
|
54302
|
+
apiToken: credential.apiToken
|
|
54303
|
+
};
|
|
54304
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54305
|
+
const leaseOwner = crypto40.randomUUID();
|
|
54306
|
+
if (!tryClaimTrafficSyncLease({
|
|
54307
|
+
db: app.db,
|
|
54308
|
+
sourceId: sourceRow2.id,
|
|
54309
|
+
owner: leaseOwner,
|
|
54310
|
+
now,
|
|
54311
|
+
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
54312
|
+
})) {
|
|
54313
|
+
throw operationInProgress("Cloudflare Queue source sync is already in progress", { sourceId: sourceRow2.id });
|
|
54314
|
+
}
|
|
54315
|
+
const runId2 = crypto40.randomUUID();
|
|
54316
|
+
const startedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
54317
|
+
const startedMs = Date.now();
|
|
54318
|
+
try {
|
|
54319
|
+
app.db.insert(runs).values({
|
|
54320
|
+
id: runId2,
|
|
54321
|
+
projectId: project.id,
|
|
54322
|
+
kind: RunKinds["traffic-sync"],
|
|
54323
|
+
status: RunStatuses.running,
|
|
54324
|
+
trigger: RunTriggers.manual,
|
|
54325
|
+
sourceId: sourceRow2.id,
|
|
54326
|
+
startedAt: startedAt2,
|
|
54327
|
+
createdAt: startedAt2
|
|
54328
|
+
}).run();
|
|
54329
|
+
let acceptedEvents = 0;
|
|
54330
|
+
let selfTrafficExcluded = 0;
|
|
54331
|
+
let crawlerHits = 0;
|
|
54332
|
+
let aiUserFetchHits = 0;
|
|
54333
|
+
let aiReferralHits = 0;
|
|
54334
|
+
let unknownHits = 0;
|
|
54335
|
+
let crawlerBucketRows2 = 0;
|
|
54336
|
+
let aiUserFetchBucketRows2 = 0;
|
|
54337
|
+
let aiReferralBucketRows2 = 0;
|
|
54338
|
+
let sampleRows2 = 0;
|
|
54339
|
+
let committedAt = startedAt2;
|
|
54340
|
+
let remainingBacklogCount = 0;
|
|
54341
|
+
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
54342
|
+
for (let batch = 0; batch < cloudflareQueueMaxBatches; batch += 1) {
|
|
54343
|
+
if (!tryClaimTrafficSyncLease({
|
|
54344
|
+
db: app.db,
|
|
54345
|
+
sourceId: sourceRow2.id,
|
|
54346
|
+
owner: leaseOwner,
|
|
54347
|
+
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
54348
|
+
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
54349
|
+
})) throw new Error("Queue sync lease was lost");
|
|
54350
|
+
const pulled = await pullQueueMessages(queueClient, {
|
|
54351
|
+
batchSize: CLOUDFLARE_QUEUE_BATCH_SIZE,
|
|
54352
|
+
visibilityTimeoutMs: CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS
|
|
54353
|
+
});
|
|
54354
|
+
remainingBacklogCount = pulled.messageBacklogCount;
|
|
54355
|
+
if (pulled.skippedUnleasedMessageCount > 0) {
|
|
54356
|
+
request.log.warn({
|
|
54357
|
+
sourceId: sourceRow2.id,
|
|
54358
|
+
skippedUnleasedMessageCount: pulled.skippedUnleasedMessageCount
|
|
54359
|
+
}, "Skipped unacknowledgeable Cloudflare Queue messages; they remain eligible for redelivery");
|
|
54360
|
+
}
|
|
54361
|
+
const validLeases = [];
|
|
54362
|
+
const poisonLeases = [];
|
|
54363
|
+
const normalized = [];
|
|
54364
|
+
let workerVersion = null;
|
|
54365
|
+
for (const message of pulled.messages) {
|
|
54366
|
+
if (message.contentType === "poison") {
|
|
54367
|
+
poisonLeases.push(message.leaseId);
|
|
54368
|
+
request.log.warn(
|
|
54369
|
+
{ sourceId: sourceRow2.id, messageId: message.id, reason: message.reason },
|
|
54370
|
+
"Dropping malformed Cloudflare Queue message"
|
|
54371
|
+
);
|
|
54372
|
+
continue;
|
|
54373
|
+
}
|
|
54374
|
+
if (message.contentType !== "json") {
|
|
54375
|
+
poisonLeases.push(message.leaseId);
|
|
54376
|
+
request.log.warn({
|
|
54377
|
+
sourceId: sourceRow2.id,
|
|
54378
|
+
messageId: message.id,
|
|
54379
|
+
contentType: message.contentType,
|
|
54380
|
+
reason: "unsupported-content-type"
|
|
54381
|
+
}, "Dropping Cloudflare Queue message with an unsupported content type");
|
|
54382
|
+
continue;
|
|
54383
|
+
}
|
|
54384
|
+
const parsedBatch = cloudflareWorkerIngestRequestSchema.safeParse(message.body);
|
|
54385
|
+
if (!parsedBatch.success) {
|
|
54386
|
+
poisonLeases.push(message.leaseId);
|
|
54387
|
+
request.log.warn(
|
|
54388
|
+
{ sourceId: sourceRow2.id, messageId: message.id },
|
|
54389
|
+
"Dropping Cloudflare Queue message with an invalid Canonry batch"
|
|
54390
|
+
);
|
|
54391
|
+
continue;
|
|
54392
|
+
}
|
|
54393
|
+
if (parsedBatch.data.events.some((event) => normalizeCloudflareEventHost(event.host) !== canonicalHost)) {
|
|
54394
|
+
poisonLeases.push(message.leaseId);
|
|
54395
|
+
request.log.warn(
|
|
54396
|
+
{ sourceId: sourceRow2.id, messageId: message.id },
|
|
54397
|
+
"Dropping Cloudflare Queue message for another host"
|
|
54398
|
+
);
|
|
54399
|
+
continue;
|
|
54400
|
+
}
|
|
54401
|
+
workerVersion = parsedBatch.data.workerVersion;
|
|
54402
|
+
for (const event of parsedBatch.data.events) {
|
|
54403
|
+
const normalizedEvent = normalizeCloudflareWorkerEvent(event);
|
|
54404
|
+
if (normalizedEvent) normalized.push(normalizedEvent);
|
|
54405
|
+
}
|
|
54406
|
+
validLeases.push(message.leaseId);
|
|
54407
|
+
}
|
|
54408
|
+
committedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54409
|
+
const writeResult = writeTrafficEventBatch({
|
|
54410
|
+
db: app.db,
|
|
54411
|
+
projectId: project.id,
|
|
54412
|
+
sourceId: sourceRow2.id,
|
|
54413
|
+
events: normalized,
|
|
54414
|
+
receivedAt: committedAt,
|
|
54415
|
+
receiptTtlMs: CLOUDFLARE_QUEUE_RECEIPT_TTL_MS,
|
|
54416
|
+
sampleLimit,
|
|
54417
|
+
validateSource: (latest) => {
|
|
54418
|
+
if (!latest || latest.projectId !== project.id || latest.status !== TrafficSourceStatuses.connected || latest.syncLeaseOwner !== leaseOwner || !isDeepStrictEqual(latest.configJson, sourceRow2.configJson)) {
|
|
54419
|
+
throw validationError("Cloudflare Queue source is no longer active");
|
|
54420
|
+
}
|
|
54421
|
+
},
|
|
54422
|
+
sourceUpdate: {
|
|
54423
|
+
status: TrafficSourceStatuses.connected,
|
|
54424
|
+
lastSyncedAt: committedAt,
|
|
54425
|
+
lastError: null,
|
|
54426
|
+
queueBacklogCount: remainingBacklogCount,
|
|
54427
|
+
queueBacklogObservedAt: committedAt,
|
|
54428
|
+
...workerVersion ? { lastWorkerVersion: workerVersion } : {},
|
|
54429
|
+
updatedAt: committedAt
|
|
54430
|
+
}
|
|
54431
|
+
});
|
|
54432
|
+
if (validLeases.length > 0 || poisonLeases.length > 0) {
|
|
54433
|
+
if (!tryClaimTrafficSyncLease({
|
|
54434
|
+
db: app.db,
|
|
54435
|
+
sourceId: sourceRow2.id,
|
|
54436
|
+
owner: leaseOwner,
|
|
54437
|
+
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
54438
|
+
ttlMs: CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS
|
|
54439
|
+
})) throw new Error("Queue sync lease was lost before acknowledgement");
|
|
54440
|
+
const latestSource = app.db.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54441
|
+
if (!latestSource || latestSource.projectId !== project.id || latestSource.status !== TrafficSourceStatuses.connected || latestSource.syncLeaseOwner !== leaseOwner || !isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
54442
|
+
throw validationError("Cloudflare Queue source was reconfigured before acknowledgement");
|
|
54443
|
+
}
|
|
54444
|
+
const ackResult = await ackQueueMessages(queueClient, { acks: [...validLeases, ...poisonLeases] });
|
|
54445
|
+
if (ackResult.warningCount > 0) {
|
|
54446
|
+
request.log.warn({
|
|
54447
|
+
sourceId: sourceRow2.id,
|
|
54448
|
+
warningCount: ackResult.warningCount
|
|
54449
|
+
}, "Cloudflare Queue acknowledgement completed with warnings");
|
|
54450
|
+
}
|
|
54451
|
+
}
|
|
54452
|
+
acceptedEvents += writeResult.acceptedEvents;
|
|
54453
|
+
selfTrafficExcluded += writeResult.selfTrafficExcluded;
|
|
54454
|
+
crawlerHits += writeResult.crawlerHits;
|
|
54455
|
+
aiUserFetchHits += writeResult.aiUserFetchHits;
|
|
54456
|
+
aiReferralHits += writeResult.aiReferralHits;
|
|
54457
|
+
unknownHits += writeResult.unknownHits;
|
|
54458
|
+
crawlerBucketRows2 += writeResult.crawlerBucketRows;
|
|
54459
|
+
aiUserFetchBucketRows2 += writeResult.aiUserFetchBucketRows;
|
|
54460
|
+
aiReferralBucketRows2 += writeResult.aiReferralBucketRows;
|
|
54461
|
+
sampleRows2 += writeResult.sampleRows;
|
|
54462
|
+
const pulledEnvelopeCount = pulled.messages.length + pulled.skippedUnleasedMessageCount;
|
|
54463
|
+
if (pulledEnvelopeCount < CLOUDFLARE_QUEUE_BATCH_SIZE || remainingBacklogCount === 0) break;
|
|
54464
|
+
}
|
|
54465
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: committedAt }).where(eq48(runs.id, runId2)).run();
|
|
54466
|
+
writeAuditLog(app.db, {
|
|
54467
|
+
projectId: project.id,
|
|
54468
|
+
actor: "api",
|
|
54469
|
+
action: "traffic.cloudflare.queue-synced",
|
|
54470
|
+
entityType: "traffic_source",
|
|
54471
|
+
entityId: sourceRow2.id
|
|
54472
|
+
});
|
|
54473
|
+
try {
|
|
54474
|
+
opts.onTrafficSynced?.({
|
|
54475
|
+
status: "completed",
|
|
54476
|
+
sourceType: sourceRow2.sourceType,
|
|
54477
|
+
sourceId: sourceRow2.id,
|
|
54478
|
+
pulledEvents: acceptedEvents,
|
|
54479
|
+
selfTrafficExcluded,
|
|
54480
|
+
crawlerHits,
|
|
54481
|
+
aiUserFetchHits,
|
|
54482
|
+
aiReferralHits,
|
|
54483
|
+
durationMs: Date.now() - startedMs
|
|
54484
|
+
});
|
|
54485
|
+
} catch {
|
|
54486
|
+
}
|
|
54487
|
+
const response2 = {
|
|
54488
|
+
sourceId: sourceRow2.id,
|
|
54489
|
+
runId: runId2,
|
|
54490
|
+
syncedAt: committedAt,
|
|
54491
|
+
pulledEvents: acceptedEvents,
|
|
54492
|
+
selfTrafficExcluded,
|
|
54493
|
+
crawlerHits,
|
|
54494
|
+
aiUserFetchHits,
|
|
54495
|
+
aiReferralHits,
|
|
54496
|
+
unknownHits,
|
|
54497
|
+
crawlerBucketRows: crawlerBucketRows2,
|
|
54498
|
+
aiUserFetchBucketRows: aiUserFetchBucketRows2,
|
|
54499
|
+
aiReferralBucketRows: aiReferralBucketRows2,
|
|
54500
|
+
sampleRows: sampleRows2,
|
|
54501
|
+
remainingBacklogCount,
|
|
54502
|
+
windowStart: startedAt2,
|
|
54503
|
+
windowEnd: committedAt
|
|
54504
|
+
};
|
|
54505
|
+
return response2;
|
|
54506
|
+
} catch (error) {
|
|
54507
|
+
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54508
|
+
const safeError = error instanceof CloudflareQueueApiError ? error.message : "Cloudflare Queue sync failed; retry the active source.";
|
|
54509
|
+
app.db.transaction((tx) => {
|
|
54510
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: safeError, finishedAt: failedAt }).where(eq48(runs.id, runId2)).run();
|
|
54511
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54512
|
+
if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
54513
|
+
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and40(
|
|
54514
|
+
eq48(trafficSources.id, sourceRow2.id),
|
|
54515
|
+
eq48(trafficSources.status, TrafficSourceStatuses.connected),
|
|
54516
|
+
eq48(trafficSources.syncLeaseOwner, leaseOwner)
|
|
54517
|
+
)).run();
|
|
54518
|
+
}
|
|
54519
|
+
});
|
|
54520
|
+
throw providerError(safeError);
|
|
54521
|
+
} finally {
|
|
54522
|
+
releaseTrafficSyncLease({
|
|
54523
|
+
db: app.db,
|
|
54524
|
+
sourceId: sourceRow2.id,
|
|
54525
|
+
owner: leaseOwner,
|
|
54526
|
+
now: (/* @__PURE__ */ new Date()).toISOString()
|
|
54527
|
+
});
|
|
54528
|
+
}
|
|
54529
|
+
}
|
|
53457
54530
|
if (sourceRow2.sourceType !== TrafficSourceTypes["cloud-run"] && sourceRow2.sourceType !== TrafficSourceTypes.wordpress && sourceRow2.sourceType !== TrafficSourceTypes.vercel) {
|
|
53458
54531
|
throw validationError(
|
|
53459
54532
|
`Sync for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
|
|
53460
54533
|
);
|
|
53461
54534
|
}
|
|
54535
|
+
const hasConnectedSibling = sourceRow2.status === TrafficSourceStatuses.error && app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
54536
|
+
if (sourceRow2.status === TrafficSourceStatuses.paused || sourceRow2.status === TrafficSourceStatuses.archived || hasConnectedSibling) {
|
|
54537
|
+
throw validationError("Traffic source must be active before it can sync");
|
|
54538
|
+
}
|
|
53462
54539
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
53463
54540
|
const startedAt = windowEnd.toISOString();
|
|
53464
54541
|
const syncStartedAtMs = windowEnd.getTime();
|
|
@@ -53476,8 +54553,11 @@ async function trafficRoutes(app, opts) {
|
|
|
53476
54553
|
const markFailed = (msg, errorCode) => {
|
|
53477
54554
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
53478
54555
|
app.db.transaction((tx) => {
|
|
53479
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
53480
|
-
tx.
|
|
54556
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq48(runs.id, runId)).run();
|
|
54557
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54558
|
+
if (latestSource?.status === TrafficSourceStatuses.connected && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson) && latestSource.updatedAt === sourceRow2.updatedAt && latestSource.lastSyncedAt === sourceRow2.lastSyncedAt) {
|
|
54559
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
54560
|
+
}
|
|
53481
54561
|
});
|
|
53482
54562
|
try {
|
|
53483
54563
|
opts.onTrafficSynced?.({
|
|
@@ -53558,7 +54638,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53558
54638
|
}
|
|
53559
54639
|
const credential = credentialStore.getConnection(project.name);
|
|
53560
54640
|
if (!credential) {
|
|
53561
|
-
app.db.delete(runs).where(
|
|
54641
|
+
app.db.delete(runs).where(eq48(runs.id, runId)).run();
|
|
53562
54642
|
throw validationError(
|
|
53563
54643
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
53564
54644
|
);
|
|
@@ -53607,12 +54687,12 @@ async function trafficRoutes(app, opts) {
|
|
|
53607
54687
|
auditAction = "traffic.vercel.synced";
|
|
53608
54688
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
53609
54689
|
if (!credentialStore) {
|
|
53610
|
-
app.db.delete(runs).where(
|
|
54690
|
+
app.db.delete(runs).where(eq48(runs.id, runId)).run();
|
|
53611
54691
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
53612
54692
|
}
|
|
53613
54693
|
const credential = credentialStore.getConnection(project.name);
|
|
53614
54694
|
if (!credential) {
|
|
53615
|
-
app.db.delete(runs).where(
|
|
54695
|
+
app.db.delete(runs).where(eq48(runs.id, runId)).run();
|
|
53616
54696
|
throw validationError(
|
|
53617
54697
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
53618
54698
|
);
|
|
@@ -53630,7 +54710,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53630
54710
|
if (cappedStartMs > clampedStartMs) {
|
|
53631
54711
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
53632
54712
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
53633
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
54713
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
53634
54714
|
request.log.warn(
|
|
53635
54715
|
{
|
|
53636
54716
|
sourceId: sourceRow2.id,
|
|
@@ -53714,8 +54794,18 @@ async function trafficRoutes(app, opts) {
|
|
|
53714
54794
|
let aiUserFetchHitsCount = 0;
|
|
53715
54795
|
let aiReferralHitsCount = 0;
|
|
53716
54796
|
let unknownHitsCount = 0;
|
|
53717
|
-
app.db.transaction((tx) => {
|
|
53718
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
54797
|
+
const commitOutcome = app.db.transaction((tx) => {
|
|
54798
|
+
const latestRow = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54799
|
+
const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
54800
|
+
if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || !isDeepStrictEqual(latestRow.configJson, sourceRow2.configJson) || latestRow.updatedAt !== sourceRow2.updatedAt || latestRow.lastSyncedAt !== sourceRow2.lastSyncedAt) {
|
|
54801
|
+
const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
54802
|
+
tx.update(runs).set({
|
|
54803
|
+
status: RunStatuses.failed,
|
|
54804
|
+
error: "Traffic source was deactivated or reconfigured during sync",
|
|
54805
|
+
finishedAt: abortedAt
|
|
54806
|
+
}).where(eq48(runs.id, runId)).run();
|
|
54807
|
+
return "source-inactive";
|
|
54808
|
+
}
|
|
53719
54809
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
53720
54810
|
const seenEventIds = new Set(previousIds);
|
|
53721
54811
|
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
@@ -53892,7 +54982,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53892
54982
|
// stuck if you first disabled the schedule and drained the in-flight run.
|
|
53893
54983
|
lastSyncedAt: new Date(
|
|
53894
54984
|
Math.max(
|
|
53895
|
-
|
|
54985
|
+
latestRow.lastSyncedAt ? new Date(latestRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY,
|
|
53896
54986
|
effectiveWindowEnd.getTime()
|
|
53897
54987
|
)
|
|
53898
54988
|
).toISOString(),
|
|
@@ -53903,9 +54993,13 @@ async function trafficRoutes(app, opts) {
|
|
|
53903
54993
|
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
53904
54994
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
53905
54995
|
}
|
|
53906
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
53907
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
54996
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
54997
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq48(runs.id, runId)).run();
|
|
54998
|
+
return "committed";
|
|
53908
54999
|
});
|
|
55000
|
+
if (commitOutcome === "source-inactive") {
|
|
55001
|
+
throw validationError("Traffic source is no longer active; discarded the in-flight sync");
|
|
55002
|
+
}
|
|
53909
55003
|
writeAuditLog(app.db, {
|
|
53910
55004
|
projectId: project.id,
|
|
53911
55005
|
actor: "api",
|
|
@@ -53956,7 +55050,7 @@ async function trafficRoutes(app, opts) {
|
|
|
53956
55050
|
});
|
|
53957
55051
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
53958
55052
|
const project = resolveProject(app.db, request.params.name);
|
|
53959
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
55053
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
53960
55054
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
53961
55055
|
throw notFound("Traffic source", request.params.id);
|
|
53962
55056
|
}
|
|
@@ -53965,6 +55059,9 @@ async function trafficRoutes(app, opts) {
|
|
|
53965
55059
|
`Backfill for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
|
|
53966
55060
|
);
|
|
53967
55061
|
}
|
|
55062
|
+
if (!isAuthoritativeTrafficSource(app.db, sourceRow2)) {
|
|
55063
|
+
throw validationError("Traffic source must be active before it can backfill");
|
|
55064
|
+
}
|
|
53968
55065
|
const requestedDays = request.body?.days ?? DEFAULT_BACKFILL_DAYS;
|
|
53969
55066
|
if (!Number.isInteger(requestedDays) || requestedDays <= 0) {
|
|
53970
55067
|
throw validationError('"days" must be a positive integer');
|
|
@@ -54107,16 +55204,22 @@ async function trafficRoutes(app, opts) {
|
|
|
54107
55204
|
}
|
|
54108
55205
|
const startedAt = windowEnd.toISOString();
|
|
54109
55206
|
const runId = crypto40.randomUUID();
|
|
54110
|
-
app.db.
|
|
54111
|
-
id
|
|
54112
|
-
|
|
54113
|
-
|
|
54114
|
-
|
|
54115
|
-
|
|
54116
|
-
|
|
54117
|
-
|
|
54118
|
-
|
|
54119
|
-
|
|
55207
|
+
app.db.transaction((tx) => {
|
|
55208
|
+
const latestSource = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
55209
|
+
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
55210
|
+
throw validationError("Traffic source must remain active and unchanged before it can backfill");
|
|
55211
|
+
}
|
|
55212
|
+
tx.insert(runs).values({
|
|
55213
|
+
id: runId,
|
|
55214
|
+
projectId: project.id,
|
|
55215
|
+
kind: RunKinds["traffic-sync"],
|
|
55216
|
+
status: RunStatuses.running,
|
|
55217
|
+
trigger: RunTriggers.backfill,
|
|
55218
|
+
sourceId: sourceRow2.id,
|
|
55219
|
+
startedAt,
|
|
55220
|
+
createdAt: startedAt
|
|
55221
|
+
}).run();
|
|
55222
|
+
});
|
|
54120
55223
|
void runBackfillTask({
|
|
54121
55224
|
app,
|
|
54122
55225
|
runId,
|
|
@@ -54144,8 +55247,8 @@ async function trafficRoutes(app, opts) {
|
|
|
54144
55247
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
54145
55248
|
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
54146
55249
|
}).from(crawlerEventsHourly).where(
|
|
54147
|
-
|
|
54148
|
-
|
|
55250
|
+
and40(
|
|
55251
|
+
eq48(crawlerEventsHourly.sourceId, row.id),
|
|
54149
55252
|
gte11(crawlerEventsHourly.tsHour, since)
|
|
54150
55253
|
)
|
|
54151
55254
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
@@ -54154,28 +55257,28 @@ async function trafficRoutes(app, opts) {
|
|
|
54154
55257
|
);
|
|
54155
55258
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
54156
55259
|
const aiUserFetchTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
54157
|
-
|
|
54158
|
-
|
|
55260
|
+
and40(
|
|
55261
|
+
eq48(aiUserFetchEventsHourly.sourceId, row.id),
|
|
54159
55262
|
gte11(aiUserFetchEventsHourly.tsHour, since)
|
|
54160
55263
|
)
|
|
54161
55264
|
).get();
|
|
54162
55265
|
const aiTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
54163
|
-
|
|
54164
|
-
|
|
55266
|
+
and40(
|
|
55267
|
+
eq48(aiReferralEventsHourly.sourceId, row.id),
|
|
54165
55268
|
gte11(aiReferralEventsHourly.tsHour, since)
|
|
54166
55269
|
)
|
|
54167
55270
|
).get();
|
|
54168
55271
|
const sampleTotals = app.db.select({ total: sql19`COUNT(*)` }).from(rawEventSamples).where(
|
|
54169
|
-
|
|
54170
|
-
|
|
55272
|
+
and40(
|
|
55273
|
+
eq48(rawEventSamples.sourceId, row.id),
|
|
54171
55274
|
gte11(rawEventSamples.ts, since)
|
|
54172
55275
|
)
|
|
54173
55276
|
).get();
|
|
54174
55277
|
const latestRun = app.db.select().from(runs).where(
|
|
54175
|
-
|
|
54176
|
-
|
|
54177
|
-
|
|
54178
|
-
|
|
55278
|
+
and40(
|
|
55279
|
+
eq48(runs.projectId, projectId),
|
|
55280
|
+
eq48(runs.kind, RunKinds["traffic-sync"]),
|
|
55281
|
+
eq48(runs.sourceId, row.id)
|
|
54179
55282
|
)
|
|
54180
55283
|
).orderBy(desc22(runs.startedAt)).limit(1).get();
|
|
54181
55284
|
return {
|
|
@@ -54206,24 +55309,36 @@ async function trafficRoutes(app, opts) {
|
|
|
54206
55309
|
"`advanceToNow` must be `true`. There is no implicit reset."
|
|
54207
55310
|
);
|
|
54208
55311
|
}
|
|
54209
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(and39(eq47(trafficSources.projectId, project.id), eq47(trafficSources.id, request.params.id))).get();
|
|
54210
|
-
if (!sourceRow2) {
|
|
54211
|
-
throw notFound("traffic source", request.params.id);
|
|
54212
|
-
}
|
|
54213
|
-
if (sourceRow2.status === TrafficSourceStatuses.archived) {
|
|
54214
|
-
throw validationError(
|
|
54215
|
-
`Traffic source "${sourceRow2.id}" is archived. Re-connect via "canonry traffic connect ..." to start tracking it again.`
|
|
54216
|
-
);
|
|
54217
|
-
}
|
|
54218
55312
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
54219
55313
|
let updatedRow;
|
|
54220
55314
|
app.db.transaction((tx) => {
|
|
55315
|
+
const sourceRow2 = tx.select().from(trafficSources).where(and40(eq48(trafficSources.projectId, project.id), eq48(trafficSources.id, request.params.id))).get();
|
|
55316
|
+
if (!sourceRow2) {
|
|
55317
|
+
throw notFound("traffic source", request.params.id);
|
|
55318
|
+
}
|
|
55319
|
+
if (sourceRow2.status === TrafficSourceStatuses.archived) {
|
|
55320
|
+
throw validationError(
|
|
55321
|
+
`Traffic source "${sourceRow2.id}" is archived. Re-connect via "canonry traffic connect ..." to start tracking it again.`
|
|
55322
|
+
);
|
|
55323
|
+
}
|
|
55324
|
+
if (sourceRow2.status === TrafficSourceStatuses.paused) {
|
|
55325
|
+
throw validationError(
|
|
55326
|
+
`Traffic source "${sourceRow2.id}" is staged. Activate it explicitly instead of resetting it.`
|
|
55327
|
+
);
|
|
55328
|
+
}
|
|
55329
|
+
const hasConnectedSibling = tx.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
55330
|
+
if (hasConnectedSibling) {
|
|
55331
|
+
throw validationError(
|
|
55332
|
+
`Traffic source "${sourceRow2.id}" is not authoritative. Activate it explicitly instead of resetting it.`
|
|
55333
|
+
);
|
|
55334
|
+
}
|
|
55335
|
+
validateTrafficSourceCredential(sourceRow2, project.name);
|
|
54221
55336
|
tx.update(trafficSources).set({
|
|
54222
55337
|
lastSyncedAt: now,
|
|
54223
55338
|
status: TrafficSourceStatuses.connected,
|
|
54224
55339
|
lastError: null,
|
|
54225
55340
|
updatedAt: now
|
|
54226
|
-
}).where(
|
|
55341
|
+
}).where(eq48(trafficSources.id, sourceRow2.id)).run();
|
|
54227
55342
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
54228
55343
|
projectId: project.id,
|
|
54229
55344
|
actor: "api",
|
|
@@ -54231,20 +55346,20 @@ async function trafficRoutes(app, opts) {
|
|
|
54231
55346
|
entityType: "traffic_source",
|
|
54232
55347
|
entityId: sourceRow2.id
|
|
54233
55348
|
}));
|
|
54234
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
55349
|
+
updatedRow = tx.select().from(trafficSources).where(eq48(trafficSources.id, sourceRow2.id)).get();
|
|
54235
55350
|
});
|
|
54236
55351
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
54237
55352
|
});
|
|
54238
55353
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
54239
55354
|
const project = resolveProject(app.db, request.params.name);
|
|
54240
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
55355
|
+
const rows = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
54241
55356
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
54242
55357
|
const response = { sources };
|
|
54243
55358
|
return response;
|
|
54244
55359
|
});
|
|
54245
55360
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
54246
55361
|
const project = resolveProject(app.db, request.params.name);
|
|
54247
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
55362
|
+
const rows = app.db.select().from(trafficSources).where(eq48(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
54248
55363
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
54249
55364
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
54250
55365
|
const response = { sources };
|
|
@@ -54254,7 +55369,7 @@ async function trafficRoutes(app, opts) {
|
|
|
54254
55369
|
"/projects/:name/traffic/sources/:id",
|
|
54255
55370
|
async (request) => {
|
|
54256
55371
|
const project = resolveProject(app.db, request.params.name);
|
|
54257
|
-
const row = app.db.select().from(trafficSources).where(
|
|
55372
|
+
const row = app.db.select().from(trafficSources).where(eq48(trafficSources.id, request.params.id)).get();
|
|
54258
55373
|
if (!row || row.projectId !== project.id) {
|
|
54259
55374
|
throw notFound("Traffic source", request.params.id);
|
|
54260
55375
|
}
|
|
@@ -54319,12 +55434,12 @@ async function trafficRoutes(app, opts) {
|
|
|
54319
55434
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
54320
55435
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
54321
55436
|
const crawlerFilters = [
|
|
54322
|
-
|
|
55437
|
+
eq48(crawlerEventsHourly.projectId, project.id),
|
|
54323
55438
|
gte11(crawlerEventsHourly.tsHour, sinceIso),
|
|
54324
|
-
|
|
55439
|
+
lte11(crawlerEventsHourly.tsHour, untilIso)
|
|
54325
55440
|
];
|
|
54326
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
54327
|
-
const crawlerWhere =
|
|
55441
|
+
if (sourceIdParam) crawlerFilters.push(eq48(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
55442
|
+
const crawlerWhere = and40(...crawlerFilters);
|
|
54328
55443
|
const pathTotals = app.db.select({
|
|
54329
55444
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
54330
55445
|
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
@@ -54361,12 +55476,12 @@ async function trafficRoutes(app, opts) {
|
|
|
54361
55476
|
}
|
|
54362
55477
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
54363
55478
|
const userFetchFilters = [
|
|
54364
|
-
|
|
55479
|
+
eq48(aiUserFetchEventsHourly.projectId, project.id),
|
|
54365
55480
|
gte11(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
54366
|
-
|
|
55481
|
+
lte11(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
54367
55482
|
];
|
|
54368
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
54369
|
-
const userFetchWhere =
|
|
55483
|
+
if (sourceIdParam) userFetchFilters.push(eq48(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
55484
|
+
const userFetchWhere = and40(...userFetchFilters);
|
|
54370
55485
|
const total = app.db.select({
|
|
54371
55486
|
total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
54372
55487
|
rows: sql19`COUNT(*)`
|
|
@@ -54398,12 +55513,12 @@ async function trafficRoutes(app, opts) {
|
|
|
54398
55513
|
}
|
|
54399
55514
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
54400
55515
|
const aiFilters = [
|
|
54401
|
-
|
|
55516
|
+
eq48(aiReferralEventsHourly.projectId, project.id),
|
|
54402
55517
|
gte11(aiReferralEventsHourly.tsHour, sinceIso),
|
|
54403
|
-
|
|
55518
|
+
lte11(aiReferralEventsHourly.tsHour, untilIso)
|
|
54404
55519
|
];
|
|
54405
|
-
if (sourceIdParam) aiFilters.push(
|
|
54406
|
-
const aiWhere =
|
|
55520
|
+
if (sourceIdParam) aiFilters.push(eq48(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
55521
|
+
const aiWhere = and40(...aiFilters);
|
|
54407
55522
|
const total = app.db.select({
|
|
54408
55523
|
total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
54409
55524
|
paid: sql19`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
@@ -54722,7 +55837,7 @@ function readInstalledManifest(skillDir) {
|
|
|
54722
55837
|
var AGENT_CHECKS = [skillsInstalledCheck, skillsCurrentCheck];
|
|
54723
55838
|
|
|
54724
55839
|
// ../api-routes/src/doctor/checks/backlinks.ts
|
|
54725
|
-
import { and as
|
|
55840
|
+
import { and as and41, eq as eq49 } from "drizzle-orm";
|
|
54726
55841
|
function skippedNoProject() {
|
|
54727
55842
|
return {
|
|
54728
55843
|
status: CheckStatuses.skipped,
|
|
@@ -54738,8 +55853,8 @@ var BACKLINKS_CHECKS = [
|
|
|
54738
55853
|
title: "Backlinks source connected",
|
|
54739
55854
|
run: (ctx) => {
|
|
54740
55855
|
if (!ctx.project) return skippedNoProject();
|
|
54741
|
-
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(
|
|
54742
|
-
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(
|
|
55856
|
+
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(eq49(projects.id, ctx.project.id)).get();
|
|
55857
|
+
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(eq49(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).limit(1).get();
|
|
54743
55858
|
const ccConnected = projectRow?.autoExtract === true && !!readySync;
|
|
54744
55859
|
const connected = [];
|
|
54745
55860
|
if (ccConnected) connected.push(BacklinkSources.commoncrawl);
|
|
@@ -54752,9 +55867,9 @@ var BACKLINKS_CHECKS = [
|
|
|
54752
55867
|
details: { commoncrawl: ccConnected }
|
|
54753
55868
|
};
|
|
54754
55869
|
}
|
|
54755
|
-
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
54756
|
-
|
|
54757
|
-
|
|
55870
|
+
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and41(
|
|
55871
|
+
eq49(backlinkSummaries.projectId, ctx.project.id),
|
|
55872
|
+
eq49(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
54758
55873
|
)).limit(1).get() : false;
|
|
54759
55874
|
return {
|
|
54760
55875
|
status: CheckStatuses.ok,
|
|
@@ -54914,7 +56029,7 @@ var BING_AUTH_CHECKS = [
|
|
|
54914
56029
|
];
|
|
54915
56030
|
|
|
54916
56031
|
// ../api-routes/src/doctor/checks/content.ts
|
|
54917
|
-
import { eq as
|
|
56032
|
+
import { eq as eq50 } from "drizzle-orm";
|
|
54918
56033
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
54919
56034
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
54920
56035
|
function skippedNoProject2() {
|
|
@@ -54927,7 +56042,7 @@ function skippedNoProject2() {
|
|
|
54927
56042
|
}
|
|
54928
56043
|
function loadProject(ctx) {
|
|
54929
56044
|
if (!ctx.project) return null;
|
|
54930
|
-
return ctx.db.select().from(projects).where(
|
|
56045
|
+
return ctx.db.select().from(projects).where(eq50(projects.id, ctx.project.id)).get() ?? null;
|
|
54931
56046
|
}
|
|
54932
56047
|
function percent(value) {
|
|
54933
56048
|
return Math.round(value * 100);
|
|
@@ -55019,7 +56134,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
55019
56134
|
);
|
|
55020
56135
|
|
|
55021
56136
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
55022
|
-
import { eq as
|
|
56137
|
+
import { eq as eq51 } from "drizzle-orm";
|
|
55023
56138
|
var RECENT_SYNC_WARN_DAYS = 7;
|
|
55024
56139
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
55025
56140
|
var adsConnectionCheck = {
|
|
@@ -55036,7 +56151,7 @@ var adsConnectionCheck = {
|
|
|
55036
56151
|
remediation: null
|
|
55037
56152
|
};
|
|
55038
56153
|
}
|
|
55039
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
56154
|
+
const row = ctx.db.select().from(adsConnections).where(eq51(adsConnections.projectId, ctx.project.id)).get();
|
|
55040
56155
|
if (!row) {
|
|
55041
56156
|
return {
|
|
55042
56157
|
status: CheckStatuses.skipped,
|
|
@@ -55086,7 +56201,7 @@ var adsRecentSyncCheck = {
|
|
|
55086
56201
|
remediation: null
|
|
55087
56202
|
};
|
|
55088
56203
|
}
|
|
55089
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
56204
|
+
const row = ctx.db.select().from(adsConnections).where(eq51(adsConnections.projectId, ctx.project.id)).get();
|
|
55090
56205
|
if (!row) {
|
|
55091
56206
|
return {
|
|
55092
56207
|
status: CheckStatuses.skipped,
|
|
@@ -55277,7 +56392,7 @@ var ga4ConnectionCheck = {
|
|
|
55277
56392
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
55278
56393
|
|
|
55279
56394
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
55280
|
-
import { and as
|
|
56395
|
+
import { and as and42, eq as eq52 } from "drizzle-orm";
|
|
55281
56396
|
var RECENT_SYNC_WARN_DAYS2 = 7;
|
|
55282
56397
|
var RECENT_SYNC_FAIL_DAYS2 = 30;
|
|
55283
56398
|
function skippedNoProject3() {
|
|
@@ -55510,7 +56625,7 @@ var recentSyncCheck = {
|
|
|
55510
56625
|
title: "GBP recent sync",
|
|
55511
56626
|
run: (ctx) => {
|
|
55512
56627
|
if (!ctx.project) return skippedNoProject3();
|
|
55513
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
56628
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and42(eq52(gbpLocations.projectId, ctx.project.id), eq52(gbpLocations.selected, true))).all();
|
|
55514
56629
|
if (selected.length === 0) {
|
|
55515
56630
|
return {
|
|
55516
56631
|
status: CheckStatuses.skipped,
|
|
@@ -55570,7 +56685,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
55570
56685
|
);
|
|
55571
56686
|
|
|
55572
56687
|
// ../api-routes/src/doctor/checks/places.ts
|
|
55573
|
-
import { eq as
|
|
56688
|
+
import { eq as eq53 } from "drizzle-orm";
|
|
55574
56689
|
var apiKeyCheck = {
|
|
55575
56690
|
id: "gbp.places.api-key",
|
|
55576
56691
|
category: CheckCategories.auth,
|
|
@@ -55615,7 +56730,7 @@ var apiKeyCheck = {
|
|
|
55615
56730
|
details: { tier: cfg.tier }
|
|
55616
56731
|
};
|
|
55617
56732
|
}
|
|
55618
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
56733
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq53(gbpLocations.projectId, ctx.project.id)).all();
|
|
55619
56734
|
const selected = rows.filter((r) => r.selected);
|
|
55620
56735
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
55621
56736
|
const details = {
|
|
@@ -56130,7 +57245,7 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
56130
57245
|
];
|
|
56131
57246
|
|
|
56132
57247
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
56133
|
-
import { and as
|
|
57248
|
+
import { and as and43, eq as eq54, gte as gte12, inArray as inArray18, ne as ne6, sql as sql20 } from "drizzle-orm";
|
|
56134
57249
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
56135
57250
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
56136
57251
|
function isCloudflareDirectPush(source) {
|
|
@@ -56138,6 +57253,12 @@ function isCloudflareDirectPush(source) {
|
|
|
56138
57253
|
const deliveryMode = source.configJson.deliveryMode;
|
|
56139
57254
|
return deliveryMode === void 0 || deliveryMode === CloudflareTrafficDeliveryModes["direct-push"];
|
|
56140
57255
|
}
|
|
57256
|
+
function isCloudflareQueuePull(source) {
|
|
57257
|
+
return source.sourceType === TrafficSourceTypes.cloudflare && source.configJson.deliveryMode === CloudflareTrafficDeliveryModes["queue-pull"];
|
|
57258
|
+
}
|
|
57259
|
+
function isActiveSource(source) {
|
|
57260
|
+
return source.status !== TrafficSourceStatuses.paused;
|
|
57261
|
+
}
|
|
56141
57262
|
function recentDataRemediation(sources, lastSyncedAt) {
|
|
56142
57263
|
const hasCloudflare = sources.some(isCloudflareDirectPush);
|
|
56143
57264
|
const hasPullSource = sources.some((source) => !isCloudflareDirectPush(source));
|
|
@@ -56160,8 +57281,8 @@ function skippedNoProject5() {
|
|
|
56160
57281
|
function loadProbes(ctx) {
|
|
56161
57282
|
if (!ctx.project) return [];
|
|
56162
57283
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
56163
|
-
|
|
56164
|
-
|
|
57284
|
+
and43(
|
|
57285
|
+
eq54(trafficSources.projectId, ctx.project.id),
|
|
56165
57286
|
ne6(trafficSources.status, TrafficSourceStatuses.archived)
|
|
56166
57287
|
)
|
|
56167
57288
|
).all();
|
|
@@ -56176,6 +57297,8 @@ function loadProbes(ctx) {
|
|
|
56176
57297
|
lastWorkerVersion: r.lastWorkerVersion,
|
|
56177
57298
|
ingestTokenHash: r.ingestTokenHash,
|
|
56178
57299
|
skippedThroughAt: r.skippedThroughAt,
|
|
57300
|
+
queueBacklogCount: r.queueBacklogCount,
|
|
57301
|
+
queueBacklogObservedAt: r.queueBacklogObservedAt,
|
|
56179
57302
|
lastError: r.lastError,
|
|
56180
57303
|
configJson: r.configJson
|
|
56181
57304
|
}));
|
|
@@ -56197,30 +57320,40 @@ var sourceConnectedCheck = {
|
|
|
56197
57320
|
details: { sourceCount: 0 }
|
|
56198
57321
|
};
|
|
56199
57322
|
}
|
|
56200
|
-
const
|
|
56201
|
-
if (
|
|
57323
|
+
const activeSources = sources.filter(isActiveSource);
|
|
57324
|
+
if (activeSources.length === 0) {
|
|
57325
|
+
return {
|
|
57326
|
+
status: CheckStatuses.skipped,
|
|
57327
|
+
code: "traffic.source.paused",
|
|
57328
|
+
summary: `${sources.length} traffic source(s) are paused or staged; none are actively ingesting.`,
|
|
57329
|
+
remediation: "Activate the intended traffic source after its deployment smoke test succeeds.",
|
|
57330
|
+
details: { sourceCount: sources.length, pausedIds: sources.map((s) => s.id) }
|
|
57331
|
+
};
|
|
57332
|
+
}
|
|
57333
|
+
const errored = activeSources.filter((s) => s.status === "error");
|
|
57334
|
+
if (errored.length > 0 && errored.length === activeSources.length) {
|
|
56202
57335
|
return {
|
|
56203
57336
|
status: CheckStatuses.fail,
|
|
56204
57337
|
code: "traffic.source.all-errored",
|
|
56205
|
-
summary: `All ${
|
|
57338
|
+
summary: `All ${activeSources.length} active traffic source(s) are in error state. No data is being ingested.`,
|
|
56206
57339
|
remediation: errored[0].lastError ? `Latest error: "${errored[0].lastError}". Re-connect the source or run \`canonry traffic sync <project> --source <id>\` to retry.` : "Run `canonry traffic sources <project>` to inspect the failing source(s) and re-connect.",
|
|
56207
|
-
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
57340
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length, erroredIds: errored.map((s) => s.id) }
|
|
56208
57341
|
};
|
|
56209
57342
|
}
|
|
56210
57343
|
if (errored.length > 0) {
|
|
56211
57344
|
return {
|
|
56212
57345
|
status: CheckStatuses.warn,
|
|
56213
57346
|
code: "traffic.source.partially-errored",
|
|
56214
|
-
summary: `${errored.length} of ${
|
|
57347
|
+
summary: `${errored.length} of ${activeSources.length} active traffic source(s) are in error state.`,
|
|
56215
57348
|
remediation: "Run `canonry traffic sources <project>` to inspect the failing sources individually.",
|
|
56216
|
-
details: { sourceCount: sources.length, erroredIds: errored.map((s) => s.id) }
|
|
57349
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length, erroredIds: errored.map((s) => s.id) }
|
|
56217
57350
|
};
|
|
56218
57351
|
}
|
|
56219
57352
|
return {
|
|
56220
57353
|
status: CheckStatuses.ok,
|
|
56221
57354
|
code: "traffic.source.connected",
|
|
56222
|
-
summary: `${
|
|
56223
|
-
details: { sourceCount: sources.length, sourceTypes: [...new Set(
|
|
57355
|
+
summary: `${activeSources.length} traffic source(s) connected: ${activeSources.map((s) => s.displayName).join(", ")}.`,
|
|
57356
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length, sourceTypes: [...new Set(activeSources.map((s) => s.sourceType))] }
|
|
56224
57357
|
};
|
|
56225
57358
|
}
|
|
56226
57359
|
};
|
|
@@ -56239,29 +57372,43 @@ var recentDataCheck = {
|
|
|
56239
57372
|
summary: "No traffic source connected \u2014 recent-data check skipped."
|
|
56240
57373
|
};
|
|
56241
57374
|
}
|
|
57375
|
+
const activeSources = sources.filter(isActiveSource);
|
|
57376
|
+
if (activeSources.length === 0) {
|
|
57377
|
+
return {
|
|
57378
|
+
status: CheckStatuses.skipped,
|
|
57379
|
+
code: "traffic.recent-data.no-active-source",
|
|
57380
|
+
summary: "Only paused or staged traffic sources are configured \u2014 recent-data check skipped.",
|
|
57381
|
+
remediation: "Activate the intended source after its deployment smoke test succeeds.",
|
|
57382
|
+
details: { sourceCount: sources.length, pausedIds: sources.map((source) => source.id) }
|
|
57383
|
+
};
|
|
57384
|
+
}
|
|
57385
|
+
const activeSourceIds = activeSources.map((source) => source.id);
|
|
56242
57386
|
const now = /* @__PURE__ */ new Date();
|
|
56243
57387
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
56244
57388
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
56245
57389
|
const recentCrawlers = Number(
|
|
56246
57390
|
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
56247
|
-
|
|
56248
|
-
|
|
57391
|
+
and43(
|
|
57392
|
+
eq54(crawlerEventsHourly.projectId, ctx.project.id),
|
|
57393
|
+
inArray18(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
56249
57394
|
gte12(crawlerEventsHourly.tsHour, warnCutoff)
|
|
56250
57395
|
)
|
|
56251
57396
|
).get()?.total ?? 0
|
|
56252
57397
|
);
|
|
56253
57398
|
const recentReferrals = Number(
|
|
56254
57399
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
56255
|
-
|
|
56256
|
-
|
|
57400
|
+
and43(
|
|
57401
|
+
eq54(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
57402
|
+
inArray18(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
56257
57403
|
gte12(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
56258
57404
|
)
|
|
56259
57405
|
).get()?.total ?? 0
|
|
56260
57406
|
);
|
|
56261
57407
|
const recentUserFetches = Number(
|
|
56262
57408
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
56263
|
-
|
|
56264
|
-
|
|
57409
|
+
and43(
|
|
57410
|
+
eq54(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
57411
|
+
inArray18(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
56265
57412
|
gte12(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
56266
57413
|
)
|
|
56267
57414
|
).get()?.total ?? 0
|
|
@@ -56281,37 +57428,40 @@ var recentDataCheck = {
|
|
|
56281
57428
|
}
|
|
56282
57429
|
const olderCrawlers = Number(
|
|
56283
57430
|
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
56284
|
-
|
|
56285
|
-
|
|
57431
|
+
and43(
|
|
57432
|
+
eq54(crawlerEventsHourly.projectId, ctx.project.id),
|
|
57433
|
+
inArray18(crawlerEventsHourly.sourceId, activeSourceIds),
|
|
56286
57434
|
gte12(crawlerEventsHourly.tsHour, failCutoff)
|
|
56287
57435
|
)
|
|
56288
57436
|
).get()?.total ?? 0
|
|
56289
57437
|
);
|
|
56290
57438
|
const olderReferrals = Number(
|
|
56291
57439
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
56292
|
-
|
|
56293
|
-
|
|
57440
|
+
and43(
|
|
57441
|
+
eq54(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
57442
|
+
inArray18(aiReferralEventsHourly.sourceId, activeSourceIds),
|
|
56294
57443
|
gte12(aiReferralEventsHourly.tsHour, failCutoff)
|
|
56295
57444
|
)
|
|
56296
57445
|
).get()?.total ?? 0
|
|
56297
57446
|
);
|
|
56298
57447
|
const olderUserFetches = Number(
|
|
56299
57448
|
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
56300
|
-
|
|
56301
|
-
|
|
57449
|
+
and43(
|
|
57450
|
+
eq54(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
57451
|
+
inArray18(aiUserFetchEventsHourly.sourceId, activeSourceIds),
|
|
56302
57452
|
gte12(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
56303
57453
|
)
|
|
56304
57454
|
).get()?.total ?? 0
|
|
56305
57455
|
);
|
|
56306
|
-
const lastSyncedAt =
|
|
57456
|
+
const lastSyncedAt = activeSources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
56307
57457
|
const hasOlderData = olderCrawlers > 0 || olderUserFetches > 0 || olderReferrals > 0;
|
|
56308
57458
|
if (hasOlderData || lastSyncedAt) {
|
|
56309
57459
|
return {
|
|
56310
57460
|
status: CheckStatuses.warn,
|
|
56311
57461
|
code: "traffic.recent-data.stale",
|
|
56312
57462
|
summary: hasOlderData ? `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.` : `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
56313
|
-
remediation: recentDataRemediation(
|
|
56314
|
-
details: { lastSyncedAt, sourceCount: sources.length }
|
|
57463
|
+
remediation: recentDataRemediation(activeSources, lastSyncedAt),
|
|
57464
|
+
details: { lastSyncedAt, sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length }
|
|
56315
57465
|
};
|
|
56316
57466
|
}
|
|
56317
57467
|
return {
|
|
@@ -56319,7 +57469,7 @@ var recentDataCheck = {
|
|
|
56319
57469
|
code: "traffic.recent-data.empty",
|
|
56320
57470
|
summary: `No traffic data in the last ${RECENT_DATA_FAIL_DAYS} days. The source is connected but isn't ingesting.`,
|
|
56321
57471
|
remediation: "Verify the source's configuration with `canonry traffic sources <project>` and run a manual sync to confirm credentials + scopes are still valid.",
|
|
56322
|
-
details: { sourceCount: sources.length }
|
|
57472
|
+
details: { sourceCount: activeSources.length, pausedCount: sources.length - activeSources.length }
|
|
56323
57473
|
};
|
|
56324
57474
|
}
|
|
56325
57475
|
};
|
|
@@ -56405,6 +57555,7 @@ function summarizePerSourceResults(fallbackId, fallbackLabel, results) {
|
|
|
56405
57555
|
status: CheckStatuses.skipped,
|
|
56406
57556
|
code: `traffic.${fallbackId}.all-skipped`,
|
|
56407
57557
|
summary: `No source-type validator was available for any of the ${results.length} connected source(s).`,
|
|
57558
|
+
remediation: skipped.find((result) => result.output.remediation)?.output.remediation,
|
|
56408
57559
|
details: detail
|
|
56409
57560
|
};
|
|
56410
57561
|
}
|
|
@@ -56509,8 +57660,16 @@ var syncLagCheck = {
|
|
|
56509
57660
|
run: (ctx) => {
|
|
56510
57661
|
if (!ctx.project) return skippedNoProject5();
|
|
56511
57662
|
const allSources = loadProbes(ctx);
|
|
56512
|
-
const
|
|
56513
|
-
if (allSources.length > 0 &&
|
|
57663
|
+
const activeSources = allSources.filter(isActiveSource);
|
|
57664
|
+
if (allSources.length > 0 && activeSources.length === 0) {
|
|
57665
|
+
return {
|
|
57666
|
+
status: CheckStatuses.skipped,
|
|
57667
|
+
code: "traffic.sync-lag.no-active-source",
|
|
57668
|
+
summary: "Only paused or staged traffic sources are configured \u2014 pull sync lag does not apply."
|
|
57669
|
+
};
|
|
57670
|
+
}
|
|
57671
|
+
const sources = activeSources.filter((source) => !isCloudflareDirectPush(source));
|
|
57672
|
+
if (activeSources.length > 0 && sources.length === 0) {
|
|
56514
57673
|
return {
|
|
56515
57674
|
status: CheckStatuses.skipped,
|
|
56516
57675
|
code: "traffic.sync-lag.push-only",
|
|
@@ -56606,6 +57765,69 @@ var syncLagCheck = {
|
|
|
56606
57765
|
};
|
|
56607
57766
|
}
|
|
56608
57767
|
};
|
|
57768
|
+
var queueBacklogCheck = {
|
|
57769
|
+
id: "traffic.source.queue-backlog",
|
|
57770
|
+
category: CheckCategories.integrations,
|
|
57771
|
+
scope: CheckScopes.project,
|
|
57772
|
+
title: "Cloudflare Queue backlog",
|
|
57773
|
+
run: (ctx) => {
|
|
57774
|
+
if (!ctx.project) return skippedNoProject5();
|
|
57775
|
+
const sources = loadProbes(ctx).filter(
|
|
57776
|
+
(source) => isActiveSource(source) && isCloudflareQueuePull(source)
|
|
57777
|
+
);
|
|
57778
|
+
if (sources.length === 0) {
|
|
57779
|
+
return {
|
|
57780
|
+
status: CheckStatuses.skipped,
|
|
57781
|
+
code: "traffic.queue-backlog.not-applicable",
|
|
57782
|
+
summary: "No active Cloudflare Queue pull source is connected."
|
|
57783
|
+
};
|
|
57784
|
+
}
|
|
57785
|
+
const measured = sources.filter((source) => source.queueBacklogCount !== null && source.queueBacklogCount !== void 0);
|
|
57786
|
+
const details = {
|
|
57787
|
+
sources: sources.map((source) => ({
|
|
57788
|
+
id: source.id,
|
|
57789
|
+
displayName: source.displayName,
|
|
57790
|
+
queueBacklogCount: source.queueBacklogCount ?? null,
|
|
57791
|
+
queueBacklogObservedAt: source.queueBacklogObservedAt ?? null
|
|
57792
|
+
}))
|
|
57793
|
+
};
|
|
57794
|
+
if (measured.length === 0) {
|
|
57795
|
+
return {
|
|
57796
|
+
status: CheckStatuses.skipped,
|
|
57797
|
+
code: "traffic.queue-backlog.not-observed",
|
|
57798
|
+
summary: "Cloudflare Queue backlog has not been observed yet.",
|
|
57799
|
+
remediation: "Run `canonry traffic sync <project> --source <id>` to pull the Queue and record its residual depth.",
|
|
57800
|
+
details
|
|
57801
|
+
};
|
|
57802
|
+
}
|
|
57803
|
+
const remaining = measured.filter((source) => (source.queueBacklogCount ?? 0) > 0);
|
|
57804
|
+
const total = remaining.reduce((sum, source) => sum + source.queueBacklogCount, 0);
|
|
57805
|
+
if (total > DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET) {
|
|
57806
|
+
return {
|
|
57807
|
+
status: CheckStatuses.warn,
|
|
57808
|
+
code: "traffic.queue-backlog.remaining",
|
|
57809
|
+
summary: `${total.toLocaleString("en-US")} message(s) remained across ${remaining.length} Cloudflare Queue source(s) after the last bounded sync, exceeding the default ${DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET.toLocaleString("en-US")}-message drain budget.`,
|
|
57810
|
+
remediation: "Run `canonry traffic sync <project> --source <id>` again. If the backlog stays above 1,000 messages, shorten the traffic-sync schedule interval.",
|
|
57811
|
+
details
|
|
57812
|
+
};
|
|
57813
|
+
}
|
|
57814
|
+
if (total > 0) {
|
|
57815
|
+
return {
|
|
57816
|
+
status: CheckStatuses.ok,
|
|
57817
|
+
code: "traffic.queue-backlog.within-drain-budget",
|
|
57818
|
+
summary: `${total.toLocaleString("en-US")} message(s) remained after the last bounded sync, within the default ${DEFAULT_CLOUDFLARE_QUEUE_DRAIN_BUDGET.toLocaleString("en-US")}-message drain budget.`,
|
|
57819
|
+
remediation: "If no new messages arrive, the next scheduled sync can drain this residual backlog. Run `canonry traffic sync <project> --source <id>` to accelerate it.",
|
|
57820
|
+
details
|
|
57821
|
+
};
|
|
57822
|
+
}
|
|
57823
|
+
return {
|
|
57824
|
+
status: CheckStatuses.ok,
|
|
57825
|
+
code: "traffic.queue-backlog.empty",
|
|
57826
|
+
summary: "The last observed Cloudflare Queue backlog was empty.",
|
|
57827
|
+
details
|
|
57828
|
+
};
|
|
57829
|
+
}
|
|
57830
|
+
};
|
|
56609
57831
|
var workerVersionCheck = {
|
|
56610
57832
|
id: "traffic.source.worker-version",
|
|
56611
57833
|
category: CheckCategories.integrations,
|
|
@@ -56613,7 +57835,7 @@ var workerVersionCheck = {
|
|
|
56613
57835
|
title: "Cloudflare Worker version",
|
|
56614
57836
|
run: (ctx) => {
|
|
56615
57837
|
if (!ctx.project) return skippedNoProject5();
|
|
56616
|
-
const sources = loadProbes(ctx).filter(isCloudflareDirectPush);
|
|
57838
|
+
const sources = loadProbes(ctx).filter((source) => isActiveSource(source) && isCloudflareDirectPush(source));
|
|
56617
57839
|
if (sources.length === 0) {
|
|
56618
57840
|
return {
|
|
56619
57841
|
status: CheckStatuses.skipped,
|
|
@@ -56667,6 +57889,7 @@ var TRAFFIC_SOURCE_CHECKS = [
|
|
|
56667
57889
|
sourceConnectedCheck,
|
|
56668
57890
|
recentDataCheck,
|
|
56669
57891
|
syncLagCheck,
|
|
57892
|
+
queueBacklogCheck,
|
|
56670
57893
|
workerVersionCheck,
|
|
56671
57894
|
credentialsCheck,
|
|
56672
57895
|
scopesCheck3,
|
|
@@ -56887,7 +58110,7 @@ async function doctorRoutes(app, opts) {
|
|
|
56887
58110
|
|
|
56888
58111
|
// ../api-routes/src/discovery/routes.ts
|
|
56889
58112
|
import crypto42 from "crypto";
|
|
56890
|
-
import { and as
|
|
58113
|
+
import { and as and44, desc as desc23, eq as eq55, gte as gte13, inArray as inArray19, isNull as isNull5, or as or11 } from "drizzle-orm";
|
|
56891
58114
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
56892
58115
|
async function discoveryRoutes(app, opts) {
|
|
56893
58116
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -56921,13 +58144,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
56921
58144
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
56922
58145
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
56923
58146
|
const decision = app.db.transaction((tx) => {
|
|
56924
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
56925
|
-
|
|
56926
|
-
|
|
58147
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and44(
|
|
58148
|
+
eq55(discoverySessions.projectId, project.id),
|
|
58149
|
+
eq55(discoverySessions.icpDescription, icpDescription),
|
|
56927
58150
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
56928
58151
|
// semantics, so a request with a different (or no) buyer must start
|
|
56929
58152
|
// its own session, never adopt another buyer's probes.
|
|
56930
|
-
parsed.data.buyerDescription == null ?
|
|
58153
|
+
parsed.data.buyerDescription == null ? isNull5(discoverySessions.buyerDescription) : eq55(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
56931
58154
|
// Locations are identity too: a different service-area subset seeds
|
|
56932
58155
|
// and probes a different geo, so it must never reuse another geo's
|
|
56933
58156
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -56937,12 +58160,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
56937
58160
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
56938
58161
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
56939
58162
|
// upgrade, never a wrong reuse.
|
|
56940
|
-
locations.length === 0 ?
|
|
58163
|
+
locations.length === 0 ? or11(isNull5(discoverySessions.locations), eq55(discoverySessions.locations, locations)) : eq55(discoverySessions.locations, locations),
|
|
56941
58164
|
// Seed provider set is identity: a different phrasing distribution
|
|
56942
58165
|
// must never reuse another set's session. Null (= Gemini-only
|
|
56943
58166
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
56944
|
-
seedProviders == null ?
|
|
56945
|
-
|
|
58167
|
+
seedProviders == null ? isNull5(discoverySessions.seedProviders) : eq55(discoverySessions.seedProviders, seedProviders),
|
|
58168
|
+
inArray19(discoverySessions.status, [
|
|
56946
58169
|
DiscoverySessionStatuses.queued,
|
|
56947
58170
|
DiscoverySessionStatuses.seeding,
|
|
56948
58171
|
DiscoverySessionStatuses.probing
|
|
@@ -57017,7 +58240,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57017
58240
|
const project = resolveProject(app.db, request.params.name);
|
|
57018
58241
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
57019
58242
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
57020
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
58243
|
+
const rows = app.db.select().from(discoverySessions).where(eq55(discoverySessions.projectId, project.id)).orderBy(desc23(discoverySessions.createdAt)).limit(limit).all();
|
|
57021
58244
|
return reply.send(rows.map(serializeSession));
|
|
57022
58245
|
}
|
|
57023
58246
|
);
|
|
@@ -57025,11 +58248,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
57025
58248
|
"/projects/:name/discover/sessions/:id",
|
|
57026
58249
|
async (request, reply) => {
|
|
57027
58250
|
const project = resolveProject(app.db, request.params.name);
|
|
57028
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58251
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57029
58252
|
if (!session || session.projectId !== project.id) {
|
|
57030
58253
|
throw notFound("Discovery session", request.params.id);
|
|
57031
58254
|
}
|
|
57032
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
58255
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
57033
58256
|
const detail = {
|
|
57034
58257
|
...serializeSession(session),
|
|
57035
58258
|
probes: probeRows.map(serializeProbe)
|
|
@@ -57041,7 +58264,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57041
58264
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
57042
58265
|
async (request, reply) => {
|
|
57043
58266
|
const project = resolveProject(app.db, request.params.name);
|
|
57044
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58267
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57045
58268
|
if (!session || session.projectId !== project.id) {
|
|
57046
58269
|
throw notFound("Discovery session", request.params.id);
|
|
57047
58270
|
}
|
|
@@ -57049,7 +58272,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57049
58272
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
57050
58273
|
const applyAnchor = request.query.anchor !== "false";
|
|
57051
58274
|
const provider = session.seedProvider ?? "gemini";
|
|
57052
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
58275
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
57053
58276
|
const extract = opts.harvestSearchQueries;
|
|
57054
58277
|
const probesWithQueries = probeRows.map((row) => {
|
|
57055
58278
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -57060,7 +58283,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57060
58283
|
return { searchQueries: [] };
|
|
57061
58284
|
}
|
|
57062
58285
|
});
|
|
57063
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
58286
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq55(queries.projectId, project.id)).all().map((r) => r.query);
|
|
57064
58287
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
57065
58288
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
57066
58289
|
effectiveDomains(project)
|
|
@@ -57107,12 +58330,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
57107
58330
|
"/projects/:name/discover/sessions/:id/promote",
|
|
57108
58331
|
async (request, reply) => {
|
|
57109
58332
|
const project = resolveProject(app.db, request.params.name);
|
|
57110
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58333
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57111
58334
|
if (!session || session.projectId !== project.id) {
|
|
57112
58335
|
throw notFound("Discovery session", request.params.id);
|
|
57113
58336
|
}
|
|
57114
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
57115
|
-
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
58337
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
58338
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq55(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
57116
58339
|
const seenCompetitors = new Set(existingCompetitors);
|
|
57117
58340
|
const cited = /* @__PURE__ */ new Set();
|
|
57118
58341
|
const aspirational = /* @__PURE__ */ new Set();
|
|
@@ -57141,7 +58364,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57141
58364
|
);
|
|
57142
58365
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
57143
58366
|
const project = resolveProject(app.db, request.params.name);
|
|
57144
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
58367
|
+
const session = app.db.select().from(discoverySessions).where(eq55(discoverySessions.id, request.params.id)).get();
|
|
57145
58368
|
if (!session || session.projectId !== project.id) {
|
|
57146
58369
|
throw notFound("Discovery session", request.params.id);
|
|
57147
58370
|
}
|
|
@@ -57164,7 +58387,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57164
58387
|
const bucketSet = new Set(buckets);
|
|
57165
58388
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
57166
58389
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
57167
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
58390
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq55(discoveryProbes.sessionId, session.id)).all();
|
|
57168
58391
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
57169
58392
|
for (const probe of probeRows) {
|
|
57170
58393
|
if (!probe.bucket) continue;
|
|
@@ -57172,7 +58395,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57172
58395
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
57173
58396
|
}
|
|
57174
58397
|
const existingQueries = new Set(
|
|
57175
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
58398
|
+
app.db.select({ query: queries.query }).from(queries).where(eq55(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
57176
58399
|
);
|
|
57177
58400
|
const promotedQueries = [];
|
|
57178
58401
|
const skippedQueries = [];
|
|
@@ -57188,7 +58411,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
57188
58411
|
const skippedCompetitors = [];
|
|
57189
58412
|
if (includeCompetitors) {
|
|
57190
58413
|
const existingCompetitors = new Set(
|
|
57191
|
-
app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
58414
|
+
app.db.select({ domain: competitors.domain }).from(competitors).where(eq55(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase())
|
|
57192
58415
|
);
|
|
57193
58416
|
const competitorMap = parseCompetitorMap(session.competitorMap);
|
|
57194
58417
|
for (const entry of selectEligibleCompetitors(competitorMap, competitorTypes)) {
|
|
@@ -57307,7 +58530,7 @@ function selectEligibleCompetitors(competitorMap, competitorTypes) {
|
|
|
57307
58530
|
|
|
57308
58531
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
57309
58532
|
import crypto43 from "crypto";
|
|
57310
|
-
import { eq as
|
|
58533
|
+
import { eq as eq56 } from "drizzle-orm";
|
|
57311
58534
|
var DEFAULT_MAX_PROBES = 100;
|
|
57312
58535
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
57313
58536
|
function classifyProbeBucket(input) {
|
|
@@ -57394,7 +58617,7 @@ async function executeDiscovery(opts) {
|
|
|
57394
58617
|
status: DiscoverySessionStatuses.seeding,
|
|
57395
58618
|
dedupThreshold,
|
|
57396
58619
|
startedAt
|
|
57397
|
-
}).where(
|
|
58620
|
+
}).where(eq56(discoverySessions.id, opts.sessionId)).run();
|
|
57398
58621
|
const seedResult = await opts.deps.seed({
|
|
57399
58622
|
project: opts.project,
|
|
57400
58623
|
icpDescription: opts.icpDescription,
|
|
@@ -57455,7 +58678,7 @@ async function executeDiscovery(opts) {
|
|
|
57455
58678
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
57456
58679
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
57457
58680
|
warning
|
|
57458
|
-
}).where(
|
|
58681
|
+
}).where(eq56(discoverySessions.id, opts.sessionId)).run();
|
|
57459
58682
|
const probeLocation = opts.locations?.[0];
|
|
57460
58683
|
const probeResults = await mapWithConcurrency(
|
|
57461
58684
|
probedCanonicals,
|
|
@@ -57509,7 +58732,7 @@ async function executeDiscovery(opts) {
|
|
|
57509
58732
|
wastedCount: buckets["wasted-surface"],
|
|
57510
58733
|
competitorMap,
|
|
57511
58734
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
57512
|
-
}).where(
|
|
58735
|
+
}).where(eq56(discoverySessions.id, opts.sessionId)).run();
|
|
57513
58736
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
57514
58737
|
return {
|
|
57515
58738
|
buckets,
|
|
@@ -57549,7 +58772,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
57549
58772
|
status: DiscoverySessionStatuses.failed,
|
|
57550
58773
|
error,
|
|
57551
58774
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
57552
|
-
}).where(
|
|
58775
|
+
}).where(eq56(discoverySessions.id, sessionId)).run();
|
|
57553
58776
|
}
|
|
57554
58777
|
function dedupeStrings(input) {
|
|
57555
58778
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -57567,7 +58790,7 @@ function dedupeStrings(input) {
|
|
|
57567
58790
|
|
|
57568
58791
|
// ../api-routes/src/technical-aeo.ts
|
|
57569
58792
|
import crypto44 from "crypto";
|
|
57570
|
-
import { and as
|
|
58793
|
+
import { and as and45, asc as asc10, count, desc as desc24, eq as eq57, inArray as inArray20, isNotNull as isNotNull3, isNull as isNull6, lt as lt8, or as or12, sql as sql21 } from "drizzle-orm";
|
|
57571
58794
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
57572
58795
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
57573
58796
|
...SiteCrawlFetchedStates,
|
|
@@ -57766,8 +58989,8 @@ function parseLinkKind(value) {
|
|
|
57766
58989
|
return allowed.data;
|
|
57767
58990
|
}
|
|
57768
58991
|
function linkKindFilter(linkKind) {
|
|
57769
|
-
if (linkKind === SiteHealthLinkKinds.content) return
|
|
57770
|
-
if (linkKind === SiteHealthLinkKinds.template) return
|
|
58992
|
+
if (linkKind === SiteHealthLinkKinds.content) return eq57(siteCrawlEdges.isTemplate, false);
|
|
58993
|
+
if (linkKind === SiteHealthLinkKinds.template) return eq57(siteCrawlEdges.isTemplate, true);
|
|
57771
58994
|
return void 0;
|
|
57772
58995
|
}
|
|
57773
58996
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
@@ -57872,50 +59095,50 @@ function changedFields(before, after, fields) {
|
|
|
57872
59095
|
async function technicalAeoRoutes(app, opts) {
|
|
57873
59096
|
const resolveCrawl = (projectId, runId) => {
|
|
57874
59097
|
const filters = [
|
|
57875
|
-
|
|
57876
|
-
|
|
57877
|
-
|
|
57878
|
-
|
|
59098
|
+
eq57(siteCrawlSnapshots.projectId, projectId),
|
|
59099
|
+
eq57(runs.projectId, projectId),
|
|
59100
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59101
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57879
59102
|
notProbeRun()
|
|
57880
59103
|
];
|
|
57881
59104
|
if (runId) {
|
|
57882
|
-
filters.push(
|
|
59105
|
+
filters.push(eq57(siteCrawlSnapshots.runId, runId));
|
|
57883
59106
|
} else {
|
|
57884
|
-
filters.push(
|
|
59107
|
+
filters.push(eq57(siteCrawlSnapshots.complete, true), eq57(runs.status, RunStatuses.completed));
|
|
57885
59108
|
}
|
|
57886
|
-
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59109
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
57887
59110
|
};
|
|
57888
59111
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
57889
59112
|
const pageInScope = (scope, selector) => {
|
|
57890
59113
|
if (!selector.nodeKey && !selector.url) return null;
|
|
57891
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
57892
|
-
|
|
57893
|
-
|
|
57894
|
-
|
|
57895
|
-
selector.nodeKey ?
|
|
59114
|
+
return app.db.select().from(siteCrawlPages).where(and45(
|
|
59115
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59116
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59117
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59118
|
+
selector.nodeKey ? eq57(siteCrawlPages.nodeKey, selector.nodeKey) : eq57(siteCrawlPages.url, selector.url)
|
|
57896
59119
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57897
59120
|
};
|
|
57898
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
57899
|
-
|
|
57900
|
-
|
|
57901
|
-
|
|
57902
|
-
|
|
59121
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and45(
|
|
59122
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59123
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59124
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59125
|
+
eq57(siteCrawlPages.depth, 0)
|
|
57903
59126
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
57904
|
-
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(
|
|
57905
|
-
|
|
57906
|
-
|
|
57907
|
-
|
|
57908
|
-
|
|
59127
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and45(
|
|
59128
|
+
eq57(runs.id, runId),
|
|
59129
|
+
eq57(runs.projectId, projectId),
|
|
59130
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59131
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57909
59132
|
notProbeRun()
|
|
57910
59133
|
)).limit(1).get());
|
|
57911
59134
|
const assertKnownAuditRun = (projectId, runId) => {
|
|
57912
59135
|
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
57913
59136
|
};
|
|
57914
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57915
|
-
|
|
57916
|
-
|
|
57917
|
-
|
|
57918
|
-
|
|
59137
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(
|
|
59138
|
+
eq57(siteAuditSnapshots.projectId, projectId),
|
|
59139
|
+
eq57(runs.projectId, projectId),
|
|
59140
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59141
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57919
59142
|
notProbeRun()
|
|
57920
59143
|
)).limit(1).get());
|
|
57921
59144
|
const emptyCrawlSummary = (projectName, legacyAuditAvailable) => ({
|
|
@@ -57936,20 +59159,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57936
59159
|
app.get("/projects/:name/technical-aeo", async (request) => {
|
|
57937
59160
|
const project = resolveProject(app.db, request.params.name);
|
|
57938
59161
|
const baseFilters = [
|
|
57939
|
-
|
|
57940
|
-
|
|
57941
|
-
|
|
57942
|
-
|
|
59162
|
+
eq57(siteAuditSnapshots.projectId, project.id),
|
|
59163
|
+
eq57(runs.projectId, project.id),
|
|
59164
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59165
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57943
59166
|
notProbeRun()
|
|
57944
59167
|
];
|
|
57945
|
-
const targetFilters = request.query.runId ? [...baseFilters,
|
|
57946
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
59168
|
+
const targetFilters = request.query.runId ? [...baseFilters, eq57(siteAuditSnapshots.runId, request.query.runId)] : baseFilters;
|
|
59169
|
+
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
57947
59170
|
if (!latest) {
|
|
57948
59171
|
if (request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
57949
59172
|
return emptyScore(project.name);
|
|
57950
59173
|
}
|
|
57951
59174
|
const snap = latest.snap;
|
|
57952
|
-
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs,
|
|
59175
|
+
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(...baseFilters, lt8(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
57953
59176
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
57954
59177
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
57955
59178
|
return {
|
|
@@ -57976,14 +59199,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57976
59199
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
57977
59200
|
const project = resolveProject(app.db, request.params.name);
|
|
57978
59201
|
const targetFilters = [
|
|
57979
|
-
|
|
57980
|
-
|
|
57981
|
-
|
|
57982
|
-
|
|
59202
|
+
eq57(siteAuditSnapshots.projectId, project.id),
|
|
59203
|
+
eq57(runs.projectId, project.id),
|
|
59204
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59205
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
57983
59206
|
notProbeRun()
|
|
57984
59207
|
];
|
|
57985
|
-
if (request.query.runId) targetFilters.push(
|
|
57986
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs,
|
|
59208
|
+
if (request.query.runId) targetFilters.push(eq57(siteAuditSnapshots.runId, request.query.runId));
|
|
59209
|
+
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
57987
59210
|
if (!latest && request.query.runId) {
|
|
57988
59211
|
throw notFound("Site audit run", request.query.runId);
|
|
57989
59212
|
}
|
|
@@ -57991,9 +59214,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57991
59214
|
return { project: project.name, runId: null, auditedAt: null, total: 0, pages: [] };
|
|
57992
59215
|
}
|
|
57993
59216
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
57994
|
-
const conds = [
|
|
57995
|
-
if (statusFilter) conds.push(
|
|
57996
|
-
const where =
|
|
59217
|
+
const conds = [eq57(siteAuditPages.projectId, project.id), eq57(siteAuditPages.runId, latest.runId)];
|
|
59218
|
+
if (statusFilter) conds.push(eq57(siteAuditPages.status, statusFilter));
|
|
59219
|
+
const where = and45(...conds);
|
|
57997
59220
|
const totalRow = app.db.select({ value: count() }).from(siteAuditPages).where(where).get();
|
|
57998
59221
|
const total = totalRow?.value ?? 0;
|
|
57999
59222
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
@@ -58017,11 +59240,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58017
59240
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
58018
59241
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
58019
59242
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
58020
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
58021
|
-
|
|
58022
|
-
|
|
58023
|
-
|
|
58024
|
-
|
|
59243
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq57(siteAuditSnapshots.runId, runs.id)).where(and45(
|
|
59244
|
+
eq57(siteAuditSnapshots.projectId, project.id),
|
|
59245
|
+
eq57(runs.projectId, project.id),
|
|
59246
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59247
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
58025
59248
|
notProbeRun()
|
|
58026
59249
|
)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
58027
59250
|
return { project: project.name, points: rows.reverse() };
|
|
@@ -58115,10 +59338,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58115
59338
|
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
58116
59339
|
snapshot.rootUrl
|
|
58117
59340
|
)?.nodeKey ?? null;
|
|
58118
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
58119
|
-
|
|
58120
|
-
|
|
58121
|
-
|
|
59341
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and45(
|
|
59342
|
+
eq57(siteCrawlGraphLayouts.projectId, project.id),
|
|
59343
|
+
eq57(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
59344
|
+
eq57(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
58122
59345
|
)).limit(1).get();
|
|
58123
59346
|
if (!persistedLayout) {
|
|
58124
59347
|
return {
|
|
@@ -58169,9 +59392,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58169
59392
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
58170
59393
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
58171
59394
|
const graphScope = [
|
|
58172
|
-
|
|
58173
|
-
|
|
58174
|
-
|
|
59395
|
+
eq57(siteCrawlGraphNodes.projectId, project.id),
|
|
59396
|
+
eq57(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
59397
|
+
eq57(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
58175
59398
|
];
|
|
58176
59399
|
const nodeRows = app.db.select({
|
|
58177
59400
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -58190,17 +59413,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58190
59413
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
58191
59414
|
x: siteCrawlGraphNodes.x,
|
|
58192
59415
|
y: siteCrawlGraphNodes.y
|
|
58193
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
58194
|
-
|
|
58195
|
-
|
|
58196
|
-
|
|
58197
|
-
|
|
58198
|
-
)).where(
|
|
59416
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and45(
|
|
59417
|
+
eq57(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
59418
|
+
eq57(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
59419
|
+
eq57(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
59420
|
+
eq57(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
59421
|
+
)).where(and45(...graphScope, lt8(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
58199
59422
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
58200
59423
|
...row,
|
|
58201
59424
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
58202
59425
|
}));
|
|
58203
|
-
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 :
|
|
59426
|
+
const graphLinkKindFilter = linkKind === SiteHealthLinkKinds.all ? void 0 : eq57(siteCrawlGraphEdges.isTemplate, linkKind === SiteHealthLinkKinds.template);
|
|
58204
59427
|
const edges = app.db.select({
|
|
58205
59428
|
edgeKey: siteCrawlGraphEdges.edgeKey,
|
|
58206
59429
|
sourceNodeKey: siteCrawlGraphEdges.sourceNodeKey,
|
|
@@ -58208,22 +59431,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58208
59431
|
followable: siteCrawlGraphEdges.followable,
|
|
58209
59432
|
occurrences: siteCrawlGraphEdges.occurrences,
|
|
58210
59433
|
isTemplate: siteCrawlGraphEdges.isTemplate
|
|
58211
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
58212
|
-
|
|
58213
|
-
|
|
58214
|
-
|
|
58215
|
-
|
|
59434
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and45(
|
|
59435
|
+
eq57(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
59436
|
+
eq57(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
59437
|
+
eq57(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
59438
|
+
eq57(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
58216
59439
|
lt8(graphSourceNode.sampleRank, maxNodes)
|
|
58217
|
-
)).innerJoin(graphTargetNode,
|
|
58218
|
-
|
|
58219
|
-
|
|
58220
|
-
|
|
58221
|
-
|
|
59440
|
+
)).innerJoin(graphTargetNode, and45(
|
|
59441
|
+
eq57(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
59442
|
+
eq57(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
59443
|
+
eq57(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
59444
|
+
eq57(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
58222
59445
|
lt8(graphTargetNode.sampleRank, maxNodes)
|
|
58223
|
-
)).where(
|
|
58224
|
-
|
|
58225
|
-
|
|
58226
|
-
|
|
59446
|
+
)).where(and45(
|
|
59447
|
+
eq57(siteCrawlGraphEdges.projectId, project.id),
|
|
59448
|
+
eq57(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
59449
|
+
eq57(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
58227
59450
|
lt8(siteCrawlGraphEdges.sampleRank, maxEdges),
|
|
58228
59451
|
graphLinkKindFilter
|
|
58229
59452
|
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
@@ -58317,15 +59540,15 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58317
59540
|
let frontier = [focus.nodeKey];
|
|
58318
59541
|
let queryTruncated = false;
|
|
58319
59542
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
58320
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
58321
|
-
|
|
58322
|
-
|
|
58323
|
-
|
|
58324
|
-
|
|
59543
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and45(
|
|
59544
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
59545
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
59546
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
59547
|
+
eq57(siteCrawlEdges.internal, true),
|
|
58325
59548
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
58326
|
-
|
|
58327
|
-
|
|
58328
|
-
|
|
59549
|
+
or12(
|
|
59550
|
+
inArray20(siteCrawlEdges.sourceNodeKey, frontier),
|
|
59551
|
+
inArray20(siteCrawlEdges.targetNodeKey, frontier)
|
|
58329
59552
|
)
|
|
58330
59553
|
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(maxEdges + maxNodes + 1).all();
|
|
58331
59554
|
if (candidates.length > maxEdges + maxNodes) queryTruncated = true;
|
|
@@ -58358,22 +59581,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58358
59581
|
frontier = [...next].sort();
|
|
58359
59582
|
}
|
|
58360
59583
|
const nodeKeys = [...distances.keys()];
|
|
58361
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
58362
|
-
|
|
58363
|
-
|
|
58364
|
-
|
|
58365
|
-
|
|
59584
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and45(
|
|
59585
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59586
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59587
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59588
|
+
inArray20(siteCrawlPages.nodeKey, nodeKeys)
|
|
58366
59589
|
)).all();
|
|
58367
59590
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
58368
59591
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
58369
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
58370
|
-
|
|
58371
|
-
|
|
58372
|
-
|
|
58373
|
-
|
|
59592
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and45(
|
|
59593
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
59594
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
59595
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
59596
|
+
eq57(siteCrawlEdges.internal, true),
|
|
58374
59597
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
58375
|
-
|
|
58376
|
-
|
|
59598
|
+
inArray20(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
59599
|
+
inArray20(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
58377
59600
|
)).limit(1).get();
|
|
58378
59601
|
if (omittedOutermostEdge && !seenEdgeKeys.has(omittedOutermostEdge.edgeKey)) {
|
|
58379
59602
|
omittedEdgeKeys.add(omittedOutermostEdge.edgeKey);
|
|
@@ -58467,23 +59690,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58467
59690
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
58468
59691
|
let truncated = false;
|
|
58469
59692
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
58470
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
58471
|
-
|
|
58472
|
-
|
|
58473
|
-
|
|
58474
|
-
|
|
58475
|
-
|
|
58476
|
-
|
|
59693
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and45(
|
|
59694
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
59695
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
59696
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
59697
|
+
eq57(siteCrawlEdges.internal, true),
|
|
59698
|
+
eq57(siteCrawlEdges.followable, true),
|
|
59699
|
+
eq57(siteCrawlEdges.relation, "anchor"),
|
|
58477
59700
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
58478
|
-
|
|
59701
|
+
inArray20(siteCrawlEdges.sourceNodeKey, frontier)
|
|
58479
59702
|
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
58480
59703
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
58481
59704
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
58482
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
58483
|
-
|
|
58484
|
-
|
|
58485
|
-
|
|
58486
|
-
|
|
59705
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and45(
|
|
59706
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59707
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59708
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59709
|
+
inArray20(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
58487
59710
|
)).all().map((row) => row.nodeKey));
|
|
58488
59711
|
const next = /* @__PURE__ */ new Set();
|
|
58489
59712
|
for (const edge of candidates.slice(0, SITE_HEALTH_PATH_MAX_VISITED_NODES)) {
|
|
@@ -58525,11 +59748,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58525
59748
|
pathEdges.unshift(step.edge);
|
|
58526
59749
|
pathKeys.unshift(step.nodeKey);
|
|
58527
59750
|
}
|
|
58528
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
58529
|
-
|
|
58530
|
-
|
|
58531
|
-
|
|
58532
|
-
|
|
59751
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and45(
|
|
59752
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
59753
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
59754
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
59755
|
+
inArray20(siteCrawlPages.nodeKey, pathKeys)
|
|
58533
59756
|
)).all();
|
|
58534
59757
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
58535
59758
|
return {
|
|
@@ -58557,21 +59780,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58557
59780
|
throw validationError("change must be all, added, removed, or changed");
|
|
58558
59781
|
}
|
|
58559
59782
|
const completeSnapshotFilters = [
|
|
58560
|
-
|
|
58561
|
-
|
|
58562
|
-
|
|
58563
|
-
|
|
58564
|
-
|
|
59783
|
+
eq57(siteCrawlSnapshots.projectId, project.id),
|
|
59784
|
+
eq57(runs.projectId, project.id),
|
|
59785
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59786
|
+
eq57(runs.status, RunStatuses.completed),
|
|
59787
|
+
eq57(siteCrawlSnapshots.complete, true),
|
|
58565
59788
|
notProbeRun()
|
|
58566
59789
|
];
|
|
58567
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59790
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
58568
59791
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
58569
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59792
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(
|
|
58570
59793
|
...completeSnapshotFilters,
|
|
58571
|
-
|
|
59794
|
+
or12(
|
|
58572
59795
|
lt8(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
58573
|
-
|
|
58574
|
-
|
|
59796
|
+
and45(
|
|
59797
|
+
eq57(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
58575
59798
|
lt8(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
58576
59799
|
)
|
|
58577
59800
|
)
|
|
@@ -58806,18 +60029,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58806
60029
|
const visibleKeys = keyRows.slice(0, limit);
|
|
58807
60030
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
58808
60031
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
58809
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
58810
|
-
|
|
58811
|
-
|
|
58812
|
-
|
|
58813
|
-
|
|
60032
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and45(
|
|
60033
|
+
eq57(siteCrawlPages.projectId, scope.projectId),
|
|
60034
|
+
eq57(siteCrawlPages.runId, scope.runId),
|
|
60035
|
+
eq57(siteCrawlPages.attemptId, scope.attemptId),
|
|
60036
|
+
inArray20(siteCrawlPages.nodeKey, pageKeys)
|
|
58814
60037
|
)).all();
|
|
58815
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
58816
|
-
|
|
58817
|
-
|
|
58818
|
-
|
|
58819
|
-
|
|
58820
|
-
|
|
60038
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and45(
|
|
60039
|
+
eq57(siteCrawlEdges.projectId, scope.projectId),
|
|
60040
|
+
eq57(siteCrawlEdges.runId, scope.runId),
|
|
60041
|
+
eq57(siteCrawlEdges.attemptId, scope.attemptId),
|
|
60042
|
+
eq57(siteCrawlEdges.internal, true),
|
|
60043
|
+
inArray20(siteCrawlEdges.edgeKey, linkKeys)
|
|
58821
60044
|
)).all();
|
|
58822
60045
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
58823
60046
|
const afterPages = new Map(pageRowsFor(afterScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -58940,30 +60163,30 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58940
60163
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, healthStateFilter: null, pages: [] };
|
|
58941
60164
|
}
|
|
58942
60165
|
const filters = [
|
|
58943
|
-
|
|
58944
|
-
|
|
58945
|
-
|
|
60166
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60167
|
+
eq57(siteCrawlPages.runId, snapshot.runId),
|
|
60168
|
+
eq57(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
58946
60169
|
];
|
|
58947
|
-
if (request.query.nodeKey) filters.push(
|
|
60170
|
+
if (request.query.nodeKey) filters.push(eq57(siteCrawlPages.nodeKey, request.query.nodeKey));
|
|
58948
60171
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
58949
|
-
if (inventoryEligible != null) filters.push(
|
|
58950
|
-
if (request.query.fetchState) filters.push(
|
|
58951
|
-
if (request.query.indexabilityState) filters.push(
|
|
58952
|
-
if (request.query.auditState) filters.push(
|
|
60172
|
+
if (inventoryEligible != null) filters.push(eq57(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
60173
|
+
if (request.query.fetchState) filters.push(eq57(siteCrawlPages.fetchState, request.query.fetchState));
|
|
60174
|
+
if (request.query.indexabilityState) filters.push(eq57(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
60175
|
+
if (request.query.auditState) filters.push(eq57(siteCrawlPages.auditState, request.query.auditState));
|
|
58953
60176
|
const healthState = parseSiteHealthState(request.query.healthState);
|
|
58954
60177
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
58955
60178
|
const offset = decodeCursor(request.query.cursor);
|
|
58956
60179
|
const orderBy = request.query.sort === "score-desc" ? [desc24(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "score-asc" ? [asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "path" ? [asc10(siteCrawlPages.path), asc10(siteCrawlPages.nodeKey)] : [asc10(siteCrawlPages.url), asc10(siteCrawlPages.nodeKey)];
|
|
58957
60180
|
let healthStateFilter = null;
|
|
58958
60181
|
if (healthState) {
|
|
58959
|
-
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(
|
|
58960
|
-
|
|
58961
|
-
|
|
58962
|
-
|
|
58963
|
-
|
|
60182
|
+
const legacyRow = app.db.select({ id: siteCrawlPages.id }).from(siteCrawlPages).where(and45(
|
|
60183
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60184
|
+
eq57(siteCrawlPages.runId, snapshot.runId),
|
|
60185
|
+
eq57(siteCrawlPages.attemptId, snapshot.attemptId),
|
|
60186
|
+
isNull6(siteCrawlPages.healthState)
|
|
58964
60187
|
)).limit(1).get();
|
|
58965
60188
|
healthStateFilter = legacyRow ? "unavailable-legacy-scan" : "applied";
|
|
58966
|
-
if (healthStateFilter === "applied") filters.push(
|
|
60189
|
+
if (healthStateFilter === "applied") filters.push(eq57(siteCrawlPages.healthState, healthState));
|
|
58967
60190
|
}
|
|
58968
60191
|
if (healthStateFilter === "unavailable-legacy-scan") {
|
|
58969
60192
|
return {
|
|
@@ -58976,7 +60199,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
58976
60199
|
pages: []
|
|
58977
60200
|
};
|
|
58978
60201
|
}
|
|
58979
|
-
const where =
|
|
60202
|
+
const where = and45(...filters);
|
|
58980
60203
|
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
58981
60204
|
const rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
58982
60205
|
const nextOffset = offset + rows.length;
|
|
@@ -59007,10 +60230,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59007
60230
|
url: siteCrawlPages.url,
|
|
59008
60231
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
59009
60232
|
fetchState: siteCrawlPages.fetchState
|
|
59010
|
-
}).from(siteCrawlPages).where(
|
|
59011
|
-
|
|
59012
|
-
|
|
59013
|
-
|
|
60233
|
+
}).from(siteCrawlPages).where(and45(
|
|
60234
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60235
|
+
eq57(siteCrawlPages.runId, snapshot.runId),
|
|
60236
|
+
eq57(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
59014
60237
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
59015
60238
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
59016
60239
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -59089,17 +60312,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59089
60312
|
};
|
|
59090
60313
|
}
|
|
59091
60314
|
const filters = [
|
|
59092
|
-
|
|
59093
|
-
|
|
59094
|
-
|
|
59095
|
-
|
|
60315
|
+
eq57(siteCrawlEdges.projectId, project.id),
|
|
60316
|
+
eq57(siteCrawlEdges.runId, snapshot.runId),
|
|
60317
|
+
eq57(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
60318
|
+
eq57(siteCrawlEdges.internal, true),
|
|
59096
60319
|
linkKindFilter(linkKind)
|
|
59097
60320
|
];
|
|
59098
|
-
if (request.query.sourceUrl) filters.push(
|
|
59099
|
-
if (request.query.targetUrl) filters.push(
|
|
60321
|
+
if (request.query.sourceUrl) filters.push(eq57(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
60322
|
+
if (request.query.targetUrl) filters.push(eq57(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
59100
60323
|
const followable = parseBoolean2(request.query.followable);
|
|
59101
|
-
if (followable != null) filters.push(
|
|
59102
|
-
const where =
|
|
60324
|
+
if (followable != null) filters.push(eq57(siteCrawlEdges.followable, followable));
|
|
60325
|
+
const where = and45(...filters);
|
|
59103
60326
|
const total = app.db.select({ value: count() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
59104
60327
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
59105
60328
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -59157,17 +60380,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59157
60380
|
};
|
|
59158
60381
|
}
|
|
59159
60382
|
const scope = [
|
|
59160
|
-
|
|
59161
|
-
|
|
59162
|
-
|
|
59163
|
-
|
|
60383
|
+
eq57(siteCrawlEdges.projectId, project.id),
|
|
60384
|
+
eq57(siteCrawlEdges.runId, snapshot.runId),
|
|
60385
|
+
eq57(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
60386
|
+
eq57(siteCrawlEdges.internal, true),
|
|
59164
60387
|
linkKindFilter(linkKind)
|
|
59165
60388
|
];
|
|
59166
|
-
const inboundMatch = request.query.nodeKey && request.query.url ?
|
|
59167
|
-
const outboundMatch = request.query.nodeKey && request.query.url ?
|
|
60389
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or12(eq57(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq57(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq57(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq57(siteCrawlEdges.targetUrl, request.query.url);
|
|
60390
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or12(eq57(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq57(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq57(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq57(siteCrawlEdges.sourceUrl, request.query.url);
|
|
59168
60391
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
59169
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
59170
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
60392
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and45(...scope, inboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
60393
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and45(...scope, outboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
59171
60394
|
return {
|
|
59172
60395
|
project: project.name,
|
|
59173
60396
|
hasCrawlData: true,
|
|
@@ -59200,11 +60423,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59200
60423
|
if (!snapshot.attemptId) {
|
|
59201
60424
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, total: 0, nextCursor: null, deadLinks: [] };
|
|
59202
60425
|
}
|
|
59203
|
-
const where =
|
|
59204
|
-
|
|
59205
|
-
|
|
59206
|
-
|
|
59207
|
-
|
|
60426
|
+
const where = and45(
|
|
60427
|
+
eq57(siteCrawlFindings.projectId, project.id),
|
|
60428
|
+
eq57(siteCrawlFindings.runId, snapshot.runId),
|
|
60429
|
+
eq57(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
60430
|
+
eq57(siteCrawlFindings.findingType, "dead-link")
|
|
59208
60431
|
);
|
|
59209
60432
|
const total = app.db.select({ value: count() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
59210
60433
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -59242,19 +60465,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59242
60465
|
createdAt: runs.createdAt,
|
|
59243
60466
|
startedAt: runs.startedAt,
|
|
59244
60467
|
finishedAt: runs.finishedAt
|
|
59245
|
-
}).from(runs).where(
|
|
59246
|
-
|
|
59247
|
-
|
|
60468
|
+
}).from(runs).where(and45(
|
|
60469
|
+
eq57(runs.projectId, project.id),
|
|
60470
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59248
60471
|
notProbeRun()
|
|
59249
60472
|
)).orderBy(desc24(runs.createdAt), desc24(runs.id)).limit(limit).all();
|
|
59250
60473
|
const runIds = rows.map((row) => row.runId);
|
|
59251
|
-
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
59252
|
-
|
|
59253
|
-
|
|
59254
|
-
|
|
59255
|
-
|
|
60474
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq57(siteCrawlSnapshots.runId, runs.id)).where(and45(
|
|
60475
|
+
eq57(siteCrawlSnapshots.projectId, project.id),
|
|
60476
|
+
eq57(runs.projectId, project.id),
|
|
60477
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
60478
|
+
inArray20(runs.status, SURFACEABLE_STATUSES),
|
|
59256
60479
|
notProbeRun(),
|
|
59257
|
-
|
|
60480
|
+
inArray20(siteCrawlSnapshots.runId, runIds)
|
|
59258
60481
|
)).all().map((row) => row.runId));
|
|
59259
60482
|
return {
|
|
59260
60483
|
project: project.name,
|
|
@@ -59267,21 +60490,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59267
60490
|
});
|
|
59268
60491
|
app.get("/projects/:name/technical-aeo/runs/:runId/progress", async (request) => {
|
|
59269
60492
|
const project = resolveProject(app.db, request.params.name);
|
|
59270
|
-
const run = app.db.select().from(runs).where(
|
|
59271
|
-
|
|
59272
|
-
|
|
59273
|
-
|
|
60493
|
+
const run = app.db.select().from(runs).where(and45(
|
|
60494
|
+
eq57(runs.id, request.params.runId),
|
|
60495
|
+
eq57(runs.projectId, project.id),
|
|
60496
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59274
60497
|
notProbeRun()
|
|
59275
60498
|
)).get();
|
|
59276
60499
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
59277
|
-
const attempt = app.db.select().from(siteCrawlAttempts).where(
|
|
59278
|
-
|
|
59279
|
-
|
|
60500
|
+
const attempt = app.db.select().from(siteCrawlAttempts).where(and45(
|
|
60501
|
+
eq57(siteCrawlAttempts.projectId, project.id),
|
|
60502
|
+
eq57(siteCrawlAttempts.runId, run.id)
|
|
59280
60503
|
)).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
59281
|
-
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(
|
|
59282
|
-
|
|
59283
|
-
|
|
59284
|
-
|
|
60504
|
+
const persistedLayout = attempt ? app.db.select().from(siteCrawlGraphLayouts).where(and45(
|
|
60505
|
+
eq57(siteCrawlGraphLayouts.projectId, project.id),
|
|
60506
|
+
eq57(siteCrawlGraphLayouts.runId, run.id),
|
|
60507
|
+
eq57(siteCrawlGraphLayouts.attemptId, attempt.id)
|
|
59285
60508
|
)).get() : void 0;
|
|
59286
60509
|
const layoutState = persistedLayout?.state === "ready" || persistedLayout?.state === "unavailable" ? persistedLayout.state : run.status === RunStatuses.completed || run.status === RunStatuses.partial ? "unavailable" : "pending";
|
|
59287
60510
|
return {
|
|
@@ -59312,18 +60535,18 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59312
60535
|
};
|
|
59313
60536
|
});
|
|
59314
60537
|
app.get("/projects/:name/technical-aeo/runs/:runId/page-health-preview", async (request) => app.db.transaction((tx) => {
|
|
59315
|
-
const project = tx.select().from(projects).where(
|
|
60538
|
+
const project = tx.select().from(projects).where(eq57(projects.name, request.params.name)).get();
|
|
59316
60539
|
if (!project) throw notFound("Project", request.params.name);
|
|
59317
|
-
const run = tx.select().from(runs).where(
|
|
59318
|
-
|
|
59319
|
-
|
|
59320
|
-
|
|
60540
|
+
const run = tx.select().from(runs).where(and45(
|
|
60541
|
+
eq57(runs.id, request.params.runId),
|
|
60542
|
+
eq57(runs.projectId, project.id),
|
|
60543
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
59321
60544
|
notProbeRun()
|
|
59322
60545
|
)).get();
|
|
59323
60546
|
if (!run) throw notFound("Site audit run", request.params.runId);
|
|
59324
|
-
const attempt = tx.select().from(siteCrawlAttempts).where(
|
|
59325
|
-
|
|
59326
|
-
|
|
60547
|
+
const attempt = tx.select().from(siteCrawlAttempts).where(and45(
|
|
60548
|
+
eq57(siteCrawlAttempts.projectId, project.id),
|
|
60549
|
+
eq57(siteCrawlAttempts.runId, run.id)
|
|
59327
60550
|
)).orderBy(desc24(siteCrawlAttempts.attemptNumber), desc24(siteCrawlAttempts.updatedAt)).limit(1).get();
|
|
59328
60551
|
const state = run.status === RunStatuses.queued ? "waiting" : run.status === RunStatuses.running ? "collecting" : "terminal";
|
|
59329
60552
|
const base = {
|
|
@@ -59337,11 +60560,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59337
60560
|
if (!attempt) {
|
|
59338
60561
|
return { ...base, pagesAudited: 0, examples: [] };
|
|
59339
60562
|
}
|
|
59340
|
-
const auditedWhere =
|
|
59341
|
-
|
|
59342
|
-
|
|
59343
|
-
|
|
59344
|
-
|
|
60563
|
+
const auditedWhere = and45(
|
|
60564
|
+
eq57(siteCrawlPages.projectId, project.id),
|
|
60565
|
+
eq57(siteCrawlPages.runId, run.id),
|
|
60566
|
+
eq57(siteCrawlPages.attemptId, attempt.id),
|
|
60567
|
+
eq57(siteCrawlPages.auditState, "success")
|
|
59345
60568
|
);
|
|
59346
60569
|
const pagesAudited = tx.select({ value: count() }).from(siteCrawlPages).where(auditedWhere).get()?.value ?? 0;
|
|
59347
60570
|
if (state !== "collecting") {
|
|
@@ -59352,7 +60575,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59352
60575
|
url: siteCrawlPages.url,
|
|
59353
60576
|
auditScore: siteCrawlPages.auditScore,
|
|
59354
60577
|
auditFields: siteCrawlPages.auditFields
|
|
59355
|
-
}).from(siteCrawlPages).where(
|
|
60578
|
+
}).from(siteCrawlPages).where(and45(
|
|
59356
60579
|
auditedWhere,
|
|
59357
60580
|
isNotNull3(siteCrawlPages.auditScore),
|
|
59358
60581
|
lt8(siteCrawlPages.auditScore, 70)
|
|
@@ -59388,13 +60611,13 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59388
60611
|
status: runs.status,
|
|
59389
60612
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
59390
60613
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
59391
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
59392
|
-
|
|
59393
|
-
|
|
59394
|
-
)).where(
|
|
59395
|
-
|
|
59396
|
-
|
|
59397
|
-
|
|
60614
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and45(
|
|
60615
|
+
eq57(siteCrawlRunRequests.projectId, runs.projectId),
|
|
60616
|
+
eq57(siteCrawlRunRequests.runId, runs.id)
|
|
60617
|
+
)).where(and45(
|
|
60618
|
+
eq57(runs.projectId, project.id),
|
|
60619
|
+
eq57(runs.kind, RunKinds["site-audit"]),
|
|
60620
|
+
inArray20(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
59398
60621
|
)).get();
|
|
59399
60622
|
if (existing) {
|
|
59400
60623
|
if (existing.identityKey === identityKey) {
|
|
@@ -59445,7 +60668,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
59445
60668
|
|
|
59446
60669
|
// ../api-routes/src/research.ts
|
|
59447
60670
|
import crypto45 from "crypto";
|
|
59448
|
-
import { and as
|
|
60671
|
+
import { and as and46, desc as desc25, eq as eq58 } from "drizzle-orm";
|
|
59449
60672
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
59450
60673
|
async function researchRoutes(app, opts) {
|
|
59451
60674
|
app.post("/projects/:name/research/runs", async (request, reply) => {
|
|
@@ -59475,7 +60698,7 @@ async function researchRoutes(app, opts) {
|
|
|
59475
60698
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
59476
60699
|
const decision = app.db.transaction((tx) => {
|
|
59477
60700
|
if (input.idempotencyKey) {
|
|
59478
|
-
const existing = tx.select().from(researchRuns).where(
|
|
60701
|
+
const existing = tx.select().from(researchRuns).where(and46(eq58(researchRuns.projectId, project.id), eq58(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
59479
60702
|
if (existing) {
|
|
59480
60703
|
if (existing.requestHash !== requestHash2) throw alreadyExists("Research idempotency key", input.idempotencyKey);
|
|
59481
60704
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
@@ -59496,7 +60719,7 @@ async function researchRoutes(app, opts) {
|
|
|
59496
60719
|
const project = resolveProject(app.db, request.params.name);
|
|
59497
60720
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
59498
60721
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
59499
|
-
const runs2 = app.db.select().from(researchRuns).where(
|
|
60722
|
+
const runs2 = app.db.select().from(researchRuns).where(eq58(researchRuns.projectId, project.id)).orderBy(desc25(researchRuns.createdAt)).limit(limit).all().map(serializeRun);
|
|
59500
60723
|
return { runs: runs2 };
|
|
59501
60724
|
});
|
|
59502
60725
|
app.get("/projects/:name/research/runs/:runId", async (request) => {
|
|
@@ -59505,9 +60728,9 @@ async function researchRoutes(app, opts) {
|
|
|
59505
60728
|
});
|
|
59506
60729
|
}
|
|
59507
60730
|
function getDetail(app, projectId, id) {
|
|
59508
|
-
const row = app.db.select().from(researchRuns).where(
|
|
60731
|
+
const row = app.db.select().from(researchRuns).where(and46(eq58(researchRuns.id, id), eq58(researchRuns.projectId, projectId))).get();
|
|
59509
60732
|
if (!row) throw notFound("Research run", id);
|
|
59510
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
60733
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq58(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
59511
60734
|
return { ...serializeRun(row), queries: queries2 };
|
|
59512
60735
|
}
|
|
59513
60736
|
function serializeRun(row) {
|
|
@@ -59718,6 +60941,9 @@ async function apiRoutes(app, opts) {
|
|
|
59718
60941
|
pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
|
|
59719
60942
|
vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,
|
|
59720
60943
|
cloudflareTrafficCredentialStore: opts.cloudflareTrafficCredentialStore,
|
|
60944
|
+
pullCloudflareQueueMessages: opts.pullCloudflareQueueMessages,
|
|
60945
|
+
ackCloudflareQueueMessages: opts.ackCloudflareQueueMessages,
|
|
60946
|
+
cloudflareQueueMaxBatches: opts.cloudflareQueueMaxBatches,
|
|
59721
60947
|
cloudflareTrafficIngestUrl: opts.cloudflareTrafficIngestUrl,
|
|
59722
60948
|
cloudflareIngestRateLimitMax: opts.cloudflareIngestRateLimitMax,
|
|
59723
60949
|
cloudflareIngestIpRateLimitMax: opts.cloudflareIngestIpRateLimitMax,
|
|
@@ -59907,22 +61133,31 @@ function buildTrafficSourceValidators(opts) {
|
|
|
59907
61133
|
validators[TrafficSourceTypes.cloudflare] = {
|
|
59908
61134
|
validateCredentials: (source) => {
|
|
59909
61135
|
const sourceMode = source.configJson.deliveryMode;
|
|
59910
|
-
|
|
59911
|
-
const record = store.
|
|
61136
|
+
const deliveryMode = sourceMode === "queue-pull" ? "queue-pull" : "direct-push";
|
|
61137
|
+
const record = store.getConnectionBySourceId(source.id);
|
|
59912
61138
|
if (!record) {
|
|
61139
|
+
const projectRecord = store.getConnection(source.projectName);
|
|
61140
|
+
if (projectRecord && projectRecord.sourceId !== source.id) {
|
|
61141
|
+
return {
|
|
61142
|
+
status: CheckStatuses.fail,
|
|
61143
|
+
code: "traffic.credentials.source-mismatch",
|
|
61144
|
+
summary: `The stored Cloudflare credential belongs to a different source than "${source.displayName}".`,
|
|
61145
|
+
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
61146
|
+
};
|
|
61147
|
+
}
|
|
59913
61148
|
return {
|
|
59914
61149
|
status: CheckStatuses.fail,
|
|
59915
61150
|
code: "traffic.credentials.missing",
|
|
59916
|
-
summary: `No Cloudflare
|
|
61151
|
+
summary: `No Cloudflare ${deliveryMode} credential is stored for source "${source.displayName}".`,
|
|
59917
61152
|
remediation: "Re-run `canonry traffic connect cloudflare <project> --zone-id <id>` from the credential-owning host."
|
|
59918
61153
|
};
|
|
59919
61154
|
}
|
|
59920
61155
|
const recordMode = record.deliveryMode;
|
|
59921
|
-
if (recordMode !==
|
|
61156
|
+
if (recordMode !== deliveryMode) {
|
|
59922
61157
|
return {
|
|
59923
61158
|
status: CheckStatuses.fail,
|
|
59924
61159
|
code: "traffic.credentials.mode-mismatch",
|
|
59925
|
-
summary: `The stored Cloudflare credential mode does not match
|
|
61160
|
+
summary: `The stored Cloudflare credential mode does not match ${deliveryMode} source "${source.displayName}".`,
|
|
59926
61161
|
remediation: "Reconnect the Cloudflare source from the credential-owning host."
|
|
59927
61162
|
};
|
|
59928
61163
|
}
|
|
@@ -59934,9 +61169,23 @@ function buildTrafficSourceValidators(opts) {
|
|
|
59934
61169
|
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
59935
61170
|
};
|
|
59936
61171
|
}
|
|
59937
|
-
|
|
59938
|
-
|
|
59939
|
-
|
|
61172
|
+
if (record.deliveryMode === "queue-pull") {
|
|
61173
|
+
const queueConfig = source.configJson;
|
|
61174
|
+
if (typeof record.apiToken !== "string" || record.apiToken.length === 0 || typeof record.accountId !== "string" || record.accountId.length === 0 || typeof record.queueId !== "string" || record.queueId.length === 0 || typeof record.queueName !== "string" || record.queueName.length === 0 || !Number.isInteger(record.retentionSeconds) || record.retentionSeconds < 60 || record.retentionSeconds > 1209600 || queueConfig.accountId !== record.accountId || queueConfig.queueId !== record.queueId || queueConfig.queueName !== record.queueName || queueConfig.retentionSeconds !== record.retentionSeconds) {
|
|
61175
|
+
return {
|
|
61176
|
+
status: CheckStatuses.fail,
|
|
61177
|
+
code: "traffic.credentials.queue-mismatch",
|
|
61178
|
+
summary: `The stored Cloudflare Queue credential does not match source "${source.displayName}".`,
|
|
61179
|
+
remediation: "Reconnect the Cloudflare Queue source from the credential-owning host."
|
|
61180
|
+
};
|
|
61181
|
+
}
|
|
61182
|
+
return {
|
|
61183
|
+
status: CheckStatuses.ok,
|
|
61184
|
+
code: "traffic.credentials.resolved",
|
|
61185
|
+
summary: `Cloudflare Queue credentials match source "${source.displayName}".`
|
|
61186
|
+
};
|
|
61187
|
+
}
|
|
61188
|
+
if (typeof record.bearerToken !== "string" || record.bearerToken.length === 0 || typeof record.hmacSecret !== "string" || record.hmacSecret.length === 0 || !source.ingestTokenHash || hashCloudflareBearerToken(record.bearerToken) !== source.ingestTokenHash) {
|
|
59940
61189
|
return {
|
|
59941
61190
|
status: CheckStatuses.fail,
|
|
59942
61191
|
code: "traffic.credentials.bearer-mismatch",
|
|
@@ -59950,7 +61199,15 @@ function buildTrafficSourceValidators(opts) {
|
|
|
59950
61199
|
summary: `Cloudflare direct-push credentials match source "${source.displayName}".`
|
|
59951
61200
|
};
|
|
59952
61201
|
},
|
|
59953
|
-
validateScopes: () =>
|
|
61202
|
+
validateScopes: (source) => {
|
|
61203
|
+
if (source.configJson.deliveryMode !== "queue-pull") return null;
|
|
61204
|
+
return {
|
|
61205
|
+
status: CheckStatuses.skipped,
|
|
61206
|
+
code: "traffic.scopes.queue-pull-static",
|
|
61207
|
+
summary: `Cloudflare Queue token scopes for "${source.displayName}" are not inspected by Doctor.`,
|
|
61208
|
+
remediation: `Verify the token has Account Queues Edit permission and run \`wrangler queues consumer http add ${source.configJson.queueName}\` for the configured Queue, then re-run the Queue smoke test.`
|
|
61209
|
+
};
|
|
61210
|
+
}
|
|
59954
61211
|
};
|
|
59955
61212
|
}
|
|
59956
61213
|
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
@@ -60165,9 +61422,9 @@ var IntelligenceService = class {
|
|
|
60165
61422
|
*/
|
|
60166
61423
|
analyzeAndPersist(runId, projectId) {
|
|
60167
61424
|
const recentRuns = this.db.select().from(runs).where(
|
|
60168
|
-
|
|
60169
|
-
|
|
60170
|
-
|
|
61425
|
+
and47(
|
|
61426
|
+
eq59(runs.projectId, projectId),
|
|
61427
|
+
or13(eq59(runs.status, "completed"), eq59(runs.status, "partial")),
|
|
60171
61428
|
// Defensive: RunCoordinator already skips probes before this is
|
|
60172
61429
|
// called, but if a future call site invokes analyzeAndPersist
|
|
60173
61430
|
// directly for a probe, probes still must not pollute the
|
|
@@ -60249,7 +61506,7 @@ var IntelligenceService = class {
|
|
|
60249
61506
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
60250
61507
|
*/
|
|
60251
61508
|
analyzeAndPersistGbp(runId, projectId) {
|
|
60252
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
61509
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq59(runs.id, runId)).get();
|
|
60253
61510
|
if (!runRow) {
|
|
60254
61511
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
60255
61512
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -60257,11 +61514,11 @@ var IntelligenceService = class {
|
|
|
60257
61514
|
}
|
|
60258
61515
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
60259
61516
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
60260
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
60261
|
-
|
|
60262
|
-
|
|
61517
|
+
const selected = this.db.select().from(gbpLocations).where(and47(
|
|
61518
|
+
eq59(gbpLocations.projectId, projectId),
|
|
61519
|
+
eq59(gbpLocations.selected, true),
|
|
60263
61520
|
gte14(gbpLocations.syncedAt, windowStart),
|
|
60264
|
-
|
|
61521
|
+
lte12(gbpLocations.syncedAt, windowEnd)
|
|
60265
61522
|
)).all();
|
|
60266
61523
|
if (selected.length === 0) {
|
|
60267
61524
|
log.info("gbp-intelligence.skip", { runId, reason: "no locations synced during run" });
|
|
@@ -60294,12 +61551,12 @@ var IntelligenceService = class {
|
|
|
60294
61551
|
}
|
|
60295
61552
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
60296
61553
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
60297
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
60298
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
60299
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
60300
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
61554
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and47(eq59(gbpDailyMetrics.projectId, projectId), eq59(gbpDailyMetrics.locationName, locationName))).all();
|
|
61555
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and47(eq59(gbpPlaceActions.projectId, projectId), eq59(gbpPlaceActions.locationName, locationName))).all();
|
|
61556
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and47(eq59(gbpLodgingSnapshots.projectId, projectId), eq59(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc26(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
61557
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and47(eq59(gbpLocations.projectId, projectId), eq59(gbpLocations.locationName, locationName))).get();
|
|
60301
61558
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
60302
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
61559
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and47(eq59(gbpPlaceDetails.projectId, projectId), eq59(gbpPlaceDetails.locationName, locationName))).orderBy(desc26(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
60303
61560
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
60304
61561
|
const summary = buildGbpSummary({
|
|
60305
61562
|
locationName,
|
|
@@ -60335,7 +61592,7 @@ var IntelligenceService = class {
|
|
|
60335
61592
|
/** Build the month-over-month keyword series for a location from the
|
|
60336
61593
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
60337
61594
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
60338
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
61595
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and47(eq59(gbpKeywordMonthly.projectId, projectId), eq59(gbpKeywordMonthly.locationName, locationName))).all();
|
|
60339
61596
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
60340
61597
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
60341
61598
|
const recentMonth = months[0] ?? null;
|
|
@@ -60366,7 +61623,7 @@ var IntelligenceService = class {
|
|
|
60366
61623
|
*/
|
|
60367
61624
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
60368
61625
|
const covered = new Set(coveredLocationNames);
|
|
60369
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
61626
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and47(eq59(insights.projectId, projectId), eq59(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
60370
61627
|
const staleIds = [];
|
|
60371
61628
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
60372
61629
|
for (const row of existing) {
|
|
@@ -60377,7 +61634,7 @@ var IntelligenceService = class {
|
|
|
60377
61634
|
}
|
|
60378
61635
|
this.db.transaction((tx) => {
|
|
60379
61636
|
for (const id of staleIds) {
|
|
60380
|
-
tx.delete(insights).where(
|
|
61637
|
+
tx.delete(insights).where(eq59(insights.id, id)).run();
|
|
60381
61638
|
}
|
|
60382
61639
|
for (const insight of gbpInsights) {
|
|
60383
61640
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -60455,7 +61712,7 @@ var IntelligenceService = class {
|
|
|
60455
61712
|
* create per run + aggregate). DB is left untouched.
|
|
60456
61713
|
*/
|
|
60457
61714
|
backfill(projectName, opts, onProgress) {
|
|
60458
|
-
const project = this.db.select().from(projects).where(
|
|
61715
|
+
const project = this.db.select().from(projects).where(eq59(projects.name, projectName)).get();
|
|
60459
61716
|
if (!project) {
|
|
60460
61717
|
throw new Error(`Project "${projectName}" not found`);
|
|
60461
61718
|
}
|
|
@@ -60468,9 +61725,9 @@ var IntelligenceService = class {
|
|
|
60468
61725
|
sinceTimestamp = parsed;
|
|
60469
61726
|
}
|
|
60470
61727
|
const allRuns = this.db.select().from(runs).where(
|
|
60471
|
-
|
|
60472
|
-
|
|
60473
|
-
|
|
61728
|
+
and47(
|
|
61729
|
+
eq59(runs.projectId, project.id),
|
|
61730
|
+
or13(eq59(runs.status, "completed"), eq59(runs.status, "partial")),
|
|
60474
61731
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
60475
61732
|
ne7(runs.trigger, RunTriggers.probe)
|
|
60476
61733
|
)
|
|
@@ -60503,7 +61760,7 @@ var IntelligenceService = class {
|
|
|
60503
61760
|
let wouldDeleteTotal = 0;
|
|
60504
61761
|
const existingByRunId = /* @__PURE__ */ new Map();
|
|
60505
61762
|
if (isDryRun && targetRuns.length > 0) {
|
|
60506
|
-
const rows = this.db.select({ runId: insights.runId }).from(insights).where(
|
|
61763
|
+
const rows = this.db.select({ runId: insights.runId }).from(insights).where(inArray21(insights.runId, targetRuns.map((r) => r.id))).all();
|
|
60507
61764
|
for (const r of rows) {
|
|
60508
61765
|
if (r.runId == null) continue;
|
|
60509
61766
|
existingByRunId.set(r.runId, (existingByRunId.get(r.runId) ?? 0) + 1);
|
|
@@ -60549,7 +61806,7 @@ var IntelligenceService = class {
|
|
|
60549
61806
|
return { processed, skipped, totalInsights };
|
|
60550
61807
|
}
|
|
60551
61808
|
loadTrackedCompetitors(projectId) {
|
|
60552
|
-
return this.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
61809
|
+
return this.db.select({ domain: competitors.domain }).from(competitors).where(eq59(competitors.projectId, projectId)).all().map((r) => r.domain);
|
|
60553
61810
|
}
|
|
60554
61811
|
/**
|
|
60555
61812
|
* Wipe transition signals from an analysis result while keeping health.
|
|
@@ -60570,15 +61827,15 @@ var IntelligenceService = class {
|
|
|
60570
61827
|
}
|
|
60571
61828
|
persistResult(result, runId, projectId) {
|
|
60572
61829
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
60573
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
61830
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq59(insights.runId, runId)).all();
|
|
60574
61831
|
for (const row of existingInsights) {
|
|
60575
61832
|
if (row.dismissed) {
|
|
60576
61833
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
60577
61834
|
}
|
|
60578
61835
|
}
|
|
60579
61836
|
this.db.transaction((tx) => {
|
|
60580
|
-
tx.delete(insights).where(
|
|
60581
|
-
tx.delete(healthSnapshots).where(
|
|
61837
|
+
tx.delete(insights).where(eq59(insights.runId, runId)).run();
|
|
61838
|
+
tx.delete(healthSnapshots).where(eq59(healthSnapshots.runId, runId)).run();
|
|
60582
61839
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
60583
61840
|
for (const insight of result.insights) {
|
|
60584
61841
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -60631,24 +61888,24 @@ var IntelligenceService = class {
|
|
|
60631
61888
|
applySeverityTiering(rawInsights, excludeRunId, projectId) {
|
|
60632
61889
|
const regressions = rawInsights.filter((i) => i.type === "regression");
|
|
60633
61890
|
if (regressions.length === 0) return rawInsights;
|
|
60634
|
-
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(
|
|
61891
|
+
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(eq59(gscSearchData.projectId, projectId)).all();
|
|
60635
61892
|
const gscConnected = gscRows.length > 0;
|
|
60636
61893
|
const gscImpressionsByQuery = /* @__PURE__ */ new Map();
|
|
60637
61894
|
for (const row of gscRows) {
|
|
60638
61895
|
const key = row.query.toLowerCase();
|
|
60639
61896
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
60640
61897
|
}
|
|
60641
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
61898
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq59(projects.id, projectId)).get();
|
|
60642
61899
|
const locationCount = Math.max(
|
|
60643
61900
|
1,
|
|
60644
61901
|
(projectRow?.locations ?? []).length
|
|
60645
61902
|
);
|
|
60646
61903
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
60647
61904
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
60648
|
-
|
|
60649
|
-
|
|
60650
|
-
|
|
60651
|
-
|
|
61905
|
+
and47(
|
|
61906
|
+
eq59(runs.projectId, projectId),
|
|
61907
|
+
eq59(runs.kind, RunKinds["answer-visibility"]),
|
|
61908
|
+
or13(eq59(runs.status, "completed"), eq59(runs.status, "partial")),
|
|
60652
61909
|
// Defensive — see top of file.
|
|
60653
61910
|
ne7(runs.trigger, RunTriggers.probe)
|
|
60654
61911
|
)
|
|
@@ -60668,7 +61925,7 @@ var IntelligenceService = class {
|
|
|
60668
61925
|
const haveHistory = recentRunIds.length > 0;
|
|
60669
61926
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
60670
61927
|
if (haveHistory) {
|
|
60671
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
61928
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and47(eq59(insights.type, "regression"), inArray21(insights.runId, recentRunIds))).all();
|
|
60672
61929
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
60673
61930
|
for (const row of priorRows) {
|
|
60674
61931
|
if (!row.runId) continue;
|
|
@@ -60697,7 +61954,7 @@ var IntelligenceService = class {
|
|
|
60697
61954
|
});
|
|
60698
61955
|
}
|
|
60699
61956
|
buildRunData(runId, projectId, completedAt, location = null) {
|
|
60700
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
61957
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq59(projects.id, projectId)).get();
|
|
60701
61958
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
60702
61959
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
60703
61960
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -60714,7 +61971,7 @@ var IntelligenceService = class {
|
|
|
60714
61971
|
citedDomains: querySnapshots.citedDomains,
|
|
60715
61972
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
60716
61973
|
snapshotLocation: querySnapshots.location
|
|
60717
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
61974
|
+
}).from(querySnapshots).leftJoin(queries, eq59(querySnapshots.queryId, queries.id)).where(eq59(querySnapshots.runId, runId)).all();
|
|
60718
61975
|
const snapshots = [];
|
|
60719
61976
|
let orphanCount = 0;
|
|
60720
61977
|
for (const r of rows) {
|