@canonry/canonry 4.156.0 → 4.157.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/assets/agent-workspace/skills/canonry/SKILL.md +8 -3
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +454 -21
- package/assets/assets/{AuditHistoryPanel-gmJMVlXX.js → AuditHistoryPanel-D12lHs6n.js} +1 -1
- package/assets/assets/{BacklinksPage-Bv2sTCyN.js → BacklinksPage-D2ScFYru.js} +1 -1
- package/assets/assets/{HistoryPage-13q7GkiI.js → HistoryPage-jZ_I4rwX.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BgBqXT1p.js → MeasurementPropertyPage-BIdQjXJT.js} +1 -1
- package/assets/assets/{ProjectPage-EsGwKhot.js → ProjectPage-CLvLOOtj.js} +3 -3
- package/assets/assets/{RunRow-bPzapIGJ.js → RunRow-Q2X0NR74.js} +1 -1
- package/assets/assets/{RunsPage-Dqfz18lH.js → RunsPage-Cw7YVsbJ.js} +1 -1
- package/assets/assets/{SettingsPage-_wMecn-V.js → SettingsPage-bFl2EXOD.js} +1 -1
- package/assets/assets/{TrafficPage-RYBdsOLB.js → TrafficPage-DJjhk7Jw.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-DpeMMyKU.js → TrafficSourceDetailPage-CGKwIu9A.js} +1 -1
- package/assets/assets/{extract-error-message-DUarAlyD.js → extract-error-message-BHm98tXH.js} +1 -1
- package/assets/assets/{index-DP-fT2uI.js → index-VO6nKq5b.js} +25 -25
- package/assets/assets/{react-sigma_core.esm.min-DnypJl5w.js → react-sigma_core.esm.min-EzswMBkr.js} +1 -1
- package/assets/index.html +1 -1
- package/dist/{chunk-ZWESOFA5.js → chunk-AEIKAWOW.js} +88 -8
- package/dist/{chunk-CPASKGUE.js → chunk-KMTRO56O.js} +35 -1
- package/dist/{chunk-ABNN2ZVR.js → chunk-MEBL4EJO.js} +1820 -431
- package/dist/{chunk-AMPT2EP7.js → chunk-TRRVBTU5.js} +71 -0
- package/dist/cli.js +667 -142
- package/dist/index.d.ts +32 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-WHC7B32V.js → intelligence-service-YUUPSJRC.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +10 -9
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
CheckScopes,
|
|
36
36
|
CheckStatuses,
|
|
37
37
|
CitationStates,
|
|
38
|
+
CloudflareTrafficDeliveryModes,
|
|
38
39
|
ContentActions,
|
|
39
40
|
DEFAULT_DISCOVERY_PROMOTE_BUCKETS,
|
|
40
41
|
DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES,
|
|
@@ -197,6 +198,9 @@ import {
|
|
|
197
198
|
classifySkillFile,
|
|
198
199
|
classifySurfaceFromCategory,
|
|
199
200
|
classifyTrafficPath,
|
|
201
|
+
cloudflareTrafficSourceConfigSchema,
|
|
202
|
+
cloudflareWorkerIngestRequestSchema,
|
|
203
|
+
cloudflareWorkerIngestResponseSchema,
|
|
200
204
|
clusterByCosine,
|
|
201
205
|
coerceSkillManifest,
|
|
202
206
|
compactDateToIso,
|
|
@@ -520,6 +524,8 @@ import {
|
|
|
520
524
|
textContainsBrandAlias,
|
|
521
525
|
textContainsDomain,
|
|
522
526
|
trafficBackfillResponseSchema,
|
|
527
|
+
trafficConnectCloudflareRequestSchema,
|
|
528
|
+
trafficConnectCloudflareResponseSchema,
|
|
523
529
|
trafficConnectVercelRequestSchema,
|
|
524
530
|
trafficConnectWordpressRequestSchema,
|
|
525
531
|
trafficEventsResponseSchema,
|
|
@@ -553,10 +559,10 @@ import {
|
|
|
553
559
|
wordpressSchemaDeployResultDtoSchema,
|
|
554
560
|
wordpressSchemaStatusResultDtoSchema,
|
|
555
561
|
wordpressStatusDtoSchema
|
|
556
|
-
} from "./chunk-
|
|
562
|
+
} from "./chunk-TRRVBTU5.js";
|
|
557
563
|
|
|
558
564
|
// src/intelligence-service.ts
|
|
559
|
-
import { eq as
|
|
565
|
+
import { eq as eq58, desc as desc26, asc as asc11, and as and46, ne as ne7, or as or12, inArray as inArray20, gte as gte14, lte as lte11 } from "drizzle-orm";
|
|
560
566
|
|
|
561
567
|
// ../db/src/client.ts
|
|
562
568
|
import { mkdirSync } from "fs";
|
|
@@ -657,6 +663,7 @@ __export(schema_exports, {
|
|
|
657
663
|
siteCrawlPages: () => siteCrawlPages,
|
|
658
664
|
siteCrawlRunRequests: () => siteCrawlRunRequests,
|
|
659
665
|
siteCrawlSnapshots: () => siteCrawlSnapshots,
|
|
666
|
+
trafficEventReceipts: () => trafficEventReceipts,
|
|
660
667
|
trafficSources: () => trafficSources,
|
|
661
668
|
usageCounters: () => usageCounters,
|
|
662
669
|
userSessions: () => userSessions,
|
|
@@ -2048,12 +2055,31 @@ var trafficSources = sqliteTable("traffic_sources", {
|
|
|
2048
2055
|
lastEventIds: text("last_event_ids", { mode: "json" }).$type(),
|
|
2049
2056
|
archivedAt: text("archived_at"),
|
|
2050
2057
|
configJson: text("config_json", { mode: "json" }).$type().notNull().default({}),
|
|
2058
|
+
// sha256 hex of the per-source bearer token issued at connect time. Only
|
|
2059
|
+
// populated for push-receive source types (currently `cloudflare`); pull
|
|
2060
|
+
// adapters leave this NULL. The cleartext bearer + HMAC secret never live
|
|
2061
|
+
// in the DB — they go to `~/.canonry/config.yaml` under the per-type
|
|
2062
|
+
// connections block.
|
|
2063
|
+
ingestTokenHash: text("ingest_token_hash"),
|
|
2064
|
+
// Semver reported by the most recent forwarded event. Drives the
|
|
2065
|
+
// `traffic.source.worker-version` doctor check. NULL until the first
|
|
2066
|
+
// event arrives or for source types that don't forward versioned events.
|
|
2067
|
+
lastWorkerVersion: text("last_worker_version"),
|
|
2051
2068
|
createdAt: text("created_at").notNull(),
|
|
2052
2069
|
updatedAt: text("updated_at").notNull()
|
|
2053
2070
|
}, (table) => [
|
|
2054
2071
|
index("idx_traffic_sources_project").on(table.projectId),
|
|
2055
2072
|
index("idx_traffic_sources_project_status").on(table.projectId, table.status)
|
|
2056
2073
|
]);
|
|
2074
|
+
var trafficEventReceipts = sqliteTable("traffic_event_receipts", {
|
|
2075
|
+
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
2076
|
+
eventId: text("event_id").notNull(),
|
|
2077
|
+
receivedAt: text("received_at").notNull(),
|
|
2078
|
+
expiresAt: text("expires_at").notNull()
|
|
2079
|
+
}, (table) => [
|
|
2080
|
+
primaryKey({ columns: [table.sourceId, table.eventId] }),
|
|
2081
|
+
index("idx_traffic_event_receipts_expires").on(table.sourceId, table.expiresAt)
|
|
2082
|
+
]);
|
|
2057
2083
|
var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
|
|
2058
2084
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
2059
2085
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -5753,6 +5779,32 @@ var MIGRATION_VERSIONS = [
|
|
|
5753
5779
|
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_target_node
|
|
5754
5780
|
ON site_crawl_graph_edges(project_id, run_id, attempt_id, target_node_key)`
|
|
5755
5781
|
]
|
|
5782
|
+
},
|
|
5783
|
+
{
|
|
5784
|
+
version: 129,
|
|
5785
|
+
name: "traffic-ingest-foundation",
|
|
5786
|
+
// Push-receive traffic sources (currently only `cloudflare`) need a
|
|
5787
|
+
// per-source bearer for the Worker to authenticate against canonry's
|
|
5788
|
+
// ingest endpoint, plus a place to remember the deployed Worker version.
|
|
5789
|
+
// Durable receipts are transport-neutral: direct push and a future Queue
|
|
5790
|
+
// pull consumer both claim an event in the same transaction as rollups.
|
|
5791
|
+
// Cleartext credentials remain outside the database.
|
|
5792
|
+
//
|
|
5793
|
+
// Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
|
|
5794
|
+
// name" on retry, which the runner already swallows.
|
|
5795
|
+
statements: [
|
|
5796
|
+
`ALTER TABLE traffic_sources ADD COLUMN ingest_token_hash TEXT`,
|
|
5797
|
+
`ALTER TABLE traffic_sources ADD COLUMN last_worker_version TEXT`,
|
|
5798
|
+
`CREATE TABLE IF NOT EXISTS traffic_event_receipts (
|
|
5799
|
+
source_id TEXT NOT NULL REFERENCES traffic_sources(id) ON DELETE CASCADE,
|
|
5800
|
+
event_id TEXT NOT NULL,
|
|
5801
|
+
received_at TEXT NOT NULL,
|
|
5802
|
+
expires_at TEXT NOT NULL,
|
|
5803
|
+
PRIMARY KEY (source_id, event_id)
|
|
5804
|
+
)`,
|
|
5805
|
+
`CREATE INDEX IF NOT EXISTS idx_traffic_event_receipts_expires
|
|
5806
|
+
ON traffic_event_receipts(source_id, expires_at)`
|
|
5807
|
+
]
|
|
5756
5808
|
}
|
|
5757
5809
|
];
|
|
5758
5810
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -8560,6 +8612,7 @@ function shouldSkipAuth(url) {
|
|
|
8560
8612
|
if (url.endsWith("/session") || url.endsWith("/session/setup")) return true;
|
|
8561
8613
|
if (url.endsWith("/auth/session") || url.endsWith("/auth/login") || url.endsWith("/auth/logout")) return true;
|
|
8562
8614
|
if (url.endsWith("/auth/sessions")) return true;
|
|
8615
|
+
if (url.endsWith("/traffic/cloudflare/ingest")) return true;
|
|
8563
8616
|
return false;
|
|
8564
8617
|
}
|
|
8565
8618
|
var USERS_READ_SCOPE = "users.read";
|
|
@@ -27365,7 +27418,11 @@ var SCHEMA_TABLE = {
|
|
|
27365
27418
|
SnapshotReportDto: snapshotReportSchema,
|
|
27366
27419
|
SourceBreakdownDto: sourceBreakdownDtoSchema,
|
|
27367
27420
|
TelemetryEventAcceptedDto: telemetryEventAcceptedDtoSchema,
|
|
27421
|
+
CloudflareWorkerIngestRequest: cloudflareWorkerIngestRequestSchema,
|
|
27422
|
+
CloudflareWorkerIngestResponse: cloudflareWorkerIngestResponseSchema,
|
|
27368
27423
|
TrafficBackfillResponse: trafficBackfillResponseSchema,
|
|
27424
|
+
TrafficConnectCloudflareRequest: trafficConnectCloudflareRequestSchema,
|
|
27425
|
+
TrafficConnectCloudflareResponse: trafficConnectCloudflareResponseSchema,
|
|
27369
27426
|
TrafficEventsResponse: trafficEventsResponseSchema,
|
|
27370
27427
|
TrafficSourceDetailDto: trafficSourceDetailDtoSchema,
|
|
27371
27428
|
TrafficSourceDto: trafficSourceDtoSchema,
|
|
@@ -32933,6 +32990,54 @@ var routeCatalog = [
|
|
|
32933
32990
|
502: errorResponse("Vercel request-logs endpoint probe failed (bad token, wrong project / team id, unreachable host, etc.).")
|
|
32934
32991
|
}
|
|
32935
32992
|
},
|
|
32993
|
+
{
|
|
32994
|
+
method: "post",
|
|
32995
|
+
path: "/api/v1/projects/{name}/traffic/connect/cloudflare",
|
|
32996
|
+
summary: "Connect a Cloudflare Worker traffic source",
|
|
32997
|
+
description: "Creates or updates a Cloudflare `direct-push` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration. Per-source bearer and HMAC credentials remain in Canonry's credential store and are installed as Worker secret bindings; neither secret appears in source, TOML, API, or MCP output. Reconnect is idempotent: it reuses a matching source and credentials, preserves omitted metadata, and emits the current deployment package. Wrangler deploys the Worker without a route. The operator must attach the exact site route manually and set its Request limit failure mode to Fail open. No upstream probe is performed at connect time.",
|
|
32998
|
+
tags: ["traffic"],
|
|
32999
|
+
parameters: [nameParameter],
|
|
33000
|
+
requestBody: {
|
|
33001
|
+
required: false,
|
|
33002
|
+
content: {
|
|
33003
|
+
"application/json": {
|
|
33004
|
+
schema: { $ref: "#/components/schemas/TrafficConnectCloudflareRequest" }
|
|
33005
|
+
}
|
|
33006
|
+
}
|
|
33007
|
+
},
|
|
33008
|
+
responses: {
|
|
33009
|
+
200: jsonResponse("Connect response with generated Worker script.", "TrafficConnectCloudflareResponse"),
|
|
33010
|
+
400: errorResponse("Invalid Cloudflare connection request or credential storage not configured."),
|
|
33011
|
+
404: errorResponse("Project not found.")
|
|
33012
|
+
}
|
|
33013
|
+
},
|
|
33014
|
+
{
|
|
33015
|
+
method: "post",
|
|
33016
|
+
path: "/api/v1/projects/{name}/traffic/cloudflare/ingest",
|
|
33017
|
+
summary: "Ingest a batch of Cloudflare Worker events",
|
|
33018
|
+
description: "Direct-push protocol endpoint. The customer Worker sends selected edge-event batches authenticated by a per-source bearer and HMAC-SHA256 over `${timestamp}.${canonicalJson(payload)}`; object keys are sorted recursively before signing. The global cnry_* bearer does not apply. Headers: `Authorization: Bearer <per-source-token>`, `X-Canonry-Timestamp` (unix seconds), `X-Canonry-Signature` (hex HMAC), `X-Canonry-Source-Id`, and `X-Canonry-Worker-Version`. Authentication completes before project resolution and every auth/path/source mismatch returns the same 401. Accepted event ids are claimed durably in the same transaction as shared crawler, user-fetch, and paid/organic referral rollups.",
|
|
33019
|
+
tags: ["traffic"],
|
|
33020
|
+
auth: false,
|
|
33021
|
+
parameters: [nameParameter],
|
|
33022
|
+
requestBody: {
|
|
33023
|
+
required: true,
|
|
33024
|
+
content: {
|
|
33025
|
+
"application/json": {
|
|
33026
|
+
schema: { $ref: "#/components/schemas/CloudflareWorkerIngestRequest" }
|
|
33027
|
+
}
|
|
33028
|
+
}
|
|
33029
|
+
},
|
|
33030
|
+
responses: {
|
|
33031
|
+
200: jsonResponse(
|
|
33032
|
+
"Ingest acknowledged; droppedEvents includes normalization failures and durable-receipt duplicates.",
|
|
33033
|
+
"CloudflareWorkerIngestResponse"
|
|
33034
|
+
),
|
|
33035
|
+
400: errorResponse("Invalid ingest payload."),
|
|
33036
|
+
401: errorResponse("Authentication failed (bearer, signature, timestamp, or source id mismatch)."),
|
|
33037
|
+
413: errorResponse("The ingest payload exceeds 256 KiB."),
|
|
33038
|
+
429: errorResponse("The per-IP or authenticated-source ingest request budget was exceeded.")
|
|
33039
|
+
}
|
|
33040
|
+
},
|
|
32936
33041
|
{
|
|
32937
33042
|
method: "post",
|
|
32938
33043
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
@@ -46811,7 +46916,7 @@ async function queryBacklinks(opts) {
|
|
|
46811
46916
|
const reversed = opts.targets.map(reverseDomain);
|
|
46812
46917
|
const targetList = reversed.map(quote).join(", ");
|
|
46813
46918
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
46814
|
-
const
|
|
46919
|
+
const sql22 = `
|
|
46815
46920
|
WITH vertices AS (
|
|
46816
46921
|
SELECT * FROM read_csv(
|
|
46817
46922
|
${quote(opts.vertexPath)},
|
|
@@ -46847,7 +46952,7 @@ async function queryBacklinks(opts) {
|
|
|
46847
46952
|
const conn = await instance.connect();
|
|
46848
46953
|
let rows;
|
|
46849
46954
|
try {
|
|
46850
|
-
const reader = await conn.runAndReadAll(
|
|
46955
|
+
const reader = await conn.runAndReadAll(sql22);
|
|
46851
46956
|
rows = reader.getRowObjects();
|
|
46852
46957
|
} finally {
|
|
46853
46958
|
conn.disconnectSync?.();
|
|
@@ -47303,9 +47408,10 @@ async function backlinksRoutes(app, opts) {
|
|
|
47303
47408
|
}
|
|
47304
47409
|
|
|
47305
47410
|
// ../api-routes/src/traffic.ts
|
|
47306
|
-
import
|
|
47411
|
+
import crypto40 from "crypto";
|
|
47412
|
+
import { isIP } from "net";
|
|
47307
47413
|
import { Agent as UndiciAgent } from "undici";
|
|
47308
|
-
import { and as
|
|
47414
|
+
import { and as and39, desc as desc22, eq as eq47, gte as gte11, lte as lte10, sql as sql19 } from "drizzle-orm";
|
|
47309
47415
|
|
|
47310
47416
|
// ../api-routes/src/traffic-limits.ts
|
|
47311
47417
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
@@ -50089,6 +50195,39 @@ var DEFAULT_AI_CRAWLER_RULES = [
|
|
|
50089
50195
|
userAgentPatterns: [/Amazonbot\//i]
|
|
50090
50196
|
}
|
|
50091
50197
|
];
|
|
50198
|
+
var DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS = [
|
|
50199
|
+
"GPTBot/",
|
|
50200
|
+
"OAI-SearchBot/",
|
|
50201
|
+
"ChatGPT-User/",
|
|
50202
|
+
"ClaudeBot/",
|
|
50203
|
+
"Claude-Web/",
|
|
50204
|
+
"Claude-SearchBot/",
|
|
50205
|
+
"Claude-User/",
|
|
50206
|
+
"anthropic-ai",
|
|
50207
|
+
"PerplexityBot/",
|
|
50208
|
+
"Google-Extended",
|
|
50209
|
+
"Google-Agent",
|
|
50210
|
+
"Bytespider",
|
|
50211
|
+
"Applebot-Extended",
|
|
50212
|
+
"Applebot/",
|
|
50213
|
+
"meta-externalagent",
|
|
50214
|
+
"CCBot/",
|
|
50215
|
+
"cohere-ai",
|
|
50216
|
+
"Diffbot",
|
|
50217
|
+
"MistralAI-User/",
|
|
50218
|
+
"MistralBot/",
|
|
50219
|
+
"DeepSeekBot",
|
|
50220
|
+
"xAI-Bot/",
|
|
50221
|
+
"Grok-Bot/",
|
|
50222
|
+
"GrokBot/",
|
|
50223
|
+
"Googlebot",
|
|
50224
|
+
"bingbot/",
|
|
50225
|
+
"DuckDuckBot",
|
|
50226
|
+
"YandexBot/",
|
|
50227
|
+
"Baiduspider",
|
|
50228
|
+
"Amazonbot/",
|
|
50229
|
+
"Perplexity-User/"
|
|
50230
|
+
];
|
|
50092
50231
|
var DEFAULT_AI_REFERRER_RULES = [
|
|
50093
50232
|
{ domain: AI_ENGINE_DOMAINS.chatgpt, operator: "OpenAI", product: "ChatGPT" },
|
|
50094
50233
|
{ domain: LEGACY_CHATGPT_DOMAIN, operator: "OpenAI", product: "ChatGPT" },
|
|
@@ -51109,6 +51248,716 @@ async function drainVercelTrafficEvents(options) {
|
|
|
51109
51248
|
return { events, subWindowCount, effectiveStartMs, retentionClamped, truncatedSliceCount, truncatedSliceStartsMs, deadlineSkippedSliceCount: deadlineSkippedSliceStartsMs.length, deadlineSkippedSliceStartsMs, drainedThroughMs: cursorMs, deadlineReached };
|
|
51110
51249
|
}
|
|
51111
51250
|
|
|
51251
|
+
// ../integration-cloudflare-worker/src/canonical-json.ts
|
|
51252
|
+
function canonicalizeCloudflareJson(value) {
|
|
51253
|
+
if (value === null) return "null";
|
|
51254
|
+
if (typeof value === "string" || typeof value === "boolean") {
|
|
51255
|
+
return JSON.stringify(value);
|
|
51256
|
+
}
|
|
51257
|
+
if (typeof value === "number") {
|
|
51258
|
+
if (!Number.isFinite(value)) {
|
|
51259
|
+
throw new TypeError("Cloudflare canonical JSON does not support non-finite numbers");
|
|
51260
|
+
}
|
|
51261
|
+
return JSON.stringify(value);
|
|
51262
|
+
}
|
|
51263
|
+
if (Array.isArray(value)) {
|
|
51264
|
+
return `[${value.map((item2) => canonicalizeCloudflareJson(item2)).join(",")}]`;
|
|
51265
|
+
}
|
|
51266
|
+
if (typeof value === "object") {
|
|
51267
|
+
const record = value;
|
|
51268
|
+
const fields = [];
|
|
51269
|
+
for (const key of Object.keys(record).sort()) {
|
|
51270
|
+
const fieldValue = record[key];
|
|
51271
|
+
if (fieldValue === void 0) continue;
|
|
51272
|
+
fields.push(`${JSON.stringify(key)}:${canonicalizeCloudflareJson(fieldValue)}`);
|
|
51273
|
+
}
|
|
51274
|
+
return `{${fields.join(",")}}`;
|
|
51275
|
+
}
|
|
51276
|
+
throw new TypeError(`Cloudflare canonical JSON does not support ${typeof value}`);
|
|
51277
|
+
}
|
|
51278
|
+
|
|
51279
|
+
// ../integration-cloudflare-worker/src/normalize.ts
|
|
51280
|
+
function emptyToNull2(value) {
|
|
51281
|
+
if (value === null || value.trim() === "") return null;
|
|
51282
|
+
return value;
|
|
51283
|
+
}
|
|
51284
|
+
function maybeLabel(value) {
|
|
51285
|
+
if (value === null) return void 0;
|
|
51286
|
+
if (typeof value === "string" && value === "") return void 0;
|
|
51287
|
+
return String(value);
|
|
51288
|
+
}
|
|
51289
|
+
function buildProviderLabels(cf) {
|
|
51290
|
+
if (!cf) return {};
|
|
51291
|
+
const candidates = {
|
|
51292
|
+
verifiedBot: maybeLabel(cf.verifiedBot),
|
|
51293
|
+
botScore: maybeLabel(cf.botScore),
|
|
51294
|
+
country: maybeLabel(cf.country),
|
|
51295
|
+
asn: maybeLabel(cf.asn),
|
|
51296
|
+
asOrganization: maybeLabel(cf.asOrganization)
|
|
51297
|
+
};
|
|
51298
|
+
return Object.fromEntries(
|
|
51299
|
+
Object.entries(candidates).filter(
|
|
51300
|
+
(entry) => typeof entry[1] === "string"
|
|
51301
|
+
)
|
|
51302
|
+
);
|
|
51303
|
+
}
|
|
51304
|
+
function normalizeCloudflareEdgeEvent(event) {
|
|
51305
|
+
if (!event.path) return null;
|
|
51306
|
+
if (!event.observedAt) return null;
|
|
51307
|
+
if (!event.eventId) return null;
|
|
51308
|
+
const host = emptyToNull2(event.host);
|
|
51309
|
+
const path8 = event.path;
|
|
51310
|
+
const queryString = emptyToNull2(event.queryString);
|
|
51311
|
+
const requestUrl = host ? `https://${host}${path8}${queryString ? `?${queryString}` : ""}` : null;
|
|
51312
|
+
return {
|
|
51313
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
51314
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
51315
|
+
confidence: TrafficEventConfidences.observed,
|
|
51316
|
+
eventId: `cloudflare-worker:${event.eventId}`,
|
|
51317
|
+
observedAt: event.observedAt,
|
|
51318
|
+
method: emptyToNull2(event.method),
|
|
51319
|
+
requestUrl,
|
|
51320
|
+
host,
|
|
51321
|
+
path: path8,
|
|
51322
|
+
queryString,
|
|
51323
|
+
status: event.status,
|
|
51324
|
+
userAgent: emptyToNull2(event.userAgent),
|
|
51325
|
+
remoteIp: emptyToNull2(event.remoteIp),
|
|
51326
|
+
referer: emptyToNull2(event.referer),
|
|
51327
|
+
latencyMs: null,
|
|
51328
|
+
requestSizeBytes: null,
|
|
51329
|
+
responseSizeBytes: null,
|
|
51330
|
+
providerResource: {
|
|
51331
|
+
type: "cloudflare_zone",
|
|
51332
|
+
labels: {}
|
|
51333
|
+
},
|
|
51334
|
+
providerLabels: buildProviderLabels(event.cf)
|
|
51335
|
+
};
|
|
51336
|
+
}
|
|
51337
|
+
var normalizeCloudflareWorkerEvent = normalizeCloudflareEdgeEvent;
|
|
51338
|
+
|
|
51339
|
+
// ../integration-cloudflare-worker/src/types.ts
|
|
51340
|
+
var CLOUDFLARE_WORKER_BINDINGS = {
|
|
51341
|
+
deliveryMode: "CANONRY_DELIVERY_MODE",
|
|
51342
|
+
sourceId: "CANONRY_SOURCE_ID",
|
|
51343
|
+
ingestUrl: "CANONRY_INGEST_URL",
|
|
51344
|
+
workerVersion: "CANONRY_WORKER_VERSION",
|
|
51345
|
+
bearerToken: "CANONRY_BEARER_TOKEN",
|
|
51346
|
+
hmacSecret: "CANONRY_HMAC_SECRET"
|
|
51347
|
+
};
|
|
51348
|
+
var CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS = [
|
|
51349
|
+
CLOUDFLARE_WORKER_BINDINGS.bearerToken,
|
|
51350
|
+
CLOUDFLARE_WORKER_BINDINGS.hmacSecret
|
|
51351
|
+
];
|
|
51352
|
+
var CLOUDFLARE_WORKER_GENERATED_MARKER = "// canonry traffic Worker \u2014 generated; do not edit by hand";
|
|
51353
|
+
var CLOUDFLARE_WRANGLER_GENERATED_MARKER = "# canonry traffic Worker configuration \u2014 generated; do not edit by hand";
|
|
51354
|
+
|
|
51355
|
+
// ../integration-cloudflare-worker/src/script.ts
|
|
51356
|
+
var DEFAULT_BOT_LIST = {
|
|
51357
|
+
version: "2026-08-09",
|
|
51358
|
+
uaKeywords: [
|
|
51359
|
+
...DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS,
|
|
51360
|
+
// Preserve a small forward-looking machine-traffic net. The shared
|
|
51361
|
+
// classifier still drops unknown products after delivery.
|
|
51362
|
+
"bot",
|
|
51363
|
+
"crawler",
|
|
51364
|
+
"spider"
|
|
51365
|
+
],
|
|
51366
|
+
refererDomains: DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain),
|
|
51367
|
+
utmSourceTokens: [
|
|
51368
|
+
.../* @__PURE__ */ new Set([
|
|
51369
|
+
...DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain),
|
|
51370
|
+
...DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain.split(".")[0]),
|
|
51371
|
+
// Legacy chat.openai.com referrals classify from their first label.
|
|
51372
|
+
"chat"
|
|
51373
|
+
])
|
|
51374
|
+
]
|
|
51375
|
+
};
|
|
51376
|
+
var DEFAULT_BOT_SCORE_MAX_FORWARD = 30;
|
|
51377
|
+
var WORKER_COMPATIBILITY_DATE = "2026-05-01";
|
|
51378
|
+
function jsString(value) {
|
|
51379
|
+
return JSON.stringify(value);
|
|
51380
|
+
}
|
|
51381
|
+
function jsArray(values) {
|
|
51382
|
+
return `[${values.map((v) => jsString(v)).join(", ")}]`;
|
|
51383
|
+
}
|
|
51384
|
+
function generateWorkerScript(opts) {
|
|
51385
|
+
const botScoreMax = opts.botScoreMaxForward ?? DEFAULT_BOT_SCORE_MAX_FORWARD;
|
|
51386
|
+
const canonicalJsonFunction = canonicalizeCloudflareJson.toString();
|
|
51387
|
+
return `${CLOUDFLARE_WORKER_GENERATED_MARKER}
|
|
51388
|
+
// worker version: ${opts.workerVersion}
|
|
51389
|
+
// bot-list version: ${opts.botList.version}
|
|
51390
|
+
// delivery mode: ${opts.deliveryMode}
|
|
51391
|
+
|
|
51392
|
+
const UA_KEYWORDS = ${jsArray(opts.botList.uaKeywords)}
|
|
51393
|
+
const REFERER_DOMAINS = ${jsArray(opts.botList.refererDomains)}
|
|
51394
|
+
const UTM_SOURCE_TOKENS = ${jsArray(opts.botList.utmSourceTokens)}
|
|
51395
|
+
const BOT_SCORE_MAX_FORWARD = ${String(botScoreMax)}
|
|
51396
|
+
const RETRY_DELAYS_MS = [250, 1000]
|
|
51397
|
+
const canonicalizeJson = (${canonicalJsonFunction})
|
|
51398
|
+
|
|
51399
|
+
function requireBinding(value, name) {
|
|
51400
|
+
if (typeof value === 'string' && value.length > 0) return value
|
|
51401
|
+
throw new Error('Missing required Worker binding: ' + name)
|
|
51402
|
+
}
|
|
51403
|
+
|
|
51404
|
+
function lower(value) {
|
|
51405
|
+
return typeof value === 'string' ? value.toLowerCase() : ''
|
|
51406
|
+
}
|
|
51407
|
+
|
|
51408
|
+
function uaMatches(ua) {
|
|
51409
|
+
const lc = lower(ua)
|
|
51410
|
+
if (!lc) return false
|
|
51411
|
+
for (const kw of UA_KEYWORDS) {
|
|
51412
|
+
if (lc.indexOf(lower(kw)) !== -1) return true
|
|
51413
|
+
}
|
|
51414
|
+
return false
|
|
51415
|
+
}
|
|
51416
|
+
|
|
51417
|
+
function hostMatchesKnownDomain(host) {
|
|
51418
|
+
let normalized = lower(host)
|
|
51419
|
+
if (normalized.startsWith('www.')) normalized = normalized.slice(4)
|
|
51420
|
+
if (normalized.endsWith('.')) normalized = normalized.slice(0, -1)
|
|
51421
|
+
if (!normalized) return false
|
|
51422
|
+
for (const domain of REFERER_DOMAINS) {
|
|
51423
|
+
if (normalized === domain || normalized.endsWith('.' + domain)) return true
|
|
51424
|
+
}
|
|
51425
|
+
return false
|
|
51426
|
+
}
|
|
51427
|
+
|
|
51428
|
+
function refererMatches(referer) {
|
|
51429
|
+
if (!referer) return false
|
|
51430
|
+
try {
|
|
51431
|
+
return hostMatchesKnownDomain(new URL(referer).hostname)
|
|
51432
|
+
} catch (_) {
|
|
51433
|
+
return false
|
|
51434
|
+
}
|
|
51435
|
+
}
|
|
51436
|
+
|
|
51437
|
+
function utmSourceMatches(value) {
|
|
51438
|
+
let normalized = lower(value).trim()
|
|
51439
|
+
if (normalized.startsWith('www.')) normalized = normalized.slice(4)
|
|
51440
|
+
if (normalized.endsWith('.')) normalized = normalized.slice(0, -1)
|
|
51441
|
+
if (!normalized) return false
|
|
51442
|
+
if (hostMatchesKnownDomain(normalized)) return true
|
|
51443
|
+
return UTM_SOURCE_TOKENS.indexOf(normalized) !== -1
|
|
51444
|
+
}
|
|
51445
|
+
|
|
51446
|
+
function urlUtmMatches(value) {
|
|
51447
|
+
if (!value) return false
|
|
51448
|
+
try {
|
|
51449
|
+
return utmSourceMatches(new URL(value).searchParams.get('utm_source'))
|
|
51450
|
+
} catch (_) {
|
|
51451
|
+
return false
|
|
51452
|
+
}
|
|
51453
|
+
}
|
|
51454
|
+
|
|
51455
|
+
function botSignals(cf) {
|
|
51456
|
+
if (!cf) return false
|
|
51457
|
+
const bm = cf.botManagement
|
|
51458
|
+
if (bm) {
|
|
51459
|
+
if (bm.verifiedBot === true) return true
|
|
51460
|
+
if (typeof bm.score === 'number' && bm.score < BOT_SCORE_MAX_FORWARD) return true
|
|
51461
|
+
}
|
|
51462
|
+
if (typeof cf.botScore === 'number' && cf.botScore < BOT_SCORE_MAX_FORWARD) return true
|
|
51463
|
+
return false
|
|
51464
|
+
}
|
|
51465
|
+
|
|
51466
|
+
function shouldForward(request) {
|
|
51467
|
+
const ua = request.headers.get('user-agent') || ''
|
|
51468
|
+
if (uaMatches(ua)) return true
|
|
51469
|
+
if (urlUtmMatches(request.url)) return true
|
|
51470
|
+
const referer = request.headers.get('referer') || ''
|
|
51471
|
+
if (refererMatches(referer)) return true
|
|
51472
|
+
if (urlUtmMatches(referer)) return true
|
|
51473
|
+
return botSignals(request.cf)
|
|
51474
|
+
}
|
|
51475
|
+
|
|
51476
|
+
function toHex(buffer) {
|
|
51477
|
+
const bytes = new Uint8Array(buffer)
|
|
51478
|
+
let out = ''
|
|
51479
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
51480
|
+
out += bytes[i].toString(16).padStart(2, '0')
|
|
51481
|
+
}
|
|
51482
|
+
return out
|
|
51483
|
+
}
|
|
51484
|
+
|
|
51485
|
+
async function signBody(secret, timestamp, body) {
|
|
51486
|
+
const key = await crypto.subtle.importKey(
|
|
51487
|
+
'raw',
|
|
51488
|
+
new TextEncoder().encode(secret),
|
|
51489
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
51490
|
+
false,
|
|
51491
|
+
['sign'],
|
|
51492
|
+
)
|
|
51493
|
+
const sig = await crypto.subtle.sign(
|
|
51494
|
+
'HMAC',
|
|
51495
|
+
key,
|
|
51496
|
+
new TextEncoder().encode(timestamp + '.' + body),
|
|
51497
|
+
)
|
|
51498
|
+
return toHex(sig)
|
|
51499
|
+
}
|
|
51500
|
+
|
|
51501
|
+
function isRetryableStatus(status) {
|
|
51502
|
+
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
51503
|
+
}
|
|
51504
|
+
|
|
51505
|
+
function sleep(ms) {
|
|
51506
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
51507
|
+
}
|
|
51508
|
+
|
|
51509
|
+
async function cancelUnusedResponseBody(response) {
|
|
51510
|
+
try {
|
|
51511
|
+
if (response.body) await response.body.cancel()
|
|
51512
|
+
} catch (_) {
|
|
51513
|
+
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
51514
|
+
}
|
|
51515
|
+
}
|
|
51516
|
+
|
|
51517
|
+
async function withRetry(operation) {
|
|
51518
|
+
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
51519
|
+
let response
|
|
51520
|
+
try {
|
|
51521
|
+
response = await operation()
|
|
51522
|
+
} catch (error) {
|
|
51523
|
+
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
51524
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
51525
|
+
continue
|
|
51526
|
+
}
|
|
51527
|
+
|
|
51528
|
+
const status = response.status
|
|
51529
|
+
const ok = response.ok
|
|
51530
|
+
await cancelUnusedResponseBody(response)
|
|
51531
|
+
if (ok) return
|
|
51532
|
+
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
51533
|
+
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
51534
|
+
}
|
|
51535
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
51536
|
+
}
|
|
51537
|
+
}
|
|
51538
|
+
|
|
51539
|
+
function pickCf(cf) {
|
|
51540
|
+
if (!cf) return null
|
|
51541
|
+
const bm = cf.botManagement || {}
|
|
51542
|
+
return {
|
|
51543
|
+
verifiedBot: typeof bm.verifiedBot === 'boolean' ? bm.verifiedBot : null,
|
|
51544
|
+
botScore: typeof bm.score === 'number' ? bm.score : (typeof cf.botScore === 'number' ? cf.botScore : null),
|
|
51545
|
+
country: typeof cf.country === 'string' ? cf.country : null,
|
|
51546
|
+
asn: typeof cf.asn === 'number' ? cf.asn : null,
|
|
51547
|
+
asOrganization: typeof cf.asOrganization === 'string' ? cf.asOrganization : null,
|
|
51548
|
+
}
|
|
51549
|
+
}
|
|
51550
|
+
|
|
51551
|
+
function buildEdgeEvent(request, status, observedAt) {
|
|
51552
|
+
const url = new URL(request.url)
|
|
51553
|
+
return {
|
|
51554
|
+
eventId: request.headers.get('cf-ray') || crypto.randomUUID(),
|
|
51555
|
+
observedAt,
|
|
51556
|
+
method: request.method || null,
|
|
51557
|
+
host: url.hostname || null,
|
|
51558
|
+
path: url.pathname || '/',
|
|
51559
|
+
queryString: url.search ? url.search.slice(1) : null,
|
|
51560
|
+
status: typeof status === 'number' ? status : null,
|
|
51561
|
+
userAgent: request.headers.get('user-agent') || null,
|
|
51562
|
+
remoteIp: request.headers.get('cf-connecting-ip') || null,
|
|
51563
|
+
referer: request.headers.get('referer') || null,
|
|
51564
|
+
cf: pickCf(request.cf),
|
|
51565
|
+
}
|
|
51566
|
+
}
|
|
51567
|
+
|
|
51568
|
+
function buildEdgeEventBatch(env, request, status, observedAt) {
|
|
51569
|
+
return {
|
|
51570
|
+
schemaVersion: 1,
|
|
51571
|
+
workerVersion: requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.workerVersion}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.workerVersion)}),
|
|
51572
|
+
events: [buildEdgeEvent(request, status, observedAt)],
|
|
51573
|
+
}
|
|
51574
|
+
}
|
|
51575
|
+
|
|
51576
|
+
async function deliverViaDirectPush(env, batch) {
|
|
51577
|
+
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
51578
|
+
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
51579
|
+
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
51580
|
+
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
51581
|
+
const body = canonicalizeJson(batch)
|
|
51582
|
+
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
51583
|
+
const signature = await signBody(hmacSecret, timestamp, body)
|
|
51584
|
+
|
|
51585
|
+
await withRetry(() => fetch(ingestUrl, {
|
|
51586
|
+
method: 'POST',
|
|
51587
|
+
headers: {
|
|
51588
|
+
'content-type': 'application/json',
|
|
51589
|
+
'Authorization': 'Bearer ' + bearerToken,
|
|
51590
|
+
'X-Canonry-Timestamp': timestamp,
|
|
51591
|
+
'X-Canonry-Signature': signature,
|
|
51592
|
+
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
51593
|
+
'X-Canonry-Source-Id': sourceId,
|
|
51594
|
+
},
|
|
51595
|
+
body,
|
|
51596
|
+
}))
|
|
51597
|
+
}
|
|
51598
|
+
|
|
51599
|
+
async function deliverEdgeEventBatch(env, batch) {
|
|
51600
|
+
const deliveryMode = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.deliveryMode}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.deliveryMode)})
|
|
51601
|
+
if (deliveryMode === 'direct-push') {
|
|
51602
|
+
return deliverViaDirectPush(env, batch)
|
|
51603
|
+
}
|
|
51604
|
+
throw new Error('Unsupported Canonry delivery mode: ' + deliveryMode)
|
|
51605
|
+
}
|
|
51606
|
+
|
|
51607
|
+
async function deliverSelectedRequest(env, request, status, observedAt) {
|
|
51608
|
+
try {
|
|
51609
|
+
const batch = buildEdgeEventBatch(env, request, status, observedAt)
|
|
51610
|
+
await deliverEdgeEventBatch(env, batch)
|
|
51611
|
+
} catch (err) {
|
|
51612
|
+
// Delivery is statistical and must never mask the customer response.
|
|
51613
|
+
// Emit binding names and source id only; never emit secret values.
|
|
51614
|
+
console.warn('Canonry traffic delivery failed', {
|
|
51615
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51616
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51617
|
+
: null,
|
|
51618
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51619
|
+
})
|
|
51620
|
+
}
|
|
51621
|
+
}
|
|
51622
|
+
|
|
51623
|
+
function shouldForwardSafely(env, request) {
|
|
51624
|
+
try {
|
|
51625
|
+
return shouldForward(request)
|
|
51626
|
+
} catch (err) {
|
|
51627
|
+
console.warn('Canonry traffic filter failed', {
|
|
51628
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51629
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51630
|
+
: null,
|
|
51631
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51632
|
+
})
|
|
51633
|
+
return false
|
|
51634
|
+
}
|
|
51635
|
+
}
|
|
51636
|
+
|
|
51637
|
+
function scheduleSelectedRequest(ctx, env, request, status, observedAt) {
|
|
51638
|
+
try {
|
|
51639
|
+
ctx.waitUntil(deliverSelectedRequest(env, request, status, observedAt))
|
|
51640
|
+
} catch (err) {
|
|
51641
|
+
console.warn('Canonry traffic delivery scheduling failed', {
|
|
51642
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51643
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51644
|
+
: null,
|
|
51645
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51646
|
+
})
|
|
51647
|
+
}
|
|
51648
|
+
}
|
|
51649
|
+
|
|
51650
|
+
export default {
|
|
51651
|
+
async fetch(request, env, ctx) {
|
|
51652
|
+
const observedAt = new Date().toISOString()
|
|
51653
|
+
const shouldLog = shouldForwardSafely(env, request)
|
|
51654
|
+
let response
|
|
51655
|
+
try {
|
|
51656
|
+
response = await fetch(request)
|
|
51657
|
+
} catch (err) {
|
|
51658
|
+
if (shouldLog) {
|
|
51659
|
+
scheduleSelectedRequest(ctx, env, request, null, observedAt)
|
|
51660
|
+
}
|
|
51661
|
+
throw err
|
|
51662
|
+
}
|
|
51663
|
+
if (shouldLog) {
|
|
51664
|
+
scheduleSelectedRequest(ctx, env, request, response.status, observedAt)
|
|
51665
|
+
}
|
|
51666
|
+
return response
|
|
51667
|
+
},
|
|
51668
|
+
}
|
|
51669
|
+
`;
|
|
51670
|
+
}
|
|
51671
|
+
function generateWranglerToml(opts) {
|
|
51672
|
+
const hostname = opts.hostname.trim().toLowerCase().replace(/\.$/, "");
|
|
51673
|
+
const accountConfig = opts.accountId ? `account_id = ${jsString(opts.accountId)}
|
|
51674
|
+
` : "";
|
|
51675
|
+
const zoneHint = opts.zoneId ? `# Target zone id: ${jsString(opts.zoneId)}
|
|
51676
|
+
` : "# Target zone id was not provided. Select the canonical site zone.\n";
|
|
51677
|
+
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
51678
|
+
name = "canonry-traffic-${opts.sourceId}"
|
|
51679
|
+
${accountConfig}main = "worker.js"
|
|
51680
|
+
compatibility_date = "${WORKER_COMPATIBILITY_DATE}"
|
|
51681
|
+
workers_dev = false
|
|
51682
|
+
|
|
51683
|
+
[vars]
|
|
51684
|
+
${CLOUDFLARE_WORKER_BINDINGS.deliveryMode} = ${jsString(opts.deliveryMode)}
|
|
51685
|
+
${CLOUDFLARE_WORKER_BINDINGS.sourceId} = ${jsString(opts.sourceId)}
|
|
51686
|
+
${CLOUDFLARE_WORKER_BINDINGS.ingestUrl} = ${jsString(opts.ingestUrl)}
|
|
51687
|
+
${CLOUDFLARE_WORKER_BINDINGS.workerVersion} = ${jsString(opts.workerVersion)}
|
|
51688
|
+
|
|
51689
|
+
[secrets]
|
|
51690
|
+
required = ${jsArray(CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS)}
|
|
51691
|
+
|
|
51692
|
+
# Install required Worker secrets interactively or through Canonry's deploy
|
|
51693
|
+
# command. Never write their values into this file:
|
|
51694
|
+
${CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS.map((name) => `# wrangler secret put ${name}`).join("\n")}
|
|
51695
|
+
|
|
51696
|
+
# Deploy this Worker via:
|
|
51697
|
+
# wrangler deploy
|
|
51698
|
+
# Canonry intentionally does not declare a route in this file.
|
|
51699
|
+
# After deploy, attach this exact route in the Cloudflare dashboard:
|
|
51700
|
+
# ${hostname}/*
|
|
51701
|
+
${zoneHint}# Set the route Request limit failure mode to Fail open before activation.
|
|
51702
|
+
# Wrangler cannot configure this route toggle.
|
|
51703
|
+
`;
|
|
51704
|
+
}
|
|
51705
|
+
|
|
51706
|
+
// ../integration-cloudflare-worker/src/verify.ts
|
|
51707
|
+
import { createHmac, timingSafeEqual } from "crypto";
|
|
51708
|
+
var DEFAULT_MAX_AGE_SECONDS = 300;
|
|
51709
|
+
function verifyRequestSignature(opts) {
|
|
51710
|
+
const { timestamp, signature, secret } = opts;
|
|
51711
|
+
if (timestamp === "" || !/^-?\d+$/.test(timestamp)) {
|
|
51712
|
+
return { ok: false, reason: "timestamp_invalid" };
|
|
51713
|
+
}
|
|
51714
|
+
const ts = Number(timestamp);
|
|
51715
|
+
if (!Number.isFinite(ts)) return { ok: false, reason: "timestamp_invalid" };
|
|
51716
|
+
const now = opts.nowSeconds ?? Math.floor(Date.now() / 1e3);
|
|
51717
|
+
const maxAge = opts.maxAgeSeconds ?? DEFAULT_MAX_AGE_SECONDS;
|
|
51718
|
+
if (Math.abs(now - ts) > maxAge) return { ok: false, reason: "timestamp_expired" };
|
|
51719
|
+
if (signature === "" || !/^[0-9a-f]+$/i.test(signature)) {
|
|
51720
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51721
|
+
}
|
|
51722
|
+
let provided;
|
|
51723
|
+
try {
|
|
51724
|
+
provided = Buffer.from(signature, "hex");
|
|
51725
|
+
} catch {
|
|
51726
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51727
|
+
}
|
|
51728
|
+
let bodies;
|
|
51729
|
+
try {
|
|
51730
|
+
if ("body" in opts) {
|
|
51731
|
+
bodies = [opts.body];
|
|
51732
|
+
} else {
|
|
51733
|
+
const canonical = canonicalizeCloudflareJson(opts.payload);
|
|
51734
|
+
const legacy = JSON.stringify(opts.payload);
|
|
51735
|
+
bodies = [canonical];
|
|
51736
|
+
if (opts.acceptLegacyJson !== false && typeof legacy === "string" && legacy !== canonical) {
|
|
51737
|
+
bodies.push(legacy);
|
|
51738
|
+
}
|
|
51739
|
+
}
|
|
51740
|
+
} catch {
|
|
51741
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51742
|
+
}
|
|
51743
|
+
let comparableLength = false;
|
|
51744
|
+
for (const body of bodies) {
|
|
51745
|
+
const expected = createHmac("sha256", secret).update(`${timestamp}.${body}`).digest();
|
|
51746
|
+
if (provided.length !== expected.length) continue;
|
|
51747
|
+
comparableLength = true;
|
|
51748
|
+
if (timingSafeEqual(provided, expected)) return { ok: true };
|
|
51749
|
+
}
|
|
51750
|
+
if (!comparableLength) return { ok: false, reason: "signature_invalid" };
|
|
51751
|
+
return { ok: false, reason: "signature_mismatch" };
|
|
51752
|
+
}
|
|
51753
|
+
|
|
51754
|
+
// ../api-routes/src/traffic-event-ingest.ts
|
|
51755
|
+
import crypto39 from "crypto";
|
|
51756
|
+
import { and as and38, eq as eq46, gte as gte10, lt as lt7, lte as lte9, sql as sql18 } from "drizzle-orm";
|
|
51757
|
+
var DIRECT_PUSH_RECEIPT_TTL_MS = 10 * 6e4;
|
|
51758
|
+
function observedUtcHourBounds(observedAt) {
|
|
51759
|
+
const instant = new Date(observedAt);
|
|
51760
|
+
if (Number.isNaN(instant.getTime())) return null;
|
|
51761
|
+
instant.setUTCMinutes(0, 0, 0);
|
|
51762
|
+
const start = instant.toISOString();
|
|
51763
|
+
instant.setUTCHours(instant.getUTCHours() + 1);
|
|
51764
|
+
return { start, end: instant.toISOString() };
|
|
51765
|
+
}
|
|
51766
|
+
function writeTrafficEventBatch(opts) {
|
|
51767
|
+
const expiresAt = new Date(Date.parse(opts.receivedAt) + opts.receiptTtlMs).toISOString();
|
|
51768
|
+
let acceptedEvents = 0;
|
|
51769
|
+
let duplicateEvents = 0;
|
|
51770
|
+
let crawlerBucketRows = 0;
|
|
51771
|
+
let aiUserFetchBucketRows = 0;
|
|
51772
|
+
let aiReferralBucketRows = 0;
|
|
51773
|
+
let sampleRows = 0;
|
|
51774
|
+
opts.db.transaction((tx) => {
|
|
51775
|
+
tx.update(trafficSources).set({ updatedAt: sql18`${trafficSources.updatedAt}` }).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
51776
|
+
const source = tx.select().from(trafficSources).where(eq46(trafficSources.id, opts.sourceId)).get();
|
|
51777
|
+
opts.validateSource(source);
|
|
51778
|
+
tx.delete(trafficEventReceipts).where(and38(
|
|
51779
|
+
eq46(trafficEventReceipts.sourceId, opts.sourceId),
|
|
51780
|
+
lte9(trafficEventReceipts.expiresAt, opts.receivedAt)
|
|
51781
|
+
)).run();
|
|
51782
|
+
const claimedEvents = [];
|
|
51783
|
+
for (const event of opts.events) {
|
|
51784
|
+
const claim = tx.insert(trafficEventReceipts).values({
|
|
51785
|
+
sourceId: opts.sourceId,
|
|
51786
|
+
eventId: event.eventId,
|
|
51787
|
+
receivedAt: opts.receivedAt,
|
|
51788
|
+
expiresAt
|
|
51789
|
+
}).onConflictDoNothing().run();
|
|
51790
|
+
if (claim.changes === 1) claimedEvents.push(event);
|
|
51791
|
+
else duplicateEvents += 1;
|
|
51792
|
+
}
|
|
51793
|
+
acceptedEvents = claimedEvents.length;
|
|
51794
|
+
if (claimedEvents.length > 0) {
|
|
51795
|
+
const report = buildTrafficProbeReport(claimedEvents, { sampleLimit: claimedEvents.length });
|
|
51796
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
51797
|
+
const status = bucket.status ?? 0;
|
|
51798
|
+
tx.insert(crawlerEventsHourly).values({
|
|
51799
|
+
projectId: opts.projectId,
|
|
51800
|
+
sourceId: opts.sourceId,
|
|
51801
|
+
tsHour: bucket.tsHour,
|
|
51802
|
+
botId: bucket.botId,
|
|
51803
|
+
operator: bucket.operator,
|
|
51804
|
+
verificationStatus: bucket.verificationStatus,
|
|
51805
|
+
pathNormalized: bucket.pathNormalized,
|
|
51806
|
+
status,
|
|
51807
|
+
hits: bucket.hits,
|
|
51808
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51809
|
+
createdAt: opts.receivedAt,
|
|
51810
|
+
updatedAt: opts.receivedAt
|
|
51811
|
+
}).onConflictDoUpdate({
|
|
51812
|
+
target: [
|
|
51813
|
+
crawlerEventsHourly.projectId,
|
|
51814
|
+
crawlerEventsHourly.sourceId,
|
|
51815
|
+
crawlerEventsHourly.tsHour,
|
|
51816
|
+
crawlerEventsHourly.botId,
|
|
51817
|
+
crawlerEventsHourly.verificationStatus,
|
|
51818
|
+
crawlerEventsHourly.pathNormalized,
|
|
51819
|
+
crawlerEventsHourly.status
|
|
51820
|
+
],
|
|
51821
|
+
set: {
|
|
51822
|
+
hits: sql18`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
51823
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51824
|
+
updatedAt: opts.receivedAt
|
|
51825
|
+
}
|
|
51826
|
+
}).run();
|
|
51827
|
+
crawlerBucketRows += 1;
|
|
51828
|
+
}
|
|
51829
|
+
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
51830
|
+
const status = bucket.status ?? 0;
|
|
51831
|
+
tx.insert(aiUserFetchEventsHourly).values({
|
|
51832
|
+
projectId: opts.projectId,
|
|
51833
|
+
sourceId: opts.sourceId,
|
|
51834
|
+
tsHour: bucket.tsHour,
|
|
51835
|
+
botId: bucket.botId,
|
|
51836
|
+
operator: bucket.operator,
|
|
51837
|
+
verificationStatus: bucket.verificationStatus,
|
|
51838
|
+
pathNormalized: bucket.pathNormalized,
|
|
51839
|
+
status,
|
|
51840
|
+
hits: bucket.hits,
|
|
51841
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51842
|
+
createdAt: opts.receivedAt,
|
|
51843
|
+
updatedAt: opts.receivedAt
|
|
51844
|
+
}).onConflictDoUpdate({
|
|
51845
|
+
target: [
|
|
51846
|
+
aiUserFetchEventsHourly.projectId,
|
|
51847
|
+
aiUserFetchEventsHourly.sourceId,
|
|
51848
|
+
aiUserFetchEventsHourly.tsHour,
|
|
51849
|
+
aiUserFetchEventsHourly.botId,
|
|
51850
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
51851
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
51852
|
+
aiUserFetchEventsHourly.status
|
|
51853
|
+
],
|
|
51854
|
+
set: {
|
|
51855
|
+
hits: sql18`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
51856
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51857
|
+
updatedAt: opts.receivedAt
|
|
51858
|
+
}
|
|
51859
|
+
}).run();
|
|
51860
|
+
aiUserFetchBucketRows += 1;
|
|
51861
|
+
}
|
|
51862
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
51863
|
+
const status = bucket.status ?? 0;
|
|
51864
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
51865
|
+
projectId: opts.projectId,
|
|
51866
|
+
sourceId: opts.sourceId,
|
|
51867
|
+
tsHour: bucket.tsHour,
|
|
51868
|
+
product: bucket.product,
|
|
51869
|
+
operator: bucket.operator,
|
|
51870
|
+
sourceDomain: bucket.sourceDomain,
|
|
51871
|
+
evidenceType: bucket.evidenceType,
|
|
51872
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
51873
|
+
status,
|
|
51874
|
+
sessionsOrHits: bucket.hits,
|
|
51875
|
+
paidSessionsOrHits: bucket.paidHits,
|
|
51876
|
+
organicSessionsOrHits: bucket.organicHits,
|
|
51877
|
+
usersEstimated: null,
|
|
51878
|
+
createdAt: opts.receivedAt,
|
|
51879
|
+
updatedAt: opts.receivedAt
|
|
51880
|
+
}).onConflictDoUpdate({
|
|
51881
|
+
target: [
|
|
51882
|
+
aiReferralEventsHourly.projectId,
|
|
51883
|
+
aiReferralEventsHourly.sourceId,
|
|
51884
|
+
aiReferralEventsHourly.tsHour,
|
|
51885
|
+
aiReferralEventsHourly.product,
|
|
51886
|
+
aiReferralEventsHourly.sourceDomain,
|
|
51887
|
+
aiReferralEventsHourly.evidenceType,
|
|
51888
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
51889
|
+
aiReferralEventsHourly.status
|
|
51890
|
+
],
|
|
51891
|
+
set: {
|
|
51892
|
+
sessionsOrHits: sql18`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
51893
|
+
paidSessionsOrHits: sql18`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
51894
|
+
organicSessionsOrHits: sql18`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
51895
|
+
updatedAt: opts.receivedAt
|
|
51896
|
+
}
|
|
51897
|
+
}).run();
|
|
51898
|
+
aiReferralBucketRows += 1;
|
|
51899
|
+
}
|
|
51900
|
+
const hourlySampleLimit = Math.max(0, Math.floor(opts.sampleLimit));
|
|
51901
|
+
const remainingSamplesByHour = /* @__PURE__ */ new Map();
|
|
51902
|
+
for (const sample of report.samples) {
|
|
51903
|
+
const hour = observedUtcHourBounds(sample.observedAt);
|
|
51904
|
+
if (!hour || hourlySampleLimit === 0) continue;
|
|
51905
|
+
let remaining = remainingSamplesByHour.get(hour.start);
|
|
51906
|
+
if (remaining === void 0) {
|
|
51907
|
+
const existing = tx.select({ total: sql18`COUNT(*)` }).from(rawEventSamples).where(and38(
|
|
51908
|
+
eq46(rawEventSamples.sourceId, opts.sourceId),
|
|
51909
|
+
gte10(rawEventSamples.ts, hour.start),
|
|
51910
|
+
lt7(rawEventSamples.ts, hour.end)
|
|
51911
|
+
)).get();
|
|
51912
|
+
remaining = Math.max(0, hourlySampleLimit - Number(existing?.total ?? 0));
|
|
51913
|
+
}
|
|
51914
|
+
if (remaining === 0) {
|
|
51915
|
+
remainingSamplesByHour.set(hour.start, 0);
|
|
51916
|
+
continue;
|
|
51917
|
+
}
|
|
51918
|
+
const eventType = sample.crawler ? "crawler" : sample.aiUserFetch ? "ai_user_fetch" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
51919
|
+
const refererHost = (() => {
|
|
51920
|
+
if (!sample.referer) return null;
|
|
51921
|
+
try {
|
|
51922
|
+
return new URL(sample.referer).hostname;
|
|
51923
|
+
} catch {
|
|
51924
|
+
return null;
|
|
51925
|
+
}
|
|
51926
|
+
})();
|
|
51927
|
+
tx.insert(rawEventSamples).values({
|
|
51928
|
+
id: crypto39.randomUUID(),
|
|
51929
|
+
projectId: opts.projectId,
|
|
51930
|
+
sourceId: opts.sourceId,
|
|
51931
|
+
ts: sample.observedAt,
|
|
51932
|
+
eventType,
|
|
51933
|
+
ipHash: null,
|
|
51934
|
+
userAgent: sample.userAgent,
|
|
51935
|
+
pathNormalized: sample.pathNormalized,
|
|
51936
|
+
status: sample.status,
|
|
51937
|
+
refererHost,
|
|
51938
|
+
classifierDetailsJson: {
|
|
51939
|
+
crawler: sample.crawler,
|
|
51940
|
+
aiUserFetch: sample.aiUserFetch,
|
|
51941
|
+
aiReferral: sample.aiReferral
|
|
51942
|
+
},
|
|
51943
|
+
createdAt: opts.receivedAt
|
|
51944
|
+
}).run();
|
|
51945
|
+
sampleRows += 1;
|
|
51946
|
+
remainingSamplesByHour.set(hour.start, remaining - 1);
|
|
51947
|
+
}
|
|
51948
|
+
}
|
|
51949
|
+
tx.update(trafficSources).set(opts.sourceUpdate).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
51950
|
+
});
|
|
51951
|
+
return {
|
|
51952
|
+
acceptedEvents,
|
|
51953
|
+
duplicateEvents,
|
|
51954
|
+
crawlerBucketRows,
|
|
51955
|
+
aiUserFetchBucketRows,
|
|
51956
|
+
aiReferralBucketRows,
|
|
51957
|
+
sampleRows
|
|
51958
|
+
};
|
|
51959
|
+
}
|
|
51960
|
+
|
|
51112
51961
|
// ../api-routes/src/traffic.ts
|
|
51113
51962
|
var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
51114
51963
|
var DEFAULT_PAGE_SIZE3 = 1e3;
|
|
@@ -51125,6 +51974,126 @@ var DEFAULT_TRAFFIC_SYNC_CRON = "*/30 * * * *";
|
|
|
51125
51974
|
var MAX_BACKFILL_DAYS = 90;
|
|
51126
51975
|
var BACKFILL_MAX_PAGES = 1e3;
|
|
51127
51976
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
51977
|
+
var CLOUDFLARE_WORKER_VERSION = "1.0.0";
|
|
51978
|
+
var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
|
|
51979
|
+
var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
|
|
51980
|
+
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
|
|
51981
|
+
function sha256Hex3(value) {
|
|
51982
|
+
return crypto40.createHash("sha256").update(value).digest("hex");
|
|
51983
|
+
}
|
|
51984
|
+
function hashCloudflareBearerToken(value) {
|
|
51985
|
+
return sha256Hex3(value);
|
|
51986
|
+
}
|
|
51987
|
+
function timingSafeEqualHex(a, b) {
|
|
51988
|
+
if (!a || !b) return false;
|
|
51989
|
+
if (!SHA256_HEX_PATTERN.test(a) || !SHA256_HEX_PATTERN.test(b)) return false;
|
|
51990
|
+
const left = Buffer.from(a, "hex");
|
|
51991
|
+
const right = Buffer.from(b, "hex");
|
|
51992
|
+
return left.length === right.length && crypto40.timingSafeEqual(left, right);
|
|
51993
|
+
}
|
|
51994
|
+
function parseDirectPushCloudflareSourceConfig(config) {
|
|
51995
|
+
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
51996
|
+
return parsed.success ? parsed.data : null;
|
|
51997
|
+
}
|
|
51998
|
+
function isDirectPushCloudflareDeliveryMode(value) {
|
|
51999
|
+
return value === CloudflareTrafficDeliveryModes["direct-push"];
|
|
52000
|
+
}
|
|
52001
|
+
function authenticateCloudflareIngest(request, store) {
|
|
52002
|
+
if (!store) return null;
|
|
52003
|
+
const authHeader = request.headers.authorization;
|
|
52004
|
+
const sourceIdHeader = request.headers["x-canonry-source-id"];
|
|
52005
|
+
const timestampHeader = request.headers["x-canonry-timestamp"];
|
|
52006
|
+
const signatureHeader = request.headers["x-canonry-signature"];
|
|
52007
|
+
const bearerToken = typeof authHeader === "string" && authHeader.startsWith("Bearer ") ? authHeader.slice(7) : "";
|
|
52008
|
+
const sourceId = typeof sourceIdHeader === "string" ? sourceIdHeader : "";
|
|
52009
|
+
const timestamp = typeof timestampHeader === "string" ? timestampHeader : "";
|
|
52010
|
+
const signature = typeof signatureHeader === "string" ? signatureHeader : "";
|
|
52011
|
+
if (!bearerToken || !sourceId || !timestamp || !signature) return null;
|
|
52012
|
+
const credential = store.getConnectionBySourceId(sourceId);
|
|
52013
|
+
if (!credential || !isDirectPushCloudflareDeliveryMode(credential.deliveryMode)) return null;
|
|
52014
|
+
const bearerHash = sha256Hex3(bearerToken);
|
|
52015
|
+
if (!timingSafeEqualHex(bearerHash, sha256Hex3(credential.bearerToken))) return null;
|
|
52016
|
+
if (!verifyRequestSignature({
|
|
52017
|
+
timestamp,
|
|
52018
|
+
signature,
|
|
52019
|
+
payload: request.body,
|
|
52020
|
+
secret: credential.hmacSecret
|
|
52021
|
+
}).ok) return null;
|
|
52022
|
+
return { bearerHash, credential };
|
|
52023
|
+
}
|
|
52024
|
+
function cloudflareIngestRateLimitKey(request, store, db) {
|
|
52025
|
+
const authenticated = authenticateCloudflareIngest(request, store);
|
|
52026
|
+
const params = request.params;
|
|
52027
|
+
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
52028
|
+
const source = db.select().from(trafficSources).where(eq47(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
52029
|
+
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
52030
|
+
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
52031
|
+
}
|
|
52032
|
+
}
|
|
52033
|
+
return `cloudflare-invalid:${request.ip}`;
|
|
52034
|
+
}
|
|
52035
|
+
function assertCloudflareIngestUrlIsPublicHttps(value) {
|
|
52036
|
+
let url;
|
|
52037
|
+
try {
|
|
52038
|
+
url = new URL(value);
|
|
52039
|
+
} catch {
|
|
52040
|
+
throw validationError("Cloudflare ingest URL must be a valid public HTTPS URL");
|
|
52041
|
+
}
|
|
52042
|
+
const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
52043
|
+
const ipv4 = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
52044
|
+
const octets = ipv4?.slice(1).map(Number);
|
|
52045
|
+
const privateIpv4 = octets !== void 0 && (octets.some((part) => part > 255) || octets[0] === 0 || octets[0] === 10 || octets[0] === 127 || octets[0] === 169 && octets[1] === 254 || octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31 || octets[0] === 192 && octets[1] === 168);
|
|
52046
|
+
const privateIpv6 = hostname.includes(":") && (hostname === "::1" || hostname.startsWith("fc") || hostname.startsWith("fd") || hostname.startsWith("fe8") || hostname.startsWith("fe9") || hostname.startsWith("fea") || hostname.startsWith("feb"));
|
|
52047
|
+
const privateHostname = hostname === "localhost" || hostname.endsWith(".localhost") || hostname.endsWith(".local") || privateIpv6 || privateIpv4;
|
|
52048
|
+
if (url.protocol !== "https:" || privateHostname || url.username.length > 0 || url.password.length > 0 || url.search.length > 0 || url.hash.length > 0) {
|
|
52049
|
+
throw validationError(
|
|
52050
|
+
"Cloudflare ingest URL must be a credential-free public HTTPS URL without a query or fragment and reachable from the public Cloudflare edge; configure publicUrl to a public origin"
|
|
52051
|
+
);
|
|
52052
|
+
}
|
|
52053
|
+
}
|
|
52054
|
+
function normalizeDnsHostname(value) {
|
|
52055
|
+
const normalized = value.trim().toLowerCase().replace(/\.+$/, "");
|
|
52056
|
+
if (normalized.length === 0 || normalized.length > 253 || normalized.includes("*") || !normalized.includes(".")) return null;
|
|
52057
|
+
const valid = normalized.split(".").every(
|
|
52058
|
+
(label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label)
|
|
52059
|
+
);
|
|
52060
|
+
return valid ? normalized : null;
|
|
52061
|
+
}
|
|
52062
|
+
function resolveCloudflareWorkerRouteHost(value) {
|
|
52063
|
+
const trimmed = value.trim();
|
|
52064
|
+
const hasScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed);
|
|
52065
|
+
if (hasScheme && !/^https?:\/\//i.test(trimmed)) {
|
|
52066
|
+
throw validationError("Project canonicalDomain must use an HTTP or HTTPS URL");
|
|
52067
|
+
}
|
|
52068
|
+
let url;
|
|
52069
|
+
try {
|
|
52070
|
+
url = new URL(hasScheme ? trimmed : `https://${trimmed}`);
|
|
52071
|
+
} catch {
|
|
52072
|
+
throw validationError("Project canonicalDomain must resolve to one exact public hostname");
|
|
52073
|
+
}
|
|
52074
|
+
if (url.username.length > 0 || url.password.length > 0 || url.pathname !== "" && url.pathname !== "/" || url.search.length > 0 || url.hash.length > 0 || url.port.length > 0) {
|
|
52075
|
+
throw validationError(
|
|
52076
|
+
"Project canonicalDomain must be a bare hostname or root HTTP(S) URL without credentials, a non-default port, path, query, or fragment"
|
|
52077
|
+
);
|
|
52078
|
+
}
|
|
52079
|
+
const hostname = normalizeDnsHostname(url.hostname);
|
|
52080
|
+
if (!hostname || isIP(hostname) !== 0) {
|
|
52081
|
+
throw validationError("Project canonicalDomain must resolve to one exact public hostname without wildcards");
|
|
52082
|
+
}
|
|
52083
|
+
return hostname;
|
|
52084
|
+
}
|
|
52085
|
+
function normalizeCloudflareEventHost(value) {
|
|
52086
|
+
if (!value || /[/:@?#*]/.test(value)) return null;
|
|
52087
|
+
return normalizeDnsHostname(value);
|
|
52088
|
+
}
|
|
52089
|
+
function assertCloudflareIngestUrlOutsideWorkerRoute(ingestUrl, workerRouteHost) {
|
|
52090
|
+
const ingestHost = normalizeDnsHostname(new URL(ingestUrl).hostname);
|
|
52091
|
+
if (ingestHost && ingestHost === workerRouteHost) {
|
|
52092
|
+
throw validationError(
|
|
52093
|
+
"Cloudflare ingest URL hostname must be outside the generated Worker route hostname to prevent recursive ingestion"
|
|
52094
|
+
);
|
|
52095
|
+
}
|
|
52096
|
+
}
|
|
51128
52097
|
function emptyTrafficSeriesPoint(bucket) {
|
|
51129
52098
|
return { bucket, crawlerHits: 0, aiUserFetchHits: 0, aiReferralHits: 0 };
|
|
51130
52099
|
}
|
|
@@ -51205,8 +52174,8 @@ async function runBackfillTask(options) {
|
|
|
51205
52174
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
51206
52175
|
try {
|
|
51207
52176
|
app.db.transaction((tx) => {
|
|
51208
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
51209
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
52177
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq47(runs.id, runId)).run();
|
|
52178
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51210
52179
|
});
|
|
51211
52180
|
} catch {
|
|
51212
52181
|
}
|
|
@@ -51221,7 +52190,7 @@ async function runBackfillTask(options) {
|
|
|
51221
52190
|
if (allEvents.length === 0) {
|
|
51222
52191
|
const finishedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
51223
52192
|
try {
|
|
51224
|
-
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(
|
|
52193
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(eq47(runs.id, runId)).run();
|
|
51225
52194
|
} catch {
|
|
51226
52195
|
}
|
|
51227
52196
|
return;
|
|
@@ -51243,31 +52212,31 @@ async function runBackfillTask(options) {
|
|
|
51243
52212
|
try {
|
|
51244
52213
|
app.db.transaction((tx) => {
|
|
51245
52214
|
tx.delete(crawlerEventsHourly).where(
|
|
51246
|
-
|
|
51247
|
-
|
|
51248
|
-
|
|
51249
|
-
|
|
52215
|
+
and39(
|
|
52216
|
+
eq47(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
52217
|
+
gte11(crawlerEventsHourly.tsHour, windowStartIso),
|
|
52218
|
+
lte10(crawlerEventsHourly.tsHour, windowEndIso)
|
|
51250
52219
|
)
|
|
51251
52220
|
).run();
|
|
51252
52221
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
51253
|
-
|
|
51254
|
-
|
|
51255
|
-
|
|
51256
|
-
|
|
52222
|
+
and39(
|
|
52223
|
+
eq47(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
52224
|
+
gte11(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
52225
|
+
lte10(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
51257
52226
|
)
|
|
51258
52227
|
).run();
|
|
51259
52228
|
tx.delete(aiReferralEventsHourly).where(
|
|
51260
|
-
|
|
51261
|
-
|
|
51262
|
-
|
|
51263
|
-
|
|
52229
|
+
and39(
|
|
52230
|
+
eq47(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
52231
|
+
gte11(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
52232
|
+
lte10(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
51264
52233
|
)
|
|
51265
52234
|
).run();
|
|
51266
52235
|
tx.delete(rawEventSamples).where(
|
|
51267
|
-
|
|
51268
|
-
|
|
51269
|
-
|
|
51270
|
-
|
|
52236
|
+
and39(
|
|
52237
|
+
eq47(rawEventSamples.sourceId, sourceRow2.id),
|
|
52238
|
+
gte11(rawEventSamples.ts, windowStartIso),
|
|
52239
|
+
lte10(rawEventSamples.ts, windowEndIso)
|
|
51271
52240
|
)
|
|
51272
52241
|
).run();
|
|
51273
52242
|
for (const bucket of report.crawlerEventsHourly) {
|
|
@@ -51332,7 +52301,7 @@ async function runBackfillTask(options) {
|
|
|
51332
52301
|
}
|
|
51333
52302
|
})();
|
|
51334
52303
|
tx.insert(rawEventSamples).values({
|
|
51335
|
-
id:
|
|
52304
|
+
id: crypto40.randomUUID(),
|
|
51336
52305
|
projectId: project.id,
|
|
51337
52306
|
sourceId: sourceRow2.id,
|
|
51338
52307
|
ts: sample.observedAt,
|
|
@@ -51359,8 +52328,8 @@ async function runBackfillTask(options) {
|
|
|
51359
52328
|
lastEventIds: newRingBuffer,
|
|
51360
52329
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
51361
52330
|
updatedAt: finishedAt
|
|
51362
|
-
}).where(
|
|
51363
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
52331
|
+
}).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
52332
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq47(runs.id, runId)).run();
|
|
51364
52333
|
});
|
|
51365
52334
|
} catch (e) {
|
|
51366
52335
|
markFailed(`Backfill rollup write failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
@@ -51405,6 +52374,11 @@ async function trafficRoutes(app, opts) {
|
|
|
51405
52374
|
const pageSize = opts.defaultPageSize ?? DEFAULT_PAGE_SIZE3;
|
|
51406
52375
|
const maxPages = opts.defaultMaxPages ?? DEFAULT_MAX_PAGES4;
|
|
51407
52376
|
const sampleLimit = opts.defaultSampleLimit ?? DEFAULT_SAMPLE_LIMIT2;
|
|
52377
|
+
const checkCloudflareSourceRateLimit = app.createRateLimit({
|
|
52378
|
+
max: opts.cloudflareIngestRateLimitMax ?? DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX,
|
|
52379
|
+
timeWindow: "1 minute",
|
|
52380
|
+
keyGenerator: (request) => cloudflareIngestRateLimitKey(request, opts.cloudflareTrafficCredentialStore, app.db)
|
|
52381
|
+
});
|
|
51408
52382
|
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
51409
52383
|
const project = resolveProject(app.db, request.params.name);
|
|
51410
52384
|
const body = request.body ?? {};
|
|
@@ -51442,7 +52416,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51442
52416
|
createdAt: existing?.createdAt ?? now,
|
|
51443
52417
|
updatedAt: now
|
|
51444
52418
|
});
|
|
51445
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52419
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
51446
52420
|
const config = {
|
|
51447
52421
|
gcpProjectId,
|
|
51448
52422
|
serviceName: serviceName ?? null,
|
|
@@ -51458,10 +52432,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51458
52432
|
lastError: null,
|
|
51459
52433
|
configJson: config,
|
|
51460
52434
|
updatedAt: now
|
|
51461
|
-
}).where(
|
|
51462
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52435
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52436
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51463
52437
|
} else {
|
|
51464
|
-
const newId =
|
|
52438
|
+
const newId = crypto40.randomUUID();
|
|
51465
52439
|
app.db.insert(trafficSources).values({
|
|
51466
52440
|
id: newId,
|
|
51467
52441
|
projectId: project.id,
|
|
@@ -51476,7 +52450,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51476
52450
|
createdAt: now,
|
|
51477
52451
|
updatedAt: now
|
|
51478
52452
|
}).run();
|
|
51479
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52453
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51480
52454
|
}
|
|
51481
52455
|
writeAuditLog(app.db, {
|
|
51482
52456
|
projectId: project.id,
|
|
@@ -51528,7 +52502,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51528
52502
|
createdAt: existing?.createdAt ?? now,
|
|
51529
52503
|
updatedAt: now
|
|
51530
52504
|
});
|
|
51531
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52505
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
51532
52506
|
const config = { baseUrl, username };
|
|
51533
52507
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
51534
52508
|
let sourceRow2;
|
|
@@ -51539,10 +52513,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51539
52513
|
lastError: null,
|
|
51540
52514
|
configJson: config,
|
|
51541
52515
|
updatedAt: now
|
|
51542
|
-
}).where(
|
|
51543
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52516
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52517
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51544
52518
|
} else {
|
|
51545
|
-
const newId =
|
|
52519
|
+
const newId = crypto40.randomUUID();
|
|
51546
52520
|
app.db.insert(trafficSources).values({
|
|
51547
52521
|
id: newId,
|
|
51548
52522
|
projectId: project.id,
|
|
@@ -51557,7 +52531,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51557
52531
|
createdAt: now,
|
|
51558
52532
|
updatedAt: now
|
|
51559
52533
|
}).run();
|
|
51560
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52534
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51561
52535
|
}
|
|
51562
52536
|
writeAuditLog(app.db, {
|
|
51563
52537
|
projectId: project.id,
|
|
@@ -51611,7 +52585,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51611
52585
|
createdAt: existing?.createdAt ?? now,
|
|
51612
52586
|
updatedAt: now
|
|
51613
52587
|
});
|
|
51614
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52588
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.vercel && row.status !== TrafficSourceStatuses.archived);
|
|
51615
52589
|
const config = { projectId, teamId, environment };
|
|
51616
52590
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
51617
52591
|
const { sourceRow: sourceRow2, scheduleCreated } = app.db.transaction((tx) => {
|
|
@@ -51623,10 +52597,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51623
52597
|
lastError: null,
|
|
51624
52598
|
configJson: config,
|
|
51625
52599
|
updatedAt: now
|
|
51626
|
-
}).where(
|
|
51627
|
-
row = tx.select().from(trafficSources).where(
|
|
52600
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52601
|
+
row = tx.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51628
52602
|
} else {
|
|
51629
|
-
const newId =
|
|
52603
|
+
const newId = crypto40.randomUUID();
|
|
51630
52604
|
tx.insert(trafficSources).values({
|
|
51631
52605
|
id: newId,
|
|
51632
52606
|
projectId: project.id,
|
|
@@ -51649,18 +52623,18 @@ async function trafficRoutes(app, opts) {
|
|
|
51649
52623
|
createdAt: now,
|
|
51650
52624
|
updatedAt: now
|
|
51651
52625
|
}).run();
|
|
51652
|
-
row = tx.select().from(trafficSources).where(
|
|
52626
|
+
row = tx.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51653
52627
|
}
|
|
51654
52628
|
const existingSchedule = tx.select().from(schedules).where(
|
|
51655
|
-
|
|
51656
|
-
|
|
51657
|
-
|
|
52629
|
+
and39(
|
|
52630
|
+
eq47(schedules.projectId, project.id),
|
|
52631
|
+
eq47(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
51658
52632
|
)
|
|
51659
52633
|
).get();
|
|
51660
52634
|
let created = false;
|
|
51661
52635
|
if (!existingSchedule) {
|
|
51662
52636
|
tx.insert(schedules).values({
|
|
51663
|
-
id:
|
|
52637
|
+
id: crypto40.randomUUID(),
|
|
51664
52638
|
projectId: project.id,
|
|
51665
52639
|
kind: SchedulableRunKinds["traffic-sync"],
|
|
51666
52640
|
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
@@ -51701,9 +52675,256 @@ async function trafficRoutes(app, opts) {
|
|
|
51701
52675
|
}
|
|
51702
52676
|
return rowToDto(sourceRow2);
|
|
51703
52677
|
});
|
|
52678
|
+
app.post("/projects/:name/traffic/connect/cloudflare", async (request) => {
|
|
52679
|
+
const project = resolveProject(app.db, request.params.name);
|
|
52680
|
+
if (!opts.cloudflareTrafficCredentialStore) {
|
|
52681
|
+
throw validationError("Cloudflare traffic credential storage is not configured for this deployment");
|
|
52682
|
+
}
|
|
52683
|
+
if (!opts.cloudflareTrafficIngestUrl) {
|
|
52684
|
+
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
52685
|
+
}
|
|
52686
|
+
const credentialStore = opts.cloudflareTrafficCredentialStore;
|
|
52687
|
+
const parsed = trafficConnectCloudflareRequestSchema.safeParse(request.body ?? {});
|
|
52688
|
+
if (!parsed.success) {
|
|
52689
|
+
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
52690
|
+
}
|
|
52691
|
+
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
52692
|
+
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
52693
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
52694
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find(
|
|
52695
|
+
(row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived
|
|
52696
|
+
);
|
|
52697
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
52698
|
+
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
52699
|
+
if (activeSource && !parsedActiveConfig) {
|
|
52700
|
+
throw validationError("Existing Cloudflare source is not configured for direct-push delivery");
|
|
52701
|
+
}
|
|
52702
|
+
const previousCredential = credentialStore.getConnection(project.name);
|
|
52703
|
+
if (previousCredential && !isDirectPushCloudflareDeliveryMode(previousCredential.deliveryMode)) {
|
|
52704
|
+
throw validationError(
|
|
52705
|
+
`Cloudflare credential delivery mode "${previousCredential.deliveryMode}" does not match "${deliveryMode}"`
|
|
52706
|
+
);
|
|
52707
|
+
}
|
|
52708
|
+
const reusableCredential = previousCredential?.sourceId === sourceId ? previousCredential : void 0;
|
|
52709
|
+
const previousZoneId = reusableCredential?.zoneId ?? parsedActiveConfig?.zoneId ?? null;
|
|
52710
|
+
const previousAccountId = reusableCredential?.accountId ?? parsedActiveConfig?.accountId ?? null;
|
|
52711
|
+
const effectiveZoneId = zoneId ?? previousZoneId;
|
|
52712
|
+
const effectiveAccountId = accountId ?? previousAccountId;
|
|
52713
|
+
const bearerToken = reusableCredential?.bearerToken ?? `cnry_cfw_${crypto40.randomBytes(32).toString("hex")}`;
|
|
52714
|
+
const hmacSecret = reusableCredential?.hmacSecret ?? crypto40.randomBytes(32).toString("hex");
|
|
52715
|
+
const ingestTokenHash = sha256Hex3(bearerToken);
|
|
52716
|
+
const ingestUrl = opts.cloudflareTrafficIngestUrl.replace("{name}", encodeURIComponent(project.name));
|
|
52717
|
+
assertCloudflareIngestUrlIsPublicHttps(ingestUrl);
|
|
52718
|
+
assertCloudflareIngestUrlOutsideWorkerRoute(ingestUrl, workerRouteHost);
|
|
52719
|
+
const workerVersion = CLOUDFLARE_WORKER_VERSION;
|
|
52720
|
+
const workerScript = generateWorkerScript({
|
|
52721
|
+
deliveryMode,
|
|
52722
|
+
workerVersion,
|
|
52723
|
+
botList: DEFAULT_BOT_LIST
|
|
52724
|
+
});
|
|
52725
|
+
const wranglerToml = generateWranglerToml({
|
|
52726
|
+
deliveryMode,
|
|
52727
|
+
sourceId,
|
|
52728
|
+
hostname: workerRouteHost,
|
|
52729
|
+
ingestUrl,
|
|
52730
|
+
workerVersion,
|
|
52731
|
+
zoneId: effectiveZoneId,
|
|
52732
|
+
accountId: effectiveAccountId
|
|
52733
|
+
});
|
|
52734
|
+
const config = {
|
|
52735
|
+
schemaVersion: 1,
|
|
52736
|
+
deliveryMode,
|
|
52737
|
+
workerVersion,
|
|
52738
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
52739
|
+
zoneId: effectiveZoneId,
|
|
52740
|
+
accountId: effectiveAccountId
|
|
52741
|
+
};
|
|
52742
|
+
const fallbackName = displayName ?? activeSource?.displayName ?? `Cloudflare \xB7 ${effectiveZoneId ?? sourceId.slice(0, 8)}`;
|
|
52743
|
+
credentialStore.upsertConnection({
|
|
52744
|
+
projectName: project.name,
|
|
52745
|
+
deliveryMode,
|
|
52746
|
+
sourceId,
|
|
52747
|
+
bearerToken,
|
|
52748
|
+
hmacSecret,
|
|
52749
|
+
workerVersion,
|
|
52750
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
52751
|
+
zoneId: effectiveZoneId,
|
|
52752
|
+
accountId: effectiveAccountId,
|
|
52753
|
+
createdAt: previousCredential?.createdAt ?? activeSource?.createdAt ?? now,
|
|
52754
|
+
updatedAt: now
|
|
52755
|
+
});
|
|
52756
|
+
let sourceRow2;
|
|
52757
|
+
try {
|
|
52758
|
+
if (activeSource) {
|
|
52759
|
+
app.db.update(trafficSources).set({
|
|
52760
|
+
displayName: fallbackName,
|
|
52761
|
+
status: TrafficSourceStatuses.connected,
|
|
52762
|
+
lastError: null,
|
|
52763
|
+
configJson: config,
|
|
52764
|
+
ingestTokenHash,
|
|
52765
|
+
updatedAt: now
|
|
52766
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52767
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
52768
|
+
} else {
|
|
52769
|
+
app.db.insert(trafficSources).values({
|
|
52770
|
+
id: sourceId,
|
|
52771
|
+
projectId: project.id,
|
|
52772
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
52773
|
+
displayName: fallbackName,
|
|
52774
|
+
status: TrafficSourceStatuses.connected,
|
|
52775
|
+
// Seed `lastSyncedAt` to NOW so the `traffic.source.recent-data`
|
|
52776
|
+
// doctor check has a non-null baseline. Successful ingest advances
|
|
52777
|
+
// it as the receiver's last-activity timestamp.
|
|
52778
|
+
lastSyncedAt: now,
|
|
52779
|
+
lastCursor: null,
|
|
52780
|
+
lastError: null,
|
|
52781
|
+
archivedAt: null,
|
|
52782
|
+
configJson: config,
|
|
52783
|
+
ingestTokenHash,
|
|
52784
|
+
createdAt: now,
|
|
52785
|
+
updatedAt: now
|
|
52786
|
+
}).run();
|
|
52787
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, sourceId)).get();
|
|
52788
|
+
}
|
|
52789
|
+
} catch (err) {
|
|
52790
|
+
try {
|
|
52791
|
+
if (previousCredential) {
|
|
52792
|
+
credentialStore.upsertConnection(previousCredential);
|
|
52793
|
+
} else {
|
|
52794
|
+
credentialStore.deleteConnection(project.name);
|
|
52795
|
+
}
|
|
52796
|
+
} catch {
|
|
52797
|
+
}
|
|
52798
|
+
throw err;
|
|
52799
|
+
}
|
|
52800
|
+
writeAuditLog(app.db, {
|
|
52801
|
+
projectId: project.id,
|
|
52802
|
+
actor: "api",
|
|
52803
|
+
action: "traffic.cloudflare.connected",
|
|
52804
|
+
entityType: "traffic_source",
|
|
52805
|
+
entityId: sourceRow2.id
|
|
52806
|
+
});
|
|
52807
|
+
const routeInstruction = effectiveZoneId ? ` 4. Deploy with \`wrangler deploy\`; wrangler.toml records zone ${effectiveZoneId} but does not claim a route` : " 4. Deploy with `wrangler deploy`; wrangler.toml intentionally does not claim a route";
|
|
52808
|
+
const instructions = [
|
|
52809
|
+
"Deploy this Worker to your Cloudflare zone:",
|
|
52810
|
+
" 1. Save worker.js and wrangler.toml; neither file contains credentials",
|
|
52811
|
+
" 2. Install the required bearer and HMAC Worker secret bindings locally; never paste their values into source or chat",
|
|
52812
|
+
` 3. Inspect existing Cloudflare Worker routes first; confirm the generated \`${workerRouteHost}/*\` catch-all will not replace or collide with another Worker`,
|
|
52813
|
+
routeInstruction,
|
|
52814
|
+
` 5. In the Cloudflare Dashboard route form, select \`${workerRouteHost}/*\` and this Worker`,
|
|
52815
|
+
" 6. Set Request limit failure mode to Fail open, then save the route; Wrangler cannot set this toggle",
|
|
52816
|
+
" 7. Keep the Canonry ingest hostname outside this Worker route to avoid recursion",
|
|
52817
|
+
"",
|
|
52818
|
+
`Source id: ${sourceRow2.id}`,
|
|
52819
|
+
"After deploy, check `canonry doctor --project " + project.name + "` to confirm events are arriving."
|
|
52820
|
+
].join("\n");
|
|
52821
|
+
return {
|
|
52822
|
+
sourceId: sourceRow2.id,
|
|
52823
|
+
deliveryMode,
|
|
52824
|
+
workerScript,
|
|
52825
|
+
wranglerToml,
|
|
52826
|
+
workerVersion,
|
|
52827
|
+
instructions
|
|
52828
|
+
};
|
|
52829
|
+
});
|
|
52830
|
+
app.post("/projects/:name/traffic/cloudflare/ingest", {
|
|
52831
|
+
bodyLimit: CLOUDFLARE_INGEST_BODY_LIMIT,
|
|
52832
|
+
preHandler: async (request, reply) => {
|
|
52833
|
+
const key = cloudflareIngestRateLimitKey(
|
|
52834
|
+
request,
|
|
52835
|
+
opts.cloudflareTrafficCredentialStore,
|
|
52836
|
+
app.db
|
|
52837
|
+
);
|
|
52838
|
+
if (key.startsWith("cloudflare-invalid:")) return;
|
|
52839
|
+
const result = await checkCloudflareSourceRateLimit(request);
|
|
52840
|
+
if (result.isAllowed || !result.isExceeded) return;
|
|
52841
|
+
reply.header("retry-after", result.ttlInSeconds);
|
|
52842
|
+
return reply.status(429).send({
|
|
52843
|
+
statusCode: 429,
|
|
52844
|
+
error: "Too Many Requests",
|
|
52845
|
+
message: `Rate limit exceeded, retry in ${result.ttlInSeconds} second(s)`
|
|
52846
|
+
});
|
|
52847
|
+
},
|
|
52848
|
+
config: {
|
|
52849
|
+
rateLimit: {
|
|
52850
|
+
hook: "onRequest",
|
|
52851
|
+
max: opts.cloudflareIngestIpRateLimitMax ?? DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX,
|
|
52852
|
+
timeWindow: "1 minute",
|
|
52853
|
+
groupId: "cloudflare-direct-push-ip",
|
|
52854
|
+
keyGenerator: (request) => `cloudflare-ingest-ip:${request.ip}`
|
|
52855
|
+
}
|
|
52856
|
+
}
|
|
52857
|
+
}, async (request, reply) => {
|
|
52858
|
+
const authenticated = authenticateCloudflareIngest(
|
|
52859
|
+
request,
|
|
52860
|
+
opts.cloudflareTrafficCredentialStore
|
|
52861
|
+
);
|
|
52862
|
+
if (!authenticated || authenticated.credential.projectName !== request.params.name || !isDirectPushCloudflareDeliveryMode(authenticated.credential.deliveryMode)) {
|
|
52863
|
+
throw authRequired();
|
|
52864
|
+
}
|
|
52865
|
+
const { bearerHash, credential } = authenticated;
|
|
52866
|
+
const sourceId = credential.sourceId;
|
|
52867
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, sourceId)).get();
|
|
52868
|
+
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status === TrafficSourceStatuses.archived || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
52869
|
+
throw authRequired();
|
|
52870
|
+
}
|
|
52871
|
+
const project = resolveProject(app.db, request.params.name);
|
|
52872
|
+
if (sourceRow2.projectId !== project.id) throw authRequired();
|
|
52873
|
+
const parsed = cloudflareWorkerIngestRequestSchema.safeParse(request.body);
|
|
52874
|
+
if (!parsed.success) {
|
|
52875
|
+
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
52876
|
+
}
|
|
52877
|
+
const { workerVersion, events } = parsed.data;
|
|
52878
|
+
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
52879
|
+
const wrongHostIndex = events.findIndex(
|
|
52880
|
+
(event) => normalizeCloudflareEventHost(event.host) !== canonicalHost
|
|
52881
|
+
);
|
|
52882
|
+
if (wrongHostIndex !== -1) {
|
|
52883
|
+
throw validationError(
|
|
52884
|
+
`Cloudflare event at index ${wrongHostIndex} does not belong to project domain "${project.canonicalDomain}"`
|
|
52885
|
+
);
|
|
52886
|
+
}
|
|
52887
|
+
const normalized = [];
|
|
52888
|
+
let normalizationDrops = 0;
|
|
52889
|
+
for (const event of events) {
|
|
52890
|
+
const result = normalizeCloudflareWorkerEvent(event);
|
|
52891
|
+
if (result) normalized.push(result);
|
|
52892
|
+
else normalizationDrops += 1;
|
|
52893
|
+
}
|
|
52894
|
+
const receivedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
52895
|
+
const writeResult = writeTrafficEventBatch({
|
|
52896
|
+
db: app.db,
|
|
52897
|
+
projectId: project.id,
|
|
52898
|
+
sourceId,
|
|
52899
|
+
events: normalized,
|
|
52900
|
+
receivedAt,
|
|
52901
|
+
receiptTtlMs: DIRECT_PUSH_RECEIPT_TTL_MS,
|
|
52902
|
+
sampleLimit,
|
|
52903
|
+
validateSource: (latestRow) => {
|
|
52904
|
+
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status === TrafficSourceStatuses.archived || !parseDirectPushCloudflareSourceConfig(latestRow.configJson) || !timingSafeEqualHex(bearerHash, latestRow.ingestTokenHash)) {
|
|
52905
|
+
throw authRequired();
|
|
52906
|
+
}
|
|
52907
|
+
},
|
|
52908
|
+
sourceUpdate: {
|
|
52909
|
+
lastWorkerVersion: workerVersion,
|
|
52910
|
+
lastSyncedAt: receivedAt,
|
|
52911
|
+
lastError: null,
|
|
52912
|
+
updatedAt: receivedAt
|
|
52913
|
+
}
|
|
52914
|
+
});
|
|
52915
|
+
return reply.status(200).send({
|
|
52916
|
+
acceptedEvents: writeResult.acceptedEvents,
|
|
52917
|
+
droppedEvents: normalizationDrops + writeResult.duplicateEvents,
|
|
52918
|
+
workerVersionAck: workerVersion,
|
|
52919
|
+
crawlerBucketRows: writeResult.crawlerBucketRows,
|
|
52920
|
+
aiUserFetchBucketRows: writeResult.aiUserFetchBucketRows,
|
|
52921
|
+
aiReferralBucketRows: writeResult.aiReferralBucketRows,
|
|
52922
|
+
sampleRows: writeResult.sampleRows
|
|
52923
|
+
});
|
|
52924
|
+
});
|
|
51704
52925
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
51705
52926
|
const project = resolveProject(app.db, request.params.name);
|
|
51706
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52927
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
51707
52928
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
51708
52929
|
throw notFound("Traffic source", request.params.id);
|
|
51709
52930
|
}
|
|
@@ -51715,7 +52936,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51715
52936
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
51716
52937
|
const startedAt = windowEnd.toISOString();
|
|
51717
52938
|
const syncStartedAtMs = windowEnd.getTime();
|
|
51718
|
-
const runId =
|
|
52939
|
+
const runId = crypto40.randomUUID();
|
|
51719
52940
|
app.db.insert(runs).values({
|
|
51720
52941
|
id: runId,
|
|
51721
52942
|
projectId: project.id,
|
|
@@ -51729,8 +52950,8 @@ async function trafficRoutes(app, opts) {
|
|
|
51729
52950
|
const markFailed = (msg, errorCode) => {
|
|
51730
52951
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
51731
52952
|
app.db.transaction((tx) => {
|
|
51732
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
51733
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
52953
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq47(runs.id, runId)).run();
|
|
52954
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51734
52955
|
});
|
|
51735
52956
|
try {
|
|
51736
52957
|
opts.onTrafficSynced?.({
|
|
@@ -51811,7 +53032,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51811
53032
|
}
|
|
51812
53033
|
const credential = credentialStore.getConnection(project.name);
|
|
51813
53034
|
if (!credential) {
|
|
51814
|
-
app.db.delete(runs).where(
|
|
53035
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51815
53036
|
throw validationError(
|
|
51816
53037
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
51817
53038
|
);
|
|
@@ -51860,12 +53081,12 @@ async function trafficRoutes(app, opts) {
|
|
|
51860
53081
|
auditAction = "traffic.vercel.synced";
|
|
51861
53082
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
51862
53083
|
if (!credentialStore) {
|
|
51863
|
-
app.db.delete(runs).where(
|
|
53084
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51864
53085
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
51865
53086
|
}
|
|
51866
53087
|
const credential = credentialStore.getConnection(project.name);
|
|
51867
53088
|
if (!credential) {
|
|
51868
|
-
app.db.delete(runs).where(
|
|
53089
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51869
53090
|
throw validationError(
|
|
51870
53091
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
51871
53092
|
);
|
|
@@ -51883,7 +53104,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51883
53104
|
if (cappedStartMs > clampedStartMs) {
|
|
51884
53105
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
51885
53106
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
51886
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
53107
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51887
53108
|
request.log.warn(
|
|
51888
53109
|
{
|
|
51889
53110
|
sourceId: sourceRow2.id,
|
|
@@ -51968,7 +53189,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51968
53189
|
let aiReferralHitsCount = 0;
|
|
51969
53190
|
let unknownHitsCount = 0;
|
|
51970
53191
|
app.db.transaction((tx) => {
|
|
51971
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
53192
|
+
const latestRow = tx.select().from(trafficSources).where(eq47(trafficSources.id, sourceRow2.id)).get();
|
|
51972
53193
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
51973
53194
|
const seenEventIds = new Set(previousIds);
|
|
51974
53195
|
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
@@ -52016,7 +53237,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52016
53237
|
crawlerEventsHourly.status
|
|
52017
53238
|
],
|
|
52018
53239
|
set: {
|
|
52019
|
-
hits:
|
|
53240
|
+
hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
52020
53241
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
52021
53242
|
updatedAt: finishedAt
|
|
52022
53243
|
}
|
|
@@ -52049,7 +53270,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52049
53270
|
aiUserFetchEventsHourly.status
|
|
52050
53271
|
],
|
|
52051
53272
|
set: {
|
|
52052
|
-
hits:
|
|
53273
|
+
hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
52053
53274
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
52054
53275
|
updatedAt: finishedAt
|
|
52055
53276
|
}
|
|
@@ -52089,9 +53310,9 @@ async function trafficRoutes(app, opts) {
|
|
|
52089
53310
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
52090
53311
|
// already there rather than overwriting a single label.
|
|
52091
53312
|
set: {
|
|
52092
|
-
sessionsOrHits:
|
|
52093
|
-
paidSessionsOrHits:
|
|
52094
|
-
organicSessionsOrHits:
|
|
53313
|
+
sessionsOrHits: sql19`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
53314
|
+
paidSessionsOrHits: sql19`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
53315
|
+
organicSessionsOrHits: sql19`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
52095
53316
|
updatedAt: finishedAt
|
|
52096
53317
|
}
|
|
52097
53318
|
}).run();
|
|
@@ -52108,7 +53329,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52108
53329
|
}
|
|
52109
53330
|
})();
|
|
52110
53331
|
tx.insert(rawEventSamples).values({
|
|
52111
|
-
id:
|
|
53332
|
+
id: crypto40.randomUUID(),
|
|
52112
53333
|
projectId: project.id,
|
|
52113
53334
|
sourceId: sourceRow2.id,
|
|
52114
53335
|
ts: sample.observedAt,
|
|
@@ -52156,8 +53377,8 @@ async function trafficRoutes(app, opts) {
|
|
|
52156
53377
|
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
52157
53378
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
52158
53379
|
}
|
|
52159
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
52160
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
53380
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
53381
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq47(runs.id, runId)).run();
|
|
52161
53382
|
});
|
|
52162
53383
|
writeAuditLog(app.db, {
|
|
52163
53384
|
projectId: project.id,
|
|
@@ -52209,7 +53430,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52209
53430
|
});
|
|
52210
53431
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
52211
53432
|
const project = resolveProject(app.db, request.params.name);
|
|
52212
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53433
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
52213
53434
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
52214
53435
|
throw notFound("Traffic source", request.params.id);
|
|
52215
53436
|
}
|
|
@@ -52359,7 +53580,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52359
53580
|
};
|
|
52360
53581
|
}
|
|
52361
53582
|
const startedAt = windowEnd.toISOString();
|
|
52362
|
-
const runId =
|
|
53583
|
+
const runId = crypto40.randomUUID();
|
|
52363
53584
|
app.db.insert(runs).values({
|
|
52364
53585
|
id: runId,
|
|
52365
53586
|
projectId: project.id,
|
|
@@ -52395,40 +53616,40 @@ async function trafficRoutes(app, opts) {
|
|
|
52395
53616
|
function buildSourceDetail(projectId, row, since) {
|
|
52396
53617
|
const crawlerPathRows = app.db.select({
|
|
52397
53618
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
52398
|
-
hits:
|
|
53619
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
52399
53620
|
}).from(crawlerEventsHourly).where(
|
|
52400
|
-
|
|
52401
|
-
|
|
52402
|
-
|
|
53621
|
+
and39(
|
|
53622
|
+
eq47(crawlerEventsHourly.sourceId, row.id),
|
|
53623
|
+
gte11(crawlerEventsHourly.tsHour, since)
|
|
52403
53624
|
)
|
|
52404
53625
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
52405
53626
|
const crawlerSegments = segmentCrawlerHits(
|
|
52406
53627
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
52407
53628
|
);
|
|
52408
53629
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
52409
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
52410
|
-
|
|
52411
|
-
|
|
52412
|
-
|
|
53630
|
+
const aiUserFetchTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
53631
|
+
and39(
|
|
53632
|
+
eq47(aiUserFetchEventsHourly.sourceId, row.id),
|
|
53633
|
+
gte11(aiUserFetchEventsHourly.tsHour, since)
|
|
52413
53634
|
)
|
|
52414
53635
|
).get();
|
|
52415
|
-
const aiTotals = app.db.select({ total:
|
|
52416
|
-
|
|
52417
|
-
|
|
52418
|
-
|
|
53636
|
+
const aiTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
53637
|
+
and39(
|
|
53638
|
+
eq47(aiReferralEventsHourly.sourceId, row.id),
|
|
53639
|
+
gte11(aiReferralEventsHourly.tsHour, since)
|
|
52419
53640
|
)
|
|
52420
53641
|
).get();
|
|
52421
|
-
const sampleTotals = app.db.select({ total:
|
|
52422
|
-
|
|
52423
|
-
|
|
52424
|
-
|
|
53642
|
+
const sampleTotals = app.db.select({ total: sql19`COUNT(*)` }).from(rawEventSamples).where(
|
|
53643
|
+
and39(
|
|
53644
|
+
eq47(rawEventSamples.sourceId, row.id),
|
|
53645
|
+
gte11(rawEventSamples.ts, since)
|
|
52425
53646
|
)
|
|
52426
53647
|
).get();
|
|
52427
53648
|
const latestRun = app.db.select().from(runs).where(
|
|
52428
|
-
|
|
52429
|
-
|
|
52430
|
-
|
|
52431
|
-
|
|
53649
|
+
and39(
|
|
53650
|
+
eq47(runs.projectId, projectId),
|
|
53651
|
+
eq47(runs.kind, RunKinds["traffic-sync"]),
|
|
53652
|
+
eq47(runs.sourceId, row.id)
|
|
52432
53653
|
)
|
|
52433
53654
|
).orderBy(desc22(runs.startedAt)).limit(1).get();
|
|
52434
53655
|
return {
|
|
@@ -52459,7 +53680,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52459
53680
|
"`advanceToNow` must be `true`. There is no implicit reset."
|
|
52460
53681
|
);
|
|
52461
53682
|
}
|
|
52462
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53683
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(and39(eq47(trafficSources.projectId, project.id), eq47(trafficSources.id, request.params.id))).get();
|
|
52463
53684
|
if (!sourceRow2) {
|
|
52464
53685
|
throw notFound("traffic source", request.params.id);
|
|
52465
53686
|
}
|
|
@@ -52476,7 +53697,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52476
53697
|
status: TrafficSourceStatuses.connected,
|
|
52477
53698
|
lastError: null,
|
|
52478
53699
|
updatedAt: now
|
|
52479
|
-
}).where(
|
|
53700
|
+
}).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
52480
53701
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
52481
53702
|
projectId: project.id,
|
|
52482
53703
|
actor: "api",
|
|
@@ -52484,20 +53705,20 @@ async function trafficRoutes(app, opts) {
|
|
|
52484
53705
|
entityType: "traffic_source",
|
|
52485
53706
|
entityId: sourceRow2.id
|
|
52486
53707
|
}));
|
|
52487
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
53708
|
+
updatedRow = tx.select().from(trafficSources).where(eq47(trafficSources.id, sourceRow2.id)).get();
|
|
52488
53709
|
});
|
|
52489
53710
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
52490
53711
|
});
|
|
52491
53712
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
52492
53713
|
const project = resolveProject(app.db, request.params.name);
|
|
52493
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
53714
|
+
const rows = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
52494
53715
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
52495
53716
|
const response = { sources };
|
|
52496
53717
|
return response;
|
|
52497
53718
|
});
|
|
52498
53719
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
52499
53720
|
const project = resolveProject(app.db, request.params.name);
|
|
52500
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
53721
|
+
const rows = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
52501
53722
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
52502
53723
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
52503
53724
|
const response = { sources };
|
|
@@ -52507,7 +53728,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52507
53728
|
"/projects/:name/traffic/sources/:id",
|
|
52508
53729
|
async (request) => {
|
|
52509
53730
|
const project = resolveProject(app.db, request.params.name);
|
|
52510
|
-
const row = app.db.select().from(trafficSources).where(
|
|
53731
|
+
const row = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
52511
53732
|
if (!row || row.projectId !== project.id) {
|
|
52512
53733
|
throw notFound("Traffic source", request.params.id);
|
|
52513
53734
|
}
|
|
@@ -52572,26 +53793,26 @@ async function trafficRoutes(app, opts) {
|
|
|
52572
53793
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
52573
53794
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
52574
53795
|
const crawlerFilters = [
|
|
52575
|
-
|
|
52576
|
-
|
|
52577
|
-
|
|
53796
|
+
eq47(crawlerEventsHourly.projectId, project.id),
|
|
53797
|
+
gte11(crawlerEventsHourly.tsHour, sinceIso),
|
|
53798
|
+
lte10(crawlerEventsHourly.tsHour, untilIso)
|
|
52578
53799
|
];
|
|
52579
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
52580
|
-
const crawlerWhere =
|
|
53800
|
+
if (sourceIdParam) crawlerFilters.push(eq47(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
53801
|
+
const crawlerWhere = and39(...crawlerFilters);
|
|
52581
53802
|
const pathTotals = app.db.select({
|
|
52582
53803
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
52583
|
-
hits:
|
|
52584
|
-
rows:
|
|
53804
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
53805
|
+
rows: sql19`COUNT(*)`
|
|
52585
53806
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
52586
53807
|
crawlerSegments = segmentCrawlerHits(
|
|
52587
53808
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
52588
53809
|
);
|
|
52589
53810
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
52590
53811
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
52591
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53812
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
52592
53813
|
const crawlerSeries = app.db.select({
|
|
52593
53814
|
bucket: crawlerSeriesBucket,
|
|
52594
|
-
hits:
|
|
53815
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
52595
53816
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
52596
53817
|
for (const row of crawlerSeries) {
|
|
52597
53818
|
trafficSeriesPoint(seriesByBucket, row.bucket).crawlerHits = Number(row.hits);
|
|
@@ -52614,22 +53835,22 @@ async function trafficRoutes(app, opts) {
|
|
|
52614
53835
|
}
|
|
52615
53836
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
52616
53837
|
const userFetchFilters = [
|
|
52617
|
-
|
|
52618
|
-
|
|
52619
|
-
|
|
53838
|
+
eq47(aiUserFetchEventsHourly.projectId, project.id),
|
|
53839
|
+
gte11(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
53840
|
+
lte10(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
52620
53841
|
];
|
|
52621
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
52622
|
-
const userFetchWhere =
|
|
53842
|
+
if (sourceIdParam) userFetchFilters.push(eq47(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
53843
|
+
const userFetchWhere = and39(...userFetchFilters);
|
|
52623
53844
|
const total = app.db.select({
|
|
52624
|
-
total:
|
|
52625
|
-
rows:
|
|
53845
|
+
total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
53846
|
+
rows: sql19`COUNT(*)`
|
|
52626
53847
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
52627
53848
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
52628
53849
|
totalEventRows += Number(total?.rows ?? 0);
|
|
52629
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53850
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
52630
53851
|
const userFetchSeries = app.db.select({
|
|
52631
53852
|
bucket: userFetchSeriesBucket,
|
|
52632
|
-
hits:
|
|
53853
|
+
hits: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
52633
53854
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
52634
53855
|
for (const row of userFetchSeries) {
|
|
52635
53856
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -52651,17 +53872,17 @@ async function trafficRoutes(app, opts) {
|
|
|
52651
53872
|
}
|
|
52652
53873
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
52653
53874
|
const aiFilters = [
|
|
52654
|
-
|
|
52655
|
-
|
|
52656
|
-
|
|
53875
|
+
eq47(aiReferralEventsHourly.projectId, project.id),
|
|
53876
|
+
gte11(aiReferralEventsHourly.tsHour, sinceIso),
|
|
53877
|
+
lte10(aiReferralEventsHourly.tsHour, untilIso)
|
|
52657
53878
|
];
|
|
52658
|
-
if (sourceIdParam) aiFilters.push(
|
|
52659
|
-
const aiWhere =
|
|
53879
|
+
if (sourceIdParam) aiFilters.push(eq47(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
53880
|
+
const aiWhere = and39(...aiFilters);
|
|
52660
53881
|
const total = app.db.select({
|
|
52661
|
-
total:
|
|
52662
|
-
paid:
|
|
52663
|
-
organic:
|
|
52664
|
-
rows:
|
|
53882
|
+
total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
53883
|
+
paid: sql19`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
53884
|
+
organic: sql19`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`,
|
|
53885
|
+
rows: sql19`COUNT(*)`
|
|
52665
53886
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
52666
53887
|
aiReferralCounts = aiReferralClassCounts(
|
|
52667
53888
|
Number(total?.total ?? 0),
|
|
@@ -52669,10 +53890,10 @@ async function trafficRoutes(app, opts) {
|
|
|
52669
53890
|
Number(total?.organic ?? 0)
|
|
52670
53891
|
);
|
|
52671
53892
|
totalEventRows += Number(total?.rows ?? 0);
|
|
52672
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53893
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
52673
53894
|
const referralSeries = app.db.select({
|
|
52674
53895
|
bucket: referralSeriesBucket,
|
|
52675
|
-
hits:
|
|
53896
|
+
hits: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
52676
53897
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
52677
53898
|
for (const row of referralSeries) {
|
|
52678
53899
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiReferralHits = Number(row.hits);
|
|
@@ -52729,7 +53950,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52729
53950
|
}
|
|
52730
53951
|
|
|
52731
53952
|
// ../api-routes/src/doctor/checks/agent.ts
|
|
52732
|
-
import
|
|
53953
|
+
import crypto41 from "crypto";
|
|
52733
53954
|
import fs6 from "fs";
|
|
52734
53955
|
import path7 from "path";
|
|
52735
53956
|
var REQUIRED_SKILLS = ["canonry", "aero"];
|
|
@@ -52960,7 +54181,7 @@ function isInstalled(dir) {
|
|
|
52960
54181
|
}
|
|
52961
54182
|
function hashInstalledFile(filePath) {
|
|
52962
54183
|
try {
|
|
52963
|
-
return
|
|
54184
|
+
return crypto41.createHash("sha256").update(fs6.readFileSync(filePath)).digest("hex");
|
|
52964
54185
|
} catch {
|
|
52965
54186
|
return void 0;
|
|
52966
54187
|
}
|
|
@@ -52975,7 +54196,7 @@ function readInstalledManifest(skillDir) {
|
|
|
52975
54196
|
var AGENT_CHECKS = [skillsInstalledCheck, skillsCurrentCheck];
|
|
52976
54197
|
|
|
52977
54198
|
// ../api-routes/src/doctor/checks/backlinks.ts
|
|
52978
|
-
import { and as
|
|
54199
|
+
import { and as and40, eq as eq48 } from "drizzle-orm";
|
|
52979
54200
|
function skippedNoProject() {
|
|
52980
54201
|
return {
|
|
52981
54202
|
status: CheckStatuses.skipped,
|
|
@@ -52991,8 +54212,8 @@ var BACKLINKS_CHECKS = [
|
|
|
52991
54212
|
title: "Backlinks source connected",
|
|
52992
54213
|
run: (ctx) => {
|
|
52993
54214
|
if (!ctx.project) return skippedNoProject();
|
|
52994
|
-
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(
|
|
52995
|
-
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(
|
|
54215
|
+
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(eq48(projects.id, ctx.project.id)).get();
|
|
54216
|
+
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(eq48(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).limit(1).get();
|
|
52996
54217
|
const ccConnected = projectRow?.autoExtract === true && !!readySync;
|
|
52997
54218
|
const connected = [];
|
|
52998
54219
|
if (ccConnected) connected.push(BacklinkSources.commoncrawl);
|
|
@@ -53005,9 +54226,9 @@ var BACKLINKS_CHECKS = [
|
|
|
53005
54226
|
details: { commoncrawl: ccConnected }
|
|
53006
54227
|
};
|
|
53007
54228
|
}
|
|
53008
|
-
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
53009
|
-
|
|
53010
|
-
|
|
54229
|
+
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and40(
|
|
54230
|
+
eq48(backlinkSummaries.projectId, ctx.project.id),
|
|
54231
|
+
eq48(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
53011
54232
|
)).limit(1).get() : false;
|
|
53012
54233
|
return {
|
|
53013
54234
|
status: CheckStatuses.ok,
|
|
@@ -53167,7 +54388,7 @@ var BING_AUTH_CHECKS = [
|
|
|
53167
54388
|
];
|
|
53168
54389
|
|
|
53169
54390
|
// ../api-routes/src/doctor/checks/content.ts
|
|
53170
|
-
import { eq as
|
|
54391
|
+
import { eq as eq49 } from "drizzle-orm";
|
|
53171
54392
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
53172
54393
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
53173
54394
|
function skippedNoProject2() {
|
|
@@ -53180,7 +54401,7 @@ function skippedNoProject2() {
|
|
|
53180
54401
|
}
|
|
53181
54402
|
function loadProject(ctx) {
|
|
53182
54403
|
if (!ctx.project) return null;
|
|
53183
|
-
return ctx.db.select().from(projects).where(
|
|
54404
|
+
return ctx.db.select().from(projects).where(eq49(projects.id, ctx.project.id)).get() ?? null;
|
|
53184
54405
|
}
|
|
53185
54406
|
function percent(value) {
|
|
53186
54407
|
return Math.round(value * 100);
|
|
@@ -53272,7 +54493,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
53272
54493
|
);
|
|
53273
54494
|
|
|
53274
54495
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
53275
|
-
import { eq as
|
|
54496
|
+
import { eq as eq50 } from "drizzle-orm";
|
|
53276
54497
|
var RECENT_SYNC_WARN_DAYS = 7;
|
|
53277
54498
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
53278
54499
|
var adsConnectionCheck = {
|
|
@@ -53289,7 +54510,7 @@ var adsConnectionCheck = {
|
|
|
53289
54510
|
remediation: null
|
|
53290
54511
|
};
|
|
53291
54512
|
}
|
|
53292
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
54513
|
+
const row = ctx.db.select().from(adsConnections).where(eq50(adsConnections.projectId, ctx.project.id)).get();
|
|
53293
54514
|
if (!row) {
|
|
53294
54515
|
return {
|
|
53295
54516
|
status: CheckStatuses.skipped,
|
|
@@ -53339,7 +54560,7 @@ var adsRecentSyncCheck = {
|
|
|
53339
54560
|
remediation: null
|
|
53340
54561
|
};
|
|
53341
54562
|
}
|
|
53342
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
54563
|
+
const row = ctx.db.select().from(adsConnections).where(eq50(adsConnections.projectId, ctx.project.id)).get();
|
|
53343
54564
|
if (!row) {
|
|
53344
54565
|
return {
|
|
53345
54566
|
status: CheckStatuses.skipped,
|
|
@@ -53530,7 +54751,7 @@ var ga4ConnectionCheck = {
|
|
|
53530
54751
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
53531
54752
|
|
|
53532
54753
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
53533
|
-
import { and as
|
|
54754
|
+
import { and as and41, eq as eq51 } from "drizzle-orm";
|
|
53534
54755
|
var RECENT_SYNC_WARN_DAYS2 = 7;
|
|
53535
54756
|
var RECENT_SYNC_FAIL_DAYS2 = 30;
|
|
53536
54757
|
function skippedNoProject3() {
|
|
@@ -53763,7 +54984,7 @@ var recentSyncCheck = {
|
|
|
53763
54984
|
title: "GBP recent sync",
|
|
53764
54985
|
run: (ctx) => {
|
|
53765
54986
|
if (!ctx.project) return skippedNoProject3();
|
|
53766
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
54987
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and41(eq51(gbpLocations.projectId, ctx.project.id), eq51(gbpLocations.selected, true))).all();
|
|
53767
54988
|
if (selected.length === 0) {
|
|
53768
54989
|
return {
|
|
53769
54990
|
status: CheckStatuses.skipped,
|
|
@@ -53823,7 +55044,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
53823
55044
|
);
|
|
53824
55045
|
|
|
53825
55046
|
// ../api-routes/src/doctor/checks/places.ts
|
|
53826
|
-
import { eq as
|
|
55047
|
+
import { eq as eq52 } from "drizzle-orm";
|
|
53827
55048
|
var apiKeyCheck = {
|
|
53828
55049
|
id: "gbp.places.api-key",
|
|
53829
55050
|
category: CheckCategories.auth,
|
|
@@ -53868,7 +55089,7 @@ var apiKeyCheck = {
|
|
|
53868
55089
|
details: { tier: cfg.tier }
|
|
53869
55090
|
};
|
|
53870
55091
|
}
|
|
53871
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
55092
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq52(gbpLocations.projectId, ctx.project.id)).all();
|
|
53872
55093
|
const selected = rows.filter((r) => r.selected);
|
|
53873
55094
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
53874
55095
|
const details = {
|
|
@@ -54383,9 +55604,25 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
54383
55604
|
];
|
|
54384
55605
|
|
|
54385
55606
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
54386
|
-
import { and as
|
|
55607
|
+
import { and as and42, eq as eq53, gte as gte12, ne as ne6, sql as sql20 } from "drizzle-orm";
|
|
54387
55608
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
54388
55609
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
55610
|
+
function isCloudflareDirectPush(source) {
|
|
55611
|
+
if (source.sourceType !== TrafficSourceTypes.cloudflare) return false;
|
|
55612
|
+
const deliveryMode = source.configJson.deliveryMode;
|
|
55613
|
+
return deliveryMode === void 0 || deliveryMode === CloudflareTrafficDeliveryModes["direct-push"];
|
|
55614
|
+
}
|
|
55615
|
+
function recentDataRemediation(sources, lastSyncedAt) {
|
|
55616
|
+
const hasCloudflare = sources.some(isCloudflareDirectPush);
|
|
55617
|
+
const hasPullSource = sources.some((source) => !isCloudflareDirectPush(source));
|
|
55618
|
+
if (hasCloudflare && !hasPullSource) {
|
|
55619
|
+
return "Confirm the Cloudflare Worker is deployed on the zone route and forwarding to canonry; reconnect Cloudflare to regenerate the Worker if needed.";
|
|
55620
|
+
}
|
|
55621
|
+
if (hasCloudflare && hasPullSource) {
|
|
55622
|
+
return "Run `canonry traffic sync <project>` for pull-based sources, and confirm the Cloudflare Worker is deployed on the zone route and forwarding to canonry.";
|
|
55623
|
+
}
|
|
55624
|
+
return lastSyncedAt ? `Last sync: ${lastSyncedAt}. Run \`canonry traffic sync <project>\` to refresh, or check the source connection.` : "Run `canonry traffic sync <project>` to pull recent events.";
|
|
55625
|
+
}
|
|
54389
55626
|
function skippedNoProject5() {
|
|
54390
55627
|
return {
|
|
54391
55628
|
status: CheckStatuses.skipped,
|
|
@@ -54397,8 +55634,8 @@ function skippedNoProject5() {
|
|
|
54397
55634
|
function loadProbes(ctx) {
|
|
54398
55635
|
if (!ctx.project) return [];
|
|
54399
55636
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
54400
|
-
|
|
54401
|
-
|
|
55637
|
+
and42(
|
|
55638
|
+
eq53(trafficSources.projectId, ctx.project.id),
|
|
54402
55639
|
ne6(trafficSources.status, TrafficSourceStatuses.archived)
|
|
54403
55640
|
)
|
|
54404
55641
|
).all();
|
|
@@ -54410,6 +55647,8 @@ function loadProbes(ctx) {
|
|
|
54410
55647
|
displayName: r.displayName,
|
|
54411
55648
|
status: r.status,
|
|
54412
55649
|
lastSyncedAt: r.lastSyncedAt,
|
|
55650
|
+
lastWorkerVersion: r.lastWorkerVersion,
|
|
55651
|
+
ingestTokenHash: r.ingestTokenHash,
|
|
54413
55652
|
skippedThroughAt: r.skippedThroughAt,
|
|
54414
55653
|
lastError: r.lastError,
|
|
54415
55654
|
configJson: r.configJson
|
|
@@ -54478,52 +55717,74 @@ var recentDataCheck = {
|
|
|
54478
55717
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
54479
55718
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
54480
55719
|
const recentCrawlers = Number(
|
|
54481
|
-
ctx.db.select({ total:
|
|
54482
|
-
|
|
54483
|
-
|
|
54484
|
-
|
|
55720
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
55721
|
+
and42(
|
|
55722
|
+
eq53(crawlerEventsHourly.projectId, ctx.project.id),
|
|
55723
|
+
gte12(crawlerEventsHourly.tsHour, warnCutoff)
|
|
54485
55724
|
)
|
|
54486
55725
|
).get()?.total ?? 0
|
|
54487
55726
|
);
|
|
54488
55727
|
const recentReferrals = Number(
|
|
54489
|
-
ctx.db.select({ total:
|
|
54490
|
-
|
|
54491
|
-
|
|
54492
|
-
|
|
55728
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
55729
|
+
and42(
|
|
55730
|
+
eq53(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
55731
|
+
gte12(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
54493
55732
|
)
|
|
54494
55733
|
).get()?.total ?? 0
|
|
54495
55734
|
);
|
|
54496
|
-
|
|
55735
|
+
const recentUserFetches = Number(
|
|
55736
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
55737
|
+
and42(
|
|
55738
|
+
eq53(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
55739
|
+
gte12(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
55740
|
+
)
|
|
55741
|
+
).get()?.total ?? 0
|
|
55742
|
+
);
|
|
55743
|
+
if (recentCrawlers > 0 || recentUserFetches > 0 || recentReferrals > 0) {
|
|
54497
55744
|
return {
|
|
54498
55745
|
status: CheckStatuses.ok,
|
|
54499
55746
|
code: "traffic.recent-data.fresh",
|
|
54500
|
-
summary: `${recentCrawlers} crawler hit(s) and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
54501
|
-
details: {
|
|
55747
|
+
summary: `${recentCrawlers} crawler hit(s), ${recentUserFetches} AI user-fetch hit(s), and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
55748
|
+
details: {
|
|
55749
|
+
crawlerHits: recentCrawlers,
|
|
55750
|
+
aiUserFetchHits: recentUserFetches,
|
|
55751
|
+
referralArrivals: recentReferrals,
|
|
55752
|
+
windowDays: RECENT_DATA_WARN_DAYS
|
|
55753
|
+
}
|
|
54502
55754
|
};
|
|
54503
55755
|
}
|
|
54504
55756
|
const olderCrawlers = Number(
|
|
54505
|
-
ctx.db.select({ total:
|
|
54506
|
-
|
|
54507
|
-
|
|
54508
|
-
|
|
55757
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
55758
|
+
and42(
|
|
55759
|
+
eq53(crawlerEventsHourly.projectId, ctx.project.id),
|
|
55760
|
+
gte12(crawlerEventsHourly.tsHour, failCutoff)
|
|
54509
55761
|
)
|
|
54510
55762
|
).get()?.total ?? 0
|
|
54511
55763
|
);
|
|
54512
55764
|
const olderReferrals = Number(
|
|
54513
|
-
ctx.db.select({ total:
|
|
54514
|
-
|
|
54515
|
-
|
|
54516
|
-
|
|
55765
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
55766
|
+
and42(
|
|
55767
|
+
eq53(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
55768
|
+
gte12(aiReferralEventsHourly.tsHour, failCutoff)
|
|
55769
|
+
)
|
|
55770
|
+
).get()?.total ?? 0
|
|
55771
|
+
);
|
|
55772
|
+
const olderUserFetches = Number(
|
|
55773
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
55774
|
+
and42(
|
|
55775
|
+
eq53(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
55776
|
+
gte12(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
54517
55777
|
)
|
|
54518
55778
|
).get()?.total ?? 0
|
|
54519
55779
|
);
|
|
54520
55780
|
const lastSyncedAt = sources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
54521
|
-
|
|
55781
|
+
const hasOlderData = olderCrawlers > 0 || olderUserFetches > 0 || olderReferrals > 0;
|
|
55782
|
+
if (hasOlderData || lastSyncedAt) {
|
|
54522
55783
|
return {
|
|
54523
55784
|
status: CheckStatuses.warn,
|
|
54524
55785
|
code: "traffic.recent-data.stale",
|
|
54525
|
-
summary: `No crawler
|
|
54526
|
-
remediation:
|
|
55786
|
+
summary: hasOlderData ? `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.` : `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
55787
|
+
remediation: recentDataRemediation(sources, lastSyncedAt),
|
|
54527
55788
|
details: { lastSyncedAt, sourceCount: sources.length }
|
|
54528
55789
|
};
|
|
54529
55790
|
}
|
|
@@ -54721,7 +55982,15 @@ var syncLagCheck = {
|
|
|
54721
55982
|
title: "Traffic source sync lag",
|
|
54722
55983
|
run: (ctx) => {
|
|
54723
55984
|
if (!ctx.project) return skippedNoProject5();
|
|
54724
|
-
const
|
|
55985
|
+
const allSources = loadProbes(ctx);
|
|
55986
|
+
const sources = allSources.filter((source) => !isCloudflareDirectPush(source));
|
|
55987
|
+
if (allSources.length > 0 && sources.length === 0) {
|
|
55988
|
+
return {
|
|
55989
|
+
status: CheckStatuses.skipped,
|
|
55990
|
+
code: "traffic.sync-lag.push-only",
|
|
55991
|
+
summary: "Only push traffic sources are connected \u2014 pull sync lag does not apply."
|
|
55992
|
+
};
|
|
55993
|
+
}
|
|
54725
55994
|
if (sources.length === 0) {
|
|
54726
55995
|
return {
|
|
54727
55996
|
status: CheckStatuses.skipped,
|
|
@@ -54811,10 +56080,68 @@ var syncLagCheck = {
|
|
|
54811
56080
|
};
|
|
54812
56081
|
}
|
|
54813
56082
|
};
|
|
56083
|
+
var workerVersionCheck = {
|
|
56084
|
+
id: "traffic.source.worker-version",
|
|
56085
|
+
category: CheckCategories.integrations,
|
|
56086
|
+
scope: CheckScopes.project,
|
|
56087
|
+
title: "Cloudflare Worker version",
|
|
56088
|
+
run: (ctx) => {
|
|
56089
|
+
if (!ctx.project) return skippedNoProject5();
|
|
56090
|
+
const sources = loadProbes(ctx).filter(isCloudflareDirectPush);
|
|
56091
|
+
if (sources.length === 0) {
|
|
56092
|
+
return {
|
|
56093
|
+
status: CheckStatuses.skipped,
|
|
56094
|
+
code: "traffic.worker-version.not-applicable",
|
|
56095
|
+
summary: "No Cloudflare direct-push source is connected."
|
|
56096
|
+
};
|
|
56097
|
+
}
|
|
56098
|
+
const measured = sources.map((source) => {
|
|
56099
|
+
const expected = source.configJson.workerVersion;
|
|
56100
|
+
const expectedVersion = typeof expected === "string" && expected.length > 0 ? expected : null;
|
|
56101
|
+
const observedVersion = source.lastWorkerVersion;
|
|
56102
|
+
const state = observedVersion === null ? "waiting-for-first-event" : expectedVersion !== null && observedVersion === expectedVersion ? "current" : "stale";
|
|
56103
|
+
return {
|
|
56104
|
+
sourceId: source.id,
|
|
56105
|
+
displayName: source.displayName,
|
|
56106
|
+
expectedVersion,
|
|
56107
|
+
observedVersion,
|
|
56108
|
+
state
|
|
56109
|
+
};
|
|
56110
|
+
});
|
|
56111
|
+
const details = { sources: measured };
|
|
56112
|
+
const stale = measured.filter((source) => source.state === "stale");
|
|
56113
|
+
if (stale.length > 0) {
|
|
56114
|
+
return {
|
|
56115
|
+
status: CheckStatuses.warn,
|
|
56116
|
+
code: "traffic.worker-version.stale",
|
|
56117
|
+
summary: `${stale.length} Cloudflare Worker source(s) report a version that differs from the generated version.`,
|
|
56118
|
+
remediation: "Redeploy the generated Worker from the credential-owning host. Use `canonry traffic connect cloudflare <project> --zone-id <id> --deploy --confirm-route --confirm-fail-open`, then attach the route manually.",
|
|
56119
|
+
details
|
|
56120
|
+
};
|
|
56121
|
+
}
|
|
56122
|
+
const waiting = measured.filter((source) => source.state === "waiting-for-first-event");
|
|
56123
|
+
if (waiting.length > 0) {
|
|
56124
|
+
return {
|
|
56125
|
+
status: CheckStatuses.warn,
|
|
56126
|
+
code: "traffic.worker-version.waiting-for-first-event",
|
|
56127
|
+
summary: `${waiting.length} Cloudflare Worker source(s) have not delivered an event yet.`,
|
|
56128
|
+
remediation: "Send a smoke-test request through the Worker. Then run `canonry doctor --project <project>` again.",
|
|
56129
|
+
details
|
|
56130
|
+
};
|
|
56131
|
+
}
|
|
56132
|
+
return {
|
|
56133
|
+
status: CheckStatuses.ok,
|
|
56134
|
+
code: "traffic.worker-version.current",
|
|
56135
|
+
summary: "All Cloudflare direct-push Workers report the generated version.",
|
|
56136
|
+
details
|
|
56137
|
+
};
|
|
56138
|
+
}
|
|
56139
|
+
};
|
|
54814
56140
|
var TRAFFIC_SOURCE_CHECKS = [
|
|
54815
56141
|
sourceConnectedCheck,
|
|
54816
56142
|
recentDataCheck,
|
|
54817
56143
|
syncLagCheck,
|
|
56144
|
+
workerVersionCheck,
|
|
54818
56145
|
credentialsCheck,
|
|
54819
56146
|
scopesCheck3,
|
|
54820
56147
|
cacheBlindSpotCheck
|
|
@@ -55033,8 +56360,8 @@ async function doctorRoutes(app, opts) {
|
|
|
55033
56360
|
}
|
|
55034
56361
|
|
|
55035
56362
|
// ../api-routes/src/discovery/routes.ts
|
|
55036
|
-
import
|
|
55037
|
-
import { and as
|
|
56363
|
+
import crypto42 from "crypto";
|
|
56364
|
+
import { and as and43, desc as desc23, eq as eq54, gte as gte13, inArray as inArray18, isNull as isNull4, or as or10 } from "drizzle-orm";
|
|
55038
56365
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
55039
56366
|
async function discoveryRoutes(app, opts) {
|
|
55040
56367
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -55068,13 +56395,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
55068
56395
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
55069
56396
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
55070
56397
|
const decision = app.db.transaction((tx) => {
|
|
55071
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
55072
|
-
|
|
55073
|
-
|
|
56398
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and43(
|
|
56399
|
+
eq54(discoverySessions.projectId, project.id),
|
|
56400
|
+
eq54(discoverySessions.icpDescription, icpDescription),
|
|
55074
56401
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
55075
56402
|
// semantics, so a request with a different (or no) buyer must start
|
|
55076
56403
|
// its own session, never adopt another buyer's probes.
|
|
55077
|
-
parsed.data.buyerDescription == null ? isNull4(discoverySessions.buyerDescription) :
|
|
56404
|
+
parsed.data.buyerDescription == null ? isNull4(discoverySessions.buyerDescription) : eq54(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
55078
56405
|
// Locations are identity too: a different service-area subset seeds
|
|
55079
56406
|
// and probes a different geo, so it must never reuse another geo's
|
|
55080
56407
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -55084,23 +56411,23 @@ async function discoveryRoutes(app, opts) {
|
|
|
55084
56411
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
55085
56412
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
55086
56413
|
// upgrade, never a wrong reuse.
|
|
55087
|
-
locations.length === 0 ? or10(isNull4(discoverySessions.locations),
|
|
56414
|
+
locations.length === 0 ? or10(isNull4(discoverySessions.locations), eq54(discoverySessions.locations, locations)) : eq54(discoverySessions.locations, locations),
|
|
55088
56415
|
// Seed provider set is identity: a different phrasing distribution
|
|
55089
56416
|
// must never reuse another set's session. Null (= Gemini-only
|
|
55090
56417
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
55091
|
-
seedProviders == null ? isNull4(discoverySessions.seedProviders) :
|
|
56418
|
+
seedProviders == null ? isNull4(discoverySessions.seedProviders) : eq54(discoverySessions.seedProviders, seedProviders),
|
|
55092
56419
|
inArray18(discoverySessions.status, [
|
|
55093
56420
|
DiscoverySessionStatuses.queued,
|
|
55094
56421
|
DiscoverySessionStatuses.seeding,
|
|
55095
56422
|
DiscoverySessionStatuses.probing
|
|
55096
56423
|
]),
|
|
55097
|
-
|
|
56424
|
+
gte13(discoverySessions.createdAt, ageFloorIso)
|
|
55098
56425
|
)).orderBy(desc23(discoverySessions.createdAt)).get();
|
|
55099
56426
|
if (existing && existing.runId) {
|
|
55100
56427
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
55101
56428
|
}
|
|
55102
|
-
const sessionId =
|
|
55103
|
-
const runId =
|
|
56429
|
+
const sessionId = crypto42.randomUUID();
|
|
56430
|
+
const runId = crypto42.randomUUID();
|
|
55104
56431
|
tx.insert(discoverySessions).values({
|
|
55105
56432
|
id: sessionId,
|
|
55106
56433
|
projectId: project.id,
|
|
@@ -55164,7 +56491,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55164
56491
|
const project = resolveProject(app.db, request.params.name);
|
|
55165
56492
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
55166
56493
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
55167
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
56494
|
+
const rows = app.db.select().from(discoverySessions).where(eq54(discoverySessions.projectId, project.id)).orderBy(desc23(discoverySessions.createdAt)).limit(limit).all();
|
|
55168
56495
|
return reply.send(rows.map(serializeSession));
|
|
55169
56496
|
}
|
|
55170
56497
|
);
|
|
@@ -55172,11 +56499,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
55172
56499
|
"/projects/:name/discover/sessions/:id",
|
|
55173
56500
|
async (request, reply) => {
|
|
55174
56501
|
const project = resolveProject(app.db, request.params.name);
|
|
55175
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56502
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55176
56503
|
if (!session || session.projectId !== project.id) {
|
|
55177
56504
|
throw notFound("Discovery session", request.params.id);
|
|
55178
56505
|
}
|
|
55179
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56506
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
55180
56507
|
const detail = {
|
|
55181
56508
|
...serializeSession(session),
|
|
55182
56509
|
probes: probeRows.map(serializeProbe)
|
|
@@ -55188,7 +56515,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55188
56515
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
55189
56516
|
async (request, reply) => {
|
|
55190
56517
|
const project = resolveProject(app.db, request.params.name);
|
|
55191
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56518
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55192
56519
|
if (!session || session.projectId !== project.id) {
|
|
55193
56520
|
throw notFound("Discovery session", request.params.id);
|
|
55194
56521
|
}
|
|
@@ -55196,7 +56523,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55196
56523
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
55197
56524
|
const applyAnchor = request.query.anchor !== "false";
|
|
55198
56525
|
const provider = session.seedProvider ?? "gemini";
|
|
55199
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56526
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
55200
56527
|
const extract = opts.harvestSearchQueries;
|
|
55201
56528
|
const probesWithQueries = probeRows.map((row) => {
|
|
55202
56529
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -55207,7 +56534,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55207
56534
|
return { searchQueries: [] };
|
|
55208
56535
|
}
|
|
55209
56536
|
});
|
|
55210
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
56537
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq54(queries.projectId, project.id)).all().map((r) => r.query);
|
|
55211
56538
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
55212
56539
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
55213
56540
|
effectiveDomains(project)
|
|
@@ -55254,12 +56581,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
55254
56581
|
"/projects/:name/discover/sessions/:id/promote",
|
|
55255
56582
|
async (request, reply) => {
|
|
55256
56583
|
const project = resolveProject(app.db, request.params.name);
|
|
55257
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56584
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55258
56585
|
if (!session || session.projectId !== project.id) {
|
|
55259
56586
|
throw notFound("Discovery session", request.params.id);
|
|
55260
56587
|
}
|
|
55261
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
55262
|
-
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
56588
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
56589
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq54(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
55263
56590
|
const seenCompetitors = new Set(existingCompetitors);
|
|
55264
56591
|
const cited = /* @__PURE__ */ new Set();
|
|
55265
56592
|
const aspirational = /* @__PURE__ */ new Set();
|
|
@@ -55288,7 +56615,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55288
56615
|
);
|
|
55289
56616
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
55290
56617
|
const project = resolveProject(app.db, request.params.name);
|
|
55291
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56618
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55292
56619
|
if (!session || session.projectId !== project.id) {
|
|
55293
56620
|
throw notFound("Discovery session", request.params.id);
|
|
55294
56621
|
}
|
|
@@ -55311,7 +56638,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55311
56638
|
const bucketSet = new Set(buckets);
|
|
55312
56639
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
55313
56640
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
55314
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56641
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
55315
56642
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
55316
56643
|
for (const probe of probeRows) {
|
|
55317
56644
|
if (!probe.bucket) continue;
|
|
@@ -55319,7 +56646,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55319
56646
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
55320
56647
|
}
|
|
55321
56648
|
const existingQueries = new Set(
|
|
55322
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
56649
|
+
app.db.select({ query: queries.query }).from(queries).where(eq54(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
55323
56650
|
);
|
|
55324
56651
|
const promotedQueries = [];
|
|
55325
56652
|
const skippedQueries = [];
|
|
@@ -55335,7 +56662,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55335
56662
|
const skippedCompetitors = [];
|
|
55336
56663
|
if (includeCompetitors) {
|
|
55337
56664
|
const existingCompetitors = new Set(
|
|
55338
|
-
app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
56665
|
+
app.db.select({ domain: competitors.domain }).from(competitors).where(eq54(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase())
|
|
55339
56666
|
);
|
|
55340
56667
|
const competitorMap = parseCompetitorMap(session.competitorMap);
|
|
55341
56668
|
for (const entry of selectEligibleCompetitors(competitorMap, competitorTypes)) {
|
|
@@ -55354,7 +56681,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55354
56681
|
app.db.transaction((tx) => {
|
|
55355
56682
|
for (const query of promotedQueries) {
|
|
55356
56683
|
tx.insert(queries).values({
|
|
55357
|
-
id:
|
|
56684
|
+
id: crypto42.randomUUID(),
|
|
55358
56685
|
projectId: project.id,
|
|
55359
56686
|
query,
|
|
55360
56687
|
provenance,
|
|
@@ -55363,7 +56690,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55363
56690
|
}
|
|
55364
56691
|
for (const domain of promotedCompetitors) {
|
|
55365
56692
|
tx.insert(competitors).values({
|
|
55366
|
-
id:
|
|
56693
|
+
id: crypto42.randomUUID(),
|
|
55367
56694
|
projectId: project.id,
|
|
55368
56695
|
domain,
|
|
55369
56696
|
provenance,
|
|
@@ -55453,8 +56780,8 @@ function selectEligibleCompetitors(competitorMap, competitorTypes) {
|
|
|
55453
56780
|
}
|
|
55454
56781
|
|
|
55455
56782
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
55456
|
-
import
|
|
55457
|
-
import { eq as
|
|
56783
|
+
import crypto43 from "crypto";
|
|
56784
|
+
import { eq as eq55 } from "drizzle-orm";
|
|
55458
56785
|
var DEFAULT_MAX_PROBES = 100;
|
|
55459
56786
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
55460
56787
|
function classifyProbeBucket(input) {
|
|
@@ -55541,7 +56868,7 @@ async function executeDiscovery(opts) {
|
|
|
55541
56868
|
status: DiscoverySessionStatuses.seeding,
|
|
55542
56869
|
dedupThreshold,
|
|
55543
56870
|
startedAt
|
|
55544
|
-
}).where(
|
|
56871
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55545
56872
|
const seedResult = await opts.deps.seed({
|
|
55546
56873
|
project: opts.project,
|
|
55547
56874
|
icpDescription: opts.icpDescription,
|
|
@@ -55602,7 +56929,7 @@ async function executeDiscovery(opts) {
|
|
|
55602
56929
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
55603
56930
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
55604
56931
|
warning
|
|
55605
|
-
}).where(
|
|
56932
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55606
56933
|
const probeLocation = opts.locations?.[0];
|
|
55607
56934
|
const probeResults = await mapWithConcurrency(
|
|
55608
56935
|
probedCanonicals,
|
|
@@ -55621,7 +56948,7 @@ async function executeDiscovery(opts) {
|
|
|
55621
56948
|
probeRows.push({ citedDomains: probe.citedDomains, bucket });
|
|
55622
56949
|
buckets[bucket]++;
|
|
55623
56950
|
return {
|
|
55624
|
-
id:
|
|
56951
|
+
id: crypto43.randomUUID(),
|
|
55625
56952
|
sessionId: opts.sessionId,
|
|
55626
56953
|
projectId: opts.project.id,
|
|
55627
56954
|
query: probedCanonicals[index2],
|
|
@@ -55656,7 +56983,7 @@ async function executeDiscovery(opts) {
|
|
|
55656
56983
|
wastedCount: buckets["wasted-surface"],
|
|
55657
56984
|
competitorMap,
|
|
55658
56985
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55659
|
-
}).where(
|
|
56986
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55660
56987
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
55661
56988
|
return {
|
|
55662
56989
|
buckets,
|
|
@@ -55673,7 +57000,7 @@ function upsertDomainClassifications(db, projectId, sessionId, competitorMap) {
|
|
|
55673
57000
|
const domain = hostOf(entry.domain) ?? "";
|
|
55674
57001
|
if (!domain) continue;
|
|
55675
57002
|
db.insert(domainClassifications).values({
|
|
55676
|
-
id:
|
|
57003
|
+
id: crypto43.randomUUID(),
|
|
55677
57004
|
projectId,
|
|
55678
57005
|
domain,
|
|
55679
57006
|
competitorType: entry.competitorType,
|
|
@@ -55696,7 +57023,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
55696
57023
|
status: DiscoverySessionStatuses.failed,
|
|
55697
57024
|
error,
|
|
55698
57025
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55699
|
-
}).where(
|
|
57026
|
+
}).where(eq55(discoverySessions.id, sessionId)).run();
|
|
55700
57027
|
}
|
|
55701
57028
|
function dedupeStrings(input) {
|
|
55702
57029
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -55713,8 +57040,8 @@ function dedupeStrings(input) {
|
|
|
55713
57040
|
}
|
|
55714
57041
|
|
|
55715
57042
|
// ../api-routes/src/technical-aeo.ts
|
|
55716
|
-
import
|
|
55717
|
-
import { and as
|
|
57043
|
+
import crypto44 from "crypto";
|
|
57044
|
+
import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
|
|
55718
57045
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
55719
57046
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
55720
57047
|
...SiteCrawlFetchedStates,
|
|
@@ -55861,7 +57188,7 @@ function mapCrawlEdge(row) {
|
|
|
55861
57188
|
}
|
|
55862
57189
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
55863
57190
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
55864
|
-
var PAGE_CHANGE_PREDICATE =
|
|
57191
|
+
var PAGE_CHANGE_PREDICATE = sql21`(
|
|
55865
57192
|
current.url IS NOT previous.url
|
|
55866
57193
|
OR current.final_url IS NOT previous.final_url
|
|
55867
57194
|
OR current.path IS NOT previous.path
|
|
@@ -55883,7 +57210,7 @@ var PAGE_CHANGE_PREDICATE = sql20`(
|
|
|
55883
57210
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
55884
57211
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
55885
57212
|
)`;
|
|
55886
|
-
var LINK_CHANGE_PREDICATE =
|
|
57213
|
+
var LINK_CHANGE_PREDICATE = sql21`(
|
|
55887
57214
|
current.source_node_key IS NOT previous.source_node_key
|
|
55888
57215
|
OR current.source_url IS NOT previous.source_url
|
|
55889
57216
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -55961,39 +57288,39 @@ function changedFields(before, after, fields) {
|
|
|
55961
57288
|
async function technicalAeoRoutes(app, opts) {
|
|
55962
57289
|
const resolveCrawl = (projectId, runId) => {
|
|
55963
57290
|
const filters = [
|
|
55964
|
-
|
|
55965
|
-
|
|
55966
|
-
|
|
57291
|
+
eq56(siteCrawlSnapshots.projectId, projectId),
|
|
57292
|
+
eq56(runs.projectId, projectId),
|
|
57293
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55967
57294
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55968
57295
|
notProbeRun()
|
|
55969
57296
|
];
|
|
55970
57297
|
if (runId) {
|
|
55971
|
-
filters.push(
|
|
57298
|
+
filters.push(eq56(siteCrawlSnapshots.runId, runId));
|
|
55972
57299
|
} else {
|
|
55973
|
-
filters.push(
|
|
57300
|
+
filters.push(eq56(siteCrawlSnapshots.complete, true), eq56(runs.status, RunStatuses.completed));
|
|
55974
57301
|
}
|
|
55975
|
-
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
57302
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
55976
57303
|
};
|
|
55977
57304
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
55978
57305
|
const pageInScope = (scope, selector) => {
|
|
55979
57306
|
if (!selector.nodeKey && !selector.url) return null;
|
|
55980
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
55981
|
-
|
|
55982
|
-
|
|
55983
|
-
|
|
55984
|
-
selector.nodeKey ?
|
|
57307
|
+
return app.db.select().from(siteCrawlPages).where(and44(
|
|
57308
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57309
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57310
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57311
|
+
selector.nodeKey ? eq56(siteCrawlPages.nodeKey, selector.nodeKey) : eq56(siteCrawlPages.url, selector.url)
|
|
55985
57312
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55986
57313
|
};
|
|
55987
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
55988
|
-
|
|
55989
|
-
|
|
55990
|
-
|
|
55991
|
-
|
|
57314
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and44(
|
|
57315
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57316
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57317
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57318
|
+
eq56(siteCrawlPages.depth, 0)
|
|
55992
57319
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55993
|
-
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs,
|
|
55994
|
-
|
|
55995
|
-
|
|
55996
|
-
|
|
57320
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57321
|
+
eq56(siteAuditSnapshots.projectId, projectId),
|
|
57322
|
+
eq56(runs.projectId, projectId),
|
|
57323
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55997
57324
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55998
57325
|
notProbeRun()
|
|
55999
57326
|
)).limit(1).get());
|
|
@@ -56015,20 +57342,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56015
57342
|
app.get("/projects/:name/technical-aeo", async (request) => {
|
|
56016
57343
|
const project = resolveProject(app.db, request.params.name);
|
|
56017
57344
|
const baseFilters = [
|
|
56018
|
-
|
|
56019
|
-
|
|
56020
|
-
|
|
57345
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57346
|
+
eq56(runs.projectId, project.id),
|
|
57347
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
56021
57348
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
56022
57349
|
notProbeRun()
|
|
56023
57350
|
];
|
|
56024
|
-
const targetFilters = request.query.runId ? [...baseFilters,
|
|
56025
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57351
|
+
const targetFilters = request.query.runId ? [...baseFilters, eq56(siteAuditSnapshots.runId, request.query.runId)] : baseFilters;
|
|
57352
|
+
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
56026
57353
|
if (!latest) {
|
|
56027
57354
|
if (request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
56028
57355
|
return emptyScore(project.name);
|
|
56029
57356
|
}
|
|
56030
57357
|
const snap = latest.snap;
|
|
56031
|
-
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57358
|
+
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...baseFilters, lt8(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
56032
57359
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
56033
57360
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
56034
57361
|
return {
|
|
@@ -56055,14 +57382,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56055
57382
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
56056
57383
|
const project = resolveProject(app.db, request.params.name);
|
|
56057
57384
|
const targetFilters = [
|
|
56058
|
-
|
|
56059
|
-
|
|
56060
|
-
|
|
57385
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57386
|
+
eq56(runs.projectId, project.id),
|
|
57387
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
56061
57388
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
56062
57389
|
notProbeRun()
|
|
56063
57390
|
];
|
|
56064
|
-
if (request.query.runId) targetFilters.push(
|
|
56065
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57391
|
+
if (request.query.runId) targetFilters.push(eq56(siteAuditSnapshots.runId, request.query.runId));
|
|
57392
|
+
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
56066
57393
|
if (!latest && request.query.runId) {
|
|
56067
57394
|
throw notFound("Site audit run", request.query.runId);
|
|
56068
57395
|
}
|
|
@@ -56070,9 +57397,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56070
57397
|
return { project: project.name, runId: null, auditedAt: null, total: 0, pages: [] };
|
|
56071
57398
|
}
|
|
56072
57399
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
56073
|
-
const conds = [
|
|
56074
|
-
if (statusFilter) conds.push(
|
|
56075
|
-
const where =
|
|
57400
|
+
const conds = [eq56(siteAuditPages.projectId, project.id), eq56(siteAuditPages.runId, latest.runId)];
|
|
57401
|
+
if (statusFilter) conds.push(eq56(siteAuditPages.status, statusFilter));
|
|
57402
|
+
const where = and44(...conds);
|
|
56076
57403
|
const totalRow = app.db.select({ value: count() }).from(siteAuditPages).where(where).get();
|
|
56077
57404
|
const total = totalRow?.value ?? 0;
|
|
56078
57405
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
@@ -56096,10 +57423,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56096
57423
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
56097
57424
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
56098
57425
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
56099
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
56100
|
-
|
|
56101
|
-
|
|
56102
|
-
|
|
57426
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57427
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57428
|
+
eq56(runs.projectId, project.id),
|
|
57429
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
56103
57430
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
56104
57431
|
notProbeRun()
|
|
56105
57432
|
)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
@@ -56177,10 +57504,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56177
57504
|
sampled: false
|
|
56178
57505
|
};
|
|
56179
57506
|
}
|
|
56180
|
-
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(
|
|
56181
|
-
|
|
56182
|
-
|
|
56183
|
-
|
|
57507
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and44(
|
|
57508
|
+
eq56(siteCrawlGraphLayouts.projectId, project.id),
|
|
57509
|
+
eq56(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
57510
|
+
eq56(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
56184
57511
|
)).limit(1).get();
|
|
56185
57512
|
if (!persistedLayout) {
|
|
56186
57513
|
return {
|
|
@@ -56221,9 +57548,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56221
57548
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
56222
57549
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
56223
57550
|
const graphScope = [
|
|
56224
|
-
|
|
56225
|
-
|
|
56226
|
-
|
|
57551
|
+
eq56(siteCrawlGraphNodes.projectId, project.id),
|
|
57552
|
+
eq56(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
57553
|
+
eq56(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
56227
57554
|
];
|
|
56228
57555
|
const nodeRows = app.db.select({
|
|
56229
57556
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -56242,12 +57569,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56242
57569
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
56243
57570
|
x: siteCrawlGraphNodes.x,
|
|
56244
57571
|
y: siteCrawlGraphNodes.y
|
|
56245
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
56246
|
-
|
|
56247
|
-
|
|
56248
|
-
|
|
56249
|
-
|
|
56250
|
-
)).where(
|
|
57572
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and44(
|
|
57573
|
+
eq56(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
57574
|
+
eq56(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
57575
|
+
eq56(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
57576
|
+
eq56(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
57577
|
+
)).where(and44(...graphScope, lt8(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
56251
57578
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
56252
57579
|
...row,
|
|
56253
57580
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
@@ -56258,23 +57585,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56258
57585
|
targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
|
|
56259
57586
|
followable: siteCrawlGraphEdges.followable,
|
|
56260
57587
|
occurrences: siteCrawlGraphEdges.occurrences
|
|
56261
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
56262
|
-
|
|
56263
|
-
|
|
56264
|
-
|
|
56265
|
-
|
|
56266
|
-
|
|
56267
|
-
)).innerJoin(graphTargetNode,
|
|
56268
|
-
|
|
56269
|
-
|
|
56270
|
-
|
|
56271
|
-
|
|
56272
|
-
|
|
56273
|
-
)).where(
|
|
56274
|
-
|
|
56275
|
-
|
|
56276
|
-
|
|
56277
|
-
|
|
57588
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and44(
|
|
57589
|
+
eq56(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57590
|
+
eq56(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
57591
|
+
eq56(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
57592
|
+
eq56(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
57593
|
+
lt8(graphSourceNode.sampleRank, maxNodes)
|
|
57594
|
+
)).innerJoin(graphTargetNode, and44(
|
|
57595
|
+
eq56(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57596
|
+
eq56(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
57597
|
+
eq56(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
57598
|
+
eq56(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
57599
|
+
lt8(graphTargetNode.sampleRank, maxNodes)
|
|
57600
|
+
)).where(and44(
|
|
57601
|
+
eq56(siteCrawlGraphEdges.projectId, project.id),
|
|
57602
|
+
eq56(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
57603
|
+
eq56(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
57604
|
+
lt8(siteCrawlGraphEdges.sampleRank, maxEdges)
|
|
56278
57605
|
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
56279
57606
|
const totalNodes = persistedLayout.totalNodes;
|
|
56280
57607
|
const totalEdges = persistedLayout.totalEdges;
|
|
@@ -56352,11 +57679,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56352
57679
|
let frontier = [focus.nodeKey];
|
|
56353
57680
|
let queryTruncated = false;
|
|
56354
57681
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
56355
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
56356
|
-
|
|
56357
|
-
|
|
56358
|
-
|
|
56359
|
-
|
|
57682
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and44(
|
|
57683
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57684
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57685
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57686
|
+
eq56(siteCrawlEdges.internal, true),
|
|
56360
57687
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56361
57688
|
or11(
|
|
56362
57689
|
inArray19(siteCrawlEdges.sourceNodeKey, frontier),
|
|
@@ -56393,19 +57720,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56393
57720
|
frontier = [...next].sort();
|
|
56394
57721
|
}
|
|
56395
57722
|
const nodeKeys = [...distances.keys()];
|
|
56396
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
56397
|
-
|
|
56398
|
-
|
|
56399
|
-
|
|
57723
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and44(
|
|
57724
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57725
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57726
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56400
57727
|
inArray19(siteCrawlPages.nodeKey, nodeKeys)
|
|
56401
57728
|
)).all();
|
|
56402
57729
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
56403
57730
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
56404
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
56405
|
-
|
|
56406
|
-
|
|
56407
|
-
|
|
56408
|
-
|
|
57731
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and44(
|
|
57732
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57733
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57734
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57735
|
+
eq56(siteCrawlEdges.internal, true),
|
|
56409
57736
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56410
57737
|
inArray19(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
56411
57738
|
inArray19(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
@@ -56502,22 +57829,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56502
57829
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
56503
57830
|
let truncated = false;
|
|
56504
57831
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
56505
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
56506
|
-
|
|
56507
|
-
|
|
56508
|
-
|
|
56509
|
-
|
|
56510
|
-
|
|
56511
|
-
|
|
57832
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and44(
|
|
57833
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57834
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57835
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57836
|
+
eq56(siteCrawlEdges.internal, true),
|
|
57837
|
+
eq56(siteCrawlEdges.followable, true),
|
|
57838
|
+
eq56(siteCrawlEdges.relation, "anchor"),
|
|
56512
57839
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56513
57840
|
inArray19(siteCrawlEdges.sourceNodeKey, frontier)
|
|
56514
57841
|
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
56515
57842
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
56516
57843
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
56517
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
56518
|
-
|
|
56519
|
-
|
|
56520
|
-
|
|
57844
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and44(
|
|
57845
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57846
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57847
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56521
57848
|
inArray19(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
56522
57849
|
)).all().map((row) => row.nodeKey));
|
|
56523
57850
|
const next = /* @__PURE__ */ new Set();
|
|
@@ -56560,10 +57887,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56560
57887
|
pathEdges.unshift(step.edge);
|
|
56561
57888
|
pathKeys.unshift(step.nodeKey);
|
|
56562
57889
|
}
|
|
56563
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
56564
|
-
|
|
56565
|
-
|
|
56566
|
-
|
|
57890
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and44(
|
|
57891
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57892
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57893
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56567
57894
|
inArray19(siteCrawlPages.nodeKey, pathKeys)
|
|
56568
57895
|
)).all();
|
|
56569
57896
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
@@ -56592,22 +57919,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56592
57919
|
throw validationError("change must be all, added, removed, or changed");
|
|
56593
57920
|
}
|
|
56594
57921
|
const completeSnapshotFilters = [
|
|
56595
|
-
|
|
56596
|
-
|
|
56597
|
-
|
|
56598
|
-
|
|
56599
|
-
|
|
57922
|
+
eq56(siteCrawlSnapshots.projectId, project.id),
|
|
57923
|
+
eq56(runs.projectId, project.id),
|
|
57924
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57925
|
+
eq56(runs.status, RunStatuses.completed),
|
|
57926
|
+
eq56(siteCrawlSnapshots.complete, true),
|
|
56600
57927
|
notProbeRun()
|
|
56601
57928
|
];
|
|
56602
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
57929
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
56603
57930
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
56604
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
57931
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(
|
|
56605
57932
|
...completeSnapshotFilters,
|
|
56606
57933
|
or11(
|
|
56607
|
-
|
|
56608
|
-
|
|
56609
|
-
|
|
56610
|
-
|
|
57934
|
+
lt8(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
57935
|
+
and44(
|
|
57936
|
+
eq56(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
57937
|
+
lt8(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
56611
57938
|
)
|
|
56612
57939
|
)
|
|
56613
57940
|
)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
@@ -56684,9 +58011,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56684
58011
|
const summary2 = { pages: zero2(), links: zero2() };
|
|
56685
58012
|
if (scopeFilter !== "links") {
|
|
56686
58013
|
if (includeAdded || includeChanged) {
|
|
56687
|
-
const addedExpression = includeAdded ?
|
|
56688
|
-
const changedExpression = includeChanged ?
|
|
56689
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
58014
|
+
const addedExpression = includeAdded ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58015
|
+
const changedExpression = includeChanged ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58016
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql21`
|
|
56690
58017
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56691
58018
|
FROM site_crawl_pages AS current
|
|
56692
58019
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -56702,7 +58029,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56702
58029
|
summary2.pages.changed = Number(counts2.changed);
|
|
56703
58030
|
}
|
|
56704
58031
|
if (includeRemoved) {
|
|
56705
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
58032
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql21`
|
|
56706
58033
|
SELECT COUNT(*) AS removed
|
|
56707
58034
|
FROM site_crawl_pages AS previous
|
|
56708
58035
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -56720,9 +58047,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56720
58047
|
}
|
|
56721
58048
|
if (scopeFilter !== "pages") {
|
|
56722
58049
|
if (includeAdded || includeChanged) {
|
|
56723
|
-
const addedExpression = includeAdded ?
|
|
56724
|
-
const changedExpression = includeChanged ?
|
|
56725
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
58050
|
+
const addedExpression = includeAdded ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58051
|
+
const changedExpression = includeChanged ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58052
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql21`
|
|
56726
58053
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56727
58054
|
FROM site_crawl_edges AS current
|
|
56728
58055
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -56740,7 +58067,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56740
58067
|
summary2.links.changed = Number(counts2.changed);
|
|
56741
58068
|
}
|
|
56742
58069
|
if (includeRemoved) {
|
|
56743
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
58070
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql21`
|
|
56744
58071
|
SELECT COUNT(*) AS removed
|
|
56745
58072
|
FROM site_crawl_edges AS previous
|
|
56746
58073
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -56770,48 +58097,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56770
58097
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
56771
58098
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
56772
58099
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
56773
|
-
const internalCurrent = entity === "link" ?
|
|
56774
|
-
const internalPreviousJoin = entity === "link" ?
|
|
56775
|
-
const internalPrevious = entity === "link" ?
|
|
56776
|
-
const internalCurrentJoin = entity === "link" ?
|
|
58100
|
+
const internalCurrent = entity === "link" ? sql21`AND current.internal = 1` : sql21``;
|
|
58101
|
+
const internalPreviousJoin = entity === "link" ? sql21`AND previous.internal = 1` : sql21``;
|
|
58102
|
+
const internalPrevious = entity === "link" ? sql21`AND previous.internal = 1` : sql21``;
|
|
58103
|
+
const internalCurrentJoin = entity === "link" ? sql21`AND current.internal = 1` : sql21``;
|
|
56777
58104
|
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
56778
|
-
const afterKind = changeFilter === "added" ?
|
|
56779
|
-
const afterRows = includeAfter ? app.db.all(
|
|
58105
|
+
const afterKind = changeFilter === "added" ? sql21`previous.id IS NULL` : changeFilter === "changed" ? sql21`previous.id IS NOT NULL AND ${difference2}` : sql21`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
58106
|
+
const afterRows = includeAfter ? app.db.all(sql21`
|
|
56780
58107
|
SELECT ${entity} AS entity,
|
|
56781
58108
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
56782
|
-
current.${
|
|
56783
|
-
FROM ${
|
|
56784
|
-
LEFT JOIN ${
|
|
58109
|
+
current.${sql21.raw(keyColumn)} AS key
|
|
58110
|
+
FROM ${sql21.raw(table)} AS current
|
|
58111
|
+
LEFT JOIN ${sql21.raw(table)} AS previous
|
|
56785
58112
|
ON previous.project_id = ${beforeScope.projectId}
|
|
56786
58113
|
AND previous.run_id = ${beforeScope.runId}
|
|
56787
58114
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56788
58115
|
${internalPreviousJoin}
|
|
56789
|
-
AND previous.${
|
|
58116
|
+
AND previous.${sql21.raw(keyColumn)} = current.${sql21.raw(keyColumn)}
|
|
56790
58117
|
WHERE current.project_id = ${afterScope.projectId}
|
|
56791
58118
|
AND current.run_id = ${afterScope.runId}
|
|
56792
58119
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
56793
58120
|
${internalCurrent}
|
|
56794
|
-
AND current.${
|
|
58121
|
+
AND current.${sql21.raw(keyColumn)} > ${afterKey}
|
|
56795
58122
|
AND (${afterKind})
|
|
56796
|
-
ORDER BY current.${
|
|
58123
|
+
ORDER BY current.${sql21.raw(keyColumn)} ASC
|
|
56797
58124
|
LIMIT ${wanted2}
|
|
56798
58125
|
`) : [];
|
|
56799
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
56800
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
56801
|
-
FROM ${
|
|
56802
|
-
LEFT JOIN ${
|
|
58126
|
+
const removedRows = includeRemoved ? app.db.all(sql21`
|
|
58127
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql21.raw(keyColumn)} AS key
|
|
58128
|
+
FROM ${sql21.raw(table)} AS previous
|
|
58129
|
+
LEFT JOIN ${sql21.raw(table)} AS current
|
|
56803
58130
|
ON current.project_id = ${afterScope.projectId}
|
|
56804
58131
|
AND current.run_id = ${afterScope.runId}
|
|
56805
58132
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
56806
58133
|
${internalCurrentJoin}
|
|
56807
|
-
AND current.${
|
|
58134
|
+
AND current.${sql21.raw(keyColumn)} = previous.${sql21.raw(keyColumn)}
|
|
56808
58135
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56809
58136
|
AND previous.run_id = ${beforeScope.runId}
|
|
56810
58137
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56811
58138
|
${internalPrevious}
|
|
56812
|
-
AND previous.${
|
|
58139
|
+
AND previous.${sql21.raw(keyColumn)} > ${afterKey}
|
|
56813
58140
|
AND current.id IS NULL
|
|
56814
|
-
ORDER BY previous.${
|
|
58141
|
+
ORDER BY previous.${sql21.raw(keyColumn)} ASC
|
|
56815
58142
|
LIMIT ${wanted2}
|
|
56816
58143
|
`) : [];
|
|
56817
58144
|
const merged = [];
|
|
@@ -56841,17 +58168,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56841
58168
|
const visibleKeys = keyRows.slice(0, limit);
|
|
56842
58169
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
56843
58170
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
56844
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
56845
|
-
|
|
56846
|
-
|
|
56847
|
-
|
|
58171
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and44(
|
|
58172
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
58173
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
58174
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56848
58175
|
inArray19(siteCrawlPages.nodeKey, pageKeys)
|
|
56849
58176
|
)).all();
|
|
56850
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
56851
|
-
|
|
56852
|
-
|
|
56853
|
-
|
|
56854
|
-
|
|
58177
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and44(
|
|
58178
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
58179
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
58180
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
58181
|
+
eq56(siteCrawlEdges.internal, true),
|
|
56855
58182
|
inArray19(siteCrawlEdges.edgeKey, linkKeys)
|
|
56856
58183
|
)).all();
|
|
56857
58184
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -56975,16 +58302,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56975
58302
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, pages: [] };
|
|
56976
58303
|
}
|
|
56977
58304
|
const filters = [
|
|
56978
|
-
|
|
56979
|
-
|
|
56980
|
-
|
|
58305
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58306
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58307
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
56981
58308
|
];
|
|
56982
58309
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
56983
|
-
if (inventoryEligible != null) filters.push(
|
|
56984
|
-
if (request.query.fetchState) filters.push(
|
|
56985
|
-
if (request.query.indexabilityState) filters.push(
|
|
56986
|
-
if (request.query.auditState) filters.push(
|
|
56987
|
-
const where =
|
|
58310
|
+
if (inventoryEligible != null) filters.push(eq56(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
58311
|
+
if (request.query.fetchState) filters.push(eq56(siteCrawlPages.fetchState, request.query.fetchState));
|
|
58312
|
+
if (request.query.indexabilityState) filters.push(eq56(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
58313
|
+
if (request.query.auditState) filters.push(eq56(siteCrawlPages.auditState, request.query.auditState));
|
|
58314
|
+
const where = and44(...filters);
|
|
56988
58315
|
const total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
56989
58316
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
56990
58317
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -57017,10 +58344,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57017
58344
|
url: siteCrawlPages.url,
|
|
57018
58345
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
57019
58346
|
fetchState: siteCrawlPages.fetchState
|
|
57020
|
-
}).from(siteCrawlPages).where(
|
|
57021
|
-
|
|
57022
|
-
|
|
57023
|
-
|
|
58347
|
+
}).from(siteCrawlPages).where(and44(
|
|
58348
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58349
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58350
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
57024
58351
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
57025
58352
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
57026
58353
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -57079,16 +58406,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57079
58406
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, edges: [] };
|
|
57080
58407
|
}
|
|
57081
58408
|
const filters = [
|
|
57082
|
-
|
|
57083
|
-
|
|
57084
|
-
|
|
57085
|
-
|
|
58409
|
+
eq56(siteCrawlEdges.projectId, project.id),
|
|
58410
|
+
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58411
|
+
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58412
|
+
eq56(siteCrawlEdges.internal, true)
|
|
57086
58413
|
];
|
|
57087
|
-
if (request.query.sourceUrl) filters.push(
|
|
57088
|
-
if (request.query.targetUrl) filters.push(
|
|
58414
|
+
if (request.query.sourceUrl) filters.push(eq56(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
58415
|
+
if (request.query.targetUrl) filters.push(eq56(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
57089
58416
|
const followable = parseBoolean2(request.query.followable);
|
|
57090
|
-
if (followable != null) filters.push(
|
|
57091
|
-
const where =
|
|
58417
|
+
if (followable != null) filters.push(eq56(siteCrawlEdges.followable, followable));
|
|
58418
|
+
const where = and44(...filters);
|
|
57092
58419
|
const total = app.db.select({ value: count() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
57093
58420
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
57094
58421
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -57138,16 +58465,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57138
58465
|
};
|
|
57139
58466
|
}
|
|
57140
58467
|
const scope = [
|
|
57141
|
-
|
|
57142
|
-
|
|
57143
|
-
|
|
57144
|
-
|
|
58468
|
+
eq56(siteCrawlEdges.projectId, project.id),
|
|
58469
|
+
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58470
|
+
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58471
|
+
eq56(siteCrawlEdges.internal, true)
|
|
57145
58472
|
];
|
|
57146
|
-
const inboundMatch = request.query.nodeKey && request.query.url ? or11(
|
|
57147
|
-
const outboundMatch = request.query.nodeKey && request.query.url ? or11(
|
|
58473
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.targetUrl, request.query.url);
|
|
58474
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.sourceUrl, request.query.url);
|
|
57148
58475
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
57149
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
57150
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
58476
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and44(...scope, inboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
58477
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and44(...scope, outboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
57151
58478
|
return {
|
|
57152
58479
|
project: project.name,
|
|
57153
58480
|
hasCrawlData: true,
|
|
@@ -57178,11 +58505,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57178
58505
|
if (!snapshot.attemptId) {
|
|
57179
58506
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, total: 0, nextCursor: null, deadLinks: [] };
|
|
57180
58507
|
}
|
|
57181
|
-
const where =
|
|
57182
|
-
|
|
57183
|
-
|
|
57184
|
-
|
|
57185
|
-
|
|
58508
|
+
const where = and44(
|
|
58509
|
+
eq56(siteCrawlFindings.projectId, project.id),
|
|
58510
|
+
eq56(siteCrawlFindings.runId, snapshot.runId),
|
|
58511
|
+
eq56(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
58512
|
+
eq56(siteCrawlFindings.findingType, "dead-link")
|
|
57186
58513
|
);
|
|
57187
58514
|
const total = app.db.select({ value: count() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
57188
58515
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -57225,12 +58552,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57225
58552
|
status: runs.status,
|
|
57226
58553
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
57227
58554
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
57228
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
57229
|
-
|
|
57230
|
-
|
|
57231
|
-
)).where(
|
|
57232
|
-
|
|
57233
|
-
|
|
58555
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and44(
|
|
58556
|
+
eq56(siteCrawlRunRequests.projectId, runs.projectId),
|
|
58557
|
+
eq56(siteCrawlRunRequests.runId, runs.id)
|
|
58558
|
+
)).where(and44(
|
|
58559
|
+
eq56(runs.projectId, project.id),
|
|
58560
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57234
58561
|
inArray19(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
57235
58562
|
)).get();
|
|
57236
58563
|
if (existing) {
|
|
@@ -57248,7 +58575,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57248
58575
|
);
|
|
57249
58576
|
}
|
|
57250
58577
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
57251
|
-
const runId =
|
|
58578
|
+
const runId = crypto44.randomUUID();
|
|
57252
58579
|
tx.insert(runs).values({
|
|
57253
58580
|
id: runId,
|
|
57254
58581
|
projectId: project.id,
|
|
@@ -57281,8 +58608,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57281
58608
|
}
|
|
57282
58609
|
|
|
57283
58610
|
// ../api-routes/src/research.ts
|
|
57284
|
-
import
|
|
57285
|
-
import { and as
|
|
58611
|
+
import crypto45 from "crypto";
|
|
58612
|
+
import { and as and45, desc as desc25, eq as eq57 } from "drizzle-orm";
|
|
57286
58613
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
57287
58614
|
async function researchRoutes(app, opts) {
|
|
57288
58615
|
app.post("/projects/:name/research/runs", async (request, reply) => {
|
|
@@ -57308,19 +58635,19 @@ async function researchRoutes(app, opts) {
|
|
|
57308
58635
|
if (!adapter.modelValidationPattern.test(resolvedModel)) throw validationError(`Invalid resolved model "${resolvedModel}" for provider "${providerName}".`, { provider: providerName, model: resolvedModel, hint: adapter.modelValidationHint });
|
|
57309
58636
|
if (new Set(input.queries.map((query) => query.toLocaleLowerCase())).size !== input.queries.length) throw validationError("Research queries must be unique within a batch.");
|
|
57310
58637
|
const normalized = { queries: input.queries, provider: providerName, model: requestedModel, location: location ?? null };
|
|
57311
|
-
const requestHash2 =
|
|
58638
|
+
const requestHash2 = crypto45.createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
|
|
57312
58639
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
57313
58640
|
const decision = app.db.transaction((tx) => {
|
|
57314
58641
|
if (input.idempotencyKey) {
|
|
57315
|
-
const existing = tx.select().from(researchRuns).where(
|
|
58642
|
+
const existing = tx.select().from(researchRuns).where(and45(eq57(researchRuns.projectId, project.id), eq57(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
57316
58643
|
if (existing) {
|
|
57317
58644
|
if (existing.requestHash !== requestHash2) throw alreadyExists("Research idempotency key", input.idempotencyKey);
|
|
57318
58645
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
57319
58646
|
}
|
|
57320
58647
|
}
|
|
57321
|
-
const id =
|
|
58648
|
+
const id = crypto45.randomUUID();
|
|
57322
58649
|
tx.insert(researchRuns).values({ id, projectId: project.id, status: ResearchRunStatuses.queued, provider: providerName, requestedModel, resolvedModel, location: location ?? null, totalQueries: input.queries.length, idempotencyKey: input.idempotencyKey ?? null, requestHash: input.idempotencyKey ? requestHash2 : null, createdAt: now }).run();
|
|
57323
|
-
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id:
|
|
58650
|
+
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id: crypto45.randomUUID(), researchRunId: id, position, queryText: query, status: ResearchQueryStatuses.queued, requestedModel, resolvedModel, groundingSources: [], citedDomains: [], searchQueries: [], createdAt: now }).run();
|
|
57324
58651
|
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
57325
58652
|
return { reused: false, id, shouldDispatch: true };
|
|
57326
58653
|
});
|
|
@@ -57333,7 +58660,7 @@ async function researchRoutes(app, opts) {
|
|
|
57333
58660
|
const project = resolveProject(app.db, request.params.name);
|
|
57334
58661
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
57335
58662
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
57336
|
-
const runs2 = app.db.select().from(researchRuns).where(
|
|
58663
|
+
const runs2 = app.db.select().from(researchRuns).where(eq57(researchRuns.projectId, project.id)).orderBy(desc25(researchRuns.createdAt)).limit(limit).all().map(serializeRun);
|
|
57337
58664
|
return { runs: runs2 };
|
|
57338
58665
|
});
|
|
57339
58666
|
app.get("/projects/:name/research/runs/:runId", async (request) => {
|
|
@@ -57342,9 +58669,9 @@ async function researchRoutes(app, opts) {
|
|
|
57342
58669
|
});
|
|
57343
58670
|
}
|
|
57344
58671
|
function getDetail(app, projectId, id) {
|
|
57345
|
-
const row = app.db.select().from(researchRuns).where(
|
|
58672
|
+
const row = app.db.select().from(researchRuns).where(and45(eq57(researchRuns.id, id), eq57(researchRuns.projectId, projectId))).get();
|
|
57346
58673
|
if (!row) throw notFound("Research run", id);
|
|
57347
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
58674
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq57(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
57348
58675
|
return { ...serializeRun(row), queries: queries2 };
|
|
57349
58676
|
}
|
|
57350
58677
|
function serializeRun(row) {
|
|
@@ -57554,6 +58881,11 @@ async function apiRoutes(app, opts) {
|
|
|
57554
58881
|
vercelTrafficCredentialStore: opts.vercelTrafficCredentialStore,
|
|
57555
58882
|
pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
|
|
57556
58883
|
vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,
|
|
58884
|
+
cloudflareTrafficCredentialStore: opts.cloudflareTrafficCredentialStore,
|
|
58885
|
+
cloudflareTrafficIngestUrl: opts.cloudflareTrafficIngestUrl,
|
|
58886
|
+
cloudflareIngestRateLimitMax: opts.cloudflareIngestRateLimitMax,
|
|
58887
|
+
cloudflareIngestIpRateLimitMax: opts.cloudflareIngestIpRateLimitMax,
|
|
58888
|
+
defaultSampleLimit: opts.defaultTrafficSampleLimit,
|
|
57557
58889
|
onTrafficSynced: opts.onTrafficSynced,
|
|
57558
58890
|
onScheduleUpdated: opts.onScheduleUpdated,
|
|
57559
58891
|
allowLoopbackWebhooks: opts.allowLoopbackWebhooks
|
|
@@ -57734,11 +59066,62 @@ function buildTrafficSourceValidators(opts) {
|
|
|
57734
59066
|
validateScopes: () => null
|
|
57735
59067
|
};
|
|
57736
59068
|
}
|
|
59069
|
+
if (opts.cloudflareTrafficCredentialStore) {
|
|
59070
|
+
const store = opts.cloudflareTrafficCredentialStore;
|
|
59071
|
+
validators[TrafficSourceTypes.cloudflare] = {
|
|
59072
|
+
validateCredentials: (source) => {
|
|
59073
|
+
const sourceMode = source.configJson.deliveryMode;
|
|
59074
|
+
if (sourceMode !== void 0 && sourceMode !== "direct-push") return null;
|
|
59075
|
+
const record = store.getConnection(source.projectName);
|
|
59076
|
+
if (!record) {
|
|
59077
|
+
return {
|
|
59078
|
+
status: CheckStatuses.fail,
|
|
59079
|
+
code: "traffic.credentials.missing",
|
|
59080
|
+
summary: `No Cloudflare direct-push credential is stored for project "${source.projectName}".`,
|
|
59081
|
+
remediation: "Re-run `canonry traffic connect cloudflare <project> --zone-id <id>` from the credential-owning host."
|
|
59082
|
+
};
|
|
59083
|
+
}
|
|
59084
|
+
const recordMode = record.deliveryMode;
|
|
59085
|
+
if (recordMode !== "direct-push") {
|
|
59086
|
+
return {
|
|
59087
|
+
status: CheckStatuses.fail,
|
|
59088
|
+
code: "traffic.credentials.mode-mismatch",
|
|
59089
|
+
summary: `The stored Cloudflare credential mode does not match direct-push source "${source.displayName}".`,
|
|
59090
|
+
remediation: "Reconnect the Cloudflare source from the credential-owning host."
|
|
59091
|
+
};
|
|
59092
|
+
}
|
|
59093
|
+
if (record.sourceId !== source.id) {
|
|
59094
|
+
return {
|
|
59095
|
+
status: CheckStatuses.fail,
|
|
59096
|
+
code: "traffic.credentials.source-mismatch",
|
|
59097
|
+
summary: `The stored Cloudflare credential belongs to a different source than "${source.displayName}".`,
|
|
59098
|
+
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
59099
|
+
};
|
|
59100
|
+
}
|
|
59101
|
+
const bearerToken = record.bearerToken;
|
|
59102
|
+
const hmacSecret = record.hmacSecret;
|
|
59103
|
+
if (typeof bearerToken !== "string" || bearerToken.length === 0 || typeof hmacSecret !== "string" || hmacSecret.length === 0 || !source.ingestTokenHash || hashCloudflareBearerToken(bearerToken) !== source.ingestTokenHash) {
|
|
59104
|
+
return {
|
|
59105
|
+
status: CheckStatuses.fail,
|
|
59106
|
+
code: "traffic.credentials.bearer-mismatch",
|
|
59107
|
+
summary: `The stored Cloudflare push credential does not match source "${source.displayName}".`,
|
|
59108
|
+
remediation: "Reconnect the Cloudflare source and redeploy the generated Worker secrets."
|
|
59109
|
+
};
|
|
59110
|
+
}
|
|
59111
|
+
return {
|
|
59112
|
+
status: CheckStatuses.ok,
|
|
59113
|
+
code: "traffic.credentials.resolved",
|
|
59114
|
+
summary: `Cloudflare direct-push credentials match source "${source.displayName}".`
|
|
59115
|
+
};
|
|
59116
|
+
},
|
|
59117
|
+
validateScopes: () => null
|
|
59118
|
+
};
|
|
59119
|
+
}
|
|
57737
59120
|
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
57738
59121
|
}
|
|
57739
59122
|
|
|
57740
59123
|
// src/intelligence-service.ts
|
|
57741
|
-
import
|
|
59124
|
+
import crypto46 from "crypto";
|
|
57742
59125
|
|
|
57743
59126
|
// src/logger.ts
|
|
57744
59127
|
var IS_TTY = process.stdout.isTTY === true;
|
|
@@ -57946,9 +59329,9 @@ var IntelligenceService = class {
|
|
|
57946
59329
|
*/
|
|
57947
59330
|
analyzeAndPersist(runId, projectId) {
|
|
57948
59331
|
const recentRuns = this.db.select().from(runs).where(
|
|
57949
|
-
|
|
57950
|
-
|
|
57951
|
-
or12(
|
|
59332
|
+
and46(
|
|
59333
|
+
eq58(runs.projectId, projectId),
|
|
59334
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
57952
59335
|
// Defensive: RunCoordinator already skips probes before this is
|
|
57953
59336
|
// called, but if a future call site invokes analyzeAndPersist
|
|
57954
59337
|
// directly for a probe, probes still must not pollute the
|
|
@@ -58030,7 +59413,7 @@ var IntelligenceService = class {
|
|
|
58030
59413
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
58031
59414
|
*/
|
|
58032
59415
|
analyzeAndPersistGbp(runId, projectId) {
|
|
58033
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
59416
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq58(runs.id, runId)).get();
|
|
58034
59417
|
if (!runRow) {
|
|
58035
59418
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
58036
59419
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -58038,11 +59421,11 @@ var IntelligenceService = class {
|
|
|
58038
59421
|
}
|
|
58039
59422
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
58040
59423
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
58041
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
58042
|
-
|
|
58043
|
-
|
|
58044
|
-
|
|
58045
|
-
|
|
59424
|
+
const selected = this.db.select().from(gbpLocations).where(and46(
|
|
59425
|
+
eq58(gbpLocations.projectId, projectId),
|
|
59426
|
+
eq58(gbpLocations.selected, true),
|
|
59427
|
+
gte14(gbpLocations.syncedAt, windowStart),
|
|
59428
|
+
lte11(gbpLocations.syncedAt, windowEnd)
|
|
58046
59429
|
)).all();
|
|
58047
59430
|
if (selected.length === 0) {
|
|
58048
59431
|
log.info("gbp-intelligence.skip", { runId, reason: "no locations synced during run" });
|
|
@@ -58075,12 +59458,12 @@ var IntelligenceService = class {
|
|
|
58075
59458
|
}
|
|
58076
59459
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
58077
59460
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
58078
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
58079
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
58080
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
58081
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
59461
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and46(eq58(gbpDailyMetrics.projectId, projectId), eq58(gbpDailyMetrics.locationName, locationName))).all();
|
|
59462
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and46(eq58(gbpPlaceActions.projectId, projectId), eq58(gbpPlaceActions.locationName, locationName))).all();
|
|
59463
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and46(eq58(gbpLodgingSnapshots.projectId, projectId), eq58(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc26(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
59464
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and46(eq58(gbpLocations.projectId, projectId), eq58(gbpLocations.locationName, locationName))).get();
|
|
58082
59465
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
58083
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
59466
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and46(eq58(gbpPlaceDetails.projectId, projectId), eq58(gbpPlaceDetails.locationName, locationName))).orderBy(desc26(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
58084
59467
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
58085
59468
|
const summary = buildGbpSummary({
|
|
58086
59469
|
locationName,
|
|
@@ -58116,7 +59499,7 @@ var IntelligenceService = class {
|
|
|
58116
59499
|
/** Build the month-over-month keyword series for a location from the
|
|
58117
59500
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
58118
59501
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
58119
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
59502
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and46(eq58(gbpKeywordMonthly.projectId, projectId), eq58(gbpKeywordMonthly.locationName, locationName))).all();
|
|
58120
59503
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
58121
59504
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
58122
59505
|
const recentMonth = months[0] ?? null;
|
|
@@ -58147,7 +59530,7 @@ var IntelligenceService = class {
|
|
|
58147
59530
|
*/
|
|
58148
59531
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
58149
59532
|
const covered = new Set(coveredLocationNames);
|
|
58150
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
59533
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and46(eq58(insights.projectId, projectId), eq58(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
58151
59534
|
const staleIds = [];
|
|
58152
59535
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
58153
59536
|
for (const row of existing) {
|
|
@@ -58158,7 +59541,7 @@ var IntelligenceService = class {
|
|
|
58158
59541
|
}
|
|
58159
59542
|
this.db.transaction((tx) => {
|
|
58160
59543
|
for (const id of staleIds) {
|
|
58161
|
-
tx.delete(insights).where(
|
|
59544
|
+
tx.delete(insights).where(eq58(insights.id, id)).run();
|
|
58162
59545
|
}
|
|
58163
59546
|
for (const insight of gbpInsights) {
|
|
58164
59547
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -58236,7 +59619,7 @@ var IntelligenceService = class {
|
|
|
58236
59619
|
* create per run + aggregate). DB is left untouched.
|
|
58237
59620
|
*/
|
|
58238
59621
|
backfill(projectName, opts, onProgress) {
|
|
58239
|
-
const project = this.db.select().from(projects).where(
|
|
59622
|
+
const project = this.db.select().from(projects).where(eq58(projects.name, projectName)).get();
|
|
58240
59623
|
if (!project) {
|
|
58241
59624
|
throw new Error(`Project "${projectName}" not found`);
|
|
58242
59625
|
}
|
|
@@ -58249,9 +59632,9 @@ var IntelligenceService = class {
|
|
|
58249
59632
|
sinceTimestamp = parsed;
|
|
58250
59633
|
}
|
|
58251
59634
|
const allRuns = this.db.select().from(runs).where(
|
|
58252
|
-
|
|
58253
|
-
|
|
58254
|
-
or12(
|
|
59635
|
+
and46(
|
|
59636
|
+
eq58(runs.projectId, project.id),
|
|
59637
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
58255
59638
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
58256
59639
|
ne7(runs.trigger, RunTriggers.probe)
|
|
58257
59640
|
)
|
|
@@ -58330,7 +59713,7 @@ var IntelligenceService = class {
|
|
|
58330
59713
|
return { processed, skipped, totalInsights };
|
|
58331
59714
|
}
|
|
58332
59715
|
loadTrackedCompetitors(projectId) {
|
|
58333
|
-
return this.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
59716
|
+
return this.db.select({ domain: competitors.domain }).from(competitors).where(eq58(competitors.projectId, projectId)).all().map((r) => r.domain);
|
|
58334
59717
|
}
|
|
58335
59718
|
/**
|
|
58336
59719
|
* Wipe transition signals from an analysis result while keeping health.
|
|
@@ -58351,15 +59734,15 @@ var IntelligenceService = class {
|
|
|
58351
59734
|
}
|
|
58352
59735
|
persistResult(result, runId, projectId) {
|
|
58353
59736
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
58354
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
59737
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq58(insights.runId, runId)).all();
|
|
58355
59738
|
for (const row of existingInsights) {
|
|
58356
59739
|
if (row.dismissed) {
|
|
58357
59740
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
58358
59741
|
}
|
|
58359
59742
|
}
|
|
58360
59743
|
this.db.transaction((tx) => {
|
|
58361
|
-
tx.delete(insights).where(
|
|
58362
|
-
tx.delete(healthSnapshots).where(
|
|
59744
|
+
tx.delete(insights).where(eq58(insights.runId, runId)).run();
|
|
59745
|
+
tx.delete(healthSnapshots).where(eq58(healthSnapshots.runId, runId)).run();
|
|
58363
59746
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
58364
59747
|
for (const insight of result.insights) {
|
|
58365
59748
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -58379,7 +59762,7 @@ var IntelligenceService = class {
|
|
|
58379
59762
|
}).run();
|
|
58380
59763
|
}
|
|
58381
59764
|
tx.insert(healthSnapshots).values({
|
|
58382
|
-
id:
|
|
59765
|
+
id: crypto46.randomUUID(),
|
|
58383
59766
|
projectId,
|
|
58384
59767
|
runId,
|
|
58385
59768
|
overallCitedRate: String(result.health.overallCitedRate),
|
|
@@ -58412,24 +59795,24 @@ var IntelligenceService = class {
|
|
|
58412
59795
|
applySeverityTiering(rawInsights, excludeRunId, projectId) {
|
|
58413
59796
|
const regressions = rawInsights.filter((i) => i.type === "regression");
|
|
58414
59797
|
if (regressions.length === 0) return rawInsights;
|
|
58415
|
-
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(
|
|
59798
|
+
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(eq58(gscSearchData.projectId, projectId)).all();
|
|
58416
59799
|
const gscConnected = gscRows.length > 0;
|
|
58417
59800
|
const gscImpressionsByQuery = /* @__PURE__ */ new Map();
|
|
58418
59801
|
for (const row of gscRows) {
|
|
58419
59802
|
const key = row.query.toLowerCase();
|
|
58420
59803
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
58421
59804
|
}
|
|
58422
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
59805
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq58(projects.id, projectId)).get();
|
|
58423
59806
|
const locationCount = Math.max(
|
|
58424
59807
|
1,
|
|
58425
59808
|
(projectRow?.locations ?? []).length
|
|
58426
59809
|
);
|
|
58427
59810
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
58428
59811
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
58429
|
-
|
|
58430
|
-
|
|
58431
|
-
|
|
58432
|
-
or12(
|
|
59812
|
+
and46(
|
|
59813
|
+
eq58(runs.projectId, projectId),
|
|
59814
|
+
eq58(runs.kind, RunKinds["answer-visibility"]),
|
|
59815
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
58433
59816
|
// Defensive — see top of file.
|
|
58434
59817
|
ne7(runs.trigger, RunTriggers.probe)
|
|
58435
59818
|
)
|
|
@@ -58449,7 +59832,7 @@ var IntelligenceService = class {
|
|
|
58449
59832
|
const haveHistory = recentRunIds.length > 0;
|
|
58450
59833
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
58451
59834
|
if (haveHistory) {
|
|
58452
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
59835
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and46(eq58(insights.type, "regression"), inArray20(insights.runId, recentRunIds))).all();
|
|
58453
59836
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
58454
59837
|
for (const row of priorRows) {
|
|
58455
59838
|
if (!row.runId) continue;
|
|
@@ -58478,7 +59861,7 @@ var IntelligenceService = class {
|
|
|
58478
59861
|
});
|
|
58479
59862
|
}
|
|
58480
59863
|
buildRunData(runId, projectId, completedAt, location = null) {
|
|
58481
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
59864
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq58(projects.id, projectId)).get();
|
|
58482
59865
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
58483
59866
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
58484
59867
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -58495,7 +59878,7 @@ var IntelligenceService = class {
|
|
|
58495
59878
|
citedDomains: querySnapshots.citedDomains,
|
|
58496
59879
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
58497
59880
|
snapshotLocation: querySnapshots.location
|
|
58498
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
59881
|
+
}).from(querySnapshots).leftJoin(queries, eq58(querySnapshots.queryId, queries.id)).where(eq58(querySnapshots.runId, runId)).all();
|
|
58499
59882
|
const snapshots = [];
|
|
58500
59883
|
let orphanCount = 0;
|
|
58501
59884
|
for (const r of rows) {
|
|
@@ -58658,6 +60041,12 @@ export {
|
|
|
58658
60041
|
listCachedReleases,
|
|
58659
60042
|
pruneCachedRelease,
|
|
58660
60043
|
resolveVercelSyncDeadlineMs,
|
|
60044
|
+
CLOUDFLARE_WORKER_BINDINGS,
|
|
60045
|
+
CLOUDFLARE_WORKER_GENERATED_MARKER,
|
|
60046
|
+
CLOUDFLARE_WRANGLER_GENERATED_MARKER,
|
|
60047
|
+
DEFAULT_BOT_LIST,
|
|
60048
|
+
generateWorkerScript,
|
|
60049
|
+
generateWranglerToml,
|
|
58661
60050
|
executeDiscovery,
|
|
58662
60051
|
markSessionFailed,
|
|
58663
60052
|
apiRoutes,
|