@canonry/canonry 4.156.0 → 4.158.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/assets/agent-workspace/skills/canonry/SKILL.md +8 -3
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +454 -21
- package/assets/assets/{AuditHistoryPanel-gmJMVlXX.js → AuditHistoryPanel-McQfjoU9.js} +1 -1
- package/assets/assets/{BacklinksPage-Bv2sTCyN.js → BacklinksPage-Da8y1Wv8.js} +1 -1
- package/assets/assets/{HistoryPage-13q7GkiI.js → HistoryPage-9MJfhRyT.js} +1 -1
- package/assets/assets/{MeasurementPropertyPage-BgBqXT1p.js → MeasurementPropertyPage-BiNAHEJW.js} +1 -1
- package/assets/assets/ProjectPage-mWSMjXrR.js +15 -0
- package/assets/assets/{RunRow-bPzapIGJ.js → RunRow-DCG2LBlS.js} +1 -1
- package/assets/assets/{RunsPage-Dqfz18lH.js → RunsPage-ByWA_FSk.js} +1 -1
- package/assets/assets/{SettingsPage-_wMecn-V.js → SettingsPage-DdSeKAKF.js} +1 -1
- package/assets/assets/{TrafficPage-RYBdsOLB.js → TrafficPage-5u8rPcBt.js} +1 -1
- package/assets/assets/{TrafficSourceDetailPage-DpeMMyKU.js → TrafficSourceDetailPage-Dm-fK87u.js} +1 -1
- package/assets/assets/{extract-error-message-DUarAlyD.js → extract-error-message-CtxkAd6E.js} +1 -1
- package/assets/assets/{index-DP-fT2uI.js → index-DUfg50Pl.js} +28 -28
- package/assets/assets/index-cXb3mfeQ.css +1 -0
- package/assets/assets/{react-sigma_core.esm.min-DnypJl5w.js → react-sigma_core.esm.min-ComIyDKi.js} +1 -1
- package/assets/index.html +2 -2
- package/dist/{chunk-ZWESOFA5.js → chunk-4VZP64GZ.js} +273 -15
- package/dist/{chunk-AMPT2EP7.js → chunk-DEVAEC2S.js} +121 -1
- package/dist/{chunk-CPASKGUE.js → chunk-LK6YFHG4.js} +35 -1
- package/dist/{chunk-ABNN2ZVR.js → chunk-YP4K2KXH.js} +1933 -451
- package/dist/cli.js +667 -142
- package/dist/index.d.ts +32 -0
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-WHC7B32V.js → intelligence-service-LQ6ZRWMF.js} +2 -2
- package/dist/mcp.js +2 -2
- package/package.json +12 -10
- package/assets/assets/ProjectPage-EsGwKhot.js +0 -15
- package/assets/assets/index-BxegEwic.css +0 -1
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
CheckScopes,
|
|
36
36
|
CheckStatuses,
|
|
37
37
|
CitationStates,
|
|
38
|
+
CloudflareTrafficDeliveryModes,
|
|
38
39
|
ContentActions,
|
|
39
40
|
DEFAULT_DISCOVERY_PROMOTE_BUCKETS,
|
|
40
41
|
DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES,
|
|
@@ -81,6 +82,8 @@ import {
|
|
|
81
82
|
SITE_HEALTH_PATH_DEFAULT_MAX_DEPTH,
|
|
82
83
|
SITE_HEALTH_PATH_MAX_DEPTH,
|
|
83
84
|
SITE_HEALTH_PATH_MAX_VISITED_NODES,
|
|
85
|
+
SITE_HEALTH_SCANS_DEFAULT_LIMIT,
|
|
86
|
+
SITE_HEALTH_SCANS_MAX_LIMIT,
|
|
84
87
|
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_EDGES,
|
|
85
88
|
SITE_HEALTH_SUBGRAPH_DEFAULT_MAX_NODES,
|
|
86
89
|
SITE_HEALTH_SUBGRAPH_MAX_EDGES,
|
|
@@ -197,6 +200,9 @@ import {
|
|
|
197
200
|
classifySkillFile,
|
|
198
201
|
classifySurfaceFromCategory,
|
|
199
202
|
classifyTrafficPath,
|
|
203
|
+
cloudflareTrafficSourceConfigSchema,
|
|
204
|
+
cloudflareWorkerIngestRequestSchema,
|
|
205
|
+
cloudflareWorkerIngestResponseSchema,
|
|
200
206
|
clusterByCosine,
|
|
201
207
|
coerceSkillManifest,
|
|
202
208
|
compactDateToIso,
|
|
@@ -504,6 +510,8 @@ import {
|
|
|
504
510
|
siteCrawlSummarySchema,
|
|
505
511
|
siteHealthChangesResponseSchema,
|
|
506
512
|
siteHealthPathResponseSchema,
|
|
513
|
+
siteHealthScansResponseSchema,
|
|
514
|
+
siteHealthStateSchema,
|
|
507
515
|
siteHealthSubgraphResponseSchema,
|
|
508
516
|
snapshotDiffResponseSchema,
|
|
509
517
|
snapshotListResponseSchema,
|
|
@@ -520,6 +528,8 @@ import {
|
|
|
520
528
|
textContainsBrandAlias,
|
|
521
529
|
textContainsDomain,
|
|
522
530
|
trafficBackfillResponseSchema,
|
|
531
|
+
trafficConnectCloudflareRequestSchema,
|
|
532
|
+
trafficConnectCloudflareResponseSchema,
|
|
523
533
|
trafficConnectVercelRequestSchema,
|
|
524
534
|
trafficConnectWordpressRequestSchema,
|
|
525
535
|
trafficEventsResponseSchema,
|
|
@@ -553,10 +563,10 @@ import {
|
|
|
553
563
|
wordpressSchemaDeployResultDtoSchema,
|
|
554
564
|
wordpressSchemaStatusResultDtoSchema,
|
|
555
565
|
wordpressStatusDtoSchema
|
|
556
|
-
} from "./chunk-
|
|
566
|
+
} from "./chunk-DEVAEC2S.js";
|
|
557
567
|
|
|
558
568
|
// src/intelligence-service.ts
|
|
559
|
-
import { eq as
|
|
569
|
+
import { eq as eq58, desc as desc26, asc as asc11, and as and46, ne as ne7, or as or12, inArray as inArray20, gte as gte14, lte as lte11 } from "drizzle-orm";
|
|
560
570
|
|
|
561
571
|
// ../db/src/client.ts
|
|
562
572
|
import { mkdirSync } from "fs";
|
|
@@ -657,6 +667,7 @@ __export(schema_exports, {
|
|
|
657
667
|
siteCrawlPages: () => siteCrawlPages,
|
|
658
668
|
siteCrawlRunRequests: () => siteCrawlRunRequests,
|
|
659
669
|
siteCrawlSnapshots: () => siteCrawlSnapshots,
|
|
670
|
+
trafficEventReceipts: () => trafficEventReceipts,
|
|
660
671
|
trafficSources: () => trafficSources,
|
|
661
672
|
usageCounters: () => usageCounters,
|
|
662
673
|
userSessions: () => userSessions,
|
|
@@ -2048,12 +2059,31 @@ var trafficSources = sqliteTable("traffic_sources", {
|
|
|
2048
2059
|
lastEventIds: text("last_event_ids", { mode: "json" }).$type(),
|
|
2049
2060
|
archivedAt: text("archived_at"),
|
|
2050
2061
|
configJson: text("config_json", { mode: "json" }).$type().notNull().default({}),
|
|
2062
|
+
// sha256 hex of the per-source bearer token issued at connect time. Only
|
|
2063
|
+
// populated for push-receive source types (currently `cloudflare`); pull
|
|
2064
|
+
// adapters leave this NULL. The cleartext bearer + HMAC secret never live
|
|
2065
|
+
// in the DB — they go to `~/.canonry/config.yaml` under the per-type
|
|
2066
|
+
// connections block.
|
|
2067
|
+
ingestTokenHash: text("ingest_token_hash"),
|
|
2068
|
+
// Semver reported by the most recent forwarded event. Drives the
|
|
2069
|
+
// `traffic.source.worker-version` doctor check. NULL until the first
|
|
2070
|
+
// event arrives or for source types that don't forward versioned events.
|
|
2071
|
+
lastWorkerVersion: text("last_worker_version"),
|
|
2051
2072
|
createdAt: text("created_at").notNull(),
|
|
2052
2073
|
updatedAt: text("updated_at").notNull()
|
|
2053
2074
|
}, (table) => [
|
|
2054
2075
|
index("idx_traffic_sources_project").on(table.projectId),
|
|
2055
2076
|
index("idx_traffic_sources_project_status").on(table.projectId, table.status)
|
|
2056
2077
|
]);
|
|
2078
|
+
var trafficEventReceipts = sqliteTable("traffic_event_receipts", {
|
|
2079
|
+
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
2080
|
+
eventId: text("event_id").notNull(),
|
|
2081
|
+
receivedAt: text("received_at").notNull(),
|
|
2082
|
+
expiresAt: text("expires_at").notNull()
|
|
2083
|
+
}, (table) => [
|
|
2084
|
+
primaryKey({ columns: [table.sourceId, table.eventId] }),
|
|
2085
|
+
index("idx_traffic_event_receipts_expires").on(table.sourceId, table.expiresAt)
|
|
2086
|
+
]);
|
|
2057
2087
|
var crawlerEventsHourly = sqliteTable("crawler_events_hourly", {
|
|
2058
2088
|
projectId: text("project_id").notNull().references(() => projects.id, { onDelete: "cascade" }),
|
|
2059
2089
|
sourceId: text("source_id").notNull().references(() => trafficSources.id, { onDelete: "cascade" }),
|
|
@@ -5753,6 +5783,32 @@ var MIGRATION_VERSIONS = [
|
|
|
5753
5783
|
`CREATE INDEX IF NOT EXISTS idx_site_crawl_graph_edges_target_node
|
|
5754
5784
|
ON site_crawl_graph_edges(project_id, run_id, attempt_id, target_node_key)`
|
|
5755
5785
|
]
|
|
5786
|
+
},
|
|
5787
|
+
{
|
|
5788
|
+
version: 129,
|
|
5789
|
+
name: "traffic-ingest-foundation",
|
|
5790
|
+
// Push-receive traffic sources (currently only `cloudflare`) need a
|
|
5791
|
+
// per-source bearer for the Worker to authenticate against canonry's
|
|
5792
|
+
// ingest endpoint, plus a place to remember the deployed Worker version.
|
|
5793
|
+
// Durable receipts are transport-neutral: direct push and a future Queue
|
|
5794
|
+
// pull consumer both claim an event in the same transaction as rollups.
|
|
5795
|
+
// Cleartext credentials remain outside the database.
|
|
5796
|
+
//
|
|
5797
|
+
// Idempotent: `ALTER TABLE ADD COLUMN` errors with "duplicate column
|
|
5798
|
+
// name" on retry, which the runner already swallows.
|
|
5799
|
+
statements: [
|
|
5800
|
+
`ALTER TABLE traffic_sources ADD COLUMN ingest_token_hash TEXT`,
|
|
5801
|
+
`ALTER TABLE traffic_sources ADD COLUMN last_worker_version TEXT`,
|
|
5802
|
+
`CREATE TABLE IF NOT EXISTS traffic_event_receipts (
|
|
5803
|
+
source_id TEXT NOT NULL REFERENCES traffic_sources(id) ON DELETE CASCADE,
|
|
5804
|
+
event_id TEXT NOT NULL,
|
|
5805
|
+
received_at TEXT NOT NULL,
|
|
5806
|
+
expires_at TEXT NOT NULL,
|
|
5807
|
+
PRIMARY KEY (source_id, event_id)
|
|
5808
|
+
)`,
|
|
5809
|
+
`CREATE INDEX IF NOT EXISTS idx_traffic_event_receipts_expires
|
|
5810
|
+
ON traffic_event_receipts(source_id, expires_at)`
|
|
5811
|
+
]
|
|
5756
5812
|
}
|
|
5757
5813
|
];
|
|
5758
5814
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -8560,6 +8616,7 @@ function shouldSkipAuth(url) {
|
|
|
8560
8616
|
if (url.endsWith("/session") || url.endsWith("/session/setup")) return true;
|
|
8561
8617
|
if (url.endsWith("/auth/session") || url.endsWith("/auth/login") || url.endsWith("/auth/logout")) return true;
|
|
8562
8618
|
if (url.endsWith("/auth/sessions")) return true;
|
|
8619
|
+
if (url.endsWith("/traffic/cloudflare/ingest")) return true;
|
|
8563
8620
|
return false;
|
|
8564
8621
|
}
|
|
8565
8622
|
var USERS_READ_SCOPE = "users.read";
|
|
@@ -8625,7 +8682,7 @@ function isOverviewRead(url) {
|
|
|
8625
8682
|
}
|
|
8626
8683
|
function isTechnicalAeoRead(url) {
|
|
8627
8684
|
const rest = projectRouteRest(url);
|
|
8628
|
-
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8685
|
+
return rest === "technical-aeo" || rest === "technical-aeo/pages" || rest === "technical-aeo/trend" || rest === "technical-aeo/crawl" || rest === "technical-aeo/graph" || rest === "technical-aeo/subgraph" || rest === "technical-aeo/path" || rest === "technical-aeo/changes" || rest === "technical-aeo/crawl/pages/audit" || rest === "technical-aeo/crawl/pages" || rest === "technical-aeo/runs" || rest === "technical-aeo/structure" || rest === "technical-aeo/internal-links" || rest === "technical-aeo/internal-links/neighbors" || rest === "technical-aeo/dead-links";
|
|
8629
8686
|
}
|
|
8630
8687
|
function isReportRead(url) {
|
|
8631
8688
|
const rest = projectRouteRest(url);
|
|
@@ -27359,13 +27416,18 @@ var SCHEMA_TABLE = {
|
|
|
27359
27416
|
SiteCrawlSummaryDto: siteCrawlSummarySchema,
|
|
27360
27417
|
SiteHealthChangesResponseDto: siteHealthChangesResponseSchema,
|
|
27361
27418
|
SiteHealthPathResponseDto: siteHealthPathResponseSchema,
|
|
27419
|
+
SiteHealthScansResponseDto: siteHealthScansResponseSchema,
|
|
27362
27420
|
SiteHealthSubgraphResponseDto: siteHealthSubgraphResponseSchema,
|
|
27363
27421
|
SnapshotDiffResponse: snapshotDiffResponseSchema,
|
|
27364
27422
|
SnapshotListResponse: snapshotListResponseSchema,
|
|
27365
27423
|
SnapshotReportDto: snapshotReportSchema,
|
|
27366
27424
|
SourceBreakdownDto: sourceBreakdownDtoSchema,
|
|
27367
27425
|
TelemetryEventAcceptedDto: telemetryEventAcceptedDtoSchema,
|
|
27426
|
+
CloudflareWorkerIngestRequest: cloudflareWorkerIngestRequestSchema,
|
|
27427
|
+
CloudflareWorkerIngestResponse: cloudflareWorkerIngestResponseSchema,
|
|
27368
27428
|
TrafficBackfillResponse: trafficBackfillResponseSchema,
|
|
27429
|
+
TrafficConnectCloudflareRequest: trafficConnectCloudflareRequestSchema,
|
|
27430
|
+
TrafficConnectCloudflareResponse: trafficConnectCloudflareResponseSchema,
|
|
27369
27431
|
TrafficEventsResponse: trafficEventsResponseSchema,
|
|
27370
27432
|
TrafficSourceDetailDto: trafficSourceDetailDtoSchema,
|
|
27371
27433
|
TrafficSourceDto: trafficSourceDtoSchema,
|
|
@@ -32933,6 +32995,54 @@ var routeCatalog = [
|
|
|
32933
32995
|
502: errorResponse("Vercel request-logs endpoint probe failed (bad token, wrong project / team id, unreachable host, etc.).")
|
|
32934
32996
|
}
|
|
32935
32997
|
},
|
|
32998
|
+
{
|
|
32999
|
+
method: "post",
|
|
33000
|
+
path: "/api/v1/projects/{name}/traffic/connect/cloudflare",
|
|
33001
|
+
summary: "Connect a Cloudflare Worker traffic source",
|
|
33002
|
+
description: "Creates or updates a Cloudflare `direct-push` traffic source and returns a secret-free ES-module Worker plus Wrangler configuration. Per-source bearer and HMAC credentials remain in Canonry's credential store and are installed as Worker secret bindings; neither secret appears in source, TOML, API, or MCP output. Reconnect is idempotent: it reuses a matching source and credentials, preserves omitted metadata, and emits the current deployment package. Wrangler deploys the Worker without a route. The operator must attach the exact site route manually and set its Request limit failure mode to Fail open. No upstream probe is performed at connect time.",
|
|
33003
|
+
tags: ["traffic"],
|
|
33004
|
+
parameters: [nameParameter],
|
|
33005
|
+
requestBody: {
|
|
33006
|
+
required: false,
|
|
33007
|
+
content: {
|
|
33008
|
+
"application/json": {
|
|
33009
|
+
schema: { $ref: "#/components/schemas/TrafficConnectCloudflareRequest" }
|
|
33010
|
+
}
|
|
33011
|
+
}
|
|
33012
|
+
},
|
|
33013
|
+
responses: {
|
|
33014
|
+
200: jsonResponse("Connect response with generated Worker script.", "TrafficConnectCloudflareResponse"),
|
|
33015
|
+
400: errorResponse("Invalid Cloudflare connection request or credential storage not configured."),
|
|
33016
|
+
404: errorResponse("Project not found.")
|
|
33017
|
+
}
|
|
33018
|
+
},
|
|
33019
|
+
{
|
|
33020
|
+
method: "post",
|
|
33021
|
+
path: "/api/v1/projects/{name}/traffic/cloudflare/ingest",
|
|
33022
|
+
summary: "Ingest a batch of Cloudflare Worker events",
|
|
33023
|
+
description: "Direct-push protocol endpoint. The customer Worker sends selected edge-event batches authenticated by a per-source bearer and HMAC-SHA256 over `${timestamp}.${canonicalJson(payload)}`; object keys are sorted recursively before signing. The global cnry_* bearer does not apply. Headers: `Authorization: Bearer <per-source-token>`, `X-Canonry-Timestamp` (unix seconds), `X-Canonry-Signature` (hex HMAC), `X-Canonry-Source-Id`, and `X-Canonry-Worker-Version`. Authentication completes before project resolution and every auth/path/source mismatch returns the same 401. Accepted event ids are claimed durably in the same transaction as shared crawler, user-fetch, and paid/organic referral rollups.",
|
|
33024
|
+
tags: ["traffic"],
|
|
33025
|
+
auth: false,
|
|
33026
|
+
parameters: [nameParameter],
|
|
33027
|
+
requestBody: {
|
|
33028
|
+
required: true,
|
|
33029
|
+
content: {
|
|
33030
|
+
"application/json": {
|
|
33031
|
+
schema: { $ref: "#/components/schemas/CloudflareWorkerIngestRequest" }
|
|
33032
|
+
}
|
|
33033
|
+
}
|
|
33034
|
+
},
|
|
33035
|
+
responses: {
|
|
33036
|
+
200: jsonResponse(
|
|
33037
|
+
"Ingest acknowledged; droppedEvents includes normalization failures and durable-receipt duplicates.",
|
|
33038
|
+
"CloudflareWorkerIngestResponse"
|
|
33039
|
+
),
|
|
33040
|
+
400: errorResponse("Invalid ingest payload."),
|
|
33041
|
+
401: errorResponse("Authentication failed (bearer, signature, timestamp, or source id mismatch)."),
|
|
33042
|
+
413: errorResponse("The ingest payload exceeds 256 KiB."),
|
|
33043
|
+
429: errorResponse("The per-IP or authenticated-source ingest request budget was exceeded.")
|
|
33044
|
+
}
|
|
33045
|
+
},
|
|
32936
33046
|
{
|
|
32937
33047
|
method: "post",
|
|
32938
33048
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
@@ -33291,7 +33401,7 @@ var routeCatalog = [
|
|
|
33291
33401
|
],
|
|
33292
33402
|
responses: {
|
|
33293
33403
|
200: jsonResponse("Persisted crawl summary returned.", "SiteCrawlSummaryDto"),
|
|
33294
|
-
404: errorResponse("Project or
|
|
33404
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33295
33405
|
}
|
|
33296
33406
|
},
|
|
33297
33407
|
{
|
|
@@ -33308,7 +33418,7 @@ var routeCatalog = [
|
|
|
33308
33418
|
],
|
|
33309
33419
|
responses: {
|
|
33310
33420
|
200: jsonResponse("Bounded Site Health graph projection returned.", "SiteCrawlGraphResponseDto"),
|
|
33311
|
-
404: errorResponse("Project or
|
|
33421
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33312
33422
|
}
|
|
33313
33423
|
},
|
|
33314
33424
|
{
|
|
@@ -33403,6 +33513,7 @@ var routeCatalog = [
|
|
|
33403
33513
|
{ name: "inventoryEligible", in: "query", description: "Filter Canonry technical inventory eligibility (`true` or `false`).", schema: booleanSchema },
|
|
33404
33514
|
{ name: "fetchState", in: "query", description: "Filter by persisted fetch state.", schema: stringSchema },
|
|
33405
33515
|
{ name: "indexabilityState", in: "query", description: "Filter by crawler-derived indexability state; this is not Google index coverage.", schema: stringSchema },
|
|
33516
|
+
{ name: "healthState", in: "query", description: "Filter by the derived Site Health state shared with the dashboard and agents. Unlike `indexabilityState` this folds fetch state, canonical identity, and the crawler reasons into one decision, so `hidden` also covers redirects, robots blocks, non-HTML, and canonical-away pages. Values: `eligible`, `hidden`, `failed`, `unchecked`.", schema: { type: "string", enum: ["eligible", "hidden", "failed", "unchecked"] } },
|
|
33406
33517
|
{ name: "auditState", in: "query", description: "Filter by audit state.", schema: stringSchema },
|
|
33407
33518
|
{ name: "sort", in: "query", description: "Sort order. Defaults to `url`.", schema: { type: "string", enum: ["url", "path", "score-asc", "score-desc"] } },
|
|
33408
33519
|
crawlCursorParameter,
|
|
@@ -33410,7 +33521,7 @@ var routeCatalog = [
|
|
|
33410
33521
|
],
|
|
33411
33522
|
responses: {
|
|
33412
33523
|
200: jsonResponse("Crawl pages returned.", "SiteCrawlPagesResponseDto"),
|
|
33413
|
-
404: errorResponse("Project or
|
|
33524
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33414
33525
|
}
|
|
33415
33526
|
},
|
|
33416
33527
|
{
|
|
@@ -33428,7 +33539,7 @@ var routeCatalog = [
|
|
|
33428
33539
|
],
|
|
33429
33540
|
responses: {
|
|
33430
33541
|
200: jsonResponse("Immediate crawl children returned.", "SiteCrawlStructureResponseDto"),
|
|
33431
|
-
404: errorResponse("Project or
|
|
33542
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33432
33543
|
}
|
|
33433
33544
|
},
|
|
33434
33545
|
{
|
|
@@ -33448,7 +33559,7 @@ var routeCatalog = [
|
|
|
33448
33559
|
],
|
|
33449
33560
|
responses: {
|
|
33450
33561
|
200: jsonResponse("Internal crawl links returned.", "SiteCrawlInternalLinksResponseDto"),
|
|
33451
|
-
404: errorResponse("Project or
|
|
33562
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33452
33563
|
}
|
|
33453
33564
|
},
|
|
33454
33565
|
{
|
|
@@ -33467,7 +33578,7 @@ var routeCatalog = [
|
|
|
33467
33578
|
responses: {
|
|
33468
33579
|
200: jsonResponse("Bounded internal-link neighborhood returned.", "SiteCrawlNeighborsResponseDto"),
|
|
33469
33580
|
400: errorResponse("A nodeKey or URL is required."),
|
|
33470
|
-
404: errorResponse("Project or
|
|
33581
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33471
33582
|
}
|
|
33472
33583
|
},
|
|
33473
33584
|
{
|
|
@@ -33484,7 +33595,22 @@ var routeCatalog = [
|
|
|
33484
33595
|
],
|
|
33485
33596
|
responses: {
|
|
33486
33597
|
200: jsonResponse("Dead-link status and findings returned.", "SiteCrawlDeadLinksResponseDto"),
|
|
33487
|
-
404: errorResponse("Project or
|
|
33598
|
+
404: errorResponse("Project or site-audit run not found. A known run that published no crawl returns 200 with the no-crawl state instead.")
|
|
33599
|
+
}
|
|
33600
|
+
},
|
|
33601
|
+
{
|
|
33602
|
+
method: "get",
|
|
33603
|
+
path: "/api/v1/projects/{name}/technical-aeo/runs",
|
|
33604
|
+
summary: "List Site Health scan history",
|
|
33605
|
+
description: "Returns every non-probe `site-audit` run for the project, newest first, including runs still queued or running and runs that failed. `hasCrawlData` says whether that scan published a page and internal-link crawl: a legacy score-only scan is listed with `hasCrawlData=false` rather than hidden, and the crawl-scoped reads answer it with their no-crawl state instead of a 404.",
|
|
33606
|
+
tags: ["technical-aeo"],
|
|
33607
|
+
parameters: [
|
|
33608
|
+
nameParameter,
|
|
33609
|
+
{ name: "limit", in: "query", description: "Max scans returned (most recent first). Defaults to 20; max 100.", schema: { type: "integer", minimum: 1, maximum: 100 } }
|
|
33610
|
+
],
|
|
33611
|
+
responses: {
|
|
33612
|
+
200: jsonResponse("Site Health scan history returned.", "SiteHealthScansResponseDto"),
|
|
33613
|
+
404: errorResponse("Project not found.")
|
|
33488
33614
|
}
|
|
33489
33615
|
},
|
|
33490
33616
|
{
|
|
@@ -46811,7 +46937,7 @@ async function queryBacklinks(opts) {
|
|
|
46811
46937
|
const reversed = opts.targets.map(reverseDomain);
|
|
46812
46938
|
const targetList = reversed.map(quote).join(", ");
|
|
46813
46939
|
const limitClause = opts.limitPerTarget ? `QUALIFY row_number() OVER (PARTITION BY t.target_rev_domain ORDER BY v.num_hosts DESC) <= ${Math.floor(opts.limitPerTarget)}` : "";
|
|
46814
|
-
const
|
|
46940
|
+
const sql22 = `
|
|
46815
46941
|
WITH vertices AS (
|
|
46816
46942
|
SELECT * FROM read_csv(
|
|
46817
46943
|
${quote(opts.vertexPath)},
|
|
@@ -46847,7 +46973,7 @@ async function queryBacklinks(opts) {
|
|
|
46847
46973
|
const conn = await instance.connect();
|
|
46848
46974
|
let rows;
|
|
46849
46975
|
try {
|
|
46850
|
-
const reader = await conn.runAndReadAll(
|
|
46976
|
+
const reader = await conn.runAndReadAll(sql22);
|
|
46851
46977
|
rows = reader.getRowObjects();
|
|
46852
46978
|
} finally {
|
|
46853
46979
|
conn.disconnectSync?.();
|
|
@@ -47303,9 +47429,10 @@ async function backlinksRoutes(app, opts) {
|
|
|
47303
47429
|
}
|
|
47304
47430
|
|
|
47305
47431
|
// ../api-routes/src/traffic.ts
|
|
47306
|
-
import
|
|
47432
|
+
import crypto40 from "crypto";
|
|
47433
|
+
import { isIP } from "net";
|
|
47307
47434
|
import { Agent as UndiciAgent } from "undici";
|
|
47308
|
-
import { and as
|
|
47435
|
+
import { and as and39, desc as desc22, eq as eq47, gte as gte11, lte as lte10, sql as sql19 } from "drizzle-orm";
|
|
47309
47436
|
|
|
47310
47437
|
// ../api-routes/src/traffic-limits.ts
|
|
47311
47438
|
var VERCEL_MAX_SYNC_WINDOW_MS = 24 * 60 * 6e4;
|
|
@@ -50089,6 +50216,39 @@ var DEFAULT_AI_CRAWLER_RULES = [
|
|
|
50089
50216
|
userAgentPatterns: [/Amazonbot\//i]
|
|
50090
50217
|
}
|
|
50091
50218
|
];
|
|
50219
|
+
var DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS = [
|
|
50220
|
+
"GPTBot/",
|
|
50221
|
+
"OAI-SearchBot/",
|
|
50222
|
+
"ChatGPT-User/",
|
|
50223
|
+
"ClaudeBot/",
|
|
50224
|
+
"Claude-Web/",
|
|
50225
|
+
"Claude-SearchBot/",
|
|
50226
|
+
"Claude-User/",
|
|
50227
|
+
"anthropic-ai",
|
|
50228
|
+
"PerplexityBot/",
|
|
50229
|
+
"Google-Extended",
|
|
50230
|
+
"Google-Agent",
|
|
50231
|
+
"Bytespider",
|
|
50232
|
+
"Applebot-Extended",
|
|
50233
|
+
"Applebot/",
|
|
50234
|
+
"meta-externalagent",
|
|
50235
|
+
"CCBot/",
|
|
50236
|
+
"cohere-ai",
|
|
50237
|
+
"Diffbot",
|
|
50238
|
+
"MistralAI-User/",
|
|
50239
|
+
"MistralBot/",
|
|
50240
|
+
"DeepSeekBot",
|
|
50241
|
+
"xAI-Bot/",
|
|
50242
|
+
"Grok-Bot/",
|
|
50243
|
+
"GrokBot/",
|
|
50244
|
+
"Googlebot",
|
|
50245
|
+
"bingbot/",
|
|
50246
|
+
"DuckDuckBot",
|
|
50247
|
+
"YandexBot/",
|
|
50248
|
+
"Baiduspider",
|
|
50249
|
+
"Amazonbot/",
|
|
50250
|
+
"Perplexity-User/"
|
|
50251
|
+
];
|
|
50092
50252
|
var DEFAULT_AI_REFERRER_RULES = [
|
|
50093
50253
|
{ domain: AI_ENGINE_DOMAINS.chatgpt, operator: "OpenAI", product: "ChatGPT" },
|
|
50094
50254
|
{ domain: LEGACY_CHATGPT_DOMAIN, operator: "OpenAI", product: "ChatGPT" },
|
|
@@ -51109,6 +51269,716 @@ async function drainVercelTrafficEvents(options) {
|
|
|
51109
51269
|
return { events, subWindowCount, effectiveStartMs, retentionClamped, truncatedSliceCount, truncatedSliceStartsMs, deadlineSkippedSliceCount: deadlineSkippedSliceStartsMs.length, deadlineSkippedSliceStartsMs, drainedThroughMs: cursorMs, deadlineReached };
|
|
51110
51270
|
}
|
|
51111
51271
|
|
|
51272
|
+
// ../integration-cloudflare-worker/src/canonical-json.ts
|
|
51273
|
+
function canonicalizeCloudflareJson(value) {
|
|
51274
|
+
if (value === null) return "null";
|
|
51275
|
+
if (typeof value === "string" || typeof value === "boolean") {
|
|
51276
|
+
return JSON.stringify(value);
|
|
51277
|
+
}
|
|
51278
|
+
if (typeof value === "number") {
|
|
51279
|
+
if (!Number.isFinite(value)) {
|
|
51280
|
+
throw new TypeError("Cloudflare canonical JSON does not support non-finite numbers");
|
|
51281
|
+
}
|
|
51282
|
+
return JSON.stringify(value);
|
|
51283
|
+
}
|
|
51284
|
+
if (Array.isArray(value)) {
|
|
51285
|
+
return `[${value.map((item2) => canonicalizeCloudflareJson(item2)).join(",")}]`;
|
|
51286
|
+
}
|
|
51287
|
+
if (typeof value === "object") {
|
|
51288
|
+
const record = value;
|
|
51289
|
+
const fields = [];
|
|
51290
|
+
for (const key of Object.keys(record).sort()) {
|
|
51291
|
+
const fieldValue = record[key];
|
|
51292
|
+
if (fieldValue === void 0) continue;
|
|
51293
|
+
fields.push(`${JSON.stringify(key)}:${canonicalizeCloudflareJson(fieldValue)}`);
|
|
51294
|
+
}
|
|
51295
|
+
return `{${fields.join(",")}}`;
|
|
51296
|
+
}
|
|
51297
|
+
throw new TypeError(`Cloudflare canonical JSON does not support ${typeof value}`);
|
|
51298
|
+
}
|
|
51299
|
+
|
|
51300
|
+
// ../integration-cloudflare-worker/src/normalize.ts
|
|
51301
|
+
function emptyToNull2(value) {
|
|
51302
|
+
if (value === null || value.trim() === "") return null;
|
|
51303
|
+
return value;
|
|
51304
|
+
}
|
|
51305
|
+
function maybeLabel(value) {
|
|
51306
|
+
if (value === null) return void 0;
|
|
51307
|
+
if (typeof value === "string" && value === "") return void 0;
|
|
51308
|
+
return String(value);
|
|
51309
|
+
}
|
|
51310
|
+
function buildProviderLabels(cf) {
|
|
51311
|
+
if (!cf) return {};
|
|
51312
|
+
const candidates = {
|
|
51313
|
+
verifiedBot: maybeLabel(cf.verifiedBot),
|
|
51314
|
+
botScore: maybeLabel(cf.botScore),
|
|
51315
|
+
country: maybeLabel(cf.country),
|
|
51316
|
+
asn: maybeLabel(cf.asn),
|
|
51317
|
+
asOrganization: maybeLabel(cf.asOrganization)
|
|
51318
|
+
};
|
|
51319
|
+
return Object.fromEntries(
|
|
51320
|
+
Object.entries(candidates).filter(
|
|
51321
|
+
(entry) => typeof entry[1] === "string"
|
|
51322
|
+
)
|
|
51323
|
+
);
|
|
51324
|
+
}
|
|
51325
|
+
function normalizeCloudflareEdgeEvent(event) {
|
|
51326
|
+
if (!event.path) return null;
|
|
51327
|
+
if (!event.observedAt) return null;
|
|
51328
|
+
if (!event.eventId) return null;
|
|
51329
|
+
const host = emptyToNull2(event.host);
|
|
51330
|
+
const path8 = event.path;
|
|
51331
|
+
const queryString = emptyToNull2(event.queryString);
|
|
51332
|
+
const requestUrl = host ? `https://${host}${path8}${queryString ? `?${queryString}` : ""}` : null;
|
|
51333
|
+
return {
|
|
51334
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
51335
|
+
evidenceKind: TrafficEvidenceKinds["raw-request"],
|
|
51336
|
+
confidence: TrafficEventConfidences.observed,
|
|
51337
|
+
eventId: `cloudflare-worker:${event.eventId}`,
|
|
51338
|
+
observedAt: event.observedAt,
|
|
51339
|
+
method: emptyToNull2(event.method),
|
|
51340
|
+
requestUrl,
|
|
51341
|
+
host,
|
|
51342
|
+
path: path8,
|
|
51343
|
+
queryString,
|
|
51344
|
+
status: event.status,
|
|
51345
|
+
userAgent: emptyToNull2(event.userAgent),
|
|
51346
|
+
remoteIp: emptyToNull2(event.remoteIp),
|
|
51347
|
+
referer: emptyToNull2(event.referer),
|
|
51348
|
+
latencyMs: null,
|
|
51349
|
+
requestSizeBytes: null,
|
|
51350
|
+
responseSizeBytes: null,
|
|
51351
|
+
providerResource: {
|
|
51352
|
+
type: "cloudflare_zone",
|
|
51353
|
+
labels: {}
|
|
51354
|
+
},
|
|
51355
|
+
providerLabels: buildProviderLabels(event.cf)
|
|
51356
|
+
};
|
|
51357
|
+
}
|
|
51358
|
+
var normalizeCloudflareWorkerEvent = normalizeCloudflareEdgeEvent;
|
|
51359
|
+
|
|
51360
|
+
// ../integration-cloudflare-worker/src/types.ts
|
|
51361
|
+
var CLOUDFLARE_WORKER_BINDINGS = {
|
|
51362
|
+
deliveryMode: "CANONRY_DELIVERY_MODE",
|
|
51363
|
+
sourceId: "CANONRY_SOURCE_ID",
|
|
51364
|
+
ingestUrl: "CANONRY_INGEST_URL",
|
|
51365
|
+
workerVersion: "CANONRY_WORKER_VERSION",
|
|
51366
|
+
bearerToken: "CANONRY_BEARER_TOKEN",
|
|
51367
|
+
hmacSecret: "CANONRY_HMAC_SECRET"
|
|
51368
|
+
};
|
|
51369
|
+
var CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS = [
|
|
51370
|
+
CLOUDFLARE_WORKER_BINDINGS.bearerToken,
|
|
51371
|
+
CLOUDFLARE_WORKER_BINDINGS.hmacSecret
|
|
51372
|
+
];
|
|
51373
|
+
var CLOUDFLARE_WORKER_GENERATED_MARKER = "// canonry traffic Worker \u2014 generated; do not edit by hand";
|
|
51374
|
+
var CLOUDFLARE_WRANGLER_GENERATED_MARKER = "# canonry traffic Worker configuration \u2014 generated; do not edit by hand";
|
|
51375
|
+
|
|
51376
|
+
// ../integration-cloudflare-worker/src/script.ts
|
|
51377
|
+
var DEFAULT_BOT_LIST = {
|
|
51378
|
+
version: "2026-08-09",
|
|
51379
|
+
uaKeywords: [
|
|
51380
|
+
...DEFAULT_AI_CRAWLER_USER_AGENT_SUBSTRINGS,
|
|
51381
|
+
// Preserve a small forward-looking machine-traffic net. The shared
|
|
51382
|
+
// classifier still drops unknown products after delivery.
|
|
51383
|
+
"bot",
|
|
51384
|
+
"crawler",
|
|
51385
|
+
"spider"
|
|
51386
|
+
],
|
|
51387
|
+
refererDomains: DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain),
|
|
51388
|
+
utmSourceTokens: [
|
|
51389
|
+
.../* @__PURE__ */ new Set([
|
|
51390
|
+
...DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain),
|
|
51391
|
+
...DEFAULT_AI_REFERRER_RULES.map((rule) => rule.domain.split(".")[0]),
|
|
51392
|
+
// Legacy chat.openai.com referrals classify from their first label.
|
|
51393
|
+
"chat"
|
|
51394
|
+
])
|
|
51395
|
+
]
|
|
51396
|
+
};
|
|
51397
|
+
var DEFAULT_BOT_SCORE_MAX_FORWARD = 30;
|
|
51398
|
+
var WORKER_COMPATIBILITY_DATE = "2026-05-01";
|
|
51399
|
+
function jsString(value) {
|
|
51400
|
+
return JSON.stringify(value);
|
|
51401
|
+
}
|
|
51402
|
+
function jsArray(values) {
|
|
51403
|
+
return `[${values.map((v) => jsString(v)).join(", ")}]`;
|
|
51404
|
+
}
|
|
51405
|
+
function generateWorkerScript(opts) {
|
|
51406
|
+
const botScoreMax = opts.botScoreMaxForward ?? DEFAULT_BOT_SCORE_MAX_FORWARD;
|
|
51407
|
+
const canonicalJsonFunction = canonicalizeCloudflareJson.toString();
|
|
51408
|
+
return `${CLOUDFLARE_WORKER_GENERATED_MARKER}
|
|
51409
|
+
// worker version: ${opts.workerVersion}
|
|
51410
|
+
// bot-list version: ${opts.botList.version}
|
|
51411
|
+
// delivery mode: ${opts.deliveryMode}
|
|
51412
|
+
|
|
51413
|
+
const UA_KEYWORDS = ${jsArray(opts.botList.uaKeywords)}
|
|
51414
|
+
const REFERER_DOMAINS = ${jsArray(opts.botList.refererDomains)}
|
|
51415
|
+
const UTM_SOURCE_TOKENS = ${jsArray(opts.botList.utmSourceTokens)}
|
|
51416
|
+
const BOT_SCORE_MAX_FORWARD = ${String(botScoreMax)}
|
|
51417
|
+
const RETRY_DELAYS_MS = [250, 1000]
|
|
51418
|
+
const canonicalizeJson = (${canonicalJsonFunction})
|
|
51419
|
+
|
|
51420
|
+
function requireBinding(value, name) {
|
|
51421
|
+
if (typeof value === 'string' && value.length > 0) return value
|
|
51422
|
+
throw new Error('Missing required Worker binding: ' + name)
|
|
51423
|
+
}
|
|
51424
|
+
|
|
51425
|
+
function lower(value) {
|
|
51426
|
+
return typeof value === 'string' ? value.toLowerCase() : ''
|
|
51427
|
+
}
|
|
51428
|
+
|
|
51429
|
+
function uaMatches(ua) {
|
|
51430
|
+
const lc = lower(ua)
|
|
51431
|
+
if (!lc) return false
|
|
51432
|
+
for (const kw of UA_KEYWORDS) {
|
|
51433
|
+
if (lc.indexOf(lower(kw)) !== -1) return true
|
|
51434
|
+
}
|
|
51435
|
+
return false
|
|
51436
|
+
}
|
|
51437
|
+
|
|
51438
|
+
function hostMatchesKnownDomain(host) {
|
|
51439
|
+
let normalized = lower(host)
|
|
51440
|
+
if (normalized.startsWith('www.')) normalized = normalized.slice(4)
|
|
51441
|
+
if (normalized.endsWith('.')) normalized = normalized.slice(0, -1)
|
|
51442
|
+
if (!normalized) return false
|
|
51443
|
+
for (const domain of REFERER_DOMAINS) {
|
|
51444
|
+
if (normalized === domain || normalized.endsWith('.' + domain)) return true
|
|
51445
|
+
}
|
|
51446
|
+
return false
|
|
51447
|
+
}
|
|
51448
|
+
|
|
51449
|
+
function refererMatches(referer) {
|
|
51450
|
+
if (!referer) return false
|
|
51451
|
+
try {
|
|
51452
|
+
return hostMatchesKnownDomain(new URL(referer).hostname)
|
|
51453
|
+
} catch (_) {
|
|
51454
|
+
return false
|
|
51455
|
+
}
|
|
51456
|
+
}
|
|
51457
|
+
|
|
51458
|
+
function utmSourceMatches(value) {
|
|
51459
|
+
let normalized = lower(value).trim()
|
|
51460
|
+
if (normalized.startsWith('www.')) normalized = normalized.slice(4)
|
|
51461
|
+
if (normalized.endsWith('.')) normalized = normalized.slice(0, -1)
|
|
51462
|
+
if (!normalized) return false
|
|
51463
|
+
if (hostMatchesKnownDomain(normalized)) return true
|
|
51464
|
+
return UTM_SOURCE_TOKENS.indexOf(normalized) !== -1
|
|
51465
|
+
}
|
|
51466
|
+
|
|
51467
|
+
function urlUtmMatches(value) {
|
|
51468
|
+
if (!value) return false
|
|
51469
|
+
try {
|
|
51470
|
+
return utmSourceMatches(new URL(value).searchParams.get('utm_source'))
|
|
51471
|
+
} catch (_) {
|
|
51472
|
+
return false
|
|
51473
|
+
}
|
|
51474
|
+
}
|
|
51475
|
+
|
|
51476
|
+
function botSignals(cf) {
|
|
51477
|
+
if (!cf) return false
|
|
51478
|
+
const bm = cf.botManagement
|
|
51479
|
+
if (bm) {
|
|
51480
|
+
if (bm.verifiedBot === true) return true
|
|
51481
|
+
if (typeof bm.score === 'number' && bm.score < BOT_SCORE_MAX_FORWARD) return true
|
|
51482
|
+
}
|
|
51483
|
+
if (typeof cf.botScore === 'number' && cf.botScore < BOT_SCORE_MAX_FORWARD) return true
|
|
51484
|
+
return false
|
|
51485
|
+
}
|
|
51486
|
+
|
|
51487
|
+
function shouldForward(request) {
|
|
51488
|
+
const ua = request.headers.get('user-agent') || ''
|
|
51489
|
+
if (uaMatches(ua)) return true
|
|
51490
|
+
if (urlUtmMatches(request.url)) return true
|
|
51491
|
+
const referer = request.headers.get('referer') || ''
|
|
51492
|
+
if (refererMatches(referer)) return true
|
|
51493
|
+
if (urlUtmMatches(referer)) return true
|
|
51494
|
+
return botSignals(request.cf)
|
|
51495
|
+
}
|
|
51496
|
+
|
|
51497
|
+
function toHex(buffer) {
|
|
51498
|
+
const bytes = new Uint8Array(buffer)
|
|
51499
|
+
let out = ''
|
|
51500
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
51501
|
+
out += bytes[i].toString(16).padStart(2, '0')
|
|
51502
|
+
}
|
|
51503
|
+
return out
|
|
51504
|
+
}
|
|
51505
|
+
|
|
51506
|
+
async function signBody(secret, timestamp, body) {
|
|
51507
|
+
const key = await crypto.subtle.importKey(
|
|
51508
|
+
'raw',
|
|
51509
|
+
new TextEncoder().encode(secret),
|
|
51510
|
+
{ name: 'HMAC', hash: 'SHA-256' },
|
|
51511
|
+
false,
|
|
51512
|
+
['sign'],
|
|
51513
|
+
)
|
|
51514
|
+
const sig = await crypto.subtle.sign(
|
|
51515
|
+
'HMAC',
|
|
51516
|
+
key,
|
|
51517
|
+
new TextEncoder().encode(timestamp + '.' + body),
|
|
51518
|
+
)
|
|
51519
|
+
return toHex(sig)
|
|
51520
|
+
}
|
|
51521
|
+
|
|
51522
|
+
function isRetryableStatus(status) {
|
|
51523
|
+
return status === 408 || status === 425 || status === 429 || status >= 500
|
|
51524
|
+
}
|
|
51525
|
+
|
|
51526
|
+
function sleep(ms) {
|
|
51527
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
51528
|
+
}
|
|
51529
|
+
|
|
51530
|
+
async function cancelUnusedResponseBody(response) {
|
|
51531
|
+
try {
|
|
51532
|
+
if (response.body) await response.body.cancel()
|
|
51533
|
+
} catch (_) {
|
|
51534
|
+
// The ingest response body is unused. Cancellation failure is non-fatal.
|
|
51535
|
+
}
|
|
51536
|
+
}
|
|
51537
|
+
|
|
51538
|
+
async function withRetry(operation) {
|
|
51539
|
+
for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) {
|
|
51540
|
+
let response
|
|
51541
|
+
try {
|
|
51542
|
+
response = await operation()
|
|
51543
|
+
} catch (error) {
|
|
51544
|
+
if (attempt === RETRY_DELAYS_MS.length) throw error
|
|
51545
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
51546
|
+
continue
|
|
51547
|
+
}
|
|
51548
|
+
|
|
51549
|
+
const status = response.status
|
|
51550
|
+
const ok = response.ok
|
|
51551
|
+
await cancelUnusedResponseBody(response)
|
|
51552
|
+
if (ok) return
|
|
51553
|
+
if (!isRetryableStatus(status) || attempt === RETRY_DELAYS_MS.length) {
|
|
51554
|
+
throw new Error('Canonry ingest returned HTTP ' + status)
|
|
51555
|
+
}
|
|
51556
|
+
await sleep(RETRY_DELAYS_MS[attempt])
|
|
51557
|
+
}
|
|
51558
|
+
}
|
|
51559
|
+
|
|
51560
|
+
function pickCf(cf) {
|
|
51561
|
+
if (!cf) return null
|
|
51562
|
+
const bm = cf.botManagement || {}
|
|
51563
|
+
return {
|
|
51564
|
+
verifiedBot: typeof bm.verifiedBot === 'boolean' ? bm.verifiedBot : null,
|
|
51565
|
+
botScore: typeof bm.score === 'number' ? bm.score : (typeof cf.botScore === 'number' ? cf.botScore : null),
|
|
51566
|
+
country: typeof cf.country === 'string' ? cf.country : null,
|
|
51567
|
+
asn: typeof cf.asn === 'number' ? cf.asn : null,
|
|
51568
|
+
asOrganization: typeof cf.asOrganization === 'string' ? cf.asOrganization : null,
|
|
51569
|
+
}
|
|
51570
|
+
}
|
|
51571
|
+
|
|
51572
|
+
function buildEdgeEvent(request, status, observedAt) {
|
|
51573
|
+
const url = new URL(request.url)
|
|
51574
|
+
return {
|
|
51575
|
+
eventId: request.headers.get('cf-ray') || crypto.randomUUID(),
|
|
51576
|
+
observedAt,
|
|
51577
|
+
method: request.method || null,
|
|
51578
|
+
host: url.hostname || null,
|
|
51579
|
+
path: url.pathname || '/',
|
|
51580
|
+
queryString: url.search ? url.search.slice(1) : null,
|
|
51581
|
+
status: typeof status === 'number' ? status : null,
|
|
51582
|
+
userAgent: request.headers.get('user-agent') || null,
|
|
51583
|
+
remoteIp: request.headers.get('cf-connecting-ip') || null,
|
|
51584
|
+
referer: request.headers.get('referer') || null,
|
|
51585
|
+
cf: pickCf(request.cf),
|
|
51586
|
+
}
|
|
51587
|
+
}
|
|
51588
|
+
|
|
51589
|
+
function buildEdgeEventBatch(env, request, status, observedAt) {
|
|
51590
|
+
return {
|
|
51591
|
+
schemaVersion: 1,
|
|
51592
|
+
workerVersion: requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.workerVersion}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.workerVersion)}),
|
|
51593
|
+
events: [buildEdgeEvent(request, status, observedAt)],
|
|
51594
|
+
}
|
|
51595
|
+
}
|
|
51596
|
+
|
|
51597
|
+
async function deliverViaDirectPush(env, batch) {
|
|
51598
|
+
const sourceId = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.sourceId)})
|
|
51599
|
+
const ingestUrl = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.ingestUrl}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.ingestUrl)})
|
|
51600
|
+
const bearerToken = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.bearerToken}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.bearerToken)})
|
|
51601
|
+
const hmacSecret = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.hmacSecret}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.hmacSecret)})
|
|
51602
|
+
const body = canonicalizeJson(batch)
|
|
51603
|
+
const timestamp = String(Math.floor(Date.now() / 1000))
|
|
51604
|
+
const signature = await signBody(hmacSecret, timestamp, body)
|
|
51605
|
+
|
|
51606
|
+
await withRetry(() => fetch(ingestUrl, {
|
|
51607
|
+
method: 'POST',
|
|
51608
|
+
headers: {
|
|
51609
|
+
'content-type': 'application/json',
|
|
51610
|
+
'Authorization': 'Bearer ' + bearerToken,
|
|
51611
|
+
'X-Canonry-Timestamp': timestamp,
|
|
51612
|
+
'X-Canonry-Signature': signature,
|
|
51613
|
+
'X-Canonry-Worker-Version': batch.workerVersion,
|
|
51614
|
+
'X-Canonry-Source-Id': sourceId,
|
|
51615
|
+
},
|
|
51616
|
+
body,
|
|
51617
|
+
}))
|
|
51618
|
+
}
|
|
51619
|
+
|
|
51620
|
+
async function deliverEdgeEventBatch(env, batch) {
|
|
51621
|
+
const deliveryMode = requireBinding(env.${CLOUDFLARE_WORKER_BINDINGS.deliveryMode}, ${jsString(CLOUDFLARE_WORKER_BINDINGS.deliveryMode)})
|
|
51622
|
+
if (deliveryMode === 'direct-push') {
|
|
51623
|
+
return deliverViaDirectPush(env, batch)
|
|
51624
|
+
}
|
|
51625
|
+
throw new Error('Unsupported Canonry delivery mode: ' + deliveryMode)
|
|
51626
|
+
}
|
|
51627
|
+
|
|
51628
|
+
async function deliverSelectedRequest(env, request, status, observedAt) {
|
|
51629
|
+
try {
|
|
51630
|
+
const batch = buildEdgeEventBatch(env, request, status, observedAt)
|
|
51631
|
+
await deliverEdgeEventBatch(env, batch)
|
|
51632
|
+
} catch (err) {
|
|
51633
|
+
// Delivery is statistical and must never mask the customer response.
|
|
51634
|
+
// Emit binding names and source id only; never emit secret values.
|
|
51635
|
+
console.warn('Canonry traffic delivery failed', {
|
|
51636
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51637
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51638
|
+
: null,
|
|
51639
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51640
|
+
})
|
|
51641
|
+
}
|
|
51642
|
+
}
|
|
51643
|
+
|
|
51644
|
+
function shouldForwardSafely(env, request) {
|
|
51645
|
+
try {
|
|
51646
|
+
return shouldForward(request)
|
|
51647
|
+
} catch (err) {
|
|
51648
|
+
console.warn('Canonry traffic filter failed', {
|
|
51649
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51650
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51651
|
+
: null,
|
|
51652
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51653
|
+
})
|
|
51654
|
+
return false
|
|
51655
|
+
}
|
|
51656
|
+
}
|
|
51657
|
+
|
|
51658
|
+
function scheduleSelectedRequest(ctx, env, request, status, observedAt) {
|
|
51659
|
+
try {
|
|
51660
|
+
ctx.waitUntil(deliverSelectedRequest(env, request, status, observedAt))
|
|
51661
|
+
} catch (err) {
|
|
51662
|
+
console.warn('Canonry traffic delivery scheduling failed', {
|
|
51663
|
+
sourceId: typeof env.${CLOUDFLARE_WORKER_BINDINGS.sourceId} === 'string'
|
|
51664
|
+
? env.${CLOUDFLARE_WORKER_BINDINGS.sourceId}
|
|
51665
|
+
: null,
|
|
51666
|
+
message: err instanceof Error ? err.message : String(err),
|
|
51667
|
+
})
|
|
51668
|
+
}
|
|
51669
|
+
}
|
|
51670
|
+
|
|
51671
|
+
export default {
|
|
51672
|
+
async fetch(request, env, ctx) {
|
|
51673
|
+
const observedAt = new Date().toISOString()
|
|
51674
|
+
const shouldLog = shouldForwardSafely(env, request)
|
|
51675
|
+
let response
|
|
51676
|
+
try {
|
|
51677
|
+
response = await fetch(request)
|
|
51678
|
+
} catch (err) {
|
|
51679
|
+
if (shouldLog) {
|
|
51680
|
+
scheduleSelectedRequest(ctx, env, request, null, observedAt)
|
|
51681
|
+
}
|
|
51682
|
+
throw err
|
|
51683
|
+
}
|
|
51684
|
+
if (shouldLog) {
|
|
51685
|
+
scheduleSelectedRequest(ctx, env, request, response.status, observedAt)
|
|
51686
|
+
}
|
|
51687
|
+
return response
|
|
51688
|
+
},
|
|
51689
|
+
}
|
|
51690
|
+
`;
|
|
51691
|
+
}
|
|
51692
|
+
function generateWranglerToml(opts) {
|
|
51693
|
+
const hostname = opts.hostname.trim().toLowerCase().replace(/\.$/, "");
|
|
51694
|
+
const accountConfig = opts.accountId ? `account_id = ${jsString(opts.accountId)}
|
|
51695
|
+
` : "";
|
|
51696
|
+
const zoneHint = opts.zoneId ? `# Target zone id: ${jsString(opts.zoneId)}
|
|
51697
|
+
` : "# Target zone id was not provided. Select the canonical site zone.\n";
|
|
51698
|
+
return `${CLOUDFLARE_WRANGLER_GENERATED_MARKER}
|
|
51699
|
+
name = "canonry-traffic-${opts.sourceId}"
|
|
51700
|
+
${accountConfig}main = "worker.js"
|
|
51701
|
+
compatibility_date = "${WORKER_COMPATIBILITY_DATE}"
|
|
51702
|
+
workers_dev = false
|
|
51703
|
+
|
|
51704
|
+
[vars]
|
|
51705
|
+
${CLOUDFLARE_WORKER_BINDINGS.deliveryMode} = ${jsString(opts.deliveryMode)}
|
|
51706
|
+
${CLOUDFLARE_WORKER_BINDINGS.sourceId} = ${jsString(opts.sourceId)}
|
|
51707
|
+
${CLOUDFLARE_WORKER_BINDINGS.ingestUrl} = ${jsString(opts.ingestUrl)}
|
|
51708
|
+
${CLOUDFLARE_WORKER_BINDINGS.workerVersion} = ${jsString(opts.workerVersion)}
|
|
51709
|
+
|
|
51710
|
+
[secrets]
|
|
51711
|
+
required = ${jsArray(CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS)}
|
|
51712
|
+
|
|
51713
|
+
# Install required Worker secrets interactively or through Canonry's deploy
|
|
51714
|
+
# command. Never write their values into this file:
|
|
51715
|
+
${CLOUDFLARE_DIRECT_PUSH_SECRET_BINDINGS.map((name) => `# wrangler secret put ${name}`).join("\n")}
|
|
51716
|
+
|
|
51717
|
+
# Deploy this Worker via:
|
|
51718
|
+
# wrangler deploy
|
|
51719
|
+
# Canonry intentionally does not declare a route in this file.
|
|
51720
|
+
# After deploy, attach this exact route in the Cloudflare dashboard:
|
|
51721
|
+
# ${hostname}/*
|
|
51722
|
+
${zoneHint}# Set the route Request limit failure mode to Fail open before activation.
|
|
51723
|
+
# Wrangler cannot configure this route toggle.
|
|
51724
|
+
`;
|
|
51725
|
+
}
|
|
51726
|
+
|
|
51727
|
+
// ../integration-cloudflare-worker/src/verify.ts
|
|
51728
|
+
import { createHmac, timingSafeEqual } from "crypto";
|
|
51729
|
+
var DEFAULT_MAX_AGE_SECONDS = 300;
|
|
51730
|
+
function verifyRequestSignature(opts) {
|
|
51731
|
+
const { timestamp, signature, secret } = opts;
|
|
51732
|
+
if (timestamp === "" || !/^-?\d+$/.test(timestamp)) {
|
|
51733
|
+
return { ok: false, reason: "timestamp_invalid" };
|
|
51734
|
+
}
|
|
51735
|
+
const ts = Number(timestamp);
|
|
51736
|
+
if (!Number.isFinite(ts)) return { ok: false, reason: "timestamp_invalid" };
|
|
51737
|
+
const now = opts.nowSeconds ?? Math.floor(Date.now() / 1e3);
|
|
51738
|
+
const maxAge = opts.maxAgeSeconds ?? DEFAULT_MAX_AGE_SECONDS;
|
|
51739
|
+
if (Math.abs(now - ts) > maxAge) return { ok: false, reason: "timestamp_expired" };
|
|
51740
|
+
if (signature === "" || !/^[0-9a-f]+$/i.test(signature)) {
|
|
51741
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51742
|
+
}
|
|
51743
|
+
let provided;
|
|
51744
|
+
try {
|
|
51745
|
+
provided = Buffer.from(signature, "hex");
|
|
51746
|
+
} catch {
|
|
51747
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51748
|
+
}
|
|
51749
|
+
let bodies;
|
|
51750
|
+
try {
|
|
51751
|
+
if ("body" in opts) {
|
|
51752
|
+
bodies = [opts.body];
|
|
51753
|
+
} else {
|
|
51754
|
+
const canonical = canonicalizeCloudflareJson(opts.payload);
|
|
51755
|
+
const legacy = JSON.stringify(opts.payload);
|
|
51756
|
+
bodies = [canonical];
|
|
51757
|
+
if (opts.acceptLegacyJson !== false && typeof legacy === "string" && legacy !== canonical) {
|
|
51758
|
+
bodies.push(legacy);
|
|
51759
|
+
}
|
|
51760
|
+
}
|
|
51761
|
+
} catch {
|
|
51762
|
+
return { ok: false, reason: "signature_invalid" };
|
|
51763
|
+
}
|
|
51764
|
+
let comparableLength = false;
|
|
51765
|
+
for (const body of bodies) {
|
|
51766
|
+
const expected = createHmac("sha256", secret).update(`${timestamp}.${body}`).digest();
|
|
51767
|
+
if (provided.length !== expected.length) continue;
|
|
51768
|
+
comparableLength = true;
|
|
51769
|
+
if (timingSafeEqual(provided, expected)) return { ok: true };
|
|
51770
|
+
}
|
|
51771
|
+
if (!comparableLength) return { ok: false, reason: "signature_invalid" };
|
|
51772
|
+
return { ok: false, reason: "signature_mismatch" };
|
|
51773
|
+
}
|
|
51774
|
+
|
|
51775
|
+
// ../api-routes/src/traffic-event-ingest.ts
|
|
51776
|
+
import crypto39 from "crypto";
|
|
51777
|
+
import { and as and38, eq as eq46, gte as gte10, lt as lt7, lte as lte9, sql as sql18 } from "drizzle-orm";
|
|
51778
|
+
var DIRECT_PUSH_RECEIPT_TTL_MS = 10 * 6e4;
|
|
51779
|
+
function observedUtcHourBounds(observedAt) {
|
|
51780
|
+
const instant = new Date(observedAt);
|
|
51781
|
+
if (Number.isNaN(instant.getTime())) return null;
|
|
51782
|
+
instant.setUTCMinutes(0, 0, 0);
|
|
51783
|
+
const start = instant.toISOString();
|
|
51784
|
+
instant.setUTCHours(instant.getUTCHours() + 1);
|
|
51785
|
+
return { start, end: instant.toISOString() };
|
|
51786
|
+
}
|
|
51787
|
+
function writeTrafficEventBatch(opts) {
|
|
51788
|
+
const expiresAt = new Date(Date.parse(opts.receivedAt) + opts.receiptTtlMs).toISOString();
|
|
51789
|
+
let acceptedEvents = 0;
|
|
51790
|
+
let duplicateEvents = 0;
|
|
51791
|
+
let crawlerBucketRows = 0;
|
|
51792
|
+
let aiUserFetchBucketRows = 0;
|
|
51793
|
+
let aiReferralBucketRows = 0;
|
|
51794
|
+
let sampleRows = 0;
|
|
51795
|
+
opts.db.transaction((tx) => {
|
|
51796
|
+
tx.update(trafficSources).set({ updatedAt: sql18`${trafficSources.updatedAt}` }).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
51797
|
+
const source = tx.select().from(trafficSources).where(eq46(trafficSources.id, opts.sourceId)).get();
|
|
51798
|
+
opts.validateSource(source);
|
|
51799
|
+
tx.delete(trafficEventReceipts).where(and38(
|
|
51800
|
+
eq46(trafficEventReceipts.sourceId, opts.sourceId),
|
|
51801
|
+
lte9(trafficEventReceipts.expiresAt, opts.receivedAt)
|
|
51802
|
+
)).run();
|
|
51803
|
+
const claimedEvents = [];
|
|
51804
|
+
for (const event of opts.events) {
|
|
51805
|
+
const claim = tx.insert(trafficEventReceipts).values({
|
|
51806
|
+
sourceId: opts.sourceId,
|
|
51807
|
+
eventId: event.eventId,
|
|
51808
|
+
receivedAt: opts.receivedAt,
|
|
51809
|
+
expiresAt
|
|
51810
|
+
}).onConflictDoNothing().run();
|
|
51811
|
+
if (claim.changes === 1) claimedEvents.push(event);
|
|
51812
|
+
else duplicateEvents += 1;
|
|
51813
|
+
}
|
|
51814
|
+
acceptedEvents = claimedEvents.length;
|
|
51815
|
+
if (claimedEvents.length > 0) {
|
|
51816
|
+
const report = buildTrafficProbeReport(claimedEvents, { sampleLimit: claimedEvents.length });
|
|
51817
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
51818
|
+
const status = bucket.status ?? 0;
|
|
51819
|
+
tx.insert(crawlerEventsHourly).values({
|
|
51820
|
+
projectId: opts.projectId,
|
|
51821
|
+
sourceId: opts.sourceId,
|
|
51822
|
+
tsHour: bucket.tsHour,
|
|
51823
|
+
botId: bucket.botId,
|
|
51824
|
+
operator: bucket.operator,
|
|
51825
|
+
verificationStatus: bucket.verificationStatus,
|
|
51826
|
+
pathNormalized: bucket.pathNormalized,
|
|
51827
|
+
status,
|
|
51828
|
+
hits: bucket.hits,
|
|
51829
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51830
|
+
createdAt: opts.receivedAt,
|
|
51831
|
+
updatedAt: opts.receivedAt
|
|
51832
|
+
}).onConflictDoUpdate({
|
|
51833
|
+
target: [
|
|
51834
|
+
crawlerEventsHourly.projectId,
|
|
51835
|
+
crawlerEventsHourly.sourceId,
|
|
51836
|
+
crawlerEventsHourly.tsHour,
|
|
51837
|
+
crawlerEventsHourly.botId,
|
|
51838
|
+
crawlerEventsHourly.verificationStatus,
|
|
51839
|
+
crawlerEventsHourly.pathNormalized,
|
|
51840
|
+
crawlerEventsHourly.status
|
|
51841
|
+
],
|
|
51842
|
+
set: {
|
|
51843
|
+
hits: sql18`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
51844
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51845
|
+
updatedAt: opts.receivedAt
|
|
51846
|
+
}
|
|
51847
|
+
}).run();
|
|
51848
|
+
crawlerBucketRows += 1;
|
|
51849
|
+
}
|
|
51850
|
+
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
51851
|
+
const status = bucket.status ?? 0;
|
|
51852
|
+
tx.insert(aiUserFetchEventsHourly).values({
|
|
51853
|
+
projectId: opts.projectId,
|
|
51854
|
+
sourceId: opts.sourceId,
|
|
51855
|
+
tsHour: bucket.tsHour,
|
|
51856
|
+
botId: bucket.botId,
|
|
51857
|
+
operator: bucket.operator,
|
|
51858
|
+
verificationStatus: bucket.verificationStatus,
|
|
51859
|
+
pathNormalized: bucket.pathNormalized,
|
|
51860
|
+
status,
|
|
51861
|
+
hits: bucket.hits,
|
|
51862
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51863
|
+
createdAt: opts.receivedAt,
|
|
51864
|
+
updatedAt: opts.receivedAt
|
|
51865
|
+
}).onConflictDoUpdate({
|
|
51866
|
+
target: [
|
|
51867
|
+
aiUserFetchEventsHourly.projectId,
|
|
51868
|
+
aiUserFetchEventsHourly.sourceId,
|
|
51869
|
+
aiUserFetchEventsHourly.tsHour,
|
|
51870
|
+
aiUserFetchEventsHourly.botId,
|
|
51871
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
51872
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
51873
|
+
aiUserFetchEventsHourly.status
|
|
51874
|
+
],
|
|
51875
|
+
set: {
|
|
51876
|
+
hits: sql18`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
51877
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
51878
|
+
updatedAt: opts.receivedAt
|
|
51879
|
+
}
|
|
51880
|
+
}).run();
|
|
51881
|
+
aiUserFetchBucketRows += 1;
|
|
51882
|
+
}
|
|
51883
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
51884
|
+
const status = bucket.status ?? 0;
|
|
51885
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
51886
|
+
projectId: opts.projectId,
|
|
51887
|
+
sourceId: opts.sourceId,
|
|
51888
|
+
tsHour: bucket.tsHour,
|
|
51889
|
+
product: bucket.product,
|
|
51890
|
+
operator: bucket.operator,
|
|
51891
|
+
sourceDomain: bucket.sourceDomain,
|
|
51892
|
+
evidenceType: bucket.evidenceType,
|
|
51893
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
51894
|
+
status,
|
|
51895
|
+
sessionsOrHits: bucket.hits,
|
|
51896
|
+
paidSessionsOrHits: bucket.paidHits,
|
|
51897
|
+
organicSessionsOrHits: bucket.organicHits,
|
|
51898
|
+
usersEstimated: null,
|
|
51899
|
+
createdAt: opts.receivedAt,
|
|
51900
|
+
updatedAt: opts.receivedAt
|
|
51901
|
+
}).onConflictDoUpdate({
|
|
51902
|
+
target: [
|
|
51903
|
+
aiReferralEventsHourly.projectId,
|
|
51904
|
+
aiReferralEventsHourly.sourceId,
|
|
51905
|
+
aiReferralEventsHourly.tsHour,
|
|
51906
|
+
aiReferralEventsHourly.product,
|
|
51907
|
+
aiReferralEventsHourly.sourceDomain,
|
|
51908
|
+
aiReferralEventsHourly.evidenceType,
|
|
51909
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
51910
|
+
aiReferralEventsHourly.status
|
|
51911
|
+
],
|
|
51912
|
+
set: {
|
|
51913
|
+
sessionsOrHits: sql18`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
51914
|
+
paidSessionsOrHits: sql18`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
51915
|
+
organicSessionsOrHits: sql18`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
51916
|
+
updatedAt: opts.receivedAt
|
|
51917
|
+
}
|
|
51918
|
+
}).run();
|
|
51919
|
+
aiReferralBucketRows += 1;
|
|
51920
|
+
}
|
|
51921
|
+
const hourlySampleLimit = Math.max(0, Math.floor(opts.sampleLimit));
|
|
51922
|
+
const remainingSamplesByHour = /* @__PURE__ */ new Map();
|
|
51923
|
+
for (const sample of report.samples) {
|
|
51924
|
+
const hour = observedUtcHourBounds(sample.observedAt);
|
|
51925
|
+
if (!hour || hourlySampleLimit === 0) continue;
|
|
51926
|
+
let remaining = remainingSamplesByHour.get(hour.start);
|
|
51927
|
+
if (remaining === void 0) {
|
|
51928
|
+
const existing = tx.select({ total: sql18`COUNT(*)` }).from(rawEventSamples).where(and38(
|
|
51929
|
+
eq46(rawEventSamples.sourceId, opts.sourceId),
|
|
51930
|
+
gte10(rawEventSamples.ts, hour.start),
|
|
51931
|
+
lt7(rawEventSamples.ts, hour.end)
|
|
51932
|
+
)).get();
|
|
51933
|
+
remaining = Math.max(0, hourlySampleLimit - Number(existing?.total ?? 0));
|
|
51934
|
+
}
|
|
51935
|
+
if (remaining === 0) {
|
|
51936
|
+
remainingSamplesByHour.set(hour.start, 0);
|
|
51937
|
+
continue;
|
|
51938
|
+
}
|
|
51939
|
+
const eventType = sample.crawler ? "crawler" : sample.aiUserFetch ? "ai_user_fetch" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
51940
|
+
const refererHost = (() => {
|
|
51941
|
+
if (!sample.referer) return null;
|
|
51942
|
+
try {
|
|
51943
|
+
return new URL(sample.referer).hostname;
|
|
51944
|
+
} catch {
|
|
51945
|
+
return null;
|
|
51946
|
+
}
|
|
51947
|
+
})();
|
|
51948
|
+
tx.insert(rawEventSamples).values({
|
|
51949
|
+
id: crypto39.randomUUID(),
|
|
51950
|
+
projectId: opts.projectId,
|
|
51951
|
+
sourceId: opts.sourceId,
|
|
51952
|
+
ts: sample.observedAt,
|
|
51953
|
+
eventType,
|
|
51954
|
+
ipHash: null,
|
|
51955
|
+
userAgent: sample.userAgent,
|
|
51956
|
+
pathNormalized: sample.pathNormalized,
|
|
51957
|
+
status: sample.status,
|
|
51958
|
+
refererHost,
|
|
51959
|
+
classifierDetailsJson: {
|
|
51960
|
+
crawler: sample.crawler,
|
|
51961
|
+
aiUserFetch: sample.aiUserFetch,
|
|
51962
|
+
aiReferral: sample.aiReferral
|
|
51963
|
+
},
|
|
51964
|
+
createdAt: opts.receivedAt
|
|
51965
|
+
}).run();
|
|
51966
|
+
sampleRows += 1;
|
|
51967
|
+
remainingSamplesByHour.set(hour.start, remaining - 1);
|
|
51968
|
+
}
|
|
51969
|
+
}
|
|
51970
|
+
tx.update(trafficSources).set(opts.sourceUpdate).where(eq46(trafficSources.id, opts.sourceId)).run();
|
|
51971
|
+
});
|
|
51972
|
+
return {
|
|
51973
|
+
acceptedEvents,
|
|
51974
|
+
duplicateEvents,
|
|
51975
|
+
crawlerBucketRows,
|
|
51976
|
+
aiUserFetchBucketRows,
|
|
51977
|
+
aiReferralBucketRows,
|
|
51978
|
+
sampleRows
|
|
51979
|
+
};
|
|
51980
|
+
}
|
|
51981
|
+
|
|
51112
51982
|
// ../api-routes/src/traffic.ts
|
|
51113
51983
|
var DEFAULT_SYNC_WINDOW_MINUTES = 43200;
|
|
51114
51984
|
var DEFAULT_PAGE_SIZE3 = 1e3;
|
|
@@ -51125,6 +51995,126 @@ var DEFAULT_TRAFFIC_SYNC_CRON = "*/30 * * * *";
|
|
|
51125
51995
|
var MAX_BACKFILL_DAYS = 90;
|
|
51126
51996
|
var BACKFILL_MAX_PAGES = 1e3;
|
|
51127
51997
|
var BACKFILL_SAMPLE_LIMIT = 500;
|
|
51998
|
+
var CLOUDFLARE_WORKER_VERSION = "1.0.0";
|
|
51999
|
+
var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
|
|
52000
|
+
var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
|
|
52001
|
+
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
|
|
52002
|
+
function sha256Hex3(value) {
|
|
52003
|
+
return crypto40.createHash("sha256").update(value).digest("hex");
|
|
52004
|
+
}
|
|
52005
|
+
function hashCloudflareBearerToken(value) {
|
|
52006
|
+
return sha256Hex3(value);
|
|
52007
|
+
}
|
|
52008
|
+
function timingSafeEqualHex(a, b) {
|
|
52009
|
+
if (!a || !b) return false;
|
|
52010
|
+
if (!SHA256_HEX_PATTERN.test(a) || !SHA256_HEX_PATTERN.test(b)) return false;
|
|
52011
|
+
const left = Buffer.from(a, "hex");
|
|
52012
|
+
const right = Buffer.from(b, "hex");
|
|
52013
|
+
return left.length === right.length && crypto40.timingSafeEqual(left, right);
|
|
52014
|
+
}
|
|
52015
|
+
function parseDirectPushCloudflareSourceConfig(config) {
|
|
52016
|
+
const parsed = cloudflareTrafficSourceConfigSchema.safeParse(config);
|
|
52017
|
+
return parsed.success ? parsed.data : null;
|
|
52018
|
+
}
|
|
52019
|
+
function isDirectPushCloudflareDeliveryMode(value) {
|
|
52020
|
+
return value === CloudflareTrafficDeliveryModes["direct-push"];
|
|
52021
|
+
}
|
|
52022
|
+
function authenticateCloudflareIngest(request, store) {
|
|
52023
|
+
if (!store) return null;
|
|
52024
|
+
const authHeader = request.headers.authorization;
|
|
52025
|
+
const sourceIdHeader = request.headers["x-canonry-source-id"];
|
|
52026
|
+
const timestampHeader = request.headers["x-canonry-timestamp"];
|
|
52027
|
+
const signatureHeader = request.headers["x-canonry-signature"];
|
|
52028
|
+
const bearerToken = typeof authHeader === "string" && authHeader.startsWith("Bearer ") ? authHeader.slice(7) : "";
|
|
52029
|
+
const sourceId = typeof sourceIdHeader === "string" ? sourceIdHeader : "";
|
|
52030
|
+
const timestamp = typeof timestampHeader === "string" ? timestampHeader : "";
|
|
52031
|
+
const signature = typeof signatureHeader === "string" ? signatureHeader : "";
|
|
52032
|
+
if (!bearerToken || !sourceId || !timestamp || !signature) return null;
|
|
52033
|
+
const credential = store.getConnectionBySourceId(sourceId);
|
|
52034
|
+
if (!credential || !isDirectPushCloudflareDeliveryMode(credential.deliveryMode)) return null;
|
|
52035
|
+
const bearerHash = sha256Hex3(bearerToken);
|
|
52036
|
+
if (!timingSafeEqualHex(bearerHash, sha256Hex3(credential.bearerToken))) return null;
|
|
52037
|
+
if (!verifyRequestSignature({
|
|
52038
|
+
timestamp,
|
|
52039
|
+
signature,
|
|
52040
|
+
payload: request.body,
|
|
52041
|
+
secret: credential.hmacSecret
|
|
52042
|
+
}).ok) return null;
|
|
52043
|
+
return { bearerHash, credential };
|
|
52044
|
+
}
|
|
52045
|
+
function cloudflareIngestRateLimitKey(request, store, db) {
|
|
52046
|
+
const authenticated = authenticateCloudflareIngest(request, store);
|
|
52047
|
+
const params = request.params;
|
|
52048
|
+
if (authenticated && typeof params.name === "string" && authenticated.credential.projectName === params.name) {
|
|
52049
|
+
const source = db.select().from(trafficSources).where(eq47(trafficSources.id, authenticated.credential.sourceId)).get();
|
|
52050
|
+
if (source && source.sourceType === TrafficSourceTypes.cloudflare && source.status !== TrafficSourceStatuses.archived && parseDirectPushCloudflareSourceConfig(source.configJson) && timingSafeEqualHex(authenticated.bearerHash, source.ingestTokenHash)) {
|
|
52051
|
+
return `cloudflare-source:${authenticated.credential.sourceId}`;
|
|
52052
|
+
}
|
|
52053
|
+
}
|
|
52054
|
+
return `cloudflare-invalid:${request.ip}`;
|
|
52055
|
+
}
|
|
52056
|
+
function assertCloudflareIngestUrlIsPublicHttps(value) {
|
|
52057
|
+
let url;
|
|
52058
|
+
try {
|
|
52059
|
+
url = new URL(value);
|
|
52060
|
+
} catch {
|
|
52061
|
+
throw validationError("Cloudflare ingest URL must be a valid public HTTPS URL");
|
|
52062
|
+
}
|
|
52063
|
+
const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "");
|
|
52064
|
+
const ipv4 = hostname.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
52065
|
+
const octets = ipv4?.slice(1).map(Number);
|
|
52066
|
+
const privateIpv4 = octets !== void 0 && (octets.some((part) => part > 255) || octets[0] === 0 || octets[0] === 10 || octets[0] === 127 || octets[0] === 169 && octets[1] === 254 || octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31 || octets[0] === 192 && octets[1] === 168);
|
|
52067
|
+
const privateIpv6 = hostname.includes(":") && (hostname === "::1" || hostname.startsWith("fc") || hostname.startsWith("fd") || hostname.startsWith("fe8") || hostname.startsWith("fe9") || hostname.startsWith("fea") || hostname.startsWith("feb"));
|
|
52068
|
+
const privateHostname = hostname === "localhost" || hostname.endsWith(".localhost") || hostname.endsWith(".local") || privateIpv6 || privateIpv4;
|
|
52069
|
+
if (url.protocol !== "https:" || privateHostname || url.username.length > 0 || url.password.length > 0 || url.search.length > 0 || url.hash.length > 0) {
|
|
52070
|
+
throw validationError(
|
|
52071
|
+
"Cloudflare ingest URL must be a credential-free public HTTPS URL without a query or fragment and reachable from the public Cloudflare edge; configure publicUrl to a public origin"
|
|
52072
|
+
);
|
|
52073
|
+
}
|
|
52074
|
+
}
|
|
52075
|
+
function normalizeDnsHostname(value) {
|
|
52076
|
+
const normalized = value.trim().toLowerCase().replace(/\.+$/, "");
|
|
52077
|
+
if (normalized.length === 0 || normalized.length > 253 || normalized.includes("*") || !normalized.includes(".")) return null;
|
|
52078
|
+
const valid = normalized.split(".").every(
|
|
52079
|
+
(label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label)
|
|
52080
|
+
);
|
|
52081
|
+
return valid ? normalized : null;
|
|
52082
|
+
}
|
|
52083
|
+
function resolveCloudflareWorkerRouteHost(value) {
|
|
52084
|
+
const trimmed = value.trim();
|
|
52085
|
+
const hasScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed);
|
|
52086
|
+
if (hasScheme && !/^https?:\/\//i.test(trimmed)) {
|
|
52087
|
+
throw validationError("Project canonicalDomain must use an HTTP or HTTPS URL");
|
|
52088
|
+
}
|
|
52089
|
+
let url;
|
|
52090
|
+
try {
|
|
52091
|
+
url = new URL(hasScheme ? trimmed : `https://${trimmed}`);
|
|
52092
|
+
} catch {
|
|
52093
|
+
throw validationError("Project canonicalDomain must resolve to one exact public hostname");
|
|
52094
|
+
}
|
|
52095
|
+
if (url.username.length > 0 || url.password.length > 0 || url.pathname !== "" && url.pathname !== "/" || url.search.length > 0 || url.hash.length > 0 || url.port.length > 0) {
|
|
52096
|
+
throw validationError(
|
|
52097
|
+
"Project canonicalDomain must be a bare hostname or root HTTP(S) URL without credentials, a non-default port, path, query, or fragment"
|
|
52098
|
+
);
|
|
52099
|
+
}
|
|
52100
|
+
const hostname = normalizeDnsHostname(url.hostname);
|
|
52101
|
+
if (!hostname || isIP(hostname) !== 0) {
|
|
52102
|
+
throw validationError("Project canonicalDomain must resolve to one exact public hostname without wildcards");
|
|
52103
|
+
}
|
|
52104
|
+
return hostname;
|
|
52105
|
+
}
|
|
52106
|
+
function normalizeCloudflareEventHost(value) {
|
|
52107
|
+
if (!value || /[/:@?#*]/.test(value)) return null;
|
|
52108
|
+
return normalizeDnsHostname(value);
|
|
52109
|
+
}
|
|
52110
|
+
function assertCloudflareIngestUrlOutsideWorkerRoute(ingestUrl, workerRouteHost) {
|
|
52111
|
+
const ingestHost = normalizeDnsHostname(new URL(ingestUrl).hostname);
|
|
52112
|
+
if (ingestHost && ingestHost === workerRouteHost) {
|
|
52113
|
+
throw validationError(
|
|
52114
|
+
"Cloudflare ingest URL hostname must be outside the generated Worker route hostname to prevent recursive ingestion"
|
|
52115
|
+
);
|
|
52116
|
+
}
|
|
52117
|
+
}
|
|
51128
52118
|
function emptyTrafficSeriesPoint(bucket) {
|
|
51129
52119
|
return { bucket, crawlerHits: 0, aiUserFetchHits: 0, aiReferralHits: 0 };
|
|
51130
52120
|
}
|
|
@@ -51205,8 +52195,8 @@ async function runBackfillTask(options) {
|
|
|
51205
52195
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
51206
52196
|
try {
|
|
51207
52197
|
app.db.transaction((tx) => {
|
|
51208
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
51209
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
52198
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq47(runs.id, runId)).run();
|
|
52199
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51210
52200
|
});
|
|
51211
52201
|
} catch {
|
|
51212
52202
|
}
|
|
@@ -51221,7 +52211,7 @@ async function runBackfillTask(options) {
|
|
|
51221
52211
|
if (allEvents.length === 0) {
|
|
51222
52212
|
const finishedAt2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
51223
52213
|
try {
|
|
51224
|
-
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(
|
|
52214
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: finishedAt2 }).where(eq47(runs.id, runId)).run();
|
|
51225
52215
|
} catch {
|
|
51226
52216
|
}
|
|
51227
52217
|
return;
|
|
@@ -51243,31 +52233,31 @@ async function runBackfillTask(options) {
|
|
|
51243
52233
|
try {
|
|
51244
52234
|
app.db.transaction((tx) => {
|
|
51245
52235
|
tx.delete(crawlerEventsHourly).where(
|
|
51246
|
-
|
|
51247
|
-
|
|
51248
|
-
|
|
51249
|
-
|
|
52236
|
+
and39(
|
|
52237
|
+
eq47(crawlerEventsHourly.sourceId, sourceRow2.id),
|
|
52238
|
+
gte11(crawlerEventsHourly.tsHour, windowStartIso),
|
|
52239
|
+
lte10(crawlerEventsHourly.tsHour, windowEndIso)
|
|
51250
52240
|
)
|
|
51251
52241
|
).run();
|
|
51252
52242
|
tx.delete(aiUserFetchEventsHourly).where(
|
|
51253
|
-
|
|
51254
|
-
|
|
51255
|
-
|
|
51256
|
-
|
|
52243
|
+
and39(
|
|
52244
|
+
eq47(aiUserFetchEventsHourly.sourceId, sourceRow2.id),
|
|
52245
|
+
gte11(aiUserFetchEventsHourly.tsHour, windowStartIso),
|
|
52246
|
+
lte10(aiUserFetchEventsHourly.tsHour, windowEndIso)
|
|
51257
52247
|
)
|
|
51258
52248
|
).run();
|
|
51259
52249
|
tx.delete(aiReferralEventsHourly).where(
|
|
51260
|
-
|
|
51261
|
-
|
|
51262
|
-
|
|
51263
|
-
|
|
52250
|
+
and39(
|
|
52251
|
+
eq47(aiReferralEventsHourly.sourceId, sourceRow2.id),
|
|
52252
|
+
gte11(aiReferralEventsHourly.tsHour, windowStartIso),
|
|
52253
|
+
lte10(aiReferralEventsHourly.tsHour, windowEndIso)
|
|
51264
52254
|
)
|
|
51265
52255
|
).run();
|
|
51266
52256
|
tx.delete(rawEventSamples).where(
|
|
51267
|
-
|
|
51268
|
-
|
|
51269
|
-
|
|
51270
|
-
|
|
52257
|
+
and39(
|
|
52258
|
+
eq47(rawEventSamples.sourceId, sourceRow2.id),
|
|
52259
|
+
gte11(rawEventSamples.ts, windowStartIso),
|
|
52260
|
+
lte10(rawEventSamples.ts, windowEndIso)
|
|
51271
52261
|
)
|
|
51272
52262
|
).run();
|
|
51273
52263
|
for (const bucket of report.crawlerEventsHourly) {
|
|
@@ -51332,7 +52322,7 @@ async function runBackfillTask(options) {
|
|
|
51332
52322
|
}
|
|
51333
52323
|
})();
|
|
51334
52324
|
tx.insert(rawEventSamples).values({
|
|
51335
|
-
id:
|
|
52325
|
+
id: crypto40.randomUUID(),
|
|
51336
52326
|
projectId: project.id,
|
|
51337
52327
|
sourceId: sourceRow2.id,
|
|
51338
52328
|
ts: sample.observedAt,
|
|
@@ -51359,8 +52349,8 @@ async function runBackfillTask(options) {
|
|
|
51359
52349
|
lastEventIds: newRingBuffer,
|
|
51360
52350
|
...skipRecovered ? { skippedThroughAt: null } : {},
|
|
51361
52351
|
updatedAt: finishedAt
|
|
51362
|
-
}).where(
|
|
51363
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
52352
|
+
}).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
52353
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq47(runs.id, runId)).run();
|
|
51364
52354
|
});
|
|
51365
52355
|
} catch (e) {
|
|
51366
52356
|
markFailed(`Backfill rollup write failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
@@ -51405,6 +52395,11 @@ async function trafficRoutes(app, opts) {
|
|
|
51405
52395
|
const pageSize = opts.defaultPageSize ?? DEFAULT_PAGE_SIZE3;
|
|
51406
52396
|
const maxPages = opts.defaultMaxPages ?? DEFAULT_MAX_PAGES4;
|
|
51407
52397
|
const sampleLimit = opts.defaultSampleLimit ?? DEFAULT_SAMPLE_LIMIT2;
|
|
52398
|
+
const checkCloudflareSourceRateLimit = app.createRateLimit({
|
|
52399
|
+
max: opts.cloudflareIngestRateLimitMax ?? DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX,
|
|
52400
|
+
timeWindow: "1 minute",
|
|
52401
|
+
keyGenerator: (request) => cloudflareIngestRateLimitKey(request, opts.cloudflareTrafficCredentialStore, app.db)
|
|
52402
|
+
});
|
|
51408
52403
|
app.post("/projects/:name/traffic/connect/cloud-run", async (request) => {
|
|
51409
52404
|
const project = resolveProject(app.db, request.params.name);
|
|
51410
52405
|
const body = request.body ?? {};
|
|
@@ -51442,7 +52437,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51442
52437
|
createdAt: existing?.createdAt ?? now,
|
|
51443
52438
|
updatedAt: now
|
|
51444
52439
|
});
|
|
51445
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52440
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes["cloud-run"] && row.status !== TrafficSourceStatuses.archived);
|
|
51446
52441
|
const config = {
|
|
51447
52442
|
gcpProjectId,
|
|
51448
52443
|
serviceName: serviceName ?? null,
|
|
@@ -51458,10 +52453,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51458
52453
|
lastError: null,
|
|
51459
52454
|
configJson: config,
|
|
51460
52455
|
updatedAt: now
|
|
51461
|
-
}).where(
|
|
51462
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52456
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52457
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51463
52458
|
} else {
|
|
51464
|
-
const newId =
|
|
52459
|
+
const newId = crypto40.randomUUID();
|
|
51465
52460
|
app.db.insert(trafficSources).values({
|
|
51466
52461
|
id: newId,
|
|
51467
52462
|
projectId: project.id,
|
|
@@ -51476,7 +52471,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51476
52471
|
createdAt: now,
|
|
51477
52472
|
updatedAt: now
|
|
51478
52473
|
}).run();
|
|
51479
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52474
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51480
52475
|
}
|
|
51481
52476
|
writeAuditLog(app.db, {
|
|
51482
52477
|
projectId: project.id,
|
|
@@ -51528,7 +52523,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51528
52523
|
createdAt: existing?.createdAt ?? now,
|
|
51529
52524
|
updatedAt: now
|
|
51530
52525
|
});
|
|
51531
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52526
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
51532
52527
|
const config = { baseUrl, username };
|
|
51533
52528
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
51534
52529
|
let sourceRow2;
|
|
@@ -51539,10 +52534,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51539
52534
|
lastError: null,
|
|
51540
52535
|
configJson: config,
|
|
51541
52536
|
updatedAt: now
|
|
51542
|
-
}).where(
|
|
51543
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52537
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52538
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51544
52539
|
} else {
|
|
51545
|
-
const newId =
|
|
52540
|
+
const newId = crypto40.randomUUID();
|
|
51546
52541
|
app.db.insert(trafficSources).values({
|
|
51547
52542
|
id: newId,
|
|
51548
52543
|
projectId: project.id,
|
|
@@ -51557,7 +52552,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51557
52552
|
createdAt: now,
|
|
51558
52553
|
updatedAt: now
|
|
51559
52554
|
}).run();
|
|
51560
|
-
sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52555
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51561
52556
|
}
|
|
51562
52557
|
writeAuditLog(app.db, {
|
|
51563
52558
|
projectId: project.id,
|
|
@@ -51611,7 +52606,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51611
52606
|
createdAt: existing?.createdAt ?? now,
|
|
51612
52607
|
updatedAt: now
|
|
51613
52608
|
});
|
|
51614
|
-
const activeSource = app.db.select().from(trafficSources).where(
|
|
52609
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.vercel && row.status !== TrafficSourceStatuses.archived);
|
|
51615
52610
|
const config = { projectId, teamId, environment };
|
|
51616
52611
|
const fallbackName = displayName ?? `Vercel \xB7 ${projectId}`;
|
|
51617
52612
|
const { sourceRow: sourceRow2, scheduleCreated } = app.db.transaction((tx) => {
|
|
@@ -51623,10 +52618,10 @@ async function trafficRoutes(app, opts) {
|
|
|
51623
52618
|
lastError: null,
|
|
51624
52619
|
configJson: config,
|
|
51625
52620
|
updatedAt: now
|
|
51626
|
-
}).where(
|
|
51627
|
-
row = tx.select().from(trafficSources).where(
|
|
52621
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52622
|
+
row = tx.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
51628
52623
|
} else {
|
|
51629
|
-
const newId =
|
|
52624
|
+
const newId = crypto40.randomUUID();
|
|
51630
52625
|
tx.insert(trafficSources).values({
|
|
51631
52626
|
id: newId,
|
|
51632
52627
|
projectId: project.id,
|
|
@@ -51649,18 +52644,18 @@ async function trafficRoutes(app, opts) {
|
|
|
51649
52644
|
createdAt: now,
|
|
51650
52645
|
updatedAt: now
|
|
51651
52646
|
}).run();
|
|
51652
|
-
row = tx.select().from(trafficSources).where(
|
|
52647
|
+
row = tx.select().from(trafficSources).where(eq47(trafficSources.id, newId)).get();
|
|
51653
52648
|
}
|
|
51654
52649
|
const existingSchedule = tx.select().from(schedules).where(
|
|
51655
|
-
|
|
51656
|
-
|
|
51657
|
-
|
|
52650
|
+
and39(
|
|
52651
|
+
eq47(schedules.projectId, project.id),
|
|
52652
|
+
eq47(schedules.kind, SchedulableRunKinds["traffic-sync"])
|
|
51658
52653
|
)
|
|
51659
52654
|
).get();
|
|
51660
52655
|
let created = false;
|
|
51661
52656
|
if (!existingSchedule) {
|
|
51662
52657
|
tx.insert(schedules).values({
|
|
51663
|
-
id:
|
|
52658
|
+
id: crypto40.randomUUID(),
|
|
51664
52659
|
projectId: project.id,
|
|
51665
52660
|
kind: SchedulableRunKinds["traffic-sync"],
|
|
51666
52661
|
cronExpr: DEFAULT_TRAFFIC_SYNC_CRON,
|
|
@@ -51701,9 +52696,256 @@ async function trafficRoutes(app, opts) {
|
|
|
51701
52696
|
}
|
|
51702
52697
|
return rowToDto(sourceRow2);
|
|
51703
52698
|
});
|
|
52699
|
+
app.post("/projects/:name/traffic/connect/cloudflare", async (request) => {
|
|
52700
|
+
const project = resolveProject(app.db, request.params.name);
|
|
52701
|
+
if (!opts.cloudflareTrafficCredentialStore) {
|
|
52702
|
+
throw validationError("Cloudflare traffic credential storage is not configured for this deployment");
|
|
52703
|
+
}
|
|
52704
|
+
if (!opts.cloudflareTrafficIngestUrl) {
|
|
52705
|
+
throw validationError("Cloudflare ingest URL is not configured for this deployment");
|
|
52706
|
+
}
|
|
52707
|
+
const credentialStore = opts.cloudflareTrafficCredentialStore;
|
|
52708
|
+
const parsed = trafficConnectCloudflareRequestSchema.safeParse(request.body ?? {});
|
|
52709
|
+
if (!parsed.success) {
|
|
52710
|
+
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
52711
|
+
}
|
|
52712
|
+
const { deliveryMode, displayName, zoneId, accountId } = parsed.data;
|
|
52713
|
+
const workerRouteHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
52714
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
52715
|
+
const activeSource = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).all().find(
|
|
52716
|
+
(row) => row.sourceType === TrafficSourceTypes.cloudflare && row.status !== TrafficSourceStatuses.archived
|
|
52717
|
+
);
|
|
52718
|
+
const sourceId = activeSource?.id ?? crypto40.randomUUID();
|
|
52719
|
+
const parsedActiveConfig = activeSource ? parseDirectPushCloudflareSourceConfig(activeSource.configJson) : null;
|
|
52720
|
+
if (activeSource && !parsedActiveConfig) {
|
|
52721
|
+
throw validationError("Existing Cloudflare source is not configured for direct-push delivery");
|
|
52722
|
+
}
|
|
52723
|
+
const previousCredential = credentialStore.getConnection(project.name);
|
|
52724
|
+
if (previousCredential && !isDirectPushCloudflareDeliveryMode(previousCredential.deliveryMode)) {
|
|
52725
|
+
throw validationError(
|
|
52726
|
+
`Cloudflare credential delivery mode "${previousCredential.deliveryMode}" does not match "${deliveryMode}"`
|
|
52727
|
+
);
|
|
52728
|
+
}
|
|
52729
|
+
const reusableCredential = previousCredential?.sourceId === sourceId ? previousCredential : void 0;
|
|
52730
|
+
const previousZoneId = reusableCredential?.zoneId ?? parsedActiveConfig?.zoneId ?? null;
|
|
52731
|
+
const previousAccountId = reusableCredential?.accountId ?? parsedActiveConfig?.accountId ?? null;
|
|
52732
|
+
const effectiveZoneId = zoneId ?? previousZoneId;
|
|
52733
|
+
const effectiveAccountId = accountId ?? previousAccountId;
|
|
52734
|
+
const bearerToken = reusableCredential?.bearerToken ?? `cnry_cfw_${crypto40.randomBytes(32).toString("hex")}`;
|
|
52735
|
+
const hmacSecret = reusableCredential?.hmacSecret ?? crypto40.randomBytes(32).toString("hex");
|
|
52736
|
+
const ingestTokenHash = sha256Hex3(bearerToken);
|
|
52737
|
+
const ingestUrl = opts.cloudflareTrafficIngestUrl.replace("{name}", encodeURIComponent(project.name));
|
|
52738
|
+
assertCloudflareIngestUrlIsPublicHttps(ingestUrl);
|
|
52739
|
+
assertCloudflareIngestUrlOutsideWorkerRoute(ingestUrl, workerRouteHost);
|
|
52740
|
+
const workerVersion = CLOUDFLARE_WORKER_VERSION;
|
|
52741
|
+
const workerScript = generateWorkerScript({
|
|
52742
|
+
deliveryMode,
|
|
52743
|
+
workerVersion,
|
|
52744
|
+
botList: DEFAULT_BOT_LIST
|
|
52745
|
+
});
|
|
52746
|
+
const wranglerToml = generateWranglerToml({
|
|
52747
|
+
deliveryMode,
|
|
52748
|
+
sourceId,
|
|
52749
|
+
hostname: workerRouteHost,
|
|
52750
|
+
ingestUrl,
|
|
52751
|
+
workerVersion,
|
|
52752
|
+
zoneId: effectiveZoneId,
|
|
52753
|
+
accountId: effectiveAccountId
|
|
52754
|
+
});
|
|
52755
|
+
const config = {
|
|
52756
|
+
schemaVersion: 1,
|
|
52757
|
+
deliveryMode,
|
|
52758
|
+
workerVersion,
|
|
52759
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
52760
|
+
zoneId: effectiveZoneId,
|
|
52761
|
+
accountId: effectiveAccountId
|
|
52762
|
+
};
|
|
52763
|
+
const fallbackName = displayName ?? activeSource?.displayName ?? `Cloudflare \xB7 ${effectiveZoneId ?? sourceId.slice(0, 8)}`;
|
|
52764
|
+
credentialStore.upsertConnection({
|
|
52765
|
+
projectName: project.name,
|
|
52766
|
+
deliveryMode,
|
|
52767
|
+
sourceId,
|
|
52768
|
+
bearerToken,
|
|
52769
|
+
hmacSecret,
|
|
52770
|
+
workerVersion,
|
|
52771
|
+
expectedBotListVersion: DEFAULT_BOT_LIST.version,
|
|
52772
|
+
zoneId: effectiveZoneId,
|
|
52773
|
+
accountId: effectiveAccountId,
|
|
52774
|
+
createdAt: previousCredential?.createdAt ?? activeSource?.createdAt ?? now,
|
|
52775
|
+
updatedAt: now
|
|
52776
|
+
});
|
|
52777
|
+
let sourceRow2;
|
|
52778
|
+
try {
|
|
52779
|
+
if (activeSource) {
|
|
52780
|
+
app.db.update(trafficSources).set({
|
|
52781
|
+
displayName: fallbackName,
|
|
52782
|
+
status: TrafficSourceStatuses.connected,
|
|
52783
|
+
lastError: null,
|
|
52784
|
+
configJson: config,
|
|
52785
|
+
ingestTokenHash,
|
|
52786
|
+
updatedAt: now
|
|
52787
|
+
}).where(eq47(trafficSources.id, activeSource.id)).run();
|
|
52788
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, activeSource.id)).get();
|
|
52789
|
+
} else {
|
|
52790
|
+
app.db.insert(trafficSources).values({
|
|
52791
|
+
id: sourceId,
|
|
52792
|
+
projectId: project.id,
|
|
52793
|
+
sourceType: TrafficSourceTypes.cloudflare,
|
|
52794
|
+
displayName: fallbackName,
|
|
52795
|
+
status: TrafficSourceStatuses.connected,
|
|
52796
|
+
// Seed `lastSyncedAt` to NOW so the `traffic.source.recent-data`
|
|
52797
|
+
// doctor check has a non-null baseline. Successful ingest advances
|
|
52798
|
+
// it as the receiver's last-activity timestamp.
|
|
52799
|
+
lastSyncedAt: now,
|
|
52800
|
+
lastCursor: null,
|
|
52801
|
+
lastError: null,
|
|
52802
|
+
archivedAt: null,
|
|
52803
|
+
configJson: config,
|
|
52804
|
+
ingestTokenHash,
|
|
52805
|
+
createdAt: now,
|
|
52806
|
+
updatedAt: now
|
|
52807
|
+
}).run();
|
|
52808
|
+
sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, sourceId)).get();
|
|
52809
|
+
}
|
|
52810
|
+
} catch (err) {
|
|
52811
|
+
try {
|
|
52812
|
+
if (previousCredential) {
|
|
52813
|
+
credentialStore.upsertConnection(previousCredential);
|
|
52814
|
+
} else {
|
|
52815
|
+
credentialStore.deleteConnection(project.name);
|
|
52816
|
+
}
|
|
52817
|
+
} catch {
|
|
52818
|
+
}
|
|
52819
|
+
throw err;
|
|
52820
|
+
}
|
|
52821
|
+
writeAuditLog(app.db, {
|
|
52822
|
+
projectId: project.id,
|
|
52823
|
+
actor: "api",
|
|
52824
|
+
action: "traffic.cloudflare.connected",
|
|
52825
|
+
entityType: "traffic_source",
|
|
52826
|
+
entityId: sourceRow2.id
|
|
52827
|
+
});
|
|
52828
|
+
const routeInstruction = effectiveZoneId ? ` 4. Deploy with \`wrangler deploy\`; wrangler.toml records zone ${effectiveZoneId} but does not claim a route` : " 4. Deploy with `wrangler deploy`; wrangler.toml intentionally does not claim a route";
|
|
52829
|
+
const instructions = [
|
|
52830
|
+
"Deploy this Worker to your Cloudflare zone:",
|
|
52831
|
+
" 1. Save worker.js and wrangler.toml; neither file contains credentials",
|
|
52832
|
+
" 2. Install the required bearer and HMAC Worker secret bindings locally; never paste their values into source or chat",
|
|
52833
|
+
` 3. Inspect existing Cloudflare Worker routes first; confirm the generated \`${workerRouteHost}/*\` catch-all will not replace or collide with another Worker`,
|
|
52834
|
+
routeInstruction,
|
|
52835
|
+
` 5. In the Cloudflare Dashboard route form, select \`${workerRouteHost}/*\` and this Worker`,
|
|
52836
|
+
" 6. Set Request limit failure mode to Fail open, then save the route; Wrangler cannot set this toggle",
|
|
52837
|
+
" 7. Keep the Canonry ingest hostname outside this Worker route to avoid recursion",
|
|
52838
|
+
"",
|
|
52839
|
+
`Source id: ${sourceRow2.id}`,
|
|
52840
|
+
"After deploy, check `canonry doctor --project " + project.name + "` to confirm events are arriving."
|
|
52841
|
+
].join("\n");
|
|
52842
|
+
return {
|
|
52843
|
+
sourceId: sourceRow2.id,
|
|
52844
|
+
deliveryMode,
|
|
52845
|
+
workerScript,
|
|
52846
|
+
wranglerToml,
|
|
52847
|
+
workerVersion,
|
|
52848
|
+
instructions
|
|
52849
|
+
};
|
|
52850
|
+
});
|
|
52851
|
+
app.post("/projects/:name/traffic/cloudflare/ingest", {
|
|
52852
|
+
bodyLimit: CLOUDFLARE_INGEST_BODY_LIMIT,
|
|
52853
|
+
preHandler: async (request, reply) => {
|
|
52854
|
+
const key = cloudflareIngestRateLimitKey(
|
|
52855
|
+
request,
|
|
52856
|
+
opts.cloudflareTrafficCredentialStore,
|
|
52857
|
+
app.db
|
|
52858
|
+
);
|
|
52859
|
+
if (key.startsWith("cloudflare-invalid:")) return;
|
|
52860
|
+
const result = await checkCloudflareSourceRateLimit(request);
|
|
52861
|
+
if (result.isAllowed || !result.isExceeded) return;
|
|
52862
|
+
reply.header("retry-after", result.ttlInSeconds);
|
|
52863
|
+
return reply.status(429).send({
|
|
52864
|
+
statusCode: 429,
|
|
52865
|
+
error: "Too Many Requests",
|
|
52866
|
+
message: `Rate limit exceeded, retry in ${result.ttlInSeconds} second(s)`
|
|
52867
|
+
});
|
|
52868
|
+
},
|
|
52869
|
+
config: {
|
|
52870
|
+
rateLimit: {
|
|
52871
|
+
hook: "onRequest",
|
|
52872
|
+
max: opts.cloudflareIngestIpRateLimitMax ?? DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX,
|
|
52873
|
+
timeWindow: "1 minute",
|
|
52874
|
+
groupId: "cloudflare-direct-push-ip",
|
|
52875
|
+
keyGenerator: (request) => `cloudflare-ingest-ip:${request.ip}`
|
|
52876
|
+
}
|
|
52877
|
+
}
|
|
52878
|
+
}, async (request, reply) => {
|
|
52879
|
+
const authenticated = authenticateCloudflareIngest(
|
|
52880
|
+
request,
|
|
52881
|
+
opts.cloudflareTrafficCredentialStore
|
|
52882
|
+
);
|
|
52883
|
+
if (!authenticated || authenticated.credential.projectName !== request.params.name || !isDirectPushCloudflareDeliveryMode(authenticated.credential.deliveryMode)) {
|
|
52884
|
+
throw authRequired();
|
|
52885
|
+
}
|
|
52886
|
+
const { bearerHash, credential } = authenticated;
|
|
52887
|
+
const sourceId = credential.sourceId;
|
|
52888
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, sourceId)).get();
|
|
52889
|
+
if (!sourceRow2 || sourceRow2.sourceType !== TrafficSourceTypes.cloudflare || sourceRow2.status === TrafficSourceStatuses.archived || !parseDirectPushCloudflareSourceConfig(sourceRow2.configJson) || !timingSafeEqualHex(bearerHash, sourceRow2.ingestTokenHash)) {
|
|
52890
|
+
throw authRequired();
|
|
52891
|
+
}
|
|
52892
|
+
const project = resolveProject(app.db, request.params.name);
|
|
52893
|
+
if (sourceRow2.projectId !== project.id) throw authRequired();
|
|
52894
|
+
const parsed = cloudflareWorkerIngestRequestSchema.safeParse(request.body);
|
|
52895
|
+
if (!parsed.success) {
|
|
52896
|
+
throw validationError(parsed.error.issues.map((i) => i.message).join("; "));
|
|
52897
|
+
}
|
|
52898
|
+
const { workerVersion, events } = parsed.data;
|
|
52899
|
+
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
52900
|
+
const wrongHostIndex = events.findIndex(
|
|
52901
|
+
(event) => normalizeCloudflareEventHost(event.host) !== canonicalHost
|
|
52902
|
+
);
|
|
52903
|
+
if (wrongHostIndex !== -1) {
|
|
52904
|
+
throw validationError(
|
|
52905
|
+
`Cloudflare event at index ${wrongHostIndex} does not belong to project domain "${project.canonicalDomain}"`
|
|
52906
|
+
);
|
|
52907
|
+
}
|
|
52908
|
+
const normalized = [];
|
|
52909
|
+
let normalizationDrops = 0;
|
|
52910
|
+
for (const event of events) {
|
|
52911
|
+
const result = normalizeCloudflareWorkerEvent(event);
|
|
52912
|
+
if (result) normalized.push(result);
|
|
52913
|
+
else normalizationDrops += 1;
|
|
52914
|
+
}
|
|
52915
|
+
const receivedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
52916
|
+
const writeResult = writeTrafficEventBatch({
|
|
52917
|
+
db: app.db,
|
|
52918
|
+
projectId: project.id,
|
|
52919
|
+
sourceId,
|
|
52920
|
+
events: normalized,
|
|
52921
|
+
receivedAt,
|
|
52922
|
+
receiptTtlMs: DIRECT_PUSH_RECEIPT_TTL_MS,
|
|
52923
|
+
sampleLimit,
|
|
52924
|
+
validateSource: (latestRow) => {
|
|
52925
|
+
if (!latestRow || latestRow.projectId !== project.id || latestRow.sourceType !== TrafficSourceTypes.cloudflare || latestRow.status === TrafficSourceStatuses.archived || !parseDirectPushCloudflareSourceConfig(latestRow.configJson) || !timingSafeEqualHex(bearerHash, latestRow.ingestTokenHash)) {
|
|
52926
|
+
throw authRequired();
|
|
52927
|
+
}
|
|
52928
|
+
},
|
|
52929
|
+
sourceUpdate: {
|
|
52930
|
+
lastWorkerVersion: workerVersion,
|
|
52931
|
+
lastSyncedAt: receivedAt,
|
|
52932
|
+
lastError: null,
|
|
52933
|
+
updatedAt: receivedAt
|
|
52934
|
+
}
|
|
52935
|
+
});
|
|
52936
|
+
return reply.status(200).send({
|
|
52937
|
+
acceptedEvents: writeResult.acceptedEvents,
|
|
52938
|
+
droppedEvents: normalizationDrops + writeResult.duplicateEvents,
|
|
52939
|
+
workerVersionAck: workerVersion,
|
|
52940
|
+
crawlerBucketRows: writeResult.crawlerBucketRows,
|
|
52941
|
+
aiUserFetchBucketRows: writeResult.aiUserFetchBucketRows,
|
|
52942
|
+
aiReferralBucketRows: writeResult.aiReferralBucketRows,
|
|
52943
|
+
sampleRows: writeResult.sampleRows
|
|
52944
|
+
});
|
|
52945
|
+
});
|
|
51704
52946
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
51705
52947
|
const project = resolveProject(app.db, request.params.name);
|
|
51706
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
52948
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
51707
52949
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
51708
52950
|
throw notFound("Traffic source", request.params.id);
|
|
51709
52951
|
}
|
|
@@ -51715,7 +52957,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51715
52957
|
const windowEnd = /* @__PURE__ */ new Date();
|
|
51716
52958
|
const startedAt = windowEnd.toISOString();
|
|
51717
52959
|
const syncStartedAtMs = windowEnd.getTime();
|
|
51718
|
-
const runId =
|
|
52960
|
+
const runId = crypto40.randomUUID();
|
|
51719
52961
|
app.db.insert(runs).values({
|
|
51720
52962
|
id: runId,
|
|
51721
52963
|
projectId: project.id,
|
|
@@ -51729,8 +52971,8 @@ async function trafficRoutes(app, opts) {
|
|
|
51729
52971
|
const markFailed = (msg, errorCode) => {
|
|
51730
52972
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
51731
52973
|
app.db.transaction((tx) => {
|
|
51732
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(
|
|
51733
|
-
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(
|
|
52974
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq47(runs.id, runId)).run();
|
|
52975
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51734
52976
|
});
|
|
51735
52977
|
try {
|
|
51736
52978
|
opts.onTrafficSynced?.({
|
|
@@ -51811,7 +53053,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51811
53053
|
}
|
|
51812
53054
|
const credential = credentialStore.getConnection(project.name);
|
|
51813
53055
|
if (!credential) {
|
|
51814
|
-
app.db.delete(runs).where(
|
|
53056
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51815
53057
|
throw validationError(
|
|
51816
53058
|
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
51817
53059
|
);
|
|
@@ -51860,12 +53102,12 @@ async function trafficRoutes(app, opts) {
|
|
|
51860
53102
|
auditAction = "traffic.vercel.synced";
|
|
51861
53103
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
51862
53104
|
if (!credentialStore) {
|
|
51863
|
-
app.db.delete(runs).where(
|
|
53105
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51864
53106
|
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
51865
53107
|
}
|
|
51866
53108
|
const credential = credentialStore.getConnection(project.name);
|
|
51867
53109
|
if (!credential) {
|
|
51868
|
-
app.db.delete(runs).where(
|
|
53110
|
+
app.db.delete(runs).where(eq47(runs.id, runId)).run();
|
|
51869
53111
|
throw validationError(
|
|
51870
53112
|
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
51871
53113
|
);
|
|
@@ -51883,7 +53125,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51883
53125
|
if (cappedStartMs > clampedStartMs) {
|
|
51884
53126
|
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
51885
53127
|
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
51886
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(
|
|
53128
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
51887
53129
|
request.log.warn(
|
|
51888
53130
|
{
|
|
51889
53131
|
sourceId: sourceRow2.id,
|
|
@@ -51968,7 +53210,7 @@ async function trafficRoutes(app, opts) {
|
|
|
51968
53210
|
let aiReferralHitsCount = 0;
|
|
51969
53211
|
let unknownHitsCount = 0;
|
|
51970
53212
|
app.db.transaction((tx) => {
|
|
51971
|
-
const latestRow = tx.select().from(trafficSources).where(
|
|
53213
|
+
const latestRow = tx.select().from(trafficSources).where(eq47(trafficSources.id, sourceRow2.id)).get();
|
|
51972
53214
|
const previousIds = latestRow.lastEventIds ?? [];
|
|
51973
53215
|
const seenEventIds = new Set(previousIds);
|
|
51974
53216
|
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
@@ -52016,7 +53258,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52016
53258
|
crawlerEventsHourly.status
|
|
52017
53259
|
],
|
|
52018
53260
|
set: {
|
|
52019
|
-
hits:
|
|
53261
|
+
hits: sql19`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
52020
53262
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
52021
53263
|
updatedAt: finishedAt
|
|
52022
53264
|
}
|
|
@@ -52049,7 +53291,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52049
53291
|
aiUserFetchEventsHourly.status
|
|
52050
53292
|
],
|
|
52051
53293
|
set: {
|
|
52052
|
-
hits:
|
|
53294
|
+
hits: sql19`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
52053
53295
|
sampledUserAgent: bucket.sampledUserAgent,
|
|
52054
53296
|
updatedAt: finishedAt
|
|
52055
53297
|
}
|
|
@@ -52089,9 +53331,9 @@ async function trafficRoutes(app, opts) {
|
|
|
52089
53331
|
// colliding bucket adds its paid and organic arrivals to the ones
|
|
52090
53332
|
// already there rather than overwriting a single label.
|
|
52091
53333
|
set: {
|
|
52092
|
-
sessionsOrHits:
|
|
52093
|
-
paidSessionsOrHits:
|
|
52094
|
-
organicSessionsOrHits:
|
|
53334
|
+
sessionsOrHits: sql19`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
53335
|
+
paidSessionsOrHits: sql19`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
53336
|
+
organicSessionsOrHits: sql19`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
52095
53337
|
updatedAt: finishedAt
|
|
52096
53338
|
}
|
|
52097
53339
|
}).run();
|
|
@@ -52108,7 +53350,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52108
53350
|
}
|
|
52109
53351
|
})();
|
|
52110
53352
|
tx.insert(rawEventSamples).values({
|
|
52111
|
-
id:
|
|
53353
|
+
id: crypto40.randomUUID(),
|
|
52112
53354
|
projectId: project.id,
|
|
52113
53355
|
sourceId: sourceRow2.id,
|
|
52114
53356
|
ts: sample.observedAt,
|
|
@@ -52156,8 +53398,8 @@ async function trafficRoutes(app, opts) {
|
|
|
52156
53398
|
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
52157
53399
|
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
52158
53400
|
}
|
|
52159
|
-
tx.update(trafficSources).set(sourceUpdate).where(
|
|
52160
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(
|
|
53401
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
53402
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq47(runs.id, runId)).run();
|
|
52161
53403
|
});
|
|
52162
53404
|
writeAuditLog(app.db, {
|
|
52163
53405
|
projectId: project.id,
|
|
@@ -52209,7 +53451,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52209
53451
|
});
|
|
52210
53452
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
52211
53453
|
const project = resolveProject(app.db, request.params.name);
|
|
52212
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53454
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
52213
53455
|
if (!sourceRow2 || sourceRow2.projectId !== project.id) {
|
|
52214
53456
|
throw notFound("Traffic source", request.params.id);
|
|
52215
53457
|
}
|
|
@@ -52359,7 +53601,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52359
53601
|
};
|
|
52360
53602
|
}
|
|
52361
53603
|
const startedAt = windowEnd.toISOString();
|
|
52362
|
-
const runId =
|
|
53604
|
+
const runId = crypto40.randomUUID();
|
|
52363
53605
|
app.db.insert(runs).values({
|
|
52364
53606
|
id: runId,
|
|
52365
53607
|
projectId: project.id,
|
|
@@ -52395,40 +53637,40 @@ async function trafficRoutes(app, opts) {
|
|
|
52395
53637
|
function buildSourceDetail(projectId, row, since) {
|
|
52396
53638
|
const crawlerPathRows = app.db.select({
|
|
52397
53639
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
52398
|
-
hits:
|
|
53640
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
52399
53641
|
}).from(crawlerEventsHourly).where(
|
|
52400
|
-
|
|
52401
|
-
|
|
52402
|
-
|
|
53642
|
+
and39(
|
|
53643
|
+
eq47(crawlerEventsHourly.sourceId, row.id),
|
|
53644
|
+
gte11(crawlerEventsHourly.tsHour, since)
|
|
52403
53645
|
)
|
|
52404
53646
|
).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
52405
53647
|
const crawlerSegments = segmentCrawlerHits(
|
|
52406
53648
|
crawlerPathRows.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
52407
53649
|
);
|
|
52408
53650
|
const crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
52409
|
-
const aiUserFetchTotals = app.db.select({ total:
|
|
52410
|
-
|
|
52411
|
-
|
|
52412
|
-
|
|
53651
|
+
const aiUserFetchTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
53652
|
+
and39(
|
|
53653
|
+
eq47(aiUserFetchEventsHourly.sourceId, row.id),
|
|
53654
|
+
gte11(aiUserFetchEventsHourly.tsHour, since)
|
|
52413
53655
|
)
|
|
52414
53656
|
).get();
|
|
52415
|
-
const aiTotals = app.db.select({ total:
|
|
52416
|
-
|
|
52417
|
-
|
|
52418
|
-
|
|
53657
|
+
const aiTotals = app.db.select({ total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
53658
|
+
and39(
|
|
53659
|
+
eq47(aiReferralEventsHourly.sourceId, row.id),
|
|
53660
|
+
gte11(aiReferralEventsHourly.tsHour, since)
|
|
52419
53661
|
)
|
|
52420
53662
|
).get();
|
|
52421
|
-
const sampleTotals = app.db.select({ total:
|
|
52422
|
-
|
|
52423
|
-
|
|
52424
|
-
|
|
53663
|
+
const sampleTotals = app.db.select({ total: sql19`COUNT(*)` }).from(rawEventSamples).where(
|
|
53664
|
+
and39(
|
|
53665
|
+
eq47(rawEventSamples.sourceId, row.id),
|
|
53666
|
+
gte11(rawEventSamples.ts, since)
|
|
52425
53667
|
)
|
|
52426
53668
|
).get();
|
|
52427
53669
|
const latestRun = app.db.select().from(runs).where(
|
|
52428
|
-
|
|
52429
|
-
|
|
52430
|
-
|
|
52431
|
-
|
|
53670
|
+
and39(
|
|
53671
|
+
eq47(runs.projectId, projectId),
|
|
53672
|
+
eq47(runs.kind, RunKinds["traffic-sync"]),
|
|
53673
|
+
eq47(runs.sourceId, row.id)
|
|
52432
53674
|
)
|
|
52433
53675
|
).orderBy(desc22(runs.startedAt)).limit(1).get();
|
|
52434
53676
|
return {
|
|
@@ -52459,7 +53701,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52459
53701
|
"`advanceToNow` must be `true`. There is no implicit reset."
|
|
52460
53702
|
);
|
|
52461
53703
|
}
|
|
52462
|
-
const sourceRow2 = app.db.select().from(trafficSources).where(
|
|
53704
|
+
const sourceRow2 = app.db.select().from(trafficSources).where(and39(eq47(trafficSources.projectId, project.id), eq47(trafficSources.id, request.params.id))).get();
|
|
52463
53705
|
if (!sourceRow2) {
|
|
52464
53706
|
throw notFound("traffic source", request.params.id);
|
|
52465
53707
|
}
|
|
@@ -52476,7 +53718,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52476
53718
|
status: TrafficSourceStatuses.connected,
|
|
52477
53719
|
lastError: null,
|
|
52478
53720
|
updatedAt: now
|
|
52479
|
-
}).where(
|
|
53721
|
+
}).where(eq47(trafficSources.id, sourceRow2.id)).run();
|
|
52480
53722
|
writeAuditLog(tx, auditFromRequest(request, {
|
|
52481
53723
|
projectId: project.id,
|
|
52482
53724
|
actor: "api",
|
|
@@ -52484,20 +53726,20 @@ async function trafficRoutes(app, opts) {
|
|
|
52484
53726
|
entityType: "traffic_source",
|
|
52485
53727
|
entityId: sourceRow2.id
|
|
52486
53728
|
}));
|
|
52487
|
-
updatedRow = tx.select().from(trafficSources).where(
|
|
53729
|
+
updatedRow = tx.select().from(trafficSources).where(eq47(trafficSources.id, sourceRow2.id)).get();
|
|
52488
53730
|
});
|
|
52489
53731
|
return buildSourceDetail(project.id, updatedRow, new Date(Date.now() - 24 * 60 * 6e4).toISOString());
|
|
52490
53732
|
});
|
|
52491
53733
|
app.get("/projects/:name/traffic/sources", async (request) => {
|
|
52492
53734
|
const project = resolveProject(app.db, request.params.name);
|
|
52493
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
53735
|
+
const rows = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
52494
53736
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map(rowToDto);
|
|
52495
53737
|
const response = { sources };
|
|
52496
53738
|
return response;
|
|
52497
53739
|
});
|
|
52498
53740
|
app.get("/projects/:name/traffic/status", async (request) => {
|
|
52499
53741
|
const project = resolveProject(app.db, request.params.name);
|
|
52500
|
-
const rows = app.db.select().from(trafficSources).where(
|
|
53742
|
+
const rows = app.db.select().from(trafficSources).where(eq47(trafficSources.projectId, project.id)).orderBy(desc22(trafficSources.createdAt)).all();
|
|
52501
53743
|
const since = new Date(Date.now() - 24 * 60 * 6e4).toISOString();
|
|
52502
53744
|
const sources = rows.filter((row) => row.status !== TrafficSourceStatuses.archived).map((row) => buildSourceDetail(project.id, row, since));
|
|
52503
53745
|
const response = { sources };
|
|
@@ -52507,7 +53749,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52507
53749
|
"/projects/:name/traffic/sources/:id",
|
|
52508
53750
|
async (request) => {
|
|
52509
53751
|
const project = resolveProject(app.db, request.params.name);
|
|
52510
|
-
const row = app.db.select().from(trafficSources).where(
|
|
53752
|
+
const row = app.db.select().from(trafficSources).where(eq47(trafficSources.id, request.params.id)).get();
|
|
52511
53753
|
if (!row || row.projectId !== project.id) {
|
|
52512
53754
|
throw notFound("Traffic source", request.params.id);
|
|
52513
53755
|
}
|
|
@@ -52572,26 +53814,26 @@ async function trafficRoutes(app, opts) {
|
|
|
52572
53814
|
const seriesByBucket = /* @__PURE__ */ new Map();
|
|
52573
53815
|
if (kind === "all" || kind === TrafficEventKinds.crawler) {
|
|
52574
53816
|
const crawlerFilters = [
|
|
52575
|
-
|
|
52576
|
-
|
|
52577
|
-
|
|
53817
|
+
eq47(crawlerEventsHourly.projectId, project.id),
|
|
53818
|
+
gte11(crawlerEventsHourly.tsHour, sinceIso),
|
|
53819
|
+
lte10(crawlerEventsHourly.tsHour, untilIso)
|
|
52578
53820
|
];
|
|
52579
|
-
if (sourceIdParam) crawlerFilters.push(
|
|
52580
|
-
const crawlerWhere =
|
|
53821
|
+
if (sourceIdParam) crawlerFilters.push(eq47(crawlerEventsHourly.sourceId, sourceIdParam));
|
|
53822
|
+
const crawlerWhere = and39(...crawlerFilters);
|
|
52581
53823
|
const pathTotals = app.db.select({
|
|
52582
53824
|
pathNormalized: crawlerEventsHourly.pathNormalized,
|
|
52583
|
-
hits:
|
|
52584
|
-
rows:
|
|
53825
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`,
|
|
53826
|
+
rows: sql19`COUNT(*)`
|
|
52585
53827
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerEventsHourly.pathNormalized).all();
|
|
52586
53828
|
crawlerSegments = segmentCrawlerHits(
|
|
52587
53829
|
pathTotals.map((r) => ({ pathNormalized: r.pathNormalized, hits: Number(r.hits) }))
|
|
52588
53830
|
);
|
|
52589
53831
|
crawlerTotal = crawlerSegments.content + crawlerSegments.sitemap + crawlerSegments.robots + crawlerSegments.asset + crawlerSegments.other;
|
|
52590
53832
|
totalEventRows += pathTotals.reduce((sum, row) => sum + Number(row.rows), 0);
|
|
52591
|
-
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53833
|
+
const crawlerSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${crawlerEventsHourly.tsHour}, 1, 10)` : crawlerEventsHourly.tsHour;
|
|
52592
53834
|
const crawlerSeries = app.db.select({
|
|
52593
53835
|
bucket: crawlerSeriesBucket,
|
|
52594
|
-
hits:
|
|
53836
|
+
hits: sql19`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)`
|
|
52595
53837
|
}).from(crawlerEventsHourly).where(crawlerWhere).groupBy(crawlerSeriesBucket).all();
|
|
52596
53838
|
for (const row of crawlerSeries) {
|
|
52597
53839
|
trafficSeriesPoint(seriesByBucket, row.bucket).crawlerHits = Number(row.hits);
|
|
@@ -52614,22 +53856,22 @@ async function trafficRoutes(app, opts) {
|
|
|
52614
53856
|
}
|
|
52615
53857
|
if (kind === "all" || kind === TrafficEventKinds["ai-user-fetch"]) {
|
|
52616
53858
|
const userFetchFilters = [
|
|
52617
|
-
|
|
52618
|
-
|
|
52619
|
-
|
|
53859
|
+
eq47(aiUserFetchEventsHourly.projectId, project.id),
|
|
53860
|
+
gte11(aiUserFetchEventsHourly.tsHour, sinceIso),
|
|
53861
|
+
lte10(aiUserFetchEventsHourly.tsHour, untilIso)
|
|
52620
53862
|
];
|
|
52621
|
-
if (sourceIdParam) userFetchFilters.push(
|
|
52622
|
-
const userFetchWhere =
|
|
53863
|
+
if (sourceIdParam) userFetchFilters.push(eq47(aiUserFetchEventsHourly.sourceId, sourceIdParam));
|
|
53864
|
+
const userFetchWhere = and39(...userFetchFilters);
|
|
52623
53865
|
const total = app.db.select({
|
|
52624
|
-
total:
|
|
52625
|
-
rows:
|
|
53866
|
+
total: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`,
|
|
53867
|
+
rows: sql19`COUNT(*)`
|
|
52626
53868
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).get();
|
|
52627
53869
|
aiUserFetchTotal = Number(total?.total ?? 0);
|
|
52628
53870
|
totalEventRows += Number(total?.rows ?? 0);
|
|
52629
|
-
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53871
|
+
const userFetchSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${aiUserFetchEventsHourly.tsHour}, 1, 10)` : aiUserFetchEventsHourly.tsHour;
|
|
52630
53872
|
const userFetchSeries = app.db.select({
|
|
52631
53873
|
bucket: userFetchSeriesBucket,
|
|
52632
|
-
hits:
|
|
53874
|
+
hits: sql19`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)`
|
|
52633
53875
|
}).from(aiUserFetchEventsHourly).where(userFetchWhere).groupBy(userFetchSeriesBucket).all();
|
|
52634
53876
|
for (const row of userFetchSeries) {
|
|
52635
53877
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiUserFetchHits = Number(row.hits);
|
|
@@ -52651,17 +53893,17 @@ async function trafficRoutes(app, opts) {
|
|
|
52651
53893
|
}
|
|
52652
53894
|
if (kind === "all" || kind === TrafficEventKinds["ai-referral"]) {
|
|
52653
53895
|
const aiFilters = [
|
|
52654
|
-
|
|
52655
|
-
|
|
52656
|
-
|
|
53896
|
+
eq47(aiReferralEventsHourly.projectId, project.id),
|
|
53897
|
+
gte11(aiReferralEventsHourly.tsHour, sinceIso),
|
|
53898
|
+
lte10(aiReferralEventsHourly.tsHour, untilIso)
|
|
52657
53899
|
];
|
|
52658
|
-
if (sourceIdParam) aiFilters.push(
|
|
52659
|
-
const aiWhere =
|
|
53900
|
+
if (sourceIdParam) aiFilters.push(eq47(aiReferralEventsHourly.sourceId, sourceIdParam));
|
|
53901
|
+
const aiWhere = and39(...aiFilters);
|
|
52660
53902
|
const total = app.db.select({
|
|
52661
|
-
total:
|
|
52662
|
-
paid:
|
|
52663
|
-
organic:
|
|
52664
|
-
rows:
|
|
53903
|
+
total: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`,
|
|
53904
|
+
paid: sql19`COALESCE(SUM(${aiReferralEventsHourly.paidSessionsOrHits}), 0)`,
|
|
53905
|
+
organic: sql19`COALESCE(SUM(${aiReferralEventsHourly.organicSessionsOrHits}), 0)`,
|
|
53906
|
+
rows: sql19`COUNT(*)`
|
|
52665
53907
|
}).from(aiReferralEventsHourly).where(aiWhere).get();
|
|
52666
53908
|
aiReferralCounts = aiReferralClassCounts(
|
|
52667
53909
|
Number(total?.total ?? 0),
|
|
@@ -52669,10 +53911,10 @@ async function trafficRoutes(app, opts) {
|
|
|
52669
53911
|
Number(total?.organic ?? 0)
|
|
52670
53912
|
);
|
|
52671
53913
|
totalEventRows += Number(total?.rows ?? 0);
|
|
52672
|
-
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ?
|
|
53914
|
+
const referralSeriesBucket = granularity === TrafficSeriesGranularities.day ? sql19`substr(${aiReferralEventsHourly.tsHour}, 1, 10)` : aiReferralEventsHourly.tsHour;
|
|
52673
53915
|
const referralSeries = app.db.select({
|
|
52674
53916
|
bucket: referralSeriesBucket,
|
|
52675
|
-
hits:
|
|
53917
|
+
hits: sql19`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)`
|
|
52676
53918
|
}).from(aiReferralEventsHourly).where(aiWhere).groupBy(referralSeriesBucket).all();
|
|
52677
53919
|
for (const row of referralSeries) {
|
|
52678
53920
|
trafficSeriesPoint(seriesByBucket, row.bucket).aiReferralHits = Number(row.hits);
|
|
@@ -52729,7 +53971,7 @@ async function trafficRoutes(app, opts) {
|
|
|
52729
53971
|
}
|
|
52730
53972
|
|
|
52731
53973
|
// ../api-routes/src/doctor/checks/agent.ts
|
|
52732
|
-
import
|
|
53974
|
+
import crypto41 from "crypto";
|
|
52733
53975
|
import fs6 from "fs";
|
|
52734
53976
|
import path7 from "path";
|
|
52735
53977
|
var REQUIRED_SKILLS = ["canonry", "aero"];
|
|
@@ -52960,7 +54202,7 @@ function isInstalled(dir) {
|
|
|
52960
54202
|
}
|
|
52961
54203
|
function hashInstalledFile(filePath) {
|
|
52962
54204
|
try {
|
|
52963
|
-
return
|
|
54205
|
+
return crypto41.createHash("sha256").update(fs6.readFileSync(filePath)).digest("hex");
|
|
52964
54206
|
} catch {
|
|
52965
54207
|
return void 0;
|
|
52966
54208
|
}
|
|
@@ -52975,7 +54217,7 @@ function readInstalledManifest(skillDir) {
|
|
|
52975
54217
|
var AGENT_CHECKS = [skillsInstalledCheck, skillsCurrentCheck];
|
|
52976
54218
|
|
|
52977
54219
|
// ../api-routes/src/doctor/checks/backlinks.ts
|
|
52978
|
-
import { and as
|
|
54220
|
+
import { and as and40, eq as eq48 } from "drizzle-orm";
|
|
52979
54221
|
function skippedNoProject() {
|
|
52980
54222
|
return {
|
|
52981
54223
|
status: CheckStatuses.skipped,
|
|
@@ -52991,8 +54233,8 @@ var BACKLINKS_CHECKS = [
|
|
|
52991
54233
|
title: "Backlinks source connected",
|
|
52992
54234
|
run: (ctx) => {
|
|
52993
54235
|
if (!ctx.project) return skippedNoProject();
|
|
52994
|
-
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(
|
|
52995
|
-
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(
|
|
54236
|
+
const projectRow = ctx.db.select({ autoExtract: projects.autoExtractBacklinks }).from(projects).where(eq48(projects.id, ctx.project.id)).get();
|
|
54237
|
+
const readySync = ctx.db.select({ id: ccReleaseSyncs.id }).from(ccReleaseSyncs).where(eq48(ccReleaseSyncs.status, CcReleaseSyncStatuses.ready)).limit(1).get();
|
|
52996
54238
|
const ccConnected = projectRow?.autoExtract === true && !!readySync;
|
|
52997
54239
|
const connected = [];
|
|
52998
54240
|
if (ccConnected) connected.push(BacklinkSources.commoncrawl);
|
|
@@ -53005,9 +54247,9 @@ var BACKLINKS_CHECKS = [
|
|
|
53005
54247
|
details: { commoncrawl: ccConnected }
|
|
53006
54248
|
};
|
|
53007
54249
|
}
|
|
53008
|
-
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(
|
|
53009
|
-
|
|
53010
|
-
|
|
54250
|
+
const ccHasData = ccConnected ? !!ctx.db.select({ id: backlinkSummaries.id }).from(backlinkSummaries).where(and40(
|
|
54251
|
+
eq48(backlinkSummaries.projectId, ctx.project.id),
|
|
54252
|
+
eq48(backlinkSummaries.source, BacklinkSources.commoncrawl)
|
|
53011
54253
|
)).limit(1).get() : false;
|
|
53012
54254
|
return {
|
|
53013
54255
|
status: CheckStatuses.ok,
|
|
@@ -53167,7 +54409,7 @@ var BING_AUTH_CHECKS = [
|
|
|
53167
54409
|
];
|
|
53168
54410
|
|
|
53169
54411
|
// ../api-routes/src/doctor/checks/content.ts
|
|
53170
|
-
import { eq as
|
|
54412
|
+
import { eq as eq49 } from "drizzle-orm";
|
|
53171
54413
|
var WINNABILITY_COVERAGE_WARN_THRESHOLD = 0.8;
|
|
53172
54414
|
var UNCLASSIFIED_DOMAIN_SAMPLE_LIMIT = 10;
|
|
53173
54415
|
function skippedNoProject2() {
|
|
@@ -53180,7 +54422,7 @@ function skippedNoProject2() {
|
|
|
53180
54422
|
}
|
|
53181
54423
|
function loadProject(ctx) {
|
|
53182
54424
|
if (!ctx.project) return null;
|
|
53183
|
-
return ctx.db.select().from(projects).where(
|
|
54425
|
+
return ctx.db.select().from(projects).where(eq49(projects.id, ctx.project.id)).get() ?? null;
|
|
53184
54426
|
}
|
|
53185
54427
|
function percent(value) {
|
|
53186
54428
|
return Math.round(value * 100);
|
|
@@ -53272,7 +54514,7 @@ var CONTENT_CHECK_BY_ID = Object.fromEntries(
|
|
|
53272
54514
|
);
|
|
53273
54515
|
|
|
53274
54516
|
// ../api-routes/src/doctor/checks/ads.ts
|
|
53275
|
-
import { eq as
|
|
54517
|
+
import { eq as eq50 } from "drizzle-orm";
|
|
53276
54518
|
var RECENT_SYNC_WARN_DAYS = 7;
|
|
53277
54519
|
var RECENT_SYNC_FAIL_DAYS = 30;
|
|
53278
54520
|
var adsConnectionCheck = {
|
|
@@ -53289,7 +54531,7 @@ var adsConnectionCheck = {
|
|
|
53289
54531
|
remediation: null
|
|
53290
54532
|
};
|
|
53291
54533
|
}
|
|
53292
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
54534
|
+
const row = ctx.db.select().from(adsConnections).where(eq50(adsConnections.projectId, ctx.project.id)).get();
|
|
53293
54535
|
if (!row) {
|
|
53294
54536
|
return {
|
|
53295
54537
|
status: CheckStatuses.skipped,
|
|
@@ -53339,7 +54581,7 @@ var adsRecentSyncCheck = {
|
|
|
53339
54581
|
remediation: null
|
|
53340
54582
|
};
|
|
53341
54583
|
}
|
|
53342
|
-
const row = ctx.db.select().from(adsConnections).where(
|
|
54584
|
+
const row = ctx.db.select().from(adsConnections).where(eq50(adsConnections.projectId, ctx.project.id)).get();
|
|
53343
54585
|
if (!row) {
|
|
53344
54586
|
return {
|
|
53345
54587
|
status: CheckStatuses.skipped,
|
|
@@ -53530,7 +54772,7 @@ var ga4ConnectionCheck = {
|
|
|
53530
54772
|
var GA_AUTH_CHECKS = [ga4ConnectionCheck];
|
|
53531
54773
|
|
|
53532
54774
|
// ../api-routes/src/doctor/checks/gbp-auth.ts
|
|
53533
|
-
import { and as
|
|
54775
|
+
import { and as and41, eq as eq51 } from "drizzle-orm";
|
|
53534
54776
|
var RECENT_SYNC_WARN_DAYS2 = 7;
|
|
53535
54777
|
var RECENT_SYNC_FAIL_DAYS2 = 30;
|
|
53536
54778
|
function skippedNoProject3() {
|
|
@@ -53763,7 +55005,7 @@ var recentSyncCheck = {
|
|
|
53763
55005
|
title: "GBP recent sync",
|
|
53764
55006
|
run: (ctx) => {
|
|
53765
55007
|
if (!ctx.project) return skippedNoProject3();
|
|
53766
|
-
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(
|
|
55008
|
+
const selected = ctx.db.select({ locationName: gbpLocations.locationName, syncedAt: gbpLocations.syncedAt }).from(gbpLocations).where(and41(eq51(gbpLocations.projectId, ctx.project.id), eq51(gbpLocations.selected, true))).all();
|
|
53767
55009
|
if (selected.length === 0) {
|
|
53768
55010
|
return {
|
|
53769
55011
|
status: CheckStatuses.skipped,
|
|
@@ -53823,7 +55065,7 @@ var GBP_AUTH_CHECK_BY_ID = Object.fromEntries(
|
|
|
53823
55065
|
);
|
|
53824
55066
|
|
|
53825
55067
|
// ../api-routes/src/doctor/checks/places.ts
|
|
53826
|
-
import { eq as
|
|
55068
|
+
import { eq as eq52 } from "drizzle-orm";
|
|
53827
55069
|
var apiKeyCheck = {
|
|
53828
55070
|
id: "gbp.places.api-key",
|
|
53829
55071
|
category: CheckCategories.auth,
|
|
@@ -53868,7 +55110,7 @@ var apiKeyCheck = {
|
|
|
53868
55110
|
details: { tier: cfg.tier }
|
|
53869
55111
|
};
|
|
53870
55112
|
}
|
|
53871
|
-
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(
|
|
55113
|
+
const rows = ctx.db.select({ placeId: gbpLocations.placeId, selected: gbpLocations.selected }).from(gbpLocations).where(eq52(gbpLocations.projectId, ctx.project.id)).all();
|
|
53872
55114
|
const selected = rows.filter((r) => r.selected);
|
|
53873
55115
|
const locationsWithPlaceId = selected.filter((r) => Boolean(r.placeId)).length;
|
|
53874
55116
|
const details = {
|
|
@@ -54383,9 +55625,25 @@ var RUNTIME_STATE_CHECKS = [
|
|
|
54383
55625
|
];
|
|
54384
55626
|
|
|
54385
55627
|
// ../api-routes/src/doctor/checks/traffic-source.ts
|
|
54386
|
-
import { and as
|
|
55628
|
+
import { and as and42, eq as eq53, gte as gte12, ne as ne6, sql as sql20 } from "drizzle-orm";
|
|
54387
55629
|
var RECENT_DATA_WARN_DAYS = 7;
|
|
54388
55630
|
var RECENT_DATA_FAIL_DAYS = 30;
|
|
55631
|
+
function isCloudflareDirectPush(source) {
|
|
55632
|
+
if (source.sourceType !== TrafficSourceTypes.cloudflare) return false;
|
|
55633
|
+
const deliveryMode = source.configJson.deliveryMode;
|
|
55634
|
+
return deliveryMode === void 0 || deliveryMode === CloudflareTrafficDeliveryModes["direct-push"];
|
|
55635
|
+
}
|
|
55636
|
+
function recentDataRemediation(sources, lastSyncedAt) {
|
|
55637
|
+
const hasCloudflare = sources.some(isCloudflareDirectPush);
|
|
55638
|
+
const hasPullSource = sources.some((source) => !isCloudflareDirectPush(source));
|
|
55639
|
+
if (hasCloudflare && !hasPullSource) {
|
|
55640
|
+
return "Confirm the Cloudflare Worker is deployed on the zone route and forwarding to canonry; reconnect Cloudflare to regenerate the Worker if needed.";
|
|
55641
|
+
}
|
|
55642
|
+
if (hasCloudflare && hasPullSource) {
|
|
55643
|
+
return "Run `canonry traffic sync <project>` for pull-based sources, and confirm the Cloudflare Worker is deployed on the zone route and forwarding to canonry.";
|
|
55644
|
+
}
|
|
55645
|
+
return lastSyncedAt ? `Last sync: ${lastSyncedAt}. Run \`canonry traffic sync <project>\` to refresh, or check the source connection.` : "Run `canonry traffic sync <project>` to pull recent events.";
|
|
55646
|
+
}
|
|
54389
55647
|
function skippedNoProject5() {
|
|
54390
55648
|
return {
|
|
54391
55649
|
status: CheckStatuses.skipped,
|
|
@@ -54397,8 +55655,8 @@ function skippedNoProject5() {
|
|
|
54397
55655
|
function loadProbes(ctx) {
|
|
54398
55656
|
if (!ctx.project) return [];
|
|
54399
55657
|
const rows = ctx.db.select().from(trafficSources).where(
|
|
54400
|
-
|
|
54401
|
-
|
|
55658
|
+
and42(
|
|
55659
|
+
eq53(trafficSources.projectId, ctx.project.id),
|
|
54402
55660
|
ne6(trafficSources.status, TrafficSourceStatuses.archived)
|
|
54403
55661
|
)
|
|
54404
55662
|
).all();
|
|
@@ -54410,6 +55668,8 @@ function loadProbes(ctx) {
|
|
|
54410
55668
|
displayName: r.displayName,
|
|
54411
55669
|
status: r.status,
|
|
54412
55670
|
lastSyncedAt: r.lastSyncedAt,
|
|
55671
|
+
lastWorkerVersion: r.lastWorkerVersion,
|
|
55672
|
+
ingestTokenHash: r.ingestTokenHash,
|
|
54413
55673
|
skippedThroughAt: r.skippedThroughAt,
|
|
54414
55674
|
lastError: r.lastError,
|
|
54415
55675
|
configJson: r.configJson
|
|
@@ -54478,52 +55738,74 @@ var recentDataCheck = {
|
|
|
54478
55738
|
const warnCutoff = new Date(now.getTime() - RECENT_DATA_WARN_DAYS * 24 * 60 * 6e4).toISOString();
|
|
54479
55739
|
const failCutoff = new Date(now.getTime() - RECENT_DATA_FAIL_DAYS * 24 * 60 * 6e4).toISOString();
|
|
54480
55740
|
const recentCrawlers = Number(
|
|
54481
|
-
ctx.db.select({ total:
|
|
54482
|
-
|
|
54483
|
-
|
|
54484
|
-
|
|
55741
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
55742
|
+
and42(
|
|
55743
|
+
eq53(crawlerEventsHourly.projectId, ctx.project.id),
|
|
55744
|
+
gte12(crawlerEventsHourly.tsHour, warnCutoff)
|
|
54485
55745
|
)
|
|
54486
55746
|
).get()?.total ?? 0
|
|
54487
55747
|
);
|
|
54488
55748
|
const recentReferrals = Number(
|
|
54489
|
-
ctx.db.select({ total:
|
|
54490
|
-
|
|
54491
|
-
|
|
54492
|
-
|
|
55749
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
55750
|
+
and42(
|
|
55751
|
+
eq53(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
55752
|
+
gte12(aiReferralEventsHourly.tsHour, warnCutoff)
|
|
54493
55753
|
)
|
|
54494
55754
|
).get()?.total ?? 0
|
|
54495
55755
|
);
|
|
54496
|
-
|
|
55756
|
+
const recentUserFetches = Number(
|
|
55757
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
55758
|
+
and42(
|
|
55759
|
+
eq53(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
55760
|
+
gte12(aiUserFetchEventsHourly.tsHour, warnCutoff)
|
|
55761
|
+
)
|
|
55762
|
+
).get()?.total ?? 0
|
|
55763
|
+
);
|
|
55764
|
+
if (recentCrawlers > 0 || recentUserFetches > 0 || recentReferrals > 0) {
|
|
54497
55765
|
return {
|
|
54498
55766
|
status: CheckStatuses.ok,
|
|
54499
55767
|
code: "traffic.recent-data.fresh",
|
|
54500
|
-
summary: `${recentCrawlers} crawler hit(s) and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
54501
|
-
details: {
|
|
55768
|
+
summary: `${recentCrawlers} crawler hit(s), ${recentUserFetches} AI user-fetch hit(s), and ${recentReferrals} AI-referral arrival(s) in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
55769
|
+
details: {
|
|
55770
|
+
crawlerHits: recentCrawlers,
|
|
55771
|
+
aiUserFetchHits: recentUserFetches,
|
|
55772
|
+
referralArrivals: recentReferrals,
|
|
55773
|
+
windowDays: RECENT_DATA_WARN_DAYS
|
|
55774
|
+
}
|
|
54502
55775
|
};
|
|
54503
55776
|
}
|
|
54504
55777
|
const olderCrawlers = Number(
|
|
54505
|
-
ctx.db.select({ total:
|
|
54506
|
-
|
|
54507
|
-
|
|
54508
|
-
|
|
55778
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${crawlerEventsHourly.hits}), 0)` }).from(crawlerEventsHourly).where(
|
|
55779
|
+
and42(
|
|
55780
|
+
eq53(crawlerEventsHourly.projectId, ctx.project.id),
|
|
55781
|
+
gte12(crawlerEventsHourly.tsHour, failCutoff)
|
|
54509
55782
|
)
|
|
54510
55783
|
).get()?.total ?? 0
|
|
54511
55784
|
);
|
|
54512
55785
|
const olderReferrals = Number(
|
|
54513
|
-
ctx.db.select({ total:
|
|
54514
|
-
|
|
54515
|
-
|
|
54516
|
-
|
|
55786
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiReferralEventsHourly.sessionsOrHits}), 0)` }).from(aiReferralEventsHourly).where(
|
|
55787
|
+
and42(
|
|
55788
|
+
eq53(aiReferralEventsHourly.projectId, ctx.project.id),
|
|
55789
|
+
gte12(aiReferralEventsHourly.tsHour, failCutoff)
|
|
55790
|
+
)
|
|
55791
|
+
).get()?.total ?? 0
|
|
55792
|
+
);
|
|
55793
|
+
const olderUserFetches = Number(
|
|
55794
|
+
ctx.db.select({ total: sql20`COALESCE(SUM(${aiUserFetchEventsHourly.hits}), 0)` }).from(aiUserFetchEventsHourly).where(
|
|
55795
|
+
and42(
|
|
55796
|
+
eq53(aiUserFetchEventsHourly.projectId, ctx.project.id),
|
|
55797
|
+
gte12(aiUserFetchEventsHourly.tsHour, failCutoff)
|
|
54517
55798
|
)
|
|
54518
55799
|
).get()?.total ?? 0
|
|
54519
55800
|
);
|
|
54520
55801
|
const lastSyncedAt = sources.map((s) => s.lastSyncedAt).filter(Boolean).sort().at(-1) ?? null;
|
|
54521
|
-
|
|
55802
|
+
const hasOlderData = olderCrawlers > 0 || olderUserFetches > 0 || olderReferrals > 0;
|
|
55803
|
+
if (hasOlderData || lastSyncedAt) {
|
|
54522
55804
|
return {
|
|
54523
55805
|
status: CheckStatuses.warn,
|
|
54524
55806
|
code: "traffic.recent-data.stale",
|
|
54525
|
-
summary: `No crawler
|
|
54526
|
-
remediation:
|
|
55807
|
+
summary: hasOlderData ? `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days, though older data exists.` : `No crawler, AI user-fetch, or AI-referral hits in the last ${RECENT_DATA_WARN_DAYS} days.`,
|
|
55808
|
+
remediation: recentDataRemediation(sources, lastSyncedAt),
|
|
54527
55809
|
details: { lastSyncedAt, sourceCount: sources.length }
|
|
54528
55810
|
};
|
|
54529
55811
|
}
|
|
@@ -54721,7 +56003,15 @@ var syncLagCheck = {
|
|
|
54721
56003
|
title: "Traffic source sync lag",
|
|
54722
56004
|
run: (ctx) => {
|
|
54723
56005
|
if (!ctx.project) return skippedNoProject5();
|
|
54724
|
-
const
|
|
56006
|
+
const allSources = loadProbes(ctx);
|
|
56007
|
+
const sources = allSources.filter((source) => !isCloudflareDirectPush(source));
|
|
56008
|
+
if (allSources.length > 0 && sources.length === 0) {
|
|
56009
|
+
return {
|
|
56010
|
+
status: CheckStatuses.skipped,
|
|
56011
|
+
code: "traffic.sync-lag.push-only",
|
|
56012
|
+
summary: "Only push traffic sources are connected \u2014 pull sync lag does not apply."
|
|
56013
|
+
};
|
|
56014
|
+
}
|
|
54725
56015
|
if (sources.length === 0) {
|
|
54726
56016
|
return {
|
|
54727
56017
|
status: CheckStatuses.skipped,
|
|
@@ -54811,10 +56101,68 @@ var syncLagCheck = {
|
|
|
54811
56101
|
};
|
|
54812
56102
|
}
|
|
54813
56103
|
};
|
|
56104
|
+
var workerVersionCheck = {
|
|
56105
|
+
id: "traffic.source.worker-version",
|
|
56106
|
+
category: CheckCategories.integrations,
|
|
56107
|
+
scope: CheckScopes.project,
|
|
56108
|
+
title: "Cloudflare Worker version",
|
|
56109
|
+
run: (ctx) => {
|
|
56110
|
+
if (!ctx.project) return skippedNoProject5();
|
|
56111
|
+
const sources = loadProbes(ctx).filter(isCloudflareDirectPush);
|
|
56112
|
+
if (sources.length === 0) {
|
|
56113
|
+
return {
|
|
56114
|
+
status: CheckStatuses.skipped,
|
|
56115
|
+
code: "traffic.worker-version.not-applicable",
|
|
56116
|
+
summary: "No Cloudflare direct-push source is connected."
|
|
56117
|
+
};
|
|
56118
|
+
}
|
|
56119
|
+
const measured = sources.map((source) => {
|
|
56120
|
+
const expected = source.configJson.workerVersion;
|
|
56121
|
+
const expectedVersion = typeof expected === "string" && expected.length > 0 ? expected : null;
|
|
56122
|
+
const observedVersion = source.lastWorkerVersion;
|
|
56123
|
+
const state = observedVersion === null ? "waiting-for-first-event" : expectedVersion !== null && observedVersion === expectedVersion ? "current" : "stale";
|
|
56124
|
+
return {
|
|
56125
|
+
sourceId: source.id,
|
|
56126
|
+
displayName: source.displayName,
|
|
56127
|
+
expectedVersion,
|
|
56128
|
+
observedVersion,
|
|
56129
|
+
state
|
|
56130
|
+
};
|
|
56131
|
+
});
|
|
56132
|
+
const details = { sources: measured };
|
|
56133
|
+
const stale = measured.filter((source) => source.state === "stale");
|
|
56134
|
+
if (stale.length > 0) {
|
|
56135
|
+
return {
|
|
56136
|
+
status: CheckStatuses.warn,
|
|
56137
|
+
code: "traffic.worker-version.stale",
|
|
56138
|
+
summary: `${stale.length} Cloudflare Worker source(s) report a version that differs from the generated version.`,
|
|
56139
|
+
remediation: "Redeploy the generated Worker from the credential-owning host. Use `canonry traffic connect cloudflare <project> --zone-id <id> --deploy --confirm-route --confirm-fail-open`, then attach the route manually.",
|
|
56140
|
+
details
|
|
56141
|
+
};
|
|
56142
|
+
}
|
|
56143
|
+
const waiting = measured.filter((source) => source.state === "waiting-for-first-event");
|
|
56144
|
+
if (waiting.length > 0) {
|
|
56145
|
+
return {
|
|
56146
|
+
status: CheckStatuses.warn,
|
|
56147
|
+
code: "traffic.worker-version.waiting-for-first-event",
|
|
56148
|
+
summary: `${waiting.length} Cloudflare Worker source(s) have not delivered an event yet.`,
|
|
56149
|
+
remediation: "Send a smoke-test request through the Worker. Then run `canonry doctor --project <project>` again.",
|
|
56150
|
+
details
|
|
56151
|
+
};
|
|
56152
|
+
}
|
|
56153
|
+
return {
|
|
56154
|
+
status: CheckStatuses.ok,
|
|
56155
|
+
code: "traffic.worker-version.current",
|
|
56156
|
+
summary: "All Cloudflare direct-push Workers report the generated version.",
|
|
56157
|
+
details
|
|
56158
|
+
};
|
|
56159
|
+
}
|
|
56160
|
+
};
|
|
54814
56161
|
var TRAFFIC_SOURCE_CHECKS = [
|
|
54815
56162
|
sourceConnectedCheck,
|
|
54816
56163
|
recentDataCheck,
|
|
54817
56164
|
syncLagCheck,
|
|
56165
|
+
workerVersionCheck,
|
|
54818
56166
|
credentialsCheck,
|
|
54819
56167
|
scopesCheck3,
|
|
54820
56168
|
cacheBlindSpotCheck
|
|
@@ -55033,8 +56381,8 @@ async function doctorRoutes(app, opts) {
|
|
|
55033
56381
|
}
|
|
55034
56382
|
|
|
55035
56383
|
// ../api-routes/src/discovery/routes.ts
|
|
55036
|
-
import
|
|
55037
|
-
import { and as
|
|
56384
|
+
import crypto42 from "crypto";
|
|
56385
|
+
import { and as and43, desc as desc23, eq as eq54, gte as gte13, inArray as inArray18, isNull as isNull4, or as or10 } from "drizzle-orm";
|
|
55038
56386
|
var MAX_INFLIGHT_DISCOVERY_AGE_MS = 2 * 60 * 60 * 1e3;
|
|
55039
56387
|
async function discoveryRoutes(app, opts) {
|
|
55040
56388
|
app.post("/projects/:name/discover/run", async (request, reply) => {
|
|
@@ -55068,13 +56416,13 @@ async function discoveryRoutes(app, opts) {
|
|
|
55068
56416
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
55069
56417
|
const ageFloorIso = new Date(Date.now() - MAX_INFLIGHT_DISCOVERY_AGE_MS).toISOString();
|
|
55070
56418
|
const decision = app.db.transaction((tx) => {
|
|
55071
|
-
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(
|
|
55072
|
-
|
|
55073
|
-
|
|
56419
|
+
const existing = tx.select({ id: discoverySessions.id, runId: discoverySessions.runId }).from(discoverySessions).where(and43(
|
|
56420
|
+
eq54(discoverySessions.projectId, project.id),
|
|
56421
|
+
eq54(discoverySessions.icpDescription, icpDescription),
|
|
55074
56422
|
// Buyer is part of session identity: it changes the seed prompt's
|
|
55075
56423
|
// semantics, so a request with a different (or no) buyer must start
|
|
55076
56424
|
// its own session, never adopt another buyer's probes.
|
|
55077
|
-
parsed.data.buyerDescription == null ? isNull4(discoverySessions.buyerDescription) :
|
|
56425
|
+
parsed.data.buyerDescription == null ? isNull4(discoverySessions.buyerDescription) : eq54(discoverySessions.buyerDescription, parsed.data.buyerDescription),
|
|
55078
56426
|
// Locations are identity too: a different service-area subset seeds
|
|
55079
56427
|
// and probes a different geo, so it must never reuse another geo's
|
|
55080
56428
|
// session. resolveLocations is deterministic (project-config order),
|
|
@@ -55084,23 +56432,23 @@ async function discoveryRoutes(app, opts) {
|
|
|
55084
56432
|
// locations a NULL row's subset is unknowable, so it conservatively
|
|
55085
56433
|
// never reuses — a one-time, bounded (2h window) non-reuse after
|
|
55086
56434
|
// upgrade, never a wrong reuse.
|
|
55087
|
-
locations.length === 0 ? or10(isNull4(discoverySessions.locations),
|
|
56435
|
+
locations.length === 0 ? or10(isNull4(discoverySessions.locations), eq54(discoverySessions.locations, locations)) : eq54(discoverySessions.locations, locations),
|
|
55088
56436
|
// Seed provider set is identity: a different phrasing distribution
|
|
55089
56437
|
// must never reuse another set's session. Null (= Gemini-only
|
|
55090
56438
|
// default, including explicit ['gemini']) matches legacy rows.
|
|
55091
|
-
seedProviders == null ? isNull4(discoverySessions.seedProviders) :
|
|
56439
|
+
seedProviders == null ? isNull4(discoverySessions.seedProviders) : eq54(discoverySessions.seedProviders, seedProviders),
|
|
55092
56440
|
inArray18(discoverySessions.status, [
|
|
55093
56441
|
DiscoverySessionStatuses.queued,
|
|
55094
56442
|
DiscoverySessionStatuses.seeding,
|
|
55095
56443
|
DiscoverySessionStatuses.probing
|
|
55096
56444
|
]),
|
|
55097
|
-
|
|
56445
|
+
gte13(discoverySessions.createdAt, ageFloorIso)
|
|
55098
56446
|
)).orderBy(desc23(discoverySessions.createdAt)).get();
|
|
55099
56447
|
if (existing && existing.runId) {
|
|
55100
56448
|
return { reused: true, sessionId: existing.id, runId: existing.runId };
|
|
55101
56449
|
}
|
|
55102
|
-
const sessionId =
|
|
55103
|
-
const runId =
|
|
56450
|
+
const sessionId = crypto42.randomUUID();
|
|
56451
|
+
const runId = crypto42.randomUUID();
|
|
55104
56452
|
tx.insert(discoverySessions).values({
|
|
55105
56453
|
id: sessionId,
|
|
55106
56454
|
projectId: project.id,
|
|
@@ -55164,7 +56512,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55164
56512
|
const project = resolveProject(app.db, request.params.name);
|
|
55165
56513
|
const parsedLimit = parseInt(request.query.limit ?? "", 10);
|
|
55166
56514
|
const limit = Number.isNaN(parsedLimit) || parsedLimit <= 0 ? 50 : parsedLimit;
|
|
55167
|
-
const rows = app.db.select().from(discoverySessions).where(
|
|
56515
|
+
const rows = app.db.select().from(discoverySessions).where(eq54(discoverySessions.projectId, project.id)).orderBy(desc23(discoverySessions.createdAt)).limit(limit).all();
|
|
55168
56516
|
return reply.send(rows.map(serializeSession));
|
|
55169
56517
|
}
|
|
55170
56518
|
);
|
|
@@ -55172,11 +56520,11 @@ async function discoveryRoutes(app, opts) {
|
|
|
55172
56520
|
"/projects/:name/discover/sessions/:id",
|
|
55173
56521
|
async (request, reply) => {
|
|
55174
56522
|
const project = resolveProject(app.db, request.params.name);
|
|
55175
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56523
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55176
56524
|
if (!session || session.projectId !== project.id) {
|
|
55177
56525
|
throw notFound("Discovery session", request.params.id);
|
|
55178
56526
|
}
|
|
55179
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56527
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
55180
56528
|
const detail = {
|
|
55181
56529
|
...serializeSession(session),
|
|
55182
56530
|
probes: probeRows.map(serializeProbe)
|
|
@@ -55188,7 +56536,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55188
56536
|
"/projects/:name/discover/sessions/:id/harvest",
|
|
55189
56537
|
async (request, reply) => {
|
|
55190
56538
|
const project = resolveProject(app.db, request.params.name);
|
|
55191
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56539
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55192
56540
|
if (!session || session.projectId !== project.id) {
|
|
55193
56541
|
throw notFound("Discovery session", request.params.id);
|
|
55194
56542
|
}
|
|
@@ -55196,7 +56544,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55196
56544
|
const minProbeHits = Number.isNaN(parsedFloor) || parsedFloor < 1 ? 1 : parsedFloor;
|
|
55197
56545
|
const applyAnchor = request.query.anchor !== "false";
|
|
55198
56546
|
const provider = session.seedProvider ?? "gemini";
|
|
55199
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56547
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
55200
56548
|
const extract = opts.harvestSearchQueries;
|
|
55201
56549
|
const probesWithQueries = probeRows.map((row) => {
|
|
55202
56550
|
if (!extract || !row.rawResponse) return { searchQueries: [] };
|
|
@@ -55207,7 +56555,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55207
56555
|
return { searchQueries: [] };
|
|
55208
56556
|
}
|
|
55209
56557
|
});
|
|
55210
|
-
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(
|
|
56558
|
+
const trackedQueries = app.db.select({ query: queries.query }).from(queries).where(eq54(queries.projectId, project.id)).all().map((r) => r.query);
|
|
55211
56559
|
const anchorTerms = buildHarvestAnchorTerms(
|
|
55212
56560
|
[session.icpDescription ?? "", ...trackedQueries],
|
|
55213
56561
|
effectiveDomains(project)
|
|
@@ -55254,12 +56602,12 @@ async function discoveryRoutes(app, opts) {
|
|
|
55254
56602
|
"/projects/:name/discover/sessions/:id/promote",
|
|
55255
56603
|
async (request, reply) => {
|
|
55256
56604
|
const project = resolveProject(app.db, request.params.name);
|
|
55257
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56605
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55258
56606
|
if (!session || session.projectId !== project.id) {
|
|
55259
56607
|
throw notFound("Discovery session", request.params.id);
|
|
55260
56608
|
}
|
|
55261
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
55262
|
-
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
56609
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
56610
|
+
const existingCompetitors = app.db.select({ domain: competitors.domain }).from(competitors).where(eq54(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase());
|
|
55263
56611
|
const seenCompetitors = new Set(existingCompetitors);
|
|
55264
56612
|
const cited = /* @__PURE__ */ new Set();
|
|
55265
56613
|
const aspirational = /* @__PURE__ */ new Set();
|
|
@@ -55288,7 +56636,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55288
56636
|
);
|
|
55289
56637
|
app.post("/projects/:name/discover/sessions/:id/promote", async (request, reply) => {
|
|
55290
56638
|
const project = resolveProject(app.db, request.params.name);
|
|
55291
|
-
const session = app.db.select().from(discoverySessions).where(
|
|
56639
|
+
const session = app.db.select().from(discoverySessions).where(eq54(discoverySessions.id, request.params.id)).get();
|
|
55292
56640
|
if (!session || session.projectId !== project.id) {
|
|
55293
56641
|
throw notFound("Discovery session", request.params.id);
|
|
55294
56642
|
}
|
|
@@ -55311,7 +56659,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55311
56659
|
const bucketSet = new Set(buckets);
|
|
55312
56660
|
const includeCompetitors = parsed.data.includeCompetitors ?? true;
|
|
55313
56661
|
const competitorTypes = parsed.data.competitorTypes ?? DEFAULT_DISCOVERY_PROMOTE_COMPETITOR_TYPES;
|
|
55314
|
-
const probeRows = app.db.select().from(discoveryProbes).where(
|
|
56662
|
+
const probeRows = app.db.select().from(discoveryProbes).where(eq54(discoveryProbes.sessionId, session.id)).all();
|
|
55315
56663
|
const candidateQueries = /* @__PURE__ */ new Set();
|
|
55316
56664
|
for (const probe of probeRows) {
|
|
55317
56665
|
if (!probe.bucket) continue;
|
|
@@ -55319,7 +56667,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55319
56667
|
if (bucket.success && bucketSet.has(bucket.data)) candidateQueries.add(probe.query);
|
|
55320
56668
|
}
|
|
55321
56669
|
const existingQueries = new Set(
|
|
55322
|
-
app.db.select({ query: queries.query }).from(queries).where(
|
|
56670
|
+
app.db.select({ query: queries.query }).from(queries).where(eq54(queries.projectId, project.id)).all().map((r) => r.query.toLowerCase())
|
|
55323
56671
|
);
|
|
55324
56672
|
const promotedQueries = [];
|
|
55325
56673
|
const skippedQueries = [];
|
|
@@ -55335,7 +56683,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55335
56683
|
const skippedCompetitors = [];
|
|
55336
56684
|
if (includeCompetitors) {
|
|
55337
56685
|
const existingCompetitors = new Set(
|
|
55338
|
-
app.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
56686
|
+
app.db.select({ domain: competitors.domain }).from(competitors).where(eq54(competitors.projectId, project.id)).all().map((r) => r.domain.toLowerCase())
|
|
55339
56687
|
);
|
|
55340
56688
|
const competitorMap = parseCompetitorMap(session.competitorMap);
|
|
55341
56689
|
for (const entry of selectEligibleCompetitors(competitorMap, competitorTypes)) {
|
|
@@ -55354,7 +56702,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55354
56702
|
app.db.transaction((tx) => {
|
|
55355
56703
|
for (const query of promotedQueries) {
|
|
55356
56704
|
tx.insert(queries).values({
|
|
55357
|
-
id:
|
|
56705
|
+
id: crypto42.randomUUID(),
|
|
55358
56706
|
projectId: project.id,
|
|
55359
56707
|
query,
|
|
55360
56708
|
provenance,
|
|
@@ -55363,7 +56711,7 @@ async function discoveryRoutes(app, opts) {
|
|
|
55363
56711
|
}
|
|
55364
56712
|
for (const domain of promotedCompetitors) {
|
|
55365
56713
|
tx.insert(competitors).values({
|
|
55366
|
-
id:
|
|
56714
|
+
id: crypto42.randomUUID(),
|
|
55367
56715
|
projectId: project.id,
|
|
55368
56716
|
domain,
|
|
55369
56717
|
provenance,
|
|
@@ -55453,8 +56801,8 @@ function selectEligibleCompetitors(competitorMap, competitorTypes) {
|
|
|
55453
56801
|
}
|
|
55454
56802
|
|
|
55455
56803
|
// ../api-routes/src/discovery/orchestrate.ts
|
|
55456
|
-
import
|
|
55457
|
-
import { eq as
|
|
56804
|
+
import crypto43 from "crypto";
|
|
56805
|
+
import { eq as eq55 } from "drizzle-orm";
|
|
55458
56806
|
var DEFAULT_MAX_PROBES = 100;
|
|
55459
56807
|
var ABSOLUTE_MAX_PROBES = 500;
|
|
55460
56808
|
function classifyProbeBucket(input) {
|
|
@@ -55541,7 +56889,7 @@ async function executeDiscovery(opts) {
|
|
|
55541
56889
|
status: DiscoverySessionStatuses.seeding,
|
|
55542
56890
|
dedupThreshold,
|
|
55543
56891
|
startedAt
|
|
55544
|
-
}).where(
|
|
56892
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55545
56893
|
const seedResult = await opts.deps.seed({
|
|
55546
56894
|
project: opts.project,
|
|
55547
56895
|
icpDescription: opts.icpDescription,
|
|
@@ -55602,7 +56950,7 @@ async function executeDiscovery(opts) {
|
|
|
55602
56950
|
dedupBandPairFraction: dedupStats.bandPairFraction,
|
|
55603
56951
|
dedupPairsTotal: dedupStats.pairsTotal,
|
|
55604
56952
|
warning
|
|
55605
|
-
}).where(
|
|
56953
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55606
56954
|
const probeLocation = opts.locations?.[0];
|
|
55607
56955
|
const probeResults = await mapWithConcurrency(
|
|
55608
56956
|
probedCanonicals,
|
|
@@ -55621,7 +56969,7 @@ async function executeDiscovery(opts) {
|
|
|
55621
56969
|
probeRows.push({ citedDomains: probe.citedDomains, bucket });
|
|
55622
56970
|
buckets[bucket]++;
|
|
55623
56971
|
return {
|
|
55624
|
-
id:
|
|
56972
|
+
id: crypto43.randomUUID(),
|
|
55625
56973
|
sessionId: opts.sessionId,
|
|
55626
56974
|
projectId: opts.project.id,
|
|
55627
56975
|
query: probedCanonicals[index2],
|
|
@@ -55656,7 +57004,7 @@ async function executeDiscovery(opts) {
|
|
|
55656
57004
|
wastedCount: buckets["wasted-surface"],
|
|
55657
57005
|
competitorMap,
|
|
55658
57006
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55659
|
-
}).where(
|
|
57007
|
+
}).where(eq55(discoverySessions.id, opts.sessionId)).run();
|
|
55660
57008
|
upsertDomainClassifications(opts.db, opts.project.id, opts.sessionId, competitorMap);
|
|
55661
57009
|
return {
|
|
55662
57010
|
buckets,
|
|
@@ -55673,7 +57021,7 @@ function upsertDomainClassifications(db, projectId, sessionId, competitorMap) {
|
|
|
55673
57021
|
const domain = hostOf(entry.domain) ?? "";
|
|
55674
57022
|
if (!domain) continue;
|
|
55675
57023
|
db.insert(domainClassifications).values({
|
|
55676
|
-
id:
|
|
57024
|
+
id: crypto43.randomUUID(),
|
|
55677
57025
|
projectId,
|
|
55678
57026
|
domain,
|
|
55679
57027
|
competitorType: entry.competitorType,
|
|
@@ -55696,7 +57044,7 @@ function markSessionFailed(db, sessionId, error) {
|
|
|
55696
57044
|
status: DiscoverySessionStatuses.failed,
|
|
55697
57045
|
error,
|
|
55698
57046
|
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55699
|
-
}).where(
|
|
57047
|
+
}).where(eq55(discoverySessions.id, sessionId)).run();
|
|
55700
57048
|
}
|
|
55701
57049
|
function dedupeStrings(input) {
|
|
55702
57050
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -55713,8 +57061,8 @@ function dedupeStrings(input) {
|
|
|
55713
57061
|
}
|
|
55714
57062
|
|
|
55715
57063
|
// ../api-routes/src/technical-aeo.ts
|
|
55716
|
-
import
|
|
55717
|
-
import { and as
|
|
57064
|
+
import crypto44 from "crypto";
|
|
57065
|
+
import { and as and44, asc as asc10, count, desc as desc24, eq as eq56, inArray as inArray19, isNotNull as isNotNull3, lt as lt8, or as or11, sql as sql21 } from "drizzle-orm";
|
|
55718
57066
|
import { alias } from "drizzle-orm/sqlite-core";
|
|
55719
57067
|
var FETCHED_SITE_CRAWL_STATES = /* @__PURE__ */ new Set([
|
|
55720
57068
|
...SiteCrawlFetchedStates,
|
|
@@ -55755,6 +57103,14 @@ function parseBoundedLimit(value, fallback, max) {
|
|
|
55755
57103
|
const parsed = parsePositiveInt(value, fallback, max);
|
|
55756
57104
|
return Math.max(1, parsed);
|
|
55757
57105
|
}
|
|
57106
|
+
function parseSiteHealthState(value) {
|
|
57107
|
+
if (value === void 0 || value === "") return null;
|
|
57108
|
+
const allowed = siteHealthStateSchema.safeParse(value);
|
|
57109
|
+
if (!allowed.success) {
|
|
57110
|
+
throw validationError(`"healthState" must be one of: ${siteHealthStateSchema.options.join(", ")}`);
|
|
57111
|
+
}
|
|
57112
|
+
return allowed.data;
|
|
57113
|
+
}
|
|
55758
57114
|
function parseBoolean2(value) {
|
|
55759
57115
|
if (value === true || value === "true" || value === "1") return true;
|
|
55760
57116
|
if (value === false || value === "false" || value === "0") return false;
|
|
@@ -55861,7 +57217,7 @@ function mapCrawlEdge(row) {
|
|
|
55861
57217
|
}
|
|
55862
57218
|
var graphSourceNode = alias(siteCrawlGraphNodes, "site_crawl_graph_source_node");
|
|
55863
57219
|
var graphTargetNode = alias(siteCrawlGraphNodes, "site_crawl_graph_target_node");
|
|
55864
|
-
var PAGE_CHANGE_PREDICATE =
|
|
57220
|
+
var PAGE_CHANGE_PREDICATE = sql21`(
|
|
55865
57221
|
current.url IS NOT previous.url
|
|
55866
57222
|
OR current.final_url IS NOT previous.final_url
|
|
55867
57223
|
OR current.path IS NOT previous.path
|
|
@@ -55883,7 +57239,7 @@ var PAGE_CHANGE_PREDICATE = sql20`(
|
|
|
55883
57239
|
OR current.link_score_raw IS NOT previous.link_score_raw
|
|
55884
57240
|
OR current.link_score_normalized IS NOT previous.link_score_normalized
|
|
55885
57241
|
)`;
|
|
55886
|
-
var LINK_CHANGE_PREDICATE =
|
|
57242
|
+
var LINK_CHANGE_PREDICATE = sql21`(
|
|
55887
57243
|
current.source_node_key IS NOT previous.source_node_key
|
|
55888
57244
|
OR current.source_url IS NOT previous.source_url
|
|
55889
57245
|
OR current.target_node_key IS NOT previous.target_node_key
|
|
@@ -55961,39 +57317,49 @@ function changedFields(before, after, fields) {
|
|
|
55961
57317
|
async function technicalAeoRoutes(app, opts) {
|
|
55962
57318
|
const resolveCrawl = (projectId, runId) => {
|
|
55963
57319
|
const filters = [
|
|
55964
|
-
|
|
55965
|
-
|
|
55966
|
-
|
|
57320
|
+
eq56(siteCrawlSnapshots.projectId, projectId),
|
|
57321
|
+
eq56(runs.projectId, projectId),
|
|
57322
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55967
57323
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55968
57324
|
notProbeRun()
|
|
55969
57325
|
];
|
|
55970
57326
|
if (runId) {
|
|
55971
|
-
filters.push(
|
|
57327
|
+
filters.push(eq56(siteCrawlSnapshots.runId, runId));
|
|
55972
57328
|
} else {
|
|
55973
|
-
filters.push(
|
|
57329
|
+
filters.push(eq56(siteCrawlSnapshots.complete, true), eq56(runs.status, RunStatuses.completed));
|
|
55974
57330
|
}
|
|
55975
|
-
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
57331
|
+
return app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(...filters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
55976
57332
|
};
|
|
55977
57333
|
const detailScopeFor = (projectId, snapshot) => snapshot.detailsAvailable && snapshot.attemptId ? { projectId, runId: snapshot.runId, attemptId: snapshot.attemptId } : null;
|
|
55978
57334
|
const pageInScope = (scope, selector) => {
|
|
55979
57335
|
if (!selector.nodeKey && !selector.url) return null;
|
|
55980
|
-
return app.db.select().from(siteCrawlPages).where(
|
|
55981
|
-
|
|
55982
|
-
|
|
55983
|
-
|
|
55984
|
-
selector.nodeKey ?
|
|
57336
|
+
return app.db.select().from(siteCrawlPages).where(and44(
|
|
57337
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57338
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57339
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57340
|
+
selector.nodeKey ? eq56(siteCrawlPages.nodeKey, selector.nodeKey) : eq56(siteCrawlPages.url, selector.url)
|
|
55985
57341
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55986
57342
|
};
|
|
55987
|
-
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(
|
|
55988
|
-
|
|
55989
|
-
|
|
55990
|
-
|
|
55991
|
-
|
|
57343
|
+
const rootPageInScope = (scope, rootUrl) => pageInScope(scope, { url: rootUrl }) ?? app.db.select().from(siteCrawlPages).where(and44(
|
|
57344
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57345
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57346
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
57347
|
+
eq56(siteCrawlPages.depth, 0)
|
|
55992
57348
|
)).orderBy(asc10(siteCrawlPages.nodeKey)).limit(1).get() ?? null;
|
|
55993
|
-
const
|
|
55994
|
-
|
|
55995
|
-
|
|
55996
|
-
|
|
57349
|
+
const isSurfaceableAuditRun = (projectId, runId) => Boolean(app.db.select({ id: runs.id }).from(runs).where(and44(
|
|
57350
|
+
eq56(runs.id, runId),
|
|
57351
|
+
eq56(runs.projectId, projectId),
|
|
57352
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57353
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
57354
|
+
notProbeRun()
|
|
57355
|
+
)).limit(1).get());
|
|
57356
|
+
const assertKnownAuditRun = (projectId, runId) => {
|
|
57357
|
+
if (runId && !isSurfaceableAuditRun(projectId, runId)) throw notFound("Site crawl run", runId);
|
|
57358
|
+
};
|
|
57359
|
+
const hasLegacyAudit = (projectId) => Boolean(app.db.select({ runId: siteAuditSnapshots.runId }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57360
|
+
eq56(siteAuditSnapshots.projectId, projectId),
|
|
57361
|
+
eq56(runs.projectId, projectId),
|
|
57362
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
55997
57363
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
55998
57364
|
notProbeRun()
|
|
55999
57365
|
)).limit(1).get());
|
|
@@ -56015,20 +57381,20 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56015
57381
|
app.get("/projects/:name/technical-aeo", async (request) => {
|
|
56016
57382
|
const project = resolveProject(app.db, request.params.name);
|
|
56017
57383
|
const baseFilters = [
|
|
56018
|
-
|
|
56019
|
-
|
|
56020
|
-
|
|
57384
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57385
|
+
eq56(runs.projectId, project.id),
|
|
57386
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
56021
57387
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
56022
57388
|
notProbeRun()
|
|
56023
57389
|
];
|
|
56024
|
-
const targetFilters = request.query.runId ? [...baseFilters,
|
|
56025
|
-
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57390
|
+
const targetFilters = request.query.runId ? [...baseFilters, eq56(siteAuditSnapshots.runId, request.query.runId)] : baseFilters;
|
|
57391
|
+
const latest = app.db.select({ snap: siteAuditSnapshots, runStatus: runs.status }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
56026
57392
|
if (!latest) {
|
|
56027
57393
|
if (request.query.runId) throw notFound("Site audit run", request.query.runId);
|
|
56028
57394
|
return emptyScore(project.name);
|
|
56029
57395
|
}
|
|
56030
57396
|
const snap = latest.snap;
|
|
56031
|
-
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57397
|
+
const previous = app.db.select({ snap: siteAuditSnapshots }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...baseFilters, lt8(siteAuditSnapshots.createdAt, snap.createdAt))).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get()?.snap ?? null;
|
|
56032
57398
|
const deltaScore = previous ? snap.aggregateScore - previous.aggregateScore : null;
|
|
56033
57399
|
const trend = deltaScore == null ? null : deltaScore > 0 ? SiteAuditTrendDirections.up : deltaScore < 0 ? SiteAuditTrendDirections.down : SiteAuditTrendDirections.flat;
|
|
56034
57400
|
return {
|
|
@@ -56055,14 +57421,14 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56055
57421
|
app.get("/projects/:name/technical-aeo/pages", async (request) => {
|
|
56056
57422
|
const project = resolveProject(app.db, request.params.name);
|
|
56057
57423
|
const targetFilters = [
|
|
56058
|
-
|
|
56059
|
-
|
|
56060
|
-
|
|
57424
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57425
|
+
eq56(runs.projectId, project.id),
|
|
57426
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
56061
57427
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
56062
57428
|
notProbeRun()
|
|
56063
57429
|
];
|
|
56064
|
-
if (request.query.runId) targetFilters.push(
|
|
56065
|
-
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs,
|
|
57430
|
+
if (request.query.runId) targetFilters.push(eq56(siteAuditSnapshots.runId, request.query.runId));
|
|
57431
|
+
const latest = app.db.select({ runId: siteAuditSnapshots.runId, auditedAt: siteAuditSnapshots.auditedAt }).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(...targetFilters)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(1).get();
|
|
56066
57432
|
if (!latest && request.query.runId) {
|
|
56067
57433
|
throw notFound("Site audit run", request.query.runId);
|
|
56068
57434
|
}
|
|
@@ -56070,9 +57436,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56070
57436
|
return { project: project.name, runId: null, auditedAt: null, total: 0, pages: [] };
|
|
56071
57437
|
}
|
|
56072
57438
|
const statusFilter = request.query.status === "success" || request.query.status === "error" ? request.query.status : null;
|
|
56073
|
-
const conds = [
|
|
56074
|
-
if (statusFilter) conds.push(
|
|
56075
|
-
const where =
|
|
57439
|
+
const conds = [eq56(siteAuditPages.projectId, project.id), eq56(siteAuditPages.runId, latest.runId)];
|
|
57440
|
+
if (statusFilter) conds.push(eq56(siteAuditPages.status, statusFilter));
|
|
57441
|
+
const where = and44(...conds);
|
|
56076
57442
|
const totalRow = app.db.select({ value: count() }).from(siteAuditPages).where(where).get();
|
|
56077
57443
|
const total = totalRow?.value ?? 0;
|
|
56078
57444
|
const limit = parsePositiveInt(request.query.limit, 100, 500);
|
|
@@ -56096,10 +57462,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56096
57462
|
auditedAt: siteAuditSnapshots.auditedAt,
|
|
56097
57463
|
aggregateScore: siteAuditSnapshots.aggregateScore,
|
|
56098
57464
|
pagesAudited: siteAuditSnapshots.pagesAudited
|
|
56099
|
-
}).from(siteAuditSnapshots).innerJoin(runs,
|
|
56100
|
-
|
|
56101
|
-
|
|
56102
|
-
|
|
57465
|
+
}).from(siteAuditSnapshots).innerJoin(runs, eq56(siteAuditSnapshots.runId, runs.id)).where(and44(
|
|
57466
|
+
eq56(siteAuditSnapshots.projectId, project.id),
|
|
57467
|
+
eq56(runs.projectId, project.id),
|
|
57468
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
56103
57469
|
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
56104
57470
|
notProbeRun()
|
|
56105
57471
|
)).orderBy(desc24(siteAuditSnapshots.createdAt)).limit(limit).all();
|
|
@@ -56110,7 +57476,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56110
57476
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
56111
57477
|
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
56112
57478
|
if (!target) {
|
|
56113
|
-
|
|
57479
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
56114
57480
|
return emptyCrawlSummary(project.name, legacyAuditAvailable);
|
|
56115
57481
|
}
|
|
56116
57482
|
const snapshot = target.snapshot;
|
|
@@ -56146,11 +57512,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56146
57512
|
const project = resolveProject(app.db, request.params.name);
|
|
56147
57513
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
56148
57514
|
if (!target) {
|
|
56149
|
-
|
|
57515
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
56150
57516
|
return {
|
|
56151
57517
|
project: project.name,
|
|
56152
57518
|
hasCrawlData: false,
|
|
56153
57519
|
runId: null,
|
|
57520
|
+
rootNodeKey: null,
|
|
56154
57521
|
layout: { state: "unavailable", version: null, reason: "no-crawl" },
|
|
56155
57522
|
totalNodes: 0,
|
|
56156
57523
|
totalEdges: 0,
|
|
@@ -56167,6 +57534,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56167
57534
|
project: project.name,
|
|
56168
57535
|
hasCrawlData: true,
|
|
56169
57536
|
runId: snapshot.runId,
|
|
57537
|
+
rootNodeKey: null,
|
|
56170
57538
|
layout: { state: "unavailable", version: null, reason: "details-unavailable" },
|
|
56171
57539
|
totalNodes: 0,
|
|
56172
57540
|
totalEdges: 0,
|
|
@@ -56177,16 +57545,21 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56177
57545
|
sampled: false
|
|
56178
57546
|
};
|
|
56179
57547
|
}
|
|
56180
|
-
const
|
|
56181
|
-
|
|
56182
|
-
|
|
56183
|
-
|
|
57548
|
+
const rootNodeKey = rootPageInScope(
|
|
57549
|
+
{ projectId: project.id, runId: snapshot.runId, attemptId: snapshot.attemptId },
|
|
57550
|
+
snapshot.rootUrl
|
|
57551
|
+
)?.nodeKey ?? null;
|
|
57552
|
+
const persistedLayout = app.db.select().from(siteCrawlGraphLayouts).where(and44(
|
|
57553
|
+
eq56(siteCrawlGraphLayouts.projectId, project.id),
|
|
57554
|
+
eq56(siteCrawlGraphLayouts.runId, snapshot.runId),
|
|
57555
|
+
eq56(siteCrawlGraphLayouts.attemptId, snapshot.attemptId)
|
|
56184
57556
|
)).limit(1).get();
|
|
56185
57557
|
if (!persistedLayout) {
|
|
56186
57558
|
return {
|
|
56187
57559
|
project: project.name,
|
|
56188
57560
|
hasCrawlData: true,
|
|
56189
57561
|
runId: snapshot.runId,
|
|
57562
|
+
rootNodeKey,
|
|
56190
57563
|
layout: { state: "unavailable", version: null, reason: "legacy-snapshot" },
|
|
56191
57564
|
totalNodes: 0,
|
|
56192
57565
|
totalEdges: 0,
|
|
@@ -56204,6 +57577,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56204
57577
|
project: project.name,
|
|
56205
57578
|
hasCrawlData: true,
|
|
56206
57579
|
runId: snapshot.runId,
|
|
57580
|
+
rootNodeKey,
|
|
56207
57581
|
layout: {
|
|
56208
57582
|
state: "unavailable",
|
|
56209
57583
|
version: null,
|
|
@@ -56221,9 +57595,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56221
57595
|
const maxNodes = parseBoundedLimit(request.query.maxNodes, SITE_CRAWL_GRAPH_DEFAULT_MAX_NODES, SITE_CRAWL_GRAPH_MAX_NODES);
|
|
56222
57596
|
const maxEdges = parseBoundedLimit(request.query.maxEdges, SITE_CRAWL_GRAPH_DEFAULT_MAX_EDGES, SITE_CRAWL_GRAPH_MAX_EDGES);
|
|
56223
57597
|
const graphScope = [
|
|
56224
|
-
|
|
56225
|
-
|
|
56226
|
-
|
|
57598
|
+
eq56(siteCrawlGraphNodes.projectId, project.id),
|
|
57599
|
+
eq56(siteCrawlGraphNodes.runId, snapshot.runId),
|
|
57600
|
+
eq56(siteCrawlGraphNodes.attemptId, snapshot.attemptId)
|
|
56227
57601
|
];
|
|
56228
57602
|
const nodeRows = app.db.select({
|
|
56229
57603
|
nodeKey: siteCrawlPages.nodeKey,
|
|
@@ -56242,12 +57616,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56242
57616
|
linkScoreNormalized: siteCrawlPages.linkScoreNormalized,
|
|
56243
57617
|
x: siteCrawlGraphNodes.x,
|
|
56244
57618
|
y: siteCrawlGraphNodes.y
|
|
56245
|
-
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages,
|
|
56246
|
-
|
|
56247
|
-
|
|
56248
|
-
|
|
56249
|
-
|
|
56250
|
-
)).where(
|
|
57619
|
+
}).from(siteCrawlGraphNodes).innerJoin(siteCrawlPages, and44(
|
|
57620
|
+
eq56(siteCrawlPages.projectId, siteCrawlGraphNodes.projectId),
|
|
57621
|
+
eq56(siteCrawlPages.runId, siteCrawlGraphNodes.runId),
|
|
57622
|
+
eq56(siteCrawlPages.attemptId, siteCrawlGraphNodes.attemptId),
|
|
57623
|
+
eq56(siteCrawlPages.nodeKey, siteCrawlGraphNodes.nodeKey)
|
|
57624
|
+
)).where(and44(...graphScope, lt8(siteCrawlGraphNodes.sampleRank, maxNodes))).orderBy(asc10(siteCrawlGraphNodes.sampleRank)).all();
|
|
56251
57625
|
const nodes = nodeRows.map(({ indexabilityReasons, canonicalNodeKey, ...row }) => ({
|
|
56252
57626
|
...row,
|
|
56253
57627
|
healthState: deriveSiteHealthState({ ...row, indexabilityReasons, canonicalNodeKey })
|
|
@@ -56258,23 +57632,23 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56258
57632
|
targetNodeKey: siteCrawlGraphEdges.targetNodeKey,
|
|
56259
57633
|
followable: siteCrawlGraphEdges.followable,
|
|
56260
57634
|
occurrences: siteCrawlGraphEdges.occurrences
|
|
56261
|
-
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode,
|
|
56262
|
-
|
|
56263
|
-
|
|
56264
|
-
|
|
56265
|
-
|
|
56266
|
-
|
|
56267
|
-
)).innerJoin(graphTargetNode,
|
|
56268
|
-
|
|
56269
|
-
|
|
56270
|
-
|
|
56271
|
-
|
|
56272
|
-
|
|
56273
|
-
)).where(
|
|
56274
|
-
|
|
56275
|
-
|
|
56276
|
-
|
|
56277
|
-
|
|
57635
|
+
}).from(siteCrawlGraphEdges).innerJoin(graphSourceNode, and44(
|
|
57636
|
+
eq56(graphSourceNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57637
|
+
eq56(graphSourceNode.runId, siteCrawlGraphEdges.runId),
|
|
57638
|
+
eq56(graphSourceNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
57639
|
+
eq56(graphSourceNode.nodeKey, siteCrawlGraphEdges.sourceNodeKey),
|
|
57640
|
+
lt8(graphSourceNode.sampleRank, maxNodes)
|
|
57641
|
+
)).innerJoin(graphTargetNode, and44(
|
|
57642
|
+
eq56(graphTargetNode.projectId, siteCrawlGraphEdges.projectId),
|
|
57643
|
+
eq56(graphTargetNode.runId, siteCrawlGraphEdges.runId),
|
|
57644
|
+
eq56(graphTargetNode.attemptId, siteCrawlGraphEdges.attemptId),
|
|
57645
|
+
eq56(graphTargetNode.nodeKey, siteCrawlGraphEdges.targetNodeKey),
|
|
57646
|
+
lt8(graphTargetNode.sampleRank, maxNodes)
|
|
57647
|
+
)).where(and44(
|
|
57648
|
+
eq56(siteCrawlGraphEdges.projectId, project.id),
|
|
57649
|
+
eq56(siteCrawlGraphEdges.runId, snapshot.runId),
|
|
57650
|
+
eq56(siteCrawlGraphEdges.attemptId, snapshot.attemptId),
|
|
57651
|
+
lt8(siteCrawlGraphEdges.sampleRank, maxEdges)
|
|
56278
57652
|
)).orderBy(asc10(siteCrawlGraphEdges.sampleRank)).all();
|
|
56279
57653
|
const totalNodes = persistedLayout.totalNodes;
|
|
56280
57654
|
const totalEdges = persistedLayout.totalEdges;
|
|
@@ -56284,6 +57658,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56284
57658
|
project: project.name,
|
|
56285
57659
|
hasCrawlData: true,
|
|
56286
57660
|
runId: snapshot.runId,
|
|
57661
|
+
rootNodeKey,
|
|
56287
57662
|
layout: { state: "ready", version: persistedLayout.layoutVersion, computedAt: persistedLayout.createdAt },
|
|
56288
57663
|
totalNodes,
|
|
56289
57664
|
totalEdges,
|
|
@@ -56328,7 +57703,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56328
57703
|
truncated: false
|
|
56329
57704
|
});
|
|
56330
57705
|
if (!target) {
|
|
56331
|
-
|
|
57706
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
56332
57707
|
return empty("no-crawl", null, false, false, null);
|
|
56333
57708
|
}
|
|
56334
57709
|
const snapshot = target.snapshot;
|
|
@@ -56352,11 +57727,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56352
57727
|
let frontier = [focus.nodeKey];
|
|
56353
57728
|
let queryTruncated = false;
|
|
56354
57729
|
for (let distance = 0; distance < hops && frontier.length > 0; distance += 1) {
|
|
56355
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
56356
|
-
|
|
56357
|
-
|
|
56358
|
-
|
|
56359
|
-
|
|
57730
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and44(
|
|
57731
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57732
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57733
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57734
|
+
eq56(siteCrawlEdges.internal, true),
|
|
56360
57735
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56361
57736
|
or11(
|
|
56362
57737
|
inArray19(siteCrawlEdges.sourceNodeKey, frontier),
|
|
@@ -56393,19 +57768,19 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56393
57768
|
frontier = [...next].sort();
|
|
56394
57769
|
}
|
|
56395
57770
|
const nodeKeys = [...distances.keys()];
|
|
56396
|
-
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
56397
|
-
|
|
56398
|
-
|
|
56399
|
-
|
|
57771
|
+
const pageRows = nodeKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and44(
|
|
57772
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57773
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57774
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56400
57775
|
inArray19(siteCrawlPages.nodeKey, nodeKeys)
|
|
56401
57776
|
)).all();
|
|
56402
57777
|
const persistedNodeKeys = new Set(pageRows.map((row) => row.nodeKey));
|
|
56403
57778
|
const outermostNodeKeys = pageRows.filter((row) => distances.get(row.nodeKey) === hops).map((row) => row.nodeKey);
|
|
56404
|
-
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(
|
|
56405
|
-
|
|
56406
|
-
|
|
56407
|
-
|
|
56408
|
-
|
|
57779
|
+
const omittedOutermostEdge = outermostNodeKeys.length === 0 ? void 0 : app.db.select({ edgeKey: siteCrawlEdges.edgeKey }).from(siteCrawlEdges).where(and44(
|
|
57780
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57781
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57782
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57783
|
+
eq56(siteCrawlEdges.internal, true),
|
|
56409
57784
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56410
57785
|
inArray19(siteCrawlEdges.sourceNodeKey, outermostNodeKeys),
|
|
56411
57786
|
inArray19(siteCrawlEdges.targetNodeKey, outermostNodeKeys)
|
|
@@ -56454,7 +57829,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56454
57829
|
}
|
|
56455
57830
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
56456
57831
|
if (!target) {
|
|
56457
|
-
|
|
57832
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
56458
57833
|
return {
|
|
56459
57834
|
project: project.name,
|
|
56460
57835
|
runId: null,
|
|
@@ -56502,22 +57877,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56502
57877
|
let found = fromPage.nodeKey === toPage.nodeKey;
|
|
56503
57878
|
let truncated = false;
|
|
56504
57879
|
for (let depth = 0; depth < maxDepth && frontier.length > 0 && !found; depth += 1) {
|
|
56505
|
-
const candidates = app.db.select().from(siteCrawlEdges).where(
|
|
56506
|
-
|
|
56507
|
-
|
|
56508
|
-
|
|
56509
|
-
|
|
56510
|
-
|
|
56511
|
-
|
|
57880
|
+
const candidates = app.db.select().from(siteCrawlEdges).where(and44(
|
|
57881
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
57882
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
57883
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
57884
|
+
eq56(siteCrawlEdges.internal, true),
|
|
57885
|
+
eq56(siteCrawlEdges.followable, true),
|
|
57886
|
+
eq56(siteCrawlEdges.relation, "anchor"),
|
|
56512
57887
|
isNotNull3(siteCrawlEdges.targetNodeKey),
|
|
56513
57888
|
inArray19(siteCrawlEdges.sourceNodeKey, frontier)
|
|
56514
57889
|
)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(SITE_HEALTH_PATH_MAX_VISITED_NODES + 1).all();
|
|
56515
57890
|
if (candidates.length > SITE_HEALTH_PATH_MAX_VISITED_NODES) truncated = true;
|
|
56516
57891
|
const candidateTargetKeys = [...new Set(candidates.map((edge) => edge.targetNodeKey).filter((nodeKey) => nodeKey != null))];
|
|
56517
|
-
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(
|
|
56518
|
-
|
|
56519
|
-
|
|
56520
|
-
|
|
57892
|
+
const persistedTargetKeys = new Set(candidateTargetKeys.length === 0 ? [] : app.db.select({ nodeKey: siteCrawlPages.nodeKey }).from(siteCrawlPages).where(and44(
|
|
57893
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57894
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57895
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56521
57896
|
inArray19(siteCrawlPages.nodeKey, candidateTargetKeys)
|
|
56522
57897
|
)).all().map((row) => row.nodeKey));
|
|
56523
57898
|
const next = /* @__PURE__ */ new Set();
|
|
@@ -56560,10 +57935,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56560
57935
|
pathEdges.unshift(step.edge);
|
|
56561
57936
|
pathKeys.unshift(step.nodeKey);
|
|
56562
57937
|
}
|
|
56563
|
-
const pathRows = app.db.select().from(siteCrawlPages).where(
|
|
56564
|
-
|
|
56565
|
-
|
|
56566
|
-
|
|
57938
|
+
const pathRows = app.db.select().from(siteCrawlPages).where(and44(
|
|
57939
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
57940
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
57941
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56567
57942
|
inArray19(siteCrawlPages.nodeKey, pathKeys)
|
|
56568
57943
|
)).all();
|
|
56569
57944
|
const pageByKey = new Map(pathRows.map((row) => [row.nodeKey, row]));
|
|
@@ -56592,22 +57967,22 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56592
57967
|
throw validationError("change must be all, added, removed, or changed");
|
|
56593
57968
|
}
|
|
56594
57969
|
const completeSnapshotFilters = [
|
|
56595
|
-
|
|
56596
|
-
|
|
56597
|
-
|
|
56598
|
-
|
|
56599
|
-
|
|
57970
|
+
eq56(siteCrawlSnapshots.projectId, project.id),
|
|
57971
|
+
eq56(runs.projectId, project.id),
|
|
57972
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57973
|
+
eq56(runs.status, RunStatuses.completed),
|
|
57974
|
+
eq56(siteCrawlSnapshots.complete, true),
|
|
56600
57975
|
notProbeRun()
|
|
56601
57976
|
];
|
|
56602
|
-
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
57977
|
+
const afterTarget = request.query.toRunId ? resolveCrawl(project.id, request.query.toRunId) : app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(...completeSnapshotFilters)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get();
|
|
56603
57978
|
if (request.query.toRunId && !afterTarget) throw notFound("Site crawl run", request.query.toRunId);
|
|
56604
|
-
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs,
|
|
57979
|
+
const beforeTarget = request.query.fromRunId ? resolveCrawl(project.id, request.query.fromRunId) : afterTarget ? app.db.select({ snapshot: siteCrawlSnapshots, runStatus: runs.status }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(
|
|
56605
57980
|
...completeSnapshotFilters,
|
|
56606
57981
|
or11(
|
|
56607
|
-
|
|
56608
|
-
|
|
56609
|
-
|
|
56610
|
-
|
|
57982
|
+
lt8(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
57983
|
+
and44(
|
|
57984
|
+
eq56(siteCrawlSnapshots.createdAt, afterTarget.snapshot.createdAt),
|
|
57985
|
+
lt8(siteCrawlSnapshots.runId, afterTarget.snapshot.runId)
|
|
56611
57986
|
)
|
|
56612
57987
|
)
|
|
56613
57988
|
)).orderBy(desc24(siteCrawlSnapshots.createdAt), desc24(siteCrawlSnapshots.runId)).limit(1).get() : void 0;
|
|
@@ -56684,9 +58059,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56684
58059
|
const summary2 = { pages: zero2(), links: zero2() };
|
|
56685
58060
|
if (scopeFilter !== "links") {
|
|
56686
58061
|
if (includeAdded || includeChanged) {
|
|
56687
|
-
const addedExpression = includeAdded ?
|
|
56688
|
-
const changedExpression = includeChanged ?
|
|
56689
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
58062
|
+
const addedExpression = includeAdded ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58063
|
+
const changedExpression = includeChanged ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${PAGE_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58064
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql21`
|
|
56690
58065
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56691
58066
|
FROM site_crawl_pages AS current
|
|
56692
58067
|
LEFT JOIN site_crawl_pages AS previous
|
|
@@ -56702,7 +58077,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56702
58077
|
summary2.pages.changed = Number(counts2.changed);
|
|
56703
58078
|
}
|
|
56704
58079
|
if (includeRemoved) {
|
|
56705
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
58080
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql21`
|
|
56706
58081
|
SELECT COUNT(*) AS removed
|
|
56707
58082
|
FROM site_crawl_pages AS previous
|
|
56708
58083
|
LEFT JOIN site_crawl_pages AS current
|
|
@@ -56720,9 +58095,9 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56720
58095
|
}
|
|
56721
58096
|
if (scopeFilter !== "pages") {
|
|
56722
58097
|
if (includeAdded || includeChanged) {
|
|
56723
|
-
const addedExpression = includeAdded ?
|
|
56724
|
-
const changedExpression = includeChanged ?
|
|
56725
|
-
const [counts2 = { added: 0, changed: 0 }] = app.db.all(
|
|
58098
|
+
const addedExpression = includeAdded ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NULL THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58099
|
+
const changedExpression = includeChanged ? sql21`COALESCE(SUM(CASE WHEN previous.id IS NOT NULL AND ${LINK_CHANGE_PREDICATE} THEN 1 ELSE 0 END), 0)` : sql21`0`;
|
|
58100
|
+
const [counts2 = { added: 0, changed: 0 }] = app.db.all(sql21`
|
|
56726
58101
|
SELECT ${addedExpression} AS added, ${changedExpression} AS changed
|
|
56727
58102
|
FROM site_crawl_edges AS current
|
|
56728
58103
|
LEFT JOIN site_crawl_edges AS previous
|
|
@@ -56740,7 +58115,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56740
58115
|
summary2.links.changed = Number(counts2.changed);
|
|
56741
58116
|
}
|
|
56742
58117
|
if (includeRemoved) {
|
|
56743
|
-
const [counts2 = { removed: 0 }] = app.db.all(
|
|
58118
|
+
const [counts2 = { removed: 0 }] = app.db.all(sql21`
|
|
56744
58119
|
SELECT COUNT(*) AS removed
|
|
56745
58120
|
FROM site_crawl_edges AS previous
|
|
56746
58121
|
LEFT JOIN site_crawl_edges AS current
|
|
@@ -56770,48 +58145,48 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56770
58145
|
const includeRemoved = changeFilter === "all" || changeFilter === "removed";
|
|
56771
58146
|
const table = entity === "page" ? "site_crawl_pages" : "site_crawl_edges";
|
|
56772
58147
|
const keyColumn = entity === "page" ? "node_key" : "edge_key";
|
|
56773
|
-
const internalCurrent = entity === "link" ?
|
|
56774
|
-
const internalPreviousJoin = entity === "link" ?
|
|
56775
|
-
const internalPrevious = entity === "link" ?
|
|
56776
|
-
const internalCurrentJoin = entity === "link" ?
|
|
58148
|
+
const internalCurrent = entity === "link" ? sql21`AND current.internal = 1` : sql21``;
|
|
58149
|
+
const internalPreviousJoin = entity === "link" ? sql21`AND previous.internal = 1` : sql21``;
|
|
58150
|
+
const internalPrevious = entity === "link" ? sql21`AND previous.internal = 1` : sql21``;
|
|
58151
|
+
const internalCurrentJoin = entity === "link" ? sql21`AND current.internal = 1` : sql21``;
|
|
56777
58152
|
const difference2 = entity === "page" ? PAGE_CHANGE_PREDICATE : LINK_CHANGE_PREDICATE;
|
|
56778
|
-
const afterKind = changeFilter === "added" ?
|
|
56779
|
-
const afterRows = includeAfter ? app.db.all(
|
|
58153
|
+
const afterKind = changeFilter === "added" ? sql21`previous.id IS NULL` : changeFilter === "changed" ? sql21`previous.id IS NOT NULL AND ${difference2}` : sql21`previous.id IS NULL OR (previous.id IS NOT NULL AND ${difference2})`;
|
|
58154
|
+
const afterRows = includeAfter ? app.db.all(sql21`
|
|
56780
58155
|
SELECT ${entity} AS entity,
|
|
56781
58156
|
CASE WHEN previous.id IS NULL THEN 'added' ELSE 'changed' END AS change,
|
|
56782
|
-
current.${
|
|
56783
|
-
FROM ${
|
|
56784
|
-
LEFT JOIN ${
|
|
58157
|
+
current.${sql21.raw(keyColumn)} AS key
|
|
58158
|
+
FROM ${sql21.raw(table)} AS current
|
|
58159
|
+
LEFT JOIN ${sql21.raw(table)} AS previous
|
|
56785
58160
|
ON previous.project_id = ${beforeScope.projectId}
|
|
56786
58161
|
AND previous.run_id = ${beforeScope.runId}
|
|
56787
58162
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56788
58163
|
${internalPreviousJoin}
|
|
56789
|
-
AND previous.${
|
|
58164
|
+
AND previous.${sql21.raw(keyColumn)} = current.${sql21.raw(keyColumn)}
|
|
56790
58165
|
WHERE current.project_id = ${afterScope.projectId}
|
|
56791
58166
|
AND current.run_id = ${afterScope.runId}
|
|
56792
58167
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
56793
58168
|
${internalCurrent}
|
|
56794
|
-
AND current.${
|
|
58169
|
+
AND current.${sql21.raw(keyColumn)} > ${afterKey}
|
|
56795
58170
|
AND (${afterKind})
|
|
56796
|
-
ORDER BY current.${
|
|
58171
|
+
ORDER BY current.${sql21.raw(keyColumn)} ASC
|
|
56797
58172
|
LIMIT ${wanted2}
|
|
56798
58173
|
`) : [];
|
|
56799
|
-
const removedRows = includeRemoved ? app.db.all(
|
|
56800
|
-
SELECT ${entity} AS entity, 'removed' AS change, previous.${
|
|
56801
|
-
FROM ${
|
|
56802
|
-
LEFT JOIN ${
|
|
58174
|
+
const removedRows = includeRemoved ? app.db.all(sql21`
|
|
58175
|
+
SELECT ${entity} AS entity, 'removed' AS change, previous.${sql21.raw(keyColumn)} AS key
|
|
58176
|
+
FROM ${sql21.raw(table)} AS previous
|
|
58177
|
+
LEFT JOIN ${sql21.raw(table)} AS current
|
|
56803
58178
|
ON current.project_id = ${afterScope.projectId}
|
|
56804
58179
|
AND current.run_id = ${afterScope.runId}
|
|
56805
58180
|
AND current.attempt_id = ${afterScope.attemptId}
|
|
56806
58181
|
${internalCurrentJoin}
|
|
56807
|
-
AND current.${
|
|
58182
|
+
AND current.${sql21.raw(keyColumn)} = previous.${sql21.raw(keyColumn)}
|
|
56808
58183
|
WHERE previous.project_id = ${beforeScope.projectId}
|
|
56809
58184
|
AND previous.run_id = ${beforeScope.runId}
|
|
56810
58185
|
AND previous.attempt_id = ${beforeScope.attemptId}
|
|
56811
58186
|
${internalPrevious}
|
|
56812
|
-
AND previous.${
|
|
58187
|
+
AND previous.${sql21.raw(keyColumn)} > ${afterKey}
|
|
56813
58188
|
AND current.id IS NULL
|
|
56814
|
-
ORDER BY previous.${
|
|
58189
|
+
ORDER BY previous.${sql21.raw(keyColumn)} ASC
|
|
56815
58190
|
LIMIT ${wanted2}
|
|
56816
58191
|
`) : [];
|
|
56817
58192
|
const merged = [];
|
|
@@ -56841,17 +58216,17 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56841
58216
|
const visibleKeys = keyRows.slice(0, limit);
|
|
56842
58217
|
const pageKeys = visibleKeys.filter((row) => row.entity === "page").map((row) => row.key);
|
|
56843
58218
|
const linkKeys = visibleKeys.filter((row) => row.entity === "link").map((row) => row.key);
|
|
56844
|
-
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(
|
|
56845
|
-
|
|
56846
|
-
|
|
56847
|
-
|
|
58219
|
+
const pageRowsFor = (scope) => pageKeys.length === 0 ? [] : app.db.select().from(siteCrawlPages).where(and44(
|
|
58220
|
+
eq56(siteCrawlPages.projectId, scope.projectId),
|
|
58221
|
+
eq56(siteCrawlPages.runId, scope.runId),
|
|
58222
|
+
eq56(siteCrawlPages.attemptId, scope.attemptId),
|
|
56848
58223
|
inArray19(siteCrawlPages.nodeKey, pageKeys)
|
|
56849
58224
|
)).all();
|
|
56850
|
-
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(
|
|
56851
|
-
|
|
56852
|
-
|
|
56853
|
-
|
|
56854
|
-
|
|
58225
|
+
const edgeRowsFor = (scope) => linkKeys.length === 0 ? [] : app.db.select().from(siteCrawlEdges).where(and44(
|
|
58226
|
+
eq56(siteCrawlEdges.projectId, scope.projectId),
|
|
58227
|
+
eq56(siteCrawlEdges.runId, scope.runId),
|
|
58228
|
+
eq56(siteCrawlEdges.attemptId, scope.attemptId),
|
|
58229
|
+
eq56(siteCrawlEdges.internal, true),
|
|
56855
58230
|
inArray19(siteCrawlEdges.edgeKey, linkKeys)
|
|
56856
58231
|
)).all();
|
|
56857
58232
|
const beforePages = new Map(pageRowsFor(beforeScope).map((row) => [row.nodeKey, mapCrawlPage(row)]));
|
|
@@ -56926,7 +58301,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56926
58301
|
}
|
|
56927
58302
|
const target = resolveCrawl(project.id, runId);
|
|
56928
58303
|
if (!target) {
|
|
56929
|
-
|
|
58304
|
+
assertKnownAuditRun(project.id, runId);
|
|
56930
58305
|
return { state: "no-crawl", project: project.name, runId: null };
|
|
56931
58306
|
}
|
|
56932
58307
|
const snapshot = target.snapshot;
|
|
@@ -56967,7 +58342,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56967
58342
|
const project = resolveProject(app.db, request.params.name);
|
|
56968
58343
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
56969
58344
|
if (!target) {
|
|
56970
|
-
|
|
58345
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
56971
58346
|
return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, pages: [] };
|
|
56972
58347
|
}
|
|
56973
58348
|
const snapshot = target.snapshot;
|
|
@@ -56975,21 +58350,34 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
56975
58350
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, pages: [] };
|
|
56976
58351
|
}
|
|
56977
58352
|
const filters = [
|
|
56978
|
-
|
|
56979
|
-
|
|
56980
|
-
|
|
58353
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58354
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58355
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
56981
58356
|
];
|
|
56982
58357
|
const inventoryEligible = parseBoolean2(request.query.inventoryEligible);
|
|
56983
|
-
if (inventoryEligible != null) filters.push(
|
|
56984
|
-
if (request.query.fetchState) filters.push(
|
|
56985
|
-
if (request.query.indexabilityState) filters.push(
|
|
56986
|
-
if (request.query.auditState) filters.push(
|
|
56987
|
-
const
|
|
56988
|
-
const
|
|
58358
|
+
if (inventoryEligible != null) filters.push(eq56(siteCrawlPages.inventoryEligible, inventoryEligible));
|
|
58359
|
+
if (request.query.fetchState) filters.push(eq56(siteCrawlPages.fetchState, request.query.fetchState));
|
|
58360
|
+
if (request.query.indexabilityState) filters.push(eq56(siteCrawlPages.indexabilityState, request.query.indexabilityState));
|
|
58361
|
+
if (request.query.auditState) filters.push(eq56(siteCrawlPages.auditState, request.query.auditState));
|
|
58362
|
+
const healthState = parseSiteHealthState(request.query.healthState);
|
|
58363
|
+
const where = and44(...filters);
|
|
56989
58364
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
56990
58365
|
const offset = decodeCursor(request.query.cursor);
|
|
56991
58366
|
const orderBy = request.query.sort === "score-desc" ? [desc24(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "score-asc" ? [asc10(siteCrawlPages.auditScore), asc10(siteCrawlPages.nodeKey)] : request.query.sort === "path" ? [asc10(siteCrawlPages.path), asc10(siteCrawlPages.nodeKey)] : [asc10(siteCrawlPages.url), asc10(siteCrawlPages.nodeKey)];
|
|
56992
|
-
|
|
58367
|
+
let total;
|
|
58368
|
+
let rows;
|
|
58369
|
+
if (healthState) {
|
|
58370
|
+
const candidates = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
58371
|
+
if (candidates.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
58372
|
+
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page health-state filter limit`);
|
|
58373
|
+
}
|
|
58374
|
+
const matching = candidates.filter((row) => deriveSiteHealthState(row) === healthState);
|
|
58375
|
+
total = matching.length;
|
|
58376
|
+
rows = matching.slice(offset, offset + limit);
|
|
58377
|
+
} else {
|
|
58378
|
+
total = app.db.select({ value: count() }).from(siteCrawlPages).where(where).get()?.value ?? 0;
|
|
58379
|
+
rows = app.db.select().from(siteCrawlPages).where(where).orderBy(...orderBy).limit(limit).offset(offset).all();
|
|
58380
|
+
}
|
|
56993
58381
|
const nextOffset = offset + rows.length;
|
|
56994
58382
|
return {
|
|
56995
58383
|
project: project.name,
|
|
@@ -57005,7 +58393,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57005
58393
|
const parentPath = normalizeParentPath(request.query.parentPath);
|
|
57006
58394
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57007
58395
|
if (!target) {
|
|
57008
|
-
|
|
58396
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57009
58397
|
return { project: project.name, hasCrawlData: false, runId: null, parentPath, nextCursor: null, children: [] };
|
|
57010
58398
|
}
|
|
57011
58399
|
const snapshot = target.snapshot;
|
|
@@ -57017,10 +58405,10 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57017
58405
|
url: siteCrawlPages.url,
|
|
57018
58406
|
inventoryEligible: siteCrawlPages.inventoryEligible,
|
|
57019
58407
|
fetchState: siteCrawlPages.fetchState
|
|
57020
|
-
}).from(siteCrawlPages).where(
|
|
57021
|
-
|
|
57022
|
-
|
|
57023
|
-
|
|
58408
|
+
}).from(siteCrawlPages).where(and44(
|
|
58409
|
+
eq56(siteCrawlPages.projectId, project.id),
|
|
58410
|
+
eq56(siteCrawlPages.runId, snapshot.runId),
|
|
58411
|
+
eq56(siteCrawlPages.attemptId, snapshot.attemptId)
|
|
57024
58412
|
)).limit(MAX_STRUCTURE_SOURCE_ROWS + 1).all();
|
|
57025
58413
|
if (sourceRows.length > MAX_STRUCTURE_SOURCE_ROWS) {
|
|
57026
58414
|
throw validationError(`Persisted crawl exceeds the ${MAX_STRUCTURE_SOURCE_ROWS}-page structure limit`);
|
|
@@ -57071,7 +58459,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57071
58459
|
const project = resolveProject(app.db, request.params.name);
|
|
57072
58460
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57073
58461
|
if (!target) {
|
|
57074
|
-
|
|
58462
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57075
58463
|
return { project: project.name, hasCrawlData: false, runId: null, total: 0, nextCursor: null, edges: [] };
|
|
57076
58464
|
}
|
|
57077
58465
|
const snapshot = target.snapshot;
|
|
@@ -57079,16 +58467,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57079
58467
|
return { project: project.name, hasCrawlData: true, runId: snapshot.runId, total: 0, nextCursor: null, edges: [] };
|
|
57080
58468
|
}
|
|
57081
58469
|
const filters = [
|
|
57082
|
-
|
|
57083
|
-
|
|
57084
|
-
|
|
57085
|
-
|
|
58470
|
+
eq56(siteCrawlEdges.projectId, project.id),
|
|
58471
|
+
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58472
|
+
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58473
|
+
eq56(siteCrawlEdges.internal, true)
|
|
57086
58474
|
];
|
|
57087
|
-
if (request.query.sourceUrl) filters.push(
|
|
57088
|
-
if (request.query.targetUrl) filters.push(
|
|
58475
|
+
if (request.query.sourceUrl) filters.push(eq56(siteCrawlEdges.sourceUrl, request.query.sourceUrl));
|
|
58476
|
+
if (request.query.targetUrl) filters.push(eq56(siteCrawlEdges.targetUrl, request.query.targetUrl));
|
|
57089
58477
|
const followable = parseBoolean2(request.query.followable);
|
|
57090
|
-
if (followable != null) filters.push(
|
|
57091
|
-
const where =
|
|
58478
|
+
if (followable != null) filters.push(eq56(siteCrawlEdges.followable, followable));
|
|
58479
|
+
const where = and44(...filters);
|
|
57092
58480
|
const total = app.db.select({ value: count() }).from(siteCrawlEdges).where(where).get()?.value ?? 0;
|
|
57093
58481
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
57094
58482
|
const offset = decodeCursor(request.query.cursor);
|
|
@@ -57110,7 +58498,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57110
58498
|
}
|
|
57111
58499
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57112
58500
|
if (!target) {
|
|
57113
|
-
|
|
58501
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57114
58502
|
return {
|
|
57115
58503
|
project: project.name,
|
|
57116
58504
|
hasCrawlData: false,
|
|
@@ -57138,16 +58526,16 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57138
58526
|
};
|
|
57139
58527
|
}
|
|
57140
58528
|
const scope = [
|
|
57141
|
-
|
|
57142
|
-
|
|
57143
|
-
|
|
57144
|
-
|
|
58529
|
+
eq56(siteCrawlEdges.projectId, project.id),
|
|
58530
|
+
eq56(siteCrawlEdges.runId, snapshot.runId),
|
|
58531
|
+
eq56(siteCrawlEdges.attemptId, snapshot.attemptId),
|
|
58532
|
+
eq56(siteCrawlEdges.internal, true)
|
|
57145
58533
|
];
|
|
57146
|
-
const inboundMatch = request.query.nodeKey && request.query.url ? or11(
|
|
57147
|
-
const outboundMatch = request.query.nodeKey && request.query.url ? or11(
|
|
58534
|
+
const inboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.targetUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.targetNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.targetUrl, request.query.url);
|
|
58535
|
+
const outboundMatch = request.query.nodeKey && request.query.url ? or11(eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey), eq56(siteCrawlEdges.sourceUrl, request.query.url)) : request.query.nodeKey ? eq56(siteCrawlEdges.sourceNodeKey, request.query.nodeKey) : eq56(siteCrawlEdges.sourceUrl, request.query.url);
|
|
57148
58536
|
const limit = parseBoundedLimit(request.query.limit, 50, 100);
|
|
57149
|
-
const inboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
57150
|
-
const outboundRows = app.db.select().from(siteCrawlEdges).where(
|
|
58537
|
+
const inboundRows = app.db.select().from(siteCrawlEdges).where(and44(...scope, inboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
58538
|
+
const outboundRows = app.db.select().from(siteCrawlEdges).where(and44(...scope, outboundMatch)).orderBy(asc10(siteCrawlEdges.edgeKey)).limit(limit + 1).all();
|
|
57151
58539
|
return {
|
|
57152
58540
|
project: project.name,
|
|
57153
58541
|
hasCrawlData: true,
|
|
@@ -57165,7 +58553,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57165
58553
|
const target = resolveCrawl(project.id, request.query.runId);
|
|
57166
58554
|
const legacyAuditAvailable = hasLegacyAudit(project.id);
|
|
57167
58555
|
if (!target) {
|
|
57168
|
-
|
|
58556
|
+
assertKnownAuditRun(project.id, request.query.runId);
|
|
57169
58557
|
return { project: project.name, runId: null, state: "unavailable", legacyAuditAvailable };
|
|
57170
58558
|
}
|
|
57171
58559
|
const snapshot = target.snapshot;
|
|
@@ -57178,11 +58566,11 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57178
58566
|
if (!snapshot.attemptId) {
|
|
57179
58567
|
return { project: project.name, runId: snapshot.runId, state: "partial", checkDeadLinks: true, checked: snapshot.deadLinksChecked, found: snapshot.deadLinksFound, total: 0, nextCursor: null, deadLinks: [] };
|
|
57180
58568
|
}
|
|
57181
|
-
const where =
|
|
57182
|
-
|
|
57183
|
-
|
|
57184
|
-
|
|
57185
|
-
|
|
58569
|
+
const where = and44(
|
|
58570
|
+
eq56(siteCrawlFindings.projectId, project.id),
|
|
58571
|
+
eq56(siteCrawlFindings.runId, snapshot.runId),
|
|
58572
|
+
eq56(siteCrawlFindings.attemptId, snapshot.attemptId),
|
|
58573
|
+
eq56(siteCrawlFindings.findingType, "dead-link")
|
|
57186
58574
|
);
|
|
57187
58575
|
const total = app.db.select({ value: count() }).from(siteCrawlFindings).where(where).get()?.value ?? 0;
|
|
57188
58576
|
const limit = parseBoundedLimit(request.query.limit, 100, 200);
|
|
@@ -57211,6 +58599,38 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57211
58599
|
deadLinks
|
|
57212
58600
|
};
|
|
57213
58601
|
});
|
|
58602
|
+
app.get("/projects/:name/technical-aeo/runs", async (request) => {
|
|
58603
|
+
const project = resolveProject(app.db, request.params.name);
|
|
58604
|
+
const limit = parseBoundedLimit(request.query.limit, SITE_HEALTH_SCANS_DEFAULT_LIMIT, SITE_HEALTH_SCANS_MAX_LIMIT);
|
|
58605
|
+
const rows = app.db.select({
|
|
58606
|
+
runId: runs.id,
|
|
58607
|
+
status: runs.status,
|
|
58608
|
+
createdAt: runs.createdAt,
|
|
58609
|
+
startedAt: runs.startedAt,
|
|
58610
|
+
finishedAt: runs.finishedAt
|
|
58611
|
+
}).from(runs).where(and44(
|
|
58612
|
+
eq56(runs.projectId, project.id),
|
|
58613
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
58614
|
+
notProbeRun()
|
|
58615
|
+
)).orderBy(desc24(runs.createdAt), desc24(runs.id)).limit(limit).all();
|
|
58616
|
+
const runIds = rows.map((row) => row.runId);
|
|
58617
|
+
const crawlRunIds = new Set(runIds.length === 0 ? [] : app.db.select({ runId: siteCrawlSnapshots.runId }).from(siteCrawlSnapshots).innerJoin(runs, eq56(siteCrawlSnapshots.runId, runs.id)).where(and44(
|
|
58618
|
+
eq56(siteCrawlSnapshots.projectId, project.id),
|
|
58619
|
+
eq56(runs.projectId, project.id),
|
|
58620
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
58621
|
+
inArray19(runs.status, SURFACEABLE_STATUSES),
|
|
58622
|
+
notProbeRun(),
|
|
58623
|
+
inArray19(siteCrawlSnapshots.runId, runIds)
|
|
58624
|
+
)).all().map((row) => row.runId));
|
|
58625
|
+
return {
|
|
58626
|
+
project: project.name,
|
|
58627
|
+
scans: rows.map((row) => ({
|
|
58628
|
+
...row,
|
|
58629
|
+
status: row.status,
|
|
58630
|
+
hasCrawlData: crawlRunIds.has(row.runId)
|
|
58631
|
+
}))
|
|
58632
|
+
};
|
|
58633
|
+
});
|
|
57214
58634
|
app.post("/projects/:name/technical-aeo/runs", async (request) => {
|
|
57215
58635
|
const project = resolveProject(app.db, request.params.name);
|
|
57216
58636
|
const parsed = siteAuditRunRequestSchema.safeParse(request.body ?? {});
|
|
@@ -57225,12 +58645,12 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57225
58645
|
status: runs.status,
|
|
57226
58646
|
identityKey: siteCrawlRunRequests.identityKey,
|
|
57227
58647
|
effectiveOptions: siteCrawlRunRequests.effectiveOptions
|
|
57228
|
-
}).from(runs).leftJoin(siteCrawlRunRequests,
|
|
57229
|
-
|
|
57230
|
-
|
|
57231
|
-
)).where(
|
|
57232
|
-
|
|
57233
|
-
|
|
58648
|
+
}).from(runs).leftJoin(siteCrawlRunRequests, and44(
|
|
58649
|
+
eq56(siteCrawlRunRequests.projectId, runs.projectId),
|
|
58650
|
+
eq56(siteCrawlRunRequests.runId, runs.id)
|
|
58651
|
+
)).where(and44(
|
|
58652
|
+
eq56(runs.projectId, project.id),
|
|
58653
|
+
eq56(runs.kind, RunKinds["site-audit"]),
|
|
57234
58654
|
inArray19(runs.status, [RunStatuses.queued, RunStatuses.running])
|
|
57235
58655
|
)).get();
|
|
57236
58656
|
if (existing) {
|
|
@@ -57248,7 +58668,7 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57248
58668
|
);
|
|
57249
58669
|
}
|
|
57250
58670
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
57251
|
-
const runId =
|
|
58671
|
+
const runId = crypto44.randomUUID();
|
|
57252
58672
|
tx.insert(runs).values({
|
|
57253
58673
|
id: runId,
|
|
57254
58674
|
projectId: project.id,
|
|
@@ -57281,8 +58701,8 @@ async function technicalAeoRoutes(app, opts) {
|
|
|
57281
58701
|
}
|
|
57282
58702
|
|
|
57283
58703
|
// ../api-routes/src/research.ts
|
|
57284
|
-
import
|
|
57285
|
-
import { and as
|
|
58704
|
+
import crypto45 from "crypto";
|
|
58705
|
+
import { and as and45, desc as desc25, eq as eq57 } from "drizzle-orm";
|
|
57286
58706
|
var sameLocation = (a, b) => a.label === b.label && a.city === b.city && a.region === b.region && a.country === b.country && a.timezone === b.timezone;
|
|
57287
58707
|
async function researchRoutes(app, opts) {
|
|
57288
58708
|
app.post("/projects/:name/research/runs", async (request, reply) => {
|
|
@@ -57308,19 +58728,19 @@ async function researchRoutes(app, opts) {
|
|
|
57308
58728
|
if (!adapter.modelValidationPattern.test(resolvedModel)) throw validationError(`Invalid resolved model "${resolvedModel}" for provider "${providerName}".`, { provider: providerName, model: resolvedModel, hint: adapter.modelValidationHint });
|
|
57309
58729
|
if (new Set(input.queries.map((query) => query.toLocaleLowerCase())).size !== input.queries.length) throw validationError("Research queries must be unique within a batch.");
|
|
57310
58730
|
const normalized = { queries: input.queries, provider: providerName, model: requestedModel, location: location ?? null };
|
|
57311
|
-
const requestHash2 =
|
|
58731
|
+
const requestHash2 = crypto45.createHash("sha256").update(JSON.stringify(normalized)).digest("hex");
|
|
57312
58732
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
57313
58733
|
const decision = app.db.transaction((tx) => {
|
|
57314
58734
|
if (input.idempotencyKey) {
|
|
57315
|
-
const existing = tx.select().from(researchRuns).where(
|
|
58735
|
+
const existing = tx.select().from(researchRuns).where(and45(eq57(researchRuns.projectId, project.id), eq57(researchRuns.idempotencyKey, input.idempotencyKey))).get();
|
|
57316
58736
|
if (existing) {
|
|
57317
58737
|
if (existing.requestHash !== requestHash2) throw alreadyExists("Research idempotency key", input.idempotencyKey);
|
|
57318
58738
|
return { reused: true, id: existing.id, shouldDispatch: existing.status === ResearchRunStatuses.queued };
|
|
57319
58739
|
}
|
|
57320
58740
|
}
|
|
57321
|
-
const id =
|
|
58741
|
+
const id = crypto45.randomUUID();
|
|
57322
58742
|
tx.insert(researchRuns).values({ id, projectId: project.id, status: ResearchRunStatuses.queued, provider: providerName, requestedModel, resolvedModel, location: location ?? null, totalQueries: input.queries.length, idempotencyKey: input.idempotencyKey ?? null, requestHash: input.idempotencyKey ? requestHash2 : null, createdAt: now }).run();
|
|
57323
|
-
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id:
|
|
58743
|
+
for (const [position, query] of input.queries.entries()) tx.insert(researchRunQueries).values({ id: crypto45.randomUUID(), researchRunId: id, position, queryText: query, status: ResearchQueryStatuses.queued, requestedModel, resolvedModel, groundingSources: [], citedDomains: [], searchQueries: [], createdAt: now }).run();
|
|
57324
58744
|
writeAuditLog(tx, { projectId: project.id, actor: "api", action: "research.created", entityType: "research_run", entityId: id });
|
|
57325
58745
|
return { reused: false, id, shouldDispatch: true };
|
|
57326
58746
|
});
|
|
@@ -57333,7 +58753,7 @@ async function researchRoutes(app, opts) {
|
|
|
57333
58753
|
const project = resolveProject(app.db, request.params.name);
|
|
57334
58754
|
const requested = Number.parseInt(request.query.limit ?? "", 10);
|
|
57335
58755
|
const limit = Number.isInteger(requested) && requested > 0 ? Math.min(requested, 100) : 20;
|
|
57336
|
-
const runs2 = app.db.select().from(researchRuns).where(
|
|
58756
|
+
const runs2 = app.db.select().from(researchRuns).where(eq57(researchRuns.projectId, project.id)).orderBy(desc25(researchRuns.createdAt)).limit(limit).all().map(serializeRun);
|
|
57337
58757
|
return { runs: runs2 };
|
|
57338
58758
|
});
|
|
57339
58759
|
app.get("/projects/:name/research/runs/:runId", async (request) => {
|
|
@@ -57342,9 +58762,9 @@ async function researchRoutes(app, opts) {
|
|
|
57342
58762
|
});
|
|
57343
58763
|
}
|
|
57344
58764
|
function getDetail(app, projectId, id) {
|
|
57345
|
-
const row = app.db.select().from(researchRuns).where(
|
|
58765
|
+
const row = app.db.select().from(researchRuns).where(and45(eq57(researchRuns.id, id), eq57(researchRuns.projectId, projectId))).get();
|
|
57346
58766
|
if (!row) throw notFound("Research run", id);
|
|
57347
|
-
const queries2 = app.db.select().from(researchRunQueries).where(
|
|
58767
|
+
const queries2 = app.db.select().from(researchRunQueries).where(eq57(researchRunQueries.researchRunId, id)).orderBy(researchRunQueries.position).all().map(serializeQuery);
|
|
57348
58768
|
return { ...serializeRun(row), queries: queries2 };
|
|
57349
58769
|
}
|
|
57350
58770
|
function serializeRun(row) {
|
|
@@ -57554,6 +58974,11 @@ async function apiRoutes(app, opts) {
|
|
|
57554
58974
|
vercelTrafficCredentialStore: opts.vercelTrafficCredentialStore,
|
|
57555
58975
|
pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
|
|
57556
58976
|
vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,
|
|
58977
|
+
cloudflareTrafficCredentialStore: opts.cloudflareTrafficCredentialStore,
|
|
58978
|
+
cloudflareTrafficIngestUrl: opts.cloudflareTrafficIngestUrl,
|
|
58979
|
+
cloudflareIngestRateLimitMax: opts.cloudflareIngestRateLimitMax,
|
|
58980
|
+
cloudflareIngestIpRateLimitMax: opts.cloudflareIngestIpRateLimitMax,
|
|
58981
|
+
defaultSampleLimit: opts.defaultTrafficSampleLimit,
|
|
57557
58982
|
onTrafficSynced: opts.onTrafficSynced,
|
|
57558
58983
|
onScheduleUpdated: opts.onScheduleUpdated,
|
|
57559
58984
|
allowLoopbackWebhooks: opts.allowLoopbackWebhooks
|
|
@@ -57734,11 +59159,62 @@ function buildTrafficSourceValidators(opts) {
|
|
|
57734
59159
|
validateScopes: () => null
|
|
57735
59160
|
};
|
|
57736
59161
|
}
|
|
59162
|
+
if (opts.cloudflareTrafficCredentialStore) {
|
|
59163
|
+
const store = opts.cloudflareTrafficCredentialStore;
|
|
59164
|
+
validators[TrafficSourceTypes.cloudflare] = {
|
|
59165
|
+
validateCredentials: (source) => {
|
|
59166
|
+
const sourceMode = source.configJson.deliveryMode;
|
|
59167
|
+
if (sourceMode !== void 0 && sourceMode !== "direct-push") return null;
|
|
59168
|
+
const record = store.getConnection(source.projectName);
|
|
59169
|
+
if (!record) {
|
|
59170
|
+
return {
|
|
59171
|
+
status: CheckStatuses.fail,
|
|
59172
|
+
code: "traffic.credentials.missing",
|
|
59173
|
+
summary: `No Cloudflare direct-push credential is stored for project "${source.projectName}".`,
|
|
59174
|
+
remediation: "Re-run `canonry traffic connect cloudflare <project> --zone-id <id>` from the credential-owning host."
|
|
59175
|
+
};
|
|
59176
|
+
}
|
|
59177
|
+
const recordMode = record.deliveryMode;
|
|
59178
|
+
if (recordMode !== "direct-push") {
|
|
59179
|
+
return {
|
|
59180
|
+
status: CheckStatuses.fail,
|
|
59181
|
+
code: "traffic.credentials.mode-mismatch",
|
|
59182
|
+
summary: `The stored Cloudflare credential mode does not match direct-push source "${source.displayName}".`,
|
|
59183
|
+
remediation: "Reconnect the Cloudflare source from the credential-owning host."
|
|
59184
|
+
};
|
|
59185
|
+
}
|
|
59186
|
+
if (record.sourceId !== source.id) {
|
|
59187
|
+
return {
|
|
59188
|
+
status: CheckStatuses.fail,
|
|
59189
|
+
code: "traffic.credentials.source-mismatch",
|
|
59190
|
+
summary: `The stored Cloudflare credential belongs to a different source than "${source.displayName}".`,
|
|
59191
|
+
remediation: "Reconnect the Cloudflare source to pair the credential and source row."
|
|
59192
|
+
};
|
|
59193
|
+
}
|
|
59194
|
+
const bearerToken = record.bearerToken;
|
|
59195
|
+
const hmacSecret = record.hmacSecret;
|
|
59196
|
+
if (typeof bearerToken !== "string" || bearerToken.length === 0 || typeof hmacSecret !== "string" || hmacSecret.length === 0 || !source.ingestTokenHash || hashCloudflareBearerToken(bearerToken) !== source.ingestTokenHash) {
|
|
59197
|
+
return {
|
|
59198
|
+
status: CheckStatuses.fail,
|
|
59199
|
+
code: "traffic.credentials.bearer-mismatch",
|
|
59200
|
+
summary: `The stored Cloudflare push credential does not match source "${source.displayName}".`,
|
|
59201
|
+
remediation: "Reconnect the Cloudflare source and redeploy the generated Worker secrets."
|
|
59202
|
+
};
|
|
59203
|
+
}
|
|
59204
|
+
return {
|
|
59205
|
+
status: CheckStatuses.ok,
|
|
59206
|
+
code: "traffic.credentials.resolved",
|
|
59207
|
+
summary: `Cloudflare direct-push credentials match source "${source.displayName}".`
|
|
59208
|
+
};
|
|
59209
|
+
},
|
|
59210
|
+
validateScopes: () => null
|
|
59211
|
+
};
|
|
59212
|
+
}
|
|
57737
59213
|
return Object.keys(validators).length > 0 ? validators : void 0;
|
|
57738
59214
|
}
|
|
57739
59215
|
|
|
57740
59216
|
// src/intelligence-service.ts
|
|
57741
|
-
import
|
|
59217
|
+
import crypto46 from "crypto";
|
|
57742
59218
|
|
|
57743
59219
|
// src/logger.ts
|
|
57744
59220
|
var IS_TTY = process.stdout.isTTY === true;
|
|
@@ -57946,9 +59422,9 @@ var IntelligenceService = class {
|
|
|
57946
59422
|
*/
|
|
57947
59423
|
analyzeAndPersist(runId, projectId) {
|
|
57948
59424
|
const recentRuns = this.db.select().from(runs).where(
|
|
57949
|
-
|
|
57950
|
-
|
|
57951
|
-
or12(
|
|
59425
|
+
and46(
|
|
59426
|
+
eq58(runs.projectId, projectId),
|
|
59427
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
57952
59428
|
// Defensive: RunCoordinator already skips probes before this is
|
|
57953
59429
|
// called, but if a future call site invokes analyzeAndPersist
|
|
57954
59430
|
// directly for a probe, probes still must not pollute the
|
|
@@ -58030,7 +59506,7 @@ var IntelligenceService = class {
|
|
|
58030
59506
|
* Returns the persisted insights so the coordinator can count critical/high.
|
|
58031
59507
|
*/
|
|
58032
59508
|
analyzeAndPersistGbp(runId, projectId) {
|
|
58033
|
-
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(
|
|
59509
|
+
const runRow = this.db.select({ createdAt: runs.createdAt, startedAt: runs.startedAt, finishedAt: runs.finishedAt }).from(runs).where(eq58(runs.id, runId)).get();
|
|
58034
59510
|
if (!runRow) {
|
|
58035
59511
|
log.info("gbp-intelligence.skip", { runId, reason: "run not found" });
|
|
58036
59512
|
this.persistGbpInsights(runId, projectId, [], []);
|
|
@@ -58038,11 +59514,11 @@ var IntelligenceService = class {
|
|
|
58038
59514
|
}
|
|
58039
59515
|
const windowStart = runRow.startedAt ?? runRow.createdAt;
|
|
58040
59516
|
const windowEnd = runRow.finishedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
58041
|
-
const selected = this.db.select().from(gbpLocations).where(
|
|
58042
|
-
|
|
58043
|
-
|
|
58044
|
-
|
|
58045
|
-
|
|
59517
|
+
const selected = this.db.select().from(gbpLocations).where(and46(
|
|
59518
|
+
eq58(gbpLocations.projectId, projectId),
|
|
59519
|
+
eq58(gbpLocations.selected, true),
|
|
59520
|
+
gte14(gbpLocations.syncedAt, windowStart),
|
|
59521
|
+
lte11(gbpLocations.syncedAt, windowEnd)
|
|
58046
59522
|
)).all();
|
|
58047
59523
|
if (selected.length === 0) {
|
|
58048
59524
|
log.info("gbp-intelligence.skip", { runId, reason: "no locations synced during run" });
|
|
@@ -58075,12 +59551,12 @@ var IntelligenceService = class {
|
|
|
58075
59551
|
}
|
|
58076
59552
|
/** Build the per-location signal bundle the GBP analyzer consumes. */
|
|
58077
59553
|
buildGbpLocationSignals(projectId, locationName, displayName, fallbackDate) {
|
|
58078
|
-
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(
|
|
58079
|
-
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(
|
|
58080
|
-
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(
|
|
58081
|
-
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(
|
|
59554
|
+
const metricRows = this.db.select({ metric: gbpDailyMetrics.metric, date: gbpDailyMetrics.date, value: gbpDailyMetrics.value }).from(gbpDailyMetrics).where(and46(eq58(gbpDailyMetrics.projectId, projectId), eq58(gbpDailyMetrics.locationName, locationName))).all();
|
|
59555
|
+
const placeActionRows = this.db.select({ placeActionType: gbpPlaceActions.placeActionType, providerType: gbpPlaceActions.providerType }).from(gbpPlaceActions).where(and46(eq58(gbpPlaceActions.projectId, projectId), eq58(gbpPlaceActions.locationName, locationName))).all();
|
|
59556
|
+
const lodgingRow = this.db.select({ populatedGroupCount: gbpLodgingSnapshots.populatedGroupCount }).from(gbpLodgingSnapshots).where(and46(eq58(gbpLodgingSnapshots.projectId, projectId), eq58(gbpLodgingSnapshots.locationName, locationName))).orderBy(desc26(gbpLodgingSnapshots.syncedAt)).limit(1).get();
|
|
59557
|
+
const ownerRow = this.db.select({ description: gbpLocations.description }).from(gbpLocations).where(and46(eq58(gbpLocations.projectId, projectId), eq58(gbpLocations.locationName, locationName))).get();
|
|
58082
59558
|
const descriptionMissing = !(ownerRow?.description ?? "").trim();
|
|
58083
|
-
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(
|
|
59559
|
+
const placeRow = this.db.select({ attributes: gbpPlaceDetails.attributes }).from(gbpPlaceDetails).where(and46(eq58(gbpPlaceDetails.projectId, projectId), eq58(gbpPlaceDetails.locationName, locationName))).orderBy(desc26(gbpPlaceDetails.syncedAt)).limit(1).get();
|
|
58084
59560
|
const placesAmenities = placeRow ? extractPlaceAmenities(placeRow.attributes) : [];
|
|
58085
59561
|
const summary = buildGbpSummary({
|
|
58086
59562
|
locationName,
|
|
@@ -58116,7 +59592,7 @@ var IntelligenceService = class {
|
|
|
58116
59592
|
/** Build the month-over-month keyword series for a location from the
|
|
58117
59593
|
* accumulating gbp_keyword_monthly table (latest complete month vs prior). */
|
|
58118
59594
|
buildGbpKeywordTrend(projectId, locationName) {
|
|
58119
|
-
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(
|
|
59595
|
+
const rows = this.db.select({ month: gbpKeywordMonthly.month, keyword: gbpKeywordMonthly.keyword, valueCount: gbpKeywordMonthly.valueCount }).from(gbpKeywordMonthly).where(and46(eq58(gbpKeywordMonthly.projectId, projectId), eq58(gbpKeywordMonthly.locationName, locationName))).all();
|
|
58120
59596
|
if (rows.length === 0) return { recentMonth: null, priorMonth: null, points: [] };
|
|
58121
59597
|
const months = [...new Set(rows.map((r) => r.month))].sort().reverse();
|
|
58122
59598
|
const recentMonth = months[0] ?? null;
|
|
@@ -58147,7 +59623,7 @@ var IntelligenceService = class {
|
|
|
58147
59623
|
*/
|
|
58148
59624
|
persistGbpInsights(runId, projectId, gbpInsights, coveredLocationNames) {
|
|
58149
59625
|
const covered = new Set(coveredLocationNames);
|
|
58150
|
-
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(
|
|
59626
|
+
const existing = this.db.select({ id: insights.id, dismissed: insights.dismissed }).from(insights).where(and46(eq58(insights.projectId, projectId), eq58(insights.provider, GBP_INSIGHT_PROVIDER))).all();
|
|
58151
59627
|
const staleIds = [];
|
|
58152
59628
|
const dismissedSlots = /* @__PURE__ */ new Set();
|
|
58153
59629
|
for (const row of existing) {
|
|
@@ -58158,7 +59634,7 @@ var IntelligenceService = class {
|
|
|
58158
59634
|
}
|
|
58159
59635
|
this.db.transaction((tx) => {
|
|
58160
59636
|
for (const id of staleIds) {
|
|
58161
|
-
tx.delete(insights).where(
|
|
59637
|
+
tx.delete(insights).where(eq58(insights.id, id)).run();
|
|
58162
59638
|
}
|
|
58163
59639
|
for (const insight of gbpInsights) {
|
|
58164
59640
|
const parsed = parseGbpInsightId(insight.id);
|
|
@@ -58236,7 +59712,7 @@ var IntelligenceService = class {
|
|
|
58236
59712
|
* create per run + aggregate). DB is left untouched.
|
|
58237
59713
|
*/
|
|
58238
59714
|
backfill(projectName, opts, onProgress) {
|
|
58239
|
-
const project = this.db.select().from(projects).where(
|
|
59715
|
+
const project = this.db.select().from(projects).where(eq58(projects.name, projectName)).get();
|
|
58240
59716
|
if (!project) {
|
|
58241
59717
|
throw new Error(`Project "${projectName}" not found`);
|
|
58242
59718
|
}
|
|
@@ -58249,9 +59725,9 @@ var IntelligenceService = class {
|
|
|
58249
59725
|
sinceTimestamp = parsed;
|
|
58250
59726
|
}
|
|
58251
59727
|
const allRuns = this.db.select().from(runs).where(
|
|
58252
|
-
|
|
58253
|
-
|
|
58254
|
-
or12(
|
|
59728
|
+
and46(
|
|
59729
|
+
eq58(runs.projectId, project.id),
|
|
59730
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
58255
59731
|
// Backfill must not replay probe runs as if they were real sweeps.
|
|
58256
59732
|
ne7(runs.trigger, RunTriggers.probe)
|
|
58257
59733
|
)
|
|
@@ -58330,7 +59806,7 @@ var IntelligenceService = class {
|
|
|
58330
59806
|
return { processed, skipped, totalInsights };
|
|
58331
59807
|
}
|
|
58332
59808
|
loadTrackedCompetitors(projectId) {
|
|
58333
|
-
return this.db.select({ domain: competitors.domain }).from(competitors).where(
|
|
59809
|
+
return this.db.select({ domain: competitors.domain }).from(competitors).where(eq58(competitors.projectId, projectId)).all().map((r) => r.domain);
|
|
58334
59810
|
}
|
|
58335
59811
|
/**
|
|
58336
59812
|
* Wipe transition signals from an analysis result while keeping health.
|
|
@@ -58351,15 +59827,15 @@ var IntelligenceService = class {
|
|
|
58351
59827
|
}
|
|
58352
59828
|
persistResult(result, runId, projectId) {
|
|
58353
59829
|
const previouslyDismissed = /* @__PURE__ */ new Set();
|
|
58354
|
-
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(
|
|
59830
|
+
const existingInsights = this.db.select({ query: insights.query, provider: insights.provider, type: insights.type, dismissed: insights.dismissed }).from(insights).where(eq58(insights.runId, runId)).all();
|
|
58355
59831
|
for (const row of existingInsights) {
|
|
58356
59832
|
if (row.dismissed) {
|
|
58357
59833
|
previouslyDismissed.add(`${row.query}:${row.provider}:${row.type}`);
|
|
58358
59834
|
}
|
|
58359
59835
|
}
|
|
58360
59836
|
this.db.transaction((tx) => {
|
|
58361
|
-
tx.delete(insights).where(
|
|
58362
|
-
tx.delete(healthSnapshots).where(
|
|
59837
|
+
tx.delete(insights).where(eq58(insights.runId, runId)).run();
|
|
59838
|
+
tx.delete(healthSnapshots).where(eq58(healthSnapshots.runId, runId)).run();
|
|
58363
59839
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
58364
59840
|
for (const insight of result.insights) {
|
|
58365
59841
|
const wasDismissed = previouslyDismissed.has(`${insight.query}:${insight.provider}:${insight.type}`);
|
|
@@ -58379,7 +59855,7 @@ var IntelligenceService = class {
|
|
|
58379
59855
|
}).run();
|
|
58380
59856
|
}
|
|
58381
59857
|
tx.insert(healthSnapshots).values({
|
|
58382
|
-
id:
|
|
59858
|
+
id: crypto46.randomUUID(),
|
|
58383
59859
|
projectId,
|
|
58384
59860
|
runId,
|
|
58385
59861
|
overallCitedRate: String(result.health.overallCitedRate),
|
|
@@ -58412,24 +59888,24 @@ var IntelligenceService = class {
|
|
|
58412
59888
|
applySeverityTiering(rawInsights, excludeRunId, projectId) {
|
|
58413
59889
|
const regressions = rawInsights.filter((i) => i.type === "regression");
|
|
58414
59890
|
if (regressions.length === 0) return rawInsights;
|
|
58415
|
-
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(
|
|
59891
|
+
const gscRows = this.db.select({ query: gscSearchData.query, impressions: gscSearchData.impressions }).from(gscSearchData).where(eq58(gscSearchData.projectId, projectId)).all();
|
|
58416
59892
|
const gscConnected = gscRows.length > 0;
|
|
58417
59893
|
const gscImpressionsByQuery = /* @__PURE__ */ new Map();
|
|
58418
59894
|
for (const row of gscRows) {
|
|
58419
59895
|
const key = row.query.toLowerCase();
|
|
58420
59896
|
gscImpressionsByQuery.set(key, (gscImpressionsByQuery.get(key) ?? 0) + row.impressions);
|
|
58421
59897
|
}
|
|
58422
|
-
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(
|
|
59898
|
+
const projectRow = this.db.select({ locations: projects.locations }).from(projects).where(eq58(projects.id, projectId)).get();
|
|
58423
59899
|
const locationCount = Math.max(
|
|
58424
59900
|
1,
|
|
58425
59901
|
(projectRow?.locations ?? []).length
|
|
58426
59902
|
);
|
|
58427
59903
|
const ROWS_PER_GROUP_BUDGET = Math.max(2, locationCount);
|
|
58428
59904
|
const recentRunRows = this.db.select({ id: runs.id, createdAt: runs.createdAt }).from(runs).where(
|
|
58429
|
-
|
|
58430
|
-
|
|
58431
|
-
|
|
58432
|
-
or12(
|
|
59905
|
+
and46(
|
|
59906
|
+
eq58(runs.projectId, projectId),
|
|
59907
|
+
eq58(runs.kind, RunKinds["answer-visibility"]),
|
|
59908
|
+
or12(eq58(runs.status, "completed"), eq58(runs.status, "partial")),
|
|
58433
59909
|
// Defensive — see top of file.
|
|
58434
59910
|
ne7(runs.trigger, RunTriggers.probe)
|
|
58435
59911
|
)
|
|
@@ -58449,7 +59925,7 @@ var IntelligenceService = class {
|
|
|
58449
59925
|
const haveHistory = recentRunIds.length > 0;
|
|
58450
59926
|
const priorRegressionsByPair = /* @__PURE__ */ new Map();
|
|
58451
59927
|
if (haveHistory) {
|
|
58452
|
-
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(
|
|
59928
|
+
const priorRows = this.db.select({ query: insights.query, provider: insights.provider, runId: insights.runId }).from(insights).where(and46(eq58(insights.type, "regression"), inArray20(insights.runId, recentRunIds))).all();
|
|
58453
59929
|
const regressionGroups = /* @__PURE__ */ new Map();
|
|
58454
59930
|
for (const row of priorRows) {
|
|
58455
59931
|
if (!row.runId) continue;
|
|
@@ -58478,7 +59954,7 @@ var IntelligenceService = class {
|
|
|
58478
59954
|
});
|
|
58479
59955
|
}
|
|
58480
59956
|
buildRunData(runId, projectId, completedAt, location = null) {
|
|
58481
|
-
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(
|
|
59957
|
+
const projectDomainRow = this.db.select({ canonicalDomain: projects.canonicalDomain, ownedDomains: projects.ownedDomains }).from(projects).where(eq58(projects.id, projectId)).get();
|
|
58482
59958
|
const projectDomains = projectDomainRow ? effectiveDomains({
|
|
58483
59959
|
canonicalDomain: projectDomainRow.canonicalDomain,
|
|
58484
59960
|
ownedDomains: projectDomainRow.ownedDomains
|
|
@@ -58495,7 +59971,7 @@ var IntelligenceService = class {
|
|
|
58495
59971
|
citedDomains: querySnapshots.citedDomains,
|
|
58496
59972
|
competitorOverlap: querySnapshots.competitorOverlap,
|
|
58497
59973
|
snapshotLocation: querySnapshots.location
|
|
58498
|
-
}).from(querySnapshots).leftJoin(queries,
|
|
59974
|
+
}).from(querySnapshots).leftJoin(queries, eq58(querySnapshots.queryId, queries.id)).where(eq58(querySnapshots.runId, runId)).all();
|
|
58499
59975
|
const snapshots = [];
|
|
58500
59976
|
let orphanCount = 0;
|
|
58501
59977
|
for (const r of rows) {
|
|
@@ -58658,6 +60134,12 @@ export {
|
|
|
58658
60134
|
listCachedReleases,
|
|
58659
60135
|
pruneCachedRelease,
|
|
58660
60136
|
resolveVercelSyncDeadlineMs,
|
|
60137
|
+
CLOUDFLARE_WORKER_BINDINGS,
|
|
60138
|
+
CLOUDFLARE_WORKER_GENERATED_MARKER,
|
|
60139
|
+
CLOUDFLARE_WRANGLER_GENERATED_MARKER,
|
|
60140
|
+
DEFAULT_BOT_LIST,
|
|
60141
|
+
generateWorkerScript,
|
|
60142
|
+
generateWranglerToml,
|
|
58661
60143
|
executeDiscovery,
|
|
58662
60144
|
markSessionFailed,
|
|
58663
60145
|
apiRoutes,
|