@canonry/canonry 4.177.0 → 4.177.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/agent-workspace/skills/canonry/SKILL.md +7 -0
- package/assets/agent-workspace/skills/canonry/references/server-side-traffic.md +40 -9
- package/dist/{chunk-F4ZZKGAC.js → chunk-6XUWDSLN.js} +10 -10
- package/dist/{chunk-X776E66Z.js → chunk-GJE334DA.js} +823 -613
- package/dist/{chunk-UXXXKDW4.js → chunk-PQT3ECM3.js} +5 -4
- package/dist/{chunk-WW7TDVLV.js → chunk-QRE3DAAU.js} +2 -2
- package/dist/{chunk-YWFNRDLK.js → chunk-XFAPF2YX.js} +4 -4
- package/dist/cli.js +9 -6
- package/dist/index.js +4 -4
- package/dist/{intelligence-service-QLLGPZOC.js → intelligence-service-C57RIB37.js} +2 -2
- package/dist/mcp.js +3 -3
- package/package.json +10 -10
|
@@ -630,7 +630,7 @@ import {
|
|
|
630
630
|
wordpressSchemaDeployResultDtoSchema,
|
|
631
631
|
wordpressSchemaStatusResultDtoSchema,
|
|
632
632
|
wordpressStatusDtoSchema
|
|
633
|
-
} from "./chunk-
|
|
633
|
+
} from "./chunk-PQT3ECM3.js";
|
|
634
634
|
|
|
635
635
|
// src/intelligence-service.ts
|
|
636
636
|
import { eq as eq61, desc as desc27, asc as asc11, and as and50, ne as ne8, or as or14, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
|
|
@@ -2242,6 +2242,11 @@ var trafficSources = sqliteTable("traffic_sources", {
|
|
|
2242
2242
|
status: text("status").notNull(),
|
|
2243
2243
|
lastSyncedAt: text("last_synced_at"),
|
|
2244
2244
|
lastCursor: text("last_cursor"),
|
|
2245
|
+
// A WordPress cursor is valid only inside the fixed [lastSyncedAt,
|
|
2246
|
+
// wordpressPendingUntil) window that created it. NULL means no such window
|
|
2247
|
+
// is pending; a non-NULL cursor with a NULL end is legacy state that cannot
|
|
2248
|
+
// safely compose with a bounded incremental pull.
|
|
2249
|
+
wordpressPendingUntil: text("wordpress_pending_until"),
|
|
2245
2250
|
lastError: text("last_error"),
|
|
2246
2251
|
// JSON-encoded array of normalized event IDs (e.g. `cloud-run:<ts>:<insertId>`)
|
|
2247
2252
|
// observed in the most recent successful sync. Bounded ring buffer used to
|
|
@@ -6849,6 +6854,18 @@ var MIGRATION_VERSIONS = [
|
|
|
6849
6854
|
statements: [
|
|
6850
6855
|
`CREATE INDEX IF NOT EXISTS idx_raw_event_samples_ts ON raw_event_samples(ts)`
|
|
6851
6856
|
]
|
|
6857
|
+
},
|
|
6858
|
+
{
|
|
6859
|
+
version: 145,
|
|
6860
|
+
name: "wordpress-traffic-pending-window",
|
|
6861
|
+
// A persisted upper bound makes a capped WordPress cursor drain a finite,
|
|
6862
|
+
// repeatable [last_synced_at, wordpress_pending_until) window. Existing
|
|
6863
|
+
// non-null cursors deliberately receive NULL here: they were created by
|
|
6864
|
+
// the old unbounded route, so treating their already-advanced watermark
|
|
6865
|
+
// as a lower bound would silently skip undrained history.
|
|
6866
|
+
statements: [
|
|
6867
|
+
`ALTER TABLE traffic_sources ADD COLUMN wordpress_pending_until TEXT`
|
|
6868
|
+
]
|
|
6852
6869
|
}
|
|
6853
6870
|
];
|
|
6854
6871
|
function addRunsMeasurementPlanVersionForeignKey(tx) {
|
|
@@ -35387,7 +35404,7 @@ var routeCatalog = [
|
|
|
35387
35404
|
method: "post",
|
|
35388
35405
|
path: "/api/v1/projects/{name}/traffic/connect/wordpress",
|
|
35389
35406
|
summary: "Connect a WordPress traffic-logger source",
|
|
35390
|
-
description: "Probes the WordPress traffic-logger plugin endpoint with the supplied Application Password (single page, `limit=1`) before persisting. On success, stores the credential in `~/.canonry/config.yaml` and creates / updates the project's active WordPress `traffic_sources` row. A probe failure (HTTP 4xx/5xx, network error) surfaces as 502 with the upstream status in the message so the caller learns about a bad credential up front instead of at the first sync.",
|
|
35407
|
+
description: "Probes the WordPress traffic-logger plugin endpoint with the supplied Application Password (single page, `limit=1`) before persisting. On success, stores the credential in `~/.canonry/config.yaml` and creates / updates the project's active WordPress `traffic_sources` row. Changing `baseUrl` archives the old source lineage and creates a fresh source so historical rollups cannot mix across endpoints. A probe failure (HTTP 4xx/5xx, network error) surfaces as 502 with the upstream status in the message so the caller learns about a bad credential up front instead of at the first sync.",
|
|
35391
35408
|
tags: ["traffic"],
|
|
35392
35409
|
parameters: [nameParameter],
|
|
35393
35410
|
requestBody: {
|
|
@@ -35514,7 +35531,7 @@ var routeCatalog = [
|
|
|
35514
35531
|
method: "post",
|
|
35515
35532
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
|
|
35516
35533
|
summary: "Trigger a sync run for a traffic source",
|
|
35517
|
-
description: "Pulls from the selected Cloud Run, WordPress, Vercel, or Cloudflare Queue source, classifies crawler hits / user fetches / AI-referral sessions, and commits hourly buckets plus a bounded sample tail. Queue pull uses a durable source lease and acknowledges each Cloudflare message only after its event receipts and rollups commit.",
|
|
35534
|
+
description: "Pulls from the selected Cloud Run, WordPress, Vercel, or Cloudflare Queue source, classifies crawler hits / user fetches / AI-referral sessions, and commits hourly buckets plus a bounded sample tail. WordPress uses a fixed half-open `[since, until)` window and a source lease; its first or idle sync defaults to the plugin maximum retention horizon (365d) unless `sinceMinutes` is supplied. Queue pull also uses a durable source lease and acknowledges each Cloudflare message only after its event receipts and rollups commit.",
|
|
35518
35535
|
tags: ["traffic"],
|
|
35519
35536
|
parameters: [
|
|
35520
35537
|
nameParameter,
|
|
@@ -35527,7 +35544,7 @@ var routeCatalog = [
|
|
|
35527
35544
|
schema: {
|
|
35528
35545
|
type: "object",
|
|
35529
35546
|
properties: {
|
|
35530
|
-
sinceMinutes: { ...integerSchema, description: "Optional lookback for time-window sources; Cloudflare Queue pull ignores it." }
|
|
35547
|
+
sinceMinutes: { ...integerSchema, description: "Optional lookback for time-window sources. Defaults are adapter-specific; a new or idle WordPress source uses 365d to cover the plugin\u2019s maximum configurable retention. Cloudflare Queue pull ignores it." }
|
|
35531
35548
|
}
|
|
35532
35549
|
}
|
|
35533
35550
|
}
|
|
@@ -35536,7 +35553,7 @@ var routeCatalog = [
|
|
|
35536
35553
|
responses: {
|
|
35537
35554
|
200: jsonResponse("Sync summary returned.", "TrafficSyncResponse"),
|
|
35538
35555
|
400: errorResponse("Invalid sync request or missing credentials."),
|
|
35539
|
-
409: errorResponse("Another Queue sync currently owns the source lease."),
|
|
35556
|
+
409: errorResponse("Another WordPress or Queue sync currently owns the source lease."),
|
|
35540
35557
|
404: errorResponse("Project or traffic source not found."),
|
|
35541
35558
|
502: errorResponse("Upstream pull, acknowledgement, or credential resolution failed.")
|
|
35542
35559
|
}
|
|
@@ -35545,7 +35562,7 @@ var routeCatalog = [
|
|
|
35545
35562
|
method: "post",
|
|
35546
35563
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/backfill",
|
|
35547
35564
|
summary: "Reclassify historical traffic-source logs",
|
|
35548
|
-
description: 'Async one-shot
|
|
35565
|
+
description: 'Async one-shot reclassification: pulls the last `days` of events, classifies them with current rules, and replaces hourly rollup buckets plus the sample slice in that window inside one transaction. Adapter limits apply (Cloud Logging `_Default` is typically 30d; request payloads are capped at 90d). Generic replace backfill supports `cloud-run` and `vercel`. WordPress is always rejected because its retained event feed cannot prove it covers every bucket; use a retention-aware repair that explicitly declares any unrecoverable span. Returns immediately with `{ runId, status: "running" }`; poll `GET /runs/{id}` for completion. lastSyncedAt only advances forward.',
|
|
35549
35566
|
tags: ["traffic"],
|
|
35550
35567
|
parameters: [
|
|
35551
35568
|
nameParameter,
|
|
@@ -35558,7 +35575,7 @@ var routeCatalog = [
|
|
|
35558
35575
|
schema: {
|
|
35559
35576
|
type: "object",
|
|
35560
35577
|
properties: {
|
|
35561
|
-
days: { ...integerSchema, description: "Lookback window in days (default 30, capped
|
|
35578
|
+
days: { ...integerSchema, description: "Lookback window in days (default 30, capped by the adapter; generic WordPress replace backfill is unavailable)." }
|
|
35562
35579
|
}
|
|
35563
35580
|
}
|
|
35564
35581
|
}
|
|
@@ -35574,7 +35591,7 @@ var routeCatalog = [
|
|
|
35574
35591
|
method: "post",
|
|
35575
35592
|
path: "/api/v1/projects/{name}/traffic/sources/{id}/reset",
|
|
35576
35593
|
summary: "Advance lastSyncedAt to NOW and clear the error state",
|
|
35577
|
-
description: "Operator recovery: advances `lastSyncedAt` to NOW, sets `status` back to `connected`, and clears `last_error`.
|
|
35594
|
+
description: "Operator recovery: advances `lastSyncedAt` to NOW, sets `status` back to `connected`, and clears `last_error`. WordPress resets also clear `last_cursor` and the pending-window marker, then record an unrecovered skip through the reset instant so the next drain cannot combine an old cursor with a new window. That prevents replay but is not a historical repair. Generic WordPress replace-mode backfill is unavailable because the plugin feed cannot prove retained coverage; use a retention-aware repair and explicitly declare any unrecoverable span. Common trigger: an idle Vercel/Cloud Run source whose `lastSyncedAt` has aged past upstream retention (`request-logs` ~14d, Cloud Logging 30d) and now throws on every sync. `advanceToNow: true` is required (no implicit reset). Archived sources are rejected with 400 \u2014 re-connect them via the appropriate `traffic/connect/*` endpoint instead.",
|
|
35578
35595
|
tags: ["traffic"],
|
|
35579
35596
|
parameters: [
|
|
35580
35597
|
nameParameter,
|
|
@@ -66747,12 +66764,92 @@ async function readErrorBody2(response) {
|
|
|
66747
66764
|
if (!text2) return void 0;
|
|
66748
66765
|
return text2.length <= 500 ? text2 : `${text2.slice(0, 500)}... [truncated]`;
|
|
66749
66766
|
}
|
|
66767
|
+
function isRecord2(value) {
|
|
66768
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
66769
|
+
}
|
|
66770
|
+
function parseRequestedBoundary(name, value) {
|
|
66771
|
+
if (value === void 0 || value === "") return void 0;
|
|
66772
|
+
const timestampMs = Date.parse(value);
|
|
66773
|
+
if (!Number.isFinite(timestampMs)) {
|
|
66774
|
+
throw new WordpressTrafficApiError(`${name} must be a valid ISO 8601 timestamp`, 400);
|
|
66775
|
+
}
|
|
66776
|
+
return { value, timestampMs };
|
|
66777
|
+
}
|
|
66778
|
+
function parseRequestedWindow(options) {
|
|
66779
|
+
const since = parseRequestedBoundary("since", options.since);
|
|
66780
|
+
const until = parseRequestedBoundary("until", options.until);
|
|
66781
|
+
if (since && until && since.timestampMs >= until.timestampMs) {
|
|
66782
|
+
throw new WordpressTrafficApiError("since must be earlier than until", 400);
|
|
66783
|
+
}
|
|
66784
|
+
return { since, until };
|
|
66785
|
+
}
|
|
66786
|
+
function parseSiteMetadata(value) {
|
|
66787
|
+
if (!isRecord2(value)) return void 0;
|
|
66788
|
+
return {
|
|
66789
|
+
url: typeof value.url === "string" ? value.url : void 0,
|
|
66790
|
+
anonymous_id: typeof value.anonymous_id === "string" ? value.anonymous_id : void 0,
|
|
66791
|
+
wordpress_version: typeof value.wordpress_version === "string" ? value.wordpress_version : void 0,
|
|
66792
|
+
plugin_version: typeof value.plugin_version === "string" ? value.plugin_version : void 0
|
|
66793
|
+
};
|
|
66794
|
+
}
|
|
66795
|
+
function invalidResponse(message) {
|
|
66796
|
+
return new WordpressTrafficApiError(
|
|
66797
|
+
`WordPress traffic endpoint returned an invalid response: ${message}`,
|
|
66798
|
+
502
|
|
66799
|
+
);
|
|
66800
|
+
}
|
|
66801
|
+
function parseResponseBody(value) {
|
|
66802
|
+
if (!isRecord2(value)) {
|
|
66803
|
+
throw invalidResponse("expected an object body");
|
|
66804
|
+
}
|
|
66805
|
+
if (!Array.isArray(value.events)) {
|
|
66806
|
+
throw invalidResponse("events must be an array");
|
|
66807
|
+
}
|
|
66808
|
+
if (typeof value.has_more !== "boolean") {
|
|
66809
|
+
throw invalidResponse("has_more must be a boolean");
|
|
66810
|
+
}
|
|
66811
|
+
const nextCursor = value.next_cursor;
|
|
66812
|
+
if (value.has_more) {
|
|
66813
|
+
if (typeof nextCursor !== "string" || nextCursor.trim().length === 0) {
|
|
66814
|
+
throw invalidResponse("has_more=true requires a nonempty next_cursor");
|
|
66815
|
+
}
|
|
66816
|
+
} else if (nextCursor !== null) {
|
|
66817
|
+
throw invalidResponse("has_more=false requires next_cursor=null");
|
|
66818
|
+
}
|
|
66819
|
+
return {
|
|
66820
|
+
events: value.events,
|
|
66821
|
+
nextCursor,
|
|
66822
|
+
hasMore: value.has_more,
|
|
66823
|
+
site: parseSiteMetadata(value.site)
|
|
66824
|
+
};
|
|
66825
|
+
}
|
|
66826
|
+
async function readResponseJson(response) {
|
|
66827
|
+
try {
|
|
66828
|
+
return await response.json();
|
|
66829
|
+
} catch {
|
|
66830
|
+
throw invalidResponse("body is not valid JSON");
|
|
66831
|
+
}
|
|
66832
|
+
}
|
|
66833
|
+
function assertEntryIsWithinRequestedWindow(entry, window) {
|
|
66834
|
+
if (!window.since && !window.until) return;
|
|
66835
|
+
const observedAt = isRecord2(entry) && typeof entry.observed_at === "string" ? entry.observed_at : void 0;
|
|
66836
|
+
const observedAtMs = observedAt === void 0 ? Number.NaN : Date.parse(observedAt);
|
|
66837
|
+
const outsideWindow = !Number.isFinite(observedAtMs) || window.since !== void 0 && observedAtMs < window.since.timestampMs || window.until !== void 0 && observedAtMs >= window.until.timestampMs;
|
|
66838
|
+
if (!outsideWindow) return;
|
|
66839
|
+
const lower = window.since?.value ?? "-infinity";
|
|
66840
|
+
const upper = window.until?.value ?? "+infinity";
|
|
66841
|
+
throw new WordpressTrafficApiError(
|
|
66842
|
+
`WordPress traffic endpoint returned an invalid or out-of-window observed_at for the requested [${lower}, ${upper}) window. Upgrade to a bounded-window-capable Canonry traffic-logger extension before syncing.`,
|
|
66843
|
+
502
|
|
66844
|
+
);
|
|
66845
|
+
}
|
|
66750
66846
|
async function listWordpressTrafficEvents(options) {
|
|
66751
66847
|
const endpoint = resolveEndpoint(options.baseUrl);
|
|
66752
66848
|
const authHeader = buildBasicAuthHeader(options.username, options.applicationPassword);
|
|
66753
66849
|
const pageSize = normalizePageSize2(options.pageSize);
|
|
66754
66850
|
const maxPages = normalizeMaxPages2(options.maxPages);
|
|
66755
66851
|
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
|
|
66852
|
+
const requestedWindow = parseRequestedWindow(options);
|
|
66756
66853
|
let cursor = options.cursor;
|
|
66757
66854
|
let rawEntryCount = 0;
|
|
66758
66855
|
let skippedEntryCount = 0;
|
|
@@ -66793,20 +66890,28 @@ async function listWordpressTrafficEvents(options) {
|
|
|
66793
66890
|
body2
|
|
66794
66891
|
);
|
|
66795
66892
|
}
|
|
66796
|
-
const body = await response
|
|
66797
|
-
const entries = body.events
|
|
66893
|
+
const body = parseResponseBody(await readResponseJson(response));
|
|
66894
|
+
const entries = body.events;
|
|
66798
66895
|
rawEntryCount += entries.length;
|
|
66799
66896
|
for (const entry of entries) {
|
|
66800
|
-
|
|
66897
|
+
assertEntryIsWithinRequestedWindow(entry, requestedWindow);
|
|
66898
|
+
if (!isRecord2(entry)) {
|
|
66899
|
+
skippedEntryCount += 1;
|
|
66900
|
+
continue;
|
|
66901
|
+
}
|
|
66902
|
+
const normalized = normalizeWordpressTrafficEvent(
|
|
66903
|
+
entry,
|
|
66904
|
+
body.site
|
|
66905
|
+
);
|
|
66801
66906
|
if (normalized) {
|
|
66802
66907
|
events.push(normalized);
|
|
66803
66908
|
} else {
|
|
66804
66909
|
skippedEntryCount += 1;
|
|
66805
66910
|
}
|
|
66806
66911
|
}
|
|
66807
|
-
cursor = body.
|
|
66808
|
-
hasMore =
|
|
66809
|
-
if (!
|
|
66912
|
+
cursor = body.nextCursor ?? void 0;
|
|
66913
|
+
hasMore = body.hasMore;
|
|
66914
|
+
if (!hasMore) break;
|
|
66810
66915
|
}
|
|
66811
66916
|
return {
|
|
66812
66917
|
events,
|
|
@@ -68470,8 +68575,7 @@ function tryClaimTrafficSyncLease(input) {
|
|
|
68470
68575
|
return input.db.transaction((tx) => {
|
|
68471
68576
|
const changed = tx.update(trafficSources).set({
|
|
68472
68577
|
syncLeaseOwner: input.owner,
|
|
68473
|
-
syncLeaseExpiresAt: expiresAt
|
|
68474
|
-
updatedAt: input.now
|
|
68578
|
+
syncLeaseExpiresAt: expiresAt
|
|
68475
68579
|
}).where(and42(
|
|
68476
68580
|
eq49(trafficSources.id, input.sourceId),
|
|
68477
68581
|
or11(
|
|
@@ -68487,8 +68591,7 @@ function releaseTrafficSyncLease(input) {
|
|
|
68487
68591
|
return input.db.transaction((tx) => {
|
|
68488
68592
|
const changed = tx.update(trafficSources).set({
|
|
68489
68593
|
syncLeaseOwner: null,
|
|
68490
|
-
syncLeaseExpiresAt: null
|
|
68491
|
-
updatedAt: input.now
|
|
68594
|
+
syncLeaseExpiresAt: null
|
|
68492
68595
|
}).where(and42(
|
|
68493
68596
|
eq49(trafficSources.id, input.sourceId),
|
|
68494
68597
|
eq49(trafficSources.syncLeaseOwner, input.owner)
|
|
@@ -68504,6 +68607,7 @@ var DEFAULT_MAX_PAGES4 = 5;
|
|
|
68504
68607
|
var DEFAULT_SAMPLE_LIMIT2 = 100;
|
|
68505
68608
|
var DEFAULT_WP_PAGE_SIZE = 500;
|
|
68506
68609
|
var DEFAULT_WP_MAX_PAGES = 20;
|
|
68610
|
+
var DEFAULT_WP_SYNC_WINDOW_MINUTES = 365 * 24 * 60;
|
|
68507
68611
|
var DEFAULT_VERCEL_MAX_PAGES = 50;
|
|
68508
68612
|
var VERCEL_MAX_SUB_WINDOWS = 5e3;
|
|
68509
68613
|
var VERCEL_BACKFILL_CHUNK_MS = 60 * 6e4;
|
|
@@ -68517,6 +68621,7 @@ var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
|
|
|
68517
68621
|
var CLOUDFLARE_QUEUE_RECEIPT_TTL_MS = 14 * 24 * 60 * 6e4 + 10 * 6e4;
|
|
68518
68622
|
var CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS = 5 * 6e4;
|
|
68519
68623
|
var CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS = 5 * 6e4;
|
|
68624
|
+
var WORDPRESS_SYNC_LEASE_TTL_MS = 5 * 6e4;
|
|
68520
68625
|
var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
|
|
68521
68626
|
var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
|
|
68522
68627
|
function sha256Hex3(value) {
|
|
@@ -68822,7 +68927,19 @@ function trafficConnectStatus(tx, projectId, existingSource) {
|
|
|
68822
68927
|
return existingSource?.status === TrafficSourceStatuses.paused || hasConnectedTrafficSourceSibling(tx, projectId, existingSource?.id ?? "") ? TrafficSourceStatuses.paused : TrafficSourceStatuses.connected;
|
|
68823
68928
|
}
|
|
68824
68929
|
function isSameTrafficSourceGeneration(current, started) {
|
|
68825
|
-
return isDeepStrictEqual(current.configJson, started.configJson) && current.updatedAt === started.updatedAt && current.lastSyncedAt === started.lastSyncedAt;
|
|
68930
|
+
return isDeepStrictEqual(current.configJson, started.configJson) && current.updatedAt === started.updatedAt && current.lastSyncedAt === started.lastSyncedAt && current.lastCursor === started.lastCursor && current.wordpressPendingUntil === started.wordpressPendingUntil;
|
|
68931
|
+
}
|
|
68932
|
+
function wordpressBaseUrlIdentity(value) {
|
|
68933
|
+
if (typeof value !== "string" || !value.trim()) return void 0;
|
|
68934
|
+
try {
|
|
68935
|
+
const url = new URL(value);
|
|
68936
|
+
url.hash = "";
|
|
68937
|
+
url.search = "";
|
|
68938
|
+
url.pathname = url.pathname.replace(/\/+$/, "") || "/";
|
|
68939
|
+
return url.toString();
|
|
68940
|
+
} catch {
|
|
68941
|
+
return void 0;
|
|
68942
|
+
}
|
|
68826
68943
|
}
|
|
68827
68944
|
function bindTrafficSyncSchedule(tx, projectId, sourceId, now, createIfMissing = true) {
|
|
68828
68945
|
const schedule = tx.select().from(schedules).where(and43(
|
|
@@ -68879,6 +68996,10 @@ async function runBackfillTask(options) {
|
|
|
68879
68996
|
} catch {
|
|
68880
68997
|
}
|
|
68881
68998
|
};
|
|
68999
|
+
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
69000
|
+
markFailed("Generic WordPress replace backfill is unavailable because retained coverage is unproven. Use a retention-aware repair that declares the unrecoverable span.");
|
|
69001
|
+
return;
|
|
69002
|
+
}
|
|
68882
69003
|
let allEvents;
|
|
68883
69004
|
try {
|
|
68884
69005
|
allEvents = await pullForBackfill();
|
|
@@ -68922,6 +69043,14 @@ async function runBackfillTask(options) {
|
|
|
68922
69043
|
try {
|
|
68923
69044
|
const commitOutcome = app.db.transaction((tx) => {
|
|
68924
69045
|
const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
|
|
69046
|
+
if (latestSource?.sourceType === TrafficSourceTypes.wordpress) {
|
|
69047
|
+
tx.update(runs).set({
|
|
69048
|
+
status: RunStatuses.failed,
|
|
69049
|
+
error: "Generic WordPress replace backfill is unavailable because retained coverage is unproven.",
|
|
69050
|
+
finishedAt
|
|
69051
|
+
}).where(eq50(runs.id, runId)).run();
|
|
69052
|
+
return "wordpress-backfill-unsupported";
|
|
69053
|
+
}
|
|
68925
69054
|
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
|
|
68926
69055
|
tx.update(runs).set({
|
|
68927
69056
|
status: RunStatuses.failed,
|
|
@@ -69142,7 +69271,7 @@ async function runBackfillTask(options) {
|
|
|
69142
69271
|
}).run();
|
|
69143
69272
|
}
|
|
69144
69273
|
const recordedSkipMs = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
69145
|
-
const skipRecovered = Number.isFinite(recordedSkipMs) && windowStart.getTime() <= recordedSkipMs;
|
|
69274
|
+
const skipRecovered = sourceRow2.sourceType !== TrafficSourceTypes.wordpress && Number.isFinite(recordedSkipMs) && windowStart.getTime() <= recordedSkipMs;
|
|
69146
69275
|
tx.update(trafficSources).set({
|
|
69147
69276
|
status: TrafficSourceStatuses.connected,
|
|
69148
69277
|
lastSyncedAt: nextLastSyncedAt,
|
|
@@ -69154,7 +69283,7 @@ async function runBackfillTask(options) {
|
|
|
69154
69283
|
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq50(runs.id, runId)).run();
|
|
69155
69284
|
return "committed";
|
|
69156
69285
|
});
|
|
69157
|
-
if (commitOutcome === "source-inactive") return;
|
|
69286
|
+
if (commitOutcome === "source-inactive" || commitOutcome === "wordpress-backfill-unsupported") return;
|
|
69158
69287
|
} catch (e) {
|
|
69159
69288
|
markFailed(`Backfill rollup write failed: ${describeError(e)}`);
|
|
69160
69289
|
}
|
|
@@ -69402,9 +69531,10 @@ async function trafficRoutes(app, opts) {
|
|
|
69402
69531
|
const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
|
|
69403
69532
|
const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
|
|
69404
69533
|
const activeSource = tx.select().from(trafficSources).where(eq50(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
|
|
69405
|
-
const
|
|
69534
|
+
const endpointChanged = activeSource !== void 0 && wordpressBaseUrlIdentity(activeSource.configJson.baseUrl) !== wordpressBaseUrlIdentity(baseUrl);
|
|
69535
|
+
const sourceId = endpointChanged ? crypto42.randomUUID() : activeSource?.id ?? crypto42.randomUUID();
|
|
69406
69536
|
const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
|
|
69407
|
-
if (activeSource) {
|
|
69537
|
+
if (activeSource && !endpointChanged) {
|
|
69408
69538
|
tx.update(trafficSources).set({
|
|
69409
69539
|
displayName: fallbackName,
|
|
69410
69540
|
status: sourceStatus,
|
|
@@ -69413,6 +69543,13 @@ async function trafficRoutes(app, opts) {
|
|
|
69413
69543
|
updatedAt: now
|
|
69414
69544
|
}).where(eq50(trafficSources.id, sourceId)).run();
|
|
69415
69545
|
} else {
|
|
69546
|
+
if (activeSource) {
|
|
69547
|
+
tx.update(trafficSources).set({
|
|
69548
|
+
status: TrafficSourceStatuses.archived,
|
|
69549
|
+
archivedAt: now,
|
|
69550
|
+
updatedAt: now
|
|
69551
|
+
}).where(eq50(trafficSources.id, activeSource.id)).run();
|
|
69552
|
+
}
|
|
69416
69553
|
tx.insert(trafficSources).values({
|
|
69417
69554
|
id: sourceId,
|
|
69418
69555
|
projectId: project.id,
|
|
@@ -70018,10 +70155,11 @@ async function trafficRoutes(app, opts) {
|
|
|
70018
70155
|
});
|
|
70019
70156
|
app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
|
|
70020
70157
|
const project = resolveProject(app.db, request.params.name);
|
|
70021
|
-
const
|
|
70022
|
-
if (!
|
|
70158
|
+
const initialSourceRow = app.db.select().from(trafficSources).where(eq50(trafficSources.id, request.params.id)).get();
|
|
70159
|
+
if (!initialSourceRow || initialSourceRow.projectId !== project.id) {
|
|
70023
70160
|
throw notFound("Traffic source", request.params.id);
|
|
70024
70161
|
}
|
|
70162
|
+
let sourceRow2 = initialSourceRow;
|
|
70025
70163
|
const queueConfig = sourceRow2.sourceType === TrafficSourceTypes.cloudflare ? parseQueuePullCloudflareSourceConfig(sourceRow2.configJson) : null;
|
|
70026
70164
|
if (queueConfig) {
|
|
70027
70165
|
if (sourceRow2.status !== TrafficSourceStatuses.connected) {
|
|
@@ -70047,19 +70185,19 @@ async function trafficRoutes(app, opts) {
|
|
|
70047
70185
|
})) {
|
|
70048
70186
|
throw operationInProgress("Cloudflare Queue source sync is already in progress", { sourceId: sourceRow2.id });
|
|
70049
70187
|
}
|
|
70050
|
-
const
|
|
70051
|
-
const
|
|
70188
|
+
const runId = crypto42.randomUUID();
|
|
70189
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70052
70190
|
const startedMs = Date.now();
|
|
70053
70191
|
try {
|
|
70054
70192
|
app.db.insert(runs).values({
|
|
70055
|
-
id:
|
|
70193
|
+
id: runId,
|
|
70056
70194
|
projectId: project.id,
|
|
70057
70195
|
kind: RunKinds["traffic-sync"],
|
|
70058
70196
|
status: RunStatuses.running,
|
|
70059
70197
|
trigger: RunTriggers.manual,
|
|
70060
70198
|
sourceId: sourceRow2.id,
|
|
70061
|
-
startedAt
|
|
70062
|
-
createdAt:
|
|
70199
|
+
startedAt,
|
|
70200
|
+
createdAt: startedAt
|
|
70063
70201
|
}).run();
|
|
70064
70202
|
let acceptedEvents = 0;
|
|
70065
70203
|
let selfTrafficExcluded = 0;
|
|
@@ -70067,11 +70205,11 @@ async function trafficRoutes(app, opts) {
|
|
|
70067
70205
|
let aiUserFetchHits = 0;
|
|
70068
70206
|
let aiReferralHits = 0;
|
|
70069
70207
|
let unknownHits = 0;
|
|
70070
|
-
let
|
|
70071
|
-
let
|
|
70072
|
-
let
|
|
70073
|
-
let
|
|
70074
|
-
let committedAt =
|
|
70208
|
+
let crawlerBucketRows = 0;
|
|
70209
|
+
let aiUserFetchBucketRows = 0;
|
|
70210
|
+
let aiReferralBucketRows = 0;
|
|
70211
|
+
let sampleRows = 0;
|
|
70212
|
+
let committedAt = startedAt;
|
|
70075
70213
|
let remainingBacklogCount = 0;
|
|
70076
70214
|
const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
|
|
70077
70215
|
for (let batch = 0; batch < cloudflareQueueMaxBatches; batch += 1) {
|
|
@@ -70190,14 +70328,14 @@ async function trafficRoutes(app, opts) {
|
|
|
70190
70328
|
aiUserFetchHits += writeResult.aiUserFetchHits;
|
|
70191
70329
|
aiReferralHits += writeResult.aiReferralHits;
|
|
70192
70330
|
unknownHits += writeResult.unknownHits;
|
|
70193
|
-
|
|
70194
|
-
|
|
70195
|
-
|
|
70196
|
-
|
|
70331
|
+
crawlerBucketRows += writeResult.crawlerBucketRows;
|
|
70332
|
+
aiUserFetchBucketRows += writeResult.aiUserFetchBucketRows;
|
|
70333
|
+
aiReferralBucketRows += writeResult.aiReferralBucketRows;
|
|
70334
|
+
sampleRows += writeResult.sampleRows;
|
|
70197
70335
|
const pulledEnvelopeCount = pulled.messages.length + pulled.skippedUnleasedMessageCount;
|
|
70198
70336
|
if (pulledEnvelopeCount < CLOUDFLARE_QUEUE_BATCH_SIZE || remainingBacklogCount === 0) break;
|
|
70199
70337
|
}
|
|
70200
|
-
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: committedAt }).where(eq50(runs.id,
|
|
70338
|
+
app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: committedAt }).where(eq50(runs.id, runId)).run();
|
|
70201
70339
|
writeAuditLog(app.db, {
|
|
70202
70340
|
projectId: project.id,
|
|
70203
70341
|
actor: "api",
|
|
@@ -70219,9 +70357,9 @@ async function trafficRoutes(app, opts) {
|
|
|
70219
70357
|
});
|
|
70220
70358
|
} catch {
|
|
70221
70359
|
}
|
|
70222
|
-
const
|
|
70360
|
+
const response = {
|
|
70223
70361
|
sourceId: sourceRow2.id,
|
|
70224
|
-
runId
|
|
70362
|
+
runId,
|
|
70225
70363
|
syncedAt: committedAt,
|
|
70226
70364
|
pulledEvents: acceptedEvents,
|
|
70227
70365
|
selfTrafficExcluded,
|
|
@@ -70229,20 +70367,20 @@ async function trafficRoutes(app, opts) {
|
|
|
70229
70367
|
aiUserFetchHits,
|
|
70230
70368
|
aiReferralHits,
|
|
70231
70369
|
unknownHits,
|
|
70232
|
-
crawlerBucketRows
|
|
70233
|
-
aiUserFetchBucketRows
|
|
70234
|
-
aiReferralBucketRows
|
|
70235
|
-
sampleRows
|
|
70370
|
+
crawlerBucketRows,
|
|
70371
|
+
aiUserFetchBucketRows,
|
|
70372
|
+
aiReferralBucketRows,
|
|
70373
|
+
sampleRows,
|
|
70236
70374
|
remainingBacklogCount,
|
|
70237
|
-
windowStart:
|
|
70375
|
+
windowStart: startedAt,
|
|
70238
70376
|
windowEnd: committedAt
|
|
70239
70377
|
};
|
|
70240
|
-
return
|
|
70378
|
+
return response;
|
|
70241
70379
|
} catch (error) {
|
|
70242
70380
|
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70243
70381
|
const safeError = error instanceof CloudflareQueueApiError ? error.message : "Cloudflare Queue sync failed; retry the active source.";
|
|
70244
70382
|
app.db.transaction((tx) => {
|
|
70245
|
-
tx.update(runs).set({ status: RunStatuses.failed, error: safeError, finishedAt: failedAt }).where(eq50(runs.id,
|
|
70383
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: safeError, finishedAt: failedAt }).where(eq50(runs.id, runId)).run();
|
|
70246
70384
|
const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
|
|
70247
70385
|
if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
|
|
70248
70386
|
tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and43(
|
|
@@ -70271,588 +70409,690 @@ async function trafficRoutes(app, opts) {
|
|
|
70271
70409
|
if (sourceRow2.status === TrafficSourceStatuses.paused || sourceRow2.status === TrafficSourceStatuses.archived || hasConnectedSibling) {
|
|
70272
70410
|
throw validationError("Traffic source must be active before it can sync");
|
|
70273
70411
|
}
|
|
70274
|
-
const
|
|
70275
|
-
|
|
70276
|
-
|
|
70277
|
-
const runId = crypto42.randomUUID();
|
|
70278
|
-
app.db.insert(runs).values({
|
|
70279
|
-
id: runId,
|
|
70280
|
-
projectId: project.id,
|
|
70281
|
-
kind: RunKinds["traffic-sync"],
|
|
70282
|
-
status: RunStatuses.running,
|
|
70283
|
-
trigger: RunTriggers.manual,
|
|
70412
|
+
const wordpressLeaseOwner = sourceRow2.sourceType === TrafficSourceTypes.wordpress ? crypto42.randomUUID() : void 0;
|
|
70413
|
+
if (wordpressLeaseOwner && !tryClaimTrafficSyncLease({
|
|
70414
|
+
db: app.db,
|
|
70284
70415
|
sourceId: sourceRow2.id,
|
|
70285
|
-
|
|
70286
|
-
|
|
70287
|
-
|
|
70288
|
-
|
|
70289
|
-
|
|
70290
|
-
|
|
70291
|
-
|
|
70292
|
-
|
|
70293
|
-
|
|
70294
|
-
|
|
70416
|
+
owner: wordpressLeaseOwner,
|
|
70417
|
+
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
70418
|
+
ttlMs: WORDPRESS_SYNC_LEASE_TTL_MS
|
|
70419
|
+
})) {
|
|
70420
|
+
throw operationInProgress("WordPress source sync is already in progress", { sourceId: sourceRow2.id });
|
|
70421
|
+
}
|
|
70422
|
+
try {
|
|
70423
|
+
const syncWindowEnd = /* @__PURE__ */ new Date();
|
|
70424
|
+
let windowEnd = syncWindowEnd;
|
|
70425
|
+
const startedAt = syncWindowEnd.toISOString();
|
|
70426
|
+
const syncStartedAtMs = syncWindowEnd.getTime();
|
|
70427
|
+
const runId = crypto42.randomUUID();
|
|
70428
|
+
app.db.insert(runs).values({
|
|
70429
|
+
id: runId,
|
|
70430
|
+
projectId: project.id,
|
|
70431
|
+
kind: RunKinds["traffic-sync"],
|
|
70432
|
+
status: RunStatuses.running,
|
|
70433
|
+
trigger: RunTriggers.manual,
|
|
70434
|
+
sourceId: sourceRow2.id,
|
|
70435
|
+
startedAt,
|
|
70436
|
+
createdAt: startedAt
|
|
70437
|
+
}).run();
|
|
70438
|
+
const markFailed = (msg, errorCode) => {
|
|
70439
|
+
const failedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70440
|
+
app.db.transaction((tx) => {
|
|
70441
|
+
tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq50(runs.id, runId)).run();
|
|
70442
|
+
const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
|
|
70443
|
+
if (latestSource?.status === TrafficSourceStatuses.connected && isSameTrafficSourceGeneration(latestSource, sourceRow2) && (wordpressLeaseOwner === void 0 || latestSource.syncLeaseOwner === wordpressLeaseOwner)) {
|
|
70444
|
+
tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq50(trafficSources.id, sourceRow2.id)).run();
|
|
70445
|
+
}
|
|
70446
|
+
});
|
|
70447
|
+
try {
|
|
70448
|
+
opts.onTrafficSynced?.({
|
|
70449
|
+
status: "failed",
|
|
70450
|
+
sourceType: sourceRow2.sourceType,
|
|
70451
|
+
sourceId: sourceRow2.id,
|
|
70452
|
+
pulledEvents: 0,
|
|
70453
|
+
selfTrafficExcluded: 0,
|
|
70454
|
+
crawlerHits: 0,
|
|
70455
|
+
aiUserFetchHits: 0,
|
|
70456
|
+
aiReferralHits: 0,
|
|
70457
|
+
durationMs: Date.now() - syncStartedAtMs,
|
|
70458
|
+
errorCode
|
|
70459
|
+
});
|
|
70460
|
+
} catch {
|
|
70295
70461
|
}
|
|
70296
|
-
}
|
|
70297
|
-
|
|
70298
|
-
|
|
70299
|
-
|
|
70300
|
-
|
|
70462
|
+
};
|
|
70463
|
+
const renewWordpressLease = () => {
|
|
70464
|
+
if (!wordpressLeaseOwner) return;
|
|
70465
|
+
if (!tryClaimTrafficSyncLease({
|
|
70466
|
+
db: app.db,
|
|
70301
70467
|
sourceId: sourceRow2.id,
|
|
70302
|
-
|
|
70303
|
-
|
|
70304
|
-
|
|
70305
|
-
|
|
70306
|
-
|
|
70307
|
-
|
|
70308
|
-
|
|
70309
|
-
|
|
70310
|
-
|
|
70311
|
-
|
|
70312
|
-
|
|
70313
|
-
|
|
70314
|
-
|
|
70315
|
-
|
|
70316
|
-
|
|
70317
|
-
|
|
70318
|
-
|
|
70319
|
-
|
|
70320
|
-
|
|
70321
|
-
|
|
70322
|
-
|
|
70323
|
-
|
|
70324
|
-
|
|
70325
|
-
|
|
70326
|
-
|
|
70327
|
-
|
|
70328
|
-
|
|
70329
|
-
|
|
70330
|
-
|
|
70331
|
-
|
|
70332
|
-
|
|
70333
|
-
|
|
70334
|
-
|
|
70335
|
-
|
|
70336
|
-
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
70337
|
-
const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
70338
|
-
windowStart = new Date(
|
|
70339
|
-
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
70340
|
-
);
|
|
70341
|
-
let accessToken;
|
|
70342
|
-
try {
|
|
70343
|
-
accessToken = await resolveAccessToken2(credential);
|
|
70344
|
-
} catch (e) {
|
|
70345
|
-
const msg = describeError(e);
|
|
70346
|
-
markFailed(msg, "PROVIDER_AUTH");
|
|
70347
|
-
throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
70348
|
-
}
|
|
70349
|
-
const isFirstSync = !sourceRow2.lastSyncedAt;
|
|
70350
|
-
try {
|
|
70351
|
-
const page = await pullEvents(accessToken, {
|
|
70352
|
-
gcpProjectId,
|
|
70353
|
-
serviceName,
|
|
70354
|
-
location,
|
|
70355
|
-
startTime: windowStart.toISOString(),
|
|
70356
|
-
endTime: windowEnd.toISOString(),
|
|
70357
|
-
pageSize,
|
|
70358
|
-
maxPages,
|
|
70359
|
-
firstSync: isFirstSync,
|
|
70360
|
-
requestUrlSubstrings: [project.canonicalDomain]
|
|
70361
|
-
});
|
|
70362
|
-
allEvents = page.events;
|
|
70363
|
-
} catch (e) {
|
|
70364
|
-
const msg = describeError(e);
|
|
70365
|
-
markFailed(msg, "PROVIDER_PULL");
|
|
70366
|
-
throw providerError(`Cloud Run pull failed: ${msg}`);
|
|
70367
|
-
}
|
|
70368
|
-
} else if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
70369
|
-
auditAction = "traffic.wordpress.synced";
|
|
70370
|
-
const credentialStore = opts.wordpressTrafficCredentialStore;
|
|
70371
|
-
if (!credentialStore) {
|
|
70372
|
-
throw validationError("WordPress traffic credential storage is not configured for this deployment");
|
|
70373
|
-
}
|
|
70374
|
-
const credential = credentialStore.getConnection(project.name);
|
|
70375
|
-
if (!credential) {
|
|
70376
|
-
app.db.delete(runs).where(eq50(runs.id, runId)).run();
|
|
70377
|
-
throw validationError(
|
|
70378
|
-
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
70468
|
+
owner: wordpressLeaseOwner,
|
|
70469
|
+
now: (/* @__PURE__ */ new Date()).toISOString(),
|
|
70470
|
+
ttlMs: WORDPRESS_SYNC_LEASE_TTL_MS
|
|
70471
|
+
})) {
|
|
70472
|
+
throw new Error("WordPress sync lease was lost");
|
|
70473
|
+
}
|
|
70474
|
+
};
|
|
70475
|
+
let windowStart;
|
|
70476
|
+
let allEvents;
|
|
70477
|
+
let nextCursor;
|
|
70478
|
+
let auditAction;
|
|
70479
|
+
let effectiveWindowEnd = windowEnd;
|
|
70480
|
+
if (sourceRow2.sourceType === TrafficSourceTypes["cloud-run"]) {
|
|
70481
|
+
auditAction = "traffic.cloud-run.synced";
|
|
70482
|
+
const credentialStore = opts.cloudRunCredentialStore;
|
|
70483
|
+
if (!credentialStore) {
|
|
70484
|
+
throw validationError("Cloud Run credential storage is not configured for this deployment");
|
|
70485
|
+
}
|
|
70486
|
+
const credential = credentialStore.getConnection(project.name);
|
|
70487
|
+
if (!credential) {
|
|
70488
|
+
throw validationError(
|
|
70489
|
+
`No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
|
|
70490
|
+
);
|
|
70491
|
+
}
|
|
70492
|
+
const config = parseSourceConfig(sourceRow2);
|
|
70493
|
+
const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
|
|
70494
|
+
const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
|
|
70495
|
+
const location = config.location ?? credential.location ?? void 0;
|
|
70496
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
70497
|
+
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
70498
|
+
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
70499
|
+
const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
70500
|
+
windowStart = new Date(
|
|
70501
|
+
Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
70379
70502
|
);
|
|
70380
|
-
|
|
70381
|
-
|
|
70382
|
-
|
|
70383
|
-
|
|
70384
|
-
|
|
70385
|
-
|
|
70386
|
-
|
|
70387
|
-
|
|
70388
|
-
const
|
|
70389
|
-
|
|
70390
|
-
|
|
70391
|
-
|
|
70392
|
-
|
|
70393
|
-
|
|
70394
|
-
|
|
70395
|
-
|
|
70396
|
-
|
|
70397
|
-
|
|
70398
|
-
|
|
70399
|
-
|
|
70400
|
-
cursor,
|
|
70401
|
-
pageSize: wpPageSize,
|
|
70402
|
-
maxPages: 1,
|
|
70403
|
-
dispatcher: pinnedDispatcher
|
|
70503
|
+
let accessToken;
|
|
70504
|
+
try {
|
|
70505
|
+
accessToken = await resolveAccessToken2(credential);
|
|
70506
|
+
} catch (e) {
|
|
70507
|
+
const msg = describeError(e);
|
|
70508
|
+
markFailed(msg, "PROVIDER_AUTH");
|
|
70509
|
+
throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
|
|
70510
|
+
}
|
|
70511
|
+
const isFirstSync = !sourceRow2.lastSyncedAt;
|
|
70512
|
+
try {
|
|
70513
|
+
const page = await pullEvents(accessToken, {
|
|
70514
|
+
gcpProjectId,
|
|
70515
|
+
serviceName,
|
|
70516
|
+
location,
|
|
70517
|
+
startTime: windowStart.toISOString(),
|
|
70518
|
+
endTime: windowEnd.toISOString(),
|
|
70519
|
+
pageSize,
|
|
70520
|
+
maxPages,
|
|
70521
|
+
firstSync: isFirstSync,
|
|
70522
|
+
requestUrlSubstrings: [project.canonicalDomain]
|
|
70404
70523
|
});
|
|
70405
|
-
|
|
70406
|
-
|
|
70407
|
-
|
|
70408
|
-
|
|
70409
|
-
|
|
70524
|
+
allEvents = page.events;
|
|
70525
|
+
} catch (e) {
|
|
70526
|
+
const msg = describeError(e);
|
|
70527
|
+
markFailed(msg, "PROVIDER_PULL");
|
|
70528
|
+
throw providerError(`Cloud Run pull failed: ${msg}`);
|
|
70410
70529
|
}
|
|
70411
|
-
|
|
70412
|
-
|
|
70413
|
-
|
|
70414
|
-
|
|
70415
|
-
|
|
70416
|
-
throw providerError(`WordPress pull failed: ${msg}`);
|
|
70417
|
-
} finally {
|
|
70418
|
-
await pinnedDispatcher.close().catch(() => {
|
|
70419
|
-
});
|
|
70420
|
-
}
|
|
70421
|
-
} else {
|
|
70422
|
-
auditAction = "traffic.vercel.synced";
|
|
70423
|
-
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
70424
|
-
if (!credentialStore) {
|
|
70425
|
-
app.db.delete(runs).where(eq50(runs.id, runId)).run();
|
|
70426
|
-
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
70427
|
-
}
|
|
70428
|
-
const credential = credentialStore.getConnection(project.name);
|
|
70429
|
-
if (!credential) {
|
|
70430
|
-
app.db.delete(runs).where(eq50(runs.id, runId)).run();
|
|
70431
|
-
throw validationError(
|
|
70432
|
-
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
70433
|
-
);
|
|
70434
|
-
}
|
|
70435
|
-
const config = parseSourceConfig(sourceRow2);
|
|
70436
|
-
const vercelProjectId = config.projectId ?? credential.projectId;
|
|
70437
|
-
const vercelTeamId = config.teamId ?? credential.teamId;
|
|
70438
|
-
const vercelEnvironment = config.environment ?? credential.environment;
|
|
70439
|
-
const requestedMinutes = request.body?.sinceMinutes;
|
|
70440
|
-
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
70441
|
-
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
70442
|
-
const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
70443
|
-
const clampedStartMs = Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs));
|
|
70444
|
-
const cappedStartMs = Math.max(clampedStartMs, windowEnd.getTime() - VERCEL_MAX_SYNC_WINDOW_MS);
|
|
70445
|
-
if (cappedStartMs > clampedStartMs) {
|
|
70446
|
-
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
70447
|
-
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
70448
|
-
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq50(trafficSources.id, sourceRow2.id)).run();
|
|
70449
|
-
request.log.warn(
|
|
70450
|
-
{
|
|
70451
|
-
sourceId: sourceRow2.id,
|
|
70452
|
-
requestedStart: new Date(clampedStartMs).toISOString(),
|
|
70453
|
-
cappedStart: new Date(cappedStartMs).toISOString()
|
|
70454
|
-
},
|
|
70455
|
-
"Vercel sync window exceeded the max single-sync span; clamped the start forward (older traffic skipped \u2014 run a backfill to recover it)"
|
|
70456
|
-
);
|
|
70457
|
-
}
|
|
70458
|
-
windowStart = new Date(cappedStartMs);
|
|
70459
|
-
try {
|
|
70460
|
-
const drained = await drainVercelTrafficEvents({
|
|
70461
|
-
pull: pullVercelEvents,
|
|
70462
|
-
token: credential.token,
|
|
70463
|
-
projectId: vercelProjectId,
|
|
70464
|
-
teamId: vercelTeamId,
|
|
70465
|
-
environment: vercelEnvironment,
|
|
70466
|
-
startDate: windowStart.getTime(),
|
|
70467
|
-
endDate: windowEnd.getTime(),
|
|
70468
|
-
pagesPerSubWindow: vercelMaxPages,
|
|
70469
|
-
maxSubWindows: VERCEL_MAX_SUB_WINDOWS,
|
|
70470
|
-
// Bound the drain's wall-clock so a dense/slow window can't run for
|
|
70471
|
-
// many minutes. On hit the drain stops and reports how far it got.
|
|
70472
|
-
deadlineMs: syncStartedAtMs + vercelSyncDeadlineMs
|
|
70473
|
-
});
|
|
70474
|
-
if (drained.retentionClamped) {
|
|
70475
|
-
throw vercelRetentionClampError(windowStart.getTime(), drained.effectiveStartMs);
|
|
70530
|
+
} else if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
70531
|
+
auditAction = "traffic.wordpress.synced";
|
|
70532
|
+
const credentialStore = opts.wordpressTrafficCredentialStore;
|
|
70533
|
+
if (!credentialStore) {
|
|
70534
|
+
throw validationError("WordPress traffic credential storage is not configured for this deployment");
|
|
70476
70535
|
}
|
|
70477
|
-
|
|
70478
|
-
|
|
70479
|
-
|
|
70480
|
-
|
|
70481
|
-
|
|
70536
|
+
const credential = credentialStore.getConnection(project.name);
|
|
70537
|
+
if (!credential) {
|
|
70538
|
+
app.db.delete(runs).where(eq50(runs.id, runId)).run();
|
|
70539
|
+
throw validationError(
|
|
70540
|
+
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
70541
|
+
);
|
|
70542
|
+
}
|
|
70543
|
+
if (sourceRow2.lastCursor && !sourceRow2.wordpressPendingUntil) {
|
|
70544
|
+
const message = "WordPress source has a legacy continuation cursor without a bounded pending window. Reset it explicitly before resuming sync.";
|
|
70545
|
+
markFailed(message, "INTERNAL");
|
|
70546
|
+
throw validationError(message);
|
|
70547
|
+
}
|
|
70548
|
+
if (sourceRow2.wordpressPendingUntil) {
|
|
70549
|
+
const pendingStartMs = sourceRow2.lastSyncedAt ? Date.parse(sourceRow2.lastSyncedAt) : Number.NaN;
|
|
70550
|
+
const pendingEndMs = Date.parse(sourceRow2.wordpressPendingUntil);
|
|
70551
|
+
if (!Number.isFinite(pendingStartMs) || !Number.isFinite(pendingEndMs) || pendingStartMs >= pendingEndMs || pendingEndMs > syncWindowEnd.getTime()) {
|
|
70552
|
+
const message = "WordPress source has an invalid pending sync window. Reset it explicitly before resuming sync.";
|
|
70553
|
+
markFailed(message, "INTERNAL");
|
|
70554
|
+
throw validationError(message);
|
|
70482
70555
|
}
|
|
70483
|
-
|
|
70484
|
-
|
|
70485
|
-
|
|
70486
|
-
|
|
70487
|
-
|
|
70488
|
-
|
|
70489
|
-
|
|
70490
|
-
|
|
70491
|
-
|
|
70556
|
+
windowStart = new Date(pendingStartMs);
|
|
70557
|
+
windowEnd = new Date(pendingEndMs);
|
|
70558
|
+
effectiveWindowEnd = windowEnd;
|
|
70559
|
+
} else {
|
|
70560
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
70561
|
+
const hasExplicitWindow = Number.isFinite(requestedMinutes) && requestedMinutes !== void 0 && requestedMinutes > 0;
|
|
70562
|
+
const windowMinutes = hasExplicitWindow ? Math.floor(requestedMinutes) : DEFAULT_WP_SYNC_WINDOW_MINUTES;
|
|
70563
|
+
const requestedStartMs = syncWindowEnd.getTime() - windowMinutes * 6e4;
|
|
70564
|
+
const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
70565
|
+
windowStart = new Date(
|
|
70566
|
+
Math.min(syncWindowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
|
|
70492
70567
|
);
|
|
70568
|
+
const windowStartIso2 = windowStart.toISOString();
|
|
70569
|
+
const windowEndIso2 = windowEnd.toISOString();
|
|
70570
|
+
const reservedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70571
|
+
const reservedSource = app.db.transaction((tx) => {
|
|
70572
|
+
const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
|
|
70573
|
+
if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2) || latestSource.syncLeaseOwner !== wordpressLeaseOwner || latestSource.lastCursor || latestSource.wordpressPendingUntil) return void 0;
|
|
70574
|
+
tx.update(trafficSources).set({
|
|
70575
|
+
lastSyncedAt: windowStartIso2,
|
|
70576
|
+
wordpressPendingUntil: windowEndIso2,
|
|
70577
|
+
updatedAt: reservedAt
|
|
70578
|
+
}).where(eq50(trafficSources.id, sourceRow2.id)).run();
|
|
70579
|
+
return tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
|
|
70580
|
+
}, { behavior: "immediate" });
|
|
70581
|
+
if (!reservedSource) {
|
|
70582
|
+
const message = "Traffic source changed while reserving the WordPress sync window; retry the sync.";
|
|
70583
|
+
markFailed(message, "INTERNAL");
|
|
70584
|
+
throw validationError(message);
|
|
70585
|
+
}
|
|
70586
|
+
sourceRow2 = reservedSource;
|
|
70493
70587
|
}
|
|
70494
|
-
|
|
70495
|
-
|
|
70496
|
-
|
|
70497
|
-
|
|
70498
|
-
|
|
70499
|
-
|
|
70500
|
-
|
|
70501
|
-
|
|
70588
|
+
const windowStartIso = windowStart.toISOString();
|
|
70589
|
+
const windowEndIso = windowEnd.toISOString();
|
|
70590
|
+
const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
|
|
70591
|
+
const wpMaxPages = opts.defaultWordpressMaxPages ?? DEFAULT_WP_MAX_PAGES;
|
|
70592
|
+
let pinnedDispatcher;
|
|
70593
|
+
try {
|
|
70594
|
+
renewWordpressLease();
|
|
70595
|
+
pinnedDispatcher = await assertWordpressTargetAllowed(credential.baseUrl);
|
|
70596
|
+
} catch (e) {
|
|
70597
|
+
const msg = describeError(e);
|
|
70598
|
+
markFailed(msg, "PROVIDER_PULL");
|
|
70599
|
+
throw e;
|
|
70600
|
+
}
|
|
70601
|
+
const collected = [];
|
|
70602
|
+
let cursor = sourceRow2.lastCursor ?? void 0;
|
|
70603
|
+
try {
|
|
70604
|
+
for (let page = 0; page < wpMaxPages; page += 1) {
|
|
70605
|
+
renewWordpressLease();
|
|
70606
|
+
const pageResult = await pullWordpressEvents({
|
|
70607
|
+
baseUrl: credential.baseUrl,
|
|
70608
|
+
username: credential.username,
|
|
70609
|
+
applicationPassword: credential.applicationPassword,
|
|
70610
|
+
cursor,
|
|
70611
|
+
pageSize: wpPageSize,
|
|
70612
|
+
maxPages: 1,
|
|
70613
|
+
since: windowStartIso,
|
|
70614
|
+
until: windowEndIso,
|
|
70615
|
+
dispatcher: pinnedDispatcher
|
|
70616
|
+
});
|
|
70617
|
+
collected.push(...pageResult.events);
|
|
70618
|
+
const previousCursor = cursor;
|
|
70619
|
+
cursor = pageResult.nextCursor;
|
|
70620
|
+
if (!pageResult.hasMore) {
|
|
70621
|
+
cursor = void 0;
|
|
70622
|
+
break;
|
|
70623
|
+
}
|
|
70624
|
+
if (!cursor || cursor === previousCursor) {
|
|
70625
|
+
throw new Error("WordPress traffic endpoint returned has_more without a new continuation cursor");
|
|
70626
|
+
}
|
|
70627
|
+
}
|
|
70628
|
+
allEvents = collected;
|
|
70629
|
+
nextCursor = cursor;
|
|
70630
|
+
} catch (e) {
|
|
70631
|
+
const msg = describeError(e);
|
|
70632
|
+
markFailed(msg, "PROVIDER_PULL");
|
|
70633
|
+
throw providerError(`WordPress pull failed: ${msg}`);
|
|
70634
|
+
} finally {
|
|
70635
|
+
await pinnedDispatcher.close().catch(() => {
|
|
70636
|
+
});
|
|
70637
|
+
}
|
|
70638
|
+
} else {
|
|
70639
|
+
auditAction = "traffic.vercel.synced";
|
|
70640
|
+
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
70641
|
+
if (!credentialStore) {
|
|
70642
|
+
app.db.delete(runs).where(eq50(runs.id, runId)).run();
|
|
70643
|
+
throw validationError("Vercel traffic credential storage is not configured for this deployment");
|
|
70644
|
+
}
|
|
70645
|
+
const credential = credentialStore.getConnection(project.name);
|
|
70646
|
+
if (!credential) {
|
|
70647
|
+
app.db.delete(runs).where(eq50(runs.id, runId)).run();
|
|
70648
|
+
throw validationError(
|
|
70649
|
+
`No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
|
|
70502
70650
|
);
|
|
70503
70651
|
}
|
|
70504
|
-
|
|
70652
|
+
const config = parseSourceConfig(sourceRow2);
|
|
70653
|
+
const vercelProjectId = config.projectId ?? credential.projectId;
|
|
70654
|
+
const vercelTeamId = config.teamId ?? credential.teamId;
|
|
70655
|
+
const vercelEnvironment = config.environment ?? credential.environment;
|
|
70656
|
+
const requestedMinutes = request.body?.sinceMinutes;
|
|
70657
|
+
const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
|
|
70658
|
+
const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
|
|
70659
|
+
const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
|
|
70660
|
+
const clampedStartMs = Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs));
|
|
70661
|
+
const cappedStartMs = Math.max(clampedStartMs, windowEnd.getTime() - VERCEL_MAX_SYNC_WINDOW_MS);
|
|
70662
|
+
if (cappedStartMs > clampedStartMs) {
|
|
70663
|
+
const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
|
|
70664
|
+
const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
|
|
70665
|
+
app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq50(trafficSources.id, sourceRow2.id)).run();
|
|
70505
70666
|
request.log.warn(
|
|
70506
70667
|
{
|
|
70507
70668
|
sourceId: sourceRow2.id,
|
|
70508
|
-
|
|
70509
|
-
|
|
70669
|
+
requestedStart: new Date(clampedStartMs).toISOString(),
|
|
70670
|
+
cappedStart: new Date(cappedStartMs).toISOString()
|
|
70510
70671
|
},
|
|
70511
|
-
"Vercel
|
|
70672
|
+
"Vercel sync window exceeded the max single-sync span; clamped the start forward (older traffic skipped \u2014 run a backfill to recover it)"
|
|
70512
70673
|
);
|
|
70513
70674
|
}
|
|
70514
|
-
|
|
70675
|
+
windowStart = new Date(cappedStartMs);
|
|
70676
|
+
try {
|
|
70677
|
+
const drained = await drainVercelTrafficEvents({
|
|
70678
|
+
pull: pullVercelEvents,
|
|
70679
|
+
token: credential.token,
|
|
70680
|
+
projectId: vercelProjectId,
|
|
70681
|
+
teamId: vercelTeamId,
|
|
70682
|
+
environment: vercelEnvironment,
|
|
70683
|
+
startDate: windowStart.getTime(),
|
|
70684
|
+
endDate: windowEnd.getTime(),
|
|
70685
|
+
pagesPerSubWindow: vercelMaxPages,
|
|
70686
|
+
maxSubWindows: VERCEL_MAX_SUB_WINDOWS,
|
|
70687
|
+
// Bound the drain's wall-clock so a dense/slow window can't run for
|
|
70688
|
+
// many minutes. On hit the drain stops and reports how far it got.
|
|
70689
|
+
deadlineMs: syncStartedAtMs + vercelSyncDeadlineMs
|
|
70690
|
+
});
|
|
70691
|
+
if (drained.retentionClamped) {
|
|
70692
|
+
throw vercelRetentionClampError(windowStart.getTime(), drained.effectiveStartMs);
|
|
70693
|
+
}
|
|
70694
|
+
if (drained.deadlineReached) {
|
|
70695
|
+
if (drained.drainedThroughMs <= windowStart.getTime()) {
|
|
70696
|
+
throw new Error(
|
|
70697
|
+
`sync exceeded its ${vercelSyncDeadlineMs}ms drain budget without completing any sub-window (request-logs slow or unavailable)`
|
|
70698
|
+
);
|
|
70699
|
+
}
|
|
70700
|
+
effectiveWindowEnd = new Date(drained.drainedThroughMs);
|
|
70701
|
+
request.log.warn(
|
|
70702
|
+
{
|
|
70703
|
+
sourceId: sourceRow2.id,
|
|
70704
|
+
drainedThrough: effectiveWindowEnd.toISOString(),
|
|
70705
|
+
requestedEnd: windowEnd.toISOString(),
|
|
70706
|
+
subWindows: drained.subWindowCount
|
|
70707
|
+
},
|
|
70708
|
+
"Vercel drain hit its time budget; committing the partial window and advancing to it \u2014 next sync resumes from here"
|
|
70709
|
+
);
|
|
70710
|
+
}
|
|
70711
|
+
if (drained.truncatedSliceCount > 0) {
|
|
70712
|
+
request.log.warn(
|
|
70713
|
+
{
|
|
70714
|
+
sourceId: sourceRow2.id,
|
|
70715
|
+
truncatedSlices: drained.truncatedSliceCount,
|
|
70716
|
+
sliceStarts: drained.truncatedSliceStartsMs.map((ms) => new Date(ms).toISOString())
|
|
70717
|
+
},
|
|
70718
|
+
"Vercel drain truncated dense one-second slice(s); ingested a sample and advanced past them"
|
|
70719
|
+
);
|
|
70720
|
+
}
|
|
70721
|
+
if (drained.deadlineSkippedSliceCount > 0) {
|
|
70722
|
+
request.log.warn(
|
|
70723
|
+
{
|
|
70724
|
+
sourceId: sourceRow2.id,
|
|
70725
|
+
skippedSlices: drained.deadlineSkippedSliceCount,
|
|
70726
|
+
sliceStarts: drained.deadlineSkippedSliceStartsMs.map((ms) => new Date(ms).toISOString())
|
|
70727
|
+
},
|
|
70728
|
+
"Vercel drain could not narrow a dense/slow head slice within its budget; skipped past it to guarantee progress"
|
|
70729
|
+
);
|
|
70730
|
+
}
|
|
70731
|
+
allEvents = drained.events;
|
|
70732
|
+
} catch (e) {
|
|
70733
|
+
const msg = describeError(e);
|
|
70734
|
+
markFailed(msg, "PROVIDER_PULL");
|
|
70735
|
+
throw providerError(`Vercel pull failed: ${msg}`);
|
|
70736
|
+
}
|
|
70737
|
+
}
|
|
70738
|
+
let crawlerBucketRows = 0;
|
|
70739
|
+
let aiUserFetchBucketRows = 0;
|
|
70740
|
+
let aiReferralBucketRows = 0;
|
|
70741
|
+
let sampleRows = 0;
|
|
70742
|
+
let finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70743
|
+
let pulledEventsCount = 0;
|
|
70744
|
+
let selfTrafficExcludedCount = 0;
|
|
70745
|
+
let crawlerHitsCount = 0;
|
|
70746
|
+
let aiUserFetchHitsCount = 0;
|
|
70747
|
+
let aiReferralHitsCount = 0;
|
|
70748
|
+
let unknownHitsCount = 0;
|
|
70749
|
+
try {
|
|
70750
|
+
renewWordpressLease();
|
|
70515
70751
|
} catch (e) {
|
|
70516
70752
|
const msg = describeError(e);
|
|
70517
|
-
markFailed(msg, "
|
|
70518
|
-
throw providerError(`
|
|
70519
|
-
}
|
|
70520
|
-
|
|
70521
|
-
|
|
70522
|
-
|
|
70523
|
-
|
|
70524
|
-
|
|
70525
|
-
|
|
70526
|
-
|
|
70527
|
-
|
|
70528
|
-
|
|
70529
|
-
|
|
70530
|
-
|
|
70531
|
-
|
|
70532
|
-
|
|
70533
|
-
|
|
70534
|
-
|
|
70535
|
-
|
|
70536
|
-
const
|
|
70537
|
-
|
|
70538
|
-
|
|
70539
|
-
|
|
70540
|
-
|
|
70541
|
-
|
|
70542
|
-
|
|
70543
|
-
|
|
70544
|
-
|
|
70545
|
-
|
|
70546
|
-
|
|
70547
|
-
|
|
70548
|
-
|
|
70549
|
-
|
|
70550
|
-
|
|
70551
|
-
|
|
70552
|
-
|
|
70553
|
-
|
|
70554
|
-
|
|
70555
|
-
|
|
70556
|
-
|
|
70557
|
-
|
|
70558
|
-
|
|
70559
|
-
|
|
70560
|
-
|
|
70561
|
-
sourceRow2.id,
|
|
70562
|
-
finishedAt
|
|
70563
|
-
);
|
|
70564
|
-
pulledEventsCount = report.totals.normalizedEvents;
|
|
70565
|
-
selfTrafficExcludedCount = report.totals.selfTrafficExcluded;
|
|
70566
|
-
crawlerHitsCount = report.totals.crawlerHits;
|
|
70567
|
-
aiUserFetchHitsCount = report.totals.aiUserFetchHits;
|
|
70568
|
-
aiReferralHitsCount = report.totals.aiReferralHits;
|
|
70569
|
-
unknownHitsCount = report.totals.unknownHits;
|
|
70570
|
-
for (const bucket of report.crawlerEventsHourly) {
|
|
70571
|
-
const status = bucket.status ?? 0;
|
|
70572
|
-
tx.insert(crawlerEventsHourly).values({
|
|
70573
|
-
projectId: project.id,
|
|
70574
|
-
sourceId: sourceRow2.id,
|
|
70575
|
-
tsHour: bucket.tsHour,
|
|
70576
|
-
botId: bucket.botId,
|
|
70577
|
-
operator: bucket.operator,
|
|
70578
|
-
verificationStatus: bucket.verificationStatus,
|
|
70579
|
-
pathNormalized: bucket.pathNormalized,
|
|
70580
|
-
status,
|
|
70581
|
-
hits: bucket.hits,
|
|
70582
|
-
sampledUserAgent: bucket.sampledUserAgent,
|
|
70583
|
-
createdAt: finishedAt,
|
|
70584
|
-
updatedAt: finishedAt
|
|
70585
|
-
}).onConflictDoUpdate({
|
|
70586
|
-
target: [
|
|
70587
|
-
crawlerEventsHourly.projectId,
|
|
70588
|
-
crawlerEventsHourly.sourceId,
|
|
70589
|
-
crawlerEventsHourly.tsHour,
|
|
70590
|
-
crawlerEventsHourly.botId,
|
|
70591
|
-
crawlerEventsHourly.verificationStatus,
|
|
70592
|
-
crawlerEventsHourly.pathNormalized,
|
|
70593
|
-
crawlerEventsHourly.status
|
|
70594
|
-
],
|
|
70595
|
-
set: {
|
|
70596
|
-
hits: sql22`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
70597
|
-
sampledUserAgent: bucket.sampledUserAgent,
|
|
70598
|
-
updatedAt: finishedAt
|
|
70599
|
-
}
|
|
70600
|
-
}).run();
|
|
70601
|
-
if (bucket.verificationManifest) {
|
|
70602
|
-
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
70753
|
+
markFailed(msg, "INTERNAL");
|
|
70754
|
+
throw providerError(`WordPress sync failed: ${msg}`);
|
|
70755
|
+
}
|
|
70756
|
+
const commitOutcome = app.db.transaction((tx) => {
|
|
70757
|
+
const latestRow = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
|
|
70758
|
+
const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq50(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
|
|
70759
|
+
if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || wordpressLeaseOwner !== void 0 && latestRow.syncLeaseOwner !== wordpressLeaseOwner || !isSameTrafficSourceGeneration(latestRow, sourceRow2)) {
|
|
70760
|
+
const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70761
|
+
tx.update(runs).set({
|
|
70762
|
+
status: RunStatuses.failed,
|
|
70763
|
+
error: "Traffic source was deactivated or reconfigured during sync",
|
|
70764
|
+
finishedAt: abortedAt
|
|
70765
|
+
}).where(eq50(runs.id, runId)).run();
|
|
70766
|
+
return "source-inactive";
|
|
70767
|
+
}
|
|
70768
|
+
const previousIds = latestRow.lastEventIds ?? [];
|
|
70769
|
+
const seenEventIds = new Set(previousIds);
|
|
70770
|
+
const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
|
|
70771
|
+
const newSorted = dedupedEvents.filter((e) => !isSelfTraffic(e)).sort((a, b) => a.observedAt < b.observedAt ? 1 : a.observedAt > b.observedAt ? -1 : 0).map((e) => e.eventId);
|
|
70772
|
+
const merged = [];
|
|
70773
|
+
const mergedSet = /* @__PURE__ */ new Set();
|
|
70774
|
+
for (const id of [...newSorted, ...previousIds]) {
|
|
70775
|
+
if (mergedSet.has(id)) continue;
|
|
70776
|
+
mergedSet.add(id);
|
|
70777
|
+
merged.push(id);
|
|
70778
|
+
if (merged.length >= MAX_TRACKED_EVENT_IDS) break;
|
|
70779
|
+
}
|
|
70780
|
+
const nextEventIds = merged;
|
|
70781
|
+
const report = buildTrafficProbeReport(dedupedEvents, { sampleLimit });
|
|
70782
|
+
finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
70783
|
+
const rawSampleCutoff = enforceRawEventSampleRetention(
|
|
70784
|
+
tx,
|
|
70785
|
+
sourceRow2.id,
|
|
70786
|
+
finishedAt
|
|
70787
|
+
);
|
|
70788
|
+
pulledEventsCount = report.totals.normalizedEvents;
|
|
70789
|
+
selfTrafficExcludedCount = report.totals.selfTrafficExcluded;
|
|
70790
|
+
crawlerHitsCount = report.totals.crawlerHits;
|
|
70791
|
+
aiUserFetchHitsCount = report.totals.aiUserFetchHits;
|
|
70792
|
+
aiReferralHitsCount = report.totals.aiReferralHits;
|
|
70793
|
+
unknownHitsCount = report.totals.unknownHits;
|
|
70794
|
+
for (const bucket of report.crawlerEventsHourly) {
|
|
70795
|
+
const status = bucket.status ?? 0;
|
|
70796
|
+
tx.insert(crawlerEventsHourly).values({
|
|
70603
70797
|
projectId: project.id,
|
|
70604
70798
|
sourceId: sourceRow2.id,
|
|
70605
70799
|
tsHour: bucket.tsHour,
|
|
70606
70800
|
botId: bucket.botId,
|
|
70801
|
+
operator: bucket.operator,
|
|
70607
70802
|
verificationStatus: bucket.verificationStatus,
|
|
70608
70803
|
pathNormalized: bucket.pathNormalized,
|
|
70609
70804
|
status,
|
|
70610
|
-
manifestId: bucket.verificationManifest.id,
|
|
70611
|
-
manifestJson: bucket.verificationManifest,
|
|
70612
70805
|
hits: bucket.hits,
|
|
70806
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
70613
70807
|
createdAt: finishedAt,
|
|
70614
70808
|
updatedAt: finishedAt
|
|
70615
70809
|
}).onConflictDoUpdate({
|
|
70616
70810
|
target: [
|
|
70617
|
-
|
|
70618
|
-
|
|
70619
|
-
|
|
70620
|
-
|
|
70621
|
-
|
|
70622
|
-
|
|
70623
|
-
|
|
70624
|
-
crawlerVerificationManifestsHourly.manifestId
|
|
70811
|
+
crawlerEventsHourly.projectId,
|
|
70812
|
+
crawlerEventsHourly.sourceId,
|
|
70813
|
+
crawlerEventsHourly.tsHour,
|
|
70814
|
+
crawlerEventsHourly.botId,
|
|
70815
|
+
crawlerEventsHourly.verificationStatus,
|
|
70816
|
+
crawlerEventsHourly.pathNormalized,
|
|
70817
|
+
crawlerEventsHourly.status
|
|
70625
70818
|
],
|
|
70626
70819
|
set: {
|
|
70627
|
-
hits: sql22`${
|
|
70628
|
-
|
|
70820
|
+
hits: sql22`${crawlerEventsHourly.hits} + ${bucket.hits}`,
|
|
70821
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
70629
70822
|
updatedAt: finishedAt
|
|
70630
70823
|
}
|
|
70631
70824
|
}).run();
|
|
70632
|
-
|
|
70633
|
-
|
|
70634
|
-
|
|
70635
|
-
|
|
70636
|
-
|
|
70637
|
-
|
|
70638
|
-
|
|
70639
|
-
|
|
70640
|
-
|
|
70641
|
-
|
|
70642
|
-
|
|
70643
|
-
|
|
70644
|
-
|
|
70645
|
-
|
|
70646
|
-
|
|
70647
|
-
|
|
70648
|
-
|
|
70649
|
-
|
|
70650
|
-
|
|
70651
|
-
|
|
70652
|
-
|
|
70653
|
-
|
|
70654
|
-
|
|
70655
|
-
|
|
70656
|
-
|
|
70657
|
-
|
|
70658
|
-
|
|
70659
|
-
|
|
70660
|
-
|
|
70661
|
-
|
|
70662
|
-
|
|
70663
|
-
updatedAt: finishedAt
|
|
70825
|
+
if (bucket.verificationManifest) {
|
|
70826
|
+
tx.insert(crawlerVerificationManifestsHourly).values({
|
|
70827
|
+
projectId: project.id,
|
|
70828
|
+
sourceId: sourceRow2.id,
|
|
70829
|
+
tsHour: bucket.tsHour,
|
|
70830
|
+
botId: bucket.botId,
|
|
70831
|
+
verificationStatus: bucket.verificationStatus,
|
|
70832
|
+
pathNormalized: bucket.pathNormalized,
|
|
70833
|
+
status,
|
|
70834
|
+
manifestId: bucket.verificationManifest.id,
|
|
70835
|
+
manifestJson: bucket.verificationManifest,
|
|
70836
|
+
hits: bucket.hits,
|
|
70837
|
+
createdAt: finishedAt,
|
|
70838
|
+
updatedAt: finishedAt
|
|
70839
|
+
}).onConflictDoUpdate({
|
|
70840
|
+
target: [
|
|
70841
|
+
crawlerVerificationManifestsHourly.projectId,
|
|
70842
|
+
crawlerVerificationManifestsHourly.sourceId,
|
|
70843
|
+
crawlerVerificationManifestsHourly.tsHour,
|
|
70844
|
+
crawlerVerificationManifestsHourly.botId,
|
|
70845
|
+
crawlerVerificationManifestsHourly.verificationStatus,
|
|
70846
|
+
crawlerVerificationManifestsHourly.pathNormalized,
|
|
70847
|
+
crawlerVerificationManifestsHourly.status,
|
|
70848
|
+
crawlerVerificationManifestsHourly.manifestId
|
|
70849
|
+
],
|
|
70850
|
+
set: {
|
|
70851
|
+
hits: sql22`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
70852
|
+
manifestJson: bucket.verificationManifest,
|
|
70853
|
+
updatedAt: finishedAt
|
|
70854
|
+
}
|
|
70855
|
+
}).run();
|
|
70664
70856
|
}
|
|
70665
|
-
|
|
70666
|
-
|
|
70667
|
-
|
|
70857
|
+
crawlerBucketRows += 1;
|
|
70858
|
+
}
|
|
70859
|
+
for (const bucket of report.aiUserFetchEventsHourly) {
|
|
70860
|
+
const status = bucket.status ?? 0;
|
|
70861
|
+
tx.insert(aiUserFetchEventsHourly).values({
|
|
70668
70862
|
projectId: project.id,
|
|
70669
70863
|
sourceId: sourceRow2.id,
|
|
70670
70864
|
tsHour: bucket.tsHour,
|
|
70671
70865
|
botId: bucket.botId,
|
|
70866
|
+
operator: bucket.operator,
|
|
70672
70867
|
verificationStatus: bucket.verificationStatus,
|
|
70673
70868
|
pathNormalized: bucket.pathNormalized,
|
|
70674
70869
|
status,
|
|
70675
|
-
manifestId: bucket.verificationManifest.id,
|
|
70676
|
-
manifestJson: bucket.verificationManifest,
|
|
70677
70870
|
hits: bucket.hits,
|
|
70871
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
70678
70872
|
createdAt: finishedAt,
|
|
70679
70873
|
updatedAt: finishedAt
|
|
70680
70874
|
}).onConflictDoUpdate({
|
|
70681
70875
|
target: [
|
|
70682
|
-
|
|
70683
|
-
|
|
70684
|
-
|
|
70685
|
-
|
|
70686
|
-
|
|
70687
|
-
|
|
70688
|
-
|
|
70689
|
-
aiUserFetchVerificationManifestsHourly.manifestId
|
|
70876
|
+
aiUserFetchEventsHourly.projectId,
|
|
70877
|
+
aiUserFetchEventsHourly.sourceId,
|
|
70878
|
+
aiUserFetchEventsHourly.tsHour,
|
|
70879
|
+
aiUserFetchEventsHourly.botId,
|
|
70880
|
+
aiUserFetchEventsHourly.verificationStatus,
|
|
70881
|
+
aiUserFetchEventsHourly.pathNormalized,
|
|
70882
|
+
aiUserFetchEventsHourly.status
|
|
70690
70883
|
],
|
|
70691
70884
|
set: {
|
|
70692
|
-
hits: sql22`${
|
|
70885
|
+
hits: sql22`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
|
|
70886
|
+
sampledUserAgent: bucket.sampledUserAgent,
|
|
70887
|
+
updatedAt: finishedAt
|
|
70888
|
+
}
|
|
70889
|
+
}).run();
|
|
70890
|
+
if (bucket.verificationManifest) {
|
|
70891
|
+
tx.insert(aiUserFetchVerificationManifestsHourly).values({
|
|
70892
|
+
projectId: project.id,
|
|
70893
|
+
sourceId: sourceRow2.id,
|
|
70894
|
+
tsHour: bucket.tsHour,
|
|
70895
|
+
botId: bucket.botId,
|
|
70896
|
+
verificationStatus: bucket.verificationStatus,
|
|
70897
|
+
pathNormalized: bucket.pathNormalized,
|
|
70898
|
+
status,
|
|
70899
|
+
manifestId: bucket.verificationManifest.id,
|
|
70693
70900
|
manifestJson: bucket.verificationManifest,
|
|
70901
|
+
hits: bucket.hits,
|
|
70902
|
+
createdAt: finishedAt,
|
|
70903
|
+
updatedAt: finishedAt
|
|
70904
|
+
}).onConflictDoUpdate({
|
|
70905
|
+
target: [
|
|
70906
|
+
aiUserFetchVerificationManifestsHourly.projectId,
|
|
70907
|
+
aiUserFetchVerificationManifestsHourly.sourceId,
|
|
70908
|
+
aiUserFetchVerificationManifestsHourly.tsHour,
|
|
70909
|
+
aiUserFetchVerificationManifestsHourly.botId,
|
|
70910
|
+
aiUserFetchVerificationManifestsHourly.verificationStatus,
|
|
70911
|
+
aiUserFetchVerificationManifestsHourly.pathNormalized,
|
|
70912
|
+
aiUserFetchVerificationManifestsHourly.status,
|
|
70913
|
+
aiUserFetchVerificationManifestsHourly.manifestId
|
|
70914
|
+
],
|
|
70915
|
+
set: {
|
|
70916
|
+
hits: sql22`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
|
|
70917
|
+
manifestJson: bucket.verificationManifest,
|
|
70918
|
+
updatedAt: finishedAt
|
|
70919
|
+
}
|
|
70920
|
+
}).run();
|
|
70921
|
+
}
|
|
70922
|
+
aiUserFetchBucketRows += 1;
|
|
70923
|
+
}
|
|
70924
|
+
for (const bucket of report.aiReferralEventsHourly) {
|
|
70925
|
+
const status = bucket.status ?? 0;
|
|
70926
|
+
tx.insert(aiReferralEventsHourly).values({
|
|
70927
|
+
projectId: project.id,
|
|
70928
|
+
sourceId: sourceRow2.id,
|
|
70929
|
+
tsHour: bucket.tsHour,
|
|
70930
|
+
product: bucket.product,
|
|
70931
|
+
operator: bucket.operator,
|
|
70932
|
+
sourceDomain: bucket.sourceDomain,
|
|
70933
|
+
evidenceType: bucket.evidenceType,
|
|
70934
|
+
landingPathNormalized: bucket.landingPathNormalized,
|
|
70935
|
+
status,
|
|
70936
|
+
sessionsOrHits: bucket.hits,
|
|
70937
|
+
paidSessionsOrHits: bucket.paidHits,
|
|
70938
|
+
organicSessionsOrHits: bucket.organicHits,
|
|
70939
|
+
usersEstimated: null,
|
|
70940
|
+
createdAt: finishedAt,
|
|
70941
|
+
updatedAt: finishedAt
|
|
70942
|
+
}).onConflictDoUpdate({
|
|
70943
|
+
target: [
|
|
70944
|
+
aiReferralEventsHourly.projectId,
|
|
70945
|
+
aiReferralEventsHourly.sourceId,
|
|
70946
|
+
aiReferralEventsHourly.tsHour,
|
|
70947
|
+
aiReferralEventsHourly.product,
|
|
70948
|
+
aiReferralEventsHourly.sourceDomain,
|
|
70949
|
+
aiReferralEventsHourly.evidenceType,
|
|
70950
|
+
aiReferralEventsHourly.landingPathNormalized,
|
|
70951
|
+
aiReferralEventsHourly.status
|
|
70952
|
+
],
|
|
70953
|
+
// All three counters accumulate. The class splits the measure, so a
|
|
70954
|
+
// colliding bucket adds its paid and organic arrivals to the ones
|
|
70955
|
+
// already there rather than overwriting a single label.
|
|
70956
|
+
set: {
|
|
70957
|
+
sessionsOrHits: sql22`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
70958
|
+
paidSessionsOrHits: sql22`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
70959
|
+
organicSessionsOrHits: sql22`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
70694
70960
|
updatedAt: finishedAt
|
|
70695
70961
|
}
|
|
70696
70962
|
}).run();
|
|
70963
|
+
aiReferralBucketRows += 1;
|
|
70697
70964
|
}
|
|
70698
|
-
|
|
70699
|
-
|
|
70700
|
-
|
|
70701
|
-
|
|
70702
|
-
|
|
70703
|
-
|
|
70704
|
-
|
|
70705
|
-
|
|
70706
|
-
|
|
70707
|
-
|
|
70708
|
-
|
|
70709
|
-
|
|
70710
|
-
|
|
70711
|
-
|
|
70712
|
-
|
|
70713
|
-
|
|
70714
|
-
|
|
70715
|
-
|
|
70716
|
-
|
|
70965
|
+
for (const sample of report.samples) {
|
|
70966
|
+
const sampleTimestamp = retainedRawEventSampleTimestamp(sample.observedAt, rawSampleCutoff);
|
|
70967
|
+
if (!sampleTimestamp) continue;
|
|
70968
|
+
const eventType = sample.crawler ? "crawler" : sample.aiUserFetch ? "ai_user_fetch" : sample.aiReferral ? "ai_referral" : "unknown";
|
|
70969
|
+
const refererHost = (() => {
|
|
70970
|
+
if (!sample.referer) return null;
|
|
70971
|
+
try {
|
|
70972
|
+
return new URL(sample.referer).hostname;
|
|
70973
|
+
} catch {
|
|
70974
|
+
return null;
|
|
70975
|
+
}
|
|
70976
|
+
})();
|
|
70977
|
+
tx.insert(rawEventSamples).values({
|
|
70978
|
+
id: crypto42.randomUUID(),
|
|
70979
|
+
projectId: project.id,
|
|
70980
|
+
sourceId: sourceRow2.id,
|
|
70981
|
+
ts: sampleTimestamp,
|
|
70982
|
+
eventType,
|
|
70983
|
+
ipHash: null,
|
|
70984
|
+
userAgent: sample.userAgent,
|
|
70985
|
+
pathNormalized: sample.pathNormalized,
|
|
70986
|
+
status: sample.status,
|
|
70987
|
+
refererHost,
|
|
70988
|
+
classifierDetailsJson: {
|
|
70989
|
+
crawler: sample.crawler,
|
|
70990
|
+
aiUserFetch: sample.aiUserFetch,
|
|
70991
|
+
aiReferral: sample.aiReferral
|
|
70992
|
+
},
|
|
70993
|
+
createdAt: finishedAt
|
|
70994
|
+
}).run();
|
|
70995
|
+
sampleRows += 1;
|
|
70996
|
+
}
|
|
70997
|
+
const nextWatermark = sourceRow2.sourceType === TrafficSourceTypes.wordpress && nextCursor ? windowStart : effectiveWindowEnd;
|
|
70998
|
+
const sourceUpdate = {
|
|
70999
|
+
status: TrafficSourceStatuses.connected,
|
|
71000
|
+
// Advance to nextWatermark, not finishedAt — events arriving at the
|
|
71001
|
+
// source between the window end and finishedAt aren't in this pull's
|
|
71002
|
+
// range. If we stored finishedAt, the next sync's clamp would skip past
|
|
71003
|
+
// them and they'd be lost. A capped WordPress drain retains its lower
|
|
71004
|
+
// bound; full pulls use effectiveWindowEnd, including a partial Vercel
|
|
71005
|
+
// boundary, so the next sync resumes exactly where this one left off.
|
|
71006
|
+
//
|
|
71007
|
+
// Never move the watermark BACKWARD. A sync already in flight when the
|
|
71008
|
+
// cursor is advanced out from under it (an operator reset, or a backfill
|
|
71009
|
+
// committing a later window) would otherwise commit its own older
|
|
71010
|
+
// window end and undo that advance — the source silently resumes from
|
|
71011
|
+
// the past and re-walks ground that was deliberately skipped. Backfill
|
|
71012
|
+
// has always guarded this; the incremental path did not, so a reset only
|
|
71013
|
+
// stuck if you first disabled the schedule and drained the in-flight run.
|
|
71014
|
+
lastSyncedAt: new Date(
|
|
71015
|
+
Math.max(
|
|
71016
|
+
latestRow.lastSyncedAt ? new Date(latestRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY,
|
|
71017
|
+
nextWatermark.getTime()
|
|
71018
|
+
)
|
|
71019
|
+
).toISOString(),
|
|
71020
|
+
lastError: null,
|
|
71021
|
+
lastEventIds: nextEventIds,
|
|
70717
71022
|
updatedAt: finishedAt
|
|
70718
|
-
}
|
|
70719
|
-
|
|
70720
|
-
|
|
70721
|
-
|
|
70722
|
-
|
|
70723
|
-
|
|
70724
|
-
|
|
70725
|
-
|
|
70726
|
-
|
|
70727
|
-
|
|
70728
|
-
|
|
70729
|
-
// All three counters accumulate. The class splits the measure, so a
|
|
70730
|
-
// colliding bucket adds its paid and organic arrivals to the ones
|
|
70731
|
-
// already there rather than overwriting a single label.
|
|
70732
|
-
set: {
|
|
70733
|
-
sessionsOrHits: sql22`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
|
|
70734
|
-
paidSessionsOrHits: sql22`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
|
|
70735
|
-
organicSessionsOrHits: sql22`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
|
|
70736
|
-
updatedAt: finishedAt
|
|
70737
|
-
}
|
|
70738
|
-
}).run();
|
|
70739
|
-
aiReferralBucketRows += 1;
|
|
71023
|
+
};
|
|
71024
|
+
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
71025
|
+
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
71026
|
+
sourceUpdate.wordpressPendingUntil = nextCursor ? windowEnd.toISOString() : null;
|
|
71027
|
+
}
|
|
71028
|
+
tx.update(trafficSources).set(sourceUpdate).where(eq50(trafficSources.id, sourceRow2.id)).run();
|
|
71029
|
+
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq50(runs.id, runId)).run();
|
|
71030
|
+
return "committed";
|
|
71031
|
+
}, { behavior: "immediate" });
|
|
71032
|
+
if (commitOutcome === "source-inactive") {
|
|
71033
|
+
throw validationError("Traffic source is no longer active; discarded the in-flight sync");
|
|
70740
71034
|
}
|
|
70741
|
-
|
|
70742
|
-
|
|
70743
|
-
|
|
70744
|
-
|
|
70745
|
-
|
|
70746
|
-
|
|
70747
|
-
|
|
70748
|
-
|
|
70749
|
-
|
|
70750
|
-
|
|
70751
|
-
|
|
70752
|
-
|
|
70753
|
-
tx.insert(rawEventSamples).values({
|
|
70754
|
-
id: crypto42.randomUUID(),
|
|
70755
|
-
projectId: project.id,
|
|
70756
|
-
sourceId: sourceRow2.id,
|
|
70757
|
-
ts: sampleTimestamp,
|
|
70758
|
-
eventType,
|
|
70759
|
-
ipHash: null,
|
|
70760
|
-
userAgent: sample.userAgent,
|
|
70761
|
-
pathNormalized: sample.pathNormalized,
|
|
70762
|
-
status: sample.status,
|
|
70763
|
-
refererHost,
|
|
70764
|
-
classifierDetailsJson: {
|
|
70765
|
-
crawler: sample.crawler,
|
|
70766
|
-
aiUserFetch: sample.aiUserFetch,
|
|
70767
|
-
aiReferral: sample.aiReferral
|
|
70768
|
-
},
|
|
70769
|
-
createdAt: finishedAt
|
|
70770
|
-
}).run();
|
|
70771
|
-
sampleRows += 1;
|
|
71035
|
+
writeAuditLog(app.db, {
|
|
71036
|
+
projectId: project.id,
|
|
71037
|
+
actor: "api",
|
|
71038
|
+
action: auditAction,
|
|
71039
|
+
entityType: "traffic_source",
|
|
71040
|
+
entityId: sourceRow2.id
|
|
71041
|
+
});
|
|
71042
|
+
if (selfTrafficExcludedCount > 0) {
|
|
71043
|
+
request.log.info(
|
|
71044
|
+
{ sourceId: sourceRow2.id, selfTrafficExcluded: selfTrafficExcludedCount },
|
|
71045
|
+
"Dropped Canonry self-traffic before rollup; excluded from pulledEvents"
|
|
71046
|
+
);
|
|
70772
71047
|
}
|
|
70773
|
-
|
|
70774
|
-
|
|
70775
|
-
|
|
70776
|
-
|
|
70777
|
-
|
|
70778
|
-
|
|
70779
|
-
|
|
70780
|
-
|
|
70781
|
-
|
|
70782
|
-
|
|
70783
|
-
|
|
70784
|
-
|
|
70785
|
-
|
|
70786
|
-
// the past and re-walks ground that was deliberately skipped. Backfill
|
|
70787
|
-
// has always guarded this; the incremental path did not, so a reset only
|
|
70788
|
-
// stuck if you first disabled the schedule and drained the in-flight run.
|
|
70789
|
-
lastSyncedAt: new Date(
|
|
70790
|
-
Math.max(
|
|
70791
|
-
latestRow.lastSyncedAt ? new Date(latestRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY,
|
|
70792
|
-
effectiveWindowEnd.getTime()
|
|
70793
|
-
)
|
|
70794
|
-
).toISOString(),
|
|
70795
|
-
lastError: null,
|
|
70796
|
-
lastEventIds: nextEventIds,
|
|
70797
|
-
updatedAt: finishedAt
|
|
70798
|
-
};
|
|
70799
|
-
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
70800
|
-
sourceUpdate.lastCursor = nextCursor ?? null;
|
|
71048
|
+
try {
|
|
71049
|
+
opts.onTrafficSynced?.({
|
|
71050
|
+
status: "completed",
|
|
71051
|
+
sourceType: sourceRow2.sourceType,
|
|
71052
|
+
sourceId: sourceRow2.id,
|
|
71053
|
+
pulledEvents: pulledEventsCount,
|
|
71054
|
+
selfTrafficExcluded: selfTrafficExcludedCount,
|
|
71055
|
+
crawlerHits: crawlerHitsCount,
|
|
71056
|
+
aiUserFetchHits: aiUserFetchHitsCount,
|
|
71057
|
+
aiReferralHits: aiReferralHitsCount,
|
|
71058
|
+
durationMs: Date.now() - syncStartedAtMs
|
|
71059
|
+
});
|
|
71060
|
+
} catch {
|
|
70801
71061
|
}
|
|
70802
|
-
|
|
70803
|
-
tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq50(runs.id, runId)).run();
|
|
70804
|
-
return "committed";
|
|
70805
|
-
});
|
|
70806
|
-
if (commitOutcome === "source-inactive") {
|
|
70807
|
-
throw validationError("Traffic source is no longer active; discarded the in-flight sync");
|
|
70808
|
-
}
|
|
70809
|
-
writeAuditLog(app.db, {
|
|
70810
|
-
projectId: project.id,
|
|
70811
|
-
actor: "api",
|
|
70812
|
-
action: auditAction,
|
|
70813
|
-
entityType: "traffic_source",
|
|
70814
|
-
entityId: sourceRow2.id
|
|
70815
|
-
});
|
|
70816
|
-
if (selfTrafficExcludedCount > 0) {
|
|
70817
|
-
request.log.info(
|
|
70818
|
-
{ sourceId: sourceRow2.id, selfTrafficExcluded: selfTrafficExcludedCount },
|
|
70819
|
-
"Dropped Canonry self-traffic before rollup; excluded from pulledEvents"
|
|
70820
|
-
);
|
|
70821
|
-
}
|
|
70822
|
-
try {
|
|
70823
|
-
opts.onTrafficSynced?.({
|
|
70824
|
-
status: "completed",
|
|
70825
|
-
sourceType: sourceRow2.sourceType,
|
|
71062
|
+
const response = {
|
|
70826
71063
|
sourceId: sourceRow2.id,
|
|
71064
|
+
runId,
|
|
71065
|
+
syncedAt: finishedAt,
|
|
70827
71066
|
pulledEvents: pulledEventsCount,
|
|
70828
71067
|
selfTrafficExcluded: selfTrafficExcludedCount,
|
|
70829
71068
|
crawlerHits: crawlerHitsCount,
|
|
70830
71069
|
aiUserFetchHits: aiUserFetchHitsCount,
|
|
70831
71070
|
aiReferralHits: aiReferralHitsCount,
|
|
70832
|
-
|
|
70833
|
-
|
|
70834
|
-
|
|
71071
|
+
unknownHits: unknownHitsCount,
|
|
71072
|
+
crawlerBucketRows,
|
|
71073
|
+
aiUserFetchBucketRows,
|
|
71074
|
+
aiReferralBucketRows,
|
|
71075
|
+
sampleRows,
|
|
71076
|
+
windowStart: windowStart.toISOString(),
|
|
71077
|
+
// The window actually synced: equals windowEnd on a full sync, or the
|
|
71078
|
+
// partial boundary when a Vercel drain stopped at its deadline.
|
|
71079
|
+
windowEnd: effectiveWindowEnd.toISOString()
|
|
71080
|
+
};
|
|
71081
|
+
return response;
|
|
71082
|
+
} finally {
|
|
71083
|
+
if (wordpressLeaseOwner) {
|
|
71084
|
+
try {
|
|
71085
|
+
releaseTrafficSyncLease({
|
|
71086
|
+
db: app.db,
|
|
71087
|
+
sourceId: sourceRow2.id,
|
|
71088
|
+
owner: wordpressLeaseOwner,
|
|
71089
|
+
now: (/* @__PURE__ */ new Date()).toISOString()
|
|
71090
|
+
});
|
|
71091
|
+
} catch (error) {
|
|
71092
|
+
request.log.warn({ sourceId: sourceRow2.id, error: describeError(error) }, "Failed to release WordPress sync lease");
|
|
71093
|
+
}
|
|
71094
|
+
}
|
|
70835
71095
|
}
|
|
70836
|
-
const response = {
|
|
70837
|
-
sourceId: sourceRow2.id,
|
|
70838
|
-
runId,
|
|
70839
|
-
syncedAt: finishedAt,
|
|
70840
|
-
pulledEvents: pulledEventsCount,
|
|
70841
|
-
selfTrafficExcluded: selfTrafficExcludedCount,
|
|
70842
|
-
crawlerHits: crawlerHitsCount,
|
|
70843
|
-
aiUserFetchHits: aiUserFetchHitsCount,
|
|
70844
|
-
aiReferralHits: aiReferralHitsCount,
|
|
70845
|
-
unknownHits: unknownHitsCount,
|
|
70846
|
-
crawlerBucketRows,
|
|
70847
|
-
aiUserFetchBucketRows,
|
|
70848
|
-
aiReferralBucketRows,
|
|
70849
|
-
sampleRows,
|
|
70850
|
-
windowStart: windowStart.toISOString(),
|
|
70851
|
-
// The window actually synced: equals windowEnd on a full sync, or the
|
|
70852
|
-
// partial boundary when a Vercel drain stopped at its deadline.
|
|
70853
|
-
windowEnd: effectiveWindowEnd.toISOString()
|
|
70854
|
-
};
|
|
70855
|
-
return response;
|
|
70856
71096
|
});
|
|
70857
71097
|
app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
|
|
70858
71098
|
const project = resolveProject(app.db, request.params.name);
|
|
@@ -70862,12 +71102,17 @@ async function trafficRoutes(app, opts) {
|
|
|
70862
71102
|
}
|
|
70863
71103
|
if (sourceRow2.sourceType !== TrafficSourceTypes["cloud-run"] && sourceRow2.sourceType !== TrafficSourceTypes.wordpress && sourceRow2.sourceType !== TrafficSourceTypes.vercel) {
|
|
70864
71104
|
throw validationError(
|
|
70865
|
-
`Backfill for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run
|
|
71105
|
+
`Backfill for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 generic replace backfill supports only cloud-run and vercel; WordPress requires a retention-aware repair.`
|
|
70866
71106
|
);
|
|
70867
71107
|
}
|
|
70868
71108
|
if (!isAuthoritativeTrafficSource(app.db, sourceRow2)) {
|
|
70869
71109
|
throw validationError("Traffic source must be active before it can backfill");
|
|
70870
71110
|
}
|
|
71111
|
+
if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
71112
|
+
throw validationError(
|
|
71113
|
+
"Generic WordPress replace backfill is unavailable because retained coverage is unproven. Use a retention-aware repair that declares the unrecoverable span."
|
|
71114
|
+
);
|
|
71115
|
+
}
|
|
70871
71116
|
const requestedDays = request.body?.days ?? DEFAULT_BACKFILL_DAYS;
|
|
70872
71117
|
if (!Number.isInteger(requestedDays) || requestedDays <= 0) {
|
|
70873
71118
|
throw validationError('"days" must be a positive integer');
|
|
@@ -70914,54 +71159,6 @@ async function trafficRoutes(app, opts) {
|
|
|
70914
71159
|
});
|
|
70915
71160
|
return page.events;
|
|
70916
71161
|
};
|
|
70917
|
-
} else if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
|
|
70918
|
-
const credentialStore = opts.wordpressTrafficCredentialStore;
|
|
70919
|
-
if (!credentialStore) {
|
|
70920
|
-
throw validationError("WordPress traffic credential storage is not configured for this deployment");
|
|
70921
|
-
}
|
|
70922
|
-
const credential = credentialStore.getConnection(project.name);
|
|
70923
|
-
if (!credential) {
|
|
70924
|
-
throw validationError(
|
|
70925
|
-
`No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
|
|
70926
|
-
);
|
|
70927
|
-
}
|
|
70928
|
-
await (await assertWordpressTargetAllowed(credential.baseUrl)).close().catch(() => {
|
|
70929
|
-
});
|
|
70930
|
-
const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
|
|
70931
|
-
pullErrorPrefix = "WordPress pull failed";
|
|
70932
|
-
pullForBackfill = async () => {
|
|
70933
|
-
const pinnedDispatcher = await assertWordpressTargetAllowed(credential.baseUrl);
|
|
70934
|
-
try {
|
|
70935
|
-
const collected = [];
|
|
70936
|
-
const windowStartIso = windowStart.toISOString();
|
|
70937
|
-
const windowEndIso = windowEnd.toISOString();
|
|
70938
|
-
let cursor = void 0;
|
|
70939
|
-
for (let page = 0; page < BACKFILL_MAX_PAGES; page += 1) {
|
|
70940
|
-
const pageResult = await pullWordpressEvents({
|
|
70941
|
-
baseUrl: credential.baseUrl,
|
|
70942
|
-
username: credential.username,
|
|
70943
|
-
applicationPassword: credential.applicationPassword,
|
|
70944
|
-
cursor,
|
|
70945
|
-
pageSize: wpPageSize,
|
|
70946
|
-
// Each call fetches a single page; the for-loop drives
|
|
70947
|
-
// continuation. Matches the WP sync path's pattern.
|
|
70948
|
-
maxPages: 1,
|
|
70949
|
-
since: windowStartIso,
|
|
70950
|
-
until: windowEndIso,
|
|
70951
|
-
dispatcher: pinnedDispatcher
|
|
70952
|
-
});
|
|
70953
|
-
collected.push(...pageResult.events);
|
|
70954
|
-
const previousCursor = cursor;
|
|
70955
|
-
cursor = pageResult.nextCursor;
|
|
70956
|
-
if (!pageResult.hasMore) break;
|
|
70957
|
-
if (!cursor || cursor === previousCursor) break;
|
|
70958
|
-
}
|
|
70959
|
-
return collected;
|
|
70960
|
-
} finally {
|
|
70961
|
-
await pinnedDispatcher.close().catch(() => {
|
|
70962
|
-
});
|
|
70963
|
-
}
|
|
70964
|
-
};
|
|
70965
71162
|
} else {
|
|
70966
71163
|
const credentialStore = opts.vercelTrafficCredentialStore;
|
|
70967
71164
|
if (!credentialStore) {
|
|
@@ -71147,6 +71344,11 @@ async function trafficRoutes(app, opts) {
|
|
|
71147
71344
|
validateTrafficSourceCredential(sourceRow2, project.name);
|
|
71148
71345
|
tx.update(trafficSources).set({
|
|
71149
71346
|
lastSyncedAt: now,
|
|
71347
|
+
...sourceRow2.sourceType === TrafficSourceTypes.wordpress ? {
|
|
71348
|
+
lastCursor: null,
|
|
71349
|
+
wordpressPendingUntil: null,
|
|
71350
|
+
skippedThroughAt: now
|
|
71351
|
+
} : {},
|
|
71150
71352
|
status: TrafficSourceStatuses.connected,
|
|
71151
71353
|
lastError: null,
|
|
71152
71354
|
updatedAt: now
|
|
@@ -73651,11 +73853,17 @@ var syncLagCheck = {
|
|
|
73651
73853
|
};
|
|
73652
73854
|
const unrecovered = measured.filter((m) => m.skippedThroughAt !== null);
|
|
73653
73855
|
if (unrecovered.length > 0) {
|
|
73856
|
+
const unrecoveredWordpress = unrecovered.filter((m) => m.sourceType === "wordpress");
|
|
73857
|
+
const unrecoveredOtherSources = unrecovered.filter((m) => m.sourceType !== "wordpress");
|
|
73858
|
+
const remediation = [
|
|
73859
|
+
unrecoveredWordpress.length > 0 ? "For WordPress, do not run generic replace-mode backfill: the plugin feed cannot prove it still retains every bucket in the gap. Keep existing rollups intact and use a retention-aware repair that explicitly declares the unrecoverable span." : null,
|
|
73860
|
+
unrecoveredOtherSources.length > 0 ? "Recover each non-WordPress gap with `canonry traffic backfill <project> --source <id> --days N --wait`, choosing N so the window reaches back past the skipped instant and stays inside the provider's log retention. This clears only when a backfill actually covers the span." : null
|
|
73861
|
+
].filter((part) => part !== null).join(" ");
|
|
73654
73862
|
return {
|
|
73655
73863
|
status: CheckStatuses.fail,
|
|
73656
73864
|
code: "traffic.sync-lag.unrecovered-skip",
|
|
73657
73865
|
summary: `${unrecovered.length} traffic source(s) previously skipped traffic that has not been backfilled: ${unrecovered.map((m) => `${m.displayName} (skipped through ${m.skippedThroughAt})`).join(", ")}. Current lag may look healthy; the gap does not close on its own.`,
|
|
73658
|
-
remediation
|
|
73866
|
+
remediation,
|
|
73659
73867
|
details
|
|
73660
73868
|
};
|
|
73661
73869
|
}
|
|
@@ -77160,6 +77368,8 @@ async function apiRoutes(app, opts) {
|
|
|
77160
77368
|
resolveCloudRunAccessToken: opts.resolveCloudRunAccessToken,
|
|
77161
77369
|
wordpressTrafficCredentialStore: opts.wordpressTrafficCredentialStore,
|
|
77162
77370
|
pullWordpressTrafficEvents: opts.pullWordpressTrafficEvents,
|
|
77371
|
+
defaultWordpressPageSize: opts.defaultWordpressPageSize,
|
|
77372
|
+
defaultWordpressMaxPages: opts.defaultWordpressMaxPages,
|
|
77163
77373
|
vercelTrafficCredentialStore: opts.vercelTrafficCredentialStore,
|
|
77164
77374
|
pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
|
|
77165
77375
|
vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,
|