@canonry/canonry 4.177.0 → 4.177.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -630,7 +630,7 @@ import {
630
630
  wordpressSchemaDeployResultDtoSchema,
631
631
  wordpressSchemaStatusResultDtoSchema,
632
632
  wordpressStatusDtoSchema
633
- } from "./chunk-UXXXKDW4.js";
633
+ } from "./chunk-PQT3ECM3.js";
634
634
 
635
635
  // src/intelligence-service.ts
636
636
  import { eq as eq61, desc as desc27, asc as asc11, and as and50, ne as ne8, or as or14, inArray as inArray21, gte as gte14, lte as lte12 } from "drizzle-orm";
@@ -2242,6 +2242,11 @@ var trafficSources = sqliteTable("traffic_sources", {
2242
2242
  status: text("status").notNull(),
2243
2243
  lastSyncedAt: text("last_synced_at"),
2244
2244
  lastCursor: text("last_cursor"),
2245
+ // A WordPress cursor is valid only inside the fixed [lastSyncedAt,
2246
+ // wordpressPendingUntil) window that created it. NULL means no such window
2247
+ // is pending; a non-NULL cursor with a NULL end is legacy state that cannot
2248
+ // safely compose with a bounded incremental pull.
2249
+ wordpressPendingUntil: text("wordpress_pending_until"),
2245
2250
  lastError: text("last_error"),
2246
2251
  // JSON-encoded array of normalized event IDs (e.g. `cloud-run:<ts>:<insertId>`)
2247
2252
  // observed in the most recent successful sync. Bounded ring buffer used to
@@ -6849,6 +6854,18 @@ var MIGRATION_VERSIONS = [
6849
6854
  statements: [
6850
6855
  `CREATE INDEX IF NOT EXISTS idx_raw_event_samples_ts ON raw_event_samples(ts)`
6851
6856
  ]
6857
+ },
6858
+ {
6859
+ version: 145,
6860
+ name: "wordpress-traffic-pending-window",
6861
+ // A persisted upper bound makes a capped WordPress cursor drain a finite,
6862
+ // repeatable [last_synced_at, wordpress_pending_until) window. Existing
6863
+ // non-null cursors deliberately receive NULL here: they were created by
6864
+ // the old unbounded route, so treating their already-advanced watermark
6865
+ // as a lower bound would silently skip undrained history.
6866
+ statements: [
6867
+ `ALTER TABLE traffic_sources ADD COLUMN wordpress_pending_until TEXT`
6868
+ ]
6852
6869
  }
6853
6870
  ];
6854
6871
  function addRunsMeasurementPlanVersionForeignKey(tx) {
@@ -35387,7 +35404,7 @@ var routeCatalog = [
35387
35404
  method: "post",
35388
35405
  path: "/api/v1/projects/{name}/traffic/connect/wordpress",
35389
35406
  summary: "Connect a WordPress traffic-logger source",
35390
- description: "Probes the WordPress traffic-logger plugin endpoint with the supplied Application Password (single page, `limit=1`) before persisting. On success, stores the credential in `~/.canonry/config.yaml` and creates / updates the project's active WordPress `traffic_sources` row. A probe failure (HTTP 4xx/5xx, network error) surfaces as 502 with the upstream status in the message so the caller learns about a bad credential up front instead of at the first sync.",
35407
+ description: "Probes the WordPress traffic-logger plugin endpoint with the supplied Application Password (single page, `limit=1`) before persisting. On success, stores the credential in `~/.canonry/config.yaml` and creates / updates the project's active WordPress `traffic_sources` row. Changing `baseUrl` archives the old source lineage and creates a fresh source so historical rollups cannot mix across endpoints. A probe failure (HTTP 4xx/5xx, network error) surfaces as 502 with the upstream status in the message so the caller learns about a bad credential up front instead of at the first sync.",
35391
35408
  tags: ["traffic"],
35392
35409
  parameters: [nameParameter],
35393
35410
  requestBody: {
@@ -35514,7 +35531,7 @@ var routeCatalog = [
35514
35531
  method: "post",
35515
35532
  path: "/api/v1/projects/{name}/traffic/sources/{id}/sync",
35516
35533
  summary: "Trigger a sync run for a traffic source",
35517
- description: "Pulls from the selected Cloud Run, WordPress, Vercel, or Cloudflare Queue source, classifies crawler hits / user fetches / AI-referral sessions, and commits hourly buckets plus a bounded sample tail. Queue pull uses a durable source lease and acknowledges each Cloudflare message only after its event receipts and rollups commit.",
35534
+ description: "Pulls from the selected Cloud Run, WordPress, Vercel, or Cloudflare Queue source, classifies crawler hits / user fetches / AI-referral sessions, and commits hourly buckets plus a bounded sample tail. WordPress uses a fixed half-open `[since, until)` window and a source lease; its first or idle sync defaults to the plugin maximum retention horizon (365d) unless `sinceMinutes` is supplied. Queue pull also uses a durable source lease and acknowledges each Cloudflare message only after its event receipts and rollups commit.",
35518
35535
  tags: ["traffic"],
35519
35536
  parameters: [
35520
35537
  nameParameter,
@@ -35527,7 +35544,7 @@ var routeCatalog = [
35527
35544
  schema: {
35528
35545
  type: "object",
35529
35546
  properties: {
35530
- sinceMinutes: { ...integerSchema, description: "Optional lookback for time-window sources; Cloudflare Queue pull ignores it." }
35547
+ sinceMinutes: { ...integerSchema, description: "Optional lookback for time-window sources. Defaults are adapter-specific; a new or idle WordPress source uses 365d to cover the plugin\u2019s maximum configurable retention. Cloudflare Queue pull ignores it." }
35531
35548
  }
35532
35549
  }
35533
35550
  }
@@ -35536,7 +35553,7 @@ var routeCatalog = [
35536
35553
  responses: {
35537
35554
  200: jsonResponse("Sync summary returned.", "TrafficSyncResponse"),
35538
35555
  400: errorResponse("Invalid sync request or missing credentials."),
35539
- 409: errorResponse("Another Queue sync currently owns the source lease."),
35556
+ 409: errorResponse("Another WordPress or Queue sync currently owns the source lease."),
35540
35557
  404: errorResponse("Project or traffic source not found."),
35541
35558
  502: errorResponse("Upstream pull, acknowledgement, or credential resolution failed.")
35542
35559
  }
@@ -35545,7 +35562,7 @@ var routeCatalog = [
35545
35562
  method: "post",
35546
35563
  path: "/api/v1/projects/{name}/traffic/sources/{id}/backfill",
35547
35564
  summary: "Reclassify historical traffic-source logs",
35548
- description: 'Async one-shot backfill: pulls the last `days` of events (clamped server-side to the upstream retention ceiling \u2014 30d for Cloud Logging `_Default`; the WordPress plugin honours the same window via `since`/`until` query params), classifies them with the current rules, and replaces the hourly rollup buckets + sample slice in the window inside one transaction. Returns immediately with `{ runId, status: "running" }`; poll `GET /runs/{id}` for completion. lastSyncedAt only advances forward, so a backfill never undoes incremental sync progress that ran ahead of it. Supported source types: `cloud-run`, `wordpress`, `vercel`.',
35565
+ description: 'Async one-shot reclassification: pulls the last `days` of events, classifies them with current rules, and replaces hourly rollup buckets plus the sample slice in that window inside one transaction. Adapter limits apply (Cloud Logging `_Default` is typically 30d; request payloads are capped at 90d). Generic replace backfill supports `cloud-run` and `vercel`. WordPress is always rejected because its retained event feed cannot prove it covers every bucket; use a retention-aware repair that explicitly declares any unrecoverable span. Returns immediately with `{ runId, status: "running" }`; poll `GET /runs/{id}` for completion. lastSyncedAt only advances forward.',
35549
35566
  tags: ["traffic"],
35550
35567
  parameters: [
35551
35568
  nameParameter,
@@ -35558,7 +35575,7 @@ var routeCatalog = [
35558
35575
  schema: {
35559
35576
  type: "object",
35560
35577
  properties: {
35561
- days: { ...integerSchema, description: "Lookback window in days (default 30, capped at the upstream retention ceiling)." }
35578
+ days: { ...integerSchema, description: "Lookback window in days (default 30, capped by the adapter; generic WordPress replace backfill is unavailable)." }
35562
35579
  }
35563
35580
  }
35564
35581
  }
@@ -35574,7 +35591,7 @@ var routeCatalog = [
35574
35591
  method: "post",
35575
35592
  path: "/api/v1/projects/{name}/traffic/sources/{id}/reset",
35576
35593
  summary: "Advance lastSyncedAt to NOW and clear the error state",
35577
- description: "Operator recovery: advances `lastSyncedAt` to NOW, sets `status` back to `connected`, and clears `last_error`. Accepts any non-archived source \u2014 the `lastSyncedAt` advance determines the next sync window for time-windowed sources (Vercel, Cloud Run) and is informational for cursor-based sources (WordPress, where `last_cursor` governs the next drain and is preserved). Common trigger: an idle Vercel/Cloud Run source whose `lastSyncedAt` has aged past the upstream retention window (`request-logs` ~14d, Cloud Logging 30d) and now throws on every sync. Any pre-existing rollup history stays in place; the skipped history is the explicit trade-off \u2014 run `traffic backfill` separately to recover any of it. `advanceToNow: true` is required (no implicit reset). Archived sources are rejected with 400 \u2014 re-connect them via the appropriate `traffic/connect/*` endpoint instead.",
35594
+ description: "Operator recovery: advances `lastSyncedAt` to NOW, sets `status` back to `connected`, and clears `last_error`. WordPress resets also clear `last_cursor` and the pending-window marker, then record an unrecovered skip through the reset instant so the next drain cannot combine an old cursor with a new window. That prevents replay but is not a historical repair. Generic WordPress replace-mode backfill is unavailable because the plugin feed cannot prove retained coverage; use a retention-aware repair and explicitly declare any unrecoverable span. Common trigger: an idle Vercel/Cloud Run source whose `lastSyncedAt` has aged past upstream retention (`request-logs` ~14d, Cloud Logging 30d) and now throws on every sync. `advanceToNow: true` is required (no implicit reset). Archived sources are rejected with 400 \u2014 re-connect them via the appropriate `traffic/connect/*` endpoint instead.",
35578
35595
  tags: ["traffic"],
35579
35596
  parameters: [
35580
35597
  nameParameter,
@@ -66747,12 +66764,92 @@ async function readErrorBody2(response) {
66747
66764
  if (!text2) return void 0;
66748
66765
  return text2.length <= 500 ? text2 : `${text2.slice(0, 500)}... [truncated]`;
66749
66766
  }
66767
+ function isRecord2(value) {
66768
+ return typeof value === "object" && value !== null && !Array.isArray(value);
66769
+ }
66770
+ function parseRequestedBoundary(name, value) {
66771
+ if (value === void 0 || value === "") return void 0;
66772
+ const timestampMs = Date.parse(value);
66773
+ if (!Number.isFinite(timestampMs)) {
66774
+ throw new WordpressTrafficApiError(`${name} must be a valid ISO 8601 timestamp`, 400);
66775
+ }
66776
+ return { value, timestampMs };
66777
+ }
66778
+ function parseRequestedWindow(options) {
66779
+ const since = parseRequestedBoundary("since", options.since);
66780
+ const until = parseRequestedBoundary("until", options.until);
66781
+ if (since && until && since.timestampMs >= until.timestampMs) {
66782
+ throw new WordpressTrafficApiError("since must be earlier than until", 400);
66783
+ }
66784
+ return { since, until };
66785
+ }
66786
+ function parseSiteMetadata(value) {
66787
+ if (!isRecord2(value)) return void 0;
66788
+ return {
66789
+ url: typeof value.url === "string" ? value.url : void 0,
66790
+ anonymous_id: typeof value.anonymous_id === "string" ? value.anonymous_id : void 0,
66791
+ wordpress_version: typeof value.wordpress_version === "string" ? value.wordpress_version : void 0,
66792
+ plugin_version: typeof value.plugin_version === "string" ? value.plugin_version : void 0
66793
+ };
66794
+ }
66795
+ function invalidResponse(message) {
66796
+ return new WordpressTrafficApiError(
66797
+ `WordPress traffic endpoint returned an invalid response: ${message}`,
66798
+ 502
66799
+ );
66800
+ }
66801
+ function parseResponseBody(value) {
66802
+ if (!isRecord2(value)) {
66803
+ throw invalidResponse("expected an object body");
66804
+ }
66805
+ if (!Array.isArray(value.events)) {
66806
+ throw invalidResponse("events must be an array");
66807
+ }
66808
+ if (typeof value.has_more !== "boolean") {
66809
+ throw invalidResponse("has_more must be a boolean");
66810
+ }
66811
+ const nextCursor = value.next_cursor;
66812
+ if (value.has_more) {
66813
+ if (typeof nextCursor !== "string" || nextCursor.trim().length === 0) {
66814
+ throw invalidResponse("has_more=true requires a nonempty next_cursor");
66815
+ }
66816
+ } else if (nextCursor !== null) {
66817
+ throw invalidResponse("has_more=false requires next_cursor=null");
66818
+ }
66819
+ return {
66820
+ events: value.events,
66821
+ nextCursor,
66822
+ hasMore: value.has_more,
66823
+ site: parseSiteMetadata(value.site)
66824
+ };
66825
+ }
66826
+ async function readResponseJson(response) {
66827
+ try {
66828
+ return await response.json();
66829
+ } catch {
66830
+ throw invalidResponse("body is not valid JSON");
66831
+ }
66832
+ }
66833
+ function assertEntryIsWithinRequestedWindow(entry, window) {
66834
+ if (!window.since && !window.until) return;
66835
+ const observedAt = isRecord2(entry) && typeof entry.observed_at === "string" ? entry.observed_at : void 0;
66836
+ const observedAtMs = observedAt === void 0 ? Number.NaN : Date.parse(observedAt);
66837
+ const outsideWindow = !Number.isFinite(observedAtMs) || window.since !== void 0 && observedAtMs < window.since.timestampMs || window.until !== void 0 && observedAtMs >= window.until.timestampMs;
66838
+ if (!outsideWindow) return;
66839
+ const lower = window.since?.value ?? "-infinity";
66840
+ const upper = window.until?.value ?? "+infinity";
66841
+ throw new WordpressTrafficApiError(
66842
+ `WordPress traffic endpoint returned an invalid or out-of-window observed_at for the requested [${lower}, ${upper}) window. Upgrade to a bounded-window-capable Canonry traffic-logger extension before syncing.`,
66843
+ 502
66844
+ );
66845
+ }
66750
66846
  async function listWordpressTrafficEvents(options) {
66751
66847
  const endpoint = resolveEndpoint(options.baseUrl);
66752
66848
  const authHeader = buildBasicAuthHeader(options.username, options.applicationPassword);
66753
66849
  const pageSize = normalizePageSize2(options.pageSize);
66754
66850
  const maxPages = normalizeMaxPages2(options.maxPages);
66755
66851
  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS2;
66852
+ const requestedWindow = parseRequestedWindow(options);
66756
66853
  let cursor = options.cursor;
66757
66854
  let rawEntryCount = 0;
66758
66855
  let skippedEntryCount = 0;
@@ -66793,20 +66890,28 @@ async function listWordpressTrafficEvents(options) {
66793
66890
  body2
66794
66891
  );
66795
66892
  }
66796
- const body = await response.json();
66797
- const entries = body.events ?? [];
66893
+ const body = parseResponseBody(await readResponseJson(response));
66894
+ const entries = body.events;
66798
66895
  rawEntryCount += entries.length;
66799
66896
  for (const entry of entries) {
66800
- const normalized = normalizeWordpressTrafficEvent(entry, body.site);
66897
+ assertEntryIsWithinRequestedWindow(entry, requestedWindow);
66898
+ if (!isRecord2(entry)) {
66899
+ skippedEntryCount += 1;
66900
+ continue;
66901
+ }
66902
+ const normalized = normalizeWordpressTrafficEvent(
66903
+ entry,
66904
+ body.site
66905
+ );
66801
66906
  if (normalized) {
66802
66907
  events.push(normalized);
66803
66908
  } else {
66804
66909
  skippedEntryCount += 1;
66805
66910
  }
66806
66911
  }
66807
- cursor = body.next_cursor ?? void 0;
66808
- hasMore = Boolean(body.has_more) && Boolean(cursor);
66809
- if (!body.has_more || !cursor) break;
66912
+ cursor = body.nextCursor ?? void 0;
66913
+ hasMore = body.hasMore;
66914
+ if (!hasMore) break;
66810
66915
  }
66811
66916
  return {
66812
66917
  events,
@@ -68470,8 +68575,7 @@ function tryClaimTrafficSyncLease(input) {
68470
68575
  return input.db.transaction((tx) => {
68471
68576
  const changed = tx.update(trafficSources).set({
68472
68577
  syncLeaseOwner: input.owner,
68473
- syncLeaseExpiresAt: expiresAt,
68474
- updatedAt: input.now
68578
+ syncLeaseExpiresAt: expiresAt
68475
68579
  }).where(and42(
68476
68580
  eq49(trafficSources.id, input.sourceId),
68477
68581
  or11(
@@ -68487,8 +68591,7 @@ function releaseTrafficSyncLease(input) {
68487
68591
  return input.db.transaction((tx) => {
68488
68592
  const changed = tx.update(trafficSources).set({
68489
68593
  syncLeaseOwner: null,
68490
- syncLeaseExpiresAt: null,
68491
- updatedAt: input.now
68594
+ syncLeaseExpiresAt: null
68492
68595
  }).where(and42(
68493
68596
  eq49(trafficSources.id, input.sourceId),
68494
68597
  eq49(trafficSources.syncLeaseOwner, input.owner)
@@ -68504,6 +68607,7 @@ var DEFAULT_MAX_PAGES4 = 5;
68504
68607
  var DEFAULT_SAMPLE_LIMIT2 = 100;
68505
68608
  var DEFAULT_WP_PAGE_SIZE = 500;
68506
68609
  var DEFAULT_WP_MAX_PAGES = 20;
68610
+ var DEFAULT_WP_SYNC_WINDOW_MINUTES = 365 * 24 * 60;
68507
68611
  var DEFAULT_VERCEL_MAX_PAGES = 50;
68508
68612
  var VERCEL_MAX_SUB_WINDOWS = 5e3;
68509
68613
  var VERCEL_BACKFILL_CHUNK_MS = 60 * 6e4;
@@ -68517,6 +68621,7 @@ var CLOUDFLARE_INGEST_BODY_LIMIT = 256 * 1024;
68517
68621
  var CLOUDFLARE_QUEUE_RECEIPT_TTL_MS = 14 * 24 * 60 * 6e4 + 10 * 6e4;
68518
68622
  var CLOUDFLARE_QUEUE_VISIBILITY_TIMEOUT_MS = 5 * 6e4;
68519
68623
  var CLOUDFLARE_QUEUE_SYNC_LEASE_TTL_MS = 5 * 6e4;
68624
+ var WORDPRESS_SYNC_LEASE_TTL_MS = 5 * 6e4;
68520
68625
  var DEFAULT_CLOUDFLARE_INGEST_RATE_LIMIT_MAX = 6e3;
68521
68626
  var SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
68522
68627
  function sha256Hex3(value) {
@@ -68822,7 +68927,19 @@ function trafficConnectStatus(tx, projectId, existingSource) {
68822
68927
  return existingSource?.status === TrafficSourceStatuses.paused || hasConnectedTrafficSourceSibling(tx, projectId, existingSource?.id ?? "") ? TrafficSourceStatuses.paused : TrafficSourceStatuses.connected;
68823
68928
  }
68824
68929
  function isSameTrafficSourceGeneration(current, started) {
68825
- return isDeepStrictEqual(current.configJson, started.configJson) && current.updatedAt === started.updatedAt && current.lastSyncedAt === started.lastSyncedAt;
68930
+ return isDeepStrictEqual(current.configJson, started.configJson) && current.updatedAt === started.updatedAt && current.lastSyncedAt === started.lastSyncedAt && current.lastCursor === started.lastCursor && current.wordpressPendingUntil === started.wordpressPendingUntil;
68931
+ }
68932
+ function wordpressBaseUrlIdentity(value) {
68933
+ if (typeof value !== "string" || !value.trim()) return void 0;
68934
+ try {
68935
+ const url = new URL(value);
68936
+ url.hash = "";
68937
+ url.search = "";
68938
+ url.pathname = url.pathname.replace(/\/+$/, "") || "/";
68939
+ return url.toString();
68940
+ } catch {
68941
+ return void 0;
68942
+ }
68826
68943
  }
68827
68944
  function bindTrafficSyncSchedule(tx, projectId, sourceId, now, createIfMissing = true) {
68828
68945
  const schedule = tx.select().from(schedules).where(and43(
@@ -68879,6 +68996,10 @@ async function runBackfillTask(options) {
68879
68996
  } catch {
68880
68997
  }
68881
68998
  };
68999
+ if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
69000
+ markFailed("Generic WordPress replace backfill is unavailable because retained coverage is unproven. Use a retention-aware repair that declares the unrecoverable span.");
69001
+ return;
69002
+ }
68882
69003
  let allEvents;
68883
69004
  try {
68884
69005
  allEvents = await pullForBackfill();
@@ -68922,6 +69043,14 @@ async function runBackfillTask(options) {
68922
69043
  try {
68923
69044
  const commitOutcome = app.db.transaction((tx) => {
68924
69045
  const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
69046
+ if (latestSource?.sourceType === TrafficSourceTypes.wordpress) {
69047
+ tx.update(runs).set({
69048
+ status: RunStatuses.failed,
69049
+ error: "Generic WordPress replace backfill is unavailable because retained coverage is unproven.",
69050
+ finishedAt
69051
+ }).where(eq50(runs.id, runId)).run();
69052
+ return "wordpress-backfill-unsupported";
69053
+ }
68925
69054
  if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2)) {
68926
69055
  tx.update(runs).set({
68927
69056
  status: RunStatuses.failed,
@@ -69142,7 +69271,7 @@ async function runBackfillTask(options) {
69142
69271
  }).run();
69143
69272
  }
69144
69273
  const recordedSkipMs = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
69145
- const skipRecovered = Number.isFinite(recordedSkipMs) && windowStart.getTime() <= recordedSkipMs;
69274
+ const skipRecovered = sourceRow2.sourceType !== TrafficSourceTypes.wordpress && Number.isFinite(recordedSkipMs) && windowStart.getTime() <= recordedSkipMs;
69146
69275
  tx.update(trafficSources).set({
69147
69276
  status: TrafficSourceStatuses.connected,
69148
69277
  lastSyncedAt: nextLastSyncedAt,
@@ -69154,7 +69283,7 @@ async function runBackfillTask(options) {
69154
69283
  tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq50(runs.id, runId)).run();
69155
69284
  return "committed";
69156
69285
  });
69157
- if (commitOutcome === "source-inactive") return;
69286
+ if (commitOutcome === "source-inactive" || commitOutcome === "wordpress-backfill-unsupported") return;
69158
69287
  } catch (e) {
69159
69288
  markFailed(`Backfill rollup write failed: ${describeError(e)}`);
69160
69289
  }
@@ -69402,9 +69531,10 @@ async function trafficRoutes(app, opts) {
69402
69531
  const fallbackName = displayName ?? `WordPress \xB7 ${new URL(baseUrl).host}`;
69403
69532
  const { sourceRow: sourceRow2, scheduleChanged } = app.db.transaction((tx) => {
69404
69533
  const activeSource = tx.select().from(trafficSources).where(eq50(trafficSources.projectId, project.id)).all().find((row) => row.sourceType === TrafficSourceTypes.wordpress && row.status !== TrafficSourceStatuses.archived);
69405
- const sourceId = activeSource?.id ?? crypto42.randomUUID();
69534
+ const endpointChanged = activeSource !== void 0 && wordpressBaseUrlIdentity(activeSource.configJson.baseUrl) !== wordpressBaseUrlIdentity(baseUrl);
69535
+ const sourceId = endpointChanged ? crypto42.randomUUID() : activeSource?.id ?? crypto42.randomUUID();
69406
69536
  const sourceStatus = trafficConnectStatus(tx, project.id, activeSource);
69407
- if (activeSource) {
69537
+ if (activeSource && !endpointChanged) {
69408
69538
  tx.update(trafficSources).set({
69409
69539
  displayName: fallbackName,
69410
69540
  status: sourceStatus,
@@ -69413,6 +69543,13 @@ async function trafficRoutes(app, opts) {
69413
69543
  updatedAt: now
69414
69544
  }).where(eq50(trafficSources.id, sourceId)).run();
69415
69545
  } else {
69546
+ if (activeSource) {
69547
+ tx.update(trafficSources).set({
69548
+ status: TrafficSourceStatuses.archived,
69549
+ archivedAt: now,
69550
+ updatedAt: now
69551
+ }).where(eq50(trafficSources.id, activeSource.id)).run();
69552
+ }
69416
69553
  tx.insert(trafficSources).values({
69417
69554
  id: sourceId,
69418
69555
  projectId: project.id,
@@ -70018,10 +70155,11 @@ async function trafficRoutes(app, opts) {
70018
70155
  });
70019
70156
  app.post("/projects/:name/traffic/sources/:id/sync", async (request) => {
70020
70157
  const project = resolveProject(app.db, request.params.name);
70021
- const sourceRow2 = app.db.select().from(trafficSources).where(eq50(trafficSources.id, request.params.id)).get();
70022
- if (!sourceRow2 || sourceRow2.projectId !== project.id) {
70158
+ const initialSourceRow = app.db.select().from(trafficSources).where(eq50(trafficSources.id, request.params.id)).get();
70159
+ if (!initialSourceRow || initialSourceRow.projectId !== project.id) {
70023
70160
  throw notFound("Traffic source", request.params.id);
70024
70161
  }
70162
+ let sourceRow2 = initialSourceRow;
70025
70163
  const queueConfig = sourceRow2.sourceType === TrafficSourceTypes.cloudflare ? parseQueuePullCloudflareSourceConfig(sourceRow2.configJson) : null;
70026
70164
  if (queueConfig) {
70027
70165
  if (sourceRow2.status !== TrafficSourceStatuses.connected) {
@@ -70047,19 +70185,19 @@ async function trafficRoutes(app, opts) {
70047
70185
  })) {
70048
70186
  throw operationInProgress("Cloudflare Queue source sync is already in progress", { sourceId: sourceRow2.id });
70049
70187
  }
70050
- const runId2 = crypto42.randomUUID();
70051
- const startedAt2 = (/* @__PURE__ */ new Date()).toISOString();
70188
+ const runId = crypto42.randomUUID();
70189
+ const startedAt = (/* @__PURE__ */ new Date()).toISOString();
70052
70190
  const startedMs = Date.now();
70053
70191
  try {
70054
70192
  app.db.insert(runs).values({
70055
- id: runId2,
70193
+ id: runId,
70056
70194
  projectId: project.id,
70057
70195
  kind: RunKinds["traffic-sync"],
70058
70196
  status: RunStatuses.running,
70059
70197
  trigger: RunTriggers.manual,
70060
70198
  sourceId: sourceRow2.id,
70061
- startedAt: startedAt2,
70062
- createdAt: startedAt2
70199
+ startedAt,
70200
+ createdAt: startedAt
70063
70201
  }).run();
70064
70202
  let acceptedEvents = 0;
70065
70203
  let selfTrafficExcluded = 0;
@@ -70067,11 +70205,11 @@ async function trafficRoutes(app, opts) {
70067
70205
  let aiUserFetchHits = 0;
70068
70206
  let aiReferralHits = 0;
70069
70207
  let unknownHits = 0;
70070
- let crawlerBucketRows2 = 0;
70071
- let aiUserFetchBucketRows2 = 0;
70072
- let aiReferralBucketRows2 = 0;
70073
- let sampleRows2 = 0;
70074
- let committedAt = startedAt2;
70208
+ let crawlerBucketRows = 0;
70209
+ let aiUserFetchBucketRows = 0;
70210
+ let aiReferralBucketRows = 0;
70211
+ let sampleRows = 0;
70212
+ let committedAt = startedAt;
70075
70213
  let remainingBacklogCount = 0;
70076
70214
  const canonicalHost = resolveCloudflareWorkerRouteHost(project.canonicalDomain);
70077
70215
  for (let batch = 0; batch < cloudflareQueueMaxBatches; batch += 1) {
@@ -70190,14 +70328,14 @@ async function trafficRoutes(app, opts) {
70190
70328
  aiUserFetchHits += writeResult.aiUserFetchHits;
70191
70329
  aiReferralHits += writeResult.aiReferralHits;
70192
70330
  unknownHits += writeResult.unknownHits;
70193
- crawlerBucketRows2 += writeResult.crawlerBucketRows;
70194
- aiUserFetchBucketRows2 += writeResult.aiUserFetchBucketRows;
70195
- aiReferralBucketRows2 += writeResult.aiReferralBucketRows;
70196
- sampleRows2 += writeResult.sampleRows;
70331
+ crawlerBucketRows += writeResult.crawlerBucketRows;
70332
+ aiUserFetchBucketRows += writeResult.aiUserFetchBucketRows;
70333
+ aiReferralBucketRows += writeResult.aiReferralBucketRows;
70334
+ sampleRows += writeResult.sampleRows;
70197
70335
  const pulledEnvelopeCount = pulled.messages.length + pulled.skippedUnleasedMessageCount;
70198
70336
  if (pulledEnvelopeCount < CLOUDFLARE_QUEUE_BATCH_SIZE || remainingBacklogCount === 0) break;
70199
70337
  }
70200
- app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: committedAt }).where(eq50(runs.id, runId2)).run();
70338
+ app.db.update(runs).set({ status: RunStatuses.completed, finishedAt: committedAt }).where(eq50(runs.id, runId)).run();
70201
70339
  writeAuditLog(app.db, {
70202
70340
  projectId: project.id,
70203
70341
  actor: "api",
@@ -70219,9 +70357,9 @@ async function trafficRoutes(app, opts) {
70219
70357
  });
70220
70358
  } catch {
70221
70359
  }
70222
- const response2 = {
70360
+ const response = {
70223
70361
  sourceId: sourceRow2.id,
70224
- runId: runId2,
70362
+ runId,
70225
70363
  syncedAt: committedAt,
70226
70364
  pulledEvents: acceptedEvents,
70227
70365
  selfTrafficExcluded,
@@ -70229,20 +70367,20 @@ async function trafficRoutes(app, opts) {
70229
70367
  aiUserFetchHits,
70230
70368
  aiReferralHits,
70231
70369
  unknownHits,
70232
- crawlerBucketRows: crawlerBucketRows2,
70233
- aiUserFetchBucketRows: aiUserFetchBucketRows2,
70234
- aiReferralBucketRows: aiReferralBucketRows2,
70235
- sampleRows: sampleRows2,
70370
+ crawlerBucketRows,
70371
+ aiUserFetchBucketRows,
70372
+ aiReferralBucketRows,
70373
+ sampleRows,
70236
70374
  remainingBacklogCount,
70237
- windowStart: startedAt2,
70375
+ windowStart: startedAt,
70238
70376
  windowEnd: committedAt
70239
70377
  };
70240
- return response2;
70378
+ return response;
70241
70379
  } catch (error) {
70242
70380
  const failedAt = (/* @__PURE__ */ new Date()).toISOString();
70243
70381
  const safeError = error instanceof CloudflareQueueApiError ? error.message : "Cloudflare Queue sync failed; retry the active source.";
70244
70382
  app.db.transaction((tx) => {
70245
- tx.update(runs).set({ status: RunStatuses.failed, error: safeError, finishedAt: failedAt }).where(eq50(runs.id, runId2)).run();
70383
+ tx.update(runs).set({ status: RunStatuses.failed, error: safeError, finishedAt: failedAt }).where(eq50(runs.id, runId)).run();
70246
70384
  const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70247
70385
  if (latestSource && latestSource.status === TrafficSourceStatuses.connected && latestSource.syncLeaseOwner === leaseOwner && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson)) {
70248
70386
  tx.update(trafficSources).set({ lastError: safeError, updatedAt: failedAt }).where(and43(
@@ -70271,588 +70409,690 @@ async function trafficRoutes(app, opts) {
70271
70409
  if (sourceRow2.status === TrafficSourceStatuses.paused || sourceRow2.status === TrafficSourceStatuses.archived || hasConnectedSibling) {
70272
70410
  throw validationError("Traffic source must be active before it can sync");
70273
70411
  }
70274
- const windowEnd = /* @__PURE__ */ new Date();
70275
- const startedAt = windowEnd.toISOString();
70276
- const syncStartedAtMs = windowEnd.getTime();
70277
- const runId = crypto42.randomUUID();
70278
- app.db.insert(runs).values({
70279
- id: runId,
70280
- projectId: project.id,
70281
- kind: RunKinds["traffic-sync"],
70282
- status: RunStatuses.running,
70283
- trigger: RunTriggers.manual,
70412
+ const wordpressLeaseOwner = sourceRow2.sourceType === TrafficSourceTypes.wordpress ? crypto42.randomUUID() : void 0;
70413
+ if (wordpressLeaseOwner && !tryClaimTrafficSyncLease({
70414
+ db: app.db,
70284
70415
  sourceId: sourceRow2.id,
70285
- startedAt,
70286
- createdAt: startedAt
70287
- }).run();
70288
- const markFailed = (msg, errorCode) => {
70289
- const failedAt = (/* @__PURE__ */ new Date()).toISOString();
70290
- app.db.transaction((tx) => {
70291
- tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq50(runs.id, runId)).run();
70292
- const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70293
- if (latestSource?.status === TrafficSourceStatuses.connected && isDeepStrictEqual(latestSource.configJson, sourceRow2.configJson) && latestSource.updatedAt === sourceRow2.updatedAt && latestSource.lastSyncedAt === sourceRow2.lastSyncedAt) {
70294
- tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq50(trafficSources.id, sourceRow2.id)).run();
70416
+ owner: wordpressLeaseOwner,
70417
+ now: (/* @__PURE__ */ new Date()).toISOString(),
70418
+ ttlMs: WORDPRESS_SYNC_LEASE_TTL_MS
70419
+ })) {
70420
+ throw operationInProgress("WordPress source sync is already in progress", { sourceId: sourceRow2.id });
70421
+ }
70422
+ try {
70423
+ const syncWindowEnd = /* @__PURE__ */ new Date();
70424
+ let windowEnd = syncWindowEnd;
70425
+ const startedAt = syncWindowEnd.toISOString();
70426
+ const syncStartedAtMs = syncWindowEnd.getTime();
70427
+ const runId = crypto42.randomUUID();
70428
+ app.db.insert(runs).values({
70429
+ id: runId,
70430
+ projectId: project.id,
70431
+ kind: RunKinds["traffic-sync"],
70432
+ status: RunStatuses.running,
70433
+ trigger: RunTriggers.manual,
70434
+ sourceId: sourceRow2.id,
70435
+ startedAt,
70436
+ createdAt: startedAt
70437
+ }).run();
70438
+ const markFailed = (msg, errorCode) => {
70439
+ const failedAt = (/* @__PURE__ */ new Date()).toISOString();
70440
+ app.db.transaction((tx) => {
70441
+ tx.update(runs).set({ status: RunStatuses.failed, error: msg, finishedAt: failedAt }).where(eq50(runs.id, runId)).run();
70442
+ const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70443
+ if (latestSource?.status === TrafficSourceStatuses.connected && isSameTrafficSourceGeneration(latestSource, sourceRow2) && (wordpressLeaseOwner === void 0 || latestSource.syncLeaseOwner === wordpressLeaseOwner)) {
70444
+ tx.update(trafficSources).set({ status: TrafficSourceStatuses.error, lastError: msg, updatedAt: failedAt }).where(eq50(trafficSources.id, sourceRow2.id)).run();
70445
+ }
70446
+ });
70447
+ try {
70448
+ opts.onTrafficSynced?.({
70449
+ status: "failed",
70450
+ sourceType: sourceRow2.sourceType,
70451
+ sourceId: sourceRow2.id,
70452
+ pulledEvents: 0,
70453
+ selfTrafficExcluded: 0,
70454
+ crawlerHits: 0,
70455
+ aiUserFetchHits: 0,
70456
+ aiReferralHits: 0,
70457
+ durationMs: Date.now() - syncStartedAtMs,
70458
+ errorCode
70459
+ });
70460
+ } catch {
70295
70461
  }
70296
- });
70297
- try {
70298
- opts.onTrafficSynced?.({
70299
- status: "failed",
70300
- sourceType: sourceRow2.sourceType,
70462
+ };
70463
+ const renewWordpressLease = () => {
70464
+ if (!wordpressLeaseOwner) return;
70465
+ if (!tryClaimTrafficSyncLease({
70466
+ db: app.db,
70301
70467
  sourceId: sourceRow2.id,
70302
- pulledEvents: 0,
70303
- selfTrafficExcluded: 0,
70304
- crawlerHits: 0,
70305
- aiUserFetchHits: 0,
70306
- aiReferralHits: 0,
70307
- durationMs: Date.now() - syncStartedAtMs,
70308
- errorCode
70309
- });
70310
- } catch {
70311
- }
70312
- };
70313
- let windowStart;
70314
- let allEvents;
70315
- let nextCursor;
70316
- let auditAction;
70317
- let effectiveWindowEnd = windowEnd;
70318
- if (sourceRow2.sourceType === TrafficSourceTypes["cloud-run"]) {
70319
- auditAction = "traffic.cloud-run.synced";
70320
- const credentialStore = opts.cloudRunCredentialStore;
70321
- if (!credentialStore) {
70322
- throw validationError("Cloud Run credential storage is not configured for this deployment");
70323
- }
70324
- const credential = credentialStore.getConnection(project.name);
70325
- if (!credential) {
70326
- throw validationError(
70327
- `No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
70328
- );
70329
- }
70330
- const config = parseSourceConfig(sourceRow2);
70331
- const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
70332
- const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
70333
- const location = config.location ?? credential.location ?? void 0;
70334
- const requestedMinutes = request.body?.sinceMinutes;
70335
- const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
70336
- const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
70337
- const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
70338
- windowStart = new Date(
70339
- Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
70340
- );
70341
- let accessToken;
70342
- try {
70343
- accessToken = await resolveAccessToken2(credential);
70344
- } catch (e) {
70345
- const msg = describeError(e);
70346
- markFailed(msg, "PROVIDER_AUTH");
70347
- throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
70348
- }
70349
- const isFirstSync = !sourceRow2.lastSyncedAt;
70350
- try {
70351
- const page = await pullEvents(accessToken, {
70352
- gcpProjectId,
70353
- serviceName,
70354
- location,
70355
- startTime: windowStart.toISOString(),
70356
- endTime: windowEnd.toISOString(),
70357
- pageSize,
70358
- maxPages,
70359
- firstSync: isFirstSync,
70360
- requestUrlSubstrings: [project.canonicalDomain]
70361
- });
70362
- allEvents = page.events;
70363
- } catch (e) {
70364
- const msg = describeError(e);
70365
- markFailed(msg, "PROVIDER_PULL");
70366
- throw providerError(`Cloud Run pull failed: ${msg}`);
70367
- }
70368
- } else if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
70369
- auditAction = "traffic.wordpress.synced";
70370
- const credentialStore = opts.wordpressTrafficCredentialStore;
70371
- if (!credentialStore) {
70372
- throw validationError("WordPress traffic credential storage is not configured for this deployment");
70373
- }
70374
- const credential = credentialStore.getConnection(project.name);
70375
- if (!credential) {
70376
- app.db.delete(runs).where(eq50(runs.id, runId)).run();
70377
- throw validationError(
70378
- `No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
70468
+ owner: wordpressLeaseOwner,
70469
+ now: (/* @__PURE__ */ new Date()).toISOString(),
70470
+ ttlMs: WORDPRESS_SYNC_LEASE_TTL_MS
70471
+ })) {
70472
+ throw new Error("WordPress sync lease was lost");
70473
+ }
70474
+ };
70475
+ let windowStart;
70476
+ let allEvents;
70477
+ let nextCursor;
70478
+ let auditAction;
70479
+ let effectiveWindowEnd = windowEnd;
70480
+ if (sourceRow2.sourceType === TrafficSourceTypes["cloud-run"]) {
70481
+ auditAction = "traffic.cloud-run.synced";
70482
+ const credentialStore = opts.cloudRunCredentialStore;
70483
+ if (!credentialStore) {
70484
+ throw validationError("Cloud Run credential storage is not configured for this deployment");
70485
+ }
70486
+ const credential = credentialStore.getConnection(project.name);
70487
+ if (!credential) {
70488
+ throw validationError(
70489
+ `No Cloud Run credential found for project "${project.name}". Run "canonry traffic connect cloud-run" first.`
70490
+ );
70491
+ }
70492
+ const config = parseSourceConfig(sourceRow2);
70493
+ const gcpProjectId = config.gcpProjectId ?? credential.gcpProjectId;
70494
+ const serviceName = config.serviceName ?? credential.serviceName ?? void 0;
70495
+ const location = config.location ?? credential.location ?? void 0;
70496
+ const requestedMinutes = request.body?.sinceMinutes;
70497
+ const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
70498
+ const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
70499
+ const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
70500
+ windowStart = new Date(
70501
+ Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
70379
70502
  );
70380
- }
70381
- windowStart = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt) : windowEnd;
70382
- const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
70383
- const wpMaxPages = opts.defaultWordpressMaxPages ?? DEFAULT_WP_MAX_PAGES;
70384
- let pinnedDispatcher;
70385
- try {
70386
- pinnedDispatcher = await assertWordpressTargetAllowed(credential.baseUrl);
70387
- } catch (e) {
70388
- const msg = describeError(e);
70389
- markFailed(msg, "PROVIDER_PULL");
70390
- throw e;
70391
- }
70392
- const collected = [];
70393
- let cursor = sourceRow2.lastCursor ?? void 0;
70394
- try {
70395
- for (let page = 0; page < wpMaxPages; page += 1) {
70396
- const pageResult = await pullWordpressEvents({
70397
- baseUrl: credential.baseUrl,
70398
- username: credential.username,
70399
- applicationPassword: credential.applicationPassword,
70400
- cursor,
70401
- pageSize: wpPageSize,
70402
- maxPages: 1,
70403
- dispatcher: pinnedDispatcher
70503
+ let accessToken;
70504
+ try {
70505
+ accessToken = await resolveAccessToken2(credential);
70506
+ } catch (e) {
70507
+ const msg = describeError(e);
70508
+ markFailed(msg, "PROVIDER_AUTH");
70509
+ throw providerError(`Failed to resolve Cloud Run access token: ${msg}`);
70510
+ }
70511
+ const isFirstSync = !sourceRow2.lastSyncedAt;
70512
+ try {
70513
+ const page = await pullEvents(accessToken, {
70514
+ gcpProjectId,
70515
+ serviceName,
70516
+ location,
70517
+ startTime: windowStart.toISOString(),
70518
+ endTime: windowEnd.toISOString(),
70519
+ pageSize,
70520
+ maxPages,
70521
+ firstSync: isFirstSync,
70522
+ requestUrlSubstrings: [project.canonicalDomain]
70404
70523
  });
70405
- collected.push(...pageResult.events);
70406
- const previousCursor = cursor;
70407
- cursor = pageResult.nextCursor;
70408
- if (!pageResult.hasMore) break;
70409
- if (!cursor || cursor === previousCursor) break;
70524
+ allEvents = page.events;
70525
+ } catch (e) {
70526
+ const msg = describeError(e);
70527
+ markFailed(msg, "PROVIDER_PULL");
70528
+ throw providerError(`Cloud Run pull failed: ${msg}`);
70410
70529
  }
70411
- allEvents = collected;
70412
- nextCursor = cursor;
70413
- } catch (e) {
70414
- const msg = describeError(e);
70415
- markFailed(msg, "PROVIDER_PULL");
70416
- throw providerError(`WordPress pull failed: ${msg}`);
70417
- } finally {
70418
- await pinnedDispatcher.close().catch(() => {
70419
- });
70420
- }
70421
- } else {
70422
- auditAction = "traffic.vercel.synced";
70423
- const credentialStore = opts.vercelTrafficCredentialStore;
70424
- if (!credentialStore) {
70425
- app.db.delete(runs).where(eq50(runs.id, runId)).run();
70426
- throw validationError("Vercel traffic credential storage is not configured for this deployment");
70427
- }
70428
- const credential = credentialStore.getConnection(project.name);
70429
- if (!credential) {
70430
- app.db.delete(runs).where(eq50(runs.id, runId)).run();
70431
- throw validationError(
70432
- `No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
70433
- );
70434
- }
70435
- const config = parseSourceConfig(sourceRow2);
70436
- const vercelProjectId = config.projectId ?? credential.projectId;
70437
- const vercelTeamId = config.teamId ?? credential.teamId;
70438
- const vercelEnvironment = config.environment ?? credential.environment;
70439
- const requestedMinutes = request.body?.sinceMinutes;
70440
- const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
70441
- const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
70442
- const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
70443
- const clampedStartMs = Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs));
70444
- const cappedStartMs = Math.max(clampedStartMs, windowEnd.getTime() - VERCEL_MAX_SYNC_WINDOW_MS);
70445
- if (cappedStartMs > clampedStartMs) {
70446
- const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
70447
- const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
70448
- app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq50(trafficSources.id, sourceRow2.id)).run();
70449
- request.log.warn(
70450
- {
70451
- sourceId: sourceRow2.id,
70452
- requestedStart: new Date(clampedStartMs).toISOString(),
70453
- cappedStart: new Date(cappedStartMs).toISOString()
70454
- },
70455
- "Vercel sync window exceeded the max single-sync span; clamped the start forward (older traffic skipped \u2014 run a backfill to recover it)"
70456
- );
70457
- }
70458
- windowStart = new Date(cappedStartMs);
70459
- try {
70460
- const drained = await drainVercelTrafficEvents({
70461
- pull: pullVercelEvents,
70462
- token: credential.token,
70463
- projectId: vercelProjectId,
70464
- teamId: vercelTeamId,
70465
- environment: vercelEnvironment,
70466
- startDate: windowStart.getTime(),
70467
- endDate: windowEnd.getTime(),
70468
- pagesPerSubWindow: vercelMaxPages,
70469
- maxSubWindows: VERCEL_MAX_SUB_WINDOWS,
70470
- // Bound the drain's wall-clock so a dense/slow window can't run for
70471
- // many minutes. On hit the drain stops and reports how far it got.
70472
- deadlineMs: syncStartedAtMs + vercelSyncDeadlineMs
70473
- });
70474
- if (drained.retentionClamped) {
70475
- throw vercelRetentionClampError(windowStart.getTime(), drained.effectiveStartMs);
70530
+ } else if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
70531
+ auditAction = "traffic.wordpress.synced";
70532
+ const credentialStore = opts.wordpressTrafficCredentialStore;
70533
+ if (!credentialStore) {
70534
+ throw validationError("WordPress traffic credential storage is not configured for this deployment");
70476
70535
  }
70477
- if (drained.deadlineReached) {
70478
- if (drained.drainedThroughMs <= windowStart.getTime()) {
70479
- throw new Error(
70480
- `sync exceeded its ${vercelSyncDeadlineMs}ms drain budget without completing any sub-window (request-logs slow or unavailable)`
70481
- );
70536
+ const credential = credentialStore.getConnection(project.name);
70537
+ if (!credential) {
70538
+ app.db.delete(runs).where(eq50(runs.id, runId)).run();
70539
+ throw validationError(
70540
+ `No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
70541
+ );
70542
+ }
70543
+ if (sourceRow2.lastCursor && !sourceRow2.wordpressPendingUntil) {
70544
+ const message = "WordPress source has a legacy continuation cursor without a bounded pending window. Reset it explicitly before resuming sync.";
70545
+ markFailed(message, "INTERNAL");
70546
+ throw validationError(message);
70547
+ }
70548
+ if (sourceRow2.wordpressPendingUntil) {
70549
+ const pendingStartMs = sourceRow2.lastSyncedAt ? Date.parse(sourceRow2.lastSyncedAt) : Number.NaN;
70550
+ const pendingEndMs = Date.parse(sourceRow2.wordpressPendingUntil);
70551
+ if (!Number.isFinite(pendingStartMs) || !Number.isFinite(pendingEndMs) || pendingStartMs >= pendingEndMs || pendingEndMs > syncWindowEnd.getTime()) {
70552
+ const message = "WordPress source has an invalid pending sync window. Reset it explicitly before resuming sync.";
70553
+ markFailed(message, "INTERNAL");
70554
+ throw validationError(message);
70482
70555
  }
70483
- effectiveWindowEnd = new Date(drained.drainedThroughMs);
70484
- request.log.warn(
70485
- {
70486
- sourceId: sourceRow2.id,
70487
- drainedThrough: effectiveWindowEnd.toISOString(),
70488
- requestedEnd: windowEnd.toISOString(),
70489
- subWindows: drained.subWindowCount
70490
- },
70491
- "Vercel drain hit its time budget; committing the partial window and advancing to it \u2014 next sync resumes from here"
70556
+ windowStart = new Date(pendingStartMs);
70557
+ windowEnd = new Date(pendingEndMs);
70558
+ effectiveWindowEnd = windowEnd;
70559
+ } else {
70560
+ const requestedMinutes = request.body?.sinceMinutes;
70561
+ const hasExplicitWindow = Number.isFinite(requestedMinutes) && requestedMinutes !== void 0 && requestedMinutes > 0;
70562
+ const windowMinutes = hasExplicitWindow ? Math.floor(requestedMinutes) : DEFAULT_WP_SYNC_WINDOW_MINUTES;
70563
+ const requestedStartMs = syncWindowEnd.getTime() - windowMinutes * 6e4;
70564
+ const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
70565
+ windowStart = new Date(
70566
+ Math.min(syncWindowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs))
70492
70567
  );
70568
+ const windowStartIso2 = windowStart.toISOString();
70569
+ const windowEndIso2 = windowEnd.toISOString();
70570
+ const reservedAt = (/* @__PURE__ */ new Date()).toISOString();
70571
+ const reservedSource = app.db.transaction((tx) => {
70572
+ const latestSource = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70573
+ if (!latestSource || !isAuthoritativeTrafficSource(tx, latestSource) || !isSameTrafficSourceGeneration(latestSource, sourceRow2) || latestSource.syncLeaseOwner !== wordpressLeaseOwner || latestSource.lastCursor || latestSource.wordpressPendingUntil) return void 0;
70574
+ tx.update(trafficSources).set({
70575
+ lastSyncedAt: windowStartIso2,
70576
+ wordpressPendingUntil: windowEndIso2,
70577
+ updatedAt: reservedAt
70578
+ }).where(eq50(trafficSources.id, sourceRow2.id)).run();
70579
+ return tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70580
+ }, { behavior: "immediate" });
70581
+ if (!reservedSource) {
70582
+ const message = "Traffic source changed while reserving the WordPress sync window; retry the sync.";
70583
+ markFailed(message, "INTERNAL");
70584
+ throw validationError(message);
70585
+ }
70586
+ sourceRow2 = reservedSource;
70493
70587
  }
70494
- if (drained.truncatedSliceCount > 0) {
70495
- request.log.warn(
70496
- {
70497
- sourceId: sourceRow2.id,
70498
- truncatedSlices: drained.truncatedSliceCount,
70499
- sliceStarts: drained.truncatedSliceStartsMs.map((ms) => new Date(ms).toISOString())
70500
- },
70501
- "Vercel drain truncated dense one-second slice(s); ingested a sample and advanced past them"
70588
+ const windowStartIso = windowStart.toISOString();
70589
+ const windowEndIso = windowEnd.toISOString();
70590
+ const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
70591
+ const wpMaxPages = opts.defaultWordpressMaxPages ?? DEFAULT_WP_MAX_PAGES;
70592
+ let pinnedDispatcher;
70593
+ try {
70594
+ renewWordpressLease();
70595
+ pinnedDispatcher = await assertWordpressTargetAllowed(credential.baseUrl);
70596
+ } catch (e) {
70597
+ const msg = describeError(e);
70598
+ markFailed(msg, "PROVIDER_PULL");
70599
+ throw e;
70600
+ }
70601
+ const collected = [];
70602
+ let cursor = sourceRow2.lastCursor ?? void 0;
70603
+ try {
70604
+ for (let page = 0; page < wpMaxPages; page += 1) {
70605
+ renewWordpressLease();
70606
+ const pageResult = await pullWordpressEvents({
70607
+ baseUrl: credential.baseUrl,
70608
+ username: credential.username,
70609
+ applicationPassword: credential.applicationPassword,
70610
+ cursor,
70611
+ pageSize: wpPageSize,
70612
+ maxPages: 1,
70613
+ since: windowStartIso,
70614
+ until: windowEndIso,
70615
+ dispatcher: pinnedDispatcher
70616
+ });
70617
+ collected.push(...pageResult.events);
70618
+ const previousCursor = cursor;
70619
+ cursor = pageResult.nextCursor;
70620
+ if (!pageResult.hasMore) {
70621
+ cursor = void 0;
70622
+ break;
70623
+ }
70624
+ if (!cursor || cursor === previousCursor) {
70625
+ throw new Error("WordPress traffic endpoint returned has_more without a new continuation cursor");
70626
+ }
70627
+ }
70628
+ allEvents = collected;
70629
+ nextCursor = cursor;
70630
+ } catch (e) {
70631
+ const msg = describeError(e);
70632
+ markFailed(msg, "PROVIDER_PULL");
70633
+ throw providerError(`WordPress pull failed: ${msg}`);
70634
+ } finally {
70635
+ await pinnedDispatcher.close().catch(() => {
70636
+ });
70637
+ }
70638
+ } else {
70639
+ auditAction = "traffic.vercel.synced";
70640
+ const credentialStore = opts.vercelTrafficCredentialStore;
70641
+ if (!credentialStore) {
70642
+ app.db.delete(runs).where(eq50(runs.id, runId)).run();
70643
+ throw validationError("Vercel traffic credential storage is not configured for this deployment");
70644
+ }
70645
+ const credential = credentialStore.getConnection(project.name);
70646
+ if (!credential) {
70647
+ app.db.delete(runs).where(eq50(runs.id, runId)).run();
70648
+ throw validationError(
70649
+ `No Vercel credential found for project "${project.name}". Run "canonry traffic connect vercel" first.`
70502
70650
  );
70503
70651
  }
70504
- if (drained.deadlineSkippedSliceCount > 0) {
70652
+ const config = parseSourceConfig(sourceRow2);
70653
+ const vercelProjectId = config.projectId ?? credential.projectId;
70654
+ const vercelTeamId = config.teamId ?? credential.teamId;
70655
+ const vercelEnvironment = config.environment ?? credential.environment;
70656
+ const requestedMinutes = request.body?.sinceMinutes;
70657
+ const windowMinutes = Number.isFinite(requestedMinutes) && requestedMinutes && requestedMinutes > 0 ? Math.floor(requestedMinutes) : syncWindowMinutes;
70658
+ const requestedStartMs = windowEnd.getTime() - windowMinutes * 6e4;
70659
+ const lastSyncedMs = sourceRow2.lastSyncedAt ? new Date(sourceRow2.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY;
70660
+ const clampedStartMs = Math.min(windowEnd.getTime(), Math.max(requestedStartMs, lastSyncedMs));
70661
+ const cappedStartMs = Math.max(clampedStartMs, windowEnd.getTime() - VERCEL_MAX_SYNC_WINDOW_MS);
70662
+ if (cappedStartMs > clampedStartMs) {
70663
+ const previousSkip = sourceRow2.skippedThroughAt ? Date.parse(sourceRow2.skippedThroughAt) : Number.NaN;
70664
+ const skippedThrough = Number.isFinite(previousSkip) ? new Date(Math.max(previousSkip, cappedStartMs)) : new Date(cappedStartMs);
70665
+ app.db.update(trafficSources).set({ skippedThroughAt: skippedThrough.toISOString() }).where(eq50(trafficSources.id, sourceRow2.id)).run();
70505
70666
  request.log.warn(
70506
70667
  {
70507
70668
  sourceId: sourceRow2.id,
70508
- skippedSlices: drained.deadlineSkippedSliceCount,
70509
- sliceStarts: drained.deadlineSkippedSliceStartsMs.map((ms) => new Date(ms).toISOString())
70669
+ requestedStart: new Date(clampedStartMs).toISOString(),
70670
+ cappedStart: new Date(cappedStartMs).toISOString()
70510
70671
  },
70511
- "Vercel drain could not narrow a dense/slow head slice within its budget; skipped past it to guarantee progress"
70672
+ "Vercel sync window exceeded the max single-sync span; clamped the start forward (older traffic skipped \u2014 run a backfill to recover it)"
70512
70673
  );
70513
70674
  }
70514
- allEvents = drained.events;
70675
+ windowStart = new Date(cappedStartMs);
70676
+ try {
70677
+ const drained = await drainVercelTrafficEvents({
70678
+ pull: pullVercelEvents,
70679
+ token: credential.token,
70680
+ projectId: vercelProjectId,
70681
+ teamId: vercelTeamId,
70682
+ environment: vercelEnvironment,
70683
+ startDate: windowStart.getTime(),
70684
+ endDate: windowEnd.getTime(),
70685
+ pagesPerSubWindow: vercelMaxPages,
70686
+ maxSubWindows: VERCEL_MAX_SUB_WINDOWS,
70687
+ // Bound the drain's wall-clock so a dense/slow window can't run for
70688
+ // many minutes. On hit the drain stops and reports how far it got.
70689
+ deadlineMs: syncStartedAtMs + vercelSyncDeadlineMs
70690
+ });
70691
+ if (drained.retentionClamped) {
70692
+ throw vercelRetentionClampError(windowStart.getTime(), drained.effectiveStartMs);
70693
+ }
70694
+ if (drained.deadlineReached) {
70695
+ if (drained.drainedThroughMs <= windowStart.getTime()) {
70696
+ throw new Error(
70697
+ `sync exceeded its ${vercelSyncDeadlineMs}ms drain budget without completing any sub-window (request-logs slow or unavailable)`
70698
+ );
70699
+ }
70700
+ effectiveWindowEnd = new Date(drained.drainedThroughMs);
70701
+ request.log.warn(
70702
+ {
70703
+ sourceId: sourceRow2.id,
70704
+ drainedThrough: effectiveWindowEnd.toISOString(),
70705
+ requestedEnd: windowEnd.toISOString(),
70706
+ subWindows: drained.subWindowCount
70707
+ },
70708
+ "Vercel drain hit its time budget; committing the partial window and advancing to it \u2014 next sync resumes from here"
70709
+ );
70710
+ }
70711
+ if (drained.truncatedSliceCount > 0) {
70712
+ request.log.warn(
70713
+ {
70714
+ sourceId: sourceRow2.id,
70715
+ truncatedSlices: drained.truncatedSliceCount,
70716
+ sliceStarts: drained.truncatedSliceStartsMs.map((ms) => new Date(ms).toISOString())
70717
+ },
70718
+ "Vercel drain truncated dense one-second slice(s); ingested a sample and advanced past them"
70719
+ );
70720
+ }
70721
+ if (drained.deadlineSkippedSliceCount > 0) {
70722
+ request.log.warn(
70723
+ {
70724
+ sourceId: sourceRow2.id,
70725
+ skippedSlices: drained.deadlineSkippedSliceCount,
70726
+ sliceStarts: drained.deadlineSkippedSliceStartsMs.map((ms) => new Date(ms).toISOString())
70727
+ },
70728
+ "Vercel drain could not narrow a dense/slow head slice within its budget; skipped past it to guarantee progress"
70729
+ );
70730
+ }
70731
+ allEvents = drained.events;
70732
+ } catch (e) {
70733
+ const msg = describeError(e);
70734
+ markFailed(msg, "PROVIDER_PULL");
70735
+ throw providerError(`Vercel pull failed: ${msg}`);
70736
+ }
70737
+ }
70738
+ let crawlerBucketRows = 0;
70739
+ let aiUserFetchBucketRows = 0;
70740
+ let aiReferralBucketRows = 0;
70741
+ let sampleRows = 0;
70742
+ let finishedAt = (/* @__PURE__ */ new Date()).toISOString();
70743
+ let pulledEventsCount = 0;
70744
+ let selfTrafficExcludedCount = 0;
70745
+ let crawlerHitsCount = 0;
70746
+ let aiUserFetchHitsCount = 0;
70747
+ let aiReferralHitsCount = 0;
70748
+ let unknownHitsCount = 0;
70749
+ try {
70750
+ renewWordpressLease();
70515
70751
  } catch (e) {
70516
70752
  const msg = describeError(e);
70517
- markFailed(msg, "PROVIDER_PULL");
70518
- throw providerError(`Vercel pull failed: ${msg}`);
70519
- }
70520
- }
70521
- let crawlerBucketRows = 0;
70522
- let aiUserFetchBucketRows = 0;
70523
- let aiReferralBucketRows = 0;
70524
- let sampleRows = 0;
70525
- let finishedAt = (/* @__PURE__ */ new Date()).toISOString();
70526
- let pulledEventsCount = 0;
70527
- let selfTrafficExcludedCount = 0;
70528
- let crawlerHitsCount = 0;
70529
- let aiUserFetchHitsCount = 0;
70530
- let aiReferralHitsCount = 0;
70531
- let unknownHitsCount = 0;
70532
- const commitOutcome = app.db.transaction((tx) => {
70533
- const latestRow = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70534
- const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq50(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
70535
- if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || !isDeepStrictEqual(latestRow.configJson, sourceRow2.configJson) || latestRow.updatedAt !== sourceRow2.updatedAt || latestRow.lastSyncedAt !== sourceRow2.lastSyncedAt) {
70536
- const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
70537
- tx.update(runs).set({
70538
- status: RunStatuses.failed,
70539
- error: "Traffic source was deactivated or reconfigured during sync",
70540
- finishedAt: abortedAt
70541
- }).where(eq50(runs.id, runId)).run();
70542
- return "source-inactive";
70543
- }
70544
- const previousIds = latestRow.lastEventIds ?? [];
70545
- const seenEventIds = new Set(previousIds);
70546
- const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
70547
- const newSorted = dedupedEvents.filter((e) => !isSelfTraffic(e)).sort((a, b) => a.observedAt < b.observedAt ? 1 : a.observedAt > b.observedAt ? -1 : 0).map((e) => e.eventId);
70548
- const merged = [];
70549
- const mergedSet = /* @__PURE__ */ new Set();
70550
- for (const id of [...newSorted, ...previousIds]) {
70551
- if (mergedSet.has(id)) continue;
70552
- mergedSet.add(id);
70553
- merged.push(id);
70554
- if (merged.length >= MAX_TRACKED_EVENT_IDS) break;
70555
- }
70556
- const nextEventIds = merged;
70557
- const report = buildTrafficProbeReport(dedupedEvents, { sampleLimit });
70558
- finishedAt = (/* @__PURE__ */ new Date()).toISOString();
70559
- const rawSampleCutoff = enforceRawEventSampleRetention(
70560
- tx,
70561
- sourceRow2.id,
70562
- finishedAt
70563
- );
70564
- pulledEventsCount = report.totals.normalizedEvents;
70565
- selfTrafficExcludedCount = report.totals.selfTrafficExcluded;
70566
- crawlerHitsCount = report.totals.crawlerHits;
70567
- aiUserFetchHitsCount = report.totals.aiUserFetchHits;
70568
- aiReferralHitsCount = report.totals.aiReferralHits;
70569
- unknownHitsCount = report.totals.unknownHits;
70570
- for (const bucket of report.crawlerEventsHourly) {
70571
- const status = bucket.status ?? 0;
70572
- tx.insert(crawlerEventsHourly).values({
70573
- projectId: project.id,
70574
- sourceId: sourceRow2.id,
70575
- tsHour: bucket.tsHour,
70576
- botId: bucket.botId,
70577
- operator: bucket.operator,
70578
- verificationStatus: bucket.verificationStatus,
70579
- pathNormalized: bucket.pathNormalized,
70580
- status,
70581
- hits: bucket.hits,
70582
- sampledUserAgent: bucket.sampledUserAgent,
70583
- createdAt: finishedAt,
70584
- updatedAt: finishedAt
70585
- }).onConflictDoUpdate({
70586
- target: [
70587
- crawlerEventsHourly.projectId,
70588
- crawlerEventsHourly.sourceId,
70589
- crawlerEventsHourly.tsHour,
70590
- crawlerEventsHourly.botId,
70591
- crawlerEventsHourly.verificationStatus,
70592
- crawlerEventsHourly.pathNormalized,
70593
- crawlerEventsHourly.status
70594
- ],
70595
- set: {
70596
- hits: sql22`${crawlerEventsHourly.hits} + ${bucket.hits}`,
70597
- sampledUserAgent: bucket.sampledUserAgent,
70598
- updatedAt: finishedAt
70599
- }
70600
- }).run();
70601
- if (bucket.verificationManifest) {
70602
- tx.insert(crawlerVerificationManifestsHourly).values({
70753
+ markFailed(msg, "INTERNAL");
70754
+ throw providerError(`WordPress sync failed: ${msg}`);
70755
+ }
70756
+ const commitOutcome = app.db.transaction((tx) => {
70757
+ const latestRow = tx.select().from(trafficSources).where(eq50(trafficSources.id, sourceRow2.id)).get();
70758
+ const latestHasConnectedSibling = tx.select().from(trafficSources).where(eq50(trafficSources.projectId, project.id)).all().some((row) => row.id !== sourceRow2.id && row.status === TrafficSourceStatuses.connected);
70759
+ if (!latestRow || latestRow.status !== TrafficSourceStatuses.connected && latestRow.status !== TrafficSourceStatuses.error || latestHasConnectedSibling || wordpressLeaseOwner !== void 0 && latestRow.syncLeaseOwner !== wordpressLeaseOwner || !isSameTrafficSourceGeneration(latestRow, sourceRow2)) {
70760
+ const abortedAt = (/* @__PURE__ */ new Date()).toISOString();
70761
+ tx.update(runs).set({
70762
+ status: RunStatuses.failed,
70763
+ error: "Traffic source was deactivated or reconfigured during sync",
70764
+ finishedAt: abortedAt
70765
+ }).where(eq50(runs.id, runId)).run();
70766
+ return "source-inactive";
70767
+ }
70768
+ const previousIds = latestRow.lastEventIds ?? [];
70769
+ const seenEventIds = new Set(previousIds);
70770
+ const dedupedEvents = seenEventIds.size === 0 ? allEvents : allEvents.filter((e) => !seenEventIds.has(e.eventId));
70771
+ const newSorted = dedupedEvents.filter((e) => !isSelfTraffic(e)).sort((a, b) => a.observedAt < b.observedAt ? 1 : a.observedAt > b.observedAt ? -1 : 0).map((e) => e.eventId);
70772
+ const merged = [];
70773
+ const mergedSet = /* @__PURE__ */ new Set();
70774
+ for (const id of [...newSorted, ...previousIds]) {
70775
+ if (mergedSet.has(id)) continue;
70776
+ mergedSet.add(id);
70777
+ merged.push(id);
70778
+ if (merged.length >= MAX_TRACKED_EVENT_IDS) break;
70779
+ }
70780
+ const nextEventIds = merged;
70781
+ const report = buildTrafficProbeReport(dedupedEvents, { sampleLimit });
70782
+ finishedAt = (/* @__PURE__ */ new Date()).toISOString();
70783
+ const rawSampleCutoff = enforceRawEventSampleRetention(
70784
+ tx,
70785
+ sourceRow2.id,
70786
+ finishedAt
70787
+ );
70788
+ pulledEventsCount = report.totals.normalizedEvents;
70789
+ selfTrafficExcludedCount = report.totals.selfTrafficExcluded;
70790
+ crawlerHitsCount = report.totals.crawlerHits;
70791
+ aiUserFetchHitsCount = report.totals.aiUserFetchHits;
70792
+ aiReferralHitsCount = report.totals.aiReferralHits;
70793
+ unknownHitsCount = report.totals.unknownHits;
70794
+ for (const bucket of report.crawlerEventsHourly) {
70795
+ const status = bucket.status ?? 0;
70796
+ tx.insert(crawlerEventsHourly).values({
70603
70797
  projectId: project.id,
70604
70798
  sourceId: sourceRow2.id,
70605
70799
  tsHour: bucket.tsHour,
70606
70800
  botId: bucket.botId,
70801
+ operator: bucket.operator,
70607
70802
  verificationStatus: bucket.verificationStatus,
70608
70803
  pathNormalized: bucket.pathNormalized,
70609
70804
  status,
70610
- manifestId: bucket.verificationManifest.id,
70611
- manifestJson: bucket.verificationManifest,
70612
70805
  hits: bucket.hits,
70806
+ sampledUserAgent: bucket.sampledUserAgent,
70613
70807
  createdAt: finishedAt,
70614
70808
  updatedAt: finishedAt
70615
70809
  }).onConflictDoUpdate({
70616
70810
  target: [
70617
- crawlerVerificationManifestsHourly.projectId,
70618
- crawlerVerificationManifestsHourly.sourceId,
70619
- crawlerVerificationManifestsHourly.tsHour,
70620
- crawlerVerificationManifestsHourly.botId,
70621
- crawlerVerificationManifestsHourly.verificationStatus,
70622
- crawlerVerificationManifestsHourly.pathNormalized,
70623
- crawlerVerificationManifestsHourly.status,
70624
- crawlerVerificationManifestsHourly.manifestId
70811
+ crawlerEventsHourly.projectId,
70812
+ crawlerEventsHourly.sourceId,
70813
+ crawlerEventsHourly.tsHour,
70814
+ crawlerEventsHourly.botId,
70815
+ crawlerEventsHourly.verificationStatus,
70816
+ crawlerEventsHourly.pathNormalized,
70817
+ crawlerEventsHourly.status
70625
70818
  ],
70626
70819
  set: {
70627
- hits: sql22`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
70628
- manifestJson: bucket.verificationManifest,
70820
+ hits: sql22`${crawlerEventsHourly.hits} + ${bucket.hits}`,
70821
+ sampledUserAgent: bucket.sampledUserAgent,
70629
70822
  updatedAt: finishedAt
70630
70823
  }
70631
70824
  }).run();
70632
- }
70633
- crawlerBucketRows += 1;
70634
- }
70635
- for (const bucket of report.aiUserFetchEventsHourly) {
70636
- const status = bucket.status ?? 0;
70637
- tx.insert(aiUserFetchEventsHourly).values({
70638
- projectId: project.id,
70639
- sourceId: sourceRow2.id,
70640
- tsHour: bucket.tsHour,
70641
- botId: bucket.botId,
70642
- operator: bucket.operator,
70643
- verificationStatus: bucket.verificationStatus,
70644
- pathNormalized: bucket.pathNormalized,
70645
- status,
70646
- hits: bucket.hits,
70647
- sampledUserAgent: bucket.sampledUserAgent,
70648
- createdAt: finishedAt,
70649
- updatedAt: finishedAt
70650
- }).onConflictDoUpdate({
70651
- target: [
70652
- aiUserFetchEventsHourly.projectId,
70653
- aiUserFetchEventsHourly.sourceId,
70654
- aiUserFetchEventsHourly.tsHour,
70655
- aiUserFetchEventsHourly.botId,
70656
- aiUserFetchEventsHourly.verificationStatus,
70657
- aiUserFetchEventsHourly.pathNormalized,
70658
- aiUserFetchEventsHourly.status
70659
- ],
70660
- set: {
70661
- hits: sql22`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
70662
- sampledUserAgent: bucket.sampledUserAgent,
70663
- updatedAt: finishedAt
70825
+ if (bucket.verificationManifest) {
70826
+ tx.insert(crawlerVerificationManifestsHourly).values({
70827
+ projectId: project.id,
70828
+ sourceId: sourceRow2.id,
70829
+ tsHour: bucket.tsHour,
70830
+ botId: bucket.botId,
70831
+ verificationStatus: bucket.verificationStatus,
70832
+ pathNormalized: bucket.pathNormalized,
70833
+ status,
70834
+ manifestId: bucket.verificationManifest.id,
70835
+ manifestJson: bucket.verificationManifest,
70836
+ hits: bucket.hits,
70837
+ createdAt: finishedAt,
70838
+ updatedAt: finishedAt
70839
+ }).onConflictDoUpdate({
70840
+ target: [
70841
+ crawlerVerificationManifestsHourly.projectId,
70842
+ crawlerVerificationManifestsHourly.sourceId,
70843
+ crawlerVerificationManifestsHourly.tsHour,
70844
+ crawlerVerificationManifestsHourly.botId,
70845
+ crawlerVerificationManifestsHourly.verificationStatus,
70846
+ crawlerVerificationManifestsHourly.pathNormalized,
70847
+ crawlerVerificationManifestsHourly.status,
70848
+ crawlerVerificationManifestsHourly.manifestId
70849
+ ],
70850
+ set: {
70851
+ hits: sql22`${crawlerVerificationManifestsHourly.hits} + ${bucket.hits}`,
70852
+ manifestJson: bucket.verificationManifest,
70853
+ updatedAt: finishedAt
70854
+ }
70855
+ }).run();
70664
70856
  }
70665
- }).run();
70666
- if (bucket.verificationManifest) {
70667
- tx.insert(aiUserFetchVerificationManifestsHourly).values({
70857
+ crawlerBucketRows += 1;
70858
+ }
70859
+ for (const bucket of report.aiUserFetchEventsHourly) {
70860
+ const status = bucket.status ?? 0;
70861
+ tx.insert(aiUserFetchEventsHourly).values({
70668
70862
  projectId: project.id,
70669
70863
  sourceId: sourceRow2.id,
70670
70864
  tsHour: bucket.tsHour,
70671
70865
  botId: bucket.botId,
70866
+ operator: bucket.operator,
70672
70867
  verificationStatus: bucket.verificationStatus,
70673
70868
  pathNormalized: bucket.pathNormalized,
70674
70869
  status,
70675
- manifestId: bucket.verificationManifest.id,
70676
- manifestJson: bucket.verificationManifest,
70677
70870
  hits: bucket.hits,
70871
+ sampledUserAgent: bucket.sampledUserAgent,
70678
70872
  createdAt: finishedAt,
70679
70873
  updatedAt: finishedAt
70680
70874
  }).onConflictDoUpdate({
70681
70875
  target: [
70682
- aiUserFetchVerificationManifestsHourly.projectId,
70683
- aiUserFetchVerificationManifestsHourly.sourceId,
70684
- aiUserFetchVerificationManifestsHourly.tsHour,
70685
- aiUserFetchVerificationManifestsHourly.botId,
70686
- aiUserFetchVerificationManifestsHourly.verificationStatus,
70687
- aiUserFetchVerificationManifestsHourly.pathNormalized,
70688
- aiUserFetchVerificationManifestsHourly.status,
70689
- aiUserFetchVerificationManifestsHourly.manifestId
70876
+ aiUserFetchEventsHourly.projectId,
70877
+ aiUserFetchEventsHourly.sourceId,
70878
+ aiUserFetchEventsHourly.tsHour,
70879
+ aiUserFetchEventsHourly.botId,
70880
+ aiUserFetchEventsHourly.verificationStatus,
70881
+ aiUserFetchEventsHourly.pathNormalized,
70882
+ aiUserFetchEventsHourly.status
70690
70883
  ],
70691
70884
  set: {
70692
- hits: sql22`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
70885
+ hits: sql22`${aiUserFetchEventsHourly.hits} + ${bucket.hits}`,
70886
+ sampledUserAgent: bucket.sampledUserAgent,
70887
+ updatedAt: finishedAt
70888
+ }
70889
+ }).run();
70890
+ if (bucket.verificationManifest) {
70891
+ tx.insert(aiUserFetchVerificationManifestsHourly).values({
70892
+ projectId: project.id,
70893
+ sourceId: sourceRow2.id,
70894
+ tsHour: bucket.tsHour,
70895
+ botId: bucket.botId,
70896
+ verificationStatus: bucket.verificationStatus,
70897
+ pathNormalized: bucket.pathNormalized,
70898
+ status,
70899
+ manifestId: bucket.verificationManifest.id,
70693
70900
  manifestJson: bucket.verificationManifest,
70901
+ hits: bucket.hits,
70902
+ createdAt: finishedAt,
70903
+ updatedAt: finishedAt
70904
+ }).onConflictDoUpdate({
70905
+ target: [
70906
+ aiUserFetchVerificationManifestsHourly.projectId,
70907
+ aiUserFetchVerificationManifestsHourly.sourceId,
70908
+ aiUserFetchVerificationManifestsHourly.tsHour,
70909
+ aiUserFetchVerificationManifestsHourly.botId,
70910
+ aiUserFetchVerificationManifestsHourly.verificationStatus,
70911
+ aiUserFetchVerificationManifestsHourly.pathNormalized,
70912
+ aiUserFetchVerificationManifestsHourly.status,
70913
+ aiUserFetchVerificationManifestsHourly.manifestId
70914
+ ],
70915
+ set: {
70916
+ hits: sql22`${aiUserFetchVerificationManifestsHourly.hits} + ${bucket.hits}`,
70917
+ manifestJson: bucket.verificationManifest,
70918
+ updatedAt: finishedAt
70919
+ }
70920
+ }).run();
70921
+ }
70922
+ aiUserFetchBucketRows += 1;
70923
+ }
70924
+ for (const bucket of report.aiReferralEventsHourly) {
70925
+ const status = bucket.status ?? 0;
70926
+ tx.insert(aiReferralEventsHourly).values({
70927
+ projectId: project.id,
70928
+ sourceId: sourceRow2.id,
70929
+ tsHour: bucket.tsHour,
70930
+ product: bucket.product,
70931
+ operator: bucket.operator,
70932
+ sourceDomain: bucket.sourceDomain,
70933
+ evidenceType: bucket.evidenceType,
70934
+ landingPathNormalized: bucket.landingPathNormalized,
70935
+ status,
70936
+ sessionsOrHits: bucket.hits,
70937
+ paidSessionsOrHits: bucket.paidHits,
70938
+ organicSessionsOrHits: bucket.organicHits,
70939
+ usersEstimated: null,
70940
+ createdAt: finishedAt,
70941
+ updatedAt: finishedAt
70942
+ }).onConflictDoUpdate({
70943
+ target: [
70944
+ aiReferralEventsHourly.projectId,
70945
+ aiReferralEventsHourly.sourceId,
70946
+ aiReferralEventsHourly.tsHour,
70947
+ aiReferralEventsHourly.product,
70948
+ aiReferralEventsHourly.sourceDomain,
70949
+ aiReferralEventsHourly.evidenceType,
70950
+ aiReferralEventsHourly.landingPathNormalized,
70951
+ aiReferralEventsHourly.status
70952
+ ],
70953
+ // All three counters accumulate. The class splits the measure, so a
70954
+ // colliding bucket adds its paid and organic arrivals to the ones
70955
+ // already there rather than overwriting a single label.
70956
+ set: {
70957
+ sessionsOrHits: sql22`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
70958
+ paidSessionsOrHits: sql22`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
70959
+ organicSessionsOrHits: sql22`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
70694
70960
  updatedAt: finishedAt
70695
70961
  }
70696
70962
  }).run();
70963
+ aiReferralBucketRows += 1;
70697
70964
  }
70698
- aiUserFetchBucketRows += 1;
70699
- }
70700
- for (const bucket of report.aiReferralEventsHourly) {
70701
- const status = bucket.status ?? 0;
70702
- tx.insert(aiReferralEventsHourly).values({
70703
- projectId: project.id,
70704
- sourceId: sourceRow2.id,
70705
- tsHour: bucket.tsHour,
70706
- product: bucket.product,
70707
- operator: bucket.operator,
70708
- sourceDomain: bucket.sourceDomain,
70709
- evidenceType: bucket.evidenceType,
70710
- landingPathNormalized: bucket.landingPathNormalized,
70711
- status,
70712
- sessionsOrHits: bucket.hits,
70713
- paidSessionsOrHits: bucket.paidHits,
70714
- organicSessionsOrHits: bucket.organicHits,
70715
- usersEstimated: null,
70716
- createdAt: finishedAt,
70965
+ for (const sample of report.samples) {
70966
+ const sampleTimestamp = retainedRawEventSampleTimestamp(sample.observedAt, rawSampleCutoff);
70967
+ if (!sampleTimestamp) continue;
70968
+ const eventType = sample.crawler ? "crawler" : sample.aiUserFetch ? "ai_user_fetch" : sample.aiReferral ? "ai_referral" : "unknown";
70969
+ const refererHost = (() => {
70970
+ if (!sample.referer) return null;
70971
+ try {
70972
+ return new URL(sample.referer).hostname;
70973
+ } catch {
70974
+ return null;
70975
+ }
70976
+ })();
70977
+ tx.insert(rawEventSamples).values({
70978
+ id: crypto42.randomUUID(),
70979
+ projectId: project.id,
70980
+ sourceId: sourceRow2.id,
70981
+ ts: sampleTimestamp,
70982
+ eventType,
70983
+ ipHash: null,
70984
+ userAgent: sample.userAgent,
70985
+ pathNormalized: sample.pathNormalized,
70986
+ status: sample.status,
70987
+ refererHost,
70988
+ classifierDetailsJson: {
70989
+ crawler: sample.crawler,
70990
+ aiUserFetch: sample.aiUserFetch,
70991
+ aiReferral: sample.aiReferral
70992
+ },
70993
+ createdAt: finishedAt
70994
+ }).run();
70995
+ sampleRows += 1;
70996
+ }
70997
+ const nextWatermark = sourceRow2.sourceType === TrafficSourceTypes.wordpress && nextCursor ? windowStart : effectiveWindowEnd;
70998
+ const sourceUpdate = {
70999
+ status: TrafficSourceStatuses.connected,
71000
+ // Advance to nextWatermark, not finishedAt — events arriving at the
71001
+ // source between the window end and finishedAt aren't in this pull's
71002
+ // range. If we stored finishedAt, the next sync's clamp would skip past
71003
+ // them and they'd be lost. A capped WordPress drain retains its lower
71004
+ // bound; full pulls use effectiveWindowEnd, including a partial Vercel
71005
+ // boundary, so the next sync resumes exactly where this one left off.
71006
+ //
71007
+ // Never move the watermark BACKWARD. A sync already in flight when the
71008
+ // cursor is advanced out from under it (an operator reset, or a backfill
71009
+ // committing a later window) would otherwise commit its own older
71010
+ // window end and undo that advance — the source silently resumes from
71011
+ // the past and re-walks ground that was deliberately skipped. Backfill
71012
+ // has always guarded this; the incremental path did not, so a reset only
71013
+ // stuck if you first disabled the schedule and drained the in-flight run.
71014
+ lastSyncedAt: new Date(
71015
+ Math.max(
71016
+ latestRow.lastSyncedAt ? new Date(latestRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY,
71017
+ nextWatermark.getTime()
71018
+ )
71019
+ ).toISOString(),
71020
+ lastError: null,
71021
+ lastEventIds: nextEventIds,
70717
71022
  updatedAt: finishedAt
70718
- }).onConflictDoUpdate({
70719
- target: [
70720
- aiReferralEventsHourly.projectId,
70721
- aiReferralEventsHourly.sourceId,
70722
- aiReferralEventsHourly.tsHour,
70723
- aiReferralEventsHourly.product,
70724
- aiReferralEventsHourly.sourceDomain,
70725
- aiReferralEventsHourly.evidenceType,
70726
- aiReferralEventsHourly.landingPathNormalized,
70727
- aiReferralEventsHourly.status
70728
- ],
70729
- // All three counters accumulate. The class splits the measure, so a
70730
- // colliding bucket adds its paid and organic arrivals to the ones
70731
- // already there rather than overwriting a single label.
70732
- set: {
70733
- sessionsOrHits: sql22`${aiReferralEventsHourly.sessionsOrHits} + ${bucket.hits}`,
70734
- paidSessionsOrHits: sql22`${aiReferralEventsHourly.paidSessionsOrHits} + ${bucket.paidHits}`,
70735
- organicSessionsOrHits: sql22`${aiReferralEventsHourly.organicSessionsOrHits} + ${bucket.organicHits}`,
70736
- updatedAt: finishedAt
70737
- }
70738
- }).run();
70739
- aiReferralBucketRows += 1;
71023
+ };
71024
+ if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
71025
+ sourceUpdate.lastCursor = nextCursor ?? null;
71026
+ sourceUpdate.wordpressPendingUntil = nextCursor ? windowEnd.toISOString() : null;
71027
+ }
71028
+ tx.update(trafficSources).set(sourceUpdate).where(eq50(trafficSources.id, sourceRow2.id)).run();
71029
+ tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq50(runs.id, runId)).run();
71030
+ return "committed";
71031
+ }, { behavior: "immediate" });
71032
+ if (commitOutcome === "source-inactive") {
71033
+ throw validationError("Traffic source is no longer active; discarded the in-flight sync");
70740
71034
  }
70741
- for (const sample of report.samples) {
70742
- const sampleTimestamp = retainedRawEventSampleTimestamp(sample.observedAt, rawSampleCutoff);
70743
- if (!sampleTimestamp) continue;
70744
- const eventType = sample.crawler ? "crawler" : sample.aiUserFetch ? "ai_user_fetch" : sample.aiReferral ? "ai_referral" : "unknown";
70745
- const refererHost = (() => {
70746
- if (!sample.referer) return null;
70747
- try {
70748
- return new URL(sample.referer).hostname;
70749
- } catch {
70750
- return null;
70751
- }
70752
- })();
70753
- tx.insert(rawEventSamples).values({
70754
- id: crypto42.randomUUID(),
70755
- projectId: project.id,
70756
- sourceId: sourceRow2.id,
70757
- ts: sampleTimestamp,
70758
- eventType,
70759
- ipHash: null,
70760
- userAgent: sample.userAgent,
70761
- pathNormalized: sample.pathNormalized,
70762
- status: sample.status,
70763
- refererHost,
70764
- classifierDetailsJson: {
70765
- crawler: sample.crawler,
70766
- aiUserFetch: sample.aiUserFetch,
70767
- aiReferral: sample.aiReferral
70768
- },
70769
- createdAt: finishedAt
70770
- }).run();
70771
- sampleRows += 1;
71035
+ writeAuditLog(app.db, {
71036
+ projectId: project.id,
71037
+ actor: "api",
71038
+ action: auditAction,
71039
+ entityType: "traffic_source",
71040
+ entityId: sourceRow2.id
71041
+ });
71042
+ if (selfTrafficExcludedCount > 0) {
71043
+ request.log.info(
71044
+ { sourceId: sourceRow2.id, selfTrafficExcluded: selfTrafficExcludedCount },
71045
+ "Dropped Canonry self-traffic before rollup; excluded from pulledEvents"
71046
+ );
70772
71047
  }
70773
- const sourceUpdate = {
70774
- status: TrafficSourceStatuses.connected,
70775
- // Advance to effectiveWindowEnd, not finishedAt — events arriving at the
70776
- // source between the window end and finishedAt aren't in this pull's
70777
- // range. If we stored finishedAt, the next sync's clamp would skip past
70778
- // them and they'd be lost. effectiveWindowEnd equals windowEnd on a full
70779
- // sync; for a Vercel drain that stopped at its deadline it is the partial
70780
- // boundary, so the next sync resumes exactly where this one left off.
70781
- //
70782
- // Never move the watermark BACKWARD. A sync already in flight when the
70783
- // cursor is advanced out from under it (an operator reset, or a backfill
70784
- // committing a later window) would otherwise commit its own older
70785
- // window end and undo that advance — the source silently resumes from
70786
- // the past and re-walks ground that was deliberately skipped. Backfill
70787
- // has always guarded this; the incremental path did not, so a reset only
70788
- // stuck if you first disabled the schedule and drained the in-flight run.
70789
- lastSyncedAt: new Date(
70790
- Math.max(
70791
- latestRow.lastSyncedAt ? new Date(latestRow.lastSyncedAt).getTime() : Number.NEGATIVE_INFINITY,
70792
- effectiveWindowEnd.getTime()
70793
- )
70794
- ).toISOString(),
70795
- lastError: null,
70796
- lastEventIds: nextEventIds,
70797
- updatedAt: finishedAt
70798
- };
70799
- if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
70800
- sourceUpdate.lastCursor = nextCursor ?? null;
71048
+ try {
71049
+ opts.onTrafficSynced?.({
71050
+ status: "completed",
71051
+ sourceType: sourceRow2.sourceType,
71052
+ sourceId: sourceRow2.id,
71053
+ pulledEvents: pulledEventsCount,
71054
+ selfTrafficExcluded: selfTrafficExcludedCount,
71055
+ crawlerHits: crawlerHitsCount,
71056
+ aiUserFetchHits: aiUserFetchHitsCount,
71057
+ aiReferralHits: aiReferralHitsCount,
71058
+ durationMs: Date.now() - syncStartedAtMs
71059
+ });
71060
+ } catch {
70801
71061
  }
70802
- tx.update(trafficSources).set(sourceUpdate).where(eq50(trafficSources.id, sourceRow2.id)).run();
70803
- tx.update(runs).set({ status: RunStatuses.completed, finishedAt }).where(eq50(runs.id, runId)).run();
70804
- return "committed";
70805
- });
70806
- if (commitOutcome === "source-inactive") {
70807
- throw validationError("Traffic source is no longer active; discarded the in-flight sync");
70808
- }
70809
- writeAuditLog(app.db, {
70810
- projectId: project.id,
70811
- actor: "api",
70812
- action: auditAction,
70813
- entityType: "traffic_source",
70814
- entityId: sourceRow2.id
70815
- });
70816
- if (selfTrafficExcludedCount > 0) {
70817
- request.log.info(
70818
- { sourceId: sourceRow2.id, selfTrafficExcluded: selfTrafficExcludedCount },
70819
- "Dropped Canonry self-traffic before rollup; excluded from pulledEvents"
70820
- );
70821
- }
70822
- try {
70823
- opts.onTrafficSynced?.({
70824
- status: "completed",
70825
- sourceType: sourceRow2.sourceType,
71062
+ const response = {
70826
71063
  sourceId: sourceRow2.id,
71064
+ runId,
71065
+ syncedAt: finishedAt,
70827
71066
  pulledEvents: pulledEventsCount,
70828
71067
  selfTrafficExcluded: selfTrafficExcludedCount,
70829
71068
  crawlerHits: crawlerHitsCount,
70830
71069
  aiUserFetchHits: aiUserFetchHitsCount,
70831
71070
  aiReferralHits: aiReferralHitsCount,
70832
- durationMs: Date.now() - syncStartedAtMs
70833
- });
70834
- } catch {
71071
+ unknownHits: unknownHitsCount,
71072
+ crawlerBucketRows,
71073
+ aiUserFetchBucketRows,
71074
+ aiReferralBucketRows,
71075
+ sampleRows,
71076
+ windowStart: windowStart.toISOString(),
71077
+ // The window actually synced: equals windowEnd on a full sync, or the
71078
+ // partial boundary when a Vercel drain stopped at its deadline.
71079
+ windowEnd: effectiveWindowEnd.toISOString()
71080
+ };
71081
+ return response;
71082
+ } finally {
71083
+ if (wordpressLeaseOwner) {
71084
+ try {
71085
+ releaseTrafficSyncLease({
71086
+ db: app.db,
71087
+ sourceId: sourceRow2.id,
71088
+ owner: wordpressLeaseOwner,
71089
+ now: (/* @__PURE__ */ new Date()).toISOString()
71090
+ });
71091
+ } catch (error) {
71092
+ request.log.warn({ sourceId: sourceRow2.id, error: describeError(error) }, "Failed to release WordPress sync lease");
71093
+ }
71094
+ }
70835
71095
  }
70836
- const response = {
70837
- sourceId: sourceRow2.id,
70838
- runId,
70839
- syncedAt: finishedAt,
70840
- pulledEvents: pulledEventsCount,
70841
- selfTrafficExcluded: selfTrafficExcludedCount,
70842
- crawlerHits: crawlerHitsCount,
70843
- aiUserFetchHits: aiUserFetchHitsCount,
70844
- aiReferralHits: aiReferralHitsCount,
70845
- unknownHits: unknownHitsCount,
70846
- crawlerBucketRows,
70847
- aiUserFetchBucketRows,
70848
- aiReferralBucketRows,
70849
- sampleRows,
70850
- windowStart: windowStart.toISOString(),
70851
- // The window actually synced: equals windowEnd on a full sync, or the
70852
- // partial boundary when a Vercel drain stopped at its deadline.
70853
- windowEnd: effectiveWindowEnd.toISOString()
70854
- };
70855
- return response;
70856
71096
  });
70857
71097
  app.post("/projects/:name/traffic/sources/:id/backfill", async (request) => {
70858
71098
  const project = resolveProject(app.db, request.params.name);
@@ -70862,12 +71102,17 @@ async function trafficRoutes(app, opts) {
70862
71102
  }
70863
71103
  if (sourceRow2.sourceType !== TrafficSourceTypes["cloud-run"] && sourceRow2.sourceType !== TrafficSourceTypes.wordpress && sourceRow2.sourceType !== TrafficSourceTypes.vercel) {
70864
71104
  throw validationError(
70865
- `Backfill for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 only cloud-run, wordpress, and vercel are supported in v1.`
71105
+ `Backfill for source type "${sourceRow2.sourceType}" is not implemented yet \u2014 generic replace backfill supports only cloud-run and vercel; WordPress requires a retention-aware repair.`
70866
71106
  );
70867
71107
  }
70868
71108
  if (!isAuthoritativeTrafficSource(app.db, sourceRow2)) {
70869
71109
  throw validationError("Traffic source must be active before it can backfill");
70870
71110
  }
71111
+ if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
71112
+ throw validationError(
71113
+ "Generic WordPress replace backfill is unavailable because retained coverage is unproven. Use a retention-aware repair that declares the unrecoverable span."
71114
+ );
71115
+ }
70871
71116
  const requestedDays = request.body?.days ?? DEFAULT_BACKFILL_DAYS;
70872
71117
  if (!Number.isInteger(requestedDays) || requestedDays <= 0) {
70873
71118
  throw validationError('"days" must be a positive integer');
@@ -70914,54 +71159,6 @@ async function trafficRoutes(app, opts) {
70914
71159
  });
70915
71160
  return page.events;
70916
71161
  };
70917
- } else if (sourceRow2.sourceType === TrafficSourceTypes.wordpress) {
70918
- const credentialStore = opts.wordpressTrafficCredentialStore;
70919
- if (!credentialStore) {
70920
- throw validationError("WordPress traffic credential storage is not configured for this deployment");
70921
- }
70922
- const credential = credentialStore.getConnection(project.name);
70923
- if (!credential) {
70924
- throw validationError(
70925
- `No WordPress credential found for project "${project.name}". Run "canonry traffic connect wordpress" first.`
70926
- );
70927
- }
70928
- await (await assertWordpressTargetAllowed(credential.baseUrl)).close().catch(() => {
70929
- });
70930
- const wpPageSize = opts.defaultWordpressPageSize ?? DEFAULT_WP_PAGE_SIZE;
70931
- pullErrorPrefix = "WordPress pull failed";
70932
- pullForBackfill = async () => {
70933
- const pinnedDispatcher = await assertWordpressTargetAllowed(credential.baseUrl);
70934
- try {
70935
- const collected = [];
70936
- const windowStartIso = windowStart.toISOString();
70937
- const windowEndIso = windowEnd.toISOString();
70938
- let cursor = void 0;
70939
- for (let page = 0; page < BACKFILL_MAX_PAGES; page += 1) {
70940
- const pageResult = await pullWordpressEvents({
70941
- baseUrl: credential.baseUrl,
70942
- username: credential.username,
70943
- applicationPassword: credential.applicationPassword,
70944
- cursor,
70945
- pageSize: wpPageSize,
70946
- // Each call fetches a single page; the for-loop drives
70947
- // continuation. Matches the WP sync path's pattern.
70948
- maxPages: 1,
70949
- since: windowStartIso,
70950
- until: windowEndIso,
70951
- dispatcher: pinnedDispatcher
70952
- });
70953
- collected.push(...pageResult.events);
70954
- const previousCursor = cursor;
70955
- cursor = pageResult.nextCursor;
70956
- if (!pageResult.hasMore) break;
70957
- if (!cursor || cursor === previousCursor) break;
70958
- }
70959
- return collected;
70960
- } finally {
70961
- await pinnedDispatcher.close().catch(() => {
70962
- });
70963
- }
70964
- };
70965
71162
  } else {
70966
71163
  const credentialStore = opts.vercelTrafficCredentialStore;
70967
71164
  if (!credentialStore) {
@@ -71147,6 +71344,11 @@ async function trafficRoutes(app, opts) {
71147
71344
  validateTrafficSourceCredential(sourceRow2, project.name);
71148
71345
  tx.update(trafficSources).set({
71149
71346
  lastSyncedAt: now,
71347
+ ...sourceRow2.sourceType === TrafficSourceTypes.wordpress ? {
71348
+ lastCursor: null,
71349
+ wordpressPendingUntil: null,
71350
+ skippedThroughAt: now
71351
+ } : {},
71150
71352
  status: TrafficSourceStatuses.connected,
71151
71353
  lastError: null,
71152
71354
  updatedAt: now
@@ -73651,11 +73853,17 @@ var syncLagCheck = {
73651
73853
  };
73652
73854
  const unrecovered = measured.filter((m) => m.skippedThroughAt !== null);
73653
73855
  if (unrecovered.length > 0) {
73856
+ const unrecoveredWordpress = unrecovered.filter((m) => m.sourceType === "wordpress");
73857
+ const unrecoveredOtherSources = unrecovered.filter((m) => m.sourceType !== "wordpress");
73858
+ const remediation = [
73859
+ unrecoveredWordpress.length > 0 ? "For WordPress, do not run generic replace-mode backfill: the plugin feed cannot prove it still retains every bucket in the gap. Keep existing rollups intact and use a retention-aware repair that explicitly declares the unrecoverable span." : null,
73860
+ unrecoveredOtherSources.length > 0 ? "Recover each non-WordPress gap with `canonry traffic backfill <project> --source <id> --days N --wait`, choosing N so the window reaches back past the skipped instant and stays inside the provider's log retention. This clears only when a backfill actually covers the span." : null
73861
+ ].filter((part) => part !== null).join(" ");
73654
73862
  return {
73655
73863
  status: CheckStatuses.fail,
73656
73864
  code: "traffic.sync-lag.unrecovered-skip",
73657
73865
  summary: `${unrecovered.length} traffic source(s) previously skipped traffic that has not been backfilled: ${unrecovered.map((m) => `${m.displayName} (skipped through ${m.skippedThroughAt})`).join(", ")}. Current lag may look healthy; the gap does not close on its own.`,
73658
- remediation: "Recover the gap with `canonry traffic backfill <project> --source <id> --days N --wait`, choosing N so the window reaches back past the skipped instant and stays inside the provider's log retention. This clears only when a backfill actually covers the span.",
73866
+ remediation,
73659
73867
  details
73660
73868
  };
73661
73869
  }
@@ -77160,6 +77368,8 @@ async function apiRoutes(app, opts) {
77160
77368
  resolveCloudRunAccessToken: opts.resolveCloudRunAccessToken,
77161
77369
  wordpressTrafficCredentialStore: opts.wordpressTrafficCredentialStore,
77162
77370
  pullWordpressTrafficEvents: opts.pullWordpressTrafficEvents,
77371
+ defaultWordpressPageSize: opts.defaultWordpressPageSize,
77372
+ defaultWordpressMaxPages: opts.defaultWordpressMaxPages,
77163
77373
  vercelTrafficCredentialStore: opts.vercelTrafficCredentialStore,
77164
77374
  pullVercelTrafficEvents: opts.pullVercelTrafficEvents,
77165
77375
  vercelSyncDeadlineMs: opts.vercelSyncDeadlineMs,