cartethyia 2.5.9 → 2.6.0

package/src/observability/telemetry-buffer.ts

@@ -6,6 +6,8 @@ import { DEFAULT_BOUNDS } from "../config/network";
 import { type SourceSurface, type UsageRecord } from "../protocol/canonical-model";
 import { log } from "./logger";
 import { scrubSecrets } from "./redaction";
+import { clientNameFromUserAgent } from "./client-name";
+
 
 /**
  * Buffered durable telemetry writer. Coalesces per-request telemetry into
@@ -103,6 +105,12 @@ function telemetryEventRow(event: TelemetryEventInput) {
   // `cachedInputTokens` stays — `health` aggregates it into the Overview
   // cache-hit rate.
   return {
+    // Stable identity: the request's own id. Retries reuse the materialized
+    // row (the buffer retries the same rows across its five attempts), and
+    // the (tenant_id, request_id) unique boundary makes a re-drain or a
+    // cross-process race converge to one row via ON CONFLICT DO NOTHING
+    // instead of double-charging.
+    id: event.requestId,
     tenantId: event.tenantId,
     requestId: event.requestId,
     // Stamped here, not by the reader. A row without it has no real time, and a
@@ -115,6 +123,8 @@ function telemetryEventRow(event: TelemetryEventInput) {
     endpoint: event.endpoint ?? null,
     apiKeyId: event.apiKeyId ?? null,
     userAgent: event.userAgent ?? null,
+    // Canonical client grouping, stamped once so SQL breakdowns group by it.
+    clientName: clientNameFromUserAgent(event.userAgent),
     clientIp: event.clientIp ?? null,
     providerId: event.providerId ?? null,
     accountId: event.accountId ?? null,
@@ -153,6 +163,9 @@ export interface TelemetryBatchBufferOptions {
   readonly maxBatch?: number;
   readonly maxItems?: number;
   readonly maxBytes?: number;
+  /** Test seam: a deterministic hook before a retry timer is installed. */
+  readonly onRetry?: (attempt: number) => void | Promise<void>;
+  readonly store?: Pick<DrizzleTelemetryStore, "insertEvents">;
 }
 
 /** Cooldown between `[telemetry] failed to record` log lines during a sustained drain outage. */
@@ -166,8 +179,8 @@ const DRAIN_RETRY_MAX_MS = 1_000;
 import { exponentialBackoff, type BackoffOptions } from "../network/backoff";
 
 /** Exponential retry delay for the `attempt`-th retry, clamped to the ceiling. */
-function drainRetryDelayMs(attempt: number): number {
-  const options: BackoffOptions = { baseDelayMs: DRAIN_RETRY_BASE_MS, maxDelayMs: DRAIN_RETRY_MAX_MS };
+function drainRetryDelayMs(attempt: number, baseDelayMs = DRAIN_RETRY_BASE_MS): number {
+  const options: BackoffOptions = { baseDelayMs, maxDelayMs: DRAIN_RETRY_MAX_MS };
   return exponentialBackoff(attempt, options);
 }
 
@@ -192,6 +205,20 @@ export function adaptiveFlushIntervalMs(
   return Math.max(ADAPTIVE_FLUSH_MIN_MS, Math.round(baseMs * (1 - fill)));
 }
 
+/** The telemetry backlog's current shape — see {@link TelemetryBatchBuffer.snapshot}. */
+export interface TelemetryBacklogSnapshot {
+  /** Queued events waiting for the next drain. */
+  readonly items: number;
+  /** Estimated queued bytes. */
+  readonly bytes: number;
+  readonly itemCapacity: number;
+  readonly byteCapacity: number;
+  /** Whether a drain is currently in flight. */
+  readonly flushing: boolean;
+  /** Consecutive fully-failed drains; 0 when healthy. */
+  readonly drainFailureStreak: number;
+}
+
 export class TelemetryBatchBuffer {
   /**
    * One entry per queued event. `bytes` is a conservative enqueue-time
@@ -199,16 +226,17 @@ export class TelemetryBatchBuffer {
    * JSON serialization allocation on every request.
    */
   private readonly queue: Array<{ event: TelemetryEventInput; bytes: number }> = [];
-  private readonly store: DrizzleTelemetryStore;
+  private readonly store: Pick<DrizzleTelemetryStore, "insertEvents">;
   private readonly flushIntervalMs: number;
   private readonly maxBatch: number;
   private readonly maxItems: number;
   private readonly maxBytes: number;
+  private readonly onRetry: ((attempt: number) => void | Promise<void>) | undefined;
   private queuedBytes = 0;
   private timer: ReturnType<typeof setTimeout> | undefined;
   private stopped = false;
   private flushing = false;
-  private flushWait: Promise<void> = Promise.resolve();
+  private flushWait: Promise<boolean> = Promise.resolve(true);
   private lastDrainOutageAt = 0;
   private suppressedDrainOutages = 0;
   /** Consecutive fully-failed drains; drives the cross-drain outage backoff. */
@@ -220,11 +248,12 @@ export class TelemetryBatchBuffer {
     db: CartethyiaDatabase,
     opts: TelemetryBatchBufferOptions = {},
   ) {
-    this.store = new DrizzleTelemetryStore(db);
+    this.store = opts.store ?? new DrizzleTelemetryStore(db);
     this.flushIntervalMs = opts.flushIntervalMs ?? 500;
     this.maxBatch = opts.maxBatch ?? 500;
     this.maxItems = opts.maxItems ?? 10000;
     this.maxBytes = opts.maxBytes ?? DEFAULT_BOUNDS.maxTelemetryQueueBytes;
+    this.onRetry = opts.onRetry;
     this.scheduleNextFlush(this.flushIntervalMs);
   }
 
@@ -246,10 +275,31 @@ export class TelemetryBatchBuffer {
     if (typeof this.timer.unref === "function") this.timer.unref();
   }
 
-  /** Non-blocking; drops the event (and counts it) when the buffer is full by count or bytes. */
+  /**
+   * The backlog's current shape, for gauges and the health surface.
+   *
+   * The buffer already drops on its bounds, but a drop nobody can see is
+   * indistinguishable from "no traffic" — an operator watching the console
+   * during an outage would see flat telemetry and conclude nothing was
+   * happening, when in fact the queue was full and discarding.
+   */
+  snapshot(): TelemetryBacklogSnapshot {
+    return {
+      items: this.queue.length,
+      bytes: this.queuedBytes,
+      itemCapacity: this.maxItems,
+      byteCapacity: this.maxBytes,
+      flushing: this.flushing,
+      // The streak is the interesting one: a single failed drain is noise, a
+      // rising streak means the database is refusing writes and every event
+      // buffered past the cap is being dropped.
+      drainFailureStreak: this.drainFailureStreak,
+    };
+  }
+
   enqueue(event: TelemetryEventInput): void {
     const bytes = estimateTelemetryEventBytes(event);
-    if (this.queue.length >= this.maxItems || this.queuedBytes + bytes > this.maxBytes) {
+    if (this.stopped || this.queue.length >= this.maxItems || this.queuedBytes + bytes > this.maxBytes) {
       metrics.cartethyia_telemetry_dropped_total.inc();
       return;
     }
@@ -266,8 +316,8 @@ export class TelemetryBatchBuffer {
     }
   }
 
-  /** Drains the buffer with a single multi-row insert. Idempotent, and safe
-   *  against concurrent invocation: overlapping callers await the in-flight
+  /** Drains the buffer with a single multi-row insert. Safe against concurrent
+   *  invocation: overlapping callers await the in-flight
    *  drain and then re-enter until the queue is fully empty. Shutdown callers
    *  rely on this loop so they can observe an "empty queue" postcondition
    *  before closing the database pool, instead of returning immediately when
@@ -284,13 +334,16 @@ export class TelemetryBatchBuffer {
       }
       this.flushing = true;
       this.flushWait = (async () => {
+        let completed: boolean;
         try {
-          await this.drain();
+          completed = await this.drain();
         } finally {
           this.flushing = false;
         }
+        return completed;
       })();
-      await this.flushWait;
+      const completed = await this.flushWait;
+      if (!completed) return;
     }
   }
 
@@ -306,7 +359,23 @@ export class TelemetryBatchBuffer {
       clearTimeout(this.timer);
       this.timer = undefined;
     }
-    if (options?.flush) await this.flush();
+    if (options?.flush) {
+      await this.flush();
+    } else if (this.flushing) {
+      // A retry interrupted by stop returns its in-flight batch to the head
+      // of the queue. Await that handoff before accounting for a drop.
+      await this.flushWait;
+    }
+    this.dropRemaining();
+  }
+
+  /** Drops only after shutdown has made the owner/timer unavailable. */
+  private dropRemaining(): void {
+    if (this.queue.length === 0) return;
+    metrics.cartethyia_telemetry_dropped_total.inc(this.queue.length);
+    metrics.cartethyia_telemetry_buffered.inc(-this.queue.length);
+    this.queue.length = 0;
+    this.queuedBytes = 0;
   }
 
   /**
@@ -320,8 +389,8 @@ export class TelemetryBatchBuffer {
     return this.flush();
   }
 
-  private async drain(): Promise<void> {
-    if (this.queue.length === 0) return;
+  private async drain(): Promise<boolean> {
+    if (this.queue.length === 0) return true;
     const batch = this.queue.splice(0, Math.min(this.queue.length, this.maxBatch));
     // Reuse the enqueue-time serialized sizes — no second JSON pass.
     const batchBytes = batch.reduce((sum, item) => sum + item.bytes, 0);
@@ -335,10 +404,23 @@ export class TelemetryBatchBuffer {
         // drains probe the database less and less often. Unref'd so a
         // mid-shutdown backoff cannot hold the event loop open.
         const { promise, resolve } = Promise.withResolvers<void>();
-        const backoff = setTimeout(resolve, drainRetryDelayMs(this.drainFailureStreak + attempt - 1));
+        if (this.stopped) {
+          this.queue.unshift(...batch);
+          this.queuedBytes += batchBytes;
+          return false;
+        }
+        await this.onRetry?.(attempt);
+        const backoff = setTimeout(
+          resolve,
+          drainRetryDelayMs(this.drainFailureStreak + attempt - 1),
+        );
         if (typeof backoff.unref === "function") backoff.unref();
         await promise;
-        if (this.stopped) return;
+        if (this.stopped) {
+          this.queue.unshift(...batch);
+          this.queuedBytes += batchBytes;
+          return false;
+        }
       }
       try {
         await this.store.insertEvents(rows);
@@ -346,7 +428,7 @@ export class TelemetryBatchBuffer {
         this.drainFailureStreak = 0;
         this.drainBlockedUntil = 0;
         metrics.cartethyia_telemetry_buffered.inc(-batch.length);
-        return;
+        return true;
       } catch (error) {
         if (attempt === 0) firstError = error;
         lastError = error;
@@ -358,6 +440,7 @@ export class TelemetryBatchBuffer {
     this.logDrainOutage(lastError, firstError);
     metrics.cartethyia_telemetry_dropped_total.inc(batch.length);
     metrics.cartethyia_telemetry_buffered.inc(-batch.length);
+    return true;
   }
 
   /**

package/src/observability/telemetry-reconciliation.ts

@@ -0,0 +1,417 @@
+/**
+ * Reconciles the raw telemetry mirror back into the SQL table.
+ *
+ * The durable record is `telemetry_events`, written transactionally on the
+ * request path. The raw JSONL mirror is at-least-once: a drain that retried
+ * after a crash can have appended a line whose SQL row never landed (or landed
+ * and was rolled back). This job is how an operator recovers that gap, and how
+ * a deployment that ran before `telemetry_events` was written gets its history
+ * back.
+ *
+ * Design constraints, in the order they bind:
+ *
+ * - **Idempotent.** The unique index on `(tenant_id, request_id)` is the
+ *   boundary: `ON CONFLICT DO NOTHING` plus `RETURNING` means a re-run
+ *   inserts nothing and charges nothing. The checkpoint can therefore be
+ *   advanced optimistically and still be safe.
+ * - **Resumable.** Progress is a `(file_name, byte_offset)` checkpoint written
+ *   in the SAME transaction as the rows it covers, so a crash replays at most
+ *   one chunk and never loses a completed one.
+ * - **Leased.** Several processes share one database; a lease row
+ *   (`telemetry_reconciliation_state`) means one runner at a time. The lease
+ *   expires, so a crashed runner does not wedge the job forever.
+ * - **Bounded.** Chunked reads, a chunk cap, and a transaction cap. A 512 MiB
+ *   backlog drains over successive runs instead of one unbounded sweep.
+ * - **Never touches `telemetry_usage_totals`.** Those counters were charged at
+ *   request time. Backfilling rows must not double-charge them, so this path
+ *   inserts events only.
+ */
+import { and, eq, sql } from "drizzle-orm";
+import { readdir, open, stat } from "node:fs/promises";
+import { join } from "node:path";
+
+import type { CartethyiaDatabase } from "../persistence/db-handle";
+import { telemetryEvents, telemetryReconciliationState } from "../persistence/schema";
+import { TELEMETRY_EVENT_DIR } from "./event-log";
+import { clientNameFromUserAgent } from "./client-name";
+import type { TelemetryEventRecord } from "./event-log";
+import { hourStamp } from "../shared/time";
+
+/** The single job name: one reconciliation stream, one checkpoint. */
+const JOB_NAME = "raw_to_sql_v1";
+
+/** How long a runner holds the job. Generous, so a slow run is not stolen. */
+const LEASE_MS = 30 * 60_000;
+
+/** One read of a file. Bounds peak memory to one chunk, not one file. */
+const CHUNK_BYTES = 256 * 1024;
+
+/**
+ * Rows per transaction. Enough to amortise commit cost, small enough that a
+ * single transaction is never a long lock or a huge WAL burst.
+ */
+const ROWS_PER_TRANSACTION = 500;
+
+/** Chunks per run. A backlog drains over successive runs, not one sweep. */
+const MAX_CHUNKS_PER_RUN = 512;
+
+/** Files per run, so a fresh backlog is not capped by one huge file. */
+const MAX_FILES_PER_RUN = 64;
+
+/**
+ * The current and previous UTC hour are skipped: the current one is still
+ * being appended to, and the previous one can still receive a batch that was
+ * flushed after the boundary. Those rows arrive through the request path
+ * itself — reconciling them would race the writer for no gain.
+ */
+const HOURS_TO_SKIP = 2;
+
+export interface ReconcileOptions {
+  /** Report only: count the gap without writing anything. */
+  readonly verify?: boolean;
+  readonly now?: Date;
+}
+
+export interface ReconcileReport {
+  readonly ran: boolean;
+  /** Why the run did nothing, when it did nothing. */
+  readonly skippedReason?: "lease_held" | "no_files" | "uptodate";
+  readonly leaseOwner: string;
+  readonly filesCompleted: number;
+  readonly eventsExamined: number;
+  readonly eventsInserted: number;
+  readonly conflicts: number;
+  readonly bytesRead: number;
+}
+
+/**
+ * The checkpoint: which file, and how far into it.
+ *
+ * `fileName` is compared as a string and files are processed in name order,
+ * so an hour that rolls over does not resurrect an older file: once the
+ * checkpoint passes a name, files sorting before it never come back.
+ */
+interface Checkpoint {
+  readonly fileName: string;
+  readonly byteOffset: number;
+}
+
+function nowMs(options: ReconcileOptions): number {
+  return (options.now ?? new Date()).getTime();
+}
+
+/** Files eligible for reconciliation: JSONL logs, newest-completed-first is irrelevant here. */
+async function candidateFiles(): Promise<readonly string[]> {
+  const all = await readdir(TELEMETRY_EVENT_DIR).catch(() => [] as string[]);
+  // The current and previous hour are still being written; see HOURS_TO_SKIP.
+  const floor = hourStamp(new Date(Date.now() - HOURS_TO_SKIP * 3_600_000));
+  return all
+    .filter((file) => file.endsWith(".jsonl") && !file.endsWith(".compacted.jsonl"))
+    .filter((file) => file.slice(0, 13) < floor)
+    .sort();
+}
+
+/**
+ * Parses the complete lines in one chunk.
+ *
+ * `carry` holds the trailing partial line: the chunk boundary falls mid-line
+ * whenever the writer's last append was split. It is prepended to the next
+ * chunk rather than parsed, because a half line is not a JSON document.
+ */
+function parseChunk(text: string): { readonly events: TelemetryEventRecord[]; readonly carry: string } {
+  const lines = text.split("\n");
+  // The last element is either "" (the chunk ended on a newline, so every line
+  // is complete) or a partial line (keep it for the next chunk).
+  const carry = lines.pop() ?? "";
+  const events: TelemetryEventRecord[] = [];
+  for (const line of lines) {
+    if (line.length === 0) continue;
+    let value: unknown;
+    try {
+      value = JSON.parse(line);
+    } catch {
+      // A torn line from a crash mid-append. Skipped rather than fatal: the
+      // rest of the file is still recoverable, and one bad line must not
+      // strand a whole backlog.
+      continue;
+    }
+    if (typeof value !== "object" || value === null) continue;
+    const event = value as Partial<TelemetryEventRecord>;
+    if (typeof event.tenantId !== "string" || typeof event.requestId !== "string") continue;
+    events.push(value as TelemetryEventRecord);
+  }
+  return { events, carry };
+}
+
+/** The columns the backfill inserts. Mirrors `telemetryEventRow` on the write path. */
+function toRow(event: TelemetryEventRecord): typeof telemetryEvents.$inferInsert {
+  return {
+    // `id = requestId` is what makes the unique index a working idempotency
+    // boundary: a retry of the same request carries the same id.
+    id: event.requestId,
+    tenantId: event.tenantId!,
+    requestId: event.requestId,
+    sourceSurface: event.sourceSurface as typeof telemetryEvents.$inferInsert.sourceSurface,
+    requestedModel: event.requestedModel,
+    reportedModel: event.reportedModel,
+    requestedEffort: event.requestedEffort,
+    providerId: event.providerId,
+    accountId: event.accountId,
+    networkPoolId: event.networkPoolId,
+    endpoint: event.endpoint,
+    apiKeyId: event.apiKeyId,
+    userAgent: event.userAgent,
+    // Derived the same way ingestion derives it, so a backfilled row groups
+    // with the rows the request path wrote instead of forming its own bucket.
+    clientName: clientNameFromUserAgent(event.userAgent),
+    clientIp: event.clientIp,
+    latencyMs: event.latencyMs,
+    ttfbMs: event.ttfbMs,
+    stream: event.stream,
+    httpStatus: event.httpStatus,
+    status: event.status as typeof telemetryEvents.$inferInsert.status,
+    errorCategory: event.errorCategory,
+    errorOrigin: event.errorOrigin,
+    inputTokens: event.inputTokens,
+    cachedInputTokens: event.cachedInputTokens,
+    outputTokens: event.outputTokens,
+    reasoningTokens: event.reasoningTokens,
+    estimatedCostUsd:
+      event.estimatedCostUsd === null ? null : String(event.estimatedCostUsd),
+    createdAt: new Date(event.createdAt),
+  };
+}
+
+/**
+ * Acquires (or steals an expired) lease.
+ *
+ * `insert ... on conflict` in one statement: the row is created on first run
+ * and refreshed afterwards without a read-modify-write that two runners could
+ * interleave. `lease_expires_at` is the only arbiter, so a runner that died
+ * mid-file is replaced once its lease lapses rather than blocking forever.
+ */
+async function acquireLease(db: CartethyiaDatabase, owner: string, now: Date): Promise<boolean> {
+  const expiresAt = new Date(now.getTime() + LEASE_MS);
+  const result = await db
+    .insert(telemetryReconciliationState)
+    .values({
+      jobName: JOB_NAME,
+      leaseOwner: owner,
+      leaseExpiresAt: expiresAt,
+      fileName: "",
+      byteOffset: 0,
+      filesCompleted: 0,
+      eventsExamined: 0,
+      eventsInserted: 0,
+      conflicts: 0,
+      updatedAt: now,
+    })
+    .onConflictDoUpdate({
+      target: telemetryReconciliationState.jobName,
+      set: { leaseOwner: owner, leaseExpiresAt: expiresAt, updatedAt: now },
+      // Take the lease only when nobody holds a live one: an expired lease or
+      // no lease at all means the previous runner is gone.
+      where: sql`(${telemetryReconciliationState.leaseExpiresAt} is null or ${telemetryReconciliationState.leaseExpiresAt} < ${now} or ${telemetryReconciliationState.leaseOwner} = ${owner})`,
+    })
+    .returning({ owner: telemetryReconciliationState.leaseOwner });
+  return result.length > 0;
+}
+
+async function releaseLease(db: CartethyiaDatabase, owner: string, now: Date): Promise<void> {
+  await db
+    .update(telemetryReconciliationState)
+    .set({ leaseExpiresAt: null, updatedAt: now })
+    .where(
+      and(
+        eq(telemetryReconciliationState.jobName, JOB_NAME),
+        eq(telemetryReconciliationState.leaseOwner, owner),
+      ),
+    );
+}
+
+async function readCheckpoint(db: CartethyiaDatabase): Promise<Checkpoint> {
+  const [row] = await db
+    .select({
+      fileName: telemetryReconciliationState.fileName,
+      byteOffset: telemetryReconciliationState.byteOffset,
+    })
+    .from(telemetryReconciliationState)
+    .where(eq(telemetryReconciliationState.jobName, JOB_NAME))
+    .limit(1);
+  return { fileName: row?.fileName ?? "", byteOffset: row?.byteOffset ?? 0 };
+}
+
+/**
+ * One run of the reconciliation job.
+ *
+ * Returns a report rather than throwing on "nothing to do": a scheduled or
+ * CLI invocation has to distinguish "no gap" from "could not run", and a
+ * thrown error would read as a failure of the whole command.
+ */
+export async function reconcileTelemetry(
+  db: CartethyiaDatabase,
+  options: ReconcileOptions = {},
+): Promise<ReconcileReport> {
+  const owner = `${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
+  const now = new Date(nowMs(options));
+  const verify = options.verify === true;
+
+  const files = await candidateFiles();
+  if (files.length === 0) {
+    return {
+      ran: false,
+      skippedReason: "no_files",
+      leaseOwner: owner,
+      filesCompleted: 0,
+      eventsExamined: 0,
+      eventsInserted: 0,
+      conflicts: 0,
+      bytesRead: 0,
+    };
+  }
+
+  // `--verify` takes no lease: it only reads the mirror and counts, so two
+  // concurrent verifications cannot conflict and neither can block a real run.
+  if (!verify && !(await acquireLease(db, owner, now))) {
+    return {
+      ran: false,
+      skippedReason: "lease_held",
+      leaseOwner: owner,
+      filesCompleted: 0,
+      eventsExamined: 0,
+      eventsInserted: 0,
+      conflicts: 0,
+      bytesRead: 0,
+    };
+  }
+
+  const checkpoint = verify ? { fileName: "", byteOffset: 0 } : await readCheckpoint(db);
+  const pending = files.filter((file) => file > checkpoint.fileName);
+  if (pending.length === 0) {
+    if (!verify) await releaseLease(db, owner, new Date());
+    return {
+      ran: false,
+      skippedReason: "uptodate",
+      leaseOwner: owner,
+      filesCompleted: 0,
+      eventsExamined: 0,
+      eventsInserted: 0,
+      conflicts: 0,
+      bytesRead: 0,
+    };
+  }
+
+  let eventsExamined = 0;
+  let eventsInserted = 0;
+  let conflicts = 0;
+  let filesCompleted = 0;
+  let bytesRead = 0;
+  let chunks = 0;
+
+  try {
+    for (const file of pending.slice(0, MAX_FILES_PER_RUN)) {
+      const path = join(TELEMETRY_EVENT_DIR, file);
+      const size = (await stat(path).catch(() => undefined))?.size ?? 0;
+      const handle = await open(path, "r").catch(() => undefined);
+      if (handle === undefined) {
+        // Deleted between the listing and the open — retention pruned it.
+        continue;
+      }
+      let offset = file === checkpoint.fileName ? checkpoint.byteOffset : 0;
+      let carry = "";
+      try {
+        while (offset < size && chunks < MAX_CHUNKS_PER_RUN) {
+          const buffer = Buffer.alloc(Math.min(CHUNK_BYTES, size - offset));
+          const { bytesRead: read } = await handle.read(buffer, 0, buffer.length, offset);
+          if (read <= 0) break;
+          chunks += 1;
+          bytesRead += read;
+          offset += read;
+          const parsed = parseChunk(carry + buffer.toString("utf8", 0, read));
+          carry = parsed.carry;
+          eventsExamined += parsed.events.length;
+          if (parsed.events.length > 0 && !verify) {
+            for (let start = 0; start < parsed.events.length; start += ROWS_PER_TRANSACTION) {
+              const slice = parsed.events.slice(start, start + ROWS_PER_TRANSACTION);
+              const inserted = await insertChunk(db, slice, file, offset);
+              eventsInserted += inserted.inserted;
+              conflicts += inserted.conflicts;
+            }
+          }
+        }
+      } finally {
+        await handle.close().catch(() => undefined);
+      }
+      // Only complete files are marked done: a run that hit its chunk cap
+      // resumes inside this file, so the offset must stay.
+      if (offset >= size) {
+        filesCompleted += 1;
+        if (!verify) await writeCheckpoint(db, file, 0, now);
+      } else if (!verify) {
+        await writeCheckpoint(db, file, offset, now);
+      }
+      if (chunks >= MAX_CHUNKS_PER_RUN) break;
+    }
+  } finally {
+    if (!verify) await releaseLease(db, owner, new Date());
+  }
+
+  return {
+    ran: true,
+    leaseOwner: owner,
+    filesCompleted,
+    eventsExamined,
+    eventsInserted,
+    conflicts,
+    bytesRead,
+  };
+}
+
+/**
+ * Inserts one batch and advances the checkpoint in the SAME transaction.
+ *
+ * `ON CONFLICT DO NOTHING` + `RETURNING` is the idempotency boundary: the
+ * unique index on `(tenant_id, request_id)` decides, and only returned rows
+ * count. Usage totals are deliberately not touched — those counters were
+ * charged at request time, and re-charging from a backfill would inflate
+ * every dashboard figure for the affected window.
+ */
+async function insertChunk(
+  db: CartethyiaDatabase,
+  events: readonly TelemetryEventRecord[],
+  fileName: string,
+  byteOffset: number,
+): Promise<{ readonly inserted: number; readonly conflicts: number }> {
+  return db.transaction(async (tx) => {
+    const rows = await tx
+      .insert(telemetryEvents)
+      .values(events.map(toRow))
+      .onConflictDoNothing()
+      .returning({ id: telemetryEvents.id });
+    await tx
+      .update(telemetryReconciliationState)
+      .set({
+        fileName,
+        byteOffset,
+        filesCompleted: sql`${telemetryReconciliationState.filesCompleted} + 0`,
+        eventsExamined: sql`${telemetryReconciliationState.eventsExamined} + ${events.length}`,
+        eventsInserted: sql`${telemetryReconciliationState.eventsInserted} + ${rows.length}`,
+        conflicts: sql`${telemetryReconciliationState.conflicts} + ${events.length - rows.length}`,
+        updatedAt: new Date(),
+      })
+      .where(eq(telemetryReconciliationState.jobName, JOB_NAME));
+    return { inserted: rows.length, conflicts: events.length - rows.length };
+  });
+}
+
+async function writeCheckpoint(
+  db: CartethyiaDatabase,
+  fileName: string,
+  byteOffset: number,
+  now: Date,
+): Promise<void> {
+  await db
+    .update(telemetryReconciliationState)
+    .set({ fileName, byteOffset, updatedAt: now })
+    .where(eq(telemetryReconciliationState.jobName, JOB_NAME));
+}