@usherlabs/cex-broker 0.2.43 → 0.2.44

package/README.md

@@ -735,7 +735,7 @@ Every successful `CreateOrder` response, successful `GetOrderDetails` response,
 - Execution values: requested quantity/notional, executed base quantity, executed quote quantity/cost, average execution price, filled amount, remaining amount, fee amount, fee currency, fee rate
 - Timing: exchange timestamp when present and broker observed timestamp
 
-Use metrics for aggregations and alerts. For the durable execution audit trail, the broker archives every order lifecycle event to `broker_execution.order_events` (and pre-order top-of-book to `broker_execution.market_metadata_snapshots`) through the **archive forwarder** — the same HTTP `/archive` → ClickHouse path used for `market_data.*`. Set `CEX_BROKER_ARCHIVE_ENABLED=true`, an explicit HTTP(S) `CEX_BROKER_ARCHIVE_FORWARDER_URL`, and a writable durable JSONL path in `CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH`; startup fails if either required sink configuration is missing or invalid. In production, that path must be on persistent writable storage or a mounted volume rather than the container's ephemeral filesystem. Queue shedding and rows that remain undeliverable during shutdown are written to that loss journal with their original `{table,row}` payload before being discarded. Setting `CEX_BROKER_ARCHIVE_OTEL_LOGS_ENABLED=true` additionally mirrors execution rows to OTel logs for observability, but OTel is never the archive sink of record. Analysts join Maker action rows to `broker_execution.order_events` using `maker_action_id`, `idempotency_id`, `client_order_id`, or the exchange `order_id`, then compare Maker propAMM execution price against `average_execution_price` and fees. Failed CreateOrder rows keep bounded exchange error detail in `error_message`; their telemetry-shaped `payload_json`, metrics, and ordinary telemetry logs remain redacted. The broker does not emit raw exchange payloads, API keys, secrets, or credentials in telemetry fields.
+Use metrics for aggregations and alerts. For the durable execution audit trail, the broker archives every order lifecycle event to `broker_execution.order_events` (and pre-order top-of-book to `broker_execution.market_metadata_snapshots`) through the **archive forwarder** — the same HTTP `/archive` → ClickHouse path used for `market_data.*`. Set `CEX_BROKER_ARCHIVE_ENABLED=true`, an explicit HTTP(S) `CEX_BROKER_ARCHIVE_FORWARDER_URL`, and a writable durable JSONL path in `CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH`; startup fails if either required sink configuration is missing or invalid. In production, that path must be on persistent writable storage or a mounted volume rather than the container's ephemeral filesystem. Queue shedding, rows that remain undeliverable during shutdown, and batches the forwarder rejected on every attempt are written to that loss journal with their original `{table,row}` payload before being discarded. Replay them with `bun run archive:loss:replay --journal <path>`, which re-posts through the forwarder and reports a per-entry verdict; it is idempotent across reruns, never modifies the journal, and only archives it once every entry is terminal and the operator asserts `--broker-stopped`. When the journal lives on an encrypted enclave mount (SGX deployments), its host-side bytes are sealed ciphertext the replay tool cannot read; set `CEX_BROKER_ARCHIVE_DEAD_LETTER_EXPORT_PATH` to a path on a plain (untrusted) mount and the broker exports a boot-time plaintext snapshot of the journal there, with an adjacent `.sha256` receipt whose digest is also logged from inside the enclave for integrity comparison. The export is one-shot — an existing target file is left untouched, so remove it to request a fresh snapshot — and a failed export fails startup. Setting `CEX_BROKER_ARCHIVE_OTEL_LOGS_ENABLED=true` additionally mirrors execution rows to OTel logs for observability, but OTel is never the archive sink of record. Analysts join Maker action rows to `broker_execution.order_events` using `maker_action_id`, `idempotency_id`, `client_order_id`, or the exchange `order_id`, then compare Maker propAMM execution price against `average_execution_price` and fees. Failed CreateOrder rows keep bounded exchange error detail in `error_message`; their telemetry-shaped `payload_json`, metrics, and ordinary telemetry logs remain redacted. The broker does not emit raw exchange payloads, API keys, secrets, or credentials in telemetry fields.
 
 ### User-stream health archive contract
 

package/dist/commands/cli.js

@@ -315811,11 +315811,118 @@ function compactUndefined2(record) {
 // src/helpers/broker-execution-archive/writer.ts
 var import_api_logs2 = __toESM(require_src7(), 1);
 import { randomUUID } from "node:crypto";
-import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
+import { closeSync as closeSync2, fsyncSync as fsyncSync2, openSync as openSync2, writeSync as writeSync2 } from "node:fs";
 import {
   request as httpRequest2
 } from "node:http";
 import { request as httpsRequest2 } from "node:https";
+
+// src/helpers/broker-execution-archive/journal-export.ts
+import { createHash as createHash3 } from "node:crypto";
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  openSync,
+  readSync,
+  renameSync,
+  writeSync
+} from "node:fs";
+import { basename } from "node:path";
+var EXPORT_CHUNK_BYTES = 8 * 1024 * 1024;
+
+class DeadLetterJournalExportError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "DeadLetterJournalExportError";
+  }
+}
+function exportDeadLetterJournalFromEnv() {
+  const exportPath = process.env.CEX_BROKER_ARCHIVE_DEAD_LETTER_EXPORT_PATH?.trim();
+  if (!exportPath) {
+    return { status: "disabled" };
+  }
+  const journalPath = process.env.CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH?.trim();
+  if (!journalPath) {
+    throw new DeadLetterJournalExportError("CEX_BROKER_ARCHIVE_DEAD_LETTER_EXPORT_PATH is set but CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH is missing");
+  }
+  return exportDeadLetterJournal(journalPath, exportPath);
+}
+function exportDeadLetterJournal(journalPath, exportPath) {
+  if (existsSync(exportPath)) {
+    log.info("Dead-letter journal export target already exists, skipping", {
+      export_path: exportPath
+    });
+    return { status: "skipped_export_exists", exportPath };
+  }
+  if (!existsSync(journalPath)) {
+    log.warn("Dead-letter journal export requested but no journal exists, skipping", { journal_path: journalPath });
+    return { status: "skipped_missing_journal", journalPath };
+  }
+  const partialPath = `${exportPath}.partial`;
+  let bytes2 = 0;
+  let digest;
+  try {
+    const hash4 = createHash3("sha256");
+    const sourceFd = openSync(journalPath, "r");
+    try {
+      const targetFd = openSync(partialPath, "w", 384);
+      try {
+        const chunk = Buffer.alloc(EXPORT_CHUNK_BYTES);
+        for (;; ) {
+          const read = readSync(sourceFd, chunk, 0, chunk.length, null);
+          if (read === 0) {
+            break;
+          }
+          const view = chunk.subarray(0, read);
+          hash4.update(view);
+          writeSync(targetFd, view);
+          bytes2 += read;
+        }
+        fsyncSync(targetFd);
+      } finally {
+        closeSync(targetFd);
+      }
+    } finally {
+      closeSync(sourceFd);
+    }
+    digest = hash4.digest("hex");
+    renameSync(partialPath, exportPath);
+    const receiptFd = openSync(`${exportPath}.sha256`, "w", 384);
+    try {
+      writeSync(receiptFd, `${digest}  ${basename(exportPath)}
+`);
+      fsyncSync(receiptFd);
+    } finally {
+      closeSync(receiptFd);
+    }
+  } catch (error) {
+    throw new DeadLetterJournalExportError("Dead-letter journal export failed", { cause: error });
+  }
+  log.info("Dead-letter journal exported", {
+    journal_path: journalPath,
+    export_path: exportPath,
+    bytes: bytes2,
+    sha256: digest
+  });
+  return {
+    status: "exported",
+    journalPath,
+    exportPath,
+    bytes: bytes2,
+    sha256: digest
+  };
+}
+
+// src/helpers/broker-execution-archive/loss-journal.ts
+var LOSS_JOURNAL_RECORD_VERSION = 1;
+var LOSS_REASONS = new Set([
+  "queue_shed",
+  "shutdown_forwarder_failure",
+  "retry_exhausted"
+]);
+
+// src/helpers/broker-execution-archive/writer.ts
 var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
   "broker_execution.order_events",
   "broker_execution.market_metadata_snapshots",
@@ -315935,7 +316042,7 @@ class BrokerExecutionArchiver {
       throw new Error("Broker execution archive is enabled but CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH is missing");
     }
     try {
-      this.deadLetterFd = openSync(this.deadLetterPath, "a", 384);
+      this.deadLetterFd = openSync2(this.deadLetterPath, "a", 384);
     } catch {
       throw new Error("Broker execution archive cannot open CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH for append");
     }
@@ -316065,11 +316172,9 @@ class BrokerExecutionArchiver {
           continue;
         }
         if (!await this.flushBatch()) {
-          const undelivered = [
-            ...this.pendingRetry?.rows ?? [],
-            ...this.queue
-          ];
-          this.appendLossRecords(undelivered, "shutdown_forwarder_failure");
+          const pinned = this.pendingRetry;
+          this.appendLossRecords(pinned?.rows ?? [], "shutdown_forwarder_failure", pinned?.batchId ?? null);
+          this.appendLossRecords(this.queue, "shutdown_forwarder_failure");
           this.pendingRetry = null;
           this.queue.length = 0;
           break;
@@ -316182,14 +316287,14 @@ class BrokerExecutionArchiver {
     }
     const fd2 = this.deadLetterFd;
     try {
-      closeSync(fd2);
+      closeSync2(fd2);
     } catch (error) {
       throw new BrokerExecutionArchiveDurabilityError("Broker execution archive failed to close the configured CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH loss journal", { cause: error });
     } finally {
       this.deadLetterFd = undefined;
     }
   }
-  appendLossRecords(rows, reason) {
+  appendLossRecords(rows, reason, batchId = null) {
     if (rows.length === 0) {
       return;
     }
@@ -316197,22 +316302,26 @@ class BrokerExecutionArchiver {
       throw new BrokerExecutionArchiveDurabilityError(`Broker execution archive cannot record ${reason}: dead-letter file is not open`);
     }
     const timestamp = new Date().toISOString();
-    const records = rows.map((payload) => ({
+    const records = rows.map((payload, index2) => ({
       timestamp,
       source: this.source,
       deployment_id: this.deploymentId,
       reason,
-      payload
+      payload,
+      record_version: LOSS_JOURNAL_RECORD_VERSION,
+      batch_id: batchId,
+      batch_row_index: index2,
+      batch_row_count: rows.length
     }));
     try {
       const bytes2 = Buffer.from(records.map((record) => JSON.stringify(record)).join(`
 `) + `
 `);
-      const written = writeSync(this.deadLetterFd, bytes2);
+      const written = writeSync2(this.deadLetterFd, bytes2);
       if (written !== bytes2.length) {
         throw new Error(`wrote ${written} of ${bytes2.length} bytes`);
       }
-      fsyncSync(this.deadLetterFd);
+      fsyncSync2(this.deadLetterFd);
       const byFeedAndTable = new Map;
       for (const row of rows) {
         const key = `${row.table}\x00${archiveFeed(row)}`;
@@ -316257,7 +316366,7 @@ class BrokerExecutionArchiver {
         try {
           if (attempts >= MAX_PINNED_BATCH_ATTEMPTS) {
             this.pendingRetry = null;
-            this.appendLossRecords(batch, "retry_exhausted");
+            this.appendLossRecords(batch, "retry_exhausted", batchId);
             log.warn("Broker execution archive gave up on a batch", {
               attempts,
               rows: batch.length
@@ -316519,6 +316628,7 @@ function flattenArchiveAttributes(entry) {
   return attributes;
 }
 function createBrokerExecutionArchiverFromEnv(otelLogs, otelMetrics) {
+  exportDeadLetterJournalFromEnv();
   if (process.env.CEX_BROKER_ARCHIVE_ENABLED !== "true") {
     return BrokerExecutionArchiver.disabled();
   }
@@ -317432,7 +317542,7 @@ class FillArchivePoller {
 }
 
 // src/helpers/market-data-archive/capture-contract.ts
-import { createHash as createHash3 } from "node:crypto";
+import { createHash as createHash4 } from "node:crypto";
 var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
 var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
 var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
@@ -317537,7 +317647,7 @@ function omitChecksumFields(value) {
   return value;
 }
 function sha256Canonical(value) {
-  return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
+  return createHash4("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
 }
 function normalizeTimestampMs(value, field) {
   let timestamp;
@@ -318118,13 +318228,13 @@ function createOtelLogsFromEnv() {
 }
 
 // src/helpers/stream-health-publisher.ts
-import { createHash as createHash4, randomUUID as randomUUID2 } from "node:crypto";
+import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
 import {
-  closeSync as closeSync2,
-  fsyncSync as fsyncSync2,
-  openSync as openSync2,
+  closeSync as closeSync3,
+  fsyncSync as fsyncSync3,
+  openSync as openSync3,
   readFileSync,
-  renameSync,
+  renameSync as renameSync2,
   statSync,
   unlinkSync,
   writeFileSync
@@ -318167,7 +318277,7 @@ function registryRevision(snapshots) {
     account_scope: snapshot.accountScope,
     registry_status: snapshot.registryStatus
   })).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
-  return createHash4("sha256").update(JSON.stringify(rows)).digest("hex");
+  return createHash5("sha256").update(JSON.stringify(rows)).digest("hex");
 }
 function validState(value) {
   if (!value || typeof value !== "object" || Array.isArray(value))
@@ -318431,21 +318541,21 @@ class StreamHealthPublisher {
     const temporary = `${this.#statePath}.${process.pid}.${randomUUID2()}.tmp`;
     let fd2;
     try {
-      fd2 = openSync2(temporary, "wx", 384);
+      fd2 = openSync3(temporary, "wx", 384);
       writeFileSync(fd2, JSON.stringify(this.#state));
-      fsyncSync2(fd2);
-      closeSync2(fd2);
+      fsyncSync3(fd2);
+      closeSync3(fd2);
       fd2 = undefined;
-      renameSync(temporary, this.#statePath);
-      const parentFd = openSync2(parent, "r");
+      renameSync2(temporary, this.#statePath);
+      const parentFd = openSync3(parent, "r");
       try {
-        fsyncSync2(parentFd);
+        fsyncSync3(parentFd);
       } finally {
-        closeSync2(parentFd);
+        closeSync3(parentFd);
       }
     } catch (error) {
       if (fd2 !== undefined)
-        closeSync2(fd2);
+        closeSync3(fd2);
       try {
         unlinkSync(temporary);
       } catch {}

package/dist/helpers/broker-execution-archive/index.d.ts

@@ -1,4 +1,5 @@
 export { archiveOrderExecutionInBackground, archiveSubscribeStreamInBackground, archiveTransferEventInBackground, archiveWithdrawalObservationsInBackground, captureMarketMetadataSnapshot, captureMarketMetadataSnapshotInBackground, } from "./capture";
+export { DeadLetterJournalExportError, type DeadLetterJournalExportResult, exportDeadLetterJournal, exportDeadLetterJournalFromEnv, } from "./journal-export";
 export { hashMarketMetadata, redactErrorForArchive, redactSecretLiterals, redactStreamPayload, } from "./redact";
 export { buildAccountBalanceSnapshotRow, buildCommonArchiveTags, buildFillEventArchiveRow, buildMarketMetadataSnapshotRow, buildOrderEventArchiveRow, buildSubscribeStreamArchiveRow, buildTransferEventArchiveRow, buildUserAssetSnapshotRow, extractBinanceInternalTransferId, type FillArchiveFields, type NormalizedCcxtBalance, type NormalizedCcxtTransfer, type NormalizedUserAssets, normalizeBinanceUserAssetsForArchive, normalizeCcxtBalanceForArchive, normalizeCcxtTradeForArchive, normalizeCcxtTransactionForArchive, normalizeTimestamp, type TransferArchiveFields, } from "./rows";
 export { ACCOUNT_BALANCE_PRECISION_BASIS, ACCOUNT_BALANCE_SCOPE, ARCHIVE_SCHEMA_VERSION, BROKER_READ_SOURCE, BROKER_WRITE_SOURCE, type BrokerArchiveCommonTags, type BrokerArchiveRow, type BrokerArchiveSource, type BrokerArchiveTable, type OrderArchiveAction, type SubscribeArchiveType, type TransferEventKind, type TransferLifecycleAction, USER_ASSET_BALANCE_SCOPE, USER_ASSET_PRECISION_BASIS, } from "./types";

package/dist/helpers/broker-execution-archive/journal-export.d.ts

@@ -0,0 +1,20 @@
+export declare class DeadLetterJournalExportError extends Error {
+    constructor(message: string, options?: ErrorOptions);
+}
+export type DeadLetterJournalExportResult = {
+    status: "disabled";
+} | {
+    status: "skipped_export_exists";
+    exportPath: string;
+} | {
+    status: "skipped_missing_journal";
+    journalPath: string;
+} | {
+    status: "exported";
+    journalPath: string;
+    exportPath: string;
+    bytes: number;
+    sha256: string;
+};
+export declare function exportDeadLetterJournalFromEnv(): DeadLetterJournalExportResult;
+export declare function exportDeadLetterJournal(journalPath: string, exportPath: string): DeadLetterJournalExportResult;

package/dist/helpers/broker-execution-archive/loss-journal.d.ts

@@ -0,0 +1,36 @@
+import type { BrokerArchiveRow, BrokerArchiveSource } from "./types";
+export declare const LOSS_JOURNAL_RECORD_VERSION: 1;
+export type ArchiveLossReason = "queue_shed" | "shutdown_forwarder_failure" | "retry_exhausted";
+export type ArchiveLossRecord = {
+    timestamp: string;
+    source: BrokerArchiveSource;
+    deployment_id: string;
+    reason: ArchiveLossReason;
+    payload: BrokerArchiveRow;
+    record_version: typeof LOSS_JOURNAL_RECORD_VERSION;
+    batch_id: string | null;
+    batch_row_index: number;
+    batch_row_count: number;
+};
+type ParsedArchiveLossRecordBase = Omit<ArchiveLossRecord, "payload" | "record_version" | "batch_id" | "batch_row_index" | "batch_row_count"> & {
+    payload: {
+        table: string;
+        row: Record<string, unknown>;
+    };
+};
+export type ParsedArchiveLossRecord = ParsedArchiveLossRecordBase & (Pick<ArchiveLossRecord, "record_version" | "batch_id" | "batch_row_index" | "batch_row_count"> | {
+    record_version?: never;
+    batch_id?: never;
+    batch_row_index?: never;
+    batch_row_count?: never;
+});
+export type ParsedLossJournalLine = {
+    ok: true;
+    record: ParsedArchiveLossRecord;
+} | {
+    ok: false;
+    reason: string;
+};
+export declare function parseLossJournalLine(line: string): ParsedLossJournalLine;
+export declare function lossJournalLineDigest(rawLine: string): string;
+export {};

package/dist/index.js

@@ -291325,11 +291325,118 @@ function compactUndefined2(record) {
 // src/helpers/broker-execution-archive/writer.ts
 var import_api_logs2 = __toESM(require_src4(), 1);
 import { randomUUID } from "node:crypto";
-import { closeSync, fsyncSync, openSync, writeSync } from "node:fs";
+import { closeSync as closeSync2, fsyncSync as fsyncSync2, openSync as openSync2, writeSync as writeSync2 } from "node:fs";
 import {
   request as httpRequest2
 } from "node:http";
 import { request as httpsRequest2 } from "node:https";
+
+// src/helpers/broker-execution-archive/journal-export.ts
+import { createHash as createHash3 } from "node:crypto";
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  openSync,
+  readSync,
+  renameSync,
+  writeSync
+} from "node:fs";
+import { basename } from "node:path";
+var EXPORT_CHUNK_BYTES = 8 * 1024 * 1024;
+
+class DeadLetterJournalExportError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "DeadLetterJournalExportError";
+  }
+}
+function exportDeadLetterJournalFromEnv() {
+  const exportPath = process.env.CEX_BROKER_ARCHIVE_DEAD_LETTER_EXPORT_PATH?.trim();
+  if (!exportPath) {
+    return { status: "disabled" };
+  }
+  const journalPath = process.env.CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH?.trim();
+  if (!journalPath) {
+    throw new DeadLetterJournalExportError("CEX_BROKER_ARCHIVE_DEAD_LETTER_EXPORT_PATH is set but CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH is missing");
+  }
+  return exportDeadLetterJournal(journalPath, exportPath);
+}
+function exportDeadLetterJournal(journalPath, exportPath) {
+  if (existsSync(exportPath)) {
+    log.info("Dead-letter journal export target already exists, skipping", {
+      export_path: exportPath
+    });
+    return { status: "skipped_export_exists", exportPath };
+  }
+  if (!existsSync(journalPath)) {
+    log.warn("Dead-letter journal export requested but no journal exists, skipping", { journal_path: journalPath });
+    return { status: "skipped_missing_journal", journalPath };
+  }
+  const partialPath = `${exportPath}.partial`;
+  let bytes2 = 0;
+  let digest;
+  try {
+    const hash4 = createHash3("sha256");
+    const sourceFd = openSync(journalPath, "r");
+    try {
+      const targetFd = openSync(partialPath, "w", 384);
+      try {
+        const chunk = Buffer.alloc(EXPORT_CHUNK_BYTES);
+        for (;; ) {
+          const read = readSync(sourceFd, chunk, 0, chunk.length, null);
+          if (read === 0) {
+            break;
+          }
+          const view = chunk.subarray(0, read);
+          hash4.update(view);
+          writeSync(targetFd, view);
+          bytes2 += read;
+        }
+        fsyncSync(targetFd);
+      } finally {
+        closeSync(targetFd);
+      }
+    } finally {
+      closeSync(sourceFd);
+    }
+    digest = hash4.digest("hex");
+    renameSync(partialPath, exportPath);
+    const receiptFd = openSync(`${exportPath}.sha256`, "w", 384);
+    try {
+      writeSync(receiptFd, `${digest}  ${basename(exportPath)}
+`);
+      fsyncSync(receiptFd);
+    } finally {
+      closeSync(receiptFd);
+    }
+  } catch (error) {
+    throw new DeadLetterJournalExportError("Dead-letter journal export failed", { cause: error });
+  }
+  log.info("Dead-letter journal exported", {
+    journal_path: journalPath,
+    export_path: exportPath,
+    bytes: bytes2,
+    sha256: digest
+  });
+  return {
+    status: "exported",
+    journalPath,
+    exportPath,
+    bytes: bytes2,
+    sha256: digest
+  };
+}
+
+// src/helpers/broker-execution-archive/loss-journal.ts
+var LOSS_JOURNAL_RECORD_VERSION = 1;
+var LOSS_REASONS = new Set([
+  "queue_shed",
+  "shutdown_forwarder_failure",
+  "retry_exhausted"
+]);
+
+// src/helpers/broker-execution-archive/writer.ts
 var BROKER_EXECUTION_ARCHIVE_TABLES = new Set([
   "broker_execution.order_events",
   "broker_execution.market_metadata_snapshots",
@@ -291449,7 +291556,7 @@ class BrokerExecutionArchiver {
       throw new Error("Broker execution archive is enabled but CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH is missing");
     }
     try {
-      this.deadLetterFd = openSync(this.deadLetterPath, "a", 384);
+      this.deadLetterFd = openSync2(this.deadLetterPath, "a", 384);
     } catch {
       throw new Error("Broker execution archive cannot open CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH for append");
     }
@@ -291579,11 +291686,9 @@ class BrokerExecutionArchiver {
           continue;
         }
         if (!await this.flushBatch()) {
-          const undelivered = [
-            ...this.pendingRetry?.rows ?? [],
-            ...this.queue
-          ];
-          this.appendLossRecords(undelivered, "shutdown_forwarder_failure");
+          const pinned = this.pendingRetry;
+          this.appendLossRecords(pinned?.rows ?? [], "shutdown_forwarder_failure", pinned?.batchId ?? null);
+          this.appendLossRecords(this.queue, "shutdown_forwarder_failure");
           this.pendingRetry = null;
           this.queue.length = 0;
           break;
@@ -291696,14 +291801,14 @@ class BrokerExecutionArchiver {
     }
     const fd2 = this.deadLetterFd;
     try {
-      closeSync(fd2);
+      closeSync2(fd2);
     } catch (error) {
       throw new BrokerExecutionArchiveDurabilityError("Broker execution archive failed to close the configured CEX_BROKER_ARCHIVE_DEAD_LETTER_PATH loss journal", { cause: error });
     } finally {
       this.deadLetterFd = undefined;
     }
   }
-  appendLossRecords(rows, reason) {
+  appendLossRecords(rows, reason, batchId = null) {
     if (rows.length === 0) {
       return;
     }
@@ -291711,22 +291816,26 @@ class BrokerExecutionArchiver {
       throw new BrokerExecutionArchiveDurabilityError(`Broker execution archive cannot record ${reason}: dead-letter file is not open`);
     }
     const timestamp = new Date().toISOString();
-    const records = rows.map((payload) => ({
+    const records = rows.map((payload, index2) => ({
       timestamp,
       source: this.source,
       deployment_id: this.deploymentId,
       reason,
-      payload
+      payload,
+      record_version: LOSS_JOURNAL_RECORD_VERSION,
+      batch_id: batchId,
+      batch_row_index: index2,
+      batch_row_count: rows.length
     }));
     try {
       const bytes2 = Buffer.from(records.map((record) => JSON.stringify(record)).join(`
 `) + `
 `);
-      const written = writeSync(this.deadLetterFd, bytes2);
+      const written = writeSync2(this.deadLetterFd, bytes2);
       if (written !== bytes2.length) {
         throw new Error(`wrote ${written} of ${bytes2.length} bytes`);
       }
-      fsyncSync(this.deadLetterFd);
+      fsyncSync2(this.deadLetterFd);
       const byFeedAndTable = new Map;
       for (const row of rows) {
         const key = `${row.table}\x00${archiveFeed(row)}`;
@@ -291771,7 +291880,7 @@ class BrokerExecutionArchiver {
         try {
           if (attempts >= MAX_PINNED_BATCH_ATTEMPTS) {
             this.pendingRetry = null;
-            this.appendLossRecords(batch, "retry_exhausted");
+            this.appendLossRecords(batch, "retry_exhausted", batchId);
             log.warn("Broker execution archive gave up on a batch", {
               attempts,
               rows: batch.length
@@ -292033,6 +292142,7 @@ function flattenArchiveAttributes(entry) {
   return attributes;
 }
 function createBrokerExecutionArchiverFromEnv(otelLogs, otelMetrics) {
+  exportDeadLetterJournalFromEnv();
   if (process.env.CEX_BROKER_ARCHIVE_ENABLED !== "true") {
     return BrokerExecutionArchiver.disabled();
   }
@@ -292946,7 +293056,7 @@ class FillArchivePoller {
 }
 
 // src/helpers/market-data-archive/capture-contract.ts
-import { createHash as createHash3 } from "node:crypto";
+import { createHash as createHash4 } from "node:crypto";
 var MARKET_CAPTURE_SCHEMA_VERSION = "1.0.0";
 var CHECKSUM_ALGORITHM = "sha256-canonical-json-v1";
 var ARCHIVE_SOURCES = ["broker_read", "broker_write"];
@@ -293051,7 +293161,7 @@ function omitChecksumFields(value) {
   return value;
 }
 function sha256Canonical(value) {
-  return createHash3("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
+  return createHash4("sha256").update(canonicalSerialize(omitChecksumFields(value))).digest("hex");
 }
 function normalizeTimestampMs(value, field) {
   let timestamp;
@@ -293632,13 +293742,13 @@ function createOtelLogsFromEnv() {
 }
 
 // src/helpers/stream-health-publisher.ts
-import { createHash as createHash4, randomUUID as randomUUID2 } from "node:crypto";
+import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
 import {
-  closeSync as closeSync2,
-  fsyncSync as fsyncSync2,
-  openSync as openSync2,
+  closeSync as closeSync3,
+  fsyncSync as fsyncSync3,
+  openSync as openSync3,
   readFileSync,
-  renameSync,
+  renameSync as renameSync2,
   statSync,
   unlinkSync,
   writeFileSync
@@ -293681,7 +293791,7 @@ function registryRevision(snapshots) {
     account_scope: snapshot.accountScope,
     registry_status: snapshot.registryStatus
   })).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
-  return createHash4("sha256").update(JSON.stringify(rows)).digest("hex");
+  return createHash5("sha256").update(JSON.stringify(rows)).digest("hex");
 }
 function validState(value) {
   if (!value || typeof value !== "object" || Array.isArray(value))
@@ -293945,21 +294055,21 @@ class StreamHealthPublisher {
     const temporary = `${this.#statePath}.${process.pid}.${randomUUID2()}.tmp`;
     let fd2;
     try {
-      fd2 = openSync2(temporary, "wx", 384);
+      fd2 = openSync3(temporary, "wx", 384);
       writeFileSync(fd2, JSON.stringify(this.#state));
-      fsyncSync2(fd2);
-      closeSync2(fd2);
+      fsyncSync3(fd2);
+      closeSync3(fd2);
       fd2 = undefined;
-      renameSync(temporary, this.#statePath);
-      const parentFd = openSync2(parent, "r");
+      renameSync2(temporary, this.#statePath);
+      const parentFd = openSync3(parent, "r");
       try {
-        fsyncSync2(parentFd);
+        fsyncSync3(parentFd);
       } finally {
-        closeSync2(parentFd);
+        closeSync3(parentFd);
       }
     } catch (error) {
       if (fd2 !== undefined)
-        closeSync2(fd2);
+        closeSync3(fd2);
       try {
         unlinkSync(temporary);
       } catch {}
@@ -312456,4 +312566,4 @@ export {
   CEXBroker as default
 };
 
-//# debugId=2BB56A1EBD617FCC64756E2164756E21
+//# debugId=B0758AED2799A60464756E2164756E21