@horizon-republic/nestjs-jetstream 2.11.1 → 2.12.0

package/dist/index.cjs

@@ -64,6 +64,7 @@ __export(index_exports, {
   JetstreamHeader: () => JetstreamHeader,
   JetstreamHealthIndicator: () => JetstreamHealthIndicator,
   JetstreamModule: () => JetstreamModule,
+  JetstreamProvisioningError: () => JetstreamProvisioningError,
   JetstreamRecord: () => JetstreamRecord,
   JetstreamRecordBuilder: () => JetstreamRecordBuilder,
   JetstreamStrategy: () => JetstreamStrategy,
@@ -210,7 +211,7 @@ var DEFAULT_ORDERED_STREAM_CONFIG = {
 };
 var DEFAULT_DLQ_STREAM_CONFIG = {
   ...baseStreamConfig,
-  retention: import_jetstream.RetentionPolicy.Workqueue,
+  retention: import_jetstream.RetentionPolicy.Limits,
   allow_rollup_hdrs: false,
   max_consumers: 100,
   max_msg_size: 10 * MB,
@@ -274,6 +275,7 @@ var RESERVED_HEADERS = /* @__PURE__ */ new Set([
   "x-reply-to" /* ReplyTo */,
   "x-error" /* Error */
 ]);
+var NATS_CONTROL_HEADER_PREFIX = "nats-";
 var internalName = (name) => `${name}__microservice`;
 var buildSubject = (serviceName, kind, pattern) => `${internalName(serviceName)}.${kind}.${pattern}`;
 var buildBroadcastSubject = (pattern) => `broadcast.${pattern}`;
@@ -326,6 +328,9 @@ var ATTR_JETSTREAM_RPC_REPLY_ERROR_CODE = "jetstream.rpc.reply.error.code";
 var ATTR_JETSTREAM_PROVISIONING_ENTITY = "jetstream.provisioning.entity";
 var ATTR_JETSTREAM_PROVISIONING_ACTION = "jetstream.provisioning.action";
 var ATTR_JETSTREAM_PROVISIONING_NAME = "jetstream.provisioning.name";
+var ATTR_JETSTREAM_PROVISIONING_MAX_BYTES = "jetstream.provisioning.max_bytes";
+var ATTR_JETSTREAM_PROVISIONING_NUM_REPLICAS = "jetstream.provisioning.num_replicas";
+var ATTR_JETSTREAM_PROVISIONING_RESERVATION = "jetstream.provisioning.reservation_bytes";
 var ATTR_JETSTREAM_SELF_HEALING_REASON = "jetstream.self_healing.reason";
 var ATTR_JETSTREAM_MIGRATION_REASON = "jetstream.migration.reason";
 var ATTR_JETSTREAM_DEAD_LETTER_REASON = "jetstream.dead_letter.reason";
@@ -593,7 +598,7 @@ var extractContext = (ctx, carrier, getter) => import_api.propagation.extract(ct
 
 // src/otel/tracer.ts
 var import_api2 = require("@opentelemetry/api");
-var PACKAGE_VERSION = true ? "2.11.1" : "0.0.0";
+var PACKAGE_VERSION = true ? "2.12.0" : "0.0.0";
 var getTracer = () => import_api2.trace.getTracer(TRACER_NAME, PACKAGE_VERSION);
 
 // src/otel/carrier.ts
@@ -1182,7 +1187,10 @@ var withProvisioningSpan = (config, ctx, op) => wrapInfra(
   {
     [ATTR_JETSTREAM_PROVISIONING_ENTITY]: ctx.entity,
     [ATTR_JETSTREAM_PROVISIONING_ACTION]: ctx.action,
-    [ATTR_JETSTREAM_PROVISIONING_NAME]: ctx.name
+    [ATTR_JETSTREAM_PROVISIONING_NAME]: ctx.name,
+    [ATTR_JETSTREAM_PROVISIONING_MAX_BYTES]: ctx.maxBytes,
+    [ATTR_JETSTREAM_PROVISIONING_NUM_REPLICAS]: ctx.numReplicas,
+    [ATTR_JETSTREAM_PROVISIONING_RESERVATION]: ctx.reservation
   },
   op
 );
@@ -1358,7 +1366,13 @@ var JetstreamRecordBuilder = class {
    * lockstep. `RESERVED_HEADERS` is defined as an all-lowercase set.
    */
   validateHeaderKey(key) {
-    if (RESERVED_HEADERS.has(key.toLowerCase())) {
+    const normalized = key.toLowerCase();
+    if (normalized.startsWith(NATS_CONTROL_HEADER_PREFIX)) {
+      throw new Error(
+        `Header "${key}" is reserved for the NATS server and cannot be set manually. Use setMessageId() for deduplication, ttl() for per-message expiry, and scheduleAt() for delayed delivery.`
+      );
+    }
+    if (RESERVED_HEADERS.has(normalized)) {
       throw new Error(
         `Header "${key}" is reserved by the JetStream transport and cannot be set manually. Reserved headers: ${[...RESERVED_HEADERS].join(", ")}`
       );
@@ -1504,13 +1518,18 @@ var JetstreamClient = class extends import_microservices.ClientProxy {
   async dispatchEvent(packet) {
     if (!this.readyForPublish) await this.connect();
     const { data, hdrs, messageId, schedule, ttl } = this.extractRecordData(packet.data);
+    const publishKind = detectEventKind(packet.pattern);
+    if (schedule && publishKind === "ordered" /* Ordered */) {
+      throw new Error(
+        `scheduleAt() is not supported for ordered events (pattern: ${packet.pattern}). Scheduled delivery is available for workqueue events and broadcasts.`
+      );
+    }
     const eventSubject = this.buildEventSubject(packet.pattern);
     const publishSubject = schedule ? this.buildScheduleSubject(eventSubject) : eventSubject;
     const msgHeaders = this.buildHeaders(hdrs, { subject: eventSubject });
     const encoded = this.codec.encode(data);
     const effectiveMsgId = messageId ?? import_nuid.nuid.next();
     const record = packet.data instanceof JetstreamRecord ? packet.data : new JetstreamRecord(data, /* @__PURE__ */ new Map());
-    const publishKind = detectEventKind(packet.pattern);
     const declaredPattern = declaredEventPattern(packet.pattern);
     const streamKind = eventStreamKind(publishKind);
     const startedAt = performance.now();
@@ -1542,10 +1561,10 @@ var JetstreamClient = class extends import_microservices.ClientProxy {
             const ack2 = await this.connection.getJetStreamClient().publish(publishSubject, encoded, {
               headers: msgHeaders,
               msgID: effectiveMsgId,
-              ttl,
               schedule: {
                 specification: schedule.at,
-                target: eventSubject
+                target: eventSubject,
+                ttl
               }
             });
             warnIfDuplicate("scheduled", ack2);
@@ -1735,13 +1754,18 @@ var JetstreamClient = class extends import_microservices.ClientProxy {
         });
         return;
       }
-      await import_api8.context.with(
+      const ack = await import_api8.context.with(
         spanHandle.activeContext,
         () => this.connection.getJetStreamClient().publish(subject, encoded, {
           headers: hdrs,
           msgID: messageId ?? import_nuid.nuid.next()
         })
       );
+      if (ack.duplicate) {
+        throw new Error(
+          `Duplicate RPC publish for ${subject}: the messageId was already used within the stream dedup window, so the reply belongs to the original request`
+        );
+      }
       this.reportPublished(declaredPattern, "cmd" /* Command */, startedAt, "success");
     } catch (err) {
       const existingTimeout = this.pendingTimeouts.get(correlationId);
@@ -1911,13 +1935,17 @@ var JetstreamClient = class extends import_microservices.ClientProxy {
    * uses a separate `_sch` namespace that is NOT matched by any consumer filter.
    * NATS holds the message and publishes it to the target subject after the delay.
    *
+   * A unique per-message suffix is appended because the server stores schedules
+   * as rollup messages — one active schedule per subject (ADR-51). Without it,
+   * concurrent schedules of the same pattern would silently replace each other.
+   *
    * Examples:
-   * - `{svc}__microservice.ev.order.reminder` → `{svc}__microservice._sch.order.reminder`
-   * - `broadcast.config.updated` → `broadcast._sch.config.updated`
+   * - `{svc}__microservice.ev.order.reminder` → `{svc}__microservice._sch.order.reminder.<nuid>`
+   * - `broadcast.config.updated` → `broadcast._sch.config.updated.<nuid>`
    */
   buildScheduleSubject(eventSubject) {
     if (eventSubject.startsWith("broadcast.")) {
-      return eventSubject.replace("broadcast.", "broadcast._sch.");
+      return `${eventSubject.replace("broadcast.", "broadcast._sch.")}.${import_nuid.nuid.next()}`;
     }
     const targetPrefix = `${internalName(this.targetName)}.`;
     if (!eventSubject.startsWith(targetPrefix)) {
@@ -1929,7 +1957,7 @@ var JetstreamClient = class extends import_microservices.ClientProxy {
       throw new Error(`Event subject missing pattern segment: ${eventSubject}`);
     }
     const pattern = withoutPrefix.slice(dotIndex + 1);
-    return `${targetPrefix}_sch.${pattern}`;
+    return `${targetPrefix}_sch.${pattern}.${import_nuid.nuid.next()}`;
   }
 };
 
@@ -1941,6 +1969,7 @@ var JsonCodec = class {
     return encoder.encode(JSON.stringify(data));
   }
   decode(data) {
+    if (data.length === 0) return void 0;
     return JSON.parse(decoder.decode(data));
   }
 };
@@ -1954,6 +1983,7 @@ var MsgpackCodec = class {
     return this.packr.pack(data);
   }
   decode(data) {
+    if (data.length === 0) return void 0;
     return this.packr.unpack(data);
   }
 };
@@ -3024,43 +3054,11 @@ var JetstreamStrategy = class extends import_microservices2.Server {
    * Called by NestJS when `connectMicroservice()` is used, or internally by the module.
    */
   async listen(callback) {
-    if (this.started) {
-      this.logger.warn("listen() called more than once \u2014 ignoring");
-      return;
-    }
-    this.started = true;
-    this.patternRegistry.registerHandlers(this.getHandlers());
-    const { streams: streamKinds, durableConsumers: durableKinds } = this.resolveRequiredKinds();
-    if (streamKinds.length > 0) {
-      await this.streamProvider.ensureStreams(streamKinds);
-      if (durableKinds.length > 0) {
-        const consumers = await this.consumerProvider.ensureConsumers(durableKinds);
-        this.populateAckWaitMap(consumers);
-        this.eventRouter.updateMaxDeliverMap(this.buildMaxDeliverMap(consumers));
-        this.messageProvider.start(consumers);
-      }
-      if (this.patternRegistry.hasOrderedHandlers()) {
-        const orderedStreamName = this.streamProvider.getStreamName("ordered" /* Ordered */);
-        await this.messageProvider.startOrdered(
-          orderedStreamName,
-          this.patternRegistry.getOrderedSubjects(),
-          this.options.ordered
-        );
-      }
-      if (this.patternRegistry.hasEventHandlers() || this.patternRegistry.hasBroadcastHandlers() || this.patternRegistry.hasOrderedHandlers()) {
-        this.eventRouter.start();
-      }
-      if (isJetStreamRpcMode(this.options.rpc) && this.patternRegistry.hasRpcHandlers()) {
-        await this.rpcRouter.start();
-      }
-    }
-    if (isCoreRpcMode(this.options.rpc) && this.patternRegistry.hasRpcHandlers()) {
-      await this.coreRpcServer.start();
-    }
-    if (this.metadataProvider && this.patternRegistry.hasMetadata()) {
-      await this.metadataProvider.publish(this.patternRegistry.getMetadataEntries());
+    try {
+      await this.doListen(callback);
+    } catch (err) {
+      callback(err);
     }
-    callback();
   }
   /** Stop all consumers, routers, subscriptions, and metadata heartbeat. Called during shutdown. */
   close() {
@@ -3119,6 +3117,33 @@ var JetstreamStrategy = class extends import_microservices2.Server {
   getPatternRegistry() {
     return this.patternRegistry;
   }
+  async doListen(callback) {
+    if (this.started) {
+      this.logger.warn("listen() called more than once \u2014 ignoring");
+      return;
+    }
+    this.started = true;
+    this.patternRegistry.registerHandlers(this.getHandlers());
+    const { streams: streamKinds, durableConsumers: durableKinds } = this.resolveRequiredKinds();
+    if (streamKinds.length > 0) {
+      await this.streamProvider.ensureStreams(streamKinds);
+      let consumers = null;
+      if (durableKinds.length > 0) {
+        consumers = await this.consumerProvider.ensureConsumers(durableKinds);
+        this.populateAckWaitMap(consumers);
+        this.eventRouter.updateMaxDeliverMap(this.buildMaxDeliverMap(consumers));
+      }
+      await this.startRouters();
+      await this.startConsumption(consumers);
+    }
+    if (isCoreRpcMode(this.options.rpc) && this.patternRegistry.hasRpcHandlers()) {
+      await this.coreRpcServer.start();
+    }
+    if (this.metadataProvider && this.patternRegistry.hasMetadata()) {
+      await this.metadataProvider.publish(this.patternRegistry.getMetadataEntries());
+    }
+    callback();
+  }
   /** Determine which streams and durable consumers are needed. */
   resolveRequiredKinds() {
     const streams = [];
@@ -3140,7 +3165,29 @@ var JetstreamStrategy = class extends import_microservices2.Server {
     }
     return { streams, durableConsumers };
   }
-  /** Populate the shared ack_wait map from actual NATS consumer configs. */
+  /** Subscribe the event and RPC routers to the message subjects. */
+  async startRouters() {
+    if (this.patternRegistry.hasEventHandlers() || this.patternRegistry.hasBroadcastHandlers() || this.patternRegistry.hasOrderedHandlers()) {
+      this.eventRouter.start();
+    }
+    if (isJetStreamRpcMode(this.options.rpc) && this.patternRegistry.hasRpcHandlers()) {
+      await this.rpcRouter.start();
+    }
+  }
+  /** Begin durable and ordered consumption; routers must already be subscribed. */
+  async startConsumption(consumers) {
+    if (consumers !== null) {
+      this.messageProvider.start(consumers);
+    }
+    if (this.patternRegistry.hasOrderedHandlers()) {
+      const orderedStreamName = this.streamProvider.getStreamName("ordered" /* Ordered */);
+      await this.messageProvider.startOrdered(
+        orderedStreamName,
+        this.patternRegistry.getOrderedSubjects(),
+        this.options.ordered
+      );
+    }
+  }
   populateAckWaitMap(consumers) {
     for (const [kind, info] of consumers) {
       if (info.config.ack_wait) {
@@ -3379,6 +3426,15 @@ var serializeError = (err) => {
   return err;
 };
 
+// src/utils/settle-quietly.ts
+var settleQuietly = (logger5, label, action) => {
+  try {
+    action();
+  } catch (err) {
+    logger5.error(label, err);
+  }
+};
+
 // src/utils/unwrap-result.ts
 var import_rxjs2 = require("rxjs");
 var unwrapResult = (result) => {
@@ -3544,16 +3600,164 @@ var CoreRpcServer = class {
 
 // src/server/infrastructure/stream.provider.ts
 var import_common14 = require("@nestjs/common");
-var import_jetstream17 = require("@nats-io/jetstream");
+var import_jetstream19 = require("@nats-io/jetstream");
 
 // src/server/infrastructure/nats-error-codes.ts
 var NatsErrorCode = /* @__PURE__ */ ((NatsErrorCode2) => {
   NatsErrorCode2[NatsErrorCode2["ConsumerNotFound"] = 10014] = "ConsumerNotFound";
   NatsErrorCode2[NatsErrorCode2["ConsumerAlreadyExists"] = 10148] = "ConsumerAlreadyExists";
   NatsErrorCode2[NatsErrorCode2["StreamNotFound"] = 10059] = "StreamNotFound";
+  NatsErrorCode2[NatsErrorCode2["StorageResourcesExceeded"] = 10047] = "StorageResourcesExceeded";
+  NatsErrorCode2[NatsErrorCode2["NoSuitablePeers"] = 10005] = "NoSuitablePeers";
   return NatsErrorCode2;
 })(NatsErrorCode || {});
 
+// src/server/infrastructure/provisioning-budget.ts
+var import_jetstream16 = require("@nats-io/jetstream");
+var GIB = 1024 ** 3;
+var fmt = (bytes) => `${(bytes / GIB).toFixed(2)} GiB`;
+var resolveTierBudget = (info, replicas) => {
+  const tier = info.tiers?.[`R${replicas}`];
+  const limits = tier?.limits ?? info.limits;
+  return {
+    maxStorage: limits?.max_storage ?? 0,
+    reserved: tier?.reserved_storage ?? info.reserved_storage ?? 0,
+    tiered: tier !== void 0
+  };
+};
+var groupByReplicas = (reservations) => {
+  const groups = /* @__PURE__ */ new Map();
+  for (const r of reservations) {
+    if (r.storage !== import_jetstream16.StorageType.File) continue;
+    const prev = groups.get(r.numReplicas) ?? 0;
+    groups.set(r.numReplicas, prev + r.maxBytes * r.numReplicas);
+  }
+  return groups;
+};
+var assertStorageBudget = async (jsm, serviceName, reservations, logger5) => {
+  try {
+    const info = await jsm.getAccountInfo();
+    const groups = groupByReplicas(reservations);
+    let limitNotSetWarned = false;
+    let okReserved = 0;
+    let anyWarned = false;
+    for (const [replicas, incremental] of groups) {
+      const { maxStorage, reserved, tiered } = resolveTierBudget(info, replicas);
+      const tierNote = tiered ? ` (tier R${replicas})` : "";
+      if (maxStorage <= 0) {
+        if (!limitNotSetWarned) {
+          limitNotSetWarned = true;
+          logger5.warn(
+            `Storage preflight for "${serviceName}": account file-storage limit not set (max_storage=${maxStorage}); the server max_file_store cannot be verified from the client.`
+          );
+        }
+        continue;
+      }
+      const remaining = maxStorage - reserved;
+      if (incremental > remaining) {
+        anyWarned = true;
+        logger5.warn(
+          `Storage preflight for "${serviceName}"${tierNote}: needs ~${fmt(incremental)} but only ~${fmt(remaining)} remains (reserved ${fmt(reserved)} / limit ${fmt(maxStorage)}). Provisioning will likely fail with insufficient storage. Lower max_bytes/num_replicas, or raise the account/server storage limit.`
+        );
+        continue;
+      }
+      okReserved += incremental;
+    }
+    if (!anyWarned && !limitNotSetWarned && okReserved > 0) {
+      logger5.log(
+        `Storage preflight for "${serviceName}" OK: reserving ~${fmt(okReserved)} across file-backed streams within account limits.`
+      );
+    }
+  } catch (err) {
+    logger5.debug(`Storage preflight skipped \u2014 account info unavailable: ${String(err)}`);
+  }
+};
+
+// src/server/infrastructure/provisioning-error.ts
+var REMEDIATION = {
+  [10047 /* StorageResourcesExceeded */]: "Aggregate stream reservation exceeds the server `max_file_store` (or account `max_storage`). Lower `max_bytes`/`num_replicas` for this service, or raise `max_file_store` on the NATS servers.",
+  [10005 /* NoSuitablePeers */]: "Fewer healthy peers than `num_replicas`, or no peer has enough reserved storage headroom. Reduce replicas or add/repair cluster nodes."
+};
+var GENERIC_REMEDIATION = "Inspect the NATS server logs and JetStream account limits for the underlying cause.";
+var JetstreamProvisioningError = class _JetstreamProvisioningError extends Error {
+  entity;
+  target;
+  kind;
+  errCode;
+  errDescription;
+  remediation;
+  maxBytes;
+  numReplicas;
+  reservation;
+  constructor(fields) {
+    const reservationNote = fields.reservation !== void 0 ? ` reservation=${fields.reservation}B (max_bytes=${fields.maxBytes}B \xD7 replicas=${fields.numReplicas}).` : "";
+    super(
+      `JetStream ${fields.entity} provisioning failed for "${fields.target}" (kind=${fields.kind}): ${fields.errDescription} [err_code=${fields.errCode}].${reservationNote} ${fields.remediation}`,
+      { cause: fields.cause }
+    );
+    this.name = "JetstreamProvisioningError";
+    this.entity = fields.entity;
+    this.target = fields.target;
+    this.kind = fields.kind;
+    this.errCode = fields.errCode;
+    this.errDescription = fields.errDescription;
+    this.remediation = fields.remediation;
+    this.maxBytes = fields.maxBytes;
+    this.numReplicas = fields.numReplicas;
+    this.reservation = fields.reservation;
+    Object.setPrototypeOf(this, _JetstreamProvisioningError.prototype);
+  }
+};
+var mapProvisioningError = (err, ctx) => {
+  const api = err.apiError();
+  const remediation = REMEDIATION[api.err_code] ?? GENERIC_REMEDIATION;
+  const reservation = ctx.maxBytes !== void 0 && ctx.numReplicas !== void 0 ? ctx.maxBytes * ctx.numReplicas : void 0;
+  return new JetstreamProvisioningError({
+    entity: ctx.entity,
+    target: ctx.name,
+    kind: ctx.kind,
+    errCode: api.err_code,
+    errDescription: api.description,
+    remediation,
+    maxBytes: ctx.maxBytes,
+    numReplicas: ctx.numReplicas,
+    reservation,
+    cause: err
+  });
+};
+
+// src/server/infrastructure/provisioning-summary.ts
+var import_jetstream17 = require("@nats-io/jetstream");
+var GIB2 = 1024 ** 3;
+var NANOS_PER_SECOND = 1e9;
+var NANOS_PER_HOUR = 3600 * NANOS_PER_SECOND;
+var NANOS_PER_DAY = 86400 * NANOS_PER_SECOND;
+var formatBytes = (bytes) => {
+  if (bytes <= 0) return "0 B";
+  return `${(bytes / GIB2).toFixed(2)} GiB`;
+};
+var formatAge = (nanos) => {
+  if (nanos <= 0) return "unlimited";
+  if (nanos >= NANOS_PER_DAY) return `${(nanos / NANOS_PER_DAY).toFixed(1)}d`;
+  if (nanos >= NANOS_PER_HOUR) return `${(nanos / NANOS_PER_HOUR).toFixed(1)}h`;
+  return `${(nanos / NANOS_PER_SECOND).toFixed(0)}s`;
+};
+var formatProvisioningSummary = (serviceName, reservations) => {
+  const lines = [`Provisioning ${reservations.length} stream(s) for "${serviceName}":`];
+  let totalFileMaxBytes = 0;
+  for (const r of reservations) {
+    if (r.storage === import_jetstream17.StorageType.File) totalFileMaxBytes += r.maxBytes;
+    const clusterReservation = r.maxBytes * r.numReplicas;
+    lines.push(
+      `  \u2022 ${r.name} [${r.kind}] storage=${r.storage} replicas=${r.numReplicas} max_bytes=${formatBytes(r.maxBytes)} max_age=${formatAge(r.maxAge)} retention=${r.retention} \u2192 cluster reservation ${formatBytes(clusterReservation)}`
+    );
+  }
+  lines.push(
+    `  \u03A3 per-node file-backed footprint \u2248 ${formatBytes(totalFileMaxBytes)} (sum of max_bytes; worst case replicas = nodes). Ensure the NATS server max_file_store accommodates the sum across ALL services.`
+  );
+  return lines.join("\n");
+};
+
 // src/server/infrastructure/stream-config-diff.ts
 var TRANSPORT_CONTROLLED_PROPERTIES = /* @__PURE__ */ new Set([
   "retention"
@@ -3611,85 +3815,199 @@ var isEqual = (a, b) => {
 
 // src/server/infrastructure/stream-migration.ts
 var import_common13 = require("@nestjs/common");
-var import_jetstream16 = require("@nats-io/jetstream");
+var import_jetstream18 = require("@nats-io/jetstream");
 var MIGRATION_BACKUP_SUFFIX = "__migration_backup";
 var DEFAULT_SOURCING_TIMEOUT_MS = 3e4;
 var SOURCING_POLL_INTERVAL_MS = 100;
+var DEFAULT_PEER_WAIT_MS = 6e4;
+var ACTIVE_MIGRATION_GRACE_MS = 9e4;
+var MIGRATION_STARTED_AT_KEY = "nestjs-jetstream-migration-started-at";
 var StreamMigration = class {
-  constructor(sourcingTimeoutMs = DEFAULT_SOURCING_TIMEOUT_MS) {
+  constructor(sourcingTimeoutMs = DEFAULT_SOURCING_TIMEOUT_MS, peerWaitMs = DEFAULT_PEER_WAIT_MS) {
     this.sourcingTimeoutMs = sourcingTimeoutMs;
+    this.peerWaitMs = peerWaitMs;
   }
   logger = new import_common13.Logger("Jetstream:Stream");
   async migrate(jsm, streamName2, newConfig) {
     const backupName = `${streamName2}${MIGRATION_BACKUP_SUFFIX}`;
     const startTime = Date.now();
+    const peerFinished = await this.waitOutPeerMigration(jsm, backupName);
     const currentInfo = await jsm.streams.info(streamName2);
-    await this.cleanupOrphanedBackup(jsm, backupName);
-    const messageCount = currentInfo.state.messages;
+    if (peerFinished && !compareStreamConfig(currentInfo.config, newConfig).hasImmutableChanges) {
+      this.logger.log(`Stream ${streamName2}: migration completed by another instance`);
+      await jsm.streams.update(streamName2, newConfig);
+      return;
+    }
     this.logger.log(`Stream ${streamName2}: destructive migration started`);
     let originalDeleted = false;
+    let drainedCount = 0;
     try {
-      if (messageCount > 0) {
-        this.logger.log(`  Phase 1/4: Backing up ${messageCount} messages \u2192 ${backupName}`);
+      this.logger.log(`  Phase 1/4: Quiescing ${streamName2} (publishes rejected during migration)`);
+      await jsm.streams.update(streamName2, { ...currentInfo.config, subjects: [] });
+      drainedCount = (await jsm.streams.info(streamName2)).state.messages;
+      if (drainedCount > 0) {
+        this.logger.log(`  Phase 2/4: Backing up ${drainedCount} messages \u2192 ${backupName}`);
         await jsm.streams.add({
           ...currentInfo.config,
           name: backupName,
           subjects: [],
-          sources: [{ name: streamName2 }]
+          sources: [{ name: streamName2 }],
+          metadata: { [MIGRATION_STARTED_AT_KEY]: (/* @__PURE__ */ new Date()).toISOString() }
         });
-        await this.waitForSourcing(jsm, backupName, messageCount);
+        await this.waitForSourceDrained(jsm, backupName, streamName2, drainedCount);
       }
-      this.logger.log(`  Phase 2/4: Deleting old stream`);
+      this.logger.log(`  Phase 3/4: Recreating ${streamName2} with the new config`);
       await jsm.streams.delete(streamName2);
       originalDeleted = true;
-      this.logger.log(`  Phase 3/4: Creating stream with new config`);
       await jsm.streams.add(newConfig);
-      if (messageCount > 0) {
-        const backupInfo = await jsm.streams.info(backupName);
-        await jsm.streams.update(backupName, { ...backupInfo.config, sources: [] });
-        this.logger.log(`  Phase 4/4: Restoring ${messageCount} messages from backup`);
-        await jsm.streams.update(streamName2, {
-          ...newConfig,
-          sources: [{ name: backupName }]
-        });
-        await this.waitForSourcing(jsm, streamName2, messageCount);
-        await jsm.streams.update(streamName2, { ...newConfig, sources: [] });
-        await jsm.streams.delete(backupName);
+      if (drainedCount > 0) {
+        this.logger.log(`  Phase 4/4: Restoring ${drainedCount} messages from backup`);
+        await this.restoreFromBackup(jsm, streamName2, newConfig, backupName);
       }
     } catch (err) {
-      if (originalDeleted && messageCount > 0) {
+      if (originalDeleted) {
         this.logger.error(
-          `Migration failed after deleting original stream. Backup ${backupName} preserved for manual recovery.`
+          `Migration of ${streamName2} failed after the original was deleted. Backup ${backupName} preserved \u2014 restoration resumes on the next startup.`
         );
       } else {
-        await this.cleanupOrphanedBackup(jsm, backupName);
+        await this.rollbackBeforeDelete(jsm, streamName2, currentInfo, backupName);
       }
       throw err;
     }
     const durationMs = Date.now() - startTime;
     this.logger.log(
-      `Stream ${streamName2}: migration complete (${messageCount} messages preserved, took ${(durationMs / 1e3).toFixed(1)}s)`
+      `Stream ${streamName2}: migration complete (${drainedCount} messages preserved, took ${(durationMs / 1e3).toFixed(1)}s)`
+    );
+  }
+  /**
+   * Detect and finish a migration that a previous process left unfinished.
+   * Safe against concurrent instances: a backup fresh enough to belong to a
+   * live migration is left alone.
+   *
+   * @returns true when recovery work was performed.
+   */
+  async recoverInterrupted(jsm, streamName2, desiredConfig) {
+    const backupName = `${streamName2}${MIGRATION_BACKUP_SUFFIX}`;
+    const backupInfo = await this.tryInfo(jsm, backupName);
+    if (backupInfo === null) return false;
+    if (this.isPeerMigrationActive(backupInfo)) return false;
+    const streamInfo = await this.tryInfo(jsm, streamName2);
+    if (streamInfo === null) {
+      this.logger.warn(`Stream ${streamName2}: resuming interrupted migration from ${backupName}`);
+      await jsm.streams.add(desiredConfig);
+      if (backupInfo.state.messages > 0) {
+        await this.restoreFromBackup(jsm, streamName2, desiredConfig, backupName);
+      } else {
+        await jsm.streams.delete(backupName);
+      }
+      return true;
+    }
+    const hasBackupSource = (streamInfo.config.sources ?? []).some((s) => s.name === backupName);
+    if (hasBackupSource) {
+      this.logger.warn(`Stream ${streamName2}: finishing interrupted restore from ${backupName}`);
+      await this.waitForSourceDrained(jsm, streamName2, backupName, backupInfo.state.messages);
+      await jsm.streams.delete(backupName);
+      await jsm.streams.update(streamName2, { ...streamInfo.config, sources: [] });
+      return true;
+    }
+    if (backupInfo.state.messages === 0) {
+      this.logger.warn(`Removing empty migration backup ${backupName}`);
+      await jsm.streams.delete(backupName);
+      return true;
+    }
+    this.logger.warn(
+      `Stream ${streamName2}: restoring ${backupInfo.state.messages} messages from stale ${backupName}`
+    );
+    await this.restoreFromBackup(
+      jsm,
+      streamName2,
+      { ...streamInfo.config, name: streamName2, subjects: streamInfo.config.subjects },
+      backupName
     );
+    return true;
+  }
+  /** Attach the backup as a source, drain it fully, then clean up. */
+  async restoreFromBackup(jsm, streamName2, streamConfig, backupName) {
+    const backupInfo = await jsm.streams.info(backupName);
+    if ((backupInfo.config.sources ?? []).length > 0) {
+      await jsm.streams.update(backupName, { ...backupInfo.config, sources: [] });
+    }
+    await jsm.streams.update(streamName2, { ...streamConfig, sources: [{ name: backupName }] });
+    await this.waitForSourceDrained(jsm, streamName2, backupName, backupInfo.state.messages);
+    await jsm.streams.delete(backupName);
+    await jsm.streams.update(streamName2, { ...streamConfig, sources: [] });
   }
-  async waitForSourcing(jsm, streamName2, expectedCount) {
+  /**
+   * Wait until `sourceName` is fully drained into `streamName`. Lag-based, so
+   * concurrent live publishes to the target cannot fake completion the way a
+   * bare message-count comparison could. A freshly attached source reports
+   * `lag: 0, active: -1` before its first sync — `active >= 0` filters that
+   * false positive out (verified against NATS 2.12.6).
+   */
+  async waitForSourceDrained(jsm, streamName2, sourceName, minimumMessages) {
     const deadline = Date.now() + this.sourcingTimeoutMs;
     while (Date.now() < deadline) {
       const info = await jsm.streams.info(streamName2);
-      if (info.state.messages >= expectedCount) return;
+      const source = (info.sources ?? []).find((s) => s.name === sourceName);
+      if (source !== void 0 && source.active >= 0 && source.lag === 0 && info.state.messages >= minimumMessages) {
+        return;
+      }
       await new Promise((r) => setTimeout(r, SOURCING_POLL_INTERVAL_MS));
     }
     throw new Error(
-      `Stream sourcing timeout: ${streamName2} has not reached ${expectedCount} messages within ${this.sourcingTimeoutMs / 1e3}s`
+      `Stream sourcing timeout: ${sourceName} has not drained into ${streamName2} within ${this.sourcingTimeoutMs / 1e3}s. The backup is preserved; restoration resumes on the next startup.`
+    );
+  }
+  /**
+   * A backup already present when migrate() begins belongs to another
+   * instance migrating right now (rolling deploy) — wait for it to finish.
+   * Stale leftovers are handled by recoverInterrupted() before migrate() runs,
+   * so a timeout here means something is genuinely stuck.
+   *
+   * @returns true when a peer's backup was observed and cleared.
+   */
+  async waitOutPeerMigration(jsm, backupName) {
+    if (await this.tryInfo(jsm, backupName) === null) return false;
+    this.logger.warn(
+      `Migration backup ${backupName} exists \u2014 another instance appears to be migrating; waiting`
+    );
+    const deadline = Date.now() + this.peerWaitMs;
+    while (Date.now() < deadline) {
+      if (await this.tryInfo(jsm, backupName) === null) return true;
+      await new Promise((r) => setTimeout(r, SOURCING_POLL_INTERVAL_MS * 5));
+    }
+    throw new Error(
+      `Migration backup ${backupName} did not clear within ${this.peerWaitMs / 1e3}s. If no other instance is migrating, recover or remove the backup manually.`
     );
   }
-  async cleanupOrphanedBackup(jsm, backupName) {
+  /** Failure before the original was deleted: undo the quiesce, drop our backup. */
+  async rollbackBeforeDelete(jsm, streamName2, originalInfo, backupName) {
     try {
-      await jsm.streams.info(backupName);
-      this.logger.warn(`Found orphaned migration backup stream: ${backupName}, cleaning up`);
-      await jsm.streams.delete(backupName);
+      await jsm.streams.update(streamName2, { ...originalInfo.config });
+      const backupInfo = await this.tryInfo(jsm, backupName);
+      if (backupInfo !== null) {
+        await jsm.streams.delete(backupName);
+      }
+    } catch (rollbackErr) {
+      this.logger.error(
+        `Rollback of ${streamName2} after a failed migration also failed \u2014 the stream may be left quiesced:`,
+        rollbackErr
+      );
+    }
+  }
+  isPeerMigrationActive(backupInfo) {
+    const startedAt = backupInfo.config.metadata?.[MIGRATION_STARTED_AT_KEY];
+    if (!startedAt) return false;
+    const startedMs = Date.parse(startedAt);
+    if (Number.isNaN(startedMs)) return false;
+    return Date.now() - startedMs < ACTIVE_MIGRATION_GRACE_MS;
+  }
+  async tryInfo(jsm, name) {
+    try {
+      return await jsm.streams.info(name);
     } catch (err) {
-      if (err instanceof import_jetstream16.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
-        return;
+      if (err instanceof import_jetstream18.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
+        return null;
       }
       throw err;
     }
@@ -3720,6 +4038,15 @@ var StreamProvider = class {
    */
   async ensureStreams(kinds) {
     const jsm = await this.connection.getJetStreamManager();
+    const reservations = kinds.map((kind) => this.buildReservation(kind, this.buildConfig(kind)));
+    if (this.options.dlq) {
+      reservations.push(this.buildReservation("dlq", this.buildDlqConfig()));
+    }
+    this.logger.log(`
+${formatProvisioningSummary(this.options.name, reservations)}`);
+    if (this.options.provisioning?.preflightStorageCheck) {
+      await assertStorageBudget(jsm, this.options.name, reservations, this.logger);
+    }
     await Promise.all(kinds.map((kind) => this.ensureStream(jsm, kind)));
     if (this.options.dlq) {
       await this.ensureDlqStream(jsm);
@@ -3756,6 +4083,7 @@ var StreamProvider = class {
   /** Ensure a single stream exists, creating or updating as needed. */
   async ensureStream(jsm, kind) {
     const config = this.buildConfig(kind);
+    const ctx = this.errorContext(kind, config);
     return withProvisioningSpan(
       this.otel,
       {
@@ -3763,17 +4091,21 @@ var StreamProvider = class {
         endpoint: this.otelEndpoint,
         entity: "stream",
         name: config.name,
-        action: "ensure"
+        action: "ensure",
+        maxBytes: ctx.maxBytes,
+        numReplicas: ctx.numReplicas,
+        reservation: ctx.maxBytes !== void 0 && ctx.numReplicas !== void 0 ? ctx.maxBytes * ctx.numReplicas : void 0
       },
       async () => {
         this.logger.log(`Ensuring stream: ${config.name}`);
+        await this.migration.recoverInterrupted(jsm, config.name, config);
         try {
           const currentInfo = await jsm.streams.info(config.name);
-          return await this.handleExistingStream(jsm, currentInfo, config);
+          return await this.handleExistingStream(jsm, currentInfo, config, ctx);
         } catch (err) {
-          if (err instanceof import_jetstream17.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
+          if (err instanceof import_jetstream19.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
             this.logger.log(`Creating stream: ${config.name}`);
-            return await jsm.streams.add(config);
+            return await this.runStreamOp(ctx, () => jsm.streams.add(config));
           }
           throw err;
         }
@@ -3783,6 +4115,7 @@ var StreamProvider = class {
   /** Ensure a dead-letter queue stream exists, creating or updating as needed. */
   async ensureDlqStream(jsm) {
     const config = this.buildDlqConfig();
+    const ctx = this.errorContext("dlq", config);
     return withProvisioningSpan(
       this.otel,
       {
@@ -3790,24 +4123,30 @@ var StreamProvider = class {
         endpoint: this.otelEndpoint,
         entity: "stream",
         name: config.name,
-        action: "ensure"
+        action: "ensure",
+        maxBytes: ctx.maxBytes,
+        numReplicas: ctx.numReplicas,
+        reservation: ctx.maxBytes !== void 0 && ctx.numReplicas !== void 0 ? ctx.maxBytes * ctx.numReplicas : void 0
       },
       async () => {
         this.logger.log(`Ensuring DLQ stream: ${config.name}`);
         try {
           const currentInfo = await jsm.streams.info(config.name);
-          return await this.handleExistingStream(jsm, currentInfo, config);
+          return await this.handleExistingStream(jsm, currentInfo, config, ctx);
         } catch (err) {
-          if (err instanceof import_jetstream17.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
+          if (err instanceof import_jetstream19.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
             this.logger.log(`Creating DLQ stream: ${config.name}`);
-            return await jsm.streams.add(config);
+            return await this.runStreamOp(ctx, () => jsm.streams.add(config));
           }
           throw err;
         }
       }
     );
   }
-  async handleExistingStream(jsm, currentInfo, config) {
+  async handleExistingStream(jsm, currentInfo, config, ctx) {
+    if (this.isSharedStream(config.name)) {
+      config.subjects = [.../* @__PURE__ */ new Set([...config.subjects, ...currentInfo.config.subjects])];
+    }
     const diff = compareStreamConfig(currentInfo.config, config);
     if (!diff.hasChanges) {
       this.logger.debug(`Stream ${config.name}: no config changes`);
@@ -3822,7 +4161,7 @@ var StreamProvider = class {
     }
     if (!diff.hasImmutableChanges) {
       this.logger.debug(`Stream exists, updating: ${config.name}`);
-      return await jsm.streams.update(config.name, config);
+      return await this.runStreamOp(ctx, () => jsm.streams.update(config.name, config));
     }
     if (!this.options.allowDestructiveMigration) {
       this.logger.warn(
@@ -3830,10 +4169,15 @@ var StreamProvider = class {
       );
       if (diff.hasMutableChanges) {
         const mutableConfig = this.buildMutableOnlyConfig(config, currentInfo.config, diff);
-        return await jsm.streams.update(config.name, mutableConfig);
+        return await this.runStreamOp(ctx, () => jsm.streams.update(config.name, mutableConfig));
       }
       return currentInfo;
     }
+    if (this.isSharedStream(config.name)) {
+      throw new Error(
+        `Stream ${config.name} is shared across services and cannot be destructively migrated: recreating it would delete every other service's durable broadcast consumers and replay retained history to them. Coordinate a manual migration instead.`
+      );
+    }
     await withMigrationSpan(
       this.otel,
       {
@@ -3872,11 +4216,47 @@ var StreamProvider = class {
       }
     }
   }
+  buildReservation(kind, config) {
+    const mb = config.max_bytes;
+    return {
+      kind,
+      name: config.name,
+      storage: config.storage ?? import_jetstream19.StorageType.File,
+      numReplicas: config.num_replicas ?? 1,
+      maxBytes: mb !== void 0 && mb >= 0 ? mb : 0,
+      // NATS uses -1 for unlimited
+      maxAge: config.max_age ?? 0,
+      retention: config.retention ?? import_jetstream19.RetentionPolicy.Limits
+    };
+  }
+  errorContext(kind, config) {
+    return {
+      entity: "stream",
+      name: config.name,
+      kind,
+      maxBytes: config.max_bytes,
+      numReplicas: config.num_replicas ?? 1
+    };
+  }
+  async runStreamOp(ctx, op) {
+    try {
+      return await op();
+    } catch (err) {
+      if (err instanceof import_jetstream19.JetStreamApiError) {
+        throw mapProvisioningError(err, ctx);
+      }
+      throw err;
+    }
+  }
+  /** The broadcast stream is global — every service in the cluster shares it. */
+  isSharedStream(name) {
+    return name === this.getStreamName("broadcast" /* Broadcast */);
+  }
   /** Build the full stream config by merging defaults with user overrides. */
   buildConfig(kind) {
     const name = this.getStreamName(kind);
     const subjects = this.getSubjects(kind);
-    const description = `JetStream ${kind} stream for ${this.options.name}`;
+    const description = kind === "broadcast" /* Broadcast */ ? "JetStream broadcast stream (shared across services)" : `JetStream ${kind} stream for ${this.options.name}`;
     const defaults = this.getDefaults(kind);
     const overrides = this.getOverrides(kind);
     return {
@@ -3962,7 +4342,7 @@ var StreamProvider = class {
 
 // src/server/infrastructure/consumer.provider.ts
 var import_common15 = require("@nestjs/common");
-var import_jetstream19 = require("@nats-io/jetstream");
+var import_jetstream21 = require("@nats-io/jetstream");
 var ConsumerProvider = class {
   constructor(options, connection, streamProvider, patternRegistry) {
     this.options = options;
@@ -4018,15 +4398,16 @@ var ConsumerProvider = class {
       },
       async () => {
         this.logger.log(`Ensuring consumer: ${name} on stream: ${stream}`);
+        const ctx = { entity: "consumer", name, kind };
         try {
           await jsm.consumers.info(stream, name);
           this.logger.debug(`Consumer exists, updating: ${name}`);
-          return await jsm.consumers.update(stream, name, config);
+          return await this.runConsumerOp(ctx, () => jsm.consumers.update(stream, name, config));
         } catch (err) {
-          if (!(err instanceof import_jetstream19.JetStreamApiError) || err.apiError().err_code !== 10014 /* ConsumerNotFound */) {
+          if (!(err instanceof import_jetstream21.JetStreamApiError) || err.apiError().err_code !== 10014 /* ConsumerNotFound */) {
             throw err;
           }
-          return await this.createConsumer(jsm, stream, name, config);
+          return await this.createConsumer(jsm, stream, name, kind, config);
         }
       }
     );
@@ -4063,10 +4444,10 @@ var ConsumerProvider = class {
         try {
           return await jsm.consumers.info(stream, name);
         } catch (err) {
-          if (!(err instanceof import_jetstream19.JetStreamApiError) || err.apiError().err_code !== 10014 /* ConsumerNotFound */) {
+          if (!(err instanceof import_jetstream21.JetStreamApiError) || err.apiError().err_code !== 10014 /* ConsumerNotFound */) {
             throw err;
           }
-          return await this.createConsumer(jsm, stream, name, config);
+          return await this.createConsumer(jsm, stream, name, kind, config);
         }
       }
     );
@@ -4084,7 +4465,7 @@ var ConsumerProvider = class {
         `Stream ${stream} is being migrated (backup ${backupName} exists). Waiting for migration to complete before recovering consumer.`
       );
     } catch (err) {
-      if (err instanceof import_jetstream19.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
+      if (err instanceof import_jetstream21.JetStreamApiError && err.apiError().err_code === 10059 /* StreamNotFound */) {
         return;
       }
       throw err;
@@ -4093,18 +4474,32 @@ var ConsumerProvider = class {
   /**
    * Create a consumer, handling the race where another pod creates it first.
    */
-  async createConsumer(jsm, stream, name, config) {
+  async createConsumer(jsm, stream, name, kind, config) {
     this.logger.log(`Creating consumer: ${name}`);
+    const ctx = { entity: "consumer", name, kind };
     try {
       return await jsm.consumers.add(stream, config);
     } catch (addErr) {
-      if (addErr instanceof import_jetstream19.JetStreamApiError && addErr.apiError().err_code === 10148 /* ConsumerAlreadyExists */) {
+      if (addErr instanceof import_jetstream21.JetStreamApiError && addErr.apiError().err_code === 10148 /* ConsumerAlreadyExists */) {
         this.logger.debug(`Consumer ${name} created by another pod, using existing`);
         return await jsm.consumers.info(stream, name);
       }
+      if (addErr instanceof import_jetstream21.JetStreamApiError) {
+        throw mapProvisioningError(addErr, ctx);
+      }
       throw addErr;
     }
   }
+  async runConsumerOp(ctx, op) {
+    try {
+      return await op();
+    } catch (err) {
+      if (err instanceof import_jetstream21.JetStreamApiError) {
+        throw mapProvisioningError(err, ctx);
+      }
+      throw err;
+    }
+  }
   /** Build consumer config by merging defaults with user overrides. */
   // eslint-disable-next-line @typescript-eslint/naming-convention -- NATS API uses snake_case
   buildConfig(kind) {
@@ -4186,7 +4581,7 @@ var ConsumerProvider = class {
 
 // src/server/infrastructure/message.provider.ts
 var import_common16 = require("@nestjs/common");
-var import_jetstream21 = require("@nats-io/jetstream");
+var import_jetstream23 = require("@nats-io/jetstream");
 var import_rxjs3 = require("rxjs");
 var MessageProvider = class {
   constructor(connection, eventBus, consumeOptionsMap = /* @__PURE__ */ new Map(), consumerRecoveryFn) {
@@ -4247,7 +4642,7 @@ var MessageProvider = class {
    */
   async startOrdered(streamName2, filterSubjects, orderedConfig) {
     const consumerOpts = { filter_subjects: filterSubjects };
-    if (orderedConfig?.deliverPolicy !== void 0 && orderedConfig.deliverPolicy !== import_jetstream21.DeliverPolicy.All) {
+    if (orderedConfig?.deliverPolicy !== void 0 && orderedConfig.deliverPolicy !== import_jetstream23.DeliverPolicy.All) {
       consumerOpts.deliver_policy = orderedConfig.deliverPolicy;
     }
     if (orderedConfig?.optStartSeq !== void 0) {
@@ -4556,6 +4951,7 @@ var MetadataProvider = class {
 // src/server/routing/event.router.ts
 var import_common18 = require("@nestjs/common");
 var import_transport_node4 = require("@nats-io/transport-node");
+var DLQ_PUBLISH_ATTEMPTS = 3;
 var eventConsumeKindFor = (kind) => {
   if (kind === "broadcast" /* Broadcast */) return "broadcast" /* Broadcast */;
   if (kind === "ordered" /* Ordered */) return "ordered" /* Ordered */;
@@ -4642,33 +5038,80 @@ var EventRouter = class {
       return msg.info.deliveryCount >= maxDeliver;
     };
     const handleDeadLetter = hasDlqCheck ? (msg, data, err) => this.handleDeadLetter(msg, data, err) : null;
-    const settleSuccess = (msg, ctx) => {
-      if (ctx.shouldTerminate) msg.term(ctx.terminateReason);
-      else if (ctx.shouldRetry) msg.nak(ctx.retryDelay);
-      else msg.ack();
+    const settleSuccess = (msg, ctx, data) => {
+      if (ctx.shouldTerminate) {
+        settleQuietly(logger5, `Failed to term ${msg.subject}:`, () => {
+          msg.term(ctx.terminateReason);
+        });
+        return void 0;
+      }
+      if (ctx.shouldRetry) {
+        if (handleDeadLetter !== null && isDeadLetter(msg)) {
+          return handleDeadLetter(
+            msg,
+            data,
+            new Error("Retry requested on the final delivery attempt")
+          );
+        }
+        settleQuietly(logger5, `Failed to nak ${msg.subject}:`, () => {
+          msg.nak(ctx.retryDelay);
+        });
+        return void 0;
+      }
+      settleQuietly(logger5, `Failed to ack ${msg.subject}:`, () => {
+        msg.ack();
+      });
+      return void 0;
     };
     const settleFailure = async (msg, data, err) => {
       if (handleDeadLetter !== null && isDeadLetter(msg)) {
         await handleDeadLetter(msg, data, err);
         return;
       }
-      msg.nak();
+      settleQuietly(logger5, `Failed to nak ${msg.subject}:`, () => {
+        msg.nak();
+      });
+    };
+    const captureUnroutable = (capture, msg, err) => {
+      let data;
+      try {
+        data = codec.decode(msg.data);
+      } catch {
+        data = void 0;
+      }
+      return capture(msg, data, err).catch((captureErr) => {
+        logger5.error(`Dead-letter capture failed for unroutable ${msg.subject}:`, captureErr);
+      });
     };
     const resolveEvent = (msg) => {
       const subject = msg.subject;
       try {
         const handler = patternRegistry.getHandler(subject);
         if (!handler) {
-          msg.term(`No handler for event: ${subject}`);
           logger5.error(`No handler for subject: ${subject}`);
+          if (handleDeadLetter !== null) {
+            return captureUnroutable(
+              handleDeadLetter,
+              msg,
+              new Error(`No handler for event: ${subject}`)
+            );
+          }
+          msg.term(`No handler for event: ${subject}`);
           return null;
         }
         let data;
         try {
           data = codec.decode(msg.data);
         } catch (err) {
-          msg.term("Decode error");
           logger5.error(`Decode error for ${subject}:`, err);
+          if (handleDeadLetter !== null) {
+            return captureUnroutable(
+              handleDeadLetter,
+              msg,
+              new Error(`Decode error: ${err instanceof Error ? err.message : String(err)}`)
+            );
+          }
+          msg.term("Decode error");
           return null;
         }
         eventBus.emitMessageRouted(subject, "event" /* Event */);
@@ -4691,6 +5134,7 @@ var EventRouter = class {
     const handleSafe = (msg) => {
       const resolved = resolveEvent(msg);
       if (resolved === null) return void 0;
+      if (isPromiseLike2(resolved)) return resolved;
       const { handler, data } = resolved;
       const ctx = new RpcContext([msg]);
       const stopAckExtension = hasAckExtension ? startAckExtensionTimer(msg, ackExtensionInterval) : null;
@@ -4723,16 +5167,24 @@ var EventRouter = class {
         });
       }
       if (!isPromiseLike2(pending)) {
-        settleSuccess(msg, ctx);
+        const settled = settleSuccess(msg, ctx, data);
         reportHandlerCompleted(msg, startedAt, statusForContext(ctx));
-        if (stopAckExtension !== null) stopAckExtension();
-        return void 0;
+        if (settled === void 0) {
+          if (stopAckExtension !== null) stopAckExtension();
+          return void 0;
+        }
+        return settled.finally(() => {
+          if (stopAckExtension !== null) stopAckExtension();
+        });
       }
       return pending.then(
-        () => {
-          settleSuccess(msg, ctx);
-          reportHandlerCompleted(msg, startedAt, statusForContext(ctx));
-          if (stopAckExtension !== null) stopAckExtension();
+        async () => {
+          try {
+            await settleSuccess(msg, ctx, data);
+            reportHandlerCompleted(msg, startedAt, statusForContext(ctx));
+          } finally {
+            if (stopAckExtension !== null) stopAckExtension();
+          }
         },
         async (err) => {
           eventBus.emit(
@@ -4826,14 +5278,28 @@ var EventRouter = class {
       active--;
       drainBacklog();
     };
+    const routeSafely = (msg) => {
+      try {
+        return route(msg);
+      } catch (err) {
+        logger5.error(`Unexpected routing failure for ${msg.subject}:`, err);
+        return void 0;
+      }
+    };
+    const trackAsync = (result, msg) => {
+      void result.catch((err) => {
+        logger5.error(`Unexpected routing failure for ${msg.subject}:`, err);
+      }).finally(onAsyncDone);
+    };
     const drainBacklog = () => {
       while (active < maxActive) {
         const next = backlog.shift();
         if (next === void 0) return;
+        next.stopAckExtension?.();
         active++;
-        const result = route(next);
+        const result = routeSafely(next.msg);
         if (result !== void 0) {
-          void result.finally(onAsyncDone);
+          trackAsync(result, next.msg);
         } else {
           active--;
         }
@@ -4843,7 +5309,10 @@ var EventRouter = class {
     const subscription = stream$.subscribe({
       next: (msg) => {
         if (active >= maxActive) {
-          backlog.push(msg);
+          backlog.push({
+            msg,
+            stopAckExtension: hasAckExtension ? startAckExtensionTimer(msg, ackExtensionInterval) : null
+          });
           if (!backlogWarned && backlog.length >= backlogWarnThreshold) {
             backlogWarned = true;
             logger5.warn(
@@ -4853,9 +5322,9 @@ var EventRouter = class {
           return;
         }
         active++;
-        const result = route(msg);
+        const result = routeSafely(msg);
         if (result !== void 0) {
-          void result.finally(onAsyncDone);
+          trackAsync(result, msg);
         } else {
           active--;
           if (backlog.length > 0) drainBacklog();
@@ -4865,6 +5334,12 @@ var EventRouter = class {
         logger5.error(`Stream error in ${kind} router`, err);
       }
     });
+    subscription.add(() => {
+      for (const queued of backlog) {
+        queued.stopAckExtension?.();
+      }
+      backlog.length = 0;
+    });
     this.subscriptions.push(subscription);
   }
   getConcurrency(kind) {
@@ -4879,25 +5354,78 @@ var EventRouter = class {
   }
   /**
    * Last-resort path for a dead letter: invoke `onDeadLetter`, then `term` on
-   * success or `nak` on hook failure so NATS retries on the next delivery
-   * cycle. Used when DLQ stream isn't configured, or when publishing to it
-   * failed and we still have to surface the message somewhere observable.
+   * success. On failure the message is nak'd to release it, but the server
+   * never redelivers past `max_deliver` — it stays in the stream for manual
+   * recovery. Used when the DLQ stream isn't configured, or when publishing
+   * to it failed and we still have to surface the message somewhere.
    */
   async fallbackToOnDeadLetterCallback(info, msg) {
-    if (!this.deadLetterConfig) {
-      msg.term("Dead letter config unavailable");
+    const onDeadLetter = this.deadLetterConfig?.onDeadLetter;
+    if (!onDeadLetter) {
+      this.logger.error(
+        `Dead letter for ${msg.subject} could not be captured (DLQ publish failed, no onDeadLetter callback) \u2014 leaving the message in the stream`
+      );
+      settleQuietly(this.logger, `Failed to nak ${msg.subject}:`, () => {
+        msg.nak();
+      });
       return;
     }
     try {
-      await this.deadLetterConfig.onDeadLetter(info);
-      msg.term("Dead letter processed via fallback callback");
+      await onDeadLetter(info);
+      settleQuietly(this.logger, `Failed to term ${msg.subject}:`, () => {
+        msg.term("Dead letter processed via fallback callback");
+      });
     } catch (hookErr) {
       this.logger.error(
-        `Fallback onDeadLetter callback failed for ${msg.subject}, nak for retry:`,
+        `Fallback onDeadLetter callback failed for ${msg.subject} \u2014 the message stays in the stream and will not be redelivered (max_deliver exhausted); recover it manually:`,
         hookErr
       );
-      msg.nak();
+      settleQuietly(this.logger, `Failed to nak ${msg.subject}:`, () => {
+        msg.nak();
+      });
+    }
+  }
+  /**
+   * Copy the original message headers for the DLQ republish, dropping NATS
+   * server control headers: a copied Nats-TTL expires the DLQ entry (or gets
+   * the publish rejected when the DLQ stream has no allow_msg_ttl), a copied
+   * Nats-Msg-Id collides with the DLQ dedup window.
+   */
+  buildDlqHeaders(msg) {
+    const hdrs = (0, import_transport_node4.headers)();
+    if (!msg.headers) return hdrs;
+    for (const [k, v] of msg.headers) {
+      if (k.toLowerCase().startsWith(NATS_CONTROL_HEADER_PREFIX)) continue;
+      for (const val of v) {
+        hdrs.append(k, val);
+      }
+    }
+    return hdrs;
+  }
+  /**
+   * Attempt the DLQ publish up to {@link DLQ_PUBLISH_ATTEMPTS} times.
+   *
+   * Past `max_deliver` the server never redelivers, so an in-process retry is
+   * the only second chance a dead letter gets. There is no artificial delay
+   * between attempts: when the broker is unreachable each publish already
+   * spends its own request timeout, which spaces the attempts naturally.
+   */
+  async publishToDlqWithRetry(connection, subject, data, headers2) {
+    let lastErr;
+    for (let attempt = 1; attempt <= DLQ_PUBLISH_ATTEMPTS; attempt += 1) {
+      try {
+        await connection.getJetStreamClient().publish(subject, data, { headers: headers2 });
+        return;
+      } catch (err) {
+        lastErr = err;
+        if (attempt < DLQ_PUBLISH_ATTEMPTS) {
+          this.logger.warn(
+            `DLQ publish attempt ${attempt}/${DLQ_PUBLISH_ATTEMPTS} failed for ${subject}, retrying`
+          );
+        }
+      }
     }
+    throw lastErr;
   }
   /**
    * Publish a dead letter to the configured Dead-Letter Queue (DLQ) stream.
@@ -4917,14 +5445,7 @@ var EventRouter = class {
       return;
     }
     const destinationSubject = dlqStreamName(serviceName);
-    const hdrs = (0, import_transport_node4.headers)();
-    if (msg.headers) {
-      for (const [k, v] of msg.headers) {
-        for (const val of v) {
-          hdrs.append(k, val);
-        }
-      }
-    }
+    const hdrs = this.buildDlqHeaders(msg);
     let reason = String(error);
     if (error instanceof Error) {
       reason = error.message;
@@ -4937,8 +5458,7 @@ var EventRouter = class {
     hdrs.set("x-failed-at" /* FailedAt */, (/* @__PURE__ */ new Date()).toISOString());
     hdrs.set("x-delivery-count" /* DeliveryCount */, msg.info.deliveryCount.toString());
     try {
-      const js = this.connection.getJetStreamClient();
-      await js.publish(destinationSubject, msg.data, { headers: hdrs });
+      await this.publishToDlqWithRetry(this.connection, destinationSubject, msg.data, hdrs);
       this.logger.log(`Message sent to DLQ: ${msg.subject}`);
       if (this.deadLetterConfig?.onDeadLetter) {
         try {
@@ -4950,7 +5470,9 @@ var EventRouter = class {
           );
         }
       }
-      msg.term("Moved to DLQ stream");
+      settleQuietly(this.logger, `Failed to term ${msg.subject}:`, () => {
+        msg.term("Moved to DLQ stream");
+      });
     } catch (publishErr) {
       this.logger.error(`Failed to publish to DLQ for ${msg.subject}:`, publishErr);
       await this.fallbackToOnDeadLetterCallback(info, msg);
@@ -5144,7 +5666,9 @@ var RpcRouter = class {
           `rpc-handler:${subject}`
         );
         publishErrorReply(replyTo, correlationId, subject, err);
-        msg.term(`Handler error: ${subject}`);
+        settleQuietly(logger5, `Failed to term ${subject}:`, () => {
+          msg.term(`Handler error: ${subject}`);
+        });
       };
       const abortController = new AbortController();
       let pending;
@@ -5172,7 +5696,9 @@ var RpcRouter = class {
       }
       if (!isPromiseLike2(pending)) {
         if (stopAckExtension !== null) stopAckExtension();
-        msg.ack();
+        settleQuietly(logger5, `Failed to ack ${subject}:`, () => {
+          msg.ack();
+        });
         publishReply(replyTo, correlationId, pending);
         reportHandlerCompleted(msg, startedAt, "success");
         return void 0;
@@ -5184,7 +5710,9 @@ var RpcRouter = class {
         if (stopAckExtension !== null) stopAckExtension();
         abortController.abort();
         emitRpcTimeout(subject, correlationId);
-        msg.term("Handler timeout");
+        settleQuietly(logger5, `Failed to term ${subject}:`, () => {
+          msg.term("Handler timeout");
+        });
         reportHandlerCompleted(msg, startedAt, "terminated");
       }, timeout);
       return pending.then(
@@ -5193,7 +5721,9 @@ var RpcRouter = class {
           settled = true;
           clearTimeout(timeoutId);
           if (stopAckExtension !== null) stopAckExtension();
-          msg.ack();
+          settleQuietly(logger5, `Failed to ack ${subject}:`, () => {
+            msg.ack();
+          });
           publishReply(replyTo, correlationId, result);
           reportHandlerCompleted(msg, startedAt, "success");
         },
@@ -5215,14 +5745,28 @@ var RpcRouter = class {
       active--;
       drainBacklog();
     };
+    const routeSafely = (msg) => {
+      try {
+        return handleSafe(msg);
+      } catch (err) {
+        logger5.error(`Unexpected routing failure for ${msg.subject}:`, err);
+        return void 0;
+      }
+    };
+    const trackAsync = (result, msg) => {
+      void result.catch((err) => {
+        logger5.error(`Unexpected routing failure for ${msg.subject}:`, err);
+      }).finally(onAsyncDone);
+    };
     const drainBacklog = () => {
       while (active < maxActive) {
         const next = backlog.shift();
         if (next === void 0) return;
+        next.stopAckExtension?.();
         active++;
-        const result = handleSafe(next);
+        const result = routeSafely(next.msg);
         if (result !== void 0) {
-          void result.finally(onAsyncDone);
+          trackAsync(result, next.msg);
         } else {
           active--;
         }
@@ -5232,7 +5776,10 @@ var RpcRouter = class {
     this.subscription = this.messageProvider.commands$.subscribe({
       next: (msg) => {
         if (active >= maxActive) {
-          backlog.push(msg);
+          backlog.push({
+            msg,
+            stopAckExtension: hasAckExtension ? startAckExtensionTimer(msg, ackExtensionInterval) : null
+          });
           if (!backlogWarned && backlog.length >= backlogWarnThreshold) {
             backlogWarned = true;
             logger5.warn(
@@ -5242,9 +5789,9 @@ var RpcRouter = class {
           return;
         }
         active++;
-        const result = handleSafe(msg);
+        const result = routeSafely(msg);
         if (result !== void 0) {
-          void result.finally(onAsyncDone);
+          trackAsync(result, msg);
         } else {
           active--;
           if (backlog.length > 0) drainBacklog();
@@ -5254,6 +5801,12 @@ var RpcRouter = class {
         logger5.error("Stream error in RPC router", err);
       }
     });
+    this.subscription.add(() => {
+      for (const queued of backlog) {
+        queued.stopAckExtension?.();
+      }
+      backlog.length = 0;
+    });
   }
   /** Stop routing and unsubscribe. */
   destroy() {
@@ -5536,7 +6089,7 @@ var JetstreamModule = class {
         ],
         useFactory: (options, messageProvider, patternRegistry, codec, eventBus, ackWaitMap, connection) => {
           if (options.consumer === false) return null;
-          const deadLetterConfig = options.onDeadLetter ? {
+          const deadLetterConfig = options.onDeadLetter || options.dlq ? {
             maxDeliverByStream: /* @__PURE__ */ new Map(),
             onDeadLetter: options.onDeadLetter
           } : void 0;
@@ -5737,6 +6290,7 @@ JetstreamModule = __decorateClass([
   JetstreamHeader,
   JetstreamHealthIndicator,
   JetstreamModule,
+  JetstreamProvisioningError,
   JetstreamRecord,
   JetstreamRecordBuilder,
   JetstreamStrategy,