@medplum/agent 5.1.21 → 5.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/cjs/index.cjs +2571 -96
  2. package/package.json +5 -5
@@ -5871,8 +5871,8 @@ var require_buffer_util = __commonJS({
5871
5871
  }
5872
5872
  return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.length);
5873
5873
  }
5874
- function toBuffer(data2) {
5875
- toBuffer.readOnly = true;
5874
+ function toBuffer2(data2) {
5875
+ toBuffer2.readOnly = true;
5876
5876
  if (Buffer.isBuffer(data2)) return data2;
5877
5877
  let buf;
5878
5878
  if (data2 instanceof ArrayBuffer) {
@@ -5881,7 +5881,7 @@ var require_buffer_util = __commonJS({
5881
5881
  buf = new FastBuffer(data2.buffer, data2.byteOffset, data2.byteLength);
5882
5882
  } else {
5883
5883
  buf = Buffer.from(data2);
5884
- toBuffer.readOnly = false;
5884
+ toBuffer2.readOnly = false;
5885
5885
  }
5886
5886
  return buf;
5887
5887
  }
@@ -5889,7 +5889,7 @@ var require_buffer_util = __commonJS({
5889
5889
  concat,
5890
5890
  mask: _mask,
5891
5891
  toArrayBuffer,
5892
- toBuffer,
5892
+ toBuffer: toBuffer2,
5893
5893
  unmask: _unmask
5894
5894
  };
5895
5895
  if (!process.env.WS_NO_BUFFER_UTIL) {
@@ -7187,7 +7187,7 @@ var require_sender = __commonJS({
7187
7187
  var PerMessageDeflate2 = require_permessage_deflate();
7188
7188
  var { EMPTY_BUFFER, kWebSocket, NOOP } = require_constants2();
7189
7189
  var { isBlob, isValidStatusCode } = require_validation();
7190
- var { mask: applyMask, toBuffer } = require_buffer_util();
7190
+ var { mask: applyMask, toBuffer: toBuffer2 } = require_buffer_util();
7191
7191
  var kByteLength = /* @__PURE__ */ Symbol("kByteLength");
7192
7192
  var maskBuffer = Buffer.alloc(4);
7193
7193
  var RANDOM_POOL_SIZE = 8 * 1024;
@@ -7377,9 +7377,9 @@ var require_sender = __commonJS({
7377
7377
  byteLength = data2.size;
7378
7378
  readOnly = false;
7379
7379
  } else {
7380
- data2 = toBuffer(data2);
7380
+ data2 = toBuffer2(data2);
7381
7381
  byteLength = data2.length;
7382
- readOnly = toBuffer.readOnly;
7382
+ readOnly = toBuffer2.readOnly;
7383
7383
  }
7384
7384
  if (byteLength > 125) {
7385
7385
  throw new RangeError("The data size must not be greater than 125 bytes");
@@ -7424,9 +7424,9 @@ var require_sender = __commonJS({
7424
7424
  byteLength = data2.size;
7425
7425
  readOnly = false;
7426
7426
  } else {
7427
- data2 = toBuffer(data2);
7427
+ data2 = toBuffer2(data2);
7428
7428
  byteLength = data2.length;
7429
- readOnly = toBuffer.readOnly;
7429
+ readOnly = toBuffer2.readOnly;
7430
7430
  }
7431
7431
  if (byteLength > 125) {
7432
7432
  throw new RangeError("The data size must not be greater than 125 bytes");
@@ -7482,9 +7482,9 @@ var require_sender = __commonJS({
7482
7482
  byteLength = data2.size;
7483
7483
  readOnly = false;
7484
7484
  } else {
7485
- data2 = toBuffer(data2);
7485
+ data2 = toBuffer2(data2);
7486
7486
  byteLength = data2.length;
7487
- readOnly = toBuffer.readOnly;
7487
+ readOnly = toBuffer2.readOnly;
7488
7488
  }
7489
7489
  if (this._firstFragment) {
7490
7490
  this._firstFragment = false;
@@ -7554,7 +7554,7 @@ var require_sender = __commonJS({
7554
7554
  return;
7555
7555
  }
7556
7556
  this._bufferedBytes -= options[kByteLength];
7557
- const data2 = toBuffer(arrayBuffer);
7557
+ const data2 = toBuffer2(arrayBuffer);
7558
7558
  if (!compress) {
7559
7559
  this._state = DEFAULT;
7560
7560
  this.sendFrame(_Sender.frame(data2, options), cb);
@@ -8081,7 +8081,7 @@ var require_websocket = __commonJS({
8081
8081
  EventTarget: { addEventListener, removeEventListener }
8082
8082
  } = require_event_target();
8083
8083
  var { format, parse } = require_extension();
8084
- var { toBuffer } = require_buffer_util();
8084
+ var { toBuffer: toBuffer2 } = require_buffer_util();
8085
8085
  var kAborted = /* @__PURE__ */ Symbol("kAborted");
8086
8086
  var protocolVersions = [8, 13];
8087
8087
  var readyStates = ["CONNECTING", "OPEN", "CLOSING", "CLOSED"];
@@ -8829,7 +8829,7 @@ var require_websocket = __commonJS({
8829
8829
  }
8830
8830
  function sendAfterClose(websocket, data2, cb) {
8831
8831
  if (data2) {
8832
- const length2 = isBlob(data2) ? data2.size : toBuffer(data2).length;
8832
+ const length2 = isBlob(data2) ? data2.size : toBuffer2(data2).length;
8833
8833
  if (websocket._socket) websocket._sender._bufferedBytes += length2;
8834
8834
  else websocket._bufferedAmount += length2;
8835
8835
  }
@@ -54825,7 +54825,7 @@ function lt(r6) {
54825
54825
  return r6.bareCriteria ? [r6.bareCriteria, ...r6.criteriaWithProps] : r6.criteriaWithProps;
54826
54826
  }
54827
54827
  var hc = new X(1e3);
54828
- var yn = "5.1.21-747496a";
54828
+ var yn = "5.1.22-7350a53";
54829
54829
  var xc = k.FHIR_JSON + ", */*; q=0.1";
54830
54830
  var vc = "https://api.medplum.com/";
54831
54831
  var Tc = 1e3;
@@ -56056,11 +56056,11 @@ async function Dy(r6) {
56056
56056
 
56057
56057
  // src/main.ts
56058
56058
  var import_node_child_process3 = require("node:child_process");
56059
- var import_node_fs7 = require("node:fs");
56059
+ var import_node_fs8 = require("node:fs");
56060
56060
  var import_node_path5 = __toESM(require("node:path"));
56061
56061
 
56062
56062
  // src/agent-main.ts
56063
- var import_node_fs5 = require("node:fs");
56063
+ var import_node_fs6 = require("node:fs");
56064
56064
 
56065
56065
  // ../hl7/dist/esm/index.mjs
56066
56066
  var import_node_assert = __toESM(require("node:assert"), 1);
@@ -56435,7 +56435,7 @@ var x = class {
56435
56435
  // src/app.ts
56436
56436
  var import_node_assert3 = __toESM(require("node:assert"));
56437
56437
  var import_node_child_process = require("node:child_process");
56438
- var import_node_fs4 = require("node:fs");
56438
+ var import_node_fs5 = require("node:fs");
56439
56439
  var import_node_net4 = require("node:net");
56440
56440
  var import_node_os4 = require("node:os");
56441
56441
  var import_node_process2 = __toESM(require("node:process"));
@@ -56597,9 +56597,11 @@ var AgentByteStreamChannel = class extends BaseChannel {
56597
56597
  }
56598
56598
  sendToRemote(msg) {
56599
56599
  const connection = this.connections.get(msg.remote);
56600
- if (connection) {
56601
- connection.write(Buffer.from(msg.body, "hex"));
56600
+ if (!connection) {
56601
+ return false;
56602
56602
  }
56603
+ connection.write(Buffer.from(msg.body, "hex"));
56604
+ return true;
56603
56605
  }
56604
56606
  handleNewConnection(socket) {
56605
56607
  const c2 = new ByteStreamChannelConnection(this, socket);
@@ -73620,6 +73622,410 @@ function calculateRttStats(rttSamples, pendingCount) {
73620
73622
  };
73621
73623
  }
73622
73624
 
73625
+ // src/queue/types.ts
73626
+ var MessageState = {
73627
+ QUEUED: "queued",
73628
+ CLAIMED: "claimed",
73629
+ INFLIGHT: "inflight",
73630
+ PROCESSED: "processed",
73631
+ REJECTED: "rejected",
73632
+ FAILED: "failed",
73633
+ NACKED: "nacked"
73634
+ };
73635
+ var AckOutcome = {
73636
+ PENDING: "pending",
73637
+ DELIVERED: "delivered",
73638
+ UNDELIVERED: "undelivered",
73639
+ NOT_OWED: "not_owed"
73640
+ };
73641
+ var QueueErrorCode = {
73642
+ /** Transient (`failed`): server returned 5xx. */
73643
+ ServerError: "server-error",
73644
+ /** Transient (`failed`): server returned 429. */
73645
+ ServerRateLimited: "server-rate-limited",
73646
+ /** Ambiguous (`failed`): timed out waiting for the server response; delivery unknown. */
73647
+ ResponseTimeout: "response-timeout",
73648
+ /** Ambiguous (`failed`): row was in `processing` when the agent restarted. */
73649
+ Interrupted: "interrupted",
73650
+ /** Ambiguous (`failed`): in-flight dispatch was cancelled by worker shutdown. */
73651
+ WorkerStopped: "worker-stopped",
73652
+ /** Ambiguous (`failed`): dispatch failed for an unclassified reason; delivery unknown. */
73653
+ DispatchFailed: "dispatch-failed",
73654
+ /** Permanent (`rejected`): server returned 4xx (other than 429) — the message was rejected. */
73655
+ ServerRejected: "server-rejected",
73656
+ /** Intake rejection (`nacked`): a storage error prevented the message from being committed. */
73657
+ StorageError: "storage-error",
73658
+ /** Intake rejection (`nacked`): the message reused a committed control ID (duplicate collision). */
73659
+ DuplicateRejected: "duplicate-rejected"
73660
+ };
73661
+ var QueueError = class extends Error {
73662
+ constructor(code, message) {
73663
+ super(message);
73664
+ __publicField(this, "code");
73665
+ this.code = code;
73666
+ }
73667
+ };
73668
+ var QueueLeaseError = class extends Error {
73669
+ constructor(localHolder, currentHolder) {
73670
+ super(
73671
+ `Queue lease held by a peer; this process is no longer leader (local holder=${localHolder ?? "unset"}, current holder=${currentHolder ?? "none"})`
73672
+ );
73673
+ /** The holder this process believes it is (the one bound via `setLeaseHolder`). */
73674
+ __publicField(this, "localHolder");
73675
+ /** The holder that actually owns the lease now (the peer that took over). */
73676
+ __publicField(this, "currentHolder");
73677
+ this.name = "QueueLeaseError";
73678
+ this.localHolder = localHolder;
73679
+ this.currentHolder = currentHolder;
73680
+ }
73681
+ };
73682
+ var DuplicateBehavior = {
73683
+ REJECT: "reject",
73684
+ IDEMPOTENT: "idempotent"
73685
+ };
73686
+
73687
+ // src/queue/worker.ts
73688
+ var DEFAULT_WORKER_RESPONSE_TIMEOUT_MS = 6e4;
73689
+ var DEFAULT_WORKER_IDLE_POLL_MS = 250;
73690
+ var RowRequeuedError = class extends Error {
73691
+ constructor() {
73692
+ super("row requeued after WebSocket disconnect");
73693
+ }
73694
+ };
73695
+ var ChannelQueueWorker = class {
73696
+ constructor(options) {
73697
+ __publicField(this, "channelName");
73698
+ __publicField(this, "app");
73699
+ __publicField(this, "queue");
73700
+ __publicField(this, "log");
73701
+ __publicField(this, "responseTimeoutMs");
73702
+ __publicField(this, "idlePollMs");
73703
+ __publicField(this, "sendAck");
73704
+ __publicField(this, "running", false);
73705
+ __publicField(this, "stopping", false);
73706
+ __publicField(this, "loopPromise");
73707
+ // Resolves whenever `notify()` is called, then is replaced with a fresh promise.
73708
+ // Lets the loop sleep without polling when the queue is known-empty.
73709
+ __publicField(this, "wakeSignal");
73710
+ __publicField(this, "pending");
73711
+ /**
73712
+ * In-flight lease check, fired on each App heartbeat tick (~10s). While a
73713
+ * dispatch is awaiting the server, a worker can't notice via `claimNext` that a
73714
+ * peer took the lease — it's wedged on the response. This cancels that wedged
73715
+ * dispatch with a {@link QueueLeaseError} so the loop tears the worker down,
73716
+ * instead of waiting out the full {@link DEFAULT_WORKER_RESPONSE_TIMEOUT_MS}. A
73717
+ * no-op while no dispatch is pending — an idle worker detects loss far sooner on
73718
+ * its next `claimNext` poll. An arrow field so add/removeEventListener share one
73719
+ * stable reference.
73720
+ */
73721
+ __publicField(this, "onHeartbeat", () => {
73722
+ const pending = this.pending;
73723
+ if (!pending || !this.queue.isLeaseHeldByPeer()) {
73724
+ return;
73725
+ }
73726
+ clearTimeout(pending.timeout);
73727
+ this.pending = void 0;
73728
+ pending.reject(new QueueLeaseError(void 0, this.queue.getCurrentLease()?.holder));
73729
+ });
73730
+ this.channelName = options.channelName;
73731
+ this.app = options.app;
73732
+ this.queue = options.queue;
73733
+ this.log = options.log;
73734
+ this.responseTimeoutMs = options.responseTimeoutMs ?? DEFAULT_WORKER_RESPONSE_TIMEOUT_MS;
73735
+ this.idlePollMs = options.idlePollMs ?? DEFAULT_WORKER_IDLE_POLL_MS;
73736
+ this.sendAck = options.sendAck;
73737
+ this.wakeSignal = makeWakeSignal();
73738
+ }
73739
+ /** Starts the dispatch loop. No-op if already started. */
73740
+ start() {
73741
+ if (this.running) {
73742
+ return;
73743
+ }
73744
+ this.running = true;
73745
+ this.stopping = false;
73746
+ this.app.heartbeatEmitter.addEventListener("heartbeat", this.onHeartbeat);
73747
+ this.loopPromise = this.loop().catch((err2) => {
73748
+ this.log.error(`Worker loop crashed: ${Re(err2)}`);
73749
+ });
73750
+ }
73751
+ /**
73752
+ * @returns True while the dispatch loop is live. Goes false after {@link stop}
73753
+ * or after the worker self-terminates on lease loss — the channel checks this
73754
+ * to reap a demoted worker before starting a fresh one on re-acquisition.
73755
+ */
73756
+ isRunning() {
73757
+ return this.running;
73758
+ }
73759
+ /**
73760
+ * Signals to the loop that work may be available. Idempotent; multiple calls
73761
+ * before the loop wakes coalesce into a single wake.
73762
+ */
73763
+ notify() {
73764
+ this.wakeSignal.resolve();
73765
+ }
73766
+ /**
73767
+ * Routes a server `agent:transmit:response` to its row.
73768
+ *
73769
+ * The common case is the response to the current in-flight dispatch, which
73770
+ * resolves the pending promise and lets {@link process} settle the row.
73771
+ *
73772
+ * A response can also arrive *late* — after the response timeout (or a requeue
73773
+ * / worker stop) already cleared the pending dispatch and left the row
73774
+ * `failed`. We don't discard those: the Medplum server is the authority on the
73775
+ * Bot-leg outcome, so a late ACK/NACK for a row that errored and isn't being
73776
+ * retried right now is applied to settle the row ({@link applyServerResponse}),
73777
+ * exactly as if it had arrived in time. This disambiguates the ambiguous
73778
+ * `ResponseTimeout` case — a row we couldn't classify becomes a definite
73779
+ * `processed`/`rejected`/`failed` — and avoids a redundant re-dispatch.
73780
+ * @param response - The response message received over the WS.
73781
+ * @returns True if the response was applied or resolved a pending dispatch;
73782
+ * false if it could not be matched to a settleable row.
73783
+ */
73784
+ onServerResponse(response2) {
73785
+ if (!response2.callback) {
73786
+ return false;
73787
+ }
73788
+ const pending = this.pending;
73789
+ if (pending?.row.callbackId === response2.callback) {
73790
+ clearTimeout(pending.timeout);
73791
+ this.pending = void 0;
73792
+ pending.resolve(response2);
73793
+ return true;
73794
+ }
73795
+ const row = this.queue.findByCallback(response2.callback);
73796
+ if (!row) {
73797
+ this.log.warn(`Discarding server response with no matching row (callback=${response2.callback})`);
73798
+ return false;
73799
+ }
73800
+ if (row.state === MessageState.FAILED) {
73801
+ this.log.info(
73802
+ `Applying late server response to errored row id=${row.id} (control id=${row.msgControlId ?? "n/a"}, status=${response2.statusCode ?? "n/a"})`
73803
+ );
73804
+ try {
73805
+ this.applyServerResponse(row, response2);
73806
+ } catch (err2) {
73807
+ if (err2 instanceof QueueLeaseError) {
73808
+ this.log.info(`Discarding late server response for row id=${row.id}: queue lease held by a peer`);
73809
+ return false;
73810
+ }
73811
+ throw err2;
73812
+ }
73813
+ return true;
73814
+ }
73815
+ this.log.warn(
73816
+ `Discarding server response for row id=${row.id} in state '${row.state}' (callback=${response2.callback})`
73817
+ );
73818
+ return false;
73819
+ }
73820
+ /**
73821
+ * Stops the dispatch loop. Cancels any in-flight dispatch by rejecting its
73822
+ * pending response Promise with `worker-stopped` — an ambiguous outcome (we
73823
+ * don't know whether the server processed the message). The row is marked
73824
+ * `failed` for operator review; an operator decides whether to replay.
73825
+ */
73826
+ async stop() {
73827
+ if (!this.running) {
73828
+ return;
73829
+ }
73830
+ this.stopping = true;
73831
+ this.app.heartbeatEmitter.removeEventListener("heartbeat", this.onHeartbeat);
73832
+ if (this.pending) {
73833
+ const pending = this.pending;
73834
+ clearTimeout(pending.timeout);
73835
+ this.pending = void 0;
73836
+ pending.reject(new QueueError(QueueErrorCode.WorkerStopped, "worker stopping"));
73837
+ }
73838
+ this.notify();
73839
+ if (this.loopPromise) {
73840
+ await this.loopPromise;
73841
+ }
73842
+ this.running = false;
73843
+ }
73844
+ /** @returns True if a row is currently in-flight (worker awaiting a server response). */
73845
+ hasInFlight() {
73846
+ return this.pending !== void 0;
73847
+ }
73848
+ /**
73849
+ * Called from `app.ts` when the agent WebSocket closes. If the in-flight
73850
+ * row's `agent:transmit:request` is still sitting unsent in the app's WS
73851
+ * queue, the server provably never saw it — remove it and return the row to
73852
+ * `queued` so it retries on reconnect instead of timing out into `failed`.
73853
+ *
73854
+ * If the request already went out on the wire, the outcome is ambiguous (the
73855
+ * server may have processed it and the response was lost), so we leave the
73856
+ * pending dispatch alone and let the response timeout mark it `errored`.
73857
+ */
73858
+ onWebSocketDisconnect() {
73859
+ const pending = this.pending;
73860
+ if (!pending) {
73861
+ return;
73862
+ }
73863
+ if (!this.app.removeUnsentTransmit(pending.row.callbackId)) {
73864
+ return;
73865
+ }
73866
+ clearTimeout(pending.timeout);
73867
+ this.pending = void 0;
73868
+ try {
73869
+ this.queue.requeue(pending.row.id);
73870
+ } catch (err2) {
73871
+ if (err2 instanceof QueueLeaseError) {
73872
+ pending.reject(err2);
73873
+ return;
73874
+ }
73875
+ throw err2;
73876
+ }
73877
+ this.log.info(
73878
+ `Row id=${pending.row.id} (control id=${pending.row.msgControlId ?? "n/a"}) requeued: WebSocket disconnected before transmit was sent`
73879
+ );
73880
+ pending.reject(new RowRequeuedError());
73881
+ }
73882
+ async loop() {
73883
+ try {
73884
+ while (!this.stopping) {
73885
+ if (!this.app.isLive()) {
73886
+ await this.waitForWork();
73887
+ continue;
73888
+ }
73889
+ const row = this.queue.claimNext(this.channelName);
73890
+ if (row) {
73891
+ await this.process(row);
73892
+ } else {
73893
+ await this.waitForWork();
73894
+ }
73895
+ }
73896
+ } catch (err2) {
73897
+ if (!(err2 instanceof QueueLeaseError)) {
73898
+ throw err2;
73899
+ }
73900
+ this.log.info(`Worker for channel '${this.channelName}' stepping down: queue lease taken by a peer.`);
73901
+ this.stopping = true;
73902
+ this.app.heartbeatEmitter.removeEventListener("heartbeat", this.onHeartbeat);
73903
+ this.running = false;
73904
+ }
73905
+ }
73906
+ async process(row) {
73907
+ let response2;
73908
+ try {
73909
+ response2 = await this.dispatch(row);
73910
+ } catch (err2) {
73911
+ if (err2 instanceof RowRequeuedError) {
73912
+ return;
73913
+ }
73914
+ if (err2 instanceof QueueLeaseError) {
73915
+ throw err2;
73916
+ }
73917
+ const code = err2 instanceof QueueError ? err2.code : QueueErrorCode.DispatchFailed;
73918
+ const msg = Re(err2);
73919
+ this.queue.markFailed(row.id, msg, code);
73920
+ this.log.error(
73921
+ `Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) failed during dispatch (${code}): ${msg}`
73922
+ );
73923
+ return;
73924
+ }
73925
+ this.applyServerResponse(row, response2);
73926
+ }
73927
+ /**
73928
+ * Settles a row from its server `agent:transmit:response`: records the raw
73929
+ * response, then transitions the row by status — 2xx → `processed` (plus the
73930
+ * source-leg ACK), permanent 4xx → `rejected`, 5xx/429 → `failed`.
73931
+ *
73932
+ * Shared by the normal in-flight path ({@link process}) and the late-response
73933
+ * path ({@link onServerResponse}), so a response that arrives after the
73934
+ * response timeout settles the row identically to one that arrived in time.
73935
+ * @param row - The row the response belongs to.
73936
+ * @param response - The server response to apply.
73937
+ */
73938
+ applyServerResponse(row, response2) {
73939
+ this.queue.recordServerResponse(row.id, response2.statusCode ?? null, response2.body ?? null);
73940
+ const statusCode = response2.statusCode ?? 0;
73941
+ if (statusCode >= 400) {
73942
+ const msg = `Server returned ${statusCode}: ${response2.body ?? ""}`;
73943
+ const code = classifyStatusCode(statusCode);
73944
+ if (code === QueueErrorCode.ServerRejected) {
73945
+ this.queue.markRejected(row.id, msg, code);
73946
+ } else {
73947
+ this.queue.markFailed(row.id, msg, code);
73948
+ }
73949
+ this.log.warn(`Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) ${code}: ${msg}`);
73950
+ return;
73951
+ }
73952
+ if (row.enhancedMode === "aaMode") {
73953
+ this.queue.markProcessed(row.id, AckOutcome.DELIVERED);
73954
+ this.log.debug(
73955
+ `Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) processed; app-level AA suppressed (aaMode commit ACK already acknowledged the source)`
73956
+ );
73957
+ return;
73958
+ }
73959
+ let ackOk = false;
73960
+ let ackError;
73961
+ try {
73962
+ ackOk = this.sendAck(response2, row);
73963
+ } catch (err2) {
73964
+ ackError = err2;
73965
+ }
73966
+ this.queue.markProcessed(row.id, ackOk ? AckOutcome.DELIVERED : AckOutcome.UNDELIVERED);
73967
+ if (!ackOk) {
73968
+ const detail = ackError ? `: ${Re(ackError)}` : "";
73969
+ this.log.warn(
73970
+ `Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) processed upstream but ACK delivery to source failed${detail}; awaiting source retransmit to replay`
73971
+ );
73972
+ }
73973
+ }
73974
+ async dispatch(row) {
73975
+ return new Promise((resolve2, reject) => {
73976
+ const timeout = setTimeout(() => {
73977
+ if (this.pending?.row.id === row.id) {
73978
+ this.pending = void 0;
73979
+ }
73980
+ reject(
73981
+ new QueueError(
73982
+ QueueErrorCode.ResponseTimeout,
73983
+ `Timed out after ${this.responseTimeoutMs}ms waiting for server response`
73984
+ )
73985
+ );
73986
+ }, this.responseTimeoutMs);
73987
+ if (typeof timeout.unref === "function") {
73988
+ timeout.unref();
73989
+ }
73990
+ this.pending = { row, resolve: resolve2, reject, timeout };
73991
+ this.app.addToWebSocketQueue({
73992
+ type: "agent:transmit:request",
73993
+ accessToken: "placeholder",
73994
+ channel: row.channelName,
73995
+ remote: row.remote,
73996
+ contentType: k.HL7_V2,
73997
+ // finalizedMessage holds the decoded HL7 text stored as UTF-8 at intake
73998
+ // (see AgentHl7ChannelConnection.handleMessage) — the same body the legacy
73999
+ // path forwards. The channel `encoding` is wire-level only and recorded
74000
+ // on the row for reference; it does not affect the forwarded text.
74001
+ body: row.finalizedMessage.toString("utf8"),
74002
+ callback: row.callbackId
74003
+ });
74004
+ });
74005
+ }
74006
+ async waitForWork() {
74007
+ const wake = this.wakeSignal;
74008
+ this.wakeSignal = makeWakeSignal();
74009
+ await Promise.race([wake.promise, rn(this.idlePollMs)]);
74010
+ }
74011
+ };
74012
+ function classifyStatusCode(statusCode) {
74013
+ if (statusCode === 429) {
74014
+ return QueueErrorCode.ServerRateLimited;
74015
+ }
74016
+ if (statusCode >= 500) {
74017
+ return QueueErrorCode.ServerError;
74018
+ }
74019
+ return QueueErrorCode.ServerRejected;
74020
+ }
74021
+ function makeWakeSignal() {
74022
+ let resolve2;
74023
+ const promise = new Promise((r6) => {
74024
+ resolve2 = r6;
74025
+ });
74026
+ return { promise, resolve: resolve2 };
74027
+ }
74028
+
73623
74029
  // src/stats.ts
73624
74030
  var currentStats = {
73625
74031
  hl7ConnectionsOpen: 0,
@@ -73649,6 +74055,14 @@ var AgentHl7Channel = class extends BaseChannel {
73649
74055
  // Default app level ack mode is AL (Always)
73650
74056
  __publicField(this, "assignSeqNo", false);
73651
74057
  __publicField(this, "lastSeqNo", -1);
74058
+ __publicField(this, "duplicateBehavior", DuplicateBehavior.IDEMPOTENT);
74059
+ // The channel's own copy of the enhanced mode, parsed from the endpoint URL.
74060
+ // In durable mode this is intentionally NOT pushed onto the Hl7Connection (so
74061
+ // the connection's synchronous auto-ACK stays off and the agent can defer the
74062
+ // commit ACK until after the DB write); the connection therefore can't be the
74063
+ // source of truth, so we track it here and the durable path reads it directly.
74064
+ __publicField(this, "enhancedMode");
74065
+ __publicField(this, "worker");
73652
74066
  this.server = new x((connection) => this.handleNewConnection(connection));
73653
74067
  this.prefix = `[HL7:${definition.name}] `;
73654
74068
  this.log = app.log.clone({ options: { prefix: this.prefix } });
@@ -73664,6 +74078,7 @@ var AgentHl7Channel = class extends BaseChannel {
73664
74078
  this.log.info(`Channel starting on ${address}...`);
73665
74079
  this.stats = new ChannelStatsTracker({ heartbeatEmitter: this.app.heartbeatEmitter, log: this.log });
73666
74080
  this.configureHl7ServerAndConnections();
74081
+ this.maybeStartWorker();
73667
74082
  await this.server.start(Number.parseInt(address.port, 10));
73668
74083
  this.log.info("Channel started successfully");
73669
74084
  }
@@ -73672,45 +74087,102 @@ var AgentHl7Channel = class extends BaseChannel {
73672
74087
  return;
73673
74088
  }
73674
74089
  this.log.info("Channel stopping...");
74090
+ if (this.worker) {
74091
+ await this.worker.stop();
74092
+ this.worker = void 0;
74093
+ }
73675
74094
  await Promise.allSettled(Array.from(this.connections.values()).map((connection) => connection.close()));
73676
74095
  await this.server.stop();
73677
74096
  this.stats.cleanup();
73678
74097
  this.started = false;
73679
74098
  this.log.info("Channel stopped successfully");
73680
74099
  }
74100
+ /**
74101
+ * Starts a {@link ChannelQueueWorker} bound to this channel and the app's
74102
+ * durable queue, but only if we currently hold the queue lease.
74103
+ *
74104
+ * Called both from `start()` (when leadership may already be established by
74105
+ * the time the channel comes up) and from `onBecameQueueLeader()` (when
74106
+ * leadership arrives later, e.g. after waiting out a peer's lease during a
74107
+ * zero-downtime upgrade overlap).
74108
+ *
74109
+ * No-op when the queue is off, we're not leader, or the worker is already
74110
+ * running — so it's safe to call from either entry point.
74111
+ */
74112
+ maybeStartWorker() {
74113
+ if (this.worker && !this.worker.isRunning()) {
74114
+ this.worker = void 0;
74115
+ }
74116
+ if (this.worker) {
74117
+ return;
74118
+ }
74119
+ const queue = this.app.getDurableQueue();
74120
+ if (!queue?.isLeader()) {
74121
+ return;
74122
+ }
74123
+ this.worker = new ChannelQueueWorker({
74124
+ channelName: this.getDefinition().name,
74125
+ app: this.app,
74126
+ queue,
74127
+ log: this.log,
74128
+ sendAck: (response2) => this.sendToRemote(response2)
74129
+ });
74130
+ this.worker.start();
74131
+ this.worker.notify();
74132
+ }
74133
+ /**
74134
+ * Notification from the App that we've taken the durable-queue lease.
74135
+ * Triggers worker bring-up for this channel if it isn't already running.
74136
+ */
74137
+ onBecameQueueLeader() {
74138
+ this.maybeStartWorker();
74139
+ }
73681
74140
  shouldAssignSeqNo() {
73682
74141
  return this.assignSeqNo;
73683
74142
  }
74143
+ /**
74144
+ * Returns the next MSH.13 sequence number from the in-memory counter (resets
74145
+ * on restart). Used only by the legacy (non-durable) path; the durable path
74146
+ * peeks/commits a persisted per-channel counter via the queue so a failed
74147
+ * intake doesn't consume a number. See {@link DurableQueue.peekNextSeqNo}.
74148
+ * @returns The next sequence number to assign.
74149
+ */
73684
74150
  takeNextSeqNo() {
73685
74151
  return ++this.lastSeqNo;
73686
74152
  }
73687
74153
  sendToRemote(msg) {
73688
74154
  const connection = this.connections.get(msg.remote);
73689
- if (connection) {
73690
- const hl7Message = Wo.parse(msg.body);
73691
- const msgControlId = hl7Message.getSegment("MSA")?.getField(2)?.toString();
73692
- const ackCode = hl7Message.getSegment("MSA")?.getField(1)?.toString()?.toUpperCase();
73693
- if (ackCode && isAppLevelAckCode(ackCode) && !shouldSendAppLevelAck({
73694
- mode: this.appLevelAckMode,
73695
- ackCode,
73696
- enhancedMode: this.server.getEnhancedMode()
73697
- })) {
73698
- this.channelLog.debug(
73699
- `[Skipping ACK -- Mode: ${this.appLevelAckMode} -- ID: ${msgControlId ?? "not provided"} -- ACK: ${ackCode ?? "unknown"}]`
73700
- );
73701
- if (msgControlId) {
73702
- this.stats.recordAckReceived(msgControlId);
73703
- }
73704
- return;
73705
- }
73706
- this.channelLog.info(`[Sending ACK -- ID: ${msgControlId}]: ${hl7Message.toString().replaceAll("\r", "\n")}`);
73707
- connection.hl7Connection.send(Wo.parse(msg.body));
74155
+ if (!connection) {
74156
+ this.log.warn(`Attempted to send message to disconnected remote: ${msg.remote}`);
74157
+ return false;
74158
+ }
74159
+ const hl7Message = Wo.parse(msg.body);
74160
+ const msgControlId = hl7Message.getSegment("MSA")?.getField(2)?.toString();
74161
+ const ackCode = hl7Message.getSegment("MSA")?.getField(1)?.toString()?.toUpperCase();
74162
+ if (ackCode && isAppLevelAckCode(ackCode) && !shouldSendAppLevelAck({
74163
+ mode: this.appLevelAckMode,
74164
+ ackCode,
74165
+ enhancedMode: this.enhancedMode
74166
+ })) {
74167
+ this.channelLog.debug(
74168
+ `[Skipping ACK -- Mode: ${this.appLevelAckMode} -- ID: ${msgControlId ?? "not provided"} -- ACK: ${ackCode ?? "unknown"}]`
74169
+ );
73708
74170
  if (msgControlId) {
73709
74171
  this.stats.recordAckReceived(msgControlId);
73710
74172
  }
73711
- } else {
73712
- this.log.warn(`Attempted to send message to disconnected remote: ${msg.remote}`);
74173
+ return true;
73713
74174
  }
74175
+ this.channelLog.info(`[Sending ACK -- ID: ${msgControlId}]: ${hl7Message.toString().replaceAll("\r", "\n")}`);
74176
+ try {
74177
+ connection.hl7Connection.send(Wo.parse(msg.body));
74178
+ } catch (err2) {
74179
+ this.channelLog.error(`Failed to send ACK to ${msg.remote}: ${Re(err2)}`);
74180
+ return false;
74181
+ }
74182
+ if (msgControlId) {
74183
+ this.stats.recordAckReceived(msgControlId);
74184
+ }
74185
+ return true;
73714
74186
  }
73715
74187
  async reloadConfig(definition, endpoint) {
73716
74188
  const previousEndpoint = this.endpoint;
@@ -73744,6 +74216,7 @@ var AgentHl7Channel = class extends BaseChannel {
73744
74216
  const assignSeqNo = address.searchParams.get("assignSeqNo")?.toLowerCase() === "true";
73745
74217
  const messagesPerMinRaw = address.searchParams.get("messagesPerMin") ?? void 0;
73746
74218
  const appLevelAckRaw = address.searchParams.get("appLevelAck") ?? void 0;
74219
+ const duplicateBehaviorRaw = address.searchParams.get("duplicateBehavior") ?? void 0;
73747
74220
  let messagesPerMin = messagesPerMinRaw ? Number.parseInt(messagesPerMinRaw, 10) : void 0;
73748
74221
  if (messagesPerMin !== void 0 && !Number.isInteger(messagesPerMin)) {
73749
74222
  this.log.warn(
@@ -73753,18 +74226,33 @@ var AgentHl7Channel = class extends BaseChannel {
73753
74226
  }
73754
74227
  this.appLevelAckMode = parseAppLevelAckMode(appLevelAckRaw, this.log);
73755
74228
  this.assignSeqNo = assignSeqNo;
74229
+ this.duplicateBehavior = parseDuplicateBehavior(duplicateBehaviorRaw, this.log);
73756
74230
  if (!assignSeqNo) {
73757
74231
  this.lastSeqNo = -1;
73758
74232
  }
74233
+ this.enhancedMode = enhancedMode;
74234
+ const queueOn = this.app.getDurableQueue() !== void 0;
74235
+ const connectionEnhancedMode = queueOn ? void 0 : enhancedMode;
73759
74236
  this.server.setEncoding(encoding);
73760
- this.server.setEnhancedMode(enhancedMode);
74237
+ this.server.setEnhancedMode(connectionEnhancedMode);
73761
74238
  this.server.setMessagesPerMin(messagesPerMin);
73762
74239
  for (const connection of this.connections.values()) {
73763
74240
  connection.hl7Connection.setEncoding(encoding);
73764
- connection.hl7Connection.setEnhancedMode(enhancedMode);
74241
+ connection.hl7Connection.setEnhancedMode(connectionEnhancedMode);
73765
74242
  connection.hl7Connection.setMessagesPerMin(messagesPerMin);
73766
74243
  }
73767
74244
  }
74245
+ getDuplicateBehavior() {
74246
+ return this.duplicateBehavior;
74247
+ }
74248
+ /**
74249
+ * @returns The channel's enhanced mode, as parsed from the endpoint URL. This is
74250
+ * the agent's source of truth in durable mode, where enhancedMode is deliberately
74251
+ * kept off the underlying {@link Hl7Connection} (see {@link enhancedMode}).
74252
+ */
74253
+ getEnhancedMode() {
74254
+ return this.enhancedMode;
74255
+ }
73768
74256
  handleNewConnection(connection) {
73769
74257
  const c2 = new AgentHl7ChannelConnection(this, connection);
73770
74258
  updateStat("hl7ConnectionsOpen", getCurrentStats().hl7ConnectionsOpen + 1);
@@ -73798,44 +74286,353 @@ var AgentHl7ChannelConnection = class {
73798
74286
  this.channel.channelLog.info(
73799
74287
  `[Received -- ID: ${msgControlId ?? "not provided"}]: ${event.message.toString().replaceAll("\r", "\n")}`
73800
74288
  );
73801
- if (this.channel.shouldAssignSeqNo()) {
73802
- const seqNo = this.channel.takeNextSeqNo();
73803
- event.message.getSegment("MSH")?.setField(13, seqNo.toString());
73804
- this.channel.channelLog.info(`Setting sequence number for message control ID '${msgControlId}': ${seqNo}`);
73805
- }
73806
- this.channel.app.addToWebSocketQueue({
73807
- type: "agent:transmit:request",
73808
- accessToken: "placeholder",
73809
- channel: this.channel.getDefinition().name,
73810
- remote: this.remote,
73811
- contentType: k.HL7_V2,
73812
- body: event.message.toString(),
73813
- callback: `Agent/${this.channel.app.agentId}-${(0, import_node_crypto3.randomUUID)()}`
73814
- });
74289
+ const originalMessage = Buffer.from(event.message.toString(), "utf8");
73815
74290
  if (msgControlId) {
73816
74291
  this.channel.stats.recordMessageSent(msgControlId);
73817
74292
  }
74293
+ const queue = this.channel.app.getDurableQueue();
74294
+ if (queue) {
74295
+ await this.handleMessageDurable(queue, event, msgControlId, originalMessage);
74296
+ } else {
74297
+ this.handleMessageLegacy(event, msgControlId);
74298
+ }
73818
74299
  } catch (err2) {
73819
74300
  this.channel.log.error(`HL7 error occurred - check channel logs`);
73820
74301
  this.channel.channelLog.error(`HL7 error: ${Re(err2)}`);
73821
74302
  }
73822
74303
  }
74304
+ /**
74305
+ * Legacy non-durable inbound path: push directly to the in-memory WS queue.
74306
+ * Behavior unchanged from the pre-durable-queue agent — used when the
74307
+ * `durableQueue` setting is off.
74308
+ * @param event - The incoming HL7 message event from the underlying connection.
74309
+ * @param msgControlId - MSH.10 of the inbound message.
74310
+ */
74311
+ handleMessageLegacy(event, msgControlId) {
74312
+ this.maybeAssignSeqNo(event, msgControlId);
74313
+ this.channel.app.addToWebSocketQueue({
74314
+ type: "agent:transmit:request",
74315
+ accessToken: "placeholder",
74316
+ channel: this.channel.getDefinition().name,
74317
+ remote: this.remote,
74318
+ contentType: k.HL7_V2,
74319
+ body: event.message.toString(),
74320
+ callback: `Agent/${this.channel.app.agentId}-${(0, import_node_crypto3.randomUUID)()}`
74321
+ });
74322
+ }
74323
+ /**
74324
+ * Assigns the next channel sequence number into MSH.13 when assignSeqNo is on.
74325
+ * The durable path calls this only after the duplicate check, so retransmits
74326
+ * don't consume a sequence number; the legacy path calls it on every message.
74327
+ * @param event - The inbound message event whose MSH.13 to set.
74328
+ * @param msgControlId - MSH.10, for logging only.
74329
+ */
74330
+ maybeAssignSeqNo(event, msgControlId) {
74331
+ if (!this.channel.shouldAssignSeqNo()) {
74332
+ return;
74333
+ }
74334
+ const seqNo = this.channel.takeNextSeqNo();
74335
+ event.message.getSegment("MSH")?.setField(13, seqNo.toString());
74336
+ this.channel.channelLog.info(`Setting sequence number for message control ID '${msgControlId ?? "n/a"}': ${seqNo}`);
74337
+ }
74338
+ /**
74339
+ * Parses MSH.13 as a non-negative integer sequence number.
74340
+ * @param event - The inbound message event.
74341
+ * @returns The parsed sequence number, or null when MSH.13 is absent/non-numeric.
74342
+ */
74343
+ parseSeqNo(event) {
74344
+ const seqNoField = event.message.getSegment("MSH")?.getField(13)?.toString();
74345
+ return seqNoField && /^\d+$/.test(seqNoField) ? Number.parseInt(seqNoField, 10) : null;
74346
+ }
74347
+ /**
74348
+ * Durable inbound path (§8 of DURABLE_QUEUE_ARCHITECTURE.md):
74349
+ *
74350
+ * 1. INSERT a `queued` row.
74351
+ * 2. On success, send CA/AA via the deferred-ack API — only now is the
74352
+ * sender allowed to drop their retransmit buffer.
74353
+ * 3. Notify the channel worker so it picks the row up immediately.
74354
+ *
74355
+ * Duplicate handling — a prior non-`nacked` row already owns this MSH.10
74356
+ * (in any state: queued/processing/processed/errored — see {@link handleDuplicate}):
74357
+ * - `idempotent`: if the re-sent body matches the prior copy byte-for-byte,
74358
+ * replay the prior server response ACK (or, if none yet, the commit ACK);
74359
+ * if it differs, send AR — a different message reused a committed control ID.
74360
+ * - `reject`: send CR (or AR in aaMode); insert a `nacked` audit row.
74361
+ *
74362
+ * Storage failure: send CR with the underlying error in MSA.3; best-effort
74363
+ * insert a `nacked` audit row (which may itself fail if the DB is unwritable —
74364
+ * intentional, not retried).
74365
+ * @param queue - The app-owned durable queue handle.
74366
+ * @param event - The incoming HL7 message event from the underlying connection.
74367
+ * @param msgControlId - MSH.10 of the inbound message (may be undefined for malformed messages).
74368
+ * @param originalMessage - The message exactly as received, before any transformation (for dedup comparison).
74369
+ */
74370
+ async handleMessageDurable(queue, event, msgControlId, originalMessage) {
74371
+ const conn = this.hl7Connection;
74372
+ const enhancedMode = this.channel.getEnhancedMode();
74373
+ const enhancedModeColumn = enhancedMode ?? null;
74374
+ const channelName = this.channel.getDefinition().name;
74375
+ const msgType = event.message.getSegment("MSH")?.getField(9)?.toString() ?? null;
74376
+ const callbackId = `Agent/${this.channel.app.agentId}-${(0, import_node_crypto3.randomUUID)()}`;
74377
+ const receivedAt = Date.now();
74378
+ const assigning = this.channel.shouldAssignSeqNo();
74379
+ let finalizedMessage = originalMessage;
74380
+ let seqNo = this.parseSeqNo(event);
74381
+ let result;
74382
+ try {
74383
+ result = queue.enqueue(
74384
+ {
74385
+ channelName,
74386
+ remote: this.remote,
74387
+ msgControlId: msgControlId ?? null,
74388
+ msgType,
74389
+ originalMessage,
74390
+ finalizedMessage,
74391
+ encoding: conn.getEncoding() ?? null,
74392
+ enhancedMode: enhancedModeColumn,
74393
+ callbackId,
74394
+ seqNo,
74395
+ receivedAt
74396
+ },
74397
+ assigning ? {
74398
+ assignSeqNo: (candidate) => {
74399
+ event.message.getSegment("MSH")?.setField(13, candidate.toString());
74400
+ this.channel.channelLog.info(
74401
+ `Setting sequence number for message control ID '${msgControlId ?? "n/a"}': ${candidate}`
74402
+ );
74403
+ finalizedMessage = Buffer.from(event.message.toString(), "utf8");
74404
+ seqNo = candidate;
74405
+ return finalizedMessage;
74406
+ }
74407
+ } : void 0
74408
+ );
74409
+ } catch (err2) {
74410
+ const reason = `storage error: ${Re(err2)}`;
74411
+ this.channel.channelLog.error(`Durable enqueue failed for ${msgControlId ?? "no-id"}: ${reason}`);
74412
+ this.sendCommitNack(event.message, enhancedMode === "aaMode" ? "AE" : "CE", "storage error");
74413
+ this.recordImmediateAck(msgControlId);
74414
+ queue.enqueueRejected({
74415
+ channelName,
74416
+ remote: this.remote,
74417
+ msgControlId: msgControlId ?? null,
74418
+ msgType,
74419
+ originalMessage,
74420
+ finalizedMessage,
74421
+ encoding: conn.getEncoding() ?? null,
74422
+ enhancedMode: enhancedModeColumn,
74423
+ callbackId,
74424
+ seqNo,
74425
+ receivedAt,
74426
+ lastError: reason,
74427
+ errorCode: QueueErrorCode.StorageError
74428
+ });
74429
+ return;
74430
+ }
74431
+ if (result.kind === "duplicate") {
74432
+ this.handleDuplicate(queue, event, result.existing, msgControlId, {
74433
+ callbackId,
74434
+ msgType,
74435
+ originalMessage,
74436
+ finalizedMessage,
74437
+ enhancedModeColumn,
74438
+ seqNo,
74439
+ receivedAt
74440
+ });
74441
+ return;
74442
+ }
74443
+ this.sendCommitAck(event.message);
74444
+ this.recordImmediateAck(msgControlId);
74445
+ this.channel.worker?.notify();
74446
+ }
74447
+ /**
74448
+ * Handles an inbound message whose `(channel, MSH.10)` already belongs to a
74449
+ * prior non-`nacked` row (`existing`), per the channel's `duplicateBehavior`:
74450
+ *
74451
+ * - `reject`: reject every collision (CR, or AR in aaMode) and write a
74452
+ * `nacked` audit row — unchanged, now spanning all prior states.
74453
+ * - `idempotent` (default): treat a byte-for-byte re-send as a benign
74454
+ * retransmit and replay the acknowledgment the sender missed —
74455
+ * {@link AgentHl7Channel.sendToRemote the prior server response ACK} if the
74456
+ * message was already dispatched, otherwise the commit ACK (CA/AA). A body
74457
+ * that differs is a *different* message reusing a committed control ID, so
74458
+ * we reject it with AR (and a `nacked` audit row) the same way `reject`
74459
+ * handles any duplicate.
74460
+ *
74461
+ * The replayed ACKs go through {@link sendCommitAck}/{@link sendCommitNack} just
74462
+ * like a fresh message — a retransmit means the sender never saw the original
74463
+ * ACK and must be re-told, and the durable row is the dedup authority.
74464
+ * @param queue - The app-owned durable queue handle.
74465
+ * @param event - The duplicate inbound message event.
74466
+ * @param existing - The prior row that owns this MSH.10.
74467
+ * @param msgControlId - MSH.10 of the inbound message.
74468
+ * @param audit - Decoded fields of the inbound message, for any `nacked` audit row.
74469
+ * @param audit.callbackId - Callback ID minted for this inbound message.
74470
+ * @param audit.msgType - MSH.9 message type.
74471
+ * @param audit.originalMessage - Bytes as received (compared against the prior row's original_message).
74472
+ * @param audit.finalizedMessage - Bytes as transformed for dispatch (persisted on any audit row).
74473
+ * @param audit.enhancedModeColumn - Enhanced-mode column value to persist.
74474
+ * @param audit.seqNo - Sequence number (MSH.13), if any.
74475
+ * @param audit.receivedAt - Intake timestamp (ms).
74476
+ */
74477
+ handleDuplicate(queue, event, existing, msgControlId, audit) {
74478
+ const conn = this.hl7Connection;
74479
+ const enhancedMode = this.channel.getEnhancedMode();
74480
+ const idLabel = msgControlId ?? "n/a";
74481
+ const behavior = this.channel.getDuplicateBehavior();
74482
+ if (behavior === DuplicateBehavior.IDEMPOTENT && existing.originalMessage.equals(audit.originalMessage)) {
74483
+ if (existing.serverResponseBody && existing.serverResponseBody.length > 0 && this.replayServerAck(existing)) {
74484
+ if (existing.ackOutcome === AckOutcome.UNDELIVERED) {
74485
+ queue.setAckOutcome(existing.id, AckOutcome.DELIVERED);
74486
+ }
74487
+ this.channel.channelLog.info(
74488
+ `[Duplicate idempotent -- ID: ${idLabel}] replayed prior server response ACK from row id=${existing.id}`
74489
+ );
74490
+ return;
74491
+ }
74492
+ this.channel.channelLog.info(
74493
+ `[Duplicate idempotent -- ID: ${idLabel}] replayed commit ACK for prior row id=${existing.id}`
74494
+ );
74495
+ this.sendCommitAck(event.message);
74496
+ this.recordImmediateAck(msgControlId);
74497
+ return;
74498
+ }
74499
+ const contentMismatch = behavior === DuplicateBehavior.IDEMPOTENT;
74500
+ const reason = contentMismatch ? `duplicate control id ${idLabel}: a message with this control ID was already committed with different content` : "duplicate control id";
74501
+ this.channel.channelLog.warn(`[Duplicate rejected -- ID: ${idLabel}] prior row id=${existing.id}: ${reason}`);
74502
+ this.sendCommitNack(event.message, enhancedMode === "aaMode" ? "AR" : "CR", reason);
74503
+ this.recordImmediateAck(msgControlId);
74504
+ queue.enqueueRejected({
74505
+ channelName: this.channel.getDefinition().name,
74506
+ remote: this.remote,
74507
+ msgControlId: msgControlId ?? null,
74508
+ msgType: audit.msgType,
74509
+ originalMessage: audit.originalMessage,
74510
+ finalizedMessage: audit.finalizedMessage,
74511
+ encoding: conn.getEncoding() ?? null,
74512
+ enhancedMode: audit.enhancedModeColumn,
74513
+ callbackId: audit.callbackId,
74514
+ seqNo: audit.seqNo,
74515
+ receivedAt: audit.receivedAt,
74516
+ lastError: reason,
74517
+ errorCode: QueueErrorCode.DuplicateRejected
74518
+ });
74519
+ }
74520
+ /**
74521
+ * Balances {@link ChannelStatsTracker} for a message we answered synchronously
74522
+ * at intake (duplicate replay / reject / storage-error NACK). `handleMessage`
74523
+ * already called `recordMessageSent`; without this the control ID would linger
74524
+ * in the pending map until the 5-minute GC, skewing pendingCount and RTT.
74525
+ * @param msgControlId - MSH.10 of the message just answered, if present.
74526
+ */
74527
+ recordImmediateAck(msgControlId) {
74528
+ if (msgControlId) {
74529
+ this.channel.stats.recordAckReceived(msgControlId);
74530
+ }
74531
+ }
74532
+ /**
74533
+ * Replays the stored server response ACK for an already-dispatched duplicate,
74534
+ * routing it through {@link AgentHl7Channel.sendToRemote} so the same
74535
+ * app-level ACK policy and encoding apply as on the original delivery.
74536
+ * @param existing - The prior row whose `serverResponseBody` to replay.
74537
+ * @returns True if the response ACK was delivered; false if it couldn't be parsed/sent.
74538
+ */
74539
+ replayServerAck(existing) {
74540
+ if (!existing.serverResponseBody) {
74541
+ return false;
74542
+ }
74543
+ try {
74544
+ return this.channel.sendToRemote({
74545
+ type: "agent:transmit:response",
74546
+ channel: this.channel.getDefinition().name,
74547
+ remote: this.remote,
74548
+ contentType: k.HL7_V2,
74549
+ body: existing.serverResponseBody.toString("utf8"),
74550
+ callback: existing.callbackId
74551
+ });
74552
+ } catch (err2) {
74553
+ this.channel.channelLog.warn(
74554
+ `[Duplicate idempotent] failed to replay server response ACK for row id=${existing.id}: ${Re(err2)}`
74555
+ );
74556
+ return false;
74557
+ }
74558
+ }
73823
74559
  async handleError(event) {
73824
74560
  this.channel.log.error(`HL7 connection error: ${Re(event.error)}`);
73825
74561
  this.channel.channelLog.error(`HL7 connection error: ${Re(event.error)}`);
73826
74562
  }
74563
+ /**
74564
+ * Sends the commit ACK (CA in `standard` enhanced mode, AA in `aaMode`) for an
74565
+ * inbound message that has been durably committed. No-op outside enhanced mode.
74566
+ *
74567
+ * This is the durable path's replacement for the connection's synchronous
74568
+ * auto-ACK: in durable mode the connection carries no enhancedMode (so it never
74569
+ * auto-ACKs), and the agent calls this only after the DB write succeeds — so the
74570
+ * CA/AA is a real promise that the message is on disk. Idempotency is not enforced
74571
+ * here; the on-disk row (channel + MSH.10) is the dedup authority, and a genuine
74572
+ * retransmit must be re-ACKed because the sender never saw the original.
74573
+ * @param message - The original inbound message to ACK.
74574
+ */
74575
+ sendCommitAck(message) {
74576
+ const enhancedMode = this.channel.getEnhancedMode();
74577
+ if (!enhancedMode) {
74578
+ return;
74579
+ }
74580
+ const ackCode = enhancedMode === "standard" ? "CA" : "AA";
74581
+ const response2 = message.buildAck({ ackCode });
74582
+ this.hl7Connection.send(response2);
74583
+ this.hl7Connection.dispatchEvent(new E(this.hl7Connection, response2));
74584
+ }
74585
+ /**
74586
+ * Sends a negative commit ACK for an inbound message. The wire code is CE/CR in
74587
+ * `standard` enhanced mode and AE/AR in `aaMode` (error = retryable, reject =
74588
+ * terminal); the caller picks which best describes the failure. An optional
74589
+ * `reason` is written to MSA.3 for the sender's logs. No-op outside enhanced mode.
74590
+ * @param message - The original inbound message to NACK.
74591
+ * @param code - The negative ACK code to send.
74592
+ * @param reason - Optional human-readable explanation placed in MSA.3.
74593
+ */
74594
+ sendCommitNack(message, code, reason) {
74595
+ const enhancedMode = this.channel.getEnhancedMode();
74596
+ if (!enhancedMode) {
74597
+ return;
74598
+ }
74599
+ const response2 = message.buildAck({ ackCode: code });
74600
+ if (reason) {
74601
+ response2.getSegment("MSA")?.setField(3, reason);
74602
+ }
74603
+ this.hl7Connection.send(response2);
74604
+ this.hl7Connection.dispatchEvent(new E(this.hl7Connection, response2));
74605
+ }
73827
74606
  handleEnhancedAckSent(event) {
73828
74607
  const hl7Message = event.message;
73829
74608
  const msgControlId = hl7Message.getSegment("MSA")?.getField(2)?.toString();
73830
74609
  const ackCode = hl7Message.getSegment("MSA")?.getField(1)?.toString()?.toUpperCase();
73831
74610
  this.channel.channelLog.info(
73832
- `[Sent ${ackCode === "CA" ? "Commit ACK (CA)" : "Immediate ACK (AA)"} -- ID: ${msgControlId ?? "not provided"}]: ${hl7Message.toString().replaceAll("\r", "\n")}`
74611
+ `[Sent ${describeAckCode(ackCode)} -- ID: ${msgControlId ?? "not provided"}]: ${hl7Message.toString().replaceAll("\r", "\n")}`
73833
74612
  );
73834
74613
  }
73835
74614
  close() {
73836
74615
  return this.hl7Connection.close();
73837
74616
  }
73838
74617
  };
74618
+ function describeAckCode(code) {
74619
+ switch (code) {
74620
+ case "CA":
74621
+ return "Commit ACK (CA)";
74622
+ case "AA":
74623
+ return "App ACK (AA)";
74624
+ case "CE":
74625
+ return "Commit Error (CE)";
74626
+ case "CR":
74627
+ return "Commit Reject (CR)";
74628
+ case "AE":
74629
+ return "App Error (AE)";
74630
+ case "AR":
74631
+ return "App Reject (AR)";
74632
+ default:
74633
+ return `ACK (${code ?? "unknown"})`;
74634
+ }
74635
+ }
73839
74636
  function parseEnhancedMode(rawValue, logger) {
73840
74637
  if (!rawValue) {
73841
74638
  return void 0;
@@ -73869,6 +74666,17 @@ function isAppLevelAckCode(code) {
73869
74666
  function isAppLevelAckMode(candidate) {
73870
74667
  return APP_LEVEL_ACK_MODES.includes(candidate);
73871
74668
  }
74669
+ function parseDuplicateBehavior(rawValue, logger) {
74670
+ if (!rawValue) {
74671
+ return DuplicateBehavior.IDEMPOTENT;
74672
+ }
74673
+ const normalized = rawValue.toLowerCase();
74674
+ if (normalized === DuplicateBehavior.REJECT || normalized === DuplicateBehavior.IDEMPOTENT) {
74675
+ return normalized;
74676
+ }
74677
+ logger.warn(`Invalid duplicateBehavior value '${rawValue}'; expected 'reject' or 'idempotent'. Using idempotent.`);
74678
+ return DuplicateBehavior.IDEMPOTENT;
74679
+ }
73872
74680
  function shouldSendAppLevelAck(options) {
73873
74681
  const { mode, ackCode, enhancedMode } = options;
73874
74682
  if (!enhancedMode) {
@@ -74641,8 +75449,1355 @@ function registerAgentCleanup() {
74641
75449
  }
74642
75450
  }
74643
75451
 
74644
- // src/upgrader-utils.ts
75452
+ // src/queue/durable-queue.ts
75453
+ var import_node_crypto4 = require("node:crypto");
74645
75454
  var import_node_fs3 = require("node:fs");
75455
+
75456
+ // src/queue/queries.ts
75457
+ var ENQUEUE = `
75458
+ INSERT INTO inbound_hl7_messages (
75459
+ channel_name, remote, msg_control_id, msg_type, original_message, finalized_message, encoding,
75460
+ enhanced_mode, state, attempt_count, callback_id,
75461
+ seq_no, received_at
75462
+ ) VALUES (
75463
+ ?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?
75464
+ )
75465
+ `;
75466
+ var ENQUEUE_REJECTED = `
75467
+ INSERT INTO inbound_hl7_messages (
75468
+ channel_name, remote, msg_control_id, msg_type, original_message, finalized_message, encoding,
75469
+ enhanced_mode, state, attempt_count, callback_id,
75470
+ ack_outcome, last_error, error_code, seq_no, received_at
75471
+ ) VALUES (
75472
+ ?, ?, ?, ?, ?, ?, ?, ?, 'nacked', 0, ?, 'not_owed', ?, ?, ?, ?
75473
+ )
75474
+ `;
75475
+ var PEEK_LAST_SEQ_NO = `
75476
+ SELECT last_seq_no FROM _channel_seq WHERE channel_name = ?
75477
+ `;
75478
+ var COMMIT_SEQ_NO = `
75479
+ INSERT INTO _channel_seq (channel_name, last_seq_no) VALUES (?, ?)
75480
+ ON CONFLICT(channel_name) DO UPDATE SET last_seq_no = excluded.last_seq_no
75481
+ `;
75482
+ var FIND_SEEN_BY_CONTROL_ID = `
75483
+ SELECT * FROM inbound_hl7_messages
75484
+ WHERE channel_name = ?
75485
+ AND msg_control_id = ?
75486
+ AND state != 'nacked'
75487
+ ORDER BY id DESC
75488
+ LIMIT 1
75489
+ `;
75490
+ var CLAIM_NEXT = `
75491
+ UPDATE inbound_hl7_messages
75492
+ SET state = 'claimed',
75493
+ processing_started_at = ?,
75494
+ attempt_count = attempt_count + 1
75495
+ WHERE id = (
75496
+ SELECT id FROM inbound_hl7_messages
75497
+ WHERE channel_name = ? AND state = 'queued'
75498
+ ORDER BY id ASC
75499
+ LIMIT 1
75500
+ )
75501
+ RETURNING *
75502
+ `;
75503
+ var MARK_SENT = `
75504
+ UPDATE inbound_hl7_messages
75505
+ SET state = 'inflight',
75506
+ sent_at = ?
75507
+ WHERE callback_id = ? AND state = 'claimed'
75508
+ `;
75509
+ var FIND_BY_CALLBACK = `
75510
+ SELECT * FROM inbound_hl7_messages WHERE callback_id = ?
75511
+ `;
75512
+ var FIND_BY_ID = `
75513
+ SELECT * FROM inbound_hl7_messages WHERE id = ?
75514
+ `;
75515
+ var RECORD_SERVER_RESPONSE = `
75516
+ UPDATE inbound_hl7_messages
75517
+ SET server_status_code = ?,
75518
+ server_response_body = ?
75519
+ WHERE id = ?
75520
+ `;
75521
+ var MARK_PROCESSED = `
75522
+ UPDATE inbound_hl7_messages
75523
+ SET state = 'processed',
75524
+ ack_outcome = ?,
75525
+ processed_at = ?
75526
+ WHERE id = ?
75527
+ `;
75528
+ var MARK_BOT_FAILED = `
75529
+ UPDATE inbound_hl7_messages
75530
+ SET state = ?,
75531
+ errored_at = ?,
75532
+ last_error = ?,
75533
+ error_code = ?,
75534
+ ack_outcome = 'not_owed'
75535
+ WHERE id = ?
75536
+ `;
75537
+ var SET_ACK_OUTCOME = `
75538
+ UPDATE inbound_hl7_messages
75539
+ SET ack_outcome = ?
75540
+ WHERE id = ?
75541
+ `;
75542
+ var REQUEUE = `
75543
+ UPDATE inbound_hl7_messages
75544
+ SET state = 'queued',
75545
+ processing_started_at = NULL,
75546
+ attempt_count = MAX(0, attempt_count - 1)
75547
+ WHERE id = ? AND state = 'claimed'
75548
+ `;
75549
+ var LIST_QUEUED_IDS_FOR_CHANNEL = `
75550
+ SELECT id FROM inbound_hl7_messages
75551
+ WHERE channel_name = ? AND state = 'queued'
75552
+ ORDER BY id ASC
75553
+ `;
75554
+ var RECOVER_INFLIGHT = `
75555
+ UPDATE inbound_hl7_messages
75556
+ SET state = 'failed',
75557
+ errored_at = ?,
75558
+ last_error = COALESCE(last_error, 'interrupted: process restart while inflight'),
75559
+ error_code = COALESCE(error_code, '${QueueErrorCode.Interrupted}')
75560
+ WHERE state = 'inflight'
75561
+ `;
75562
+ var RECOVER_CLAIMED = `
75563
+ UPDATE inbound_hl7_messages
75564
+ SET state = 'queued',
75565
+ processing_started_at = NULL,
75566
+ attempt_count = MAX(0, attempt_count - 1)
75567
+ WHERE state = 'claimed'
75568
+ `;
75569
+ var COUNT_BY_STATE = `
75570
+ SELECT state, COUNT(*) AS n FROM inbound_hl7_messages GROUP BY state
75571
+ `;
75572
+ var CHANNEL_DEPTH = `
75573
+ SELECT
75574
+ SUM(state = 'queued') AS queued,
75575
+ SUM(state = 'claimed') AS claimed,
75576
+ SUM(state = 'inflight') AS inflight,
75577
+ MIN(CASE WHEN state = 'queued' THEN received_at ELSE NULL END) AS oldest_queued_at
75578
+ FROM inbound_hl7_messages
75579
+ WHERE channel_name = ?
75580
+ `;
75581
+ var DB_SIZE_BYTES = `
75582
+ SELECT page_count * page_size AS bytes FROM pragma_page_count, pragma_page_size
75583
+ `;
75584
+ var TRY_ACQUIRE_LEASE = `
75585
+ INSERT INTO _lease (id, holder, acquired_at, expires_at)
75586
+ VALUES (1, ?, ?, ?)
75587
+ ON CONFLICT(id) DO UPDATE
75588
+ SET holder = excluded.holder,
75589
+ acquired_at = excluded.acquired_at,
75590
+ expires_at = excluded.expires_at
75591
+ WHERE _lease.holder = ? OR _lease.expires_at <= ?
75592
+ `;
75593
+ var HEARTBEAT_LEASE = `
75594
+ UPDATE _lease SET expires_at = ? WHERE id = 1 AND holder = ?
75595
+ `;
75596
+ var RELEASE_LEASE = `
75597
+ DELETE FROM _lease WHERE id = 1 AND holder = ?
75598
+ `;
75599
+ var GET_LEASE = `
75600
+ SELECT holder, expires_at FROM _lease WHERE id = 1
75601
+ `;
75602
+ var CHECKPOINT_WAL = `PRAGMA wal_checkpoint(TRUNCATE)`;
75603
+ var RETENTION_PHASE1_DELETE = `
75604
+ DELETE FROM inbound_hl7_messages
75605
+ WHERE state = 'processed' AND ack_outcome != 'undelivered' AND processed_at < ?
75606
+ `;
75607
+ var RETENTION_PHASE2_DELETE = `
75608
+ DELETE FROM inbound_hl7_messages
75609
+ WHERE id IN (
75610
+ SELECT id FROM inbound_hl7_messages
75611
+ WHERE state = 'processed' AND ack_outcome != 'undelivered'
75612
+ ORDER BY processed_at ASC
75613
+ LIMIT ?
75614
+ )
75615
+ `;
75616
+ var RETENTION_PHASE3_DELETE = `
75617
+ DELETE FROM inbound_hl7_messages
75618
+ WHERE id IN (
75619
+ SELECT id FROM inbound_hl7_messages
75620
+ WHERE (
75621
+ state IN ('rejected', 'failed', 'nacked')
75622
+ OR (state = 'processed' AND ack_outcome = 'undelivered')
75623
+ )
75624
+ AND COALESCE(errored_at, processed_at) < ?
75625
+ ORDER BY COALESCE(errored_at, processed_at) ASC
75626
+ LIMIT ?
75627
+ )
75628
+ `;
75629
+
75630
+ // src/queue/schema.ts
75631
+ var MIGRATIONS = [
75632
+ {
75633
+ version: 1,
75634
+ sql: `
75635
+ CREATE TABLE IF NOT EXISTS _schema (
75636
+ version INTEGER PRIMARY KEY,
75637
+ applied_at INTEGER NOT NULL
75638
+ );
75639
+
75640
+ CREATE TABLE IF NOT EXISTS inbound_hl7_messages (
75641
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
75642
+ channel_name TEXT NOT NULL,
75643
+ remote TEXT NOT NULL,
75644
+ msg_control_id TEXT,
75645
+ msg_type TEXT,
75646
+ -- original_message is the message exactly as received, used for the
75647
+ -- intake duplicate-content comparison. finalized_message is what the
75648
+ -- worker dispatches upstream; it differs from original only when the
75649
+ -- channel rewrites the message (e.g. assignSeqNo sets MSH.13). When no
75650
+ -- transformation applies the two are byte-identical.
75651
+ original_message BLOB NOT NULL,
75652
+ finalized_message BLOB NOT NULL,
75653
+ encoding TEXT,
75654
+ enhanced_mode TEXT,
75655
+ -- state tracks the Bot leg: queued \u2192 claimed (worker owns it, request not
75656
+ -- yet on the wire) \u2192 inflight (request written to the socket, sent_at
75657
+ -- stamped, awaiting response) \u2192 processed/rejected/failed, plus the
75658
+ -- intake-reject 'nacked'. The claimed/inflight split is what lets crash
75659
+ -- recovery tell a provably-unsent row (safe to requeue) from an ambiguous
75660
+ -- in-flight one (must fail for review). The source-leg ACK-delivery outcome
75661
+ -- is tracked independently in ack_outcome
75662
+ -- (pending/delivered/undelivered/not_owed) so the two legs never conflate.
75663
+ state TEXT NOT NULL,
75664
+ attempt_count INTEGER NOT NULL DEFAULT 0,
75665
+ callback_id TEXT NOT NULL,
75666
+ server_response_body BLOB,
75667
+ server_status_code INTEGER,
75668
+ ack_outcome TEXT NOT NULL DEFAULT 'pending',
75669
+ last_error TEXT,
75670
+ error_code TEXT,
75671
+ seq_no INTEGER,
75672
+ received_at INTEGER NOT NULL,
75673
+ -- processing_started_at is stamped when the worker claims the row (enters
75674
+ -- the claimed state); sent_at is stamped when the request is written to the
75675
+ -- socket (enters inflight). A NULL sent_at on a claimed row is the durable
75676
+ -- discriminator crash recovery uses to know the request never left.
75677
+ processing_started_at INTEGER,
75678
+ sent_at INTEGER,
75679
+ processed_at INTEGER,
75680
+ errored_at INTEGER
75681
+ ) STRICT;
75682
+
75683
+ CREATE INDEX IF NOT EXISTS idx_inbound_channel_state_id
75684
+ ON inbound_hl7_messages (channel_name, state, id);
75685
+
75686
+ -- Also serves the retention phase-3 sweep: its (state IN ... OR ...)
75687
+ -- predicate resolves to a MULTI-INDEX OR over this index's leading state
75688
+ -- column, so the sweep filters by index rather than full-scanning. A
75689
+ -- dedicated COALESCE(errored_at, processed_at) expression index was
75690
+ -- evaluated to also satisfy phase 3's ORDER BY, but SQLite's partial-index
75691
+ -- prover can't match that disjunctive predicate (so a partial index goes
75692
+ -- unused), and a non-partial one would key every queued row as NULL and tax
75693
+ -- the hot intake path -- not worth it for an hourly, size-gated sweep.
75694
+ CREATE INDEX IF NOT EXISTS idx_inbound_state_processed_at
75695
+ ON inbound_hl7_messages (state, processed_at);
75696
+
75697
+ CREATE UNIQUE INDEX IF NOT EXISTS uq_inbound_dup_active
75698
+ ON inbound_hl7_messages (channel_name, msg_control_id)
75699
+ WHERE msg_control_id IS NOT NULL
75700
+ AND state IN ('queued', 'claimed', 'inflight');
75701
+
75702
+ -- Intake dedup reads the most recent prior row for a (channel, control_id)
75703
+ -- across ALL non-nacked states (uq_inbound_dup_active only covers the
75704
+ -- active window). This runs once per inbound message that carries an
75705
+ -- MSH.10, so without it the lookup scans every row for the channel. The
75706
+ -- trailing id makes ORDER BY id DESC LIMIT 1 a reverse index seek; the
75707
+ -- partial predicate matches the query so there's no residual recheck.
75708
+ CREATE INDEX IF NOT EXISTS idx_inbound_dup_lookup
75709
+ ON inbound_hl7_messages (channel_name, msg_control_id, id)
75710
+ WHERE msg_control_id IS NOT NULL
75711
+ AND state != 'nacked';
75712
+
75713
+ CREATE UNIQUE INDEX IF NOT EXISTS uq_inbound_callback
75714
+ ON inbound_hl7_messages (callback_id);
75715
+
75716
+ -- Single-row table holding the current queue lease, used for coordinating
75717
+ -- the active queue worker across zero-downtime agent upgrades (two
75718
+ -- processes share the DB for the upgrade overlap window; only the
75719
+ -- leaseholder drains rows and runs recovery). The CHECK constraint keeps
75720
+ -- it strictly single-row.
75721
+ CREATE TABLE IF NOT EXISTS _lease (
75722
+ id INTEGER PRIMARY KEY CHECK (id = 1),
75723
+ holder TEXT NOT NULL,
75724
+ acquired_at INTEGER NOT NULL,
75725
+ expires_at INTEGER NOT NULL
75726
+ ) STRICT;
75727
+
75728
+ -- Per-channel monotonic sequence counter for the assignSeqNo feature.
75729
+ -- Persisting it here (rather than only in memory) keeps MSH.13 sequence
75730
+ -- numbers monotonic across agent restarts. last_seq_no holds the most
75731
+ -- recently assigned value; the next assignment is last_seq_no + 1.
75732
+ CREATE TABLE IF NOT EXISTS _channel_seq (
75733
+ channel_name TEXT PRIMARY KEY,
75734
+ last_seq_no INTEGER NOT NULL
75735
+ ) STRICT;
75736
+ `
75737
+ }
75738
+ ];
75739
+ function runMigrations(db) {
75740
+ db.exec(`
75741
+ CREATE TABLE IF NOT EXISTS _schema (
75742
+ version INTEGER PRIMARY KEY,
75743
+ applied_at INTEGER NOT NULL
75744
+ );
75745
+ `);
75746
+ const currentRow = db.prepare("SELECT MAX(version) AS v FROM _schema").get();
75747
+ let current = currentRow?.v ?? 0;
75748
+ const insertVersion = db.prepare("INSERT INTO _schema (version, applied_at) VALUES (?, ?)");
75749
+ for (const migration of MIGRATIONS) {
75750
+ if (migration.version <= current) {
75751
+ continue;
75752
+ }
75753
+ db.exec("BEGIN");
75754
+ try {
75755
+ db.exec(migration.sql);
75756
+ insertVersion.run(migration.version, Date.now());
75757
+ db.exec("COMMIT");
75758
+ current = migration.version;
75759
+ } catch (err2) {
75760
+ db.exec("ROLLBACK");
75761
+ throw err2;
75762
+ }
75763
+ }
75764
+ return current;
75765
+ }
75766
+
75767
+ // src/queue/durable-queue.ts
75768
+ var DEFAULT_LEASE_TTL_MS = 3e4;
75769
+ var DEFAULT_LEASE_HEARTBEAT_MS = 1e4;
75770
+ var DEFAULT_LEASE_ACQUIRE_RETRY_MS = 2e3;
75771
+ var DEFAULT_CHECKPOINT_INTERVAL_MS = 1e4;
75772
+ var DurableQueue = class _DurableQueue {
75773
+ constructor(db, options) {
75774
+ __publicField(this, "db");
75775
+ __publicField(this, "log");
75776
+ __publicField(this, "path");
75777
+ __publicField(this, "closed", false);
75778
+ // The dispatch-lease holder ID of THIS process, set when startDispatchLease()
75779
+ // begins the acquire/heartbeat loop (or bound directly via setLeaseHolder in
75780
+ // tests). Dispatch-class mutations are gated on the live lease still belonging
75781
+ // to this holder; when a peer takes over, the gate throws QueueLeaseError so the
75782
+ // demoted worker stops driving the queue. Undefined when the dispatch lease was
75783
+ // never started (single-process use and most unit tests), in which case the gate
75784
+ // is inert because no lease row exists.
75785
+ __publicField(this, "leaseHolderId");
75786
+ // ── Dispatch-lease orchestration (formerly DispatchLeaseManager) ──
75787
+ // Lightweight leader election for the DISPATCH path. The lease gates *dispatch
75788
+ // only* — claiming rows and driving them to the server (the worker +
75789
+ // recoverOnStartup). It deliberately does NOT gate intake (followers still
75790
+ // persist inbound messages via enqueue), maintenance (WAL checkpoint, retention
75791
+ // sweep), or diagnostics (stats): those are valid on any process with the queue
75792
+ // file open. Exactly one process may dispatch at a time, which is the
75793
+ // load-bearing primitive that makes zero-downtime upgrades safe: during the
75794
+ // overlap window where the old and new agent processes both have the SQLite file
75795
+ // open, only the leaseholder claims/sends rows; the non-leader keeps accepting
75796
+ // and persisting inbound traffic but does not dispatch.
75797
+ //
75798
+ // There is deliberately NO lost-leadership callback: loss is enforced at the
75799
+ // data layer instead — the dispatch ops throw QueueLeaseError once a peer holds
75800
+ // the lease (see assertNotDemoted), so the worker self-detects and drains
75801
+ // without pushing an event down a chain of callbacks.
75802
+ __publicField(this, "leaseTtlMs");
75803
+ __publicField(this, "leaseHeartbeatMs");
75804
+ __publicField(this, "leaseAcquireRetryMs");
75805
+ __publicField(this, "leaseLeader", false);
75806
+ __publicField(this, "leaseLoopActive", false);
75807
+ __publicField(this, "leaseAcquireTimer");
75808
+ __publicField(this, "leaseHeartbeatTimer");
75809
+ __publicField(this, "onBecameLeader");
75810
+ // True when the WAL may contain frames not yet checkpointed into the main DB
75811
+ // file. SQLite only attempts checkpoints piggybacked on commits, so once
75812
+ // traffic stops nothing would ever drain the WAL — the queue's own checkpoint
75813
+ // loop (startCheckpointLoop) polls this via checkpointWalIfDirty(). Starts true
75814
+ // because open() itself writes (pragmas, migrations, lease).
75815
+ __publicField(this, "walDirty", true);
75816
+ // WAL-checkpoint loop. Runs while the queue is open, INDEPENDENT of the
75817
+ // dispatch lease: a follower still accepts intake writes (enqueue runs on any
75818
+ // process with the file open), so WAL draining can't be gated on leadership the
75819
+ // way the lease timers are. Started in open(), cleared in close().
75820
+ __publicField(this, "checkpointIntervalMs");
75821
+ __publicField(this, "checkpointTimer");
75822
+ // Prepared statements — created once at open(), reused for every call.
75823
+ // Names mirror the public methods that use them.
75824
+ __publicField(this, "enqueueStmt");
75825
+ __publicField(this, "enqueueRejectedStmt");
75826
+ __publicField(this, "peekLastSeqNoStmt");
75827
+ __publicField(this, "commitSeqNoStmt");
75828
+ __publicField(this, "findSeenByControlIdStmt");
75829
+ __publicField(this, "claimNextStmt");
75830
+ __publicField(this, "markSentStmt");
75831
+ __publicField(this, "findByCallbackStmt");
75832
+ __publicField(this, "findByIdStmt");
75833
+ __publicField(this, "recordServerResponseStmt");
75834
+ __publicField(this, "markProcessedStmt");
75835
+ __publicField(this, "markBotFailedStmt");
75836
+ __publicField(this, "setAckOutcomeStmt");
75837
+ __publicField(this, "requeueStmt");
75838
+ __publicField(this, "recoverInflightStmt");
75839
+ __publicField(this, "recoverClaimedStmt");
75840
+ __publicField(this, "listQueuedIdsForChannelStmt");
75841
+ __publicField(this, "countByStateStmt");
75842
+ __publicField(this, "channelDepthStmt");
75843
+ __publicField(this, "tryAcquireLeaseStmt");
75844
+ __publicField(this, "heartbeatLeaseStmt");
75845
+ __publicField(this, "releaseLeaseStmt");
75846
+ __publicField(this, "getLeaseStmt");
75847
+ __publicField(this, "checkpointStmt");
75848
+ __publicField(this, "dbSizeBytesStmt");
75849
+ this.db = db;
75850
+ this.log = options.log;
75851
+ this.path = options.path;
75852
+ this.leaseHolderId = options.leaseHolder;
75853
+ this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
75854
+ this.leaseHeartbeatMs = options.leaseHeartbeatMs ?? DEFAULT_LEASE_HEARTBEAT_MS;
75855
+ this.leaseAcquireRetryMs = options.leaseAcquireRetryMs ?? DEFAULT_LEASE_ACQUIRE_RETRY_MS;
75856
+ this.checkpointIntervalMs = options.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
75857
+ this.db.exec(`
75858
+ PRAGMA journal_mode = WAL;
75859
+ PRAGMA synchronous = NORMAL;
75860
+ PRAGMA temp_store = MEMORY;
75861
+ PRAGMA cache_size = -65536;
75862
+ PRAGMA mmap_size = 268435456;
75863
+ PRAGMA wal_autocheckpoint = 1000;
75864
+ PRAGMA busy_timeout = 5000;
75865
+ PRAGMA foreign_keys = ON;
75866
+ `);
75867
+ runMigrations(this.db);
75868
+ this.enqueueStmt = this.db.prepare(ENQUEUE);
75869
+ this.enqueueRejectedStmt = this.db.prepare(ENQUEUE_REJECTED);
75870
+ this.peekLastSeqNoStmt = this.db.prepare(PEEK_LAST_SEQ_NO);
75871
+ this.commitSeqNoStmt = this.db.prepare(COMMIT_SEQ_NO);
75872
+ this.findSeenByControlIdStmt = this.db.prepare(FIND_SEEN_BY_CONTROL_ID);
75873
+ this.claimNextStmt = this.db.prepare(CLAIM_NEXT);
75874
+ this.markSentStmt = this.db.prepare(MARK_SENT);
75875
+ this.findByCallbackStmt = this.db.prepare(FIND_BY_CALLBACK);
75876
+ this.findByIdStmt = this.db.prepare(FIND_BY_ID);
75877
+ this.recordServerResponseStmt = this.db.prepare(RECORD_SERVER_RESPONSE);
75878
+ this.markProcessedStmt = this.db.prepare(MARK_PROCESSED);
75879
+ this.markBotFailedStmt = this.db.prepare(MARK_BOT_FAILED);
75880
+ this.setAckOutcomeStmt = this.db.prepare(SET_ACK_OUTCOME);
75881
+ this.requeueStmt = this.db.prepare(REQUEUE);
75882
+ this.recoverInflightStmt = this.db.prepare(RECOVER_INFLIGHT);
75883
+ this.recoverClaimedStmt = this.db.prepare(RECOVER_CLAIMED);
75884
+ this.listQueuedIdsForChannelStmt = this.db.prepare(LIST_QUEUED_IDS_FOR_CHANNEL);
75885
+ this.countByStateStmt = this.db.prepare(COUNT_BY_STATE);
75886
+ this.channelDepthStmt = this.db.prepare(CHANNEL_DEPTH);
75887
+ this.tryAcquireLeaseStmt = this.db.prepare(TRY_ACQUIRE_LEASE);
75888
+ this.heartbeatLeaseStmt = this.db.prepare(HEARTBEAT_LEASE);
75889
+ this.releaseLeaseStmt = this.db.prepare(RELEASE_LEASE);
75890
+ this.getLeaseStmt = this.db.prepare(GET_LEASE);
75891
+ this.checkpointStmt = this.db.prepare(CHECKPOINT_WAL);
75892
+ this.dbSizeBytesStmt = this.db.prepare(DB_SIZE_BYTES);
75893
+ }
75894
+ /**
75895
+ * Opens (or creates) the DB file at `options.path`, runs migrations, and
75896
+ * returns a ready DurableQueue.
75897
+ *
75898
+ * Constructs synchronously — `node:sqlite` is itself synchronous so there's
75899
+ * no benefit to making this `async`.
75900
+ *
75901
+ * @param options - Path + logger.
75902
+ * @returns The opened DurableQueue.
75903
+ */
75904
+ static open(options) {
75905
+ const { DatabaseSync } = require("node:sqlite");
75906
+ const db = new DatabaseSync(options.path);
75907
+ const queue = new _DurableQueue(db, options);
75908
+ if ((0, import_node_fs3.existsSync)(options.path)) {
75909
+ try {
75910
+ (0, import_node_fs3.chmodSync)(options.path, 384);
75911
+ } catch {
75912
+ }
75913
+ }
75914
+ queue.startCheckpointLoop();
75915
+ return queue;
75916
+ }
75917
+ /** Closes the underlying SQLite handle. Idempotent. */
75918
+ close() {
75919
+ if (this.closed) {
75920
+ return;
75921
+ }
75922
+ this.closed = true;
75923
+ this.stopCheckpointLoop();
75924
+ this.stopDispatchLease();
75925
+ this.checkpointWal();
75926
+ try {
75927
+ this.db.close();
75928
+ } catch (err2) {
75929
+ this.log.warn(`Error while closing durable queue DB: ${Re(err2)}`);
75930
+ }
75931
+ }
75932
+ /**
75933
+ * Unconditionally runs a TRUNCATE checkpoint, folding the WAL into the main
75934
+ * DB file and truncating the WAL to zero bytes. Best-effort: failures are
75935
+ * logged, and an incomplete checkpoint (a peer connection pinning part of the
75936
+ * WAL) leaves the dirty flag set so the next attempt retries.
75937
+ * @returns True when the checkpoint fully completed.
75938
+ */
75939
+ checkpointWal() {
75940
+ try {
75941
+ const row = this.checkpointStmt.get();
75942
+ if (row?.busy) {
75943
+ return false;
75944
+ }
75945
+ this.walDirty = false;
75946
+ return true;
75947
+ } catch (err2) {
75948
+ this.log.warn(`wal_checkpoint failed: ${Re(err2)}`);
75949
+ return false;
75950
+ }
75951
+ }
75952
+ /**
75953
+ * Runs {@link DurableQueue.checkpointWal} only when writes have landed since
75954
+ * the last completed checkpoint. Driven by the queue's own checkpoint loop
75955
+ * (see {@link DurableQueue.startCheckpointLoop}) every `checkpointIntervalMs`,
75956
+ * so the WAL drains shortly after traffic stops instead of waiting for the next
75957
+ * retention sweep or for close(). A no-op on an idle queue.
75958
+ * @returns True when a checkpoint ran and fully completed.
75959
+ */
75960
+ checkpointWalIfDirty() {
75961
+ if (this.closed || !this.walDirty) {
75962
+ return false;
75963
+ }
75964
+ return this.checkpointWal();
75965
+ }
75966
+ /**
75967
+ * Starts the periodic WAL-checkpoint loop. Idempotent. Called once by
75968
+ * {@link DurableQueue.open}; runs regardless of dispatch leadership (see the
75969
+ * `checkpointTimer` field) and is torn down by {@link DurableQueue.close}.
75970
+ */
75971
+ startCheckpointLoop() {
75972
+ if (this.checkpointTimer) {
75973
+ return;
75974
+ }
75975
+ this.checkpointTimer = setInterval(() => this.checkpointWalIfDirty(), this.checkpointIntervalMs);
75976
+ if (typeof this.checkpointTimer.unref === "function") {
75977
+ this.checkpointTimer.unref();
75978
+ }
75979
+ }
75980
+ /** Stops the WAL-checkpoint loop. Idempotent and a no-op when never started. */
75981
+ stopCheckpointLoop() {
75982
+ if (this.checkpointTimer) {
75983
+ clearInterval(this.checkpointTimer);
75984
+ this.checkpointTimer = void 0;
75985
+ }
75986
+ }
75987
+ /** @returns Filesystem path of the underlying SQLite file. */
75988
+ getPath() {
75989
+ return this.path;
75990
+ }
75991
+ /** @returns Direct access to the underlying handle. Used by retention sweeper for PRAGMA queries. */
75992
+ getDb() {
75993
+ return this.db;
75994
+ }
75995
+ /**
75996
+ * Runs `fn` inside a single SQLite transaction, committing if it returns and
75997
+ * rolling back if it throws. Used to make a row state transition atomic with an
75998
+ * external side effect — e.g. flipping a row `claimed` → `inflight` only if the
75999
+ * socket write that puts it on the wire also succeeds (see `App.sendToWebSocket`):
76000
+ * if the write throws, the marker rolls back and the row stays `claimed`
76001
+ * (provably unsent, safe to requeue) rather than a phantom `inflight`.
76002
+ *
76003
+ * `node:sqlite` is synchronous, so `fn` must be synchronous too — do any `await`
76004
+ * (e.g. token refresh) before calling, not inside.
76005
+ * @param fn - The work to run transactionally.
76006
+ * @returns Whatever `fn` returns.
76007
+ */
76008
+ runInTransaction(fn2) {
76009
+ this.db.exec("BEGIN");
76010
+ try {
76011
+ const result = fn2();
76012
+ this.db.exec("COMMIT");
76013
+ return result;
76014
+ } catch (err2) {
76015
+ this.db.exec("ROLLBACK");
76016
+ throw err2;
76017
+ }
76018
+ }
76019
+ /**
76020
+ * Inserts a new `queued` row.
76021
+ *
76022
+ * If any prior non-`nacked` row already owns this `(channel_name,
76023
+ * msg_control_id)` — in `queued`, `claimed`, `inflight`, `processed`,
76024
+ * `rejected`, or `failed` — we don't insert; we surface a `duplicate` result
76025
+ * carrying that row so the caller can compare bodies and decide between
76026
+ * replaying the prior ACK and rejecting the collision (§8). This SELECT is the
76027
+ * single dedup authority on the intake path: the partial unique index only
76028
+ * covers the live `queued`/`claimed`/`inflight` window, so it can't recognize a
76029
+ * retransmit of an already-`processed`/`rejected`/`failed` row —
76030
+ * it remains only as a last-resort guard against an active-window race (not
76031
+ * expected in the single-process agent).
76032
+ * @param input - Fields to persist on the new row.
76033
+ * @param options - Optional behavior.
76034
+ * @param options.assignSeqNo - When provided, enqueue assigns the channel's next
76035
+ * sequence number — but only after the duplicate check passes, so a retransmit
76036
+ * never consumes one. It peeks the persisted counter (a non-consuming read),
76037
+ * calls this callback with the candidate so the caller can stamp it into MSH.13
76038
+ * and return the finalized bytes to persist, then advances the counter in the
76039
+ * SAME transaction as the insert (so a crash can't store the row while leaving
76040
+ * the counter behind, which would reuse the number on restart). On a duplicate
76041
+ * the callback is never invoked.
76042
+ * @returns Either the inserted row, or the prior row that owns the MSH.10.
76043
+ */
76044
+ enqueue(input, options) {
76045
+ if (input.msgControlId) {
76046
+ const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
76047
+ if (existing) {
76048
+ return { kind: "duplicate", existing };
76049
+ }
76050
+ }
76051
+ let finalizedMessage = input.finalizedMessage;
76052
+ let seqNo = input.seqNo;
76053
+ let seqNoToCommit;
76054
+ if (options?.assignSeqNo) {
76055
+ const candidate = this.peekNextSeqNo(input.channelName);
76056
+ finalizedMessage = options.assignSeqNo(candidate);
76057
+ seqNo = candidate;
76058
+ seqNoToCommit = candidate;
76059
+ }
76060
+ try {
76061
+ const runInsert = () => {
76062
+ const info = this.enqueueStmt.run(
76063
+ input.channelName,
76064
+ input.remote,
76065
+ input.msgControlId,
76066
+ input.msgType,
76067
+ toBlob(input.originalMessage),
76068
+ toBlob(finalizedMessage),
76069
+ input.encoding,
76070
+ input.enhancedMode,
76071
+ input.callbackId,
76072
+ seqNo,
76073
+ input.receivedAt
76074
+ );
76075
+ return Number(info.lastInsertRowid);
76076
+ };
76077
+ let id;
76078
+ if (seqNoToCommit !== void 0) {
76079
+ this.db.exec("BEGIN");
76080
+ try {
76081
+ id = runInsert();
76082
+ this.commitSeqNo(input.channelName, seqNoToCommit);
76083
+ this.db.exec("COMMIT");
76084
+ } catch (txErr) {
76085
+ this.db.exec("ROLLBACK");
76086
+ throw txErr;
76087
+ }
76088
+ } else {
76089
+ id = runInsert();
76090
+ }
76091
+ this.walDirty = true;
76092
+ const row = this.getById(id);
76093
+ if (!row) {
76094
+ throw new Error(`enqueue: inserted row id=${id} could not be re-read`);
76095
+ }
76096
+ return { kind: "inserted", row };
76097
+ } catch (err2) {
76098
+ if (isUniqueConstraintError(err2) && input.msgControlId) {
76099
+ const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
76100
+ if (existing) {
76101
+ return { kind: "duplicate", existing };
76102
+ }
76103
+ }
76104
+ throw err2;
76105
+ }
76106
+ }
76107
+ /**
76108
+ * Inserts an audit row in the `nacked` terminal state — used when intake
76109
+ * rejected the message (duplicate, malformed) but we still want a forensics
76110
+ * record. Failure to write this is non-fatal; the caller is expected to log.
76111
+ * @param input - Fields to persist plus the `lastError`/`errorCode` describing why we rejected.
76112
+ * @returns The newly inserted audit row, or null if even the audit insert failed.
76113
+ */
76114
+ enqueueRejected(input) {
76115
+ try {
76116
+ const info = this.enqueueRejectedStmt.run(
76117
+ input.channelName,
76118
+ input.remote,
76119
+ input.msgControlId,
76120
+ input.msgType,
76121
+ toBlob(input.originalMessage),
76122
+ toBlob(input.finalizedMessage),
76123
+ input.encoding,
76124
+ input.enhancedMode,
76125
+ input.callbackId,
76126
+ input.lastError,
76127
+ input.errorCode,
76128
+ input.seqNo,
76129
+ input.receivedAt
76130
+ );
76131
+ this.walDirty = true;
76132
+ return this.getById(Number(info.lastInsertRowid));
76133
+ } catch (err2) {
76134
+ this.log.warn(`enqueueRejected failed: ${Re(err2)}`);
76135
+ return null;
76136
+ }
76137
+ }
76138
+ /**
76139
+ * Atomically claims the next `queued` row for `channelName`, flipping it to
76140
+ * `claimed` and bumping `attempt_count`. The row stays `claimed` until
76141
+ * {@link markSent} flips it to `inflight` once the request hits the socket.
76142
+ * @param channelName - The channel to claim from.
76143
+ * @param now - Override the timestamp written to `processing_started_at` (for tests).
76144
+ * @returns The claimed row, or `null` if the channel queue is empty.
76145
+ */
76146
+ claimNext(channelName, now = Date.now()) {
76147
+ this.assertNotDemoted();
76148
+ const raw = this.claimNextStmt.get(now, channelName);
76149
+ if (raw) {
76150
+ this.walDirty = true;
76151
+ }
76152
+ return raw ? rowFromSql(raw) : null;
76153
+ }
76154
+ /**
76155
+ * Phase A → B transition: records that the transmit request for `callbackId`
76156
+ * was written to the WebSocket, flipping the row from `claimed` to `inflight`
76157
+ * and stamping `sent_at`. Called from the App's send path the instant the
76158
+ * request leaves the process — the durable marker that distinguishes a
76159
+ * provably-unsent row (safe to requeue on crash) from an ambiguous in-flight
76160
+ * one (failed for review). Guarded on `state = 'claimed'`, so it's a no-op for
76161
+ * legacy (non-durable) sends and for any row already past `claimed`.
76162
+ * @param callbackId - The callback ID of the transmit request just sent.
76163
+ * @param now - Override for `sent_at` (for tests).
76164
+ * @returns True if a `claimed` row was flipped to `inflight`.
76165
+ */
76166
+ markSent(callbackId, now = Date.now()) {
76167
+ const info = this.markSentStmt.run(now, callbackId);
76168
+ if (Number(info.changes) > 0) {
76169
+ this.walDirty = true;
76170
+ return true;
76171
+ }
76172
+ return false;
76173
+ }
76174
+ /**
76175
+ * Looks up a row by its server-callback ID. Returns `null` if not found.
76176
+ * @param callbackId - The callback ID echoed back by the Medplum server.
76177
+ * @returns The matching row, or null.
76178
+ */
76179
+ findByCallback(callbackId) {
76180
+ const raw = this.findByCallbackStmt.get(callbackId);
76181
+ return raw ? rowFromSql(raw) : null;
76182
+ }
76183
+ /**
76184
+ * Looks up a row by its primary key.
76185
+ * @param id - The row's primary key.
76186
+ * @returns The matching row, or null.
76187
+ */
76188
+ getById(id) {
76189
+ const raw = this.findByIdStmt.get(id);
76190
+ return raw ? rowFromSql(raw) : null;
76191
+ }
76192
+ /**
76193
+ * Records the server's `agent:transmit:response` body + status against the row.
76194
+ * @param id - Row primary key.
76195
+ * @param statusCode - HTTP-style status code returned by the server.
76196
+ * @param body - Response payload, or null when the server omitted one.
76197
+ */
76198
+ recordServerResponse(id, statusCode, body) {
76199
+ this.assertNotDemoted();
76200
+ this.recordServerResponseStmt.run(statusCode, body === null ? null : toBlob(body), id);
76201
+ this.walDirty = true;
76202
+ }
76203
+ /**
76204
+ * Terminal Bot-leg transition: the server accepted the message (2xx). The
76205
+ * source leg is recorded independently via `ackOutcome` — `delivered` when the
76206
+ * app-level ACK reached the source, `undelivered` when it couldn't (e.g. the
76207
+ * source connection had closed). An `undelivered` row is fully processed
76208
+ * upstream and must never be re-dispatched; it recovers when the source
76209
+ * retransmits and the stored ACK is replayed (see {@link setAckOutcome}).
76210
+ * @param id - Row primary key.
76211
+ * @param ackOutcome - Source-leg result: `delivered` or `undelivered`.
76212
+ * @param now - Override for `processed_at` (for tests).
76213
+ */
76214
+ markProcessed(id, ackOutcome, now = Date.now()) {
76215
+ this.assertNotDemoted();
76216
+ this.markProcessedStmt.run(ackOutcome, now, id);
76217
+ this.walDirty = true;
76218
+ }
76219
+ /**
76220
+ * Terminal Bot-leg transition: the server **rejected** the message itself
76221
+ * (permanent 4xx). Retrying can never help — the content must be triaged.
76222
+ * @param id - Row primary key.
76223
+ * @param error - Human-readable error string, written to `last_error`.
76224
+ * @param errorCode - Machine-readable classification, written to `error_code`.
76225
+ * @param now - Override for `errored_at` (for tests).
76226
+ */
76227
+ markRejected(id, error, errorCode, now = Date.now()) {
76228
+ this.assertNotDemoted();
76229
+ this.markBotFailedStmt.run(MessageState.REJECTED, now, error, errorCode, id);
76230
+ this.walDirty = true;
76231
+ }
76232
+ /**
76233
+ * Terminal-for-now Bot-leg transition: a **transient/ambiguous** failure
76234
+ * (5xx, 429, response timeout, dispatch error, interrupted). The retry/
76235
+ * operator-review candidate — distinct from a `rejected` message so a future
76236
+ * retry policy can re-dispatch `failed` rows without ever touching `rejected`
76237
+ * ones (or `processed` + `undelivered` ones, whose Bot leg already succeeded).
76238
+ * @param id - Row primary key.
76239
+ * @param error - Human-readable error string, written to `last_error`.
76240
+ * @param errorCode - Machine-readable classification, written to `error_code`.
76241
+ * @param now - Override for `errored_at` (for tests).
76242
+ */
76243
+ markFailed(id, error, errorCode, now = Date.now()) {
76244
+ this.assertNotDemoted();
76245
+ this.markBotFailedStmt.run(MessageState.FAILED, now, error, errorCode, id);
76246
+ this.walDirty = true;
76247
+ }
76248
+ /**
76249
+ * Updates only the source-leg {@link AckOutcome}, leaving the Bot-leg `state`
76250
+ * untouched. Used when a duplicate retransmit replays a previously
76251
+ * `undelivered` ACK and it lands — flipping the row to `delivered` so it no
76252
+ * longer reads as awaiting source delivery.
76253
+ * @param id - Row primary key.
76254
+ * @param ackOutcome - The new source-leg outcome.
76255
+ */
76256
+ setAckOutcome(id, ackOutcome) {
76257
+ this.setAckOutcomeStmt.run(ackOutcome, id);
76258
+ this.walDirty = true;
76259
+ }
76260
+ /**
76261
+ * Returns a `claimed` row to `queued` — used when the WS connection drops
76262
+ * before the transmit request was ever written to the socket, so retrying on
76263
+ * reconnect carries no duplicate-delivery risk. (A row that already reached the
76264
+ * socket is `inflight`, not `claimed`, and this is a no-op for it — its outcome
76265
+ * is ambiguous and owned by the response timeout.) Because the row keeps its
76266
+ * original id and claims are ordered by id, it goes back to the front of its
76267
+ * channel's FIFO.
76268
+ * @param id - Row primary key.
76269
+ * @returns True if the row was requeued; false if it was not in `claimed`.
76270
+ */
76271
+ requeue(id) {
76272
+ this.assertNotDemoted();
76273
+ const info = this.requeueStmt.run(id);
76274
+ if (Number(info.changes) > 0) {
76275
+ this.walDirty = true;
76276
+ return true;
76277
+ }
76278
+ return false;
76279
+ }
76280
+ /**
76281
+ * Recovers rows left mid-flight by a previous process, splitting on whether the
76282
+ * request reached the wire. Runs once at startup (§10):
76283
+ * - `inflight` rows are ambiguous (the server may have processed them) → `failed`
76284
+ * with error code `interrupted`, surfaced for operator review, never silently retried.
76285
+ * - `claimed` rows provably never left the process (`sent_at` is NULL) → returned
76286
+ * to `queued` for a clean re-dispatch with no duplicate-delivery risk.
76287
+ * @param now - Override for `errored_at` (for tests).
76288
+ * @returns Counts of `claimed` rows requeued and `inflight` rows failed.
76289
+ */
76290
+ recoverOnStartup(now = Date.now()) {
76291
+ const failedInfo = this.recoverInflightStmt.run(now);
76292
+ const requeuedInfo = this.recoverClaimedStmt.run();
76293
+ const failed = Number(failedInfo.changes);
76294
+ const requeued = Number(requeuedInfo.changes);
76295
+ if (failed > 0 || requeued > 0) {
76296
+ this.walDirty = true;
76297
+ }
76298
+ return { requeued, failed };
76299
+ }
76300
+ /**
76301
+ * Lists all `queued` row IDs for a channel, in FIFO order. Used by the
76302
+ * worker on startup to repopulate its in-memory wake signal.
76303
+ * @param channelName - The channel to query.
76304
+ * @returns Row IDs in FIFO order.
76305
+ */
76306
+ listQueuedIdsForChannel(channelName) {
76307
+ const rows = this.listQueuedIdsForChannelStmt.all(channelName);
76308
+ return rows.map((r6) => r6.id);
76309
+ }
76310
+ /** @returns Counts of rows by state. Missing states are reported as 0. */
76311
+ countByState() {
76312
+ const counts = {
76313
+ queued: 0,
76314
+ claimed: 0,
76315
+ inflight: 0,
76316
+ processed: 0,
76317
+ rejected: 0,
76318
+ failed: 0,
76319
+ nacked: 0
76320
+ };
76321
+ const rows = this.countByStateStmt.all();
76322
+ for (const r6 of rows) {
76323
+ counts[r6.state] = r6.n;
76324
+ }
76325
+ return counts;
76326
+ }
76327
+ /**
76328
+ * @param channelName - The channel to query.
76329
+ * @param now - Override for the "now" timestamp used in `oldestQueuedAgeMs`.
76330
+ * @returns Depth snapshot for `channelName` (queued/claimed/inflight counts + oldest queued age).
76331
+ */
76332
+ getChannelDepth(channelName, now = Date.now()) {
76333
+ const row = this.channelDepthStmt.get(channelName);
76334
+ return {
76335
+ queued: row?.queued ?? 0,
76336
+ claimed: row?.claimed ?? 0,
76337
+ inflight: row?.inflight ?? 0,
76338
+ oldestQueuedAgeMs: row?.oldest_queued_at ? now - row.oldest_queued_at : null
76339
+ };
76340
+ }
76341
+ /**
76342
+ * Attempts to acquire (or re-acquire) the queue lease as `holder`.
76343
+ *
76344
+ * Succeeds when there is no current lease, the current lease is already held
76345
+ * by us (refresh case), or the prior holder's lease has expired. Fails when a
76346
+ * different holder still has a valid lease — caller should wait and retry.
76347
+ *
76348
+ * The lease is the cross-process coordination primitive that makes zero-downtime
76349
+ * upgrades safe: only the holder runs workers and `recoverOnStartup`, so two
76350
+ * processes sharing the DB during the upgrade overlap don't fight over rows.
76351
+ * @param holder - Stable identifier for this process (a per-process UUID).
76352
+ * @param ttlMs - How long the new lease should remain valid before another
76353
+ * process can take over (also drives the heartbeat cadence).
76354
+ * @param now - Override the "now" timestamp (for tests).
76355
+ * @returns True if the lease is now held by `holder`; false if a foreign holder still owns it.
76356
+ */
76357
+ tryAcquireLease(holder, ttlMs, now = Date.now()) {
76358
+ const expiresAt = now + ttlMs;
76359
+ const info = this.tryAcquireLeaseStmt.run(holder, now, expiresAt, holder, now);
76360
+ if (Number(info.changes) > 0) {
76361
+ this.walDirty = true;
76362
+ }
76363
+ return Number(info.changes) > 0;
76364
+ }
76365
+ /**
76366
+ * Extends the existing lease held by `holder`. Fails (returns false) when the
76367
+ * lease is no longer ours — most commonly because a peer took over after our
76368
+ * TTL elapsed. A leader that sees `false` here must stop driving the queue.
76369
+ * @param holder - The same identifier passed to {@link DurableQueue.tryAcquireLease}.
76370
+ * @param ttlMs - New TTL for the lease (added to `now`).
76371
+ * @param now - Override the "now" timestamp (for tests).
76372
+ * @returns True if the heartbeat extended our lease; false if we lost it.
76373
+ */
76374
+ heartbeatLease(holder, ttlMs, now = Date.now()) {
76375
+ const info = this.heartbeatLeaseStmt.run(now + ttlMs, holder);
76376
+ if (Number(info.changes) > 0) {
76377
+ this.walDirty = true;
76378
+ }
76379
+ return Number(info.changes) > 0;
76380
+ }
76381
+ /**
76382
+ * Releases the lease if (and only if) `holder` still owns it. Idempotent. The
76383
+ * holder check means a process that already lost its lease won't accidentally
76384
+ * delete a newer leader's row.
76385
+ * @param holder - The identifier of the lease to release.
76386
+ */
76387
+ releaseLease(holder) {
76388
+ const info = this.releaseLeaseStmt.run(holder);
76389
+ if (Number(info.changes) > 0) {
76390
+ this.walDirty = true;
76391
+ }
76392
+ }
76393
+ /**
76394
+ * @returns The current lease, or null if no row exists. Used for diagnostics
76395
+ * and stats; the acquire/heartbeat methods do the actual coordination.
76396
+ */
76397
+ getCurrentLease() {
76398
+ const row = this.getLeaseStmt.get();
76399
+ return row ? { holder: row.holder, expiresAt: row.expires_at } : null;
76400
+ }
76401
+ /**
76402
+ * Binds this queue to the lease holder ID of the local process, so the
76403
+ * dispatch gate ({@link assertNotDemoted}) can tell "us" from a peer.
76404
+ * {@link startDispatchLease} sets this automatically; call it directly only in
76405
+ * tests that drive {@link tryAcquireLease}/{@link heartbeatLease} by hand.
76406
+ * @param holderId - This process's lease holder ID.
76407
+ */
76408
+ setLeaseHolder(holderId) {
76409
+ this.leaseHolderId = holderId;
76410
+ }
76411
+ /**
76412
+ * Begins the dispatch-lease acquire-and-heartbeat loop for this process.
76413
+ *
76414
+ * Tries to acquire the lease immediately. On success it fires `onBecameLeader`
76415
+ * and begins heartbeating every `leaseHeartbeatMs`; on failure it retries every
76416
+ * `leaseAcquireRetryMs` until the lease frees up. `onBecameLeader` fires every
76417
+ * time this process transitions from follower to leader (typically once, but
76418
+ * again if it loses the lease mid-run and reclaims it). There is no
76419
+ * lost-leadership callback by design — loss surfaces as a {@link QueueLeaseError}
76420
+ * from the dispatch ops, which the worker catches to drain itself.
76421
+ *
76422
+ * Idempotent: calling it again while the loop is already active is a no-op,
76423
+ * regardless of whether this process is currently a retrying follower or the
76424
+ * leader. A repeat call does not re-acquire, re-heartbeat, re-fire
76425
+ * `onBecameLeader`, or replace the callback — so the App can call it on every
76426
+ * heartbeat tick (and when already the leader) without restarting the loop.
76427
+ * @param onBecameLeader - Callback invoked when this process takes the lease.
76428
+ */
76429
+ startDispatchLease(onBecameLeader) {
76430
+ if (this.leaseLoopActive) {
76431
+ return;
76432
+ }
76433
+ this.onBecameLeader = onBecameLeader;
76434
+ this.leaseLoopActive = true;
76435
+ if (!this.leaseHolderId) {
76436
+ this.leaseHolderId = (0, import_node_crypto4.randomUUID)();
76437
+ }
76438
+ this.tryAcquireDispatchLease();
76439
+ }
76440
+ /**
76441
+ * Stops the dispatch-lease loop and releases the lease if we hold it.
76442
+ * Idempotent and a no-op when the loop was never started.
76443
+ * @returns True if we were the leader when stopping (and released the lease).
76444
+ */
76445
+ stopDispatchLease() {
76446
+ this.leaseLoopActive = false;
76447
+ this.onBecameLeader = void 0;
76448
+ this.clearLeaseAcquireTimer();
76449
+ this.clearLeaseHeartbeatTimer();
76450
+ const wasLeader = this.leaseLeader;
76451
+ if (this.leaseLeader && this.leaseHolderId) {
76452
+ try {
76453
+ this.releaseLease(this.leaseHolderId);
76454
+ } catch (err2) {
76455
+ this.log.warn(`Failed to release queue lease: ${Re(err2)}`);
76456
+ }
76457
+ this.leaseLeader = false;
76458
+ }
76459
+ return wasLeader;
76460
+ }
76461
+ /** @returns True if this process currently holds the dispatch lease. */
76462
+ isLeader() {
76463
+ return this.leaseLeader;
76464
+ }
76465
+ /** @returns The dispatch-lease holder ID for this process (for diagnostics), or undefined if never started. */
76466
+ getLeaseHolderId() {
76467
+ return this.leaseHolderId;
76468
+ }
76469
+ tryAcquireDispatchLease() {
76470
+ if (!this.leaseLoopActive || this.closed || !this.leaseHolderId) {
76471
+ return;
76472
+ }
76473
+ let acquired = false;
76474
+ try {
76475
+ acquired = this.tryAcquireLease(this.leaseHolderId, this.leaseTtlMs);
76476
+ } catch (err2) {
76477
+ this.log.warn(`Queue lease acquire threw: ${Re(err2)}`);
76478
+ }
76479
+ if (acquired) {
76480
+ this.leaseLeader = true;
76481
+ this.clearLeaseAcquireTimer();
76482
+ this.log.info(`Acquired queue lease (holder=${this.leaseHolderId}).`);
76483
+ this.scheduleLeaseHeartbeat();
76484
+ try {
76485
+ this.onBecameLeader?.();
76486
+ } catch (err2) {
76487
+ this.log.error(`onBecameLeader callback threw: ${Re(err2)}`);
76488
+ }
76489
+ return;
76490
+ }
76491
+ this.scheduleLeaseAcquireRetry();
76492
+ }
76493
+ scheduleLeaseAcquireRetry() {
76494
+ if (!this.leaseLoopActive || this.leaseAcquireTimer) {
76495
+ return;
76496
+ }
76497
+ this.leaseAcquireTimer = setTimeout(() => {
76498
+ this.leaseAcquireTimer = void 0;
76499
+ this.tryAcquireDispatchLease();
76500
+ }, this.leaseAcquireRetryMs);
76501
+ if (typeof this.leaseAcquireTimer.unref === "function") {
76502
+ this.leaseAcquireTimer.unref();
76503
+ }
76504
+ }
76505
+ scheduleLeaseHeartbeat() {
76506
+ if (!this.leaseLoopActive || this.leaseHeartbeatTimer) {
76507
+ return;
76508
+ }
76509
+ this.leaseHeartbeatTimer = setInterval(() => this.leaseHeartbeat(), this.leaseHeartbeatMs);
76510
+ if (typeof this.leaseHeartbeatTimer.unref === "function") {
76511
+ this.leaseHeartbeatTimer.unref();
76512
+ }
76513
+ }
76514
+ leaseHeartbeat() {
76515
+ if (!this.leaseLoopActive || !this.leaseLeader || !this.leaseHolderId) {
76516
+ return;
76517
+ }
76518
+ let extended = false;
76519
+ try {
76520
+ extended = this.heartbeatLease(this.leaseHolderId, this.leaseTtlMs);
76521
+ } catch (err2) {
76522
+ this.log.warn(`Queue lease heartbeat threw: ${Re(err2)}`);
76523
+ return;
76524
+ }
76525
+ if (!extended) {
76526
+ this.log.error(`Lost queue lease (holder=${this.leaseHolderId}); peer took over.`);
76527
+ this.leaseLeader = false;
76528
+ this.clearLeaseHeartbeatTimer();
76529
+ this.scheduleLeaseAcquireRetry();
76530
+ }
76531
+ }
76532
+ clearLeaseAcquireTimer() {
76533
+ if (this.leaseAcquireTimer) {
76534
+ clearTimeout(this.leaseAcquireTimer);
76535
+ this.leaseAcquireTimer = void 0;
76536
+ }
76537
+ }
76538
+ clearLeaseHeartbeatTimer() {
76539
+ if (this.leaseHeartbeatTimer) {
76540
+ clearInterval(this.leaseHeartbeatTimer);
76541
+ this.leaseHeartbeatTimer = void 0;
76542
+ }
76543
+ }
76544
+ /**
76545
+ * @returns True when a lease row exists and a DIFFERENT holder owns it — i.e. a
76546
+ * peer has taken over and this process has been demoted. False when no lease
76547
+ * exists (no coordination in play) or the lease is ours. Note this intentionally
76548
+ * does NOT consider expiry: a merely-expired-but-still-ours lease will be
76549
+ * re-extended by our next heartbeat, so only a foreign holder means "demoted",
76550
+ * which keeps this exactly aligned with {@link DurableQueue.isLeader}.
76551
+ */
76552
+ isLeaseHeldByPeer() {
76553
+ const lease = this.getCurrentLease();
76554
+ return lease !== null && lease.holder !== this.leaseHolderId;
76555
+ }
76556
+ /**
76557
+ * Throws {@link QueueLeaseError} when a peer holds the lease. Called at the top
76558
+ * of every dispatch-class mutation so a demoted process can't claim, dispatch,
76559
+ * or settle rows the new leader now owns — the authoritative, data-layer half of
76560
+ * leadership enforcement (the {@link DurableQueue.isLeader} flag is the
76561
+ * cheap optimistic half). Intake, maintenance, diagnostics, and the physical
76562
+ * `markSent` marker are deliberately NOT gated.
76563
+ */
76564
+ assertNotDemoted() {
76565
+ if (this.isLeaseHeldByPeer()) {
76566
+ throw new QueueLeaseError(this.leaseHolderId, this.getCurrentLease()?.holder);
76567
+ }
76568
+ }
76569
+ /**
76570
+ * @returns Size of the underlying database file in bytes, computed from
76571
+ * `page_count * page_size`. Used by the retention sweeper.
76572
+ */
76573
+ getDbSizeBytes() {
76574
+ const row = this.dbSizeBytesStmt.get();
76575
+ return row?.bytes ?? 0;
76576
+ }
76577
+ /**
76578
+ * @param channelName - The channel to search.
76579
+ * @param msgControlId - MSH.10 to look up.
76580
+ * @returns The most recent non-`nacked` row owning this `(channel, control_id)`, or null.
76581
+ */
76582
+ findSeenByControlId(channelName, msgControlId) {
76583
+ const raw = this.findSeenByControlIdStmt.get(channelName, msgControlId);
76584
+ return raw ? rowFromSql(raw) : null;
76585
+ }
76586
+ /**
76587
+ * Returns the sequence number that {@link commitSeqNo} would next persist for
76588
+ * a channel, WITHOUT advancing the counter. The first value for a channel is
76589
+ * 0; thereafter it is the last committed value + 1. Read-only: callers stamp
76590
+ * this candidate into MSH.13 and only {@link commitSeqNo} it once the row is
76591
+ * durably enqueued, so a failed insert never burns a sequence number.
76592
+ * @param channelName - The channel whose counter to peek.
76593
+ * @returns The next sequence number to assign.
76594
+ */
76595
+ peekNextSeqNo(channelName) {
76596
+ const row = this.peekLastSeqNoStmt.get(channelName);
76597
+ return row === void 0 ? 0 : row.last_seq_no + 1;
76598
+ }
76599
+ /**
76600
+ * Persists `seqNo` as the channel's last assigned sequence number. Production
76601
+ * intake commits this inside the insert transaction via
76602
+ * {@link enqueue}'s `commitSeqNo` option, so the row and the counter advance
76603
+ * atomically; call it directly only when not pairing it with an insert. The
76604
+ * counter survives restarts, keeping MSH.13 sequence numbers monotonic.
76605
+ * @param channelName - The channel whose counter to advance.
76606
+ * @param seqNo - The sequence number that was assigned and successfully enqueued.
76607
+ */
76608
+ commitSeqNo(channelName, seqNo) {
76609
+ this.commitSeqNoStmt.run(channelName, seqNo);
76610
+ this.walDirty = true;
76611
+ }
76612
+ };
76613
+ function rowFromSql(raw) {
76614
+ return {
76615
+ id: raw.id,
76616
+ channelName: raw.channel_name,
76617
+ remote: raw.remote,
76618
+ msgControlId: raw.msg_control_id ?? null,
76619
+ msgType: raw.msg_type ?? null,
76620
+ originalMessage: toBuffer(raw.original_message),
76621
+ finalizedMessage: toBuffer(raw.finalized_message),
76622
+ encoding: raw.encoding ?? null,
76623
+ enhancedMode: raw.enhanced_mode ?? null,
76624
+ state: raw.state,
76625
+ attemptCount: raw.attempt_count,
76626
+ callbackId: raw.callback_id,
76627
+ serverResponseBody: raw.server_response_body === null || raw.server_response_body === void 0 ? null : toBuffer(raw.server_response_body),
76628
+ serverStatusCode: raw.server_status_code ?? null,
76629
+ ackOutcome: raw.ack_outcome ?? AckOutcome.PENDING,
76630
+ lastError: raw.last_error ?? null,
76631
+ errorCode: raw.error_code ?? null,
76632
+ seqNo: raw.seq_no ?? null,
76633
+ receivedAt: raw.received_at,
76634
+ processingStartedAt: raw.processing_started_at ?? null,
76635
+ sentAt: raw.sent_at ?? null,
76636
+ processedAt: raw.processed_at ?? null,
76637
+ erroredAt: raw.errored_at ?? null
76638
+ };
76639
+ }
76640
+ function toBlob(value) {
76641
+ return typeof value === "string" ? Buffer.from(value) : value;
76642
+ }
76643
+ function toBuffer(value) {
76644
+ if (Buffer.isBuffer(value)) {
76645
+ return value;
76646
+ }
76647
+ if (value instanceof Uint8Array) {
76648
+ return Buffer.from(value);
76649
+ }
76650
+ if (typeof value === "string") {
76651
+ return Buffer.from(value);
76652
+ }
76653
+ return Buffer.alloc(0);
76654
+ }
76655
+ function isUniqueConstraintError(err2) {
76656
+ if (err2 === null || typeof err2 !== "object") {
76657
+ return false;
76658
+ }
76659
+ const asAny = err2;
76660
+ if (typeof asAny.code === "string" && asAny.code.includes("CONSTRAINT_UNIQUE")) {
76661
+ return true;
76662
+ }
76663
+ if (asAny.errcode === 2067) {
76664
+ return true;
76665
+ }
76666
+ if (typeof asAny.message === "string" && /UNIQUE constraint failed/i.test(asAny.message)) {
76667
+ return true;
76668
+ }
76669
+ return false;
76670
+ }
76671
+
76672
+ // src/queue/retention.ts
76673
+ var DEFAULT_RETENTION_DAYS = 7;
76674
+ var DEFAULT_MAX_SIZE_MB = 512;
76675
+ var DEFAULT_ERRORED_RETENTION_DAYS = 90;
76676
+ var DEFAULT_SWEEP_INTERVAL_SECS = 3600;
76677
+ var RetentionSweeper = class {
76678
+ constructor(options) {
76679
+ __publicField(this, "queue");
76680
+ __publicField(this, "log");
76681
+ __publicField(this, "retentionMs");
76682
+ __publicField(this, "maxSizeBytes");
76683
+ __publicField(this, "erroredRetentionMs");
76684
+ __publicField(this, "intervalMs");
76685
+ __publicField(this, "timer");
76686
+ __publicField(this, "running", false);
76687
+ __publicField(this, "lastSweepAt", null);
76688
+ __publicField(this, "lastResult", null);
76689
+ // Prepared once at construction and reused on every sweep — re-preparing per
76690
+ // sweep would leak GC-finalised statement objects proportional to sweep count.
76691
+ __publicField(this, "phase1Stmt");
76692
+ __publicField(this, "phase2Stmt");
76693
+ __publicField(this, "phase3Stmt");
76694
+ this.queue = options.queue;
76695
+ this.log = options.log;
76696
+ this.retentionMs = (options.retentionDays ?? DEFAULT_RETENTION_DAYS) * 24 * 60 * 60 * 1e3;
76697
+ this.maxSizeBytes = (options.maxSizeMb ?? DEFAULT_MAX_SIZE_MB) * 1024 * 1024;
76698
+ this.erroredRetentionMs = (options.erroredRetentionDays ?? DEFAULT_ERRORED_RETENTION_DAYS) * 24 * 60 * 60 * 1e3;
76699
+ this.intervalMs = (options.sweepIntervalSecs ?? DEFAULT_SWEEP_INTERVAL_SECS) * 1e3;
76700
+ const db = this.queue.getDb();
76701
+ this.phase1Stmt = db.prepare(RETENTION_PHASE1_DELETE);
76702
+ this.phase2Stmt = db.prepare(RETENTION_PHASE2_DELETE);
76703
+ this.phase3Stmt = db.prepare(RETENTION_PHASE3_DELETE);
76704
+ }
76705
+ /** Starts the periodic sweep timer and runs one sweep right away. No-op if already started. */
76706
+ start() {
76707
+ if (this.timer) {
76708
+ return;
76709
+ }
76710
+ this.timer = setInterval(() => this.runSweep(), this.intervalMs);
76711
+ if (typeof this.timer.unref === "function") {
76712
+ this.timer.unref();
76713
+ }
76714
+ this.runSweep();
76715
+ }
76716
+ /** Wrapper around {@link RetentionSweeper.sweep} that logs failures instead of throwing. */
76717
+ runSweep() {
76718
+ try {
76719
+ this.sweep();
76720
+ } catch (err2) {
76721
+ this.log.error(`Retention sweep crashed: ${Re(err2)}`);
76722
+ }
76723
+ }
76724
+ /** Stops the periodic sweep timer. */
76725
+ stop() {
76726
+ if (this.timer) {
76727
+ clearInterval(this.timer);
76728
+ this.timer = void 0;
76729
+ }
76730
+ }
76731
+ /**
76732
+ * Run a single sweep cycle immediately. Safe to call from `start`'s timer or tests.
76733
+ *
76734
+ * Synchronous: every SQLite call here is synchronous, so there is no `await` to
76735
+ * make. Keeping the signature non-`async` means the `running` guard can't be
76736
+ * defeated by a future edit that adds an `await` mid-sweep and opens a
76737
+ * re-entrancy window.
76738
+ * @param now - Override the "now" timestamp used for retention comparisons.
76739
+ * @returns The deletion counts and DB size after this sweep cycle.
76740
+ */
76741
+ sweep(now = Date.now()) {
76742
+ if (this.running) {
76743
+ return this.lastResult ?? { deletedProcessed: 0, deletedErrored: 0, dbSizeBytesAfter: this.queue.getDbSizeBytes() };
76744
+ }
76745
+ this.running = true;
76746
+ const startedAt = Date.now();
76747
+ try {
76748
+ const cutoffProcessed = now - this.retentionMs;
76749
+ const cutoffErrored = now - this.erroredRetentionMs;
76750
+ const phase1 = this.phase1Stmt.run(cutoffProcessed);
76751
+ let deletedProcessed = Number(phase1.changes);
76752
+ let deletedErrored = 0;
76753
+ const sizeBudget = this.maxSizeBytes;
76754
+ const batchSize = 1e3;
76755
+ while (this.queue.getDbSizeBytes() > sizeBudget) {
76756
+ const info = this.phase2Stmt.run(batchSize);
76757
+ if (info.changes === 0) {
76758
+ break;
76759
+ }
76760
+ deletedProcessed += Number(info.changes);
76761
+ }
76762
+ if (this.queue.getDbSizeBytes() > sizeBudget) {
76763
+ while (this.queue.getDbSizeBytes() > sizeBudget) {
76764
+ const info = this.phase3Stmt.run(cutoffErrored, batchSize);
76765
+ if (info.changes === 0) {
76766
+ break;
76767
+ }
76768
+ deletedErrored += Number(info.changes);
76769
+ }
76770
+ }
76771
+ this.queue.checkpointWal();
76772
+ const result = {
76773
+ deletedProcessed,
76774
+ deletedErrored,
76775
+ dbSizeBytesAfter: this.queue.getDbSizeBytes()
76776
+ };
76777
+ this.lastResult = result;
76778
+ this.lastSweepAt = now;
76779
+ if (deletedProcessed > 0 || deletedErrored > 0) {
76780
+ this.log.info(
76781
+ `Retention sweep: deleted ${deletedProcessed} processed, ${deletedErrored} errored. DB ${result.dbSizeBytesAfter} bytes. Took ${Date.now() - startedAt} ms.`
76782
+ );
76783
+ }
76784
+ return result;
76785
+ } finally {
76786
+ this.running = false;
76787
+ }
76788
+ }
76789
+ /** @returns Time of the most recent sweep, or null if none has run yet. */
76790
+ getLastSweepAt() {
76791
+ return this.lastSweepAt;
76792
+ }
76793
+ /** @returns Result of the most recent sweep, or null if none has run yet. */
76794
+ getLastResult() {
76795
+ return this.lastResult;
76796
+ }
76797
+ };
76798
+
76799
+ // src/upgrader-utils.ts
76800
+ var import_node_fs4 = require("node:fs");
74646
76801
  var import_node_os3 = require("node:os");
74647
76802
  var import_node_path4 = require("node:path");
74648
76803
  var import_node_stream = require("node:stream");
@@ -74665,10 +76820,10 @@ async function downloadRelease(version, path4) {
74665
76820
  }
74666
76821
  const readable = import_node_stream.Readable.fromWeb(result.body);
74667
76822
  try {
74668
- await (0, import_promises.pipeline)(readable, (0, import_node_fs3.createWriteStream)(path4));
76823
+ await (0, import_promises.pipeline)(readable, (0, import_node_fs4.createWriteStream)(path4));
74669
76824
  } catch (err2) {
74670
- if ((0, import_node_fs3.existsSync)(path4)) {
74671
- (0, import_node_fs3.unlinkSync)(path4);
76825
+ if ((0, import_node_fs4.existsSync)(path4)) {
76826
+ (0, import_node_fs4.unlinkSync)(path4);
74672
76827
  }
74673
76828
  throw new Error(`Error while downloading release version ${version} to ${path4}`, { cause: err2 });
74674
76829
  }
@@ -74720,6 +76875,22 @@ async function execAsync(command, options) {
74720
76875
  });
74721
76876
  });
74722
76877
  }
76878
+ var STOP_DRAIN_TIMEOUT_MS = 1e4;
76879
+ function raceWithTimeout(promise, ms) {
76880
+ return new Promise((resolve2, reject) => {
76881
+ const timer = setTimeout(() => resolve2("timeout"), ms);
76882
+ promise.then(
76883
+ (value) => {
76884
+ clearTimeout(timer);
76885
+ resolve2(value);
76886
+ },
76887
+ (err2) => {
76888
+ clearTimeout(timer);
76889
+ reject(err2);
76890
+ }
76891
+ );
76892
+ });
76893
+ }
74723
76894
  var _App = class _App {
74724
76895
  constructor(medplum, agentId, logLevel, options) {
74725
76896
  __publicField(this, "medplum");
@@ -74746,6 +76917,8 @@ var _App = class _App {
74746
76917
  __publicField(this, "logStatsTimer");
74747
76918
  __publicField(this, "config");
74748
76919
  __publicField(this, "lastHeartbeatSentTime", -1);
76920
+ __publicField(this, "durableQueue");
76921
+ __publicField(this, "retentionSweeper");
74749
76922
  // Whether this process owns the `medplum-agent` PID, i.e. it is the sole agent that should
74750
76923
  // touch the data plane. A normally-started agent is primary from the outset (main.ts creates
74751
76924
  // the PID before start()). An upgrading agent stays non-primary until it wins the PID from the
@@ -74759,7 +76932,7 @@ var _App = class _App {
74759
76932
  }
74760
76933
  async start() {
74761
76934
  this.log.info("Medplum service starting...");
74762
- this.isPrimary = !(0, import_node_fs4.existsSync)(UPGRADE_MANIFEST_PATH);
76935
+ this.isPrimary = !(0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH);
74763
76936
  await this.startWebSocket();
74764
76937
  const { listenersStarted } = await this.beginReloadConfig();
74765
76938
  const upgradeManifest = this.consumeUpgradeManifest();
@@ -74787,12 +76960,12 @@ var _App = class _App {
74787
76960
  * @returns The parsed manifest, or undefined if no upgrade is in progress.
74788
76961
  */
74789
76962
  consumeUpgradeManifest() {
74790
- if (!(0, import_node_fs4.existsSync)(UPGRADE_MANIFEST_PATH)) {
76963
+ if (!(0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH)) {
74791
76964
  return void 0;
74792
76965
  }
74793
- const upgradeFile = (0, import_node_fs4.readFileSync)(UPGRADE_MANIFEST_PATH, { encoding: "utf-8" });
76966
+ const upgradeFile = (0, import_node_fs5.readFileSync)(UPGRADE_MANIFEST_PATH, { encoding: "utf-8" });
74794
76967
  const upgradeDetails = JSON.parse(upgradeFile);
74795
- (0, import_node_fs4.unlinkSync)(UPGRADE_MANIFEST_PATH);
76968
+ (0, import_node_fs5.unlinkSync)(UPGRADE_MANIFEST_PATH);
74796
76969
  return upgradeDetails;
74797
76970
  }
74798
76971
  async maybeFinalizeUpgrade(upgradeDetails) {
@@ -74887,9 +77060,10 @@ var _App = class _App {
74887
77060
  WebSocket: wrapper_default,
74888
77061
  binaryType: "nodebuffer"
74889
77062
  });
74890
- this.webSocket.addEventListener("error", () => {
77063
+ this.webSocket.addEventListener("error", (event) => {
74891
77064
  if (!this.shutdown) {
74892
- this.log.error("WebSocket closed due to an error");
77065
+ const error = event.error;
77066
+ this.log.error("WebSocket closed due to an error", error);
74893
77067
  }
74894
77068
  });
74895
77069
  this.webSocket.addEventListener("open", async () => {
@@ -74907,6 +77081,7 @@ var _App = class _App {
74907
77081
  if (!this.shutdown && this.live) {
74908
77082
  this.live = false;
74909
77083
  this.log.info("WebSocket closed");
77084
+ this.forEachChannelWorker((worker) => worker.onWebSocketDisconnect());
74910
77085
  }
74911
77086
  });
74912
77087
  this.webSocket.addEventListener("message", async (e) => {
@@ -74924,6 +77099,7 @@ var _App = class _App {
74924
77099
  this.notLiveHeartbeats = 0;
74925
77100
  this.outstandingHeartbeats = 0;
74926
77101
  this.startWebSocketWorker();
77102
+ this.forEachChannelWorker((worker) => worker.notify());
74927
77103
  this.log.info("Successfully connected to Medplum server");
74928
77104
  break;
74929
77105
  case "agent:heartbeat:request":
@@ -74944,9 +77120,12 @@ var _App = class _App {
74944
77120
  if (!command.callback) {
74945
77121
  this.log.warn("Transmit response missing callback");
74946
77122
  }
77123
+ if (this.routeServerResponseToWorker(command)) {
77124
+ break;
77125
+ }
74947
77126
  if (this.config?.status !== "active") {
74948
77127
  this.sendAgentDisabledError(command);
74949
- } else if (!(command.statusCode && command.statusCode >= 400)) {
77128
+ } else if (command.statusCode === void 0 || command.statusCode < 400) {
74950
77129
  this.addToHl7Queue(command);
74951
77130
  } else {
74952
77131
  this.log.error(`Error during handling transmit request: ${command.body}`);
@@ -75049,6 +77228,16 @@ var _App = class _App {
75049
77228
  const keepAlive = agent?.setting?.find((setting) => setting.name === "keepAlive")?.valueBoolean;
75050
77229
  const maxClientsPerRemote = agent?.setting?.find((setting) => setting.name === "maxClientsPerRemote")?.valueInteger;
75051
77230
  const logStatsFreqSecs = agent?.setting?.find((setting) => setting.name === "logStatsFreqSecs")?.valueInteger;
77231
+ const durableQueueOn = agent?.setting?.find((setting) => setting.name === "durableQueue")?.valueBoolean ?? false;
77232
+ const queueDbPath = agent?.setting?.find((setting) => setting.name === "queueDbPath")?.valueString;
77233
+ const queueRetentionDays = agent?.setting?.find((setting) => setting.name === "queueRetentionDays")?.valueInteger;
77234
+ const queueRetentionMaxMb = agent?.setting?.find((setting) => setting.name === "queueRetentionMaxMb")?.valueInteger;
77235
+ const queueErroredRetentionDays = agent?.setting?.find(
77236
+ (setting) => setting.name === "queueErroredRetentionDays"
77237
+ )?.valueInteger;
77238
+ const queueSweepIntervalSecs = agent?.setting?.find(
77239
+ (setting) => setting.name === "queueSweepIntervalSecs"
77240
+ )?.valueInteger;
75052
77241
  if (this.keepAlive !== keepAlive) {
75053
77242
  const results = await Promise.allSettled(Array.from(this.hl7Clients.values()).map((pool) => pool.closeAll()));
75054
77243
  for (const result of results) {
@@ -75079,9 +77268,136 @@ var _App = class _App {
75079
77268
  this.log.info(`Stats logging enabled. Logging stats every ${this.logStatsFreqSecs} seconds...`);
75080
77269
  this.logStatsTimer ??= setInterval(() => this.logStats(), this.logStatsFreqSecs * 1e3);
75081
77270
  }
77271
+ this.reconcileDurableQueue({
77272
+ durableQueueOn,
77273
+ queueDbPath,
77274
+ queueRetentionDays,
77275
+ queueRetentionMaxMb,
77276
+ queueErroredRetentionDays,
77277
+ queueSweepIntervalSecs
77278
+ });
75082
77279
  const startPromises = await this.hydrateListeners();
75083
77280
  return { listenersStarted: this.waitForChannelsToStart(startPromises) };
75084
77281
  }
77282
+ /**
77283
+ * Opens, closes, or reconfigures the durable queue based on the latest config.
77284
+ *
77285
+ * Toggling `durableQueue` between true and false at runtime triggers a queue
77286
+ * open/close. Toggling other queue settings (retention, sweep interval) starts
77287
+ * a fresh {@link RetentionSweeper} against the existing DB without reopening it.
77288
+ *
77289
+ * Changing `queueDbPath` while the queue is already open is intentionally NOT
77290
+ * supported — that would require moving / closing the existing DB. Operators
77291
+ * who need to change the path should disable the queue, then re-enable with the
77292
+ * new path.
77293
+ * @param args - The current queue-related settings drawn from the Agent resource.
77294
+ * @param args.durableQueueOn - Master switch — true to open the queue, false to close it.
77295
+ * @param args.queueDbPath - Optional override for the DB file path (defaults to `<logDir>/medplum-agent-queue.sqlite`).
77296
+ * @param args.queueRetentionDays - Time-based retention window for `processed` rows, in days.
77297
+ * @param args.queueRetentionMaxMb - Soft cap on DB size, in MiB.
77298
+ * @param args.queueErroredRetentionDays - Floor on `errored` / `nacked` retention, in days.
77299
+ * @param args.queueSweepIntervalSecs - How often the retention sweeper runs, in seconds.
77300
+ */
77301
+ reconcileDurableQueue(args) {
77302
+ if (!args.durableQueueOn) {
77303
+ if (this.durableQueue) {
77304
+ this.log.info("durableQueue disabled \u2014 closing queue.");
77305
+ this.retentionSweeper?.stop();
77306
+ this.retentionSweeper = void 0;
77307
+ this.durableQueue.close();
77308
+ this.durableQueue = void 0;
77309
+ }
77310
+ return;
77311
+ }
77312
+ if (!this.durableQueue) {
77313
+ const path4 = args.queueDbPath ?? this.defaultQueueDbPath();
77314
+ try {
77315
+ this.durableQueue = DurableQueue.open({ path: path4, log: this.log });
77316
+ this.log.info(`Durable queue opened at ${path4}.`);
77317
+ } catch (err2) {
77318
+ this.log.error(`Failed to open durable queue at ${path4}: ${Re(err2)}`);
77319
+ this.durableQueue = void 0;
77320
+ return;
77321
+ }
77322
+ }
77323
+ this.durableQueue.startDispatchLease(() => this.onBecameQueueLeader());
77324
+ this.retentionSweeper?.stop();
77325
+ this.retentionSweeper = new RetentionSweeper({
77326
+ queue: this.durableQueue,
77327
+ log: this.log,
77328
+ retentionDays: args.queueRetentionDays,
77329
+ maxSizeMb: args.queueRetentionMaxMb,
77330
+ erroredRetentionDays: args.queueErroredRetentionDays,
77331
+ sweepIntervalSecs: args.queueSweepIntervalSecs
77332
+ });
77333
+ this.retentionSweeper.start();
77334
+ }
77335
+ /**
77336
+ * Called by the {@link DurableQueue} dispatch-lease loop the first time we take the lease.
77337
+ *
77338
+ * This is the single point that runs `recoverOnStartup` and spins up the
77339
+ * channel workers. Both depend on us being the only writer — running them at
77340
+ * raw queue-open time would race with any peer that still holds the lease.
77341
+ *
77342
+ * Re-entrancy: if we lose and regain the lease later, this fires again. The
77343
+ * recovery sweep is idempotent (no `claimed`/`inflight` rows means no work), and
77344
+ * `maybeStartWorker` is a no-op if the worker is already running.
77345
+ */
77346
+ onBecameQueueLeader() {
77347
+ const queue = this.durableQueue;
77348
+ if (!queue) {
77349
+ return;
77350
+ }
77351
+ const recovered = queue.recoverOnStartup();
77352
+ if (recovered.requeued > 0 || recovered.failed > 0) {
77353
+ this.log.info(
77354
+ `Acquired queue lease \u2014 recovered interrupted rows: ${recovered.requeued} requeued (unsent), ${recovered.failed} failed (ambiguous, in-flight).`
77355
+ );
77356
+ }
77357
+ const errors = [];
77358
+ for (const channel of this.channels.values()) {
77359
+ if (channel instanceof AgentHl7Channel) {
77360
+ try {
77361
+ channel.onBecameQueueLeader();
77362
+ } catch (err2) {
77363
+ errors.push(err2);
77364
+ }
77365
+ }
77366
+ }
77367
+ if (errors.length > 0) {
77368
+ throw new Error(`Failed to start ${errors.length} HL7 channel worker(s) after acquiring queue lease`, {
77369
+ cause: errors
77370
+ });
77371
+ }
77372
+ }
77373
+ /** @returns True when this agent currently holds the durable-queue lease. */
77374
+ isQueueLeader() {
77375
+ return this.durableQueue?.isLeader() ?? false;
77376
+ }
77377
+ /**
77378
+ * Default location for the queue DB file when no override is provided.
77379
+ *
77380
+ * Co-locating with the main logger's log directory keeps everything an
77381
+ * operator needs to mount a persistent volume in one place. The fallback is
77382
+ * the current working directory — same default an unconfigured agent uses.
77383
+ * @returns Absolute path to the default queue DB file.
77384
+ */
77385
+ defaultQueueDbPath() {
77386
+ const baseDir = isWinstonWrapperLogger(this.log) && this.log.logDir || import_node_process2.default.cwd();
77387
+ const sep = baseDir.endsWith("/") || baseDir.endsWith("\\") ? "" : "/";
77388
+ return `${baseDir}${sep}medplum-agent-queue.sqlite`;
77389
+ }
77390
+ /**
77391
+ * @returns The opened {@link DurableQueue}, or undefined when the queue setting
77392
+ * is off. This is the OPEN handle, valid regardless of leadership — for intake,
77393
+ * maintenance (WAL checkpoint, retention sweep), and diagnostics. For the
77394
+ * dispatch path, gate worker startup on {@link DurableQueue.isLeader} (the cheap
77395
+ * optimistic check); the authoritative gate is implicit in the dispatch ops
77396
+ * themselves, which throw `QueueLeaseError` once a peer holds the lease.
77397
+ */
77398
+ getDurableQueue() {
77399
+ return this.durableQueue;
77400
+ }
75085
77401
  getStats() {
75086
77402
  const stats = getCurrentStats();
75087
77403
  let totalHl7Clients = 0;
@@ -75107,7 +77423,49 @@ var _App = class _App {
75107
77423
  live: this.live,
75108
77424
  outstandingHeartbeats: this.outstandingHeartbeats,
75109
77425
  channelStats,
75110
- clientStats
77426
+ clientStats,
77427
+ ...this.durableQueue ? { durableQueue: this.getDurableQueueStats(this.durableQueue) } : {}
77428
+ };
77429
+ }
77430
+ /**
77431
+ * Snapshot of durable-queue health, surfaced in `agent:stats:response`.
77432
+ *
77433
+ * Structured to fit the `AgentStatValue` shape — 3 nested levels of primitive
77434
+ * records — so `AgentStats`'s index signature stays satisfied. Null sentinels
77435
+ * become `-1` for the same reason ("never swept" reads as -1 on the wire).
77436
+ *
77437
+ * The field is only included when the queue is on; consumers that detect its
77438
+ * absence know the queue is disabled, which is more honest than reporting a
77439
+ * zeroed structure that a dashboard could misread as "queue on but idle."
77440
+ * @param queue - The opened durable queue to read counters from.
77441
+ * @returns A primitive-friendly snapshot fit for `AgentStats`.
77442
+ */
77443
+ getDurableQueueStats(queue) {
77444
+ const counts = queue.countByState();
77445
+ const channelDepth = {};
77446
+ for (const channel of this.channels.values()) {
77447
+ if (channel instanceof AgentHl7Channel) {
77448
+ const d2 = queue.getChannelDepth(channel.getDefinition().name);
77449
+ channelDepth[channel.getDefinition().name] = {
77450
+ queued: d2.queued,
77451
+ claimed: d2.claimed,
77452
+ inflight: d2.inflight,
77453
+ // -1 means "no queued rows" (a zero-aged-row would be 0).
77454
+ oldestQueuedAgeMs: d2.oldestQueuedAgeMs ?? -1
77455
+ };
77456
+ }
77457
+ }
77458
+ const lastResult = this.retentionSweeper?.getLastResult();
77459
+ return {
77460
+ enabled: true,
77461
+ isLeader: this.isQueueLeader(),
77462
+ dbSizeBytes: queue.getDbSizeBytes(),
77463
+ countsByState: counts,
77464
+ channelDepth,
77465
+ // -1 means "never swept yet."
77466
+ lastSweepAt: this.retentionSweeper?.getLastSweepAt() ?? -1,
77467
+ lastSweepDeletedProcessed: lastResult?.deletedProcessed ?? -1,
77468
+ lastSweepDeletedErrored: lastResult?.deletedErrored ?? -1
75111
77469
  };
75112
77470
  }
75113
77471
  logStats() {
@@ -75334,25 +77692,131 @@ var _App = class _App {
75334
77692
  this.webSocket.close();
75335
77693
  this.webSocket = void 0;
75336
77694
  }
77695
+ try {
77696
+ const drained = await raceWithTimeout(this.drainForStop(), STOP_DRAIN_TIMEOUT_MS);
77697
+ if (drained === "timeout") {
77698
+ this.log.warn(
77699
+ `Timed out after ${STOP_DRAIN_TIMEOUT_MS}ms waiting for clients/channels to stop \u2014 proceeding with shutdown.`
77700
+ );
77701
+ }
77702
+ } catch (err2) {
77703
+ this.log.error(`Error while stopping clients/channels: ${Re(err2)}`);
77704
+ }
77705
+ if (this.retentionSweeper) {
77706
+ this.retentionSweeper.stop();
77707
+ this.retentionSweeper = void 0;
77708
+ }
77709
+ if (this.durableQueue) {
77710
+ this.durableQueue.close();
77711
+ this.durableQueue = void 0;
77712
+ }
77713
+ this.log.info("Medplum service stopped successfully");
77714
+ }
77715
+ /**
77716
+ * Closes all outbound HL7 client pools, then stops every channel. Factored out
77717
+ * of {@link App.stop} so the whole drain can be raced against a single timeout.
77718
+ *
77719
+ * Uses allSettled rather than all: one channel failing to stop must not
77720
+ * abandon the wait on its siblings — the durable-queue DB is closed right
77721
+ * after this returns, so every channel that *can* drain must finish first.
77722
+ */
77723
+ async drainForStop() {
75337
77724
  if (this.hl7Clients.size !== 0) {
75338
77725
  const poolClosePromises = [];
75339
77726
  for (const pool of this.hl7Clients.values()) {
75340
77727
  poolClosePromises.push(pool.closeAll());
75341
77728
  }
75342
- await Promise.all(poolClosePromises);
77729
+ const poolResults = await Promise.allSettled(poolClosePromises);
77730
+ for (const result of poolResults) {
77731
+ if (result.status === "rejected") {
77732
+ this.log.error(`Error while closing HL7 client pool: ${Re(result.reason)}`);
77733
+ }
77734
+ }
75343
77735
  this.hl7Clients.clear();
75344
77736
  }
75345
77737
  const channelStopPromises = [];
75346
77738
  for (const channel of this.channels.values()) {
75347
77739
  channelStopPromises.push(channel.stop());
75348
77740
  }
75349
- await Promise.all(channelStopPromises);
75350
- this.log.info("Medplum service stopped successfully");
77741
+ const results = await Promise.allSettled(channelStopPromises);
77742
+ for (const result of results) {
77743
+ if (result.status === "rejected") {
77744
+ this.log.error(`Error while stopping channel: ${Re(result.reason)}`);
77745
+ }
77746
+ }
77747
+ }
77748
+ /**
77749
+ * Dispatches an `agent:transmit:response` to the owning channel's worker, if any.
77750
+ * @param response - The response message received over the agent WebSocket.
77751
+ * @returns True if a worker claimed the response (caller should stop here);
77752
+ * false if no worker matched and legacy handling should run.
77753
+ */
77754
+ routeServerResponseToWorker(response2) {
77755
+ if (!this.durableQueue) {
77756
+ return false;
77757
+ }
77758
+ if (!response2.channel) {
77759
+ return false;
77760
+ }
77761
+ const channel = this.channels.get(response2.channel);
77762
+ if (!(channel instanceof AgentHl7Channel) || !channel.worker) {
77763
+ return false;
77764
+ }
77765
+ channel.worker.onServerResponse(response2);
77766
+ return true;
75351
77767
  }
75352
77768
  addToWebSocketQueue(message) {
75353
77769
  this.webSocketQueue.push(message);
75354
77770
  this.startWebSocketWorker();
75355
77771
  }
77772
+ /** @returns True when the agent WebSocket is connected and the server has acknowledged the connect request. */
77773
+ isLive() {
77774
+ return this.live;
77775
+ }
77776
+ /**
77777
+ * Removes a not-yet-sent `agent:transmit:request` from the WebSocket queue.
77778
+ *
77779
+ * Used by {@link ChannelQueueWorker.onWebSocketDisconnect} to decide whether
77780
+ * an in-flight row can be safely requeued: a request still in this queue
77781
+ * provably never reached the server. A request not found here was either
77782
+ * already written to the socket or is mid-send — both ambiguous, so the
77783
+ * caller must treat `false` as "may have been delivered".
77784
+ * @param callbackId - The `callback` ID of the transmit request to remove.
77785
+ * @returns True if the request was found and removed before being sent.
77786
+ */
77787
+ removeUnsentTransmit(callbackId) {
77788
+ const index = this.webSocketQueue.findIndex(
77789
+ (msg) => msg.type === "agent:transmit:request" && msg.callback === callbackId
77790
+ );
77791
+ if (index === -1) {
77792
+ return false;
77793
+ }
77794
+ this.webSocketQueue.splice(index, 1);
77795
+ return true;
77796
+ }
77797
+ /**
77798
+ * Invokes `fn` for every channel that currently has a durable-queue worker running.
77799
+ *
77800
+ * Collects any failures so one worker throwing can't stop `fn` from reaching the
77801
+ * rest, then surfaces them together as an aggregate error with the collected
77802
+ * errors as its `cause`.
77803
+ * @param fn - Callback applied to each running {@link ChannelQueueWorker}.
77804
+ */
77805
+ forEachChannelWorker(fn2) {
77806
+ const errors = [];
77807
+ for (const channel of this.channels.values()) {
77808
+ if (channel instanceof AgentHl7Channel && channel.worker) {
77809
+ try {
77810
+ fn2(channel.worker);
77811
+ } catch (err2) {
77812
+ errors.push(err2);
77813
+ }
77814
+ }
77815
+ }
77816
+ if (errors.length > 0) {
77817
+ throw new Error(`Failed to run channel worker callback for ${errors.length} channel(s)`, { cause: errors });
77818
+ }
77819
+ }
75356
77820
  addToHl7Queue(message) {
75357
77821
  this.hl7Queue.push(message);
75358
77822
  this.trySendToHl7Connection();
@@ -75531,8 +77995,8 @@ ${result}`);
75531
77995
  forceKillApp("medplum-agent-upgrader");
75532
77996
  removePidFile("medplum-agent-upgrader");
75533
77997
  }
75534
- if ((0, import_node_fs4.existsSync)(UPGRADE_MANIFEST_PATH)) {
75535
- (0, import_node_fs4.unlinkSync)(UPGRADE_MANIFEST_PATH);
77998
+ if ((0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH)) {
77999
+ (0, import_node_fs5.unlinkSync)(UPGRADE_MANIFEST_PATH);
75536
78000
  }
75537
78001
  }
75538
78002
  try {
@@ -75551,7 +78015,7 @@ ${result}`);
75551
78015
  }
75552
78016
  try {
75553
78017
  const command = __filename;
75554
- const logFile = (0, import_node_fs4.openSync)(UPGRADER_LOG_PATH, "w+");
78018
+ const logFile = (0, import_node_fs5.openSync)(UPGRADER_LOG_PATH, "w+");
75555
78019
  child = (0, import_node_child_process.spawn)(command, message.version ? ["--upgrade", message.version] : ["--upgrade"], {
75556
78020
  detached: true,
75557
78021
  stdio: ["ignore", logFile, logFile, "ipc"]
@@ -75591,7 +78055,7 @@ ${result}`);
75591
78055
  }
75592
78056
  try {
75593
78057
  this.log.info("Writing upgrade manifest...", { previousVersion: yn, targetVersion });
75594
- (0, import_node_fs4.writeFileSync)(
78058
+ (0, import_node_fs5.writeFileSync)(
75595
78059
  UPGRADE_MANIFEST_PATH,
75596
78060
  JSON.stringify({
75597
78061
  previousVersion: yn,
@@ -75655,14 +78119,25 @@ ${result}`);
75655
78119
  }
75656
78120
  }
75657
78121
  async sendToWebSocket(message) {
75658
- if (!this.webSocket) {
78122
+ const ws2 = this.webSocket;
78123
+ if (!ws2) {
75659
78124
  throw new Error("WebSocket not connected");
75660
78125
  }
75661
78126
  if ("accessToken" in message) {
75662
78127
  await this.medplum.refreshIfExpired();
75663
78128
  message.accessToken = this.medplum.getAccessToken();
75664
78129
  }
75665
- this.webSocket.send(JSON.stringify(message));
78130
+ const payload = JSON.stringify(message);
78131
+ const callback = message.type === "agent:transmit:request" ? message.callback : void 0;
78132
+ if (callback && this.durableQueue) {
78133
+ const durableQueue = this.durableQueue;
78134
+ durableQueue.runInTransaction(() => {
78135
+ durableQueue.markSent(callback);
78136
+ ws2.send(payload);
78137
+ });
78138
+ } else {
78139
+ ws2.send(payload);
78140
+ }
75666
78141
  }
75667
78142
  sendAgentDisabledError(command) {
75668
78143
  const errMsg = "Agent.status is currently set to off";
@@ -75791,7 +78266,7 @@ ${result}`);
75791
78266
  });
75792
78267
  }
75793
78268
  isAgentUpgrading() {
75794
- if ((0, import_node_fs4.existsSync)(UPGRADE_MANIFEST_PATH)) {
78269
+ if ((0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH)) {
75795
78270
  return true;
75796
78271
  }
75797
78272
  if (isAppRunning("medplum-upgrading-agent") || isAppRunning("medplum-agent-upgrader")) {
@@ -75834,7 +78309,7 @@ async function agentMain(argv) {
75834
78309
  console.log(" clientSecret: The OAuth client secret.");
75835
78310
  console.log(" agentId: The Medplum agent ID.");
75836
78311
  process.exit(0);
75837
- } else if ((0, import_node_fs5.existsSync)("agent.properties")) {
78312
+ } else if ((0, import_node_fs6.existsSync)("agent.properties")) {
75838
78313
  args = readPropertiesFile("agent.properties");
75839
78314
  } else {
75840
78315
  console.log("Missing arguments");
@@ -75895,13 +78370,13 @@ function readCommandLineArgs(argv) {
75895
78370
  }
75896
78371
  function readPropertiesFile(fileName) {
75897
78372
  return Object.fromEntries(
75898
- (0, import_node_fs5.readFileSync)(fileName).toString().split("\n").map((line) => line.split("=").map((s) => s.trim()))
78373
+ (0, import_node_fs6.readFileSync)(fileName).toString().split("\n").map((line) => line.split("=").map((s) => s.trim()))
75899
78374
  );
75900
78375
  }
75901
78376
 
75902
78377
  // src/upgrader.ts
75903
78378
  var import_node_child_process2 = require("node:child_process");
75904
- var import_node_fs6 = require("node:fs");
78379
+ var import_node_fs7 = require("node:fs");
75905
78380
  var import_node_os5 = require("node:os");
75906
78381
  var import_node_process3 = __toESM(require("node:process"));
75907
78382
  var semver2 = __toESM(require_semver2());
@@ -75928,7 +78403,7 @@ async function upgraderMain(argv) {
75928
78403
  }
75929
78404
  version = argv[3] ?? await Dy("agent-upgrader");
75930
78405
  binPath = getReleaseBinPath(version);
75931
- if (!(0, import_node_fs6.existsSync)(binPath)) {
78406
+ if (!(0, import_node_fs7.existsSync)(binPath)) {
75932
78407
  globalLogger.info(`Could not find binary at "${binPath}". Downloading release from GitHub...`);
75933
78408
  await downloadRelease(version, binPath);
75934
78409
  globalLogger.info("Release successfully downloaded");
@@ -75979,53 +78454,53 @@ async function main(argv) {
75979
78454
  createPidFile("medplum-agent-upgrader");
75980
78455
  await upgraderMain(argv);
75981
78456
  } else if (argv[2] === "--remove-old-services") {
75982
- const logFileFd = (0, import_node_fs7.openSync)(TEMP_LOG_FILE, "a");
78457
+ const logFileFd = (0, import_node_fs8.openSync)(TEMP_LOG_FILE, "a");
75983
78458
  let allAgentServices = [];
75984
78459
  const currentServiceName = `MedplumAgent_${yn}`;
75985
78460
  while (!allAgentServices.includes(currentServiceName)) {
75986
78461
  const output = (0, import_node_child_process3.execSync)('cmd.exe /c sc query type= service state= all | findstr /i "SERVICE_NAME.*MedplumAgent"');
75987
- (0, import_node_fs7.appendFileSync)(logFileFd, `${output}\r
78462
+ (0, import_node_fs8.appendFileSync)(logFileFd, `${output}\r
75988
78463
  `, { encoding: "utf-8" });
75989
78464
  allAgentServices = output.toString().trim().split("\n").map((line) => line.replace("SERVICE_NAME: ", "").trim());
75990
- (0, import_node_fs7.appendFileSync)(logFileFd, `All services: \r
78465
+ (0, import_node_fs8.appendFileSync)(logFileFd, `All services: \r
75991
78466
  ${allAgentServices.join("\r\n")}\r
75992
78467
  `, { encoding: "utf-8" });
75993
78468
  }
75994
78469
  const servicesToRemove = argv[3] === "--all" ? allAgentServices : allAgentServices.filter((serviceName) => serviceName !== `MedplumAgent_${yn}`);
75995
- (0, import_node_fs7.appendFileSync)(logFileFd, `Medplum agent service to filter out: MedplumAgent_${yn}\r
78470
+ (0, import_node_fs8.appendFileSync)(logFileFd, `Medplum agent service to filter out: MedplumAgent_${yn}\r
75996
78471
  `, {
75997
78472
  encoding: "utf-8"
75998
78473
  });
75999
78474
  for (const serviceName of servicesToRemove) {
76000
78475
  try {
76001
78476
  (0, import_node_child_process3.execSync)(`net stop ${serviceName}`);
76002
- (0, import_node_fs7.appendFileSync)(logFileFd, `${serviceName} stopped\r
78477
+ (0, import_node_fs8.appendFileSync)(logFileFd, `${serviceName} stopped\r
76003
78478
  `, { encoding: "utf-8" });
76004
78479
  console.log(`${serviceName} stopped`);
76005
78480
  } catch (err2) {
76006
- (0, import_node_fs7.appendFileSync)(logFileFd, `Failed to stop service: ${serviceName}\r
78481
+ (0, import_node_fs8.appendFileSync)(logFileFd, `Failed to stop service: ${serviceName}\r
76007
78482
  `, { encoding: "utf-8" });
76008
- (0, import_node_fs7.appendFileSync)(logFileFd, `${Re(err2)}\r
78483
+ (0, import_node_fs8.appendFileSync)(logFileFd, `${Re(err2)}\r
76009
78484
  `, { encoding: "utf-8" });
76010
78485
  console.error(`Failed to stop service: ${serviceName}`);
76011
78486
  console.error(Re(err2));
76012
78487
  }
76013
78488
  try {
76014
78489
  (0, import_node_child_process3.execSync)(`sc.exe delete ${serviceName}`);
76015
- (0, import_node_fs7.appendFileSync)(logFileFd, `${serviceName} deleted\r
78490
+ (0, import_node_fs8.appendFileSync)(logFileFd, `${serviceName} deleted\r
76016
78491
  `, { encoding: "utf-8" });
76017
78492
  console.log(`${serviceName} deleted`);
76018
78493
  } catch (err2) {
76019
- (0, import_node_fs7.appendFileSync)(logFileFd, `Failed to delete service: ${serviceName}\r
78494
+ (0, import_node_fs8.appendFileSync)(logFileFd, `Failed to delete service: ${serviceName}\r
76020
78495
  `, { encoding: "utf-8" });
76021
- (0, import_node_fs7.appendFileSync)(logFileFd, `${Re(err2)}\r
78496
+ (0, import_node_fs8.appendFileSync)(logFileFd, `${Re(err2)}\r
76022
78497
  `, { encoding: "utf-8" });
76023
78498
  console.error(`Failed to delete service: ${serviceName}`);
76024
78499
  console.error(Re(err2));
76025
78500
  }
76026
78501
  }
76027
- (0, import_node_fs7.closeSync)(logFileFd);
76028
- } else if ((0, import_node_fs7.existsSync)(UPGRADE_MANIFEST_PATH)) {
78502
+ (0, import_node_fs8.closeSync)(logFileFd);
78503
+ } else if ((0, import_node_fs8.existsSync)(UPGRADE_MANIFEST_PATH)) {
76029
78504
  createPidFile("medplum-upgrading-agent");
76030
78505
  await agentMain(argv);
76031
78506
  } else {