@medplum/agent 5.1.21 → 5.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +2571 -96
- package/package.json +5 -5
package/dist/cjs/index.cjs
CHANGED
|
@@ -5871,8 +5871,8 @@ var require_buffer_util = __commonJS({
|
|
|
5871
5871
|
}
|
|
5872
5872
|
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.length);
|
|
5873
5873
|
}
|
|
5874
|
-
function
|
|
5875
|
-
|
|
5874
|
+
function toBuffer2(data2) {
|
|
5875
|
+
toBuffer2.readOnly = true;
|
|
5876
5876
|
if (Buffer.isBuffer(data2)) return data2;
|
|
5877
5877
|
let buf;
|
|
5878
5878
|
if (data2 instanceof ArrayBuffer) {
|
|
@@ -5881,7 +5881,7 @@ var require_buffer_util = __commonJS({
|
|
|
5881
5881
|
buf = new FastBuffer(data2.buffer, data2.byteOffset, data2.byteLength);
|
|
5882
5882
|
} else {
|
|
5883
5883
|
buf = Buffer.from(data2);
|
|
5884
|
-
|
|
5884
|
+
toBuffer2.readOnly = false;
|
|
5885
5885
|
}
|
|
5886
5886
|
return buf;
|
|
5887
5887
|
}
|
|
@@ -5889,7 +5889,7 @@ var require_buffer_util = __commonJS({
|
|
|
5889
5889
|
concat,
|
|
5890
5890
|
mask: _mask,
|
|
5891
5891
|
toArrayBuffer,
|
|
5892
|
-
toBuffer,
|
|
5892
|
+
toBuffer: toBuffer2,
|
|
5893
5893
|
unmask: _unmask
|
|
5894
5894
|
};
|
|
5895
5895
|
if (!process.env.WS_NO_BUFFER_UTIL) {
|
|
@@ -7187,7 +7187,7 @@ var require_sender = __commonJS({
|
|
|
7187
7187
|
var PerMessageDeflate2 = require_permessage_deflate();
|
|
7188
7188
|
var { EMPTY_BUFFER, kWebSocket, NOOP } = require_constants2();
|
|
7189
7189
|
var { isBlob, isValidStatusCode } = require_validation();
|
|
7190
|
-
var { mask: applyMask, toBuffer } = require_buffer_util();
|
|
7190
|
+
var { mask: applyMask, toBuffer: toBuffer2 } = require_buffer_util();
|
|
7191
7191
|
var kByteLength = /* @__PURE__ */ Symbol("kByteLength");
|
|
7192
7192
|
var maskBuffer = Buffer.alloc(4);
|
|
7193
7193
|
var RANDOM_POOL_SIZE = 8 * 1024;
|
|
@@ -7377,9 +7377,9 @@ var require_sender = __commonJS({
|
|
|
7377
7377
|
byteLength = data2.size;
|
|
7378
7378
|
readOnly = false;
|
|
7379
7379
|
} else {
|
|
7380
|
-
data2 =
|
|
7380
|
+
data2 = toBuffer2(data2);
|
|
7381
7381
|
byteLength = data2.length;
|
|
7382
|
-
readOnly =
|
|
7382
|
+
readOnly = toBuffer2.readOnly;
|
|
7383
7383
|
}
|
|
7384
7384
|
if (byteLength > 125) {
|
|
7385
7385
|
throw new RangeError("The data size must not be greater than 125 bytes");
|
|
@@ -7424,9 +7424,9 @@ var require_sender = __commonJS({
|
|
|
7424
7424
|
byteLength = data2.size;
|
|
7425
7425
|
readOnly = false;
|
|
7426
7426
|
} else {
|
|
7427
|
-
data2 =
|
|
7427
|
+
data2 = toBuffer2(data2);
|
|
7428
7428
|
byteLength = data2.length;
|
|
7429
|
-
readOnly =
|
|
7429
|
+
readOnly = toBuffer2.readOnly;
|
|
7430
7430
|
}
|
|
7431
7431
|
if (byteLength > 125) {
|
|
7432
7432
|
throw new RangeError("The data size must not be greater than 125 bytes");
|
|
@@ -7482,9 +7482,9 @@ var require_sender = __commonJS({
|
|
|
7482
7482
|
byteLength = data2.size;
|
|
7483
7483
|
readOnly = false;
|
|
7484
7484
|
} else {
|
|
7485
|
-
data2 =
|
|
7485
|
+
data2 = toBuffer2(data2);
|
|
7486
7486
|
byteLength = data2.length;
|
|
7487
|
-
readOnly =
|
|
7487
|
+
readOnly = toBuffer2.readOnly;
|
|
7488
7488
|
}
|
|
7489
7489
|
if (this._firstFragment) {
|
|
7490
7490
|
this._firstFragment = false;
|
|
@@ -7554,7 +7554,7 @@ var require_sender = __commonJS({
|
|
|
7554
7554
|
return;
|
|
7555
7555
|
}
|
|
7556
7556
|
this._bufferedBytes -= options[kByteLength];
|
|
7557
|
-
const data2 =
|
|
7557
|
+
const data2 = toBuffer2(arrayBuffer);
|
|
7558
7558
|
if (!compress) {
|
|
7559
7559
|
this._state = DEFAULT;
|
|
7560
7560
|
this.sendFrame(_Sender.frame(data2, options), cb);
|
|
@@ -8081,7 +8081,7 @@ var require_websocket = __commonJS({
|
|
|
8081
8081
|
EventTarget: { addEventListener, removeEventListener }
|
|
8082
8082
|
} = require_event_target();
|
|
8083
8083
|
var { format, parse } = require_extension();
|
|
8084
|
-
var { toBuffer } = require_buffer_util();
|
|
8084
|
+
var { toBuffer: toBuffer2 } = require_buffer_util();
|
|
8085
8085
|
var kAborted = /* @__PURE__ */ Symbol("kAborted");
|
|
8086
8086
|
var protocolVersions = [8, 13];
|
|
8087
8087
|
var readyStates = ["CONNECTING", "OPEN", "CLOSING", "CLOSED"];
|
|
@@ -8829,7 +8829,7 @@ var require_websocket = __commonJS({
|
|
|
8829
8829
|
}
|
|
8830
8830
|
function sendAfterClose(websocket, data2, cb) {
|
|
8831
8831
|
if (data2) {
|
|
8832
|
-
const length2 = isBlob(data2) ? data2.size :
|
|
8832
|
+
const length2 = isBlob(data2) ? data2.size : toBuffer2(data2).length;
|
|
8833
8833
|
if (websocket._socket) websocket._sender._bufferedBytes += length2;
|
|
8834
8834
|
else websocket._bufferedAmount += length2;
|
|
8835
8835
|
}
|
|
@@ -54825,7 +54825,7 @@ function lt(r6) {
|
|
|
54825
54825
|
return r6.bareCriteria ? [r6.bareCriteria, ...r6.criteriaWithProps] : r6.criteriaWithProps;
|
|
54826
54826
|
}
|
|
54827
54827
|
var hc = new X(1e3);
|
|
54828
|
-
var yn = "5.1.
|
|
54828
|
+
var yn = "5.1.22-7350a53";
|
|
54829
54829
|
var xc = k.FHIR_JSON + ", */*; q=0.1";
|
|
54830
54830
|
var vc = "https://api.medplum.com/";
|
|
54831
54831
|
var Tc = 1e3;
|
|
@@ -56056,11 +56056,11 @@ async function Dy(r6) {
|
|
|
56056
56056
|
|
|
56057
56057
|
// src/main.ts
|
|
56058
56058
|
var import_node_child_process3 = require("node:child_process");
|
|
56059
|
-
var
|
|
56059
|
+
var import_node_fs8 = require("node:fs");
|
|
56060
56060
|
var import_node_path5 = __toESM(require("node:path"));
|
|
56061
56061
|
|
|
56062
56062
|
// src/agent-main.ts
|
|
56063
|
-
var
|
|
56063
|
+
var import_node_fs6 = require("node:fs");
|
|
56064
56064
|
|
|
56065
56065
|
// ../hl7/dist/esm/index.mjs
|
|
56066
56066
|
var import_node_assert = __toESM(require("node:assert"), 1);
|
|
@@ -56435,7 +56435,7 @@ var x = class {
|
|
|
56435
56435
|
// src/app.ts
|
|
56436
56436
|
var import_node_assert3 = __toESM(require("node:assert"));
|
|
56437
56437
|
var import_node_child_process = require("node:child_process");
|
|
56438
|
-
var
|
|
56438
|
+
var import_node_fs5 = require("node:fs");
|
|
56439
56439
|
var import_node_net4 = require("node:net");
|
|
56440
56440
|
var import_node_os4 = require("node:os");
|
|
56441
56441
|
var import_node_process2 = __toESM(require("node:process"));
|
|
@@ -56597,9 +56597,11 @@ var AgentByteStreamChannel = class extends BaseChannel {
|
|
|
56597
56597
|
}
|
|
56598
56598
|
sendToRemote(msg) {
|
|
56599
56599
|
const connection = this.connections.get(msg.remote);
|
|
56600
|
-
if (connection) {
|
|
56601
|
-
|
|
56600
|
+
if (!connection) {
|
|
56601
|
+
return false;
|
|
56602
56602
|
}
|
|
56603
|
+
connection.write(Buffer.from(msg.body, "hex"));
|
|
56604
|
+
return true;
|
|
56603
56605
|
}
|
|
56604
56606
|
handleNewConnection(socket) {
|
|
56605
56607
|
const c2 = new ByteStreamChannelConnection(this, socket);
|
|
@@ -73620,6 +73622,410 @@ function calculateRttStats(rttSamples, pendingCount) {
|
|
|
73620
73622
|
};
|
|
73621
73623
|
}
|
|
73622
73624
|
|
|
73625
|
+
// src/queue/types.ts
|
|
73626
|
+
var MessageState = {
|
|
73627
|
+
QUEUED: "queued",
|
|
73628
|
+
CLAIMED: "claimed",
|
|
73629
|
+
INFLIGHT: "inflight",
|
|
73630
|
+
PROCESSED: "processed",
|
|
73631
|
+
REJECTED: "rejected",
|
|
73632
|
+
FAILED: "failed",
|
|
73633
|
+
NACKED: "nacked"
|
|
73634
|
+
};
|
|
73635
|
+
var AckOutcome = {
|
|
73636
|
+
PENDING: "pending",
|
|
73637
|
+
DELIVERED: "delivered",
|
|
73638
|
+
UNDELIVERED: "undelivered",
|
|
73639
|
+
NOT_OWED: "not_owed"
|
|
73640
|
+
};
|
|
73641
|
+
var QueueErrorCode = {
|
|
73642
|
+
/** Transient (`failed`): server returned 5xx. */
|
|
73643
|
+
ServerError: "server-error",
|
|
73644
|
+
/** Transient (`failed`): server returned 429. */
|
|
73645
|
+
ServerRateLimited: "server-rate-limited",
|
|
73646
|
+
/** Ambiguous (`failed`): timed out waiting for the server response; delivery unknown. */
|
|
73647
|
+
ResponseTimeout: "response-timeout",
|
|
73648
|
+
/** Ambiguous (`failed`): row was in `processing` when the agent restarted. */
|
|
73649
|
+
Interrupted: "interrupted",
|
|
73650
|
+
/** Ambiguous (`failed`): in-flight dispatch was cancelled by worker shutdown. */
|
|
73651
|
+
WorkerStopped: "worker-stopped",
|
|
73652
|
+
/** Ambiguous (`failed`): dispatch failed for an unclassified reason; delivery unknown. */
|
|
73653
|
+
DispatchFailed: "dispatch-failed",
|
|
73654
|
+
/** Permanent (`rejected`): server returned 4xx (other than 429) — the message was rejected. */
|
|
73655
|
+
ServerRejected: "server-rejected",
|
|
73656
|
+
/** Intake rejection (`nacked`): a storage error prevented the message from being committed. */
|
|
73657
|
+
StorageError: "storage-error",
|
|
73658
|
+
/** Intake rejection (`nacked`): the message reused a committed control ID (duplicate collision). */
|
|
73659
|
+
DuplicateRejected: "duplicate-rejected"
|
|
73660
|
+
};
|
|
73661
|
+
var QueueError = class extends Error {
|
|
73662
|
+
constructor(code, message) {
|
|
73663
|
+
super(message);
|
|
73664
|
+
__publicField(this, "code");
|
|
73665
|
+
this.code = code;
|
|
73666
|
+
}
|
|
73667
|
+
};
|
|
73668
|
+
var QueueLeaseError = class extends Error {
|
|
73669
|
+
constructor(localHolder, currentHolder) {
|
|
73670
|
+
super(
|
|
73671
|
+
`Queue lease held by a peer; this process is no longer leader (local holder=${localHolder ?? "unset"}, current holder=${currentHolder ?? "none"})`
|
|
73672
|
+
);
|
|
73673
|
+
/** The holder this process believes it is (the one bound via `setLeaseHolder`). */
|
|
73674
|
+
__publicField(this, "localHolder");
|
|
73675
|
+
/** The holder that actually owns the lease now (the peer that took over). */
|
|
73676
|
+
__publicField(this, "currentHolder");
|
|
73677
|
+
this.name = "QueueLeaseError";
|
|
73678
|
+
this.localHolder = localHolder;
|
|
73679
|
+
this.currentHolder = currentHolder;
|
|
73680
|
+
}
|
|
73681
|
+
};
|
|
73682
|
+
var DuplicateBehavior = {
|
|
73683
|
+
REJECT: "reject",
|
|
73684
|
+
IDEMPOTENT: "idempotent"
|
|
73685
|
+
};
|
|
73686
|
+
|
|
73687
|
+
// src/queue/worker.ts
|
|
73688
|
+
var DEFAULT_WORKER_RESPONSE_TIMEOUT_MS = 6e4;
|
|
73689
|
+
var DEFAULT_WORKER_IDLE_POLL_MS = 250;
|
|
73690
|
+
var RowRequeuedError = class extends Error {
|
|
73691
|
+
constructor() {
|
|
73692
|
+
super("row requeued after WebSocket disconnect");
|
|
73693
|
+
}
|
|
73694
|
+
};
|
|
73695
|
+
var ChannelQueueWorker = class {
|
|
73696
|
+
constructor(options) {
|
|
73697
|
+
__publicField(this, "channelName");
|
|
73698
|
+
__publicField(this, "app");
|
|
73699
|
+
__publicField(this, "queue");
|
|
73700
|
+
__publicField(this, "log");
|
|
73701
|
+
__publicField(this, "responseTimeoutMs");
|
|
73702
|
+
__publicField(this, "idlePollMs");
|
|
73703
|
+
__publicField(this, "sendAck");
|
|
73704
|
+
__publicField(this, "running", false);
|
|
73705
|
+
__publicField(this, "stopping", false);
|
|
73706
|
+
__publicField(this, "loopPromise");
|
|
73707
|
+
// Resolves whenever `notify()` is called, then is replaced with a fresh promise.
|
|
73708
|
+
// Lets the loop sleep without polling when the queue is known-empty.
|
|
73709
|
+
__publicField(this, "wakeSignal");
|
|
73710
|
+
__publicField(this, "pending");
|
|
73711
|
+
/**
|
|
73712
|
+
* In-flight lease check, fired on each App heartbeat tick (~10s). While a
|
|
73713
|
+
* dispatch is awaiting the server, a worker can't notice via `claimNext` that a
|
|
73714
|
+
* peer took the lease — it's wedged on the response. This cancels that wedged
|
|
73715
|
+
* dispatch with a {@link QueueLeaseError} so the loop tears the worker down,
|
|
73716
|
+
* instead of waiting out the full {@link DEFAULT_WORKER_RESPONSE_TIMEOUT_MS}. A
|
|
73717
|
+
* no-op while no dispatch is pending — an idle worker detects loss far sooner on
|
|
73718
|
+
* its next `claimNext` poll. An arrow field so add/removeEventListener share one
|
|
73719
|
+
* stable reference.
|
|
73720
|
+
*/
|
|
73721
|
+
__publicField(this, "onHeartbeat", () => {
|
|
73722
|
+
const pending = this.pending;
|
|
73723
|
+
if (!pending || !this.queue.isLeaseHeldByPeer()) {
|
|
73724
|
+
return;
|
|
73725
|
+
}
|
|
73726
|
+
clearTimeout(pending.timeout);
|
|
73727
|
+
this.pending = void 0;
|
|
73728
|
+
pending.reject(new QueueLeaseError(void 0, this.queue.getCurrentLease()?.holder));
|
|
73729
|
+
});
|
|
73730
|
+
this.channelName = options.channelName;
|
|
73731
|
+
this.app = options.app;
|
|
73732
|
+
this.queue = options.queue;
|
|
73733
|
+
this.log = options.log;
|
|
73734
|
+
this.responseTimeoutMs = options.responseTimeoutMs ?? DEFAULT_WORKER_RESPONSE_TIMEOUT_MS;
|
|
73735
|
+
this.idlePollMs = options.idlePollMs ?? DEFAULT_WORKER_IDLE_POLL_MS;
|
|
73736
|
+
this.sendAck = options.sendAck;
|
|
73737
|
+
this.wakeSignal = makeWakeSignal();
|
|
73738
|
+
}
|
|
73739
|
+
/** Starts the dispatch loop. No-op if already started. */
|
|
73740
|
+
start() {
|
|
73741
|
+
if (this.running) {
|
|
73742
|
+
return;
|
|
73743
|
+
}
|
|
73744
|
+
this.running = true;
|
|
73745
|
+
this.stopping = false;
|
|
73746
|
+
this.app.heartbeatEmitter.addEventListener("heartbeat", this.onHeartbeat);
|
|
73747
|
+
this.loopPromise = this.loop().catch((err2) => {
|
|
73748
|
+
this.log.error(`Worker loop crashed: ${Re(err2)}`);
|
|
73749
|
+
});
|
|
73750
|
+
}
|
|
73751
|
+
/**
|
|
73752
|
+
* @returns True while the dispatch loop is live. Goes false after {@link stop}
|
|
73753
|
+
* or after the worker self-terminates on lease loss — the channel checks this
|
|
73754
|
+
* to reap a demoted worker before starting a fresh one on re-acquisition.
|
|
73755
|
+
*/
|
|
73756
|
+
isRunning() {
|
|
73757
|
+
return this.running;
|
|
73758
|
+
}
|
|
73759
|
+
/**
|
|
73760
|
+
* Signals to the loop that work may be available. Idempotent; multiple calls
|
|
73761
|
+
* before the loop wakes coalesce into a single wake.
|
|
73762
|
+
*/
|
|
73763
|
+
notify() {
|
|
73764
|
+
this.wakeSignal.resolve();
|
|
73765
|
+
}
|
|
73766
|
+
/**
|
|
73767
|
+
* Routes a server `agent:transmit:response` to its row.
|
|
73768
|
+
*
|
|
73769
|
+
* The common case is the response to the current in-flight dispatch, which
|
|
73770
|
+
* resolves the pending promise and lets {@link process} settle the row.
|
|
73771
|
+
*
|
|
73772
|
+
* A response can also arrive *late* — after the response timeout (or a requeue
|
|
73773
|
+
* / worker stop) already cleared the pending dispatch and left the row
|
|
73774
|
+
* `failed`. We don't discard those: the Medplum server is the authority on the
|
|
73775
|
+
* Bot-leg outcome, so a late ACK/NACK for a row that errored and isn't being
|
|
73776
|
+
* retried right now is applied to settle the row ({@link applyServerResponse}),
|
|
73777
|
+
* exactly as if it had arrived in time. This disambiguates the ambiguous
|
|
73778
|
+
* `ResponseTimeout` case — a row we couldn't classify becomes a definite
|
|
73779
|
+
* `processed`/`rejected`/`failed` — and avoids a redundant re-dispatch.
|
|
73780
|
+
* @param response - The response message received over the WS.
|
|
73781
|
+
* @returns True if the response was applied or resolved a pending dispatch;
|
|
73782
|
+
* false if it could not be matched to a settleable row.
|
|
73783
|
+
*/
|
|
73784
|
+
onServerResponse(response2) {
|
|
73785
|
+
if (!response2.callback) {
|
|
73786
|
+
return false;
|
|
73787
|
+
}
|
|
73788
|
+
const pending = this.pending;
|
|
73789
|
+
if (pending?.row.callbackId === response2.callback) {
|
|
73790
|
+
clearTimeout(pending.timeout);
|
|
73791
|
+
this.pending = void 0;
|
|
73792
|
+
pending.resolve(response2);
|
|
73793
|
+
return true;
|
|
73794
|
+
}
|
|
73795
|
+
const row = this.queue.findByCallback(response2.callback);
|
|
73796
|
+
if (!row) {
|
|
73797
|
+
this.log.warn(`Discarding server response with no matching row (callback=${response2.callback})`);
|
|
73798
|
+
return false;
|
|
73799
|
+
}
|
|
73800
|
+
if (row.state === MessageState.FAILED) {
|
|
73801
|
+
this.log.info(
|
|
73802
|
+
`Applying late server response to errored row id=${row.id} (control id=${row.msgControlId ?? "n/a"}, status=${response2.statusCode ?? "n/a"})`
|
|
73803
|
+
);
|
|
73804
|
+
try {
|
|
73805
|
+
this.applyServerResponse(row, response2);
|
|
73806
|
+
} catch (err2) {
|
|
73807
|
+
if (err2 instanceof QueueLeaseError) {
|
|
73808
|
+
this.log.info(`Discarding late server response for row id=${row.id}: queue lease held by a peer`);
|
|
73809
|
+
return false;
|
|
73810
|
+
}
|
|
73811
|
+
throw err2;
|
|
73812
|
+
}
|
|
73813
|
+
return true;
|
|
73814
|
+
}
|
|
73815
|
+
this.log.warn(
|
|
73816
|
+
`Discarding server response for row id=${row.id} in state '${row.state}' (callback=${response2.callback})`
|
|
73817
|
+
);
|
|
73818
|
+
return false;
|
|
73819
|
+
}
|
|
73820
|
+
/**
|
|
73821
|
+
* Stops the dispatch loop. Cancels any in-flight dispatch by rejecting its
|
|
73822
|
+
* pending response Promise with `worker-stopped` — an ambiguous outcome (we
|
|
73823
|
+
* don't know whether the server processed the message). The row is marked
|
|
73824
|
+
* `failed` for operator review; an operator decides whether to replay.
|
|
73825
|
+
*/
|
|
73826
|
+
async stop() {
|
|
73827
|
+
if (!this.running) {
|
|
73828
|
+
return;
|
|
73829
|
+
}
|
|
73830
|
+
this.stopping = true;
|
|
73831
|
+
this.app.heartbeatEmitter.removeEventListener("heartbeat", this.onHeartbeat);
|
|
73832
|
+
if (this.pending) {
|
|
73833
|
+
const pending = this.pending;
|
|
73834
|
+
clearTimeout(pending.timeout);
|
|
73835
|
+
this.pending = void 0;
|
|
73836
|
+
pending.reject(new QueueError(QueueErrorCode.WorkerStopped, "worker stopping"));
|
|
73837
|
+
}
|
|
73838
|
+
this.notify();
|
|
73839
|
+
if (this.loopPromise) {
|
|
73840
|
+
await this.loopPromise;
|
|
73841
|
+
}
|
|
73842
|
+
this.running = false;
|
|
73843
|
+
}
|
|
73844
|
+
/** @returns True if a row is currently in-flight (worker awaiting a server response). */
|
|
73845
|
+
hasInFlight() {
|
|
73846
|
+
return this.pending !== void 0;
|
|
73847
|
+
}
|
|
73848
|
+
/**
|
|
73849
|
+
* Called from `app.ts` when the agent WebSocket closes. If the in-flight
|
|
73850
|
+
* row's `agent:transmit:request` is still sitting unsent in the app's WS
|
|
73851
|
+
* queue, the server provably never saw it — remove it and return the row to
|
|
73852
|
+
* `queued` so it retries on reconnect instead of timing out into `failed`.
|
|
73853
|
+
*
|
|
73854
|
+
* If the request already went out on the wire, the outcome is ambiguous (the
|
|
73855
|
+
* server may have processed it and the response was lost), so we leave the
|
|
73856
|
+
* pending dispatch alone and let the response timeout mark it `errored`.
|
|
73857
|
+
*/
|
|
73858
|
+
onWebSocketDisconnect() {
|
|
73859
|
+
const pending = this.pending;
|
|
73860
|
+
if (!pending) {
|
|
73861
|
+
return;
|
|
73862
|
+
}
|
|
73863
|
+
if (!this.app.removeUnsentTransmit(pending.row.callbackId)) {
|
|
73864
|
+
return;
|
|
73865
|
+
}
|
|
73866
|
+
clearTimeout(pending.timeout);
|
|
73867
|
+
this.pending = void 0;
|
|
73868
|
+
try {
|
|
73869
|
+
this.queue.requeue(pending.row.id);
|
|
73870
|
+
} catch (err2) {
|
|
73871
|
+
if (err2 instanceof QueueLeaseError) {
|
|
73872
|
+
pending.reject(err2);
|
|
73873
|
+
return;
|
|
73874
|
+
}
|
|
73875
|
+
throw err2;
|
|
73876
|
+
}
|
|
73877
|
+
this.log.info(
|
|
73878
|
+
`Row id=${pending.row.id} (control id=${pending.row.msgControlId ?? "n/a"}) requeued: WebSocket disconnected before transmit was sent`
|
|
73879
|
+
);
|
|
73880
|
+
pending.reject(new RowRequeuedError());
|
|
73881
|
+
}
|
|
73882
|
+
async loop() {
|
|
73883
|
+
try {
|
|
73884
|
+
while (!this.stopping) {
|
|
73885
|
+
if (!this.app.isLive()) {
|
|
73886
|
+
await this.waitForWork();
|
|
73887
|
+
continue;
|
|
73888
|
+
}
|
|
73889
|
+
const row = this.queue.claimNext(this.channelName);
|
|
73890
|
+
if (row) {
|
|
73891
|
+
await this.process(row);
|
|
73892
|
+
} else {
|
|
73893
|
+
await this.waitForWork();
|
|
73894
|
+
}
|
|
73895
|
+
}
|
|
73896
|
+
} catch (err2) {
|
|
73897
|
+
if (!(err2 instanceof QueueLeaseError)) {
|
|
73898
|
+
throw err2;
|
|
73899
|
+
}
|
|
73900
|
+
this.log.info(`Worker for channel '${this.channelName}' stepping down: queue lease taken by a peer.`);
|
|
73901
|
+
this.stopping = true;
|
|
73902
|
+
this.app.heartbeatEmitter.removeEventListener("heartbeat", this.onHeartbeat);
|
|
73903
|
+
this.running = false;
|
|
73904
|
+
}
|
|
73905
|
+
}
|
|
73906
|
+
async process(row) {
|
|
73907
|
+
let response2;
|
|
73908
|
+
try {
|
|
73909
|
+
response2 = await this.dispatch(row);
|
|
73910
|
+
} catch (err2) {
|
|
73911
|
+
if (err2 instanceof RowRequeuedError) {
|
|
73912
|
+
return;
|
|
73913
|
+
}
|
|
73914
|
+
if (err2 instanceof QueueLeaseError) {
|
|
73915
|
+
throw err2;
|
|
73916
|
+
}
|
|
73917
|
+
const code = err2 instanceof QueueError ? err2.code : QueueErrorCode.DispatchFailed;
|
|
73918
|
+
const msg = Re(err2);
|
|
73919
|
+
this.queue.markFailed(row.id, msg, code);
|
|
73920
|
+
this.log.error(
|
|
73921
|
+
`Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) failed during dispatch (${code}): ${msg}`
|
|
73922
|
+
);
|
|
73923
|
+
return;
|
|
73924
|
+
}
|
|
73925
|
+
this.applyServerResponse(row, response2);
|
|
73926
|
+
}
|
|
73927
|
+
/**
|
|
73928
|
+
* Settles a row from its server `agent:transmit:response`: records the raw
|
|
73929
|
+
* response, then transitions the row by status — 2xx → `processed` (plus the
|
|
73930
|
+
* source-leg ACK), permanent 4xx → `rejected`, 5xx/429 → `failed`.
|
|
73931
|
+
*
|
|
73932
|
+
* Shared by the normal in-flight path ({@link process}) and the late-response
|
|
73933
|
+
* path ({@link onServerResponse}), so a response that arrives after the
|
|
73934
|
+
* response timeout settles the row identically to one that arrived in time.
|
|
73935
|
+
* @param row - The row the response belongs to.
|
|
73936
|
+
* @param response - The server response to apply.
|
|
73937
|
+
*/
|
|
73938
|
+
applyServerResponse(row, response2) {
|
|
73939
|
+
this.queue.recordServerResponse(row.id, response2.statusCode ?? null, response2.body ?? null);
|
|
73940
|
+
const statusCode = response2.statusCode ?? 0;
|
|
73941
|
+
if (statusCode >= 400) {
|
|
73942
|
+
const msg = `Server returned ${statusCode}: ${response2.body ?? ""}`;
|
|
73943
|
+
const code = classifyStatusCode(statusCode);
|
|
73944
|
+
if (code === QueueErrorCode.ServerRejected) {
|
|
73945
|
+
this.queue.markRejected(row.id, msg, code);
|
|
73946
|
+
} else {
|
|
73947
|
+
this.queue.markFailed(row.id, msg, code);
|
|
73948
|
+
}
|
|
73949
|
+
this.log.warn(`Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) ${code}: ${msg}`);
|
|
73950
|
+
return;
|
|
73951
|
+
}
|
|
73952
|
+
if (row.enhancedMode === "aaMode") {
|
|
73953
|
+
this.queue.markProcessed(row.id, AckOutcome.DELIVERED);
|
|
73954
|
+
this.log.debug(
|
|
73955
|
+
`Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) processed; app-level AA suppressed (aaMode commit ACK already acknowledged the source)`
|
|
73956
|
+
);
|
|
73957
|
+
return;
|
|
73958
|
+
}
|
|
73959
|
+
let ackOk = false;
|
|
73960
|
+
let ackError;
|
|
73961
|
+
try {
|
|
73962
|
+
ackOk = this.sendAck(response2, row);
|
|
73963
|
+
} catch (err2) {
|
|
73964
|
+
ackError = err2;
|
|
73965
|
+
}
|
|
73966
|
+
this.queue.markProcessed(row.id, ackOk ? AckOutcome.DELIVERED : AckOutcome.UNDELIVERED);
|
|
73967
|
+
if (!ackOk) {
|
|
73968
|
+
const detail = ackError ? `: ${Re(ackError)}` : "";
|
|
73969
|
+
this.log.warn(
|
|
73970
|
+
`Row id=${row.id} (control id=${row.msgControlId ?? "n/a"}) processed upstream but ACK delivery to source failed${detail}; awaiting source retransmit to replay`
|
|
73971
|
+
);
|
|
73972
|
+
}
|
|
73973
|
+
}
|
|
73974
|
+
async dispatch(row) {
|
|
73975
|
+
return new Promise((resolve2, reject) => {
|
|
73976
|
+
const timeout = setTimeout(() => {
|
|
73977
|
+
if (this.pending?.row.id === row.id) {
|
|
73978
|
+
this.pending = void 0;
|
|
73979
|
+
}
|
|
73980
|
+
reject(
|
|
73981
|
+
new QueueError(
|
|
73982
|
+
QueueErrorCode.ResponseTimeout,
|
|
73983
|
+
`Timed out after ${this.responseTimeoutMs}ms waiting for server response`
|
|
73984
|
+
)
|
|
73985
|
+
);
|
|
73986
|
+
}, this.responseTimeoutMs);
|
|
73987
|
+
if (typeof timeout.unref === "function") {
|
|
73988
|
+
timeout.unref();
|
|
73989
|
+
}
|
|
73990
|
+
this.pending = { row, resolve: resolve2, reject, timeout };
|
|
73991
|
+
this.app.addToWebSocketQueue({
|
|
73992
|
+
type: "agent:transmit:request",
|
|
73993
|
+
accessToken: "placeholder",
|
|
73994
|
+
channel: row.channelName,
|
|
73995
|
+
remote: row.remote,
|
|
73996
|
+
contentType: k.HL7_V2,
|
|
73997
|
+
// finalizedMessage holds the decoded HL7 text stored as UTF-8 at intake
|
|
73998
|
+
// (see AgentHl7ChannelConnection.handleMessage) — the same body the legacy
|
|
73999
|
+
// path forwards. The channel `encoding` is wire-level only and recorded
|
|
74000
|
+
// on the row for reference; it does not affect the forwarded text.
|
|
74001
|
+
body: row.finalizedMessage.toString("utf8"),
|
|
74002
|
+
callback: row.callbackId
|
|
74003
|
+
});
|
|
74004
|
+
});
|
|
74005
|
+
}
|
|
74006
|
+
async waitForWork() {
|
|
74007
|
+
const wake = this.wakeSignal;
|
|
74008
|
+
this.wakeSignal = makeWakeSignal();
|
|
74009
|
+
await Promise.race([wake.promise, rn(this.idlePollMs)]);
|
|
74010
|
+
}
|
|
74011
|
+
};
|
|
74012
|
+
function classifyStatusCode(statusCode) {
|
|
74013
|
+
if (statusCode === 429) {
|
|
74014
|
+
return QueueErrorCode.ServerRateLimited;
|
|
74015
|
+
}
|
|
74016
|
+
if (statusCode >= 500) {
|
|
74017
|
+
return QueueErrorCode.ServerError;
|
|
74018
|
+
}
|
|
74019
|
+
return QueueErrorCode.ServerRejected;
|
|
74020
|
+
}
|
|
74021
|
+
function makeWakeSignal() {
|
|
74022
|
+
let resolve2;
|
|
74023
|
+
const promise = new Promise((r6) => {
|
|
74024
|
+
resolve2 = r6;
|
|
74025
|
+
});
|
|
74026
|
+
return { promise, resolve: resolve2 };
|
|
74027
|
+
}
|
|
74028
|
+
|
|
73623
74029
|
// src/stats.ts
|
|
73624
74030
|
var currentStats = {
|
|
73625
74031
|
hl7ConnectionsOpen: 0,
|
|
@@ -73649,6 +74055,14 @@ var AgentHl7Channel = class extends BaseChannel {
|
|
|
73649
74055
|
// Default app level ack mode is AL (Always)
|
|
73650
74056
|
__publicField(this, "assignSeqNo", false);
|
|
73651
74057
|
__publicField(this, "lastSeqNo", -1);
|
|
74058
|
+
__publicField(this, "duplicateBehavior", DuplicateBehavior.IDEMPOTENT);
|
|
74059
|
+
// The channel's own copy of the enhanced mode, parsed from the endpoint URL.
|
|
74060
|
+
// In durable mode this is intentionally NOT pushed onto the Hl7Connection (so
|
|
74061
|
+
// the connection's synchronous auto-ACK stays off and the agent can defer the
|
|
74062
|
+
// commit ACK until after the DB write); the connection therefore can't be the
|
|
74063
|
+
// source of truth, so we track it here and the durable path reads it directly.
|
|
74064
|
+
__publicField(this, "enhancedMode");
|
|
74065
|
+
__publicField(this, "worker");
|
|
73652
74066
|
this.server = new x((connection) => this.handleNewConnection(connection));
|
|
73653
74067
|
this.prefix = `[HL7:${definition.name}] `;
|
|
73654
74068
|
this.log = app.log.clone({ options: { prefix: this.prefix } });
|
|
@@ -73664,6 +74078,7 @@ var AgentHl7Channel = class extends BaseChannel {
|
|
|
73664
74078
|
this.log.info(`Channel starting on ${address}...`);
|
|
73665
74079
|
this.stats = new ChannelStatsTracker({ heartbeatEmitter: this.app.heartbeatEmitter, log: this.log });
|
|
73666
74080
|
this.configureHl7ServerAndConnections();
|
|
74081
|
+
this.maybeStartWorker();
|
|
73667
74082
|
await this.server.start(Number.parseInt(address.port, 10));
|
|
73668
74083
|
this.log.info("Channel started successfully");
|
|
73669
74084
|
}
|
|
@@ -73672,45 +74087,102 @@ var AgentHl7Channel = class extends BaseChannel {
|
|
|
73672
74087
|
return;
|
|
73673
74088
|
}
|
|
73674
74089
|
this.log.info("Channel stopping...");
|
|
74090
|
+
if (this.worker) {
|
|
74091
|
+
await this.worker.stop();
|
|
74092
|
+
this.worker = void 0;
|
|
74093
|
+
}
|
|
73675
74094
|
await Promise.allSettled(Array.from(this.connections.values()).map((connection) => connection.close()));
|
|
73676
74095
|
await this.server.stop();
|
|
73677
74096
|
this.stats.cleanup();
|
|
73678
74097
|
this.started = false;
|
|
73679
74098
|
this.log.info("Channel stopped successfully");
|
|
73680
74099
|
}
|
|
74100
|
+
/**
|
|
74101
|
+
* Starts a {@link ChannelQueueWorker} bound to this channel and the app's
|
|
74102
|
+
* durable queue, but only if we currently hold the queue lease.
|
|
74103
|
+
*
|
|
74104
|
+
* Called both from `start()` (when leadership may already be established by
|
|
74105
|
+
* the time the channel comes up) and from `onBecameQueueLeader()` (when
|
|
74106
|
+
* leadership arrives later, e.g. after waiting out a peer's lease during a
|
|
74107
|
+
* zero-downtime upgrade overlap).
|
|
74108
|
+
*
|
|
74109
|
+
* No-op when the queue is off, we're not leader, or the worker is already
|
|
74110
|
+
* running — so it's safe to call from either entry point.
|
|
74111
|
+
*/
|
|
74112
|
+
maybeStartWorker() {
|
|
74113
|
+
if (this.worker && !this.worker.isRunning()) {
|
|
74114
|
+
this.worker = void 0;
|
|
74115
|
+
}
|
|
74116
|
+
if (this.worker) {
|
|
74117
|
+
return;
|
|
74118
|
+
}
|
|
74119
|
+
const queue = this.app.getDurableQueue();
|
|
74120
|
+
if (!queue?.isLeader()) {
|
|
74121
|
+
return;
|
|
74122
|
+
}
|
|
74123
|
+
this.worker = new ChannelQueueWorker({
|
|
74124
|
+
channelName: this.getDefinition().name,
|
|
74125
|
+
app: this.app,
|
|
74126
|
+
queue,
|
|
74127
|
+
log: this.log,
|
|
74128
|
+
sendAck: (response2) => this.sendToRemote(response2)
|
|
74129
|
+
});
|
|
74130
|
+
this.worker.start();
|
|
74131
|
+
this.worker.notify();
|
|
74132
|
+
}
|
|
74133
|
+
/**
|
|
74134
|
+
* Notification from the App that we've taken the durable-queue lease.
|
|
74135
|
+
* Triggers worker bring-up for this channel if it isn't already running.
|
|
74136
|
+
*/
|
|
74137
|
+
onBecameQueueLeader() {
|
|
74138
|
+
this.maybeStartWorker();
|
|
74139
|
+
}
|
|
73681
74140
|
shouldAssignSeqNo() {
|
|
73682
74141
|
return this.assignSeqNo;
|
|
73683
74142
|
}
|
|
74143
|
+
/**
|
|
74144
|
+
* Returns the next MSH.13 sequence number from the in-memory counter (resets
|
|
74145
|
+
* on restart). Used only by the legacy (non-durable) path; the durable path
|
|
74146
|
+
* peeks/commits a persisted per-channel counter via the queue so a failed
|
|
74147
|
+
* intake doesn't consume a number. See {@link DurableQueue.peekNextSeqNo}.
|
|
74148
|
+
* @returns The next sequence number to assign.
|
|
74149
|
+
*/
|
|
73684
74150
|
takeNextSeqNo() {
|
|
73685
74151
|
return ++this.lastSeqNo;
|
|
73686
74152
|
}
|
|
73687
74153
|
sendToRemote(msg) {
|
|
73688
74154
|
const connection = this.connections.get(msg.remote);
|
|
73689
|
-
if (connection) {
|
|
73690
|
-
|
|
73691
|
-
|
|
73692
|
-
|
|
73693
|
-
|
|
73694
|
-
|
|
73695
|
-
|
|
73696
|
-
|
|
73697
|
-
|
|
73698
|
-
|
|
73699
|
-
|
|
73700
|
-
|
|
73701
|
-
|
|
73702
|
-
|
|
73703
|
-
|
|
73704
|
-
return;
|
|
73705
|
-
}
|
|
73706
|
-
this.channelLog.info(`[Sending ACK -- ID: ${msgControlId}]: ${hl7Message.toString().replaceAll("\r", "\n")}`);
|
|
73707
|
-
connection.hl7Connection.send(Wo.parse(msg.body));
|
|
74155
|
+
if (!connection) {
|
|
74156
|
+
this.log.warn(`Attempted to send message to disconnected remote: ${msg.remote}`);
|
|
74157
|
+
return false;
|
|
74158
|
+
}
|
|
74159
|
+
const hl7Message = Wo.parse(msg.body);
|
|
74160
|
+
const msgControlId = hl7Message.getSegment("MSA")?.getField(2)?.toString();
|
|
74161
|
+
const ackCode = hl7Message.getSegment("MSA")?.getField(1)?.toString()?.toUpperCase();
|
|
74162
|
+
if (ackCode && isAppLevelAckCode(ackCode) && !shouldSendAppLevelAck({
|
|
74163
|
+
mode: this.appLevelAckMode,
|
|
74164
|
+
ackCode,
|
|
74165
|
+
enhancedMode: this.enhancedMode
|
|
74166
|
+
})) {
|
|
74167
|
+
this.channelLog.debug(
|
|
74168
|
+
`[Skipping ACK -- Mode: ${this.appLevelAckMode} -- ID: ${msgControlId ?? "not provided"} -- ACK: ${ackCode ?? "unknown"}]`
|
|
74169
|
+
);
|
|
73708
74170
|
if (msgControlId) {
|
|
73709
74171
|
this.stats.recordAckReceived(msgControlId);
|
|
73710
74172
|
}
|
|
73711
|
-
|
|
73712
|
-
this.log.warn(`Attempted to send message to disconnected remote: ${msg.remote}`);
|
|
74173
|
+
return true;
|
|
73713
74174
|
}
|
|
74175
|
+
this.channelLog.info(`[Sending ACK -- ID: ${msgControlId}]: ${hl7Message.toString().replaceAll("\r", "\n")}`);
|
|
74176
|
+
try {
|
|
74177
|
+
connection.hl7Connection.send(Wo.parse(msg.body));
|
|
74178
|
+
} catch (err2) {
|
|
74179
|
+
this.channelLog.error(`Failed to send ACK to ${msg.remote}: ${Re(err2)}`);
|
|
74180
|
+
return false;
|
|
74181
|
+
}
|
|
74182
|
+
if (msgControlId) {
|
|
74183
|
+
this.stats.recordAckReceived(msgControlId);
|
|
74184
|
+
}
|
|
74185
|
+
return true;
|
|
73714
74186
|
}
|
|
73715
74187
|
async reloadConfig(definition, endpoint) {
|
|
73716
74188
|
const previousEndpoint = this.endpoint;
|
|
@@ -73744,6 +74216,7 @@ var AgentHl7Channel = class extends BaseChannel {
|
|
|
73744
74216
|
const assignSeqNo = address.searchParams.get("assignSeqNo")?.toLowerCase() === "true";
|
|
73745
74217
|
const messagesPerMinRaw = address.searchParams.get("messagesPerMin") ?? void 0;
|
|
73746
74218
|
const appLevelAckRaw = address.searchParams.get("appLevelAck") ?? void 0;
|
|
74219
|
+
const duplicateBehaviorRaw = address.searchParams.get("duplicateBehavior") ?? void 0;
|
|
73747
74220
|
let messagesPerMin = messagesPerMinRaw ? Number.parseInt(messagesPerMinRaw, 10) : void 0;
|
|
73748
74221
|
if (messagesPerMin !== void 0 && !Number.isInteger(messagesPerMin)) {
|
|
73749
74222
|
this.log.warn(
|
|
@@ -73753,18 +74226,33 @@ var AgentHl7Channel = class extends BaseChannel {
|
|
|
73753
74226
|
}
|
|
73754
74227
|
this.appLevelAckMode = parseAppLevelAckMode(appLevelAckRaw, this.log);
|
|
73755
74228
|
this.assignSeqNo = assignSeqNo;
|
|
74229
|
+
this.duplicateBehavior = parseDuplicateBehavior(duplicateBehaviorRaw, this.log);
|
|
73756
74230
|
if (!assignSeqNo) {
|
|
73757
74231
|
this.lastSeqNo = -1;
|
|
73758
74232
|
}
|
|
74233
|
+
this.enhancedMode = enhancedMode;
|
|
74234
|
+
const queueOn = this.app.getDurableQueue() !== void 0;
|
|
74235
|
+
const connectionEnhancedMode = queueOn ? void 0 : enhancedMode;
|
|
73759
74236
|
this.server.setEncoding(encoding);
|
|
73760
|
-
this.server.setEnhancedMode(
|
|
74237
|
+
this.server.setEnhancedMode(connectionEnhancedMode);
|
|
73761
74238
|
this.server.setMessagesPerMin(messagesPerMin);
|
|
73762
74239
|
for (const connection of this.connections.values()) {
|
|
73763
74240
|
connection.hl7Connection.setEncoding(encoding);
|
|
73764
|
-
connection.hl7Connection.setEnhancedMode(
|
|
74241
|
+
connection.hl7Connection.setEnhancedMode(connectionEnhancedMode);
|
|
73765
74242
|
connection.hl7Connection.setMessagesPerMin(messagesPerMin);
|
|
73766
74243
|
}
|
|
73767
74244
|
}
|
|
74245
|
+
getDuplicateBehavior() {
|
|
74246
|
+
return this.duplicateBehavior;
|
|
74247
|
+
}
|
|
74248
|
+
/**
|
|
74249
|
+
* @returns The channel's enhanced mode, as parsed from the endpoint URL. This is
|
|
74250
|
+
* the agent's source of truth in durable mode, where enhancedMode is deliberately
|
|
74251
|
+
* kept off the underlying {@link Hl7Connection} (see {@link enhancedMode}).
|
|
74252
|
+
*/
|
|
74253
|
+
getEnhancedMode() {
|
|
74254
|
+
return this.enhancedMode;
|
|
74255
|
+
}
|
|
73768
74256
|
handleNewConnection(connection) {
|
|
73769
74257
|
const c2 = new AgentHl7ChannelConnection(this, connection);
|
|
73770
74258
|
updateStat("hl7ConnectionsOpen", getCurrentStats().hl7ConnectionsOpen + 1);
|
|
@@ -73798,44 +74286,353 @@ var AgentHl7ChannelConnection = class {
|
|
|
73798
74286
|
this.channel.channelLog.info(
|
|
73799
74287
|
`[Received -- ID: ${msgControlId ?? "not provided"}]: ${event.message.toString().replaceAll("\r", "\n")}`
|
|
73800
74288
|
);
|
|
73801
|
-
|
|
73802
|
-
const seqNo = this.channel.takeNextSeqNo();
|
|
73803
|
-
event.message.getSegment("MSH")?.setField(13, seqNo.toString());
|
|
73804
|
-
this.channel.channelLog.info(`Setting sequence number for message control ID '${msgControlId}': ${seqNo}`);
|
|
73805
|
-
}
|
|
73806
|
-
this.channel.app.addToWebSocketQueue({
|
|
73807
|
-
type: "agent:transmit:request",
|
|
73808
|
-
accessToken: "placeholder",
|
|
73809
|
-
channel: this.channel.getDefinition().name,
|
|
73810
|
-
remote: this.remote,
|
|
73811
|
-
contentType: k.HL7_V2,
|
|
73812
|
-
body: event.message.toString(),
|
|
73813
|
-
callback: `Agent/${this.channel.app.agentId}-${(0, import_node_crypto3.randomUUID)()}`
|
|
73814
|
-
});
|
|
74289
|
+
const originalMessage = Buffer.from(event.message.toString(), "utf8");
|
|
73815
74290
|
if (msgControlId) {
|
|
73816
74291
|
this.channel.stats.recordMessageSent(msgControlId);
|
|
73817
74292
|
}
|
|
74293
|
+
const queue = this.channel.app.getDurableQueue();
|
|
74294
|
+
if (queue) {
|
|
74295
|
+
await this.handleMessageDurable(queue, event, msgControlId, originalMessage);
|
|
74296
|
+
} else {
|
|
74297
|
+
this.handleMessageLegacy(event, msgControlId);
|
|
74298
|
+
}
|
|
73818
74299
|
} catch (err2) {
|
|
73819
74300
|
this.channel.log.error(`HL7 error occurred - check channel logs`);
|
|
73820
74301
|
this.channel.channelLog.error(`HL7 error: ${Re(err2)}`);
|
|
73821
74302
|
}
|
|
73822
74303
|
}
|
|
74304
|
+
/**
|
|
74305
|
+
* Legacy non-durable inbound path: push directly to the in-memory WS queue.
|
|
74306
|
+
* Behavior unchanged from the pre-durable-queue agent — used when the
|
|
74307
|
+
* `durableQueue` setting is off.
|
|
74308
|
+
* @param event - The incoming HL7 message event from the underlying connection.
|
|
74309
|
+
* @param msgControlId - MSH.10 of the inbound message.
|
|
74310
|
+
*/
|
|
74311
|
+
handleMessageLegacy(event, msgControlId) {
|
|
74312
|
+
this.maybeAssignSeqNo(event, msgControlId);
|
|
74313
|
+
this.channel.app.addToWebSocketQueue({
|
|
74314
|
+
type: "agent:transmit:request",
|
|
74315
|
+
accessToken: "placeholder",
|
|
74316
|
+
channel: this.channel.getDefinition().name,
|
|
74317
|
+
remote: this.remote,
|
|
74318
|
+
contentType: k.HL7_V2,
|
|
74319
|
+
body: event.message.toString(),
|
|
74320
|
+
callback: `Agent/${this.channel.app.agentId}-${(0, import_node_crypto3.randomUUID)()}`
|
|
74321
|
+
});
|
|
74322
|
+
}
|
|
74323
|
+
/**
|
|
74324
|
+
* Assigns the next channel sequence number into MSH.13 when assignSeqNo is on.
|
|
74325
|
+
* The durable path calls this only after the duplicate check, so retransmits
|
|
74326
|
+
* don't consume a sequence number; the legacy path calls it on every message.
|
|
74327
|
+
* @param event - The inbound message event whose MSH.13 to set.
|
|
74328
|
+
* @param msgControlId - MSH.10, for logging only.
|
|
74329
|
+
*/
|
|
74330
|
+
maybeAssignSeqNo(event, msgControlId) {
|
|
74331
|
+
if (!this.channel.shouldAssignSeqNo()) {
|
|
74332
|
+
return;
|
|
74333
|
+
}
|
|
74334
|
+
const seqNo = this.channel.takeNextSeqNo();
|
|
74335
|
+
event.message.getSegment("MSH")?.setField(13, seqNo.toString());
|
|
74336
|
+
this.channel.channelLog.info(`Setting sequence number for message control ID '${msgControlId ?? "n/a"}': ${seqNo}`);
|
|
74337
|
+
}
|
|
74338
|
+
/**
|
|
74339
|
+
* Parses MSH.13 as a non-negative integer sequence number.
|
|
74340
|
+
* @param event - The inbound message event.
|
|
74341
|
+
* @returns The parsed sequence number, or null when MSH.13 is absent/non-numeric.
|
|
74342
|
+
*/
|
|
74343
|
+
parseSeqNo(event) {
|
|
74344
|
+
const seqNoField = event.message.getSegment("MSH")?.getField(13)?.toString();
|
|
74345
|
+
return seqNoField && /^\d+$/.test(seqNoField) ? Number.parseInt(seqNoField, 10) : null;
|
|
74346
|
+
}
|
|
74347
|
+
/**
|
|
74348
|
+
* Durable inbound path (§8 of DURABLE_QUEUE_ARCHITECTURE.md):
|
|
74349
|
+
*
|
|
74350
|
+
* 1. INSERT a `queued` row.
|
|
74351
|
+
* 2. On success, send CA/AA via the deferred-ack API — only now is the
|
|
74352
|
+
* sender allowed to drop their retransmit buffer.
|
|
74353
|
+
* 3. Notify the channel worker so it picks the row up immediately.
|
|
74354
|
+
*
|
|
74355
|
+
* Duplicate handling — a prior non-`nacked` row already owns this MSH.10
|
|
74356
|
+
* (in any state: queued/processing/processed/errored — see {@link handleDuplicate}):
|
|
74357
|
+
* - `idempotent`: if the re-sent body matches the prior copy byte-for-byte,
|
|
74358
|
+
* replay the prior server response ACK (or, if none yet, the commit ACK);
|
|
74359
|
+
* if it differs, send AR — a different message reused a committed control ID.
|
|
74360
|
+
* - `reject`: send CR (or AR in aaMode); insert a `nacked` audit row.
|
|
74361
|
+
*
|
|
74362
|
+
* Storage failure: send CR with the underlying error in MSA.3; best-effort
|
|
74363
|
+
* insert a `nacked` audit row (which may itself fail if the DB is unwritable —
|
|
74364
|
+
* intentional, not retried).
|
|
74365
|
+
* @param queue - The app-owned durable queue handle.
|
|
74366
|
+
* @param event - The incoming HL7 message event from the underlying connection.
|
|
74367
|
+
* @param msgControlId - MSH.10 of the inbound message (may be undefined for malformed messages).
|
|
74368
|
+
* @param originalMessage - The message exactly as received, before any transformation (for dedup comparison).
|
|
74369
|
+
*/
|
|
74370
|
+
async handleMessageDurable(queue, event, msgControlId, originalMessage) {
|
|
74371
|
+
const conn = this.hl7Connection;
|
|
74372
|
+
const enhancedMode = this.channel.getEnhancedMode();
|
|
74373
|
+
const enhancedModeColumn = enhancedMode ?? null;
|
|
74374
|
+
const channelName = this.channel.getDefinition().name;
|
|
74375
|
+
const msgType = event.message.getSegment("MSH")?.getField(9)?.toString() ?? null;
|
|
74376
|
+
const callbackId = `Agent/${this.channel.app.agentId}-${(0, import_node_crypto3.randomUUID)()}`;
|
|
74377
|
+
const receivedAt = Date.now();
|
|
74378
|
+
const assigning = this.channel.shouldAssignSeqNo();
|
|
74379
|
+
let finalizedMessage = originalMessage;
|
|
74380
|
+
let seqNo = this.parseSeqNo(event);
|
|
74381
|
+
let result;
|
|
74382
|
+
try {
|
|
74383
|
+
result = queue.enqueue(
|
|
74384
|
+
{
|
|
74385
|
+
channelName,
|
|
74386
|
+
remote: this.remote,
|
|
74387
|
+
msgControlId: msgControlId ?? null,
|
|
74388
|
+
msgType,
|
|
74389
|
+
originalMessage,
|
|
74390
|
+
finalizedMessage,
|
|
74391
|
+
encoding: conn.getEncoding() ?? null,
|
|
74392
|
+
enhancedMode: enhancedModeColumn,
|
|
74393
|
+
callbackId,
|
|
74394
|
+
seqNo,
|
|
74395
|
+
receivedAt
|
|
74396
|
+
},
|
|
74397
|
+
assigning ? {
|
|
74398
|
+
assignSeqNo: (candidate) => {
|
|
74399
|
+
event.message.getSegment("MSH")?.setField(13, candidate.toString());
|
|
74400
|
+
this.channel.channelLog.info(
|
|
74401
|
+
`Setting sequence number for message control ID '${msgControlId ?? "n/a"}': ${candidate}`
|
|
74402
|
+
);
|
|
74403
|
+
finalizedMessage = Buffer.from(event.message.toString(), "utf8");
|
|
74404
|
+
seqNo = candidate;
|
|
74405
|
+
return finalizedMessage;
|
|
74406
|
+
}
|
|
74407
|
+
} : void 0
|
|
74408
|
+
);
|
|
74409
|
+
} catch (err2) {
|
|
74410
|
+
const reason = `storage error: ${Re(err2)}`;
|
|
74411
|
+
this.channel.channelLog.error(`Durable enqueue failed for ${msgControlId ?? "no-id"}: ${reason}`);
|
|
74412
|
+
this.sendCommitNack(event.message, enhancedMode === "aaMode" ? "AE" : "CE", "storage error");
|
|
74413
|
+
this.recordImmediateAck(msgControlId);
|
|
74414
|
+
queue.enqueueRejected({
|
|
74415
|
+
channelName,
|
|
74416
|
+
remote: this.remote,
|
|
74417
|
+
msgControlId: msgControlId ?? null,
|
|
74418
|
+
msgType,
|
|
74419
|
+
originalMessage,
|
|
74420
|
+
finalizedMessage,
|
|
74421
|
+
encoding: conn.getEncoding() ?? null,
|
|
74422
|
+
enhancedMode: enhancedModeColumn,
|
|
74423
|
+
callbackId,
|
|
74424
|
+
seqNo,
|
|
74425
|
+
receivedAt,
|
|
74426
|
+
lastError: reason,
|
|
74427
|
+
errorCode: QueueErrorCode.StorageError
|
|
74428
|
+
});
|
|
74429
|
+
return;
|
|
74430
|
+
}
|
|
74431
|
+
if (result.kind === "duplicate") {
|
|
74432
|
+
this.handleDuplicate(queue, event, result.existing, msgControlId, {
|
|
74433
|
+
callbackId,
|
|
74434
|
+
msgType,
|
|
74435
|
+
originalMessage,
|
|
74436
|
+
finalizedMessage,
|
|
74437
|
+
enhancedModeColumn,
|
|
74438
|
+
seqNo,
|
|
74439
|
+
receivedAt
|
|
74440
|
+
});
|
|
74441
|
+
return;
|
|
74442
|
+
}
|
|
74443
|
+
this.sendCommitAck(event.message);
|
|
74444
|
+
this.recordImmediateAck(msgControlId);
|
|
74445
|
+
this.channel.worker?.notify();
|
|
74446
|
+
}
|
|
74447
|
+
/**
|
|
74448
|
+
* Handles an inbound message whose `(channel, MSH.10)` already belongs to a
|
|
74449
|
+
* prior non-`nacked` row (`existing`), per the channel's `duplicateBehavior`:
|
|
74450
|
+
*
|
|
74451
|
+
* - `reject`: reject every collision (CR, or AR in aaMode) and write a
|
|
74452
|
+
* `nacked` audit row — unchanged, now spanning all prior states.
|
|
74453
|
+
* - `idempotent` (default): treat a byte-for-byte re-send as a benign
|
|
74454
|
+
* retransmit and replay the acknowledgment the sender missed —
|
|
74455
|
+
* {@link AgentHl7Channel.sendToRemote the prior server response ACK} if the
|
|
74456
|
+
* message was already dispatched, otherwise the commit ACK (CA/AA). A body
|
|
74457
|
+
* that differs is a *different* message reusing a committed control ID, so
|
|
74458
|
+
* we reject it with AR (and a `nacked` audit row) the same way `reject`
|
|
74459
|
+
* handles any duplicate.
|
|
74460
|
+
*
|
|
74461
|
+
* The replayed ACKs go through {@link sendCommitAck}/{@link sendCommitNack} just
|
|
74462
|
+
* like a fresh message — a retransmit means the sender never saw the original
|
|
74463
|
+
* ACK and must be re-told, and the durable row is the dedup authority.
|
|
74464
|
+
* @param queue - The app-owned durable queue handle.
|
|
74465
|
+
* @param event - The duplicate inbound message event.
|
|
74466
|
+
* @param existing - The prior row that owns this MSH.10.
|
|
74467
|
+
* @param msgControlId - MSH.10 of the inbound message.
|
|
74468
|
+
* @param audit - Decoded fields of the inbound message, for any `nacked` audit row.
|
|
74469
|
+
* @param audit.callbackId - Callback ID minted for this inbound message.
|
|
74470
|
+
* @param audit.msgType - MSH.9 message type.
|
|
74471
|
+
* @param audit.originalMessage - Bytes as received (compared against the prior row's original_message).
|
|
74472
|
+
* @param audit.finalizedMessage - Bytes as transformed for dispatch (persisted on any audit row).
|
|
74473
|
+
* @param audit.enhancedModeColumn - Enhanced-mode column value to persist.
|
|
74474
|
+
* @param audit.seqNo - Sequence number (MSH.13), if any.
|
|
74475
|
+
* @param audit.receivedAt - Intake timestamp (ms).
|
|
74476
|
+
*/
|
|
74477
|
+
handleDuplicate(queue, event, existing, msgControlId, audit) {
|
|
74478
|
+
const conn = this.hl7Connection;
|
|
74479
|
+
const enhancedMode = this.channel.getEnhancedMode();
|
|
74480
|
+
const idLabel = msgControlId ?? "n/a";
|
|
74481
|
+
const behavior = this.channel.getDuplicateBehavior();
|
|
74482
|
+
if (behavior === DuplicateBehavior.IDEMPOTENT && existing.originalMessage.equals(audit.originalMessage)) {
|
|
74483
|
+
if (existing.serverResponseBody && existing.serverResponseBody.length > 0 && this.replayServerAck(existing)) {
|
|
74484
|
+
if (existing.ackOutcome === AckOutcome.UNDELIVERED) {
|
|
74485
|
+
queue.setAckOutcome(existing.id, AckOutcome.DELIVERED);
|
|
74486
|
+
}
|
|
74487
|
+
this.channel.channelLog.info(
|
|
74488
|
+
`[Duplicate idempotent -- ID: ${idLabel}] replayed prior server response ACK from row id=${existing.id}`
|
|
74489
|
+
);
|
|
74490
|
+
return;
|
|
74491
|
+
}
|
|
74492
|
+
this.channel.channelLog.info(
|
|
74493
|
+
`[Duplicate idempotent -- ID: ${idLabel}] replayed commit ACK for prior row id=${existing.id}`
|
|
74494
|
+
);
|
|
74495
|
+
this.sendCommitAck(event.message);
|
|
74496
|
+
this.recordImmediateAck(msgControlId);
|
|
74497
|
+
return;
|
|
74498
|
+
}
|
|
74499
|
+
const contentMismatch = behavior === DuplicateBehavior.IDEMPOTENT;
|
|
74500
|
+
const reason = contentMismatch ? `duplicate control id ${idLabel}: a message with this control ID was already committed with different content` : "duplicate control id";
|
|
74501
|
+
this.channel.channelLog.warn(`[Duplicate rejected -- ID: ${idLabel}] prior row id=${existing.id}: ${reason}`);
|
|
74502
|
+
this.sendCommitNack(event.message, enhancedMode === "aaMode" ? "AR" : "CR", reason);
|
|
74503
|
+
this.recordImmediateAck(msgControlId);
|
|
74504
|
+
queue.enqueueRejected({
|
|
74505
|
+
channelName: this.channel.getDefinition().name,
|
|
74506
|
+
remote: this.remote,
|
|
74507
|
+
msgControlId: msgControlId ?? null,
|
|
74508
|
+
msgType: audit.msgType,
|
|
74509
|
+
originalMessage: audit.originalMessage,
|
|
74510
|
+
finalizedMessage: audit.finalizedMessage,
|
|
74511
|
+
encoding: conn.getEncoding() ?? null,
|
|
74512
|
+
enhancedMode: audit.enhancedModeColumn,
|
|
74513
|
+
callbackId: audit.callbackId,
|
|
74514
|
+
seqNo: audit.seqNo,
|
|
74515
|
+
receivedAt: audit.receivedAt,
|
|
74516
|
+
lastError: reason,
|
|
74517
|
+
errorCode: QueueErrorCode.DuplicateRejected
|
|
74518
|
+
});
|
|
74519
|
+
}
|
|
74520
|
+
/**
|
|
74521
|
+
* Balances {@link ChannelStatsTracker} for a message we answered synchronously
|
|
74522
|
+
* at intake (duplicate replay / reject / storage-error NACK). `handleMessage`
|
|
74523
|
+
* already called `recordMessageSent`; without this the control ID would linger
|
|
74524
|
+
* in the pending map until the 5-minute GC, skewing pendingCount and RTT.
|
|
74525
|
+
* @param msgControlId - MSH.10 of the message just answered, if present.
|
|
74526
|
+
*/
|
|
74527
|
+
recordImmediateAck(msgControlId) {
|
|
74528
|
+
if (msgControlId) {
|
|
74529
|
+
this.channel.stats.recordAckReceived(msgControlId);
|
|
74530
|
+
}
|
|
74531
|
+
}
|
|
74532
|
+
/**
|
|
74533
|
+
* Replays the stored server response ACK for an already-dispatched duplicate,
|
|
74534
|
+
* routing it through {@link AgentHl7Channel.sendToRemote} so the same
|
|
74535
|
+
* app-level ACK policy and encoding apply as on the original delivery.
|
|
74536
|
+
* @param existing - The prior row whose `serverResponseBody` to replay.
|
|
74537
|
+
* @returns True if the response ACK was delivered; false if it couldn't be parsed/sent.
|
|
74538
|
+
*/
|
|
74539
|
+
replayServerAck(existing) {
|
|
74540
|
+
if (!existing.serverResponseBody) {
|
|
74541
|
+
return false;
|
|
74542
|
+
}
|
|
74543
|
+
try {
|
|
74544
|
+
return this.channel.sendToRemote({
|
|
74545
|
+
type: "agent:transmit:response",
|
|
74546
|
+
channel: this.channel.getDefinition().name,
|
|
74547
|
+
remote: this.remote,
|
|
74548
|
+
contentType: k.HL7_V2,
|
|
74549
|
+
body: existing.serverResponseBody.toString("utf8"),
|
|
74550
|
+
callback: existing.callbackId
|
|
74551
|
+
});
|
|
74552
|
+
} catch (err2) {
|
|
74553
|
+
this.channel.channelLog.warn(
|
|
74554
|
+
`[Duplicate idempotent] failed to replay server response ACK for row id=${existing.id}: ${Re(err2)}`
|
|
74555
|
+
);
|
|
74556
|
+
return false;
|
|
74557
|
+
}
|
|
74558
|
+
}
|
|
73823
74559
|
async handleError(event) {
|
|
73824
74560
|
this.channel.log.error(`HL7 connection error: ${Re(event.error)}`);
|
|
73825
74561
|
this.channel.channelLog.error(`HL7 connection error: ${Re(event.error)}`);
|
|
73826
74562
|
}
|
|
74563
|
+
/**
|
|
74564
|
+
* Sends the commit ACK (CA in `standard` enhanced mode, AA in `aaMode`) for an
|
|
74565
|
+
* inbound message that has been durably committed. No-op outside enhanced mode.
|
|
74566
|
+
*
|
|
74567
|
+
* This is the durable path's replacement for the connection's synchronous
|
|
74568
|
+
* auto-ACK: in durable mode the connection carries no enhancedMode (so it never
|
|
74569
|
+
* auto-ACKs), and the agent calls this only after the DB write succeeds — so the
|
|
74570
|
+
* CA/AA is a real promise that the message is on disk. Idempotency is not enforced
|
|
74571
|
+
* here; the on-disk row (channel + MSH.10) is the dedup authority, and a genuine
|
|
74572
|
+
* retransmit must be re-ACKed because the sender never saw the original.
|
|
74573
|
+
* @param message - The original inbound message to ACK.
|
|
74574
|
+
*/
|
|
74575
|
+
sendCommitAck(message) {
|
|
74576
|
+
const enhancedMode = this.channel.getEnhancedMode();
|
|
74577
|
+
if (!enhancedMode) {
|
|
74578
|
+
return;
|
|
74579
|
+
}
|
|
74580
|
+
const ackCode = enhancedMode === "standard" ? "CA" : "AA";
|
|
74581
|
+
const response2 = message.buildAck({ ackCode });
|
|
74582
|
+
this.hl7Connection.send(response2);
|
|
74583
|
+
this.hl7Connection.dispatchEvent(new E(this.hl7Connection, response2));
|
|
74584
|
+
}
|
|
74585
|
+
/**
|
|
74586
|
+
* Sends a negative commit ACK for an inbound message. The wire code is CE/CR in
|
|
74587
|
+
* `standard` enhanced mode and AE/AR in `aaMode` (error = retryable, reject =
|
|
74588
|
+
* terminal); the caller picks which best describes the failure. An optional
|
|
74589
|
+
* `reason` is written to MSA.3 for the sender's logs. No-op outside enhanced mode.
|
|
74590
|
+
* @param message - The original inbound message to NACK.
|
|
74591
|
+
* @param code - The negative ACK code to send.
|
|
74592
|
+
* @param reason - Optional human-readable explanation placed in MSA.3.
|
|
74593
|
+
*/
|
|
74594
|
+
sendCommitNack(message, code, reason) {
|
|
74595
|
+
const enhancedMode = this.channel.getEnhancedMode();
|
|
74596
|
+
if (!enhancedMode) {
|
|
74597
|
+
return;
|
|
74598
|
+
}
|
|
74599
|
+
const response2 = message.buildAck({ ackCode: code });
|
|
74600
|
+
if (reason) {
|
|
74601
|
+
response2.getSegment("MSA")?.setField(3, reason);
|
|
74602
|
+
}
|
|
74603
|
+
this.hl7Connection.send(response2);
|
|
74604
|
+
this.hl7Connection.dispatchEvent(new E(this.hl7Connection, response2));
|
|
74605
|
+
}
|
|
73827
74606
|
handleEnhancedAckSent(event) {
|
|
73828
74607
|
const hl7Message = event.message;
|
|
73829
74608
|
const msgControlId = hl7Message.getSegment("MSA")?.getField(2)?.toString();
|
|
73830
74609
|
const ackCode = hl7Message.getSegment("MSA")?.getField(1)?.toString()?.toUpperCase();
|
|
73831
74610
|
this.channel.channelLog.info(
|
|
73832
|
-
`[Sent ${ackCode
|
|
74611
|
+
`[Sent ${describeAckCode(ackCode)} -- ID: ${msgControlId ?? "not provided"}]: ${hl7Message.toString().replaceAll("\r", "\n")}`
|
|
73833
74612
|
);
|
|
73834
74613
|
}
|
|
73835
74614
|
close() {
|
|
73836
74615
|
return this.hl7Connection.close();
|
|
73837
74616
|
}
|
|
73838
74617
|
};
|
|
74618
|
+
function describeAckCode(code) {
|
|
74619
|
+
switch (code) {
|
|
74620
|
+
case "CA":
|
|
74621
|
+
return "Commit ACK (CA)";
|
|
74622
|
+
case "AA":
|
|
74623
|
+
return "App ACK (AA)";
|
|
74624
|
+
case "CE":
|
|
74625
|
+
return "Commit Error (CE)";
|
|
74626
|
+
case "CR":
|
|
74627
|
+
return "Commit Reject (CR)";
|
|
74628
|
+
case "AE":
|
|
74629
|
+
return "App Error (AE)";
|
|
74630
|
+
case "AR":
|
|
74631
|
+
return "App Reject (AR)";
|
|
74632
|
+
default:
|
|
74633
|
+
return `ACK (${code ?? "unknown"})`;
|
|
74634
|
+
}
|
|
74635
|
+
}
|
|
73839
74636
|
function parseEnhancedMode(rawValue, logger) {
|
|
73840
74637
|
if (!rawValue) {
|
|
73841
74638
|
return void 0;
|
|
@@ -73869,6 +74666,17 @@ function isAppLevelAckCode(code) {
|
|
|
73869
74666
|
function isAppLevelAckMode(candidate) {
|
|
73870
74667
|
return APP_LEVEL_ACK_MODES.includes(candidate);
|
|
73871
74668
|
}
|
|
74669
|
+
function parseDuplicateBehavior(rawValue, logger) {
|
|
74670
|
+
if (!rawValue) {
|
|
74671
|
+
return DuplicateBehavior.IDEMPOTENT;
|
|
74672
|
+
}
|
|
74673
|
+
const normalized = rawValue.toLowerCase();
|
|
74674
|
+
if (normalized === DuplicateBehavior.REJECT || normalized === DuplicateBehavior.IDEMPOTENT) {
|
|
74675
|
+
return normalized;
|
|
74676
|
+
}
|
|
74677
|
+
logger.warn(`Invalid duplicateBehavior value '${rawValue}'; expected 'reject' or 'idempotent'. Using idempotent.`);
|
|
74678
|
+
return DuplicateBehavior.IDEMPOTENT;
|
|
74679
|
+
}
|
|
73872
74680
|
function shouldSendAppLevelAck(options) {
|
|
73873
74681
|
const { mode, ackCode, enhancedMode } = options;
|
|
73874
74682
|
if (!enhancedMode) {
|
|
@@ -74641,8 +75449,1355 @@ function registerAgentCleanup() {
|
|
|
74641
75449
|
}
|
|
74642
75450
|
}
|
|
74643
75451
|
|
|
74644
|
-
// src/
|
|
75452
|
+
// src/queue/durable-queue.ts
|
|
75453
|
+
var import_node_crypto4 = require("node:crypto");
|
|
74645
75454
|
var import_node_fs3 = require("node:fs");
|
|
75455
|
+
|
|
75456
|
+
// src/queue/queries.ts
|
|
75457
|
+
var ENQUEUE = `
|
|
75458
|
+
INSERT INTO inbound_hl7_messages (
|
|
75459
|
+
channel_name, remote, msg_control_id, msg_type, original_message, finalized_message, encoding,
|
|
75460
|
+
enhanced_mode, state, attempt_count, callback_id,
|
|
75461
|
+
seq_no, received_at
|
|
75462
|
+
) VALUES (
|
|
75463
|
+
?, ?, ?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?, ?
|
|
75464
|
+
)
|
|
75465
|
+
`;
|
|
75466
|
+
var ENQUEUE_REJECTED = `
|
|
75467
|
+
INSERT INTO inbound_hl7_messages (
|
|
75468
|
+
channel_name, remote, msg_control_id, msg_type, original_message, finalized_message, encoding,
|
|
75469
|
+
enhanced_mode, state, attempt_count, callback_id,
|
|
75470
|
+
ack_outcome, last_error, error_code, seq_no, received_at
|
|
75471
|
+
) VALUES (
|
|
75472
|
+
?, ?, ?, ?, ?, ?, ?, ?, 'nacked', 0, ?, 'not_owed', ?, ?, ?, ?
|
|
75473
|
+
)
|
|
75474
|
+
`;
|
|
75475
|
+
var PEEK_LAST_SEQ_NO = `
|
|
75476
|
+
SELECT last_seq_no FROM _channel_seq WHERE channel_name = ?
|
|
75477
|
+
`;
|
|
75478
|
+
var COMMIT_SEQ_NO = `
|
|
75479
|
+
INSERT INTO _channel_seq (channel_name, last_seq_no) VALUES (?, ?)
|
|
75480
|
+
ON CONFLICT(channel_name) DO UPDATE SET last_seq_no = excluded.last_seq_no
|
|
75481
|
+
`;
|
|
75482
|
+
var FIND_SEEN_BY_CONTROL_ID = `
|
|
75483
|
+
SELECT * FROM inbound_hl7_messages
|
|
75484
|
+
WHERE channel_name = ?
|
|
75485
|
+
AND msg_control_id = ?
|
|
75486
|
+
AND state != 'nacked'
|
|
75487
|
+
ORDER BY id DESC
|
|
75488
|
+
LIMIT 1
|
|
75489
|
+
`;
|
|
75490
|
+
var CLAIM_NEXT = `
|
|
75491
|
+
UPDATE inbound_hl7_messages
|
|
75492
|
+
SET state = 'claimed',
|
|
75493
|
+
processing_started_at = ?,
|
|
75494
|
+
attempt_count = attempt_count + 1
|
|
75495
|
+
WHERE id = (
|
|
75496
|
+
SELECT id FROM inbound_hl7_messages
|
|
75497
|
+
WHERE channel_name = ? AND state = 'queued'
|
|
75498
|
+
ORDER BY id ASC
|
|
75499
|
+
LIMIT 1
|
|
75500
|
+
)
|
|
75501
|
+
RETURNING *
|
|
75502
|
+
`;
|
|
75503
|
+
var MARK_SENT = `
|
|
75504
|
+
UPDATE inbound_hl7_messages
|
|
75505
|
+
SET state = 'inflight',
|
|
75506
|
+
sent_at = ?
|
|
75507
|
+
WHERE callback_id = ? AND state = 'claimed'
|
|
75508
|
+
`;
|
|
75509
|
+
var FIND_BY_CALLBACK = `
|
|
75510
|
+
SELECT * FROM inbound_hl7_messages WHERE callback_id = ?
|
|
75511
|
+
`;
|
|
75512
|
+
var FIND_BY_ID = `
|
|
75513
|
+
SELECT * FROM inbound_hl7_messages WHERE id = ?
|
|
75514
|
+
`;
|
|
75515
|
+
var RECORD_SERVER_RESPONSE = `
|
|
75516
|
+
UPDATE inbound_hl7_messages
|
|
75517
|
+
SET server_status_code = ?,
|
|
75518
|
+
server_response_body = ?
|
|
75519
|
+
WHERE id = ?
|
|
75520
|
+
`;
|
|
75521
|
+
var MARK_PROCESSED = `
|
|
75522
|
+
UPDATE inbound_hl7_messages
|
|
75523
|
+
SET state = 'processed',
|
|
75524
|
+
ack_outcome = ?,
|
|
75525
|
+
processed_at = ?
|
|
75526
|
+
WHERE id = ?
|
|
75527
|
+
`;
|
|
75528
|
+
var MARK_BOT_FAILED = `
|
|
75529
|
+
UPDATE inbound_hl7_messages
|
|
75530
|
+
SET state = ?,
|
|
75531
|
+
errored_at = ?,
|
|
75532
|
+
last_error = ?,
|
|
75533
|
+
error_code = ?,
|
|
75534
|
+
ack_outcome = 'not_owed'
|
|
75535
|
+
WHERE id = ?
|
|
75536
|
+
`;
|
|
75537
|
+
var SET_ACK_OUTCOME = `
|
|
75538
|
+
UPDATE inbound_hl7_messages
|
|
75539
|
+
SET ack_outcome = ?
|
|
75540
|
+
WHERE id = ?
|
|
75541
|
+
`;
|
|
75542
|
+
var REQUEUE = `
|
|
75543
|
+
UPDATE inbound_hl7_messages
|
|
75544
|
+
SET state = 'queued',
|
|
75545
|
+
processing_started_at = NULL,
|
|
75546
|
+
attempt_count = MAX(0, attempt_count - 1)
|
|
75547
|
+
WHERE id = ? AND state = 'claimed'
|
|
75548
|
+
`;
|
|
75549
|
+
var LIST_QUEUED_IDS_FOR_CHANNEL = `
|
|
75550
|
+
SELECT id FROM inbound_hl7_messages
|
|
75551
|
+
WHERE channel_name = ? AND state = 'queued'
|
|
75552
|
+
ORDER BY id ASC
|
|
75553
|
+
`;
|
|
75554
|
+
var RECOVER_INFLIGHT = `
|
|
75555
|
+
UPDATE inbound_hl7_messages
|
|
75556
|
+
SET state = 'failed',
|
|
75557
|
+
errored_at = ?,
|
|
75558
|
+
last_error = COALESCE(last_error, 'interrupted: process restart while inflight'),
|
|
75559
|
+
error_code = COALESCE(error_code, '${QueueErrorCode.Interrupted}')
|
|
75560
|
+
WHERE state = 'inflight'
|
|
75561
|
+
`;
|
|
75562
|
+
var RECOVER_CLAIMED = `
|
|
75563
|
+
UPDATE inbound_hl7_messages
|
|
75564
|
+
SET state = 'queued',
|
|
75565
|
+
processing_started_at = NULL,
|
|
75566
|
+
attempt_count = MAX(0, attempt_count - 1)
|
|
75567
|
+
WHERE state = 'claimed'
|
|
75568
|
+
`;
|
|
75569
|
+
var COUNT_BY_STATE = `
|
|
75570
|
+
SELECT state, COUNT(*) AS n FROM inbound_hl7_messages GROUP BY state
|
|
75571
|
+
`;
|
|
75572
|
+
var CHANNEL_DEPTH = `
|
|
75573
|
+
SELECT
|
|
75574
|
+
SUM(state = 'queued') AS queued,
|
|
75575
|
+
SUM(state = 'claimed') AS claimed,
|
|
75576
|
+
SUM(state = 'inflight') AS inflight,
|
|
75577
|
+
MIN(CASE WHEN state = 'queued' THEN received_at ELSE NULL END) AS oldest_queued_at
|
|
75578
|
+
FROM inbound_hl7_messages
|
|
75579
|
+
WHERE channel_name = ?
|
|
75580
|
+
`;
|
|
75581
|
+
var DB_SIZE_BYTES = `
|
|
75582
|
+
SELECT page_count * page_size AS bytes FROM pragma_page_count, pragma_page_size
|
|
75583
|
+
`;
|
|
75584
|
+
var TRY_ACQUIRE_LEASE = `
|
|
75585
|
+
INSERT INTO _lease (id, holder, acquired_at, expires_at)
|
|
75586
|
+
VALUES (1, ?, ?, ?)
|
|
75587
|
+
ON CONFLICT(id) DO UPDATE
|
|
75588
|
+
SET holder = excluded.holder,
|
|
75589
|
+
acquired_at = excluded.acquired_at,
|
|
75590
|
+
expires_at = excluded.expires_at
|
|
75591
|
+
WHERE _lease.holder = ? OR _lease.expires_at <= ?
|
|
75592
|
+
`;
|
|
75593
|
+
var HEARTBEAT_LEASE = `
|
|
75594
|
+
UPDATE _lease SET expires_at = ? WHERE id = 1 AND holder = ?
|
|
75595
|
+
`;
|
|
75596
|
+
var RELEASE_LEASE = `
|
|
75597
|
+
DELETE FROM _lease WHERE id = 1 AND holder = ?
|
|
75598
|
+
`;
|
|
75599
|
+
var GET_LEASE = `
|
|
75600
|
+
SELECT holder, expires_at FROM _lease WHERE id = 1
|
|
75601
|
+
`;
|
|
75602
|
+
var CHECKPOINT_WAL = `PRAGMA wal_checkpoint(TRUNCATE)`;
|
|
75603
|
+
var RETENTION_PHASE1_DELETE = `
|
|
75604
|
+
DELETE FROM inbound_hl7_messages
|
|
75605
|
+
WHERE state = 'processed' AND ack_outcome != 'undelivered' AND processed_at < ?
|
|
75606
|
+
`;
|
|
75607
|
+
var RETENTION_PHASE2_DELETE = `
|
|
75608
|
+
DELETE FROM inbound_hl7_messages
|
|
75609
|
+
WHERE id IN (
|
|
75610
|
+
SELECT id FROM inbound_hl7_messages
|
|
75611
|
+
WHERE state = 'processed' AND ack_outcome != 'undelivered'
|
|
75612
|
+
ORDER BY processed_at ASC
|
|
75613
|
+
LIMIT ?
|
|
75614
|
+
)
|
|
75615
|
+
`;
|
|
75616
|
+
var RETENTION_PHASE3_DELETE = `
|
|
75617
|
+
DELETE FROM inbound_hl7_messages
|
|
75618
|
+
WHERE id IN (
|
|
75619
|
+
SELECT id FROM inbound_hl7_messages
|
|
75620
|
+
WHERE (
|
|
75621
|
+
state IN ('rejected', 'failed', 'nacked')
|
|
75622
|
+
OR (state = 'processed' AND ack_outcome = 'undelivered')
|
|
75623
|
+
)
|
|
75624
|
+
AND COALESCE(errored_at, processed_at) < ?
|
|
75625
|
+
ORDER BY COALESCE(errored_at, processed_at) ASC
|
|
75626
|
+
LIMIT ?
|
|
75627
|
+
)
|
|
75628
|
+
`;
|
|
75629
|
+
|
|
75630
|
+
// src/queue/schema.ts
|
|
75631
|
+
var MIGRATIONS = [
|
|
75632
|
+
{
|
|
75633
|
+
version: 1,
|
|
75634
|
+
sql: `
|
|
75635
|
+
CREATE TABLE IF NOT EXISTS _schema (
|
|
75636
|
+
version INTEGER PRIMARY KEY,
|
|
75637
|
+
applied_at INTEGER NOT NULL
|
|
75638
|
+
);
|
|
75639
|
+
|
|
75640
|
+
CREATE TABLE IF NOT EXISTS inbound_hl7_messages (
|
|
75641
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
75642
|
+
channel_name TEXT NOT NULL,
|
|
75643
|
+
remote TEXT NOT NULL,
|
|
75644
|
+
msg_control_id TEXT,
|
|
75645
|
+
msg_type TEXT,
|
|
75646
|
+
-- original_message is the message exactly as received, used for the
|
|
75647
|
+
-- intake duplicate-content comparison. finalized_message is what the
|
|
75648
|
+
-- worker dispatches upstream; it differs from original only when the
|
|
75649
|
+
-- channel rewrites the message (e.g. assignSeqNo sets MSH.13). When no
|
|
75650
|
+
-- transformation applies the two are byte-identical.
|
|
75651
|
+
original_message BLOB NOT NULL,
|
|
75652
|
+
finalized_message BLOB NOT NULL,
|
|
75653
|
+
encoding TEXT,
|
|
75654
|
+
enhanced_mode TEXT,
|
|
75655
|
+
-- state tracks the Bot leg: queued \u2192 claimed (worker owns it, request not
|
|
75656
|
+
-- yet on the wire) \u2192 inflight (request written to the socket, sent_at
|
|
75657
|
+
-- stamped, awaiting response) \u2192 processed/rejected/failed, plus the
|
|
75658
|
+
-- intake-reject 'nacked'. The claimed/inflight split is what lets crash
|
|
75659
|
+
-- recovery tell a provably-unsent row (safe to requeue) from an ambiguous
|
|
75660
|
+
-- in-flight one (must fail for review). The source-leg ACK-delivery outcome
|
|
75661
|
+
-- is tracked independently in ack_outcome
|
|
75662
|
+
-- (pending/delivered/undelivered/not_owed) so the two legs never conflate.
|
|
75663
|
+
state TEXT NOT NULL,
|
|
75664
|
+
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
75665
|
+
callback_id TEXT NOT NULL,
|
|
75666
|
+
server_response_body BLOB,
|
|
75667
|
+
server_status_code INTEGER,
|
|
75668
|
+
ack_outcome TEXT NOT NULL DEFAULT 'pending',
|
|
75669
|
+
last_error TEXT,
|
|
75670
|
+
error_code TEXT,
|
|
75671
|
+
seq_no INTEGER,
|
|
75672
|
+
received_at INTEGER NOT NULL,
|
|
75673
|
+
-- processing_started_at is stamped when the worker claims the row (enters
|
|
75674
|
+
-- the claimed state); sent_at is stamped when the request is written to the
|
|
75675
|
+
-- socket (enters inflight). A NULL sent_at on a claimed row is the durable
|
|
75676
|
+
-- discriminator crash recovery uses to know the request never left.
|
|
75677
|
+
processing_started_at INTEGER,
|
|
75678
|
+
sent_at INTEGER,
|
|
75679
|
+
processed_at INTEGER,
|
|
75680
|
+
errored_at INTEGER
|
|
75681
|
+
) STRICT;
|
|
75682
|
+
|
|
75683
|
+
CREATE INDEX IF NOT EXISTS idx_inbound_channel_state_id
|
|
75684
|
+
ON inbound_hl7_messages (channel_name, state, id);
|
|
75685
|
+
|
|
75686
|
+
-- Also serves the retention phase-3 sweep: its (state IN ... OR ...)
|
|
75687
|
+
-- predicate resolves to a MULTI-INDEX OR over this index's leading state
|
|
75688
|
+
-- column, so the sweep filters by index rather than full-scanning. A
|
|
75689
|
+
-- dedicated COALESCE(errored_at, processed_at) expression index was
|
|
75690
|
+
-- evaluated to also satisfy phase 3's ORDER BY, but SQLite's partial-index
|
|
75691
|
+
-- prover can't match that disjunctive predicate (so a partial index goes
|
|
75692
|
+
-- unused), and a non-partial one would key every queued row as NULL and tax
|
|
75693
|
+
-- the hot intake path -- not worth it for an hourly, size-gated sweep.
|
|
75694
|
+
CREATE INDEX IF NOT EXISTS idx_inbound_state_processed_at
|
|
75695
|
+
ON inbound_hl7_messages (state, processed_at);
|
|
75696
|
+
|
|
75697
|
+
CREATE UNIQUE INDEX IF NOT EXISTS uq_inbound_dup_active
|
|
75698
|
+
ON inbound_hl7_messages (channel_name, msg_control_id)
|
|
75699
|
+
WHERE msg_control_id IS NOT NULL
|
|
75700
|
+
AND state IN ('queued', 'claimed', 'inflight');
|
|
75701
|
+
|
|
75702
|
+
-- Intake dedup reads the most recent prior row for a (channel, control_id)
|
|
75703
|
+
-- across ALL non-nacked states (uq_inbound_dup_active only covers the
|
|
75704
|
+
-- active window). This runs once per inbound message that carries an
|
|
75705
|
+
-- MSH.10, so without it the lookup scans every row for the channel. The
|
|
75706
|
+
-- trailing id makes ORDER BY id DESC LIMIT 1 a reverse index seek; the
|
|
75707
|
+
-- partial predicate matches the query so there's no residual recheck.
|
|
75708
|
+
CREATE INDEX IF NOT EXISTS idx_inbound_dup_lookup
|
|
75709
|
+
ON inbound_hl7_messages (channel_name, msg_control_id, id)
|
|
75710
|
+
WHERE msg_control_id IS NOT NULL
|
|
75711
|
+
AND state != 'nacked';
|
|
75712
|
+
|
|
75713
|
+
CREATE UNIQUE INDEX IF NOT EXISTS uq_inbound_callback
|
|
75714
|
+
ON inbound_hl7_messages (callback_id);
|
|
75715
|
+
|
|
75716
|
+
-- Single-row table holding the current queue lease, used for coordinating
|
|
75717
|
+
-- the active queue worker across zero-downtime agent upgrades (two
|
|
75718
|
+
-- processes share the DB for the upgrade overlap window; only the
|
|
75719
|
+
-- leaseholder drains rows and runs recovery). The CHECK constraint keeps
|
|
75720
|
+
-- it strictly single-row.
|
|
75721
|
+
CREATE TABLE IF NOT EXISTS _lease (
|
|
75722
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
75723
|
+
holder TEXT NOT NULL,
|
|
75724
|
+
acquired_at INTEGER NOT NULL,
|
|
75725
|
+
expires_at INTEGER NOT NULL
|
|
75726
|
+
) STRICT;
|
|
75727
|
+
|
|
75728
|
+
-- Per-channel monotonic sequence counter for the assignSeqNo feature.
|
|
75729
|
+
-- Persisting it here (rather than only in memory) keeps MSH.13 sequence
|
|
75730
|
+
-- numbers monotonic across agent restarts. last_seq_no holds the most
|
|
75731
|
+
-- recently assigned value; the next assignment is last_seq_no + 1.
|
|
75732
|
+
CREATE TABLE IF NOT EXISTS _channel_seq (
|
|
75733
|
+
channel_name TEXT PRIMARY KEY,
|
|
75734
|
+
last_seq_no INTEGER NOT NULL
|
|
75735
|
+
) STRICT;
|
|
75736
|
+
`
|
|
75737
|
+
}
|
|
75738
|
+
];
|
|
75739
|
+
function runMigrations(db) {
|
|
75740
|
+
db.exec(`
|
|
75741
|
+
CREATE TABLE IF NOT EXISTS _schema (
|
|
75742
|
+
version INTEGER PRIMARY KEY,
|
|
75743
|
+
applied_at INTEGER NOT NULL
|
|
75744
|
+
);
|
|
75745
|
+
`);
|
|
75746
|
+
const currentRow = db.prepare("SELECT MAX(version) AS v FROM _schema").get();
|
|
75747
|
+
let current = currentRow?.v ?? 0;
|
|
75748
|
+
const insertVersion = db.prepare("INSERT INTO _schema (version, applied_at) VALUES (?, ?)");
|
|
75749
|
+
for (const migration of MIGRATIONS) {
|
|
75750
|
+
if (migration.version <= current) {
|
|
75751
|
+
continue;
|
|
75752
|
+
}
|
|
75753
|
+
db.exec("BEGIN");
|
|
75754
|
+
try {
|
|
75755
|
+
db.exec(migration.sql);
|
|
75756
|
+
insertVersion.run(migration.version, Date.now());
|
|
75757
|
+
db.exec("COMMIT");
|
|
75758
|
+
current = migration.version;
|
|
75759
|
+
} catch (err2) {
|
|
75760
|
+
db.exec("ROLLBACK");
|
|
75761
|
+
throw err2;
|
|
75762
|
+
}
|
|
75763
|
+
}
|
|
75764
|
+
return current;
|
|
75765
|
+
}
|
|
75766
|
+
|
|
75767
|
+
// src/queue/durable-queue.ts
|
|
75768
|
+
var DEFAULT_LEASE_TTL_MS = 3e4;
|
|
75769
|
+
var DEFAULT_LEASE_HEARTBEAT_MS = 1e4;
|
|
75770
|
+
var DEFAULT_LEASE_ACQUIRE_RETRY_MS = 2e3;
|
|
75771
|
+
var DEFAULT_CHECKPOINT_INTERVAL_MS = 1e4;
|
|
75772
|
+
var DurableQueue = class _DurableQueue {
|
|
75773
|
+
constructor(db, options) {
|
|
75774
|
+
__publicField(this, "db");
|
|
75775
|
+
__publicField(this, "log");
|
|
75776
|
+
__publicField(this, "path");
|
|
75777
|
+
__publicField(this, "closed", false);
|
|
75778
|
+
// The dispatch-lease holder ID of THIS process, set when startDispatchLease()
|
|
75779
|
+
// begins the acquire/heartbeat loop (or bound directly via setLeaseHolder in
|
|
75780
|
+
// tests). Dispatch-class mutations are gated on the live lease still belonging
|
|
75781
|
+
// to this holder; when a peer takes over, the gate throws QueueLeaseError so the
|
|
75782
|
+
// demoted worker stops driving the queue. Undefined when the dispatch lease was
|
|
75783
|
+
// never started (single-process use and most unit tests), in which case the gate
|
|
75784
|
+
// is inert because no lease row exists.
|
|
75785
|
+
__publicField(this, "leaseHolderId");
|
|
75786
|
+
// ── Dispatch-lease orchestration (formerly DispatchLeaseManager) ──
|
|
75787
|
+
// Lightweight leader election for the DISPATCH path. The lease gates *dispatch
|
|
75788
|
+
// only* — claiming rows and driving them to the server (the worker +
|
|
75789
|
+
// recoverOnStartup). It deliberately does NOT gate intake (followers still
|
|
75790
|
+
// persist inbound messages via enqueue), maintenance (WAL checkpoint, retention
|
|
75791
|
+
// sweep), or diagnostics (stats): those are valid on any process with the queue
|
|
75792
|
+
// file open. Exactly one process may dispatch at a time, which is the
|
|
75793
|
+
// load-bearing primitive that makes zero-downtime upgrades safe: during the
|
|
75794
|
+
// overlap window where the old and new agent processes both have the SQLite file
|
|
75795
|
+
// open, only the leaseholder claims/sends rows; the non-leader keeps accepting
|
|
75796
|
+
// and persisting inbound traffic but does not dispatch.
|
|
75797
|
+
//
|
|
75798
|
+
// There is deliberately NO lost-leadership callback: loss is enforced at the
|
|
75799
|
+
// data layer instead — the dispatch ops throw QueueLeaseError once a peer holds
|
|
75800
|
+
// the lease (see assertNotDemoted), so the worker self-detects and drains
|
|
75801
|
+
// without pushing an event down a chain of callbacks.
|
|
75802
|
+
__publicField(this, "leaseTtlMs");
|
|
75803
|
+
__publicField(this, "leaseHeartbeatMs");
|
|
75804
|
+
__publicField(this, "leaseAcquireRetryMs");
|
|
75805
|
+
__publicField(this, "leaseLeader", false);
|
|
75806
|
+
__publicField(this, "leaseLoopActive", false);
|
|
75807
|
+
__publicField(this, "leaseAcquireTimer");
|
|
75808
|
+
__publicField(this, "leaseHeartbeatTimer");
|
|
75809
|
+
__publicField(this, "onBecameLeader");
|
|
75810
|
+
// True when the WAL may contain frames not yet checkpointed into the main DB
|
|
75811
|
+
// file. SQLite only attempts checkpoints piggybacked on commits, so once
|
|
75812
|
+
// traffic stops nothing would ever drain the WAL — the queue's own checkpoint
|
|
75813
|
+
// loop (startCheckpointLoop) polls this via checkpointWalIfDirty(). Starts true
|
|
75814
|
+
// because open() itself writes (pragmas, migrations, lease).
|
|
75815
|
+
__publicField(this, "walDirty", true);
|
|
75816
|
+
// WAL-checkpoint loop. Runs while the queue is open, INDEPENDENT of the
|
|
75817
|
+
// dispatch lease: a follower still accepts intake writes (enqueue runs on any
|
|
75818
|
+
// process with the file open), so WAL draining can't be gated on leadership the
|
|
75819
|
+
// way the lease timers are. Started in open(), cleared in close().
|
|
75820
|
+
__publicField(this, "checkpointIntervalMs");
|
|
75821
|
+
__publicField(this, "checkpointTimer");
|
|
75822
|
+
// Prepared statements — created once at open(), reused for every call.
|
|
75823
|
+
// Names mirror the public methods that use them.
|
|
75824
|
+
__publicField(this, "enqueueStmt");
|
|
75825
|
+
__publicField(this, "enqueueRejectedStmt");
|
|
75826
|
+
__publicField(this, "peekLastSeqNoStmt");
|
|
75827
|
+
__publicField(this, "commitSeqNoStmt");
|
|
75828
|
+
__publicField(this, "findSeenByControlIdStmt");
|
|
75829
|
+
__publicField(this, "claimNextStmt");
|
|
75830
|
+
__publicField(this, "markSentStmt");
|
|
75831
|
+
__publicField(this, "findByCallbackStmt");
|
|
75832
|
+
__publicField(this, "findByIdStmt");
|
|
75833
|
+
__publicField(this, "recordServerResponseStmt");
|
|
75834
|
+
__publicField(this, "markProcessedStmt");
|
|
75835
|
+
__publicField(this, "markBotFailedStmt");
|
|
75836
|
+
__publicField(this, "setAckOutcomeStmt");
|
|
75837
|
+
__publicField(this, "requeueStmt");
|
|
75838
|
+
__publicField(this, "recoverInflightStmt");
|
|
75839
|
+
__publicField(this, "recoverClaimedStmt");
|
|
75840
|
+
__publicField(this, "listQueuedIdsForChannelStmt");
|
|
75841
|
+
__publicField(this, "countByStateStmt");
|
|
75842
|
+
__publicField(this, "channelDepthStmt");
|
|
75843
|
+
__publicField(this, "tryAcquireLeaseStmt");
|
|
75844
|
+
__publicField(this, "heartbeatLeaseStmt");
|
|
75845
|
+
__publicField(this, "releaseLeaseStmt");
|
|
75846
|
+
__publicField(this, "getLeaseStmt");
|
|
75847
|
+
__publicField(this, "checkpointStmt");
|
|
75848
|
+
__publicField(this, "dbSizeBytesStmt");
|
|
75849
|
+
this.db = db;
|
|
75850
|
+
this.log = options.log;
|
|
75851
|
+
this.path = options.path;
|
|
75852
|
+
this.leaseHolderId = options.leaseHolder;
|
|
75853
|
+
this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
|
|
75854
|
+
this.leaseHeartbeatMs = options.leaseHeartbeatMs ?? DEFAULT_LEASE_HEARTBEAT_MS;
|
|
75855
|
+
this.leaseAcquireRetryMs = options.leaseAcquireRetryMs ?? DEFAULT_LEASE_ACQUIRE_RETRY_MS;
|
|
75856
|
+
this.checkpointIntervalMs = options.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
75857
|
+
this.db.exec(`
|
|
75858
|
+
PRAGMA journal_mode = WAL;
|
|
75859
|
+
PRAGMA synchronous = NORMAL;
|
|
75860
|
+
PRAGMA temp_store = MEMORY;
|
|
75861
|
+
PRAGMA cache_size = -65536;
|
|
75862
|
+
PRAGMA mmap_size = 268435456;
|
|
75863
|
+
PRAGMA wal_autocheckpoint = 1000;
|
|
75864
|
+
PRAGMA busy_timeout = 5000;
|
|
75865
|
+
PRAGMA foreign_keys = ON;
|
|
75866
|
+
`);
|
|
75867
|
+
runMigrations(this.db);
|
|
75868
|
+
this.enqueueStmt = this.db.prepare(ENQUEUE);
|
|
75869
|
+
this.enqueueRejectedStmt = this.db.prepare(ENQUEUE_REJECTED);
|
|
75870
|
+
this.peekLastSeqNoStmt = this.db.prepare(PEEK_LAST_SEQ_NO);
|
|
75871
|
+
this.commitSeqNoStmt = this.db.prepare(COMMIT_SEQ_NO);
|
|
75872
|
+
this.findSeenByControlIdStmt = this.db.prepare(FIND_SEEN_BY_CONTROL_ID);
|
|
75873
|
+
this.claimNextStmt = this.db.prepare(CLAIM_NEXT);
|
|
75874
|
+
this.markSentStmt = this.db.prepare(MARK_SENT);
|
|
75875
|
+
this.findByCallbackStmt = this.db.prepare(FIND_BY_CALLBACK);
|
|
75876
|
+
this.findByIdStmt = this.db.prepare(FIND_BY_ID);
|
|
75877
|
+
this.recordServerResponseStmt = this.db.prepare(RECORD_SERVER_RESPONSE);
|
|
75878
|
+
this.markProcessedStmt = this.db.prepare(MARK_PROCESSED);
|
|
75879
|
+
this.markBotFailedStmt = this.db.prepare(MARK_BOT_FAILED);
|
|
75880
|
+
this.setAckOutcomeStmt = this.db.prepare(SET_ACK_OUTCOME);
|
|
75881
|
+
this.requeueStmt = this.db.prepare(REQUEUE);
|
|
75882
|
+
this.recoverInflightStmt = this.db.prepare(RECOVER_INFLIGHT);
|
|
75883
|
+
this.recoverClaimedStmt = this.db.prepare(RECOVER_CLAIMED);
|
|
75884
|
+
this.listQueuedIdsForChannelStmt = this.db.prepare(LIST_QUEUED_IDS_FOR_CHANNEL);
|
|
75885
|
+
this.countByStateStmt = this.db.prepare(COUNT_BY_STATE);
|
|
75886
|
+
this.channelDepthStmt = this.db.prepare(CHANNEL_DEPTH);
|
|
75887
|
+
this.tryAcquireLeaseStmt = this.db.prepare(TRY_ACQUIRE_LEASE);
|
|
75888
|
+
this.heartbeatLeaseStmt = this.db.prepare(HEARTBEAT_LEASE);
|
|
75889
|
+
this.releaseLeaseStmt = this.db.prepare(RELEASE_LEASE);
|
|
75890
|
+
this.getLeaseStmt = this.db.prepare(GET_LEASE);
|
|
75891
|
+
this.checkpointStmt = this.db.prepare(CHECKPOINT_WAL);
|
|
75892
|
+
this.dbSizeBytesStmt = this.db.prepare(DB_SIZE_BYTES);
|
|
75893
|
+
}
|
|
75894
|
+
/**
|
|
75895
|
+
* Opens (or creates) the DB file at `options.path`, runs migrations, and
|
|
75896
|
+
* returns a ready DurableQueue.
|
|
75897
|
+
*
|
|
75898
|
+
* Constructs synchronously — `node:sqlite` is itself synchronous so there's
|
|
75899
|
+
* no benefit to making this `async`.
|
|
75900
|
+
*
|
|
75901
|
+
* @param options - Path + logger.
|
|
75902
|
+
* @returns The opened DurableQueue.
|
|
75903
|
+
*/
|
|
75904
|
+
static open(options) {
|
|
75905
|
+
const { DatabaseSync } = require("node:sqlite");
|
|
75906
|
+
const db = new DatabaseSync(options.path);
|
|
75907
|
+
const queue = new _DurableQueue(db, options);
|
|
75908
|
+
if ((0, import_node_fs3.existsSync)(options.path)) {
|
|
75909
|
+
try {
|
|
75910
|
+
(0, import_node_fs3.chmodSync)(options.path, 384);
|
|
75911
|
+
} catch {
|
|
75912
|
+
}
|
|
75913
|
+
}
|
|
75914
|
+
queue.startCheckpointLoop();
|
|
75915
|
+
return queue;
|
|
75916
|
+
}
|
|
75917
|
+
/** Closes the underlying SQLite handle. Idempotent. */
|
|
75918
|
+
close() {
|
|
75919
|
+
if (this.closed) {
|
|
75920
|
+
return;
|
|
75921
|
+
}
|
|
75922
|
+
this.closed = true;
|
|
75923
|
+
this.stopCheckpointLoop();
|
|
75924
|
+
this.stopDispatchLease();
|
|
75925
|
+
this.checkpointWal();
|
|
75926
|
+
try {
|
|
75927
|
+
this.db.close();
|
|
75928
|
+
} catch (err2) {
|
|
75929
|
+
this.log.warn(`Error while closing durable queue DB: ${Re(err2)}`);
|
|
75930
|
+
}
|
|
75931
|
+
}
|
|
75932
|
+
/**
|
|
75933
|
+
* Unconditionally runs a TRUNCATE checkpoint, folding the WAL into the main
|
|
75934
|
+
* DB file and truncating the WAL to zero bytes. Best-effort: failures are
|
|
75935
|
+
* logged, and an incomplete checkpoint (a peer connection pinning part of the
|
|
75936
|
+
* WAL) leaves the dirty flag set so the next attempt retries.
|
|
75937
|
+
* @returns True when the checkpoint fully completed.
|
|
75938
|
+
*/
|
|
75939
|
+
checkpointWal() {
|
|
75940
|
+
try {
|
|
75941
|
+
const row = this.checkpointStmt.get();
|
|
75942
|
+
if (row?.busy) {
|
|
75943
|
+
return false;
|
|
75944
|
+
}
|
|
75945
|
+
this.walDirty = false;
|
|
75946
|
+
return true;
|
|
75947
|
+
} catch (err2) {
|
|
75948
|
+
this.log.warn(`wal_checkpoint failed: ${Re(err2)}`);
|
|
75949
|
+
return false;
|
|
75950
|
+
}
|
|
75951
|
+
}
|
|
75952
|
+
/**
|
|
75953
|
+
* Runs {@link DurableQueue.checkpointWal} only when writes have landed since
|
|
75954
|
+
* the last completed checkpoint. Driven by the queue's own checkpoint loop
|
|
75955
|
+
* (see {@link DurableQueue.startCheckpointLoop}) every `checkpointIntervalMs`,
|
|
75956
|
+
* so the WAL drains shortly after traffic stops instead of waiting for the next
|
|
75957
|
+
* retention sweep or for close(). A no-op on an idle queue.
|
|
75958
|
+
* @returns True when a checkpoint ran and fully completed.
|
|
75959
|
+
*/
|
|
75960
|
+
checkpointWalIfDirty() {
|
|
75961
|
+
if (this.closed || !this.walDirty) {
|
|
75962
|
+
return false;
|
|
75963
|
+
}
|
|
75964
|
+
return this.checkpointWal();
|
|
75965
|
+
}
|
|
75966
|
+
/**
|
|
75967
|
+
* Starts the periodic WAL-checkpoint loop. Idempotent. Called once by
|
|
75968
|
+
* {@link DurableQueue.open}; runs regardless of dispatch leadership (see the
|
|
75969
|
+
* `checkpointTimer` field) and is torn down by {@link DurableQueue.close}.
|
|
75970
|
+
*/
|
|
75971
|
+
startCheckpointLoop() {
|
|
75972
|
+
if (this.checkpointTimer) {
|
|
75973
|
+
return;
|
|
75974
|
+
}
|
|
75975
|
+
this.checkpointTimer = setInterval(() => this.checkpointWalIfDirty(), this.checkpointIntervalMs);
|
|
75976
|
+
if (typeof this.checkpointTimer.unref === "function") {
|
|
75977
|
+
this.checkpointTimer.unref();
|
|
75978
|
+
}
|
|
75979
|
+
}
|
|
75980
|
+
/** Stops the WAL-checkpoint loop. Idempotent and a no-op when never started. */
|
|
75981
|
+
stopCheckpointLoop() {
|
|
75982
|
+
if (this.checkpointTimer) {
|
|
75983
|
+
clearInterval(this.checkpointTimer);
|
|
75984
|
+
this.checkpointTimer = void 0;
|
|
75985
|
+
}
|
|
75986
|
+
}
|
|
75987
|
+
/** @returns Filesystem path of the underlying SQLite file. */
|
|
75988
|
+
getPath() {
|
|
75989
|
+
return this.path;
|
|
75990
|
+
}
|
|
75991
|
+
/** @returns Direct access to the underlying handle. Used by retention sweeper for PRAGMA queries. */
|
|
75992
|
+
getDb() {
|
|
75993
|
+
return this.db;
|
|
75994
|
+
}
|
|
75995
|
+
/**
|
|
75996
|
+
* Runs `fn` inside a single SQLite transaction, committing if it returns and
|
|
75997
|
+
* rolling back if it throws. Used to make a row state transition atomic with an
|
|
75998
|
+
* external side effect — e.g. flipping a row `claimed` → `inflight` only if the
|
|
75999
|
+
* socket write that puts it on the wire also succeeds (see `App.sendToWebSocket`):
|
|
76000
|
+
* if the write throws, the marker rolls back and the row stays `claimed`
|
|
76001
|
+
* (provably unsent, safe to requeue) rather than a phantom `inflight`.
|
|
76002
|
+
*
|
|
76003
|
+
* `node:sqlite` is synchronous, so `fn` must be synchronous too — do any `await`
|
|
76004
|
+
* (e.g. token refresh) before calling, not inside.
|
|
76005
|
+
* @param fn - The work to run transactionally.
|
|
76006
|
+
* @returns Whatever `fn` returns.
|
|
76007
|
+
*/
|
|
76008
|
+
runInTransaction(fn2) {
|
|
76009
|
+
this.db.exec("BEGIN");
|
|
76010
|
+
try {
|
|
76011
|
+
const result = fn2();
|
|
76012
|
+
this.db.exec("COMMIT");
|
|
76013
|
+
return result;
|
|
76014
|
+
} catch (err2) {
|
|
76015
|
+
this.db.exec("ROLLBACK");
|
|
76016
|
+
throw err2;
|
|
76017
|
+
}
|
|
76018
|
+
}
|
|
76019
|
+
/**
|
|
76020
|
+
* Inserts a new `queued` row.
|
|
76021
|
+
*
|
|
76022
|
+
* If any prior non-`nacked` row already owns this `(channel_name,
|
|
76023
|
+
* msg_control_id)` — in `queued`, `claimed`, `inflight`, `processed`,
|
|
76024
|
+
* `rejected`, or `failed` — we don't insert; we surface a `duplicate` result
|
|
76025
|
+
* carrying that row so the caller can compare bodies and decide between
|
|
76026
|
+
* replaying the prior ACK and rejecting the collision (§8). This SELECT is the
|
|
76027
|
+
* single dedup authority on the intake path: the partial unique index only
|
|
76028
|
+
* covers the live `queued`/`claimed`/`inflight` window, so it can't recognize a
|
|
76029
|
+
* retransmit of an already-`processed`/`rejected`/`failed` row —
|
|
76030
|
+
* it remains only as a last-resort guard against an active-window race (not
|
|
76031
|
+
* expected in the single-process agent).
|
|
76032
|
+
* @param input - Fields to persist on the new row.
|
|
76033
|
+
* @param options - Optional behavior.
|
|
76034
|
+
* @param options.assignSeqNo - When provided, enqueue assigns the channel's next
|
|
76035
|
+
* sequence number — but only after the duplicate check passes, so a retransmit
|
|
76036
|
+
* never consumes one. It peeks the persisted counter (a non-consuming read),
|
|
76037
|
+
* calls this callback with the candidate so the caller can stamp it into MSH.13
|
|
76038
|
+
* and return the finalized bytes to persist, then advances the counter in the
|
|
76039
|
+
* SAME transaction as the insert (so a crash can't store the row while leaving
|
|
76040
|
+
* the counter behind, which would reuse the number on restart). On a duplicate
|
|
76041
|
+
* the callback is never invoked.
|
|
76042
|
+
* @returns Either the inserted row, or the prior row that owns the MSH.10.
|
|
76043
|
+
*/
|
|
76044
|
+
enqueue(input, options) {
|
|
76045
|
+
if (input.msgControlId) {
|
|
76046
|
+
const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
|
|
76047
|
+
if (existing) {
|
|
76048
|
+
return { kind: "duplicate", existing };
|
|
76049
|
+
}
|
|
76050
|
+
}
|
|
76051
|
+
let finalizedMessage = input.finalizedMessage;
|
|
76052
|
+
let seqNo = input.seqNo;
|
|
76053
|
+
let seqNoToCommit;
|
|
76054
|
+
if (options?.assignSeqNo) {
|
|
76055
|
+
const candidate = this.peekNextSeqNo(input.channelName);
|
|
76056
|
+
finalizedMessage = options.assignSeqNo(candidate);
|
|
76057
|
+
seqNo = candidate;
|
|
76058
|
+
seqNoToCommit = candidate;
|
|
76059
|
+
}
|
|
76060
|
+
try {
|
|
76061
|
+
const runInsert = () => {
|
|
76062
|
+
const info = this.enqueueStmt.run(
|
|
76063
|
+
input.channelName,
|
|
76064
|
+
input.remote,
|
|
76065
|
+
input.msgControlId,
|
|
76066
|
+
input.msgType,
|
|
76067
|
+
toBlob(input.originalMessage),
|
|
76068
|
+
toBlob(finalizedMessage),
|
|
76069
|
+
input.encoding,
|
|
76070
|
+
input.enhancedMode,
|
|
76071
|
+
input.callbackId,
|
|
76072
|
+
seqNo,
|
|
76073
|
+
input.receivedAt
|
|
76074
|
+
);
|
|
76075
|
+
return Number(info.lastInsertRowid);
|
|
76076
|
+
};
|
|
76077
|
+
let id;
|
|
76078
|
+
if (seqNoToCommit !== void 0) {
|
|
76079
|
+
this.db.exec("BEGIN");
|
|
76080
|
+
try {
|
|
76081
|
+
id = runInsert();
|
|
76082
|
+
this.commitSeqNo(input.channelName, seqNoToCommit);
|
|
76083
|
+
this.db.exec("COMMIT");
|
|
76084
|
+
} catch (txErr) {
|
|
76085
|
+
this.db.exec("ROLLBACK");
|
|
76086
|
+
throw txErr;
|
|
76087
|
+
}
|
|
76088
|
+
} else {
|
|
76089
|
+
id = runInsert();
|
|
76090
|
+
}
|
|
76091
|
+
this.walDirty = true;
|
|
76092
|
+
const row = this.getById(id);
|
|
76093
|
+
if (!row) {
|
|
76094
|
+
throw new Error(`enqueue: inserted row id=${id} could not be re-read`);
|
|
76095
|
+
}
|
|
76096
|
+
return { kind: "inserted", row };
|
|
76097
|
+
} catch (err2) {
|
|
76098
|
+
if (isUniqueConstraintError(err2) && input.msgControlId) {
|
|
76099
|
+
const existing = this.findSeenByControlId(input.channelName, input.msgControlId);
|
|
76100
|
+
if (existing) {
|
|
76101
|
+
return { kind: "duplicate", existing };
|
|
76102
|
+
}
|
|
76103
|
+
}
|
|
76104
|
+
throw err2;
|
|
76105
|
+
}
|
|
76106
|
+
}
|
|
76107
|
+
/**
|
|
76108
|
+
* Inserts an audit row in the `nacked` terminal state — used when intake
|
|
76109
|
+
* rejected the message (duplicate, malformed) but we still want a forensics
|
|
76110
|
+
* record. Failure to write this is non-fatal; the caller is expected to log.
|
|
76111
|
+
* @param input - Fields to persist plus the `lastError`/`errorCode` describing why we rejected.
|
|
76112
|
+
* @returns The newly inserted audit row, or null if even the audit insert failed.
|
|
76113
|
+
*/
|
|
76114
|
+
enqueueRejected(input) {
|
|
76115
|
+
try {
|
|
76116
|
+
const info = this.enqueueRejectedStmt.run(
|
|
76117
|
+
input.channelName,
|
|
76118
|
+
input.remote,
|
|
76119
|
+
input.msgControlId,
|
|
76120
|
+
input.msgType,
|
|
76121
|
+
toBlob(input.originalMessage),
|
|
76122
|
+
toBlob(input.finalizedMessage),
|
|
76123
|
+
input.encoding,
|
|
76124
|
+
input.enhancedMode,
|
|
76125
|
+
input.callbackId,
|
|
76126
|
+
input.lastError,
|
|
76127
|
+
input.errorCode,
|
|
76128
|
+
input.seqNo,
|
|
76129
|
+
input.receivedAt
|
|
76130
|
+
);
|
|
76131
|
+
this.walDirty = true;
|
|
76132
|
+
return this.getById(Number(info.lastInsertRowid));
|
|
76133
|
+
} catch (err2) {
|
|
76134
|
+
this.log.warn(`enqueueRejected failed: ${Re(err2)}`);
|
|
76135
|
+
return null;
|
|
76136
|
+
}
|
|
76137
|
+
}
|
|
76138
|
+
/**
|
|
76139
|
+
* Atomically claims the next `queued` row for `channelName`, flipping it to
|
|
76140
|
+
* `claimed` and bumping `attempt_count`. The row stays `claimed` until
|
|
76141
|
+
* {@link markSent} flips it to `inflight` once the request hits the socket.
|
|
76142
|
+
* @param channelName - The channel to claim from.
|
|
76143
|
+
* @param now - Override the timestamp written to `processing_started_at` (for tests).
|
|
76144
|
+
* @returns The claimed row, or `null` if the channel queue is empty.
|
|
76145
|
+
*/
|
|
76146
|
+
claimNext(channelName, now = Date.now()) {
|
|
76147
|
+
this.assertNotDemoted();
|
|
76148
|
+
const raw = this.claimNextStmt.get(now, channelName);
|
|
76149
|
+
if (raw) {
|
|
76150
|
+
this.walDirty = true;
|
|
76151
|
+
}
|
|
76152
|
+
return raw ? rowFromSql(raw) : null;
|
|
76153
|
+
}
|
|
76154
|
+
/**
|
|
76155
|
+
* Phase A → B transition: records that the transmit request for `callbackId`
|
|
76156
|
+
* was written to the WebSocket, flipping the row from `claimed` to `inflight`
|
|
76157
|
+
* and stamping `sent_at`. Called from the App's send path the instant the
|
|
76158
|
+
* request leaves the process — the durable marker that distinguishes a
|
|
76159
|
+
* provably-unsent row (safe to requeue on crash) from an ambiguous in-flight
|
|
76160
|
+
* one (failed for review). Guarded on `state = 'claimed'`, so it's a no-op for
|
|
76161
|
+
* legacy (non-durable) sends and for any row already past `claimed`.
|
|
76162
|
+
* @param callbackId - The callback ID of the transmit request just sent.
|
|
76163
|
+
* @param now - Override for `sent_at` (for tests).
|
|
76164
|
+
* @returns True if a `claimed` row was flipped to `inflight`.
|
|
76165
|
+
*/
|
|
76166
|
+
markSent(callbackId, now = Date.now()) {
|
|
76167
|
+
const info = this.markSentStmt.run(now, callbackId);
|
|
76168
|
+
if (Number(info.changes) > 0) {
|
|
76169
|
+
this.walDirty = true;
|
|
76170
|
+
return true;
|
|
76171
|
+
}
|
|
76172
|
+
return false;
|
|
76173
|
+
}
|
|
76174
|
+
/**
|
|
76175
|
+
* Looks up a row by its server-callback ID. Returns `null` if not found.
|
|
76176
|
+
* @param callbackId - The callback ID echoed back by the Medplum server.
|
|
76177
|
+
* @returns The matching row, or null.
|
|
76178
|
+
*/
|
|
76179
|
+
findByCallback(callbackId) {
|
|
76180
|
+
const raw = this.findByCallbackStmt.get(callbackId);
|
|
76181
|
+
return raw ? rowFromSql(raw) : null;
|
|
76182
|
+
}
|
|
76183
|
+
/**
|
|
76184
|
+
* Looks up a row by its primary key.
|
|
76185
|
+
* @param id - The row's primary key.
|
|
76186
|
+
* @returns The matching row, or null.
|
|
76187
|
+
*/
|
|
76188
|
+
getById(id) {
|
|
76189
|
+
const raw = this.findByIdStmt.get(id);
|
|
76190
|
+
return raw ? rowFromSql(raw) : null;
|
|
76191
|
+
}
|
|
76192
|
+
/**
|
|
76193
|
+
* Records the server's `agent:transmit:response` body + status against the row.
|
|
76194
|
+
* @param id - Row primary key.
|
|
76195
|
+
* @param statusCode - HTTP-style status code returned by the server.
|
|
76196
|
+
* @param body - Response payload, or null when the server omitted one.
|
|
76197
|
+
*/
|
|
76198
|
+
recordServerResponse(id, statusCode, body) {
|
|
76199
|
+
this.assertNotDemoted();
|
|
76200
|
+
this.recordServerResponseStmt.run(statusCode, body === null ? null : toBlob(body), id);
|
|
76201
|
+
this.walDirty = true;
|
|
76202
|
+
}
|
|
76203
|
+
/**
|
|
76204
|
+
* Terminal Bot-leg transition: the server accepted the message (2xx). The
|
|
76205
|
+
* source leg is recorded independently via `ackOutcome` — `delivered` when the
|
|
76206
|
+
* app-level ACK reached the source, `undelivered` when it couldn't (e.g. the
|
|
76207
|
+
* source connection had closed). An `undelivered` row is fully processed
|
|
76208
|
+
* upstream and must never be re-dispatched; it recovers when the source
|
|
76209
|
+
* retransmits and the stored ACK is replayed (see {@link setAckOutcome}).
|
|
76210
|
+
* @param id - Row primary key.
|
|
76211
|
+
* @param ackOutcome - Source-leg result: `delivered` or `undelivered`.
|
|
76212
|
+
* @param now - Override for `processed_at` (for tests).
|
|
76213
|
+
*/
|
|
76214
|
+
markProcessed(id, ackOutcome, now = Date.now()) {
|
|
76215
|
+
this.assertNotDemoted();
|
|
76216
|
+
this.markProcessedStmt.run(ackOutcome, now, id);
|
|
76217
|
+
this.walDirty = true;
|
|
76218
|
+
}
|
|
76219
|
+
/**
|
|
76220
|
+
* Terminal Bot-leg transition: the server **rejected** the message itself
|
|
76221
|
+
* (permanent 4xx). Retrying can never help — the content must be triaged.
|
|
76222
|
+
* @param id - Row primary key.
|
|
76223
|
+
* @param error - Human-readable error string, written to `last_error`.
|
|
76224
|
+
* @param errorCode - Machine-readable classification, written to `error_code`.
|
|
76225
|
+
* @param now - Override for `errored_at` (for tests).
|
|
76226
|
+
*/
|
|
76227
|
+
markRejected(id, error, errorCode, now = Date.now()) {
|
|
76228
|
+
this.assertNotDemoted();
|
|
76229
|
+
this.markBotFailedStmt.run(MessageState.REJECTED, now, error, errorCode, id);
|
|
76230
|
+
this.walDirty = true;
|
|
76231
|
+
}
|
|
76232
|
+
/**
|
|
76233
|
+
* Terminal-for-now Bot-leg transition: a **transient/ambiguous** failure
|
|
76234
|
+
* (5xx, 429, response timeout, dispatch error, interrupted). The retry/
|
|
76235
|
+
* operator-review candidate — distinct from a `rejected` message so a future
|
|
76236
|
+
* retry policy can re-dispatch `failed` rows without ever touching `rejected`
|
|
76237
|
+
* ones (or `processed` + `undelivered` ones, whose Bot leg already succeeded).
|
|
76238
|
+
* @param id - Row primary key.
|
|
76239
|
+
* @param error - Human-readable error string, written to `last_error`.
|
|
76240
|
+
* @param errorCode - Machine-readable classification, written to `error_code`.
|
|
76241
|
+
* @param now - Override for `errored_at` (for tests).
|
|
76242
|
+
*/
|
|
76243
|
+
markFailed(id, error, errorCode, now = Date.now()) {
|
|
76244
|
+
this.assertNotDemoted();
|
|
76245
|
+
this.markBotFailedStmt.run(MessageState.FAILED, now, error, errorCode, id);
|
|
76246
|
+
this.walDirty = true;
|
|
76247
|
+
}
|
|
76248
|
+
/**
|
|
76249
|
+
* Updates only the source-leg {@link AckOutcome}, leaving the Bot-leg `state`
|
|
76250
|
+
* untouched. Used when a duplicate retransmit replays a previously
|
|
76251
|
+
* `undelivered` ACK and it lands — flipping the row to `delivered` so it no
|
|
76252
|
+
* longer reads as awaiting source delivery.
|
|
76253
|
+
* @param id - Row primary key.
|
|
76254
|
+
* @param ackOutcome - The new source-leg outcome.
|
|
76255
|
+
*/
|
|
76256
|
+
setAckOutcome(id, ackOutcome) {
|
|
76257
|
+
this.setAckOutcomeStmt.run(ackOutcome, id);
|
|
76258
|
+
this.walDirty = true;
|
|
76259
|
+
}
|
|
76260
|
+
/**
|
|
76261
|
+
* Returns a `claimed` row to `queued` — used when the WS connection drops
|
|
76262
|
+
* before the transmit request was ever written to the socket, so retrying on
|
|
76263
|
+
* reconnect carries no duplicate-delivery risk. (A row that already reached the
|
|
76264
|
+
* socket is `inflight`, not `claimed`, and this is a no-op for it — its outcome
|
|
76265
|
+
* is ambiguous and owned by the response timeout.) Because the row keeps its
|
|
76266
|
+
* original id and claims are ordered by id, it goes back to the front of its
|
|
76267
|
+
* channel's FIFO.
|
|
76268
|
+
* @param id - Row primary key.
|
|
76269
|
+
* @returns True if the row was requeued; false if it was not in `claimed`.
|
|
76270
|
+
*/
|
|
76271
|
+
requeue(id) {
|
|
76272
|
+
this.assertNotDemoted();
|
|
76273
|
+
const info = this.requeueStmt.run(id);
|
|
76274
|
+
if (Number(info.changes) > 0) {
|
|
76275
|
+
this.walDirty = true;
|
|
76276
|
+
return true;
|
|
76277
|
+
}
|
|
76278
|
+
return false;
|
|
76279
|
+
}
|
|
76280
|
+
/**
|
|
76281
|
+
* Recovers rows left mid-flight by a previous process, splitting on whether the
|
|
76282
|
+
* request reached the wire. Runs once at startup (§10):
|
|
76283
|
+
* - `inflight` rows are ambiguous (the server may have processed them) → `failed`
|
|
76284
|
+
* with error code `interrupted`, surfaced for operator review, never silently retried.
|
|
76285
|
+
* - `claimed` rows provably never left the process (`sent_at` is NULL) → returned
|
|
76286
|
+
* to `queued` for a clean re-dispatch with no duplicate-delivery risk.
|
|
76287
|
+
* @param now - Override for `errored_at` (for tests).
|
|
76288
|
+
* @returns Counts of `claimed` rows requeued and `inflight` rows failed.
|
|
76289
|
+
*/
|
|
76290
|
+
recoverOnStartup(now = Date.now()) {
|
|
76291
|
+
const failedInfo = this.recoverInflightStmt.run(now);
|
|
76292
|
+
const requeuedInfo = this.recoverClaimedStmt.run();
|
|
76293
|
+
const failed = Number(failedInfo.changes);
|
|
76294
|
+
const requeued = Number(requeuedInfo.changes);
|
|
76295
|
+
if (failed > 0 || requeued > 0) {
|
|
76296
|
+
this.walDirty = true;
|
|
76297
|
+
}
|
|
76298
|
+
return { requeued, failed };
|
|
76299
|
+
}
|
|
76300
|
+
/**
|
|
76301
|
+
* Lists all `queued` row IDs for a channel, in FIFO order. Used by the
|
|
76302
|
+
* worker on startup to repopulate its in-memory wake signal.
|
|
76303
|
+
* @param channelName - The channel to query.
|
|
76304
|
+
* @returns Row IDs in FIFO order.
|
|
76305
|
+
*/
|
|
76306
|
+
listQueuedIdsForChannel(channelName) {
|
|
76307
|
+
const rows = this.listQueuedIdsForChannelStmt.all(channelName);
|
|
76308
|
+
return rows.map((r6) => r6.id);
|
|
76309
|
+
}
|
|
76310
|
+
/** @returns Counts of rows by state. Missing states are reported as 0. */
|
|
76311
|
+
countByState() {
|
|
76312
|
+
const counts = {
|
|
76313
|
+
queued: 0,
|
|
76314
|
+
claimed: 0,
|
|
76315
|
+
inflight: 0,
|
|
76316
|
+
processed: 0,
|
|
76317
|
+
rejected: 0,
|
|
76318
|
+
failed: 0,
|
|
76319
|
+
nacked: 0
|
|
76320
|
+
};
|
|
76321
|
+
const rows = this.countByStateStmt.all();
|
|
76322
|
+
for (const r6 of rows) {
|
|
76323
|
+
counts[r6.state] = r6.n;
|
|
76324
|
+
}
|
|
76325
|
+
return counts;
|
|
76326
|
+
}
|
|
76327
|
+
/**
|
|
76328
|
+
* @param channelName - The channel to query.
|
|
76329
|
+
* @param now - Override for the "now" timestamp used in `oldestQueuedAgeMs`.
|
|
76330
|
+
* @returns Depth snapshot for `channelName` (queued/claimed/inflight counts + oldest queued age).
|
|
76331
|
+
*/
|
|
76332
|
+
getChannelDepth(channelName, now = Date.now()) {
|
|
76333
|
+
const row = this.channelDepthStmt.get(channelName);
|
|
76334
|
+
return {
|
|
76335
|
+
queued: row?.queued ?? 0,
|
|
76336
|
+
claimed: row?.claimed ?? 0,
|
|
76337
|
+
inflight: row?.inflight ?? 0,
|
|
76338
|
+
oldestQueuedAgeMs: row?.oldest_queued_at ? now - row.oldest_queued_at : null
|
|
76339
|
+
};
|
|
76340
|
+
}
|
|
76341
|
+
/**
|
|
76342
|
+
* Attempts to acquire (or re-acquire) the queue lease as `holder`.
|
|
76343
|
+
*
|
|
76344
|
+
* Succeeds when there is no current lease, the current lease is already held
|
|
76345
|
+
* by us (refresh case), or the prior holder's lease has expired. Fails when a
|
|
76346
|
+
* different holder still has a valid lease — caller should wait and retry.
|
|
76347
|
+
*
|
|
76348
|
+
* The lease is the cross-process coordination primitive that makes zero-downtime
|
|
76349
|
+
* upgrades safe: only the holder runs workers and `recoverOnStartup`, so two
|
|
76350
|
+
* processes sharing the DB during the upgrade overlap don't fight over rows.
|
|
76351
|
+
* @param holder - Stable identifier for this process (a per-process UUID).
|
|
76352
|
+
* @param ttlMs - How long the new lease should remain valid before another
|
|
76353
|
+
* process can take over (also drives the heartbeat cadence).
|
|
76354
|
+
* @param now - Override the "now" timestamp (for tests).
|
|
76355
|
+
* @returns True if the lease is now held by `holder`; false if a foreign holder still owns it.
|
|
76356
|
+
*/
|
|
76357
|
+
tryAcquireLease(holder, ttlMs, now = Date.now()) {
|
|
76358
|
+
const expiresAt = now + ttlMs;
|
|
76359
|
+
const info = this.tryAcquireLeaseStmt.run(holder, now, expiresAt, holder, now);
|
|
76360
|
+
if (Number(info.changes) > 0) {
|
|
76361
|
+
this.walDirty = true;
|
|
76362
|
+
}
|
|
76363
|
+
return Number(info.changes) > 0;
|
|
76364
|
+
}
|
|
76365
|
+
/**
|
|
76366
|
+
* Extends the existing lease held by `holder`. Fails (returns false) when the
|
|
76367
|
+
* lease is no longer ours — most commonly because a peer took over after our
|
|
76368
|
+
* TTL elapsed. A leader that sees `false` here must stop driving the queue.
|
|
76369
|
+
* @param holder - The same identifier passed to {@link DurableQueue.tryAcquireLease}.
|
|
76370
|
+
* @param ttlMs - New TTL for the lease (added to `now`).
|
|
76371
|
+
* @param now - Override the "now" timestamp (for tests).
|
|
76372
|
+
* @returns True if the heartbeat extended our lease; false if we lost it.
|
|
76373
|
+
*/
|
|
76374
|
+
heartbeatLease(holder, ttlMs, now = Date.now()) {
|
|
76375
|
+
const info = this.heartbeatLeaseStmt.run(now + ttlMs, holder);
|
|
76376
|
+
if (Number(info.changes) > 0) {
|
|
76377
|
+
this.walDirty = true;
|
|
76378
|
+
}
|
|
76379
|
+
return Number(info.changes) > 0;
|
|
76380
|
+
}
|
|
76381
|
+
/**
|
|
76382
|
+
* Releases the lease if (and only if) `holder` still owns it. Idempotent. The
|
|
76383
|
+
* holder check means a process that already lost its lease won't accidentally
|
|
76384
|
+
* delete a newer leader's row.
|
|
76385
|
+
* @param holder - The identifier of the lease to release.
|
|
76386
|
+
*/
|
|
76387
|
+
releaseLease(holder) {
|
|
76388
|
+
const info = this.releaseLeaseStmt.run(holder);
|
|
76389
|
+
if (Number(info.changes) > 0) {
|
|
76390
|
+
this.walDirty = true;
|
|
76391
|
+
}
|
|
76392
|
+
}
|
|
76393
|
+
/**
|
|
76394
|
+
* @returns The current lease, or null if no row exists. Used for diagnostics
|
|
76395
|
+
* and stats; the acquire/heartbeat methods do the actual coordination.
|
|
76396
|
+
*/
|
|
76397
|
+
getCurrentLease() {
|
|
76398
|
+
const row = this.getLeaseStmt.get();
|
|
76399
|
+
return row ? { holder: row.holder, expiresAt: row.expires_at } : null;
|
|
76400
|
+
}
|
|
76401
|
+
/**
|
|
76402
|
+
* Binds this queue to the lease holder ID of the local process, so the
|
|
76403
|
+
* dispatch gate ({@link assertNotDemoted}) can tell "us" from a peer.
|
|
76404
|
+
* {@link startDispatchLease} sets this automatically; call it directly only in
|
|
76405
|
+
* tests that drive {@link tryAcquireLease}/{@link heartbeatLease} by hand.
|
|
76406
|
+
* @param holderId - This process's lease holder ID.
|
|
76407
|
+
*/
|
|
76408
|
+
setLeaseHolder(holderId) {
|
|
76409
|
+
this.leaseHolderId = holderId;
|
|
76410
|
+
}
|
|
76411
|
+
/**
|
|
76412
|
+
* Begins the dispatch-lease acquire-and-heartbeat loop for this process.
|
|
76413
|
+
*
|
|
76414
|
+
* Tries to acquire the lease immediately. On success it fires `onBecameLeader`
|
|
76415
|
+
* and begins heartbeating every `leaseHeartbeatMs`; on failure it retries every
|
|
76416
|
+
* `leaseAcquireRetryMs` until the lease frees up. `onBecameLeader` fires every
|
|
76417
|
+
* time this process transitions from follower to leader (typically once, but
|
|
76418
|
+
* again if it loses the lease mid-run and reclaims it). There is no
|
|
76419
|
+
* lost-leadership callback by design — loss surfaces as a {@link QueueLeaseError}
|
|
76420
|
+
* from the dispatch ops, which the worker catches to drain itself.
|
|
76421
|
+
*
|
|
76422
|
+
* Idempotent: calling it again while the loop is already active is a no-op,
|
|
76423
|
+
* regardless of whether this process is currently a retrying follower or the
|
|
76424
|
+
* leader. A repeat call does not re-acquire, re-heartbeat, re-fire
|
|
76425
|
+
* `onBecameLeader`, or replace the callback — so the App can call it on every
|
|
76426
|
+
* heartbeat tick (and when already the leader) without restarting the loop.
|
|
76427
|
+
* @param onBecameLeader - Callback invoked when this process takes the lease.
|
|
76428
|
+
*/
|
|
76429
|
+
startDispatchLease(onBecameLeader) {
|
|
76430
|
+
if (this.leaseLoopActive) {
|
|
76431
|
+
return;
|
|
76432
|
+
}
|
|
76433
|
+
this.onBecameLeader = onBecameLeader;
|
|
76434
|
+
this.leaseLoopActive = true;
|
|
76435
|
+
if (!this.leaseHolderId) {
|
|
76436
|
+
this.leaseHolderId = (0, import_node_crypto4.randomUUID)();
|
|
76437
|
+
}
|
|
76438
|
+
this.tryAcquireDispatchLease();
|
|
76439
|
+
}
|
|
76440
|
+
/**
|
|
76441
|
+
* Stops the dispatch-lease loop and releases the lease if we hold it.
|
|
76442
|
+
* Idempotent and a no-op when the loop was never started.
|
|
76443
|
+
* @returns True if we were the leader when stopping (and released the lease).
|
|
76444
|
+
*/
|
|
76445
|
+
stopDispatchLease() {
|
|
76446
|
+
this.leaseLoopActive = false;
|
|
76447
|
+
this.onBecameLeader = void 0;
|
|
76448
|
+
this.clearLeaseAcquireTimer();
|
|
76449
|
+
this.clearLeaseHeartbeatTimer();
|
|
76450
|
+
const wasLeader = this.leaseLeader;
|
|
76451
|
+
if (this.leaseLeader && this.leaseHolderId) {
|
|
76452
|
+
try {
|
|
76453
|
+
this.releaseLease(this.leaseHolderId);
|
|
76454
|
+
} catch (err2) {
|
|
76455
|
+
this.log.warn(`Failed to release queue lease: ${Re(err2)}`);
|
|
76456
|
+
}
|
|
76457
|
+
this.leaseLeader = false;
|
|
76458
|
+
}
|
|
76459
|
+
return wasLeader;
|
|
76460
|
+
}
|
|
76461
|
+
/** @returns True if this process currently holds the dispatch lease. */
|
|
76462
|
+
isLeader() {
|
|
76463
|
+
return this.leaseLeader;
|
|
76464
|
+
}
|
|
76465
|
+
/** @returns The dispatch-lease holder ID for this process (for diagnostics), or undefined if never started. */
|
|
76466
|
+
getLeaseHolderId() {
|
|
76467
|
+
return this.leaseHolderId;
|
|
76468
|
+
}
|
|
76469
|
+
tryAcquireDispatchLease() {
|
|
76470
|
+
if (!this.leaseLoopActive || this.closed || !this.leaseHolderId) {
|
|
76471
|
+
return;
|
|
76472
|
+
}
|
|
76473
|
+
let acquired = false;
|
|
76474
|
+
try {
|
|
76475
|
+
acquired = this.tryAcquireLease(this.leaseHolderId, this.leaseTtlMs);
|
|
76476
|
+
} catch (err2) {
|
|
76477
|
+
this.log.warn(`Queue lease acquire threw: ${Re(err2)}`);
|
|
76478
|
+
}
|
|
76479
|
+
if (acquired) {
|
|
76480
|
+
this.leaseLeader = true;
|
|
76481
|
+
this.clearLeaseAcquireTimer();
|
|
76482
|
+
this.log.info(`Acquired queue lease (holder=${this.leaseHolderId}).`);
|
|
76483
|
+
this.scheduleLeaseHeartbeat();
|
|
76484
|
+
try {
|
|
76485
|
+
this.onBecameLeader?.();
|
|
76486
|
+
} catch (err2) {
|
|
76487
|
+
this.log.error(`onBecameLeader callback threw: ${Re(err2)}`);
|
|
76488
|
+
}
|
|
76489
|
+
return;
|
|
76490
|
+
}
|
|
76491
|
+
this.scheduleLeaseAcquireRetry();
|
|
76492
|
+
}
|
|
76493
|
+
scheduleLeaseAcquireRetry() {
|
|
76494
|
+
if (!this.leaseLoopActive || this.leaseAcquireTimer) {
|
|
76495
|
+
return;
|
|
76496
|
+
}
|
|
76497
|
+
this.leaseAcquireTimer = setTimeout(() => {
|
|
76498
|
+
this.leaseAcquireTimer = void 0;
|
|
76499
|
+
this.tryAcquireDispatchLease();
|
|
76500
|
+
}, this.leaseAcquireRetryMs);
|
|
76501
|
+
if (typeof this.leaseAcquireTimer.unref === "function") {
|
|
76502
|
+
this.leaseAcquireTimer.unref();
|
|
76503
|
+
}
|
|
76504
|
+
}
|
|
76505
|
+
scheduleLeaseHeartbeat() {
|
|
76506
|
+
if (!this.leaseLoopActive || this.leaseHeartbeatTimer) {
|
|
76507
|
+
return;
|
|
76508
|
+
}
|
|
76509
|
+
this.leaseHeartbeatTimer = setInterval(() => this.leaseHeartbeat(), this.leaseHeartbeatMs);
|
|
76510
|
+
if (typeof this.leaseHeartbeatTimer.unref === "function") {
|
|
76511
|
+
this.leaseHeartbeatTimer.unref();
|
|
76512
|
+
}
|
|
76513
|
+
}
|
|
76514
|
+
leaseHeartbeat() {
|
|
76515
|
+
if (!this.leaseLoopActive || !this.leaseLeader || !this.leaseHolderId) {
|
|
76516
|
+
return;
|
|
76517
|
+
}
|
|
76518
|
+
let extended = false;
|
|
76519
|
+
try {
|
|
76520
|
+
extended = this.heartbeatLease(this.leaseHolderId, this.leaseTtlMs);
|
|
76521
|
+
} catch (err2) {
|
|
76522
|
+
this.log.warn(`Queue lease heartbeat threw: ${Re(err2)}`);
|
|
76523
|
+
return;
|
|
76524
|
+
}
|
|
76525
|
+
if (!extended) {
|
|
76526
|
+
this.log.error(`Lost queue lease (holder=${this.leaseHolderId}); peer took over.`);
|
|
76527
|
+
this.leaseLeader = false;
|
|
76528
|
+
this.clearLeaseHeartbeatTimer();
|
|
76529
|
+
this.scheduleLeaseAcquireRetry();
|
|
76530
|
+
}
|
|
76531
|
+
}
|
|
76532
|
+
clearLeaseAcquireTimer() {
|
|
76533
|
+
if (this.leaseAcquireTimer) {
|
|
76534
|
+
clearTimeout(this.leaseAcquireTimer);
|
|
76535
|
+
this.leaseAcquireTimer = void 0;
|
|
76536
|
+
}
|
|
76537
|
+
}
|
|
76538
|
+
clearLeaseHeartbeatTimer() {
|
|
76539
|
+
if (this.leaseHeartbeatTimer) {
|
|
76540
|
+
clearInterval(this.leaseHeartbeatTimer);
|
|
76541
|
+
this.leaseHeartbeatTimer = void 0;
|
|
76542
|
+
}
|
|
76543
|
+
}
|
|
76544
|
+
/**
|
|
76545
|
+
* @returns True when a lease row exists and a DIFFERENT holder owns it — i.e. a
|
|
76546
|
+
* peer has taken over and this process has been demoted. False when no lease
|
|
76547
|
+
* exists (no coordination in play) or the lease is ours. Note this intentionally
|
|
76548
|
+
* does NOT consider expiry: a merely-expired-but-still-ours lease will be
|
|
76549
|
+
* re-extended by our next heartbeat, so only a foreign holder means "demoted",
|
|
76550
|
+
* which keeps this exactly aligned with {@link DurableQueue.isLeader}.
|
|
76551
|
+
*/
|
|
76552
|
+
isLeaseHeldByPeer() {
|
|
76553
|
+
const lease = this.getCurrentLease();
|
|
76554
|
+
return lease !== null && lease.holder !== this.leaseHolderId;
|
|
76555
|
+
}
|
|
76556
|
+
/**
|
|
76557
|
+
* Throws {@link QueueLeaseError} when a peer holds the lease. Called at the top
|
|
76558
|
+
* of every dispatch-class mutation so a demoted process can't claim, dispatch,
|
|
76559
|
+
* or settle rows the new leader now owns — the authoritative, data-layer half of
|
|
76560
|
+
* leadership enforcement (the {@link DurableQueue.isLeader} flag is the
|
|
76561
|
+
* cheap optimistic half). Intake, maintenance, diagnostics, and the physical
|
|
76562
|
+
* `markSent` marker are deliberately NOT gated.
|
|
76563
|
+
*/
|
|
76564
|
+
assertNotDemoted() {
|
|
76565
|
+
if (this.isLeaseHeldByPeer()) {
|
|
76566
|
+
throw new QueueLeaseError(this.leaseHolderId, this.getCurrentLease()?.holder);
|
|
76567
|
+
}
|
|
76568
|
+
}
|
|
76569
|
+
/**
|
|
76570
|
+
* @returns Size of the underlying database file in bytes, computed from
|
|
76571
|
+
* `page_count * page_size`. Used by the retention sweeper.
|
|
76572
|
+
*/
|
|
76573
|
+
getDbSizeBytes() {
|
|
76574
|
+
const row = this.dbSizeBytesStmt.get();
|
|
76575
|
+
return row?.bytes ?? 0;
|
|
76576
|
+
}
|
|
76577
|
+
/**
|
|
76578
|
+
* @param channelName - The channel to search.
|
|
76579
|
+
* @param msgControlId - MSH.10 to look up.
|
|
76580
|
+
* @returns The most recent non-`nacked` row owning this `(channel, control_id)`, or null.
|
|
76581
|
+
*/
|
|
76582
|
+
findSeenByControlId(channelName, msgControlId) {
|
|
76583
|
+
const raw = this.findSeenByControlIdStmt.get(channelName, msgControlId);
|
|
76584
|
+
return raw ? rowFromSql(raw) : null;
|
|
76585
|
+
}
|
|
76586
|
+
/**
|
|
76587
|
+
* Returns the sequence number that {@link commitSeqNo} would next persist for
|
|
76588
|
+
* a channel, WITHOUT advancing the counter. The first value for a channel is
|
|
76589
|
+
* 0; thereafter it is the last committed value + 1. Read-only: callers stamp
|
|
76590
|
+
* this candidate into MSH.13 and only {@link commitSeqNo} it once the row is
|
|
76591
|
+
* durably enqueued, so a failed insert never burns a sequence number.
|
|
76592
|
+
* @param channelName - The channel whose counter to peek.
|
|
76593
|
+
* @returns The next sequence number to assign.
|
|
76594
|
+
*/
|
|
76595
|
+
peekNextSeqNo(channelName) {
|
|
76596
|
+
const row = this.peekLastSeqNoStmt.get(channelName);
|
|
76597
|
+
return row === void 0 ? 0 : row.last_seq_no + 1;
|
|
76598
|
+
}
|
|
76599
|
+
/**
|
|
76600
|
+
* Persists `seqNo` as the channel's last assigned sequence number. Production
|
|
76601
|
+
* intake commits this inside the insert transaction via
|
|
76602
|
+
* {@link enqueue}'s `commitSeqNo` option, so the row and the counter advance
|
|
76603
|
+
* atomically; call it directly only when not pairing it with an insert. The
|
|
76604
|
+
* counter survives restarts, keeping MSH.13 sequence numbers monotonic.
|
|
76605
|
+
* @param channelName - The channel whose counter to advance.
|
|
76606
|
+
* @param seqNo - The sequence number that was assigned and successfully enqueued.
|
|
76607
|
+
*/
|
|
76608
|
+
commitSeqNo(channelName, seqNo) {
|
|
76609
|
+
this.commitSeqNoStmt.run(channelName, seqNo);
|
|
76610
|
+
this.walDirty = true;
|
|
76611
|
+
}
|
|
76612
|
+
};
|
|
76613
|
+
function rowFromSql(raw) {
|
|
76614
|
+
return {
|
|
76615
|
+
id: raw.id,
|
|
76616
|
+
channelName: raw.channel_name,
|
|
76617
|
+
remote: raw.remote,
|
|
76618
|
+
msgControlId: raw.msg_control_id ?? null,
|
|
76619
|
+
msgType: raw.msg_type ?? null,
|
|
76620
|
+
originalMessage: toBuffer(raw.original_message),
|
|
76621
|
+
finalizedMessage: toBuffer(raw.finalized_message),
|
|
76622
|
+
encoding: raw.encoding ?? null,
|
|
76623
|
+
enhancedMode: raw.enhanced_mode ?? null,
|
|
76624
|
+
state: raw.state,
|
|
76625
|
+
attemptCount: raw.attempt_count,
|
|
76626
|
+
callbackId: raw.callback_id,
|
|
76627
|
+
serverResponseBody: raw.server_response_body === null || raw.server_response_body === void 0 ? null : toBuffer(raw.server_response_body),
|
|
76628
|
+
serverStatusCode: raw.server_status_code ?? null,
|
|
76629
|
+
ackOutcome: raw.ack_outcome ?? AckOutcome.PENDING,
|
|
76630
|
+
lastError: raw.last_error ?? null,
|
|
76631
|
+
errorCode: raw.error_code ?? null,
|
|
76632
|
+
seqNo: raw.seq_no ?? null,
|
|
76633
|
+
receivedAt: raw.received_at,
|
|
76634
|
+
processingStartedAt: raw.processing_started_at ?? null,
|
|
76635
|
+
sentAt: raw.sent_at ?? null,
|
|
76636
|
+
processedAt: raw.processed_at ?? null,
|
|
76637
|
+
erroredAt: raw.errored_at ?? null
|
|
76638
|
+
};
|
|
76639
|
+
}
|
|
76640
|
+
function toBlob(value) {
|
|
76641
|
+
return typeof value === "string" ? Buffer.from(value) : value;
|
|
76642
|
+
}
|
|
76643
|
+
function toBuffer(value) {
|
|
76644
|
+
if (Buffer.isBuffer(value)) {
|
|
76645
|
+
return value;
|
|
76646
|
+
}
|
|
76647
|
+
if (value instanceof Uint8Array) {
|
|
76648
|
+
return Buffer.from(value);
|
|
76649
|
+
}
|
|
76650
|
+
if (typeof value === "string") {
|
|
76651
|
+
return Buffer.from(value);
|
|
76652
|
+
}
|
|
76653
|
+
return Buffer.alloc(0);
|
|
76654
|
+
}
|
|
76655
|
+
function isUniqueConstraintError(err2) {
|
|
76656
|
+
if (err2 === null || typeof err2 !== "object") {
|
|
76657
|
+
return false;
|
|
76658
|
+
}
|
|
76659
|
+
const asAny = err2;
|
|
76660
|
+
if (typeof asAny.code === "string" && asAny.code.includes("CONSTRAINT_UNIQUE")) {
|
|
76661
|
+
return true;
|
|
76662
|
+
}
|
|
76663
|
+
if (asAny.errcode === 2067) {
|
|
76664
|
+
return true;
|
|
76665
|
+
}
|
|
76666
|
+
if (typeof asAny.message === "string" && /UNIQUE constraint failed/i.test(asAny.message)) {
|
|
76667
|
+
return true;
|
|
76668
|
+
}
|
|
76669
|
+
return false;
|
|
76670
|
+
}
|
|
76671
|
+
|
|
76672
|
+
// src/queue/retention.ts
|
|
76673
|
+
var DEFAULT_RETENTION_DAYS = 7;
|
|
76674
|
+
var DEFAULT_MAX_SIZE_MB = 512;
|
|
76675
|
+
var DEFAULT_ERRORED_RETENTION_DAYS = 90;
|
|
76676
|
+
var DEFAULT_SWEEP_INTERVAL_SECS = 3600;
|
|
76677
|
+
var RetentionSweeper = class {
|
|
76678
|
+
constructor(options) {
|
|
76679
|
+
__publicField(this, "queue");
|
|
76680
|
+
__publicField(this, "log");
|
|
76681
|
+
__publicField(this, "retentionMs");
|
|
76682
|
+
__publicField(this, "maxSizeBytes");
|
|
76683
|
+
__publicField(this, "erroredRetentionMs");
|
|
76684
|
+
__publicField(this, "intervalMs");
|
|
76685
|
+
__publicField(this, "timer");
|
|
76686
|
+
__publicField(this, "running", false);
|
|
76687
|
+
__publicField(this, "lastSweepAt", null);
|
|
76688
|
+
__publicField(this, "lastResult", null);
|
|
76689
|
+
// Prepared once at construction and reused on every sweep — re-preparing per
|
|
76690
|
+
// sweep would leak GC-finalised statement objects proportional to sweep count.
|
|
76691
|
+
__publicField(this, "phase1Stmt");
|
|
76692
|
+
__publicField(this, "phase2Stmt");
|
|
76693
|
+
__publicField(this, "phase3Stmt");
|
|
76694
|
+
this.queue = options.queue;
|
|
76695
|
+
this.log = options.log;
|
|
76696
|
+
this.retentionMs = (options.retentionDays ?? DEFAULT_RETENTION_DAYS) * 24 * 60 * 60 * 1e3;
|
|
76697
|
+
this.maxSizeBytes = (options.maxSizeMb ?? DEFAULT_MAX_SIZE_MB) * 1024 * 1024;
|
|
76698
|
+
this.erroredRetentionMs = (options.erroredRetentionDays ?? DEFAULT_ERRORED_RETENTION_DAYS) * 24 * 60 * 60 * 1e3;
|
|
76699
|
+
this.intervalMs = (options.sweepIntervalSecs ?? DEFAULT_SWEEP_INTERVAL_SECS) * 1e3;
|
|
76700
|
+
const db = this.queue.getDb();
|
|
76701
|
+
this.phase1Stmt = db.prepare(RETENTION_PHASE1_DELETE);
|
|
76702
|
+
this.phase2Stmt = db.prepare(RETENTION_PHASE2_DELETE);
|
|
76703
|
+
this.phase3Stmt = db.prepare(RETENTION_PHASE3_DELETE);
|
|
76704
|
+
}
|
|
76705
|
+
/** Starts the periodic sweep timer and runs one sweep right away. No-op if already started. */
|
|
76706
|
+
start() {
|
|
76707
|
+
if (this.timer) {
|
|
76708
|
+
return;
|
|
76709
|
+
}
|
|
76710
|
+
this.timer = setInterval(() => this.runSweep(), this.intervalMs);
|
|
76711
|
+
if (typeof this.timer.unref === "function") {
|
|
76712
|
+
this.timer.unref();
|
|
76713
|
+
}
|
|
76714
|
+
this.runSweep();
|
|
76715
|
+
}
|
|
76716
|
+
/** Wrapper around {@link RetentionSweeper.sweep} that logs failures instead of throwing. */
|
|
76717
|
+
runSweep() {
|
|
76718
|
+
try {
|
|
76719
|
+
this.sweep();
|
|
76720
|
+
} catch (err2) {
|
|
76721
|
+
this.log.error(`Retention sweep crashed: ${Re(err2)}`);
|
|
76722
|
+
}
|
|
76723
|
+
}
|
|
76724
|
+
/** Stops the periodic sweep timer. */
|
|
76725
|
+
stop() {
|
|
76726
|
+
if (this.timer) {
|
|
76727
|
+
clearInterval(this.timer);
|
|
76728
|
+
this.timer = void 0;
|
|
76729
|
+
}
|
|
76730
|
+
}
|
|
76731
|
+
/**
|
|
76732
|
+
* Run a single sweep cycle immediately. Safe to call from `start`'s timer or tests.
|
|
76733
|
+
*
|
|
76734
|
+
* Synchronous: every SQLite call here is synchronous, so there is no `await` to
|
|
76735
|
+
* make. Keeping the signature non-`async` means the `running` guard can't be
|
|
76736
|
+
* defeated by a future edit that adds an `await` mid-sweep and opens a
|
|
76737
|
+
* re-entrancy window.
|
|
76738
|
+
* @param now - Override the "now" timestamp used for retention comparisons.
|
|
76739
|
+
* @returns The deletion counts and DB size after this sweep cycle.
|
|
76740
|
+
*/
|
|
76741
|
+
sweep(now = Date.now()) {
|
|
76742
|
+
if (this.running) {
|
|
76743
|
+
return this.lastResult ?? { deletedProcessed: 0, deletedErrored: 0, dbSizeBytesAfter: this.queue.getDbSizeBytes() };
|
|
76744
|
+
}
|
|
76745
|
+
this.running = true;
|
|
76746
|
+
const startedAt = Date.now();
|
|
76747
|
+
try {
|
|
76748
|
+
const cutoffProcessed = now - this.retentionMs;
|
|
76749
|
+
const cutoffErrored = now - this.erroredRetentionMs;
|
|
76750
|
+
const phase1 = this.phase1Stmt.run(cutoffProcessed);
|
|
76751
|
+
let deletedProcessed = Number(phase1.changes);
|
|
76752
|
+
let deletedErrored = 0;
|
|
76753
|
+
const sizeBudget = this.maxSizeBytes;
|
|
76754
|
+
const batchSize = 1e3;
|
|
76755
|
+
while (this.queue.getDbSizeBytes() > sizeBudget) {
|
|
76756
|
+
const info = this.phase2Stmt.run(batchSize);
|
|
76757
|
+
if (info.changes === 0) {
|
|
76758
|
+
break;
|
|
76759
|
+
}
|
|
76760
|
+
deletedProcessed += Number(info.changes);
|
|
76761
|
+
}
|
|
76762
|
+
if (this.queue.getDbSizeBytes() > sizeBudget) {
|
|
76763
|
+
while (this.queue.getDbSizeBytes() > sizeBudget) {
|
|
76764
|
+
const info = this.phase3Stmt.run(cutoffErrored, batchSize);
|
|
76765
|
+
if (info.changes === 0) {
|
|
76766
|
+
break;
|
|
76767
|
+
}
|
|
76768
|
+
deletedErrored += Number(info.changes);
|
|
76769
|
+
}
|
|
76770
|
+
}
|
|
76771
|
+
this.queue.checkpointWal();
|
|
76772
|
+
const result = {
|
|
76773
|
+
deletedProcessed,
|
|
76774
|
+
deletedErrored,
|
|
76775
|
+
dbSizeBytesAfter: this.queue.getDbSizeBytes()
|
|
76776
|
+
};
|
|
76777
|
+
this.lastResult = result;
|
|
76778
|
+
this.lastSweepAt = now;
|
|
76779
|
+
if (deletedProcessed > 0 || deletedErrored > 0) {
|
|
76780
|
+
this.log.info(
|
|
76781
|
+
`Retention sweep: deleted ${deletedProcessed} processed, ${deletedErrored} errored. DB ${result.dbSizeBytesAfter} bytes. Took ${Date.now() - startedAt} ms.`
|
|
76782
|
+
);
|
|
76783
|
+
}
|
|
76784
|
+
return result;
|
|
76785
|
+
} finally {
|
|
76786
|
+
this.running = false;
|
|
76787
|
+
}
|
|
76788
|
+
}
|
|
76789
|
+
/** @returns Time of the most recent sweep, or null if none has run yet. */
|
|
76790
|
+
getLastSweepAt() {
|
|
76791
|
+
return this.lastSweepAt;
|
|
76792
|
+
}
|
|
76793
|
+
/** @returns Result of the most recent sweep, or null if none has run yet. */
|
|
76794
|
+
getLastResult() {
|
|
76795
|
+
return this.lastResult;
|
|
76796
|
+
}
|
|
76797
|
+
};
|
|
76798
|
+
|
|
76799
|
+
// src/upgrader-utils.ts
|
|
76800
|
+
var import_node_fs4 = require("node:fs");
|
|
74646
76801
|
var import_node_os3 = require("node:os");
|
|
74647
76802
|
var import_node_path4 = require("node:path");
|
|
74648
76803
|
var import_node_stream = require("node:stream");
|
|
@@ -74665,10 +76820,10 @@ async function downloadRelease(version, path4) {
|
|
|
74665
76820
|
}
|
|
74666
76821
|
const readable = import_node_stream.Readable.fromWeb(result.body);
|
|
74667
76822
|
try {
|
|
74668
|
-
await (0, import_promises.pipeline)(readable, (0,
|
|
76823
|
+
await (0, import_promises.pipeline)(readable, (0, import_node_fs4.createWriteStream)(path4));
|
|
74669
76824
|
} catch (err2) {
|
|
74670
|
-
if ((0,
|
|
74671
|
-
(0,
|
|
76825
|
+
if ((0, import_node_fs4.existsSync)(path4)) {
|
|
76826
|
+
(0, import_node_fs4.unlinkSync)(path4);
|
|
74672
76827
|
}
|
|
74673
76828
|
throw new Error(`Error while downloading release version ${version} to ${path4}`, { cause: err2 });
|
|
74674
76829
|
}
|
|
@@ -74720,6 +76875,22 @@ async function execAsync(command, options) {
|
|
|
74720
76875
|
});
|
|
74721
76876
|
});
|
|
74722
76877
|
}
|
|
76878
|
+
var STOP_DRAIN_TIMEOUT_MS = 1e4;
|
|
76879
|
+
function raceWithTimeout(promise, ms) {
|
|
76880
|
+
return new Promise((resolve2, reject) => {
|
|
76881
|
+
const timer = setTimeout(() => resolve2("timeout"), ms);
|
|
76882
|
+
promise.then(
|
|
76883
|
+
(value) => {
|
|
76884
|
+
clearTimeout(timer);
|
|
76885
|
+
resolve2(value);
|
|
76886
|
+
},
|
|
76887
|
+
(err2) => {
|
|
76888
|
+
clearTimeout(timer);
|
|
76889
|
+
reject(err2);
|
|
76890
|
+
}
|
|
76891
|
+
);
|
|
76892
|
+
});
|
|
76893
|
+
}
|
|
74723
76894
|
var _App = class _App {
|
|
74724
76895
|
constructor(medplum, agentId, logLevel, options) {
|
|
74725
76896
|
__publicField(this, "medplum");
|
|
@@ -74746,6 +76917,8 @@ var _App = class _App {
|
|
|
74746
76917
|
__publicField(this, "logStatsTimer");
|
|
74747
76918
|
__publicField(this, "config");
|
|
74748
76919
|
__publicField(this, "lastHeartbeatSentTime", -1);
|
|
76920
|
+
__publicField(this, "durableQueue");
|
|
76921
|
+
__publicField(this, "retentionSweeper");
|
|
74749
76922
|
// Whether this process owns the `medplum-agent` PID, i.e. it is the sole agent that should
|
|
74750
76923
|
// touch the data plane. A normally-started agent is primary from the outset (main.ts creates
|
|
74751
76924
|
// the PID before start()). An upgrading agent stays non-primary until it wins the PID from the
|
|
@@ -74759,7 +76932,7 @@ var _App = class _App {
|
|
|
74759
76932
|
}
|
|
74760
76933
|
async start() {
|
|
74761
76934
|
this.log.info("Medplum service starting...");
|
|
74762
|
-
this.isPrimary = !(0,
|
|
76935
|
+
this.isPrimary = !(0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH);
|
|
74763
76936
|
await this.startWebSocket();
|
|
74764
76937
|
const { listenersStarted } = await this.beginReloadConfig();
|
|
74765
76938
|
const upgradeManifest = this.consumeUpgradeManifest();
|
|
@@ -74787,12 +76960,12 @@ var _App = class _App {
|
|
|
74787
76960
|
* @returns The parsed manifest, or undefined if no upgrade is in progress.
|
|
74788
76961
|
*/
|
|
74789
76962
|
consumeUpgradeManifest() {
|
|
74790
|
-
if (!(0,
|
|
76963
|
+
if (!(0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH)) {
|
|
74791
76964
|
return void 0;
|
|
74792
76965
|
}
|
|
74793
|
-
const upgradeFile = (0,
|
|
76966
|
+
const upgradeFile = (0, import_node_fs5.readFileSync)(UPGRADE_MANIFEST_PATH, { encoding: "utf-8" });
|
|
74794
76967
|
const upgradeDetails = JSON.parse(upgradeFile);
|
|
74795
|
-
(0,
|
|
76968
|
+
(0, import_node_fs5.unlinkSync)(UPGRADE_MANIFEST_PATH);
|
|
74796
76969
|
return upgradeDetails;
|
|
74797
76970
|
}
|
|
74798
76971
|
async maybeFinalizeUpgrade(upgradeDetails) {
|
|
@@ -74887,9 +77060,10 @@ var _App = class _App {
|
|
|
74887
77060
|
WebSocket: wrapper_default,
|
|
74888
77061
|
binaryType: "nodebuffer"
|
|
74889
77062
|
});
|
|
74890
|
-
this.webSocket.addEventListener("error", () => {
|
|
77063
|
+
this.webSocket.addEventListener("error", (event) => {
|
|
74891
77064
|
if (!this.shutdown) {
|
|
74892
|
-
|
|
77065
|
+
const error = event.error;
|
|
77066
|
+
this.log.error("WebSocket closed due to an error", error);
|
|
74893
77067
|
}
|
|
74894
77068
|
});
|
|
74895
77069
|
this.webSocket.addEventListener("open", async () => {
|
|
@@ -74907,6 +77081,7 @@ var _App = class _App {
|
|
|
74907
77081
|
if (!this.shutdown && this.live) {
|
|
74908
77082
|
this.live = false;
|
|
74909
77083
|
this.log.info("WebSocket closed");
|
|
77084
|
+
this.forEachChannelWorker((worker) => worker.onWebSocketDisconnect());
|
|
74910
77085
|
}
|
|
74911
77086
|
});
|
|
74912
77087
|
this.webSocket.addEventListener("message", async (e) => {
|
|
@@ -74924,6 +77099,7 @@ var _App = class _App {
|
|
|
74924
77099
|
this.notLiveHeartbeats = 0;
|
|
74925
77100
|
this.outstandingHeartbeats = 0;
|
|
74926
77101
|
this.startWebSocketWorker();
|
|
77102
|
+
this.forEachChannelWorker((worker) => worker.notify());
|
|
74927
77103
|
this.log.info("Successfully connected to Medplum server");
|
|
74928
77104
|
break;
|
|
74929
77105
|
case "agent:heartbeat:request":
|
|
@@ -74944,9 +77120,12 @@ var _App = class _App {
|
|
|
74944
77120
|
if (!command.callback) {
|
|
74945
77121
|
this.log.warn("Transmit response missing callback");
|
|
74946
77122
|
}
|
|
77123
|
+
if (this.routeServerResponseToWorker(command)) {
|
|
77124
|
+
break;
|
|
77125
|
+
}
|
|
74947
77126
|
if (this.config?.status !== "active") {
|
|
74948
77127
|
this.sendAgentDisabledError(command);
|
|
74949
|
-
} else if (
|
|
77128
|
+
} else if (command.statusCode === void 0 || command.statusCode < 400) {
|
|
74950
77129
|
this.addToHl7Queue(command);
|
|
74951
77130
|
} else {
|
|
74952
77131
|
this.log.error(`Error during handling transmit request: ${command.body}`);
|
|
@@ -75049,6 +77228,16 @@ var _App = class _App {
|
|
|
75049
77228
|
const keepAlive = agent?.setting?.find((setting) => setting.name === "keepAlive")?.valueBoolean;
|
|
75050
77229
|
const maxClientsPerRemote = agent?.setting?.find((setting) => setting.name === "maxClientsPerRemote")?.valueInteger;
|
|
75051
77230
|
const logStatsFreqSecs = agent?.setting?.find((setting) => setting.name === "logStatsFreqSecs")?.valueInteger;
|
|
77231
|
+
const durableQueueOn = agent?.setting?.find((setting) => setting.name === "durableQueue")?.valueBoolean ?? false;
|
|
77232
|
+
const queueDbPath = agent?.setting?.find((setting) => setting.name === "queueDbPath")?.valueString;
|
|
77233
|
+
const queueRetentionDays = agent?.setting?.find((setting) => setting.name === "queueRetentionDays")?.valueInteger;
|
|
77234
|
+
const queueRetentionMaxMb = agent?.setting?.find((setting) => setting.name === "queueRetentionMaxMb")?.valueInteger;
|
|
77235
|
+
const queueErroredRetentionDays = agent?.setting?.find(
|
|
77236
|
+
(setting) => setting.name === "queueErroredRetentionDays"
|
|
77237
|
+
)?.valueInteger;
|
|
77238
|
+
const queueSweepIntervalSecs = agent?.setting?.find(
|
|
77239
|
+
(setting) => setting.name === "queueSweepIntervalSecs"
|
|
77240
|
+
)?.valueInteger;
|
|
75052
77241
|
if (this.keepAlive !== keepAlive) {
|
|
75053
77242
|
const results = await Promise.allSettled(Array.from(this.hl7Clients.values()).map((pool) => pool.closeAll()));
|
|
75054
77243
|
for (const result of results) {
|
|
@@ -75079,9 +77268,136 @@ var _App = class _App {
|
|
|
75079
77268
|
this.log.info(`Stats logging enabled. Logging stats every ${this.logStatsFreqSecs} seconds...`);
|
|
75080
77269
|
this.logStatsTimer ??= setInterval(() => this.logStats(), this.logStatsFreqSecs * 1e3);
|
|
75081
77270
|
}
|
|
77271
|
+
this.reconcileDurableQueue({
|
|
77272
|
+
durableQueueOn,
|
|
77273
|
+
queueDbPath,
|
|
77274
|
+
queueRetentionDays,
|
|
77275
|
+
queueRetentionMaxMb,
|
|
77276
|
+
queueErroredRetentionDays,
|
|
77277
|
+
queueSweepIntervalSecs
|
|
77278
|
+
});
|
|
75082
77279
|
const startPromises = await this.hydrateListeners();
|
|
75083
77280
|
return { listenersStarted: this.waitForChannelsToStart(startPromises) };
|
|
75084
77281
|
}
|
|
77282
|
+
/**
|
|
77283
|
+
* Opens, closes, or reconfigures the durable queue based on the latest config.
|
|
77284
|
+
*
|
|
77285
|
+
* Toggling `durableQueue` between true and false at runtime triggers a queue
|
|
77286
|
+
* open/close. Toggling other queue settings (retention, sweep interval) starts
|
|
77287
|
+
* a fresh {@link RetentionSweeper} against the existing DB without reopening it.
|
|
77288
|
+
*
|
|
77289
|
+
* Changing `queueDbPath` while the queue is already open is intentionally NOT
|
|
77290
|
+
* supported — that would require moving / closing the existing DB. Operators
|
|
77291
|
+
* who need to change the path should disable the queue, then re-enable with the
|
|
77292
|
+
* new path.
|
|
77293
|
+
* @param args - The current queue-related settings drawn from the Agent resource.
|
|
77294
|
+
* @param args.durableQueueOn - Master switch — true to open the queue, false to close it.
|
|
77295
|
+
* @param args.queueDbPath - Optional override for the DB file path (defaults to `<logDir>/medplum-agent-queue.sqlite`).
|
|
77296
|
+
* @param args.queueRetentionDays - Time-based retention window for `processed` rows, in days.
|
|
77297
|
+
* @param args.queueRetentionMaxMb - Soft cap on DB size, in MiB.
|
|
77298
|
+
* @param args.queueErroredRetentionDays - Floor on `errored` / `nacked` retention, in days.
|
|
77299
|
+
* @param args.queueSweepIntervalSecs - How often the retention sweeper runs, in seconds.
|
|
77300
|
+
*/
|
|
77301
|
+
reconcileDurableQueue(args) {
|
|
77302
|
+
if (!args.durableQueueOn) {
|
|
77303
|
+
if (this.durableQueue) {
|
|
77304
|
+
this.log.info("durableQueue disabled \u2014 closing queue.");
|
|
77305
|
+
this.retentionSweeper?.stop();
|
|
77306
|
+
this.retentionSweeper = void 0;
|
|
77307
|
+
this.durableQueue.close();
|
|
77308
|
+
this.durableQueue = void 0;
|
|
77309
|
+
}
|
|
77310
|
+
return;
|
|
77311
|
+
}
|
|
77312
|
+
if (!this.durableQueue) {
|
|
77313
|
+
const path4 = args.queueDbPath ?? this.defaultQueueDbPath();
|
|
77314
|
+
try {
|
|
77315
|
+
this.durableQueue = DurableQueue.open({ path: path4, log: this.log });
|
|
77316
|
+
this.log.info(`Durable queue opened at ${path4}.`);
|
|
77317
|
+
} catch (err2) {
|
|
77318
|
+
this.log.error(`Failed to open durable queue at ${path4}: ${Re(err2)}`);
|
|
77319
|
+
this.durableQueue = void 0;
|
|
77320
|
+
return;
|
|
77321
|
+
}
|
|
77322
|
+
}
|
|
77323
|
+
this.durableQueue.startDispatchLease(() => this.onBecameQueueLeader());
|
|
77324
|
+
this.retentionSweeper?.stop();
|
|
77325
|
+
this.retentionSweeper = new RetentionSweeper({
|
|
77326
|
+
queue: this.durableQueue,
|
|
77327
|
+
log: this.log,
|
|
77328
|
+
retentionDays: args.queueRetentionDays,
|
|
77329
|
+
maxSizeMb: args.queueRetentionMaxMb,
|
|
77330
|
+
erroredRetentionDays: args.queueErroredRetentionDays,
|
|
77331
|
+
sweepIntervalSecs: args.queueSweepIntervalSecs
|
|
77332
|
+
});
|
|
77333
|
+
this.retentionSweeper.start();
|
|
77334
|
+
}
|
|
77335
|
+
/**
|
|
77336
|
+
* Called by the {@link DurableQueue} dispatch-lease loop the first time we take the lease.
|
|
77337
|
+
*
|
|
77338
|
+
* This is the single point that runs `recoverOnStartup` and spins up the
|
|
77339
|
+
* channel workers. Both depend on us being the only writer — running them at
|
|
77340
|
+
* raw queue-open time would race with any peer that still holds the lease.
|
|
77341
|
+
*
|
|
77342
|
+
* Re-entrancy: if we lose and regain the lease later, this fires again. The
|
|
77343
|
+
* recovery sweep is idempotent (no `claimed`/`inflight` rows means no work), and
|
|
77344
|
+
* `maybeStartWorker` is a no-op if the worker is already running.
|
|
77345
|
+
*/
|
|
77346
|
+
onBecameQueueLeader() {
|
|
77347
|
+
const queue = this.durableQueue;
|
|
77348
|
+
if (!queue) {
|
|
77349
|
+
return;
|
|
77350
|
+
}
|
|
77351
|
+
const recovered = queue.recoverOnStartup();
|
|
77352
|
+
if (recovered.requeued > 0 || recovered.failed > 0) {
|
|
77353
|
+
this.log.info(
|
|
77354
|
+
`Acquired queue lease \u2014 recovered interrupted rows: ${recovered.requeued} requeued (unsent), ${recovered.failed} failed (ambiguous, in-flight).`
|
|
77355
|
+
);
|
|
77356
|
+
}
|
|
77357
|
+
const errors = [];
|
|
77358
|
+
for (const channel of this.channels.values()) {
|
|
77359
|
+
if (channel instanceof AgentHl7Channel) {
|
|
77360
|
+
try {
|
|
77361
|
+
channel.onBecameQueueLeader();
|
|
77362
|
+
} catch (err2) {
|
|
77363
|
+
errors.push(err2);
|
|
77364
|
+
}
|
|
77365
|
+
}
|
|
77366
|
+
}
|
|
77367
|
+
if (errors.length > 0) {
|
|
77368
|
+
throw new Error(`Failed to start ${errors.length} HL7 channel worker(s) after acquiring queue lease`, {
|
|
77369
|
+
cause: errors
|
|
77370
|
+
});
|
|
77371
|
+
}
|
|
77372
|
+
}
|
|
77373
|
+
/** @returns True when this agent currently holds the durable-queue lease. */
|
|
77374
|
+
isQueueLeader() {
|
|
77375
|
+
return this.durableQueue?.isLeader() ?? false;
|
|
77376
|
+
}
|
|
77377
|
+
/**
|
|
77378
|
+
* Default location for the queue DB file when no override is provided.
|
|
77379
|
+
*
|
|
77380
|
+
* Co-locating with the main logger's log directory keeps everything an
|
|
77381
|
+
* operator needs to mount a persistent volume in one place. The fallback is
|
|
77382
|
+
* the current working directory — same default an unconfigured agent uses.
|
|
77383
|
+
* @returns Absolute path to the default queue DB file.
|
|
77384
|
+
*/
|
|
77385
|
+
defaultQueueDbPath() {
|
|
77386
|
+
const baseDir = isWinstonWrapperLogger(this.log) && this.log.logDir || import_node_process2.default.cwd();
|
|
77387
|
+
const sep = baseDir.endsWith("/") || baseDir.endsWith("\\") ? "" : "/";
|
|
77388
|
+
return `${baseDir}${sep}medplum-agent-queue.sqlite`;
|
|
77389
|
+
}
|
|
77390
|
+
/**
|
|
77391
|
+
* @returns The opened {@link DurableQueue}, or undefined when the queue setting
|
|
77392
|
+
* is off. This is the OPEN handle, valid regardless of leadership — for intake,
|
|
77393
|
+
* maintenance (WAL checkpoint, retention sweep), and diagnostics. For the
|
|
77394
|
+
* dispatch path, gate worker startup on {@link DurableQueue.isLeader} (the cheap
|
|
77395
|
+
* optimistic check); the authoritative gate is implicit in the dispatch ops
|
|
77396
|
+
* themselves, which throw `QueueLeaseError` once a peer holds the lease.
|
|
77397
|
+
*/
|
|
77398
|
+
getDurableQueue() {
|
|
77399
|
+
return this.durableQueue;
|
|
77400
|
+
}
|
|
75085
77401
|
getStats() {
|
|
75086
77402
|
const stats = getCurrentStats();
|
|
75087
77403
|
let totalHl7Clients = 0;
|
|
@@ -75107,7 +77423,49 @@ var _App = class _App {
|
|
|
75107
77423
|
live: this.live,
|
|
75108
77424
|
outstandingHeartbeats: this.outstandingHeartbeats,
|
|
75109
77425
|
channelStats,
|
|
75110
|
-
clientStats
|
|
77426
|
+
clientStats,
|
|
77427
|
+
...this.durableQueue ? { durableQueue: this.getDurableQueueStats(this.durableQueue) } : {}
|
|
77428
|
+
};
|
|
77429
|
+
}
|
|
77430
|
+
/**
|
|
77431
|
+
* Snapshot of durable-queue health, surfaced in `agent:stats:response`.
|
|
77432
|
+
*
|
|
77433
|
+
* Structured to fit the `AgentStatValue` shape — 3 nested levels of primitive
|
|
77434
|
+
* records — so `AgentStats`'s index signature stays satisfied. Null sentinels
|
|
77435
|
+
* become `-1` for the same reason ("never swept" reads as -1 on the wire).
|
|
77436
|
+
*
|
|
77437
|
+
* The field is only included when the queue is on; consumers that detect its
|
|
77438
|
+
* absence know the queue is disabled, which is more honest than reporting a
|
|
77439
|
+
* zeroed structure that a dashboard could misread as "queue on but idle."
|
|
77440
|
+
* @param queue - The opened durable queue to read counters from.
|
|
77441
|
+
* @returns A primitive-friendly snapshot fit for `AgentStats`.
|
|
77442
|
+
*/
|
|
77443
|
+
getDurableQueueStats(queue) {
|
|
77444
|
+
const counts = queue.countByState();
|
|
77445
|
+
const channelDepth = {};
|
|
77446
|
+
for (const channel of this.channels.values()) {
|
|
77447
|
+
if (channel instanceof AgentHl7Channel) {
|
|
77448
|
+
const d2 = queue.getChannelDepth(channel.getDefinition().name);
|
|
77449
|
+
channelDepth[channel.getDefinition().name] = {
|
|
77450
|
+
queued: d2.queued,
|
|
77451
|
+
claimed: d2.claimed,
|
|
77452
|
+
inflight: d2.inflight,
|
|
77453
|
+
// -1 means "no queued rows" (a zero-aged-row would be 0).
|
|
77454
|
+
oldestQueuedAgeMs: d2.oldestQueuedAgeMs ?? -1
|
|
77455
|
+
};
|
|
77456
|
+
}
|
|
77457
|
+
}
|
|
77458
|
+
const lastResult = this.retentionSweeper?.getLastResult();
|
|
77459
|
+
return {
|
|
77460
|
+
enabled: true,
|
|
77461
|
+
isLeader: this.isQueueLeader(),
|
|
77462
|
+
dbSizeBytes: queue.getDbSizeBytes(),
|
|
77463
|
+
countsByState: counts,
|
|
77464
|
+
channelDepth,
|
|
77465
|
+
// -1 means "never swept yet."
|
|
77466
|
+
lastSweepAt: this.retentionSweeper?.getLastSweepAt() ?? -1,
|
|
77467
|
+
lastSweepDeletedProcessed: lastResult?.deletedProcessed ?? -1,
|
|
77468
|
+
lastSweepDeletedErrored: lastResult?.deletedErrored ?? -1
|
|
75111
77469
|
};
|
|
75112
77470
|
}
|
|
75113
77471
|
logStats() {
|
|
@@ -75334,25 +77692,131 @@ var _App = class _App {
|
|
|
75334
77692
|
this.webSocket.close();
|
|
75335
77693
|
this.webSocket = void 0;
|
|
75336
77694
|
}
|
|
77695
|
+
try {
|
|
77696
|
+
const drained = await raceWithTimeout(this.drainForStop(), STOP_DRAIN_TIMEOUT_MS);
|
|
77697
|
+
if (drained === "timeout") {
|
|
77698
|
+
this.log.warn(
|
|
77699
|
+
`Timed out after ${STOP_DRAIN_TIMEOUT_MS}ms waiting for clients/channels to stop \u2014 proceeding with shutdown.`
|
|
77700
|
+
);
|
|
77701
|
+
}
|
|
77702
|
+
} catch (err2) {
|
|
77703
|
+
this.log.error(`Error while stopping clients/channels: ${Re(err2)}`);
|
|
77704
|
+
}
|
|
77705
|
+
if (this.retentionSweeper) {
|
|
77706
|
+
this.retentionSweeper.stop();
|
|
77707
|
+
this.retentionSweeper = void 0;
|
|
77708
|
+
}
|
|
77709
|
+
if (this.durableQueue) {
|
|
77710
|
+
this.durableQueue.close();
|
|
77711
|
+
this.durableQueue = void 0;
|
|
77712
|
+
}
|
|
77713
|
+
this.log.info("Medplum service stopped successfully");
|
|
77714
|
+
}
|
|
77715
|
+
/**
|
|
77716
|
+
* Closes all outbound HL7 client pools, then stops every channel. Factored out
|
|
77717
|
+
* of {@link App.stop} so the whole drain can be raced against a single timeout.
|
|
77718
|
+
*
|
|
77719
|
+
* Uses allSettled rather than all: one channel failing to stop must not
|
|
77720
|
+
* abandon the wait on its siblings — the durable-queue DB is closed right
|
|
77721
|
+
* after this returns, so every channel that *can* drain must finish first.
|
|
77722
|
+
*/
|
|
77723
|
+
async drainForStop() {
|
|
75337
77724
|
if (this.hl7Clients.size !== 0) {
|
|
75338
77725
|
const poolClosePromises = [];
|
|
75339
77726
|
for (const pool of this.hl7Clients.values()) {
|
|
75340
77727
|
poolClosePromises.push(pool.closeAll());
|
|
75341
77728
|
}
|
|
75342
|
-
await Promise.
|
|
77729
|
+
const poolResults = await Promise.allSettled(poolClosePromises);
|
|
77730
|
+
for (const result of poolResults) {
|
|
77731
|
+
if (result.status === "rejected") {
|
|
77732
|
+
this.log.error(`Error while closing HL7 client pool: ${Re(result.reason)}`);
|
|
77733
|
+
}
|
|
77734
|
+
}
|
|
75343
77735
|
this.hl7Clients.clear();
|
|
75344
77736
|
}
|
|
75345
77737
|
const channelStopPromises = [];
|
|
75346
77738
|
for (const channel of this.channels.values()) {
|
|
75347
77739
|
channelStopPromises.push(channel.stop());
|
|
75348
77740
|
}
|
|
75349
|
-
await Promise.
|
|
75350
|
-
|
|
77741
|
+
const results = await Promise.allSettled(channelStopPromises);
|
|
77742
|
+
for (const result of results) {
|
|
77743
|
+
if (result.status === "rejected") {
|
|
77744
|
+
this.log.error(`Error while stopping channel: ${Re(result.reason)}`);
|
|
77745
|
+
}
|
|
77746
|
+
}
|
|
77747
|
+
}
|
|
77748
|
+
/**
|
|
77749
|
+
* Dispatches an `agent:transmit:response` to the owning channel's worker, if any.
|
|
77750
|
+
* @param response - The response message received over the agent WebSocket.
|
|
77751
|
+
* @returns True if a worker claimed the response (caller should stop here);
|
|
77752
|
+
* false if no worker matched and legacy handling should run.
|
|
77753
|
+
*/
|
|
77754
|
+
routeServerResponseToWorker(response2) {
|
|
77755
|
+
if (!this.durableQueue) {
|
|
77756
|
+
return false;
|
|
77757
|
+
}
|
|
77758
|
+
if (!response2.channel) {
|
|
77759
|
+
return false;
|
|
77760
|
+
}
|
|
77761
|
+
const channel = this.channels.get(response2.channel);
|
|
77762
|
+
if (!(channel instanceof AgentHl7Channel) || !channel.worker) {
|
|
77763
|
+
return false;
|
|
77764
|
+
}
|
|
77765
|
+
channel.worker.onServerResponse(response2);
|
|
77766
|
+
return true;
|
|
75351
77767
|
}
|
|
75352
77768
|
addToWebSocketQueue(message) {
|
|
75353
77769
|
this.webSocketQueue.push(message);
|
|
75354
77770
|
this.startWebSocketWorker();
|
|
75355
77771
|
}
|
|
77772
|
+
/** @returns True when the agent WebSocket is connected and the server has acknowledged the connect request. */
|
|
77773
|
+
isLive() {
|
|
77774
|
+
return this.live;
|
|
77775
|
+
}
|
|
77776
|
+
/**
|
|
77777
|
+
* Removes a not-yet-sent `agent:transmit:request` from the WebSocket queue.
|
|
77778
|
+
*
|
|
77779
|
+
* Used by {@link ChannelQueueWorker.onWebSocketDisconnect} to decide whether
|
|
77780
|
+
* an in-flight row can be safely requeued: a request still in this queue
|
|
77781
|
+
* provably never reached the server. A request not found here was either
|
|
77782
|
+
* already written to the socket or is mid-send — both ambiguous, so the
|
|
77783
|
+
* caller must treat `false` as "may have been delivered".
|
|
77784
|
+
* @param callbackId - The `callback` ID of the transmit request to remove.
|
|
77785
|
+
* @returns True if the request was found and removed before being sent.
|
|
77786
|
+
*/
|
|
77787
|
+
removeUnsentTransmit(callbackId) {
|
|
77788
|
+
const index = this.webSocketQueue.findIndex(
|
|
77789
|
+
(msg) => msg.type === "agent:transmit:request" && msg.callback === callbackId
|
|
77790
|
+
);
|
|
77791
|
+
if (index === -1) {
|
|
77792
|
+
return false;
|
|
77793
|
+
}
|
|
77794
|
+
this.webSocketQueue.splice(index, 1);
|
|
77795
|
+
return true;
|
|
77796
|
+
}
|
|
77797
|
+
/**
|
|
77798
|
+
* Invokes `fn` for every channel that currently has a durable-queue worker running.
|
|
77799
|
+
*
|
|
77800
|
+
* Collects any failures so one worker throwing can't stop `fn` from reaching the
|
|
77801
|
+
* rest, then surfaces them together as an aggregate error with the collected
|
|
77802
|
+
* errors as its `cause`.
|
|
77803
|
+
* @param fn - Callback applied to each running {@link ChannelQueueWorker}.
|
|
77804
|
+
*/
|
|
77805
|
+
forEachChannelWorker(fn2) {
|
|
77806
|
+
const errors = [];
|
|
77807
|
+
for (const channel of this.channels.values()) {
|
|
77808
|
+
if (channel instanceof AgentHl7Channel && channel.worker) {
|
|
77809
|
+
try {
|
|
77810
|
+
fn2(channel.worker);
|
|
77811
|
+
} catch (err2) {
|
|
77812
|
+
errors.push(err2);
|
|
77813
|
+
}
|
|
77814
|
+
}
|
|
77815
|
+
}
|
|
77816
|
+
if (errors.length > 0) {
|
|
77817
|
+
throw new Error(`Failed to run channel worker callback for ${errors.length} channel(s)`, { cause: errors });
|
|
77818
|
+
}
|
|
77819
|
+
}
|
|
75356
77820
|
addToHl7Queue(message) {
|
|
75357
77821
|
this.hl7Queue.push(message);
|
|
75358
77822
|
this.trySendToHl7Connection();
|
|
@@ -75531,8 +77995,8 @@ ${result}`);
|
|
|
75531
77995
|
forceKillApp("medplum-agent-upgrader");
|
|
75532
77996
|
removePidFile("medplum-agent-upgrader");
|
|
75533
77997
|
}
|
|
75534
|
-
if ((0,
|
|
75535
|
-
(0,
|
|
77998
|
+
if ((0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH)) {
|
|
77999
|
+
(0, import_node_fs5.unlinkSync)(UPGRADE_MANIFEST_PATH);
|
|
75536
78000
|
}
|
|
75537
78001
|
}
|
|
75538
78002
|
try {
|
|
@@ -75551,7 +78015,7 @@ ${result}`);
|
|
|
75551
78015
|
}
|
|
75552
78016
|
try {
|
|
75553
78017
|
const command = __filename;
|
|
75554
|
-
const logFile = (0,
|
|
78018
|
+
const logFile = (0, import_node_fs5.openSync)(UPGRADER_LOG_PATH, "w+");
|
|
75555
78019
|
child = (0, import_node_child_process.spawn)(command, message.version ? ["--upgrade", message.version] : ["--upgrade"], {
|
|
75556
78020
|
detached: true,
|
|
75557
78021
|
stdio: ["ignore", logFile, logFile, "ipc"]
|
|
@@ -75591,7 +78055,7 @@ ${result}`);
|
|
|
75591
78055
|
}
|
|
75592
78056
|
try {
|
|
75593
78057
|
this.log.info("Writing upgrade manifest...", { previousVersion: yn, targetVersion });
|
|
75594
|
-
(0,
|
|
78058
|
+
(0, import_node_fs5.writeFileSync)(
|
|
75595
78059
|
UPGRADE_MANIFEST_PATH,
|
|
75596
78060
|
JSON.stringify({
|
|
75597
78061
|
previousVersion: yn,
|
|
@@ -75655,14 +78119,25 @@ ${result}`);
|
|
|
75655
78119
|
}
|
|
75656
78120
|
}
|
|
75657
78121
|
async sendToWebSocket(message) {
|
|
75658
|
-
|
|
78122
|
+
const ws2 = this.webSocket;
|
|
78123
|
+
if (!ws2) {
|
|
75659
78124
|
throw new Error("WebSocket not connected");
|
|
75660
78125
|
}
|
|
75661
78126
|
if ("accessToken" in message) {
|
|
75662
78127
|
await this.medplum.refreshIfExpired();
|
|
75663
78128
|
message.accessToken = this.medplum.getAccessToken();
|
|
75664
78129
|
}
|
|
75665
|
-
|
|
78130
|
+
const payload = JSON.stringify(message);
|
|
78131
|
+
const callback = message.type === "agent:transmit:request" ? message.callback : void 0;
|
|
78132
|
+
if (callback && this.durableQueue) {
|
|
78133
|
+
const durableQueue = this.durableQueue;
|
|
78134
|
+
durableQueue.runInTransaction(() => {
|
|
78135
|
+
durableQueue.markSent(callback);
|
|
78136
|
+
ws2.send(payload);
|
|
78137
|
+
});
|
|
78138
|
+
} else {
|
|
78139
|
+
ws2.send(payload);
|
|
78140
|
+
}
|
|
75666
78141
|
}
|
|
75667
78142
|
sendAgentDisabledError(command) {
|
|
75668
78143
|
const errMsg = "Agent.status is currently set to off";
|
|
@@ -75791,7 +78266,7 @@ ${result}`);
|
|
|
75791
78266
|
});
|
|
75792
78267
|
}
|
|
75793
78268
|
isAgentUpgrading() {
|
|
75794
|
-
if ((0,
|
|
78269
|
+
if ((0, import_node_fs5.existsSync)(UPGRADE_MANIFEST_PATH)) {
|
|
75795
78270
|
return true;
|
|
75796
78271
|
}
|
|
75797
78272
|
if (isAppRunning("medplum-upgrading-agent") || isAppRunning("medplum-agent-upgrader")) {
|
|
@@ -75834,7 +78309,7 @@ async function agentMain(argv) {
|
|
|
75834
78309
|
console.log(" clientSecret: The OAuth client secret.");
|
|
75835
78310
|
console.log(" agentId: The Medplum agent ID.");
|
|
75836
78311
|
process.exit(0);
|
|
75837
|
-
} else if ((0,
|
|
78312
|
+
} else if ((0, import_node_fs6.existsSync)("agent.properties")) {
|
|
75838
78313
|
args = readPropertiesFile("agent.properties");
|
|
75839
78314
|
} else {
|
|
75840
78315
|
console.log("Missing arguments");
|
|
@@ -75895,13 +78370,13 @@ function readCommandLineArgs(argv) {
|
|
|
75895
78370
|
}
|
|
75896
78371
|
function readPropertiesFile(fileName) {
|
|
75897
78372
|
return Object.fromEntries(
|
|
75898
|
-
(0,
|
|
78373
|
+
(0, import_node_fs6.readFileSync)(fileName).toString().split("\n").map((line) => line.split("=").map((s) => s.trim()))
|
|
75899
78374
|
);
|
|
75900
78375
|
}
|
|
75901
78376
|
|
|
75902
78377
|
// src/upgrader.ts
|
|
75903
78378
|
var import_node_child_process2 = require("node:child_process");
|
|
75904
|
-
var
|
|
78379
|
+
var import_node_fs7 = require("node:fs");
|
|
75905
78380
|
var import_node_os5 = require("node:os");
|
|
75906
78381
|
var import_node_process3 = __toESM(require("node:process"));
|
|
75907
78382
|
var semver2 = __toESM(require_semver2());
|
|
@@ -75928,7 +78403,7 @@ async function upgraderMain(argv) {
|
|
|
75928
78403
|
}
|
|
75929
78404
|
version = argv[3] ?? await Dy("agent-upgrader");
|
|
75930
78405
|
binPath = getReleaseBinPath(version);
|
|
75931
|
-
if (!(0,
|
|
78406
|
+
if (!(0, import_node_fs7.existsSync)(binPath)) {
|
|
75932
78407
|
globalLogger.info(`Could not find binary at "${binPath}". Downloading release from GitHub...`);
|
|
75933
78408
|
await downloadRelease(version, binPath);
|
|
75934
78409
|
globalLogger.info("Release successfully downloaded");
|
|
@@ -75979,53 +78454,53 @@ async function main(argv) {
|
|
|
75979
78454
|
createPidFile("medplum-agent-upgrader");
|
|
75980
78455
|
await upgraderMain(argv);
|
|
75981
78456
|
} else if (argv[2] === "--remove-old-services") {
|
|
75982
|
-
const logFileFd = (0,
|
|
78457
|
+
const logFileFd = (0, import_node_fs8.openSync)(TEMP_LOG_FILE, "a");
|
|
75983
78458
|
let allAgentServices = [];
|
|
75984
78459
|
const currentServiceName = `MedplumAgent_${yn}`;
|
|
75985
78460
|
while (!allAgentServices.includes(currentServiceName)) {
|
|
75986
78461
|
const output = (0, import_node_child_process3.execSync)('cmd.exe /c sc query type= service state= all | findstr /i "SERVICE_NAME.*MedplumAgent"');
|
|
75987
|
-
(0,
|
|
78462
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `${output}\r
|
|
75988
78463
|
`, { encoding: "utf-8" });
|
|
75989
78464
|
allAgentServices = output.toString().trim().split("\n").map((line) => line.replace("SERVICE_NAME: ", "").trim());
|
|
75990
|
-
(0,
|
|
78465
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `All services: \r
|
|
75991
78466
|
${allAgentServices.join("\r\n")}\r
|
|
75992
78467
|
`, { encoding: "utf-8" });
|
|
75993
78468
|
}
|
|
75994
78469
|
const servicesToRemove = argv[3] === "--all" ? allAgentServices : allAgentServices.filter((serviceName) => serviceName !== `MedplumAgent_${yn}`);
|
|
75995
|
-
(0,
|
|
78470
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `Medplum agent service to filter out: MedplumAgent_${yn}\r
|
|
75996
78471
|
`, {
|
|
75997
78472
|
encoding: "utf-8"
|
|
75998
78473
|
});
|
|
75999
78474
|
for (const serviceName of servicesToRemove) {
|
|
76000
78475
|
try {
|
|
76001
78476
|
(0, import_node_child_process3.execSync)(`net stop ${serviceName}`);
|
|
76002
|
-
(0,
|
|
78477
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `${serviceName} stopped\r
|
|
76003
78478
|
`, { encoding: "utf-8" });
|
|
76004
78479
|
console.log(`${serviceName} stopped`);
|
|
76005
78480
|
} catch (err2) {
|
|
76006
|
-
(0,
|
|
78481
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `Failed to stop service: ${serviceName}\r
|
|
76007
78482
|
`, { encoding: "utf-8" });
|
|
76008
|
-
(0,
|
|
78483
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `${Re(err2)}\r
|
|
76009
78484
|
`, { encoding: "utf-8" });
|
|
76010
78485
|
console.error(`Failed to stop service: ${serviceName}`);
|
|
76011
78486
|
console.error(Re(err2));
|
|
76012
78487
|
}
|
|
76013
78488
|
try {
|
|
76014
78489
|
(0, import_node_child_process3.execSync)(`sc.exe delete ${serviceName}`);
|
|
76015
|
-
(0,
|
|
78490
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `${serviceName} deleted\r
|
|
76016
78491
|
`, { encoding: "utf-8" });
|
|
76017
78492
|
console.log(`${serviceName} deleted`);
|
|
76018
78493
|
} catch (err2) {
|
|
76019
|
-
(0,
|
|
78494
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `Failed to delete service: ${serviceName}\r
|
|
76020
78495
|
`, { encoding: "utf-8" });
|
|
76021
|
-
(0,
|
|
78496
|
+
(0, import_node_fs8.appendFileSync)(logFileFd, `${Re(err2)}\r
|
|
76022
78497
|
`, { encoding: "utf-8" });
|
|
76023
78498
|
console.error(`Failed to delete service: ${serviceName}`);
|
|
76024
78499
|
console.error(Re(err2));
|
|
76025
78500
|
}
|
|
76026
78501
|
}
|
|
76027
|
-
(0,
|
|
76028
|
-
} else if ((0,
|
|
78502
|
+
(0, import_node_fs8.closeSync)(logFileFd);
|
|
78503
|
+
} else if ((0, import_node_fs8.existsSync)(UPGRADE_MANIFEST_PATH)) {
|
|
76029
78504
|
createPidFile("medplum-upgrading-agent");
|
|
76030
78505
|
await agentMain(argv);
|
|
76031
78506
|
} else {
|