@holokai/holo 1.5.2 → 1.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/holo.js +794 -314
- package/dist/holo.js.map +1 -1
- package/package.json +3 -3
package/dist/holo.js
CHANGED
|
@@ -661,7 +661,8 @@ var init_protocol = __esm({
|
|
|
661
661
|
EMBED: "embed",
|
|
662
662
|
RERANK: "rerank",
|
|
663
663
|
MODELS: "models",
|
|
664
|
-
METRICS: "metrics"
|
|
664
|
+
METRICS: "metrics",
|
|
665
|
+
IMAGE: "image"
|
|
665
666
|
};
|
|
666
667
|
}
|
|
667
668
|
});
|
|
@@ -760,6 +761,7 @@ var init_types4 = __esm({
|
|
|
760
761
|
ProviderEventType = {
|
|
761
762
|
STREAM_EVENT: "stream_event",
|
|
762
763
|
TEXT_DELTA: "text_delta",
|
|
764
|
+
REASONING_DELTA: "reasoning_delta",
|
|
763
765
|
TOOL_CALL_DELTA: "tool_call_delta",
|
|
764
766
|
DONE: "done",
|
|
765
767
|
ERROR: "error"
|
|
@@ -1275,6 +1277,11 @@ data: ${JSON.stringify(data)}
|
|
|
1275
1277
|
chunks.push(await this.chunkify(ev, async () => this.formatWire(deltaEvent)));
|
|
1276
1278
|
return chunks;
|
|
1277
1279
|
}
|
|
1280
|
+
case "reasoning_delta": {
|
|
1281
|
+
const reasoningEvent = this.emitReasoningDelta(0, ev.text);
|
|
1282
|
+
chunks.push(await this.chunkify(ev, async () => this.formatWire(reasoningEvent)));
|
|
1283
|
+
return chunks;
|
|
1284
|
+
}
|
|
1278
1285
|
case "tool_call_delta": {
|
|
1279
1286
|
const tcEv = ev;
|
|
1280
1287
|
const holoEvent = this.emitToolCallDelta(tcEv.index, tcEv.delta);
|
|
@@ -1755,7 +1762,6 @@ var init_env2 = __esm({
|
|
|
1755
1762
|
})(env2.redis || (env2.redis = {}));
|
|
1756
1763
|
(function(queue) {
|
|
1757
1764
|
queue.url = process.env.RABBITMQ_URL || "amqp://localhost:5672";
|
|
1758
|
-
queue.reconnectAttempts = parseNumber(process.env.RABBITMQ_RECONNECT_ATTEMPTS, 5);
|
|
1759
1765
|
queue.reconnectDelayMs = parseNumber(process.env.RABBITMQ_RECONNECT_DELAY_MS, 5e3);
|
|
1760
1766
|
queue.prefetch = parseNumber(process.env.RABBITMQ_PREFETCH, 10);
|
|
1761
1767
|
queue.maxMessageAgeMs = parseNumber(process.env.RABBITMQ_MAX_MESSAGE_AGE_MS, 18e5);
|
|
@@ -1779,14 +1785,13 @@ var init_env2 = __esm({
|
|
|
1779
1785
|
queue.auditRoutingKey = process.env.AUDIT_ROUTING_KEY || q("audit");
|
|
1780
1786
|
queue.batchQueue = process.env.RABBITMQ_BATCH_QUEUE || q("batch-jobs");
|
|
1781
1787
|
queue.batchRoutingKey = process.env.BATCH_ROUTING_KEY || q("batch-jobs");
|
|
1782
|
-
queue.queueExpiration = process.env.QUEUE_EXPIRATION
|
|
1788
|
+
queue.queueExpiration = parseNumber(process.env.QUEUE_EXPIRATION, 36e5);
|
|
1783
1789
|
queue.managementQueue = process.env.PROXY_MANAGEMENT_QUEUE || q("proxy_management");
|
|
1784
1790
|
queue.notificationExchange = process.env.RABBITMQ_NOTIFICATION_EXCHANGE || q("notifications_exchange");
|
|
1785
1791
|
queue.notificationQueue = process.env.RABBITMQ_NOTIFICATION_QUEUE || q("notifications");
|
|
1786
1792
|
queue.auditNotificationQueue = process.env.RABBITMQ_AUDIT_NOTIFICATION_QUEUE || q("notifications_audit");
|
|
1787
1793
|
queue.config = {
|
|
1788
1794
|
url: queue.url,
|
|
1789
|
-
reconnectAttempts: queue.reconnectAttempts,
|
|
1790
1795
|
reconnectDelayMs: queue.reconnectDelayMs
|
|
1791
1796
|
};
|
|
1792
1797
|
})(env2.queue || (env2.queue = {}));
|
|
@@ -1870,7 +1875,7 @@ var VERSION;
|
|
|
1870
1875
|
var init_version2 = __esm({
|
|
1871
1876
|
"src/version.ts"() {
|
|
1872
1877
|
"use strict";
|
|
1873
|
-
VERSION = "1.5.
|
|
1878
|
+
VERSION = "1.5.3";
|
|
1874
1879
|
}
|
|
1875
1880
|
});
|
|
1876
1881
|
|
|
@@ -2786,8 +2791,9 @@ var init_effective_model_db = __esm({
|
|
|
2786
2791
|
this.db = db;
|
|
2787
2792
|
}
|
|
2788
2793
|
/**
|
|
2789
|
-
* Resolve the effective binding for `(orgId, selector)`. `p_org` is always the
|
|
2790
|
-
* `org_id`
|
|
2794
|
+
* Resolve the effective binding for `(orgId, selector, profileId)`. `p_org` is always the
|
|
2795
|
+
* token's `org_id`, and `profileId` the token's mint-validated `profile_id` claim (or null
|
|
2796
|
+
* when absent) — never client-supplied — and the SRF filters org internally.
|
|
2791
2797
|
*
|
|
2792
2798
|
* The SRF returns the org's set; Holo keys it by the exact selector (never another
|
|
2793
2799
|
* selector's default — an unknown selector yields no row → 404). The selector is
|
|
@@ -2795,18 +2801,18 @@ var init_effective_model_db = __esm({
|
|
|
2795
2801
|
* (`widget-jpql` vs `THINK_MAX`), so any normalization would silently break one
|
|
2796
2802
|
* vocabulary or collide the two.
|
|
2797
2803
|
*
|
|
2798
|
-
* **`profile_id` is
|
|
2799
|
-
*
|
|
2800
|
-
*
|
|
2801
|
-
*
|
|
2802
|
-
*
|
|
2803
|
-
*
|
|
2804
|
-
*
|
|
2804
|
+
* **`profile_id` is the third selection key** (moku P3 fan-out: one row per selectable
|
|
2805
|
+
* profile, unique on `(org, purpose, profile_id)`). A concrete `profileId` selects its
|
|
2806
|
+
* row; `null` selects the profile-less row (`profile_id IS NULL`) — the shape moku
|
|
2807
|
+
* projects for internal purposes and for orgs with no authored profiles, the only orgs
|
|
2808
|
+
* whose tokens omit the claim (the mint stamps the concrete default-profile id whenever
|
|
2809
|
+
* any profile applies — `SystemModelTokenService.decideProfile`). No row for the triple
|
|
2810
|
+
* fails closed (404); there is no fallback to another profile's row.
|
|
2805
2811
|
*
|
|
2806
2812
|
* The returned row may carry a null binding — callers must gate on
|
|
2807
2813
|
* {@link isServableEffectiveModel} before any provider lookup.
|
|
2808
2814
|
*/
|
|
2809
|
-
async getEffectiveModel(orgId, selector) {
|
|
2815
|
+
async getEffectiveModel(orgId, selector, profileId) {
|
|
2810
2816
|
return this.db.queryOne(`SELECT emr.provider_id,
|
|
2811
2817
|
emr.model_slug,
|
|
2812
2818
|
emr.source,
|
|
@@ -2816,9 +2822,11 @@ var init_effective_model_db = __esm({
|
|
|
2816
2822
|
emr.profile_id
|
|
2817
2823
|
FROM ${SRF}($1::uuid) emr
|
|
2818
2824
|
WHERE emr.${SELECTOR_COLUMN} = $2
|
|
2825
|
+
AND (($3::uuid IS NULL AND emr.profile_id IS NULL) OR emr.profile_id = $3::uuid)
|
|
2819
2826
|
LIMIT 1`, [
|
|
2820
2827
|
orgId,
|
|
2821
|
-
selector
|
|
2828
|
+
selector,
|
|
2829
|
+
profileId
|
|
2822
2830
|
]);
|
|
2823
2831
|
}
|
|
2824
2832
|
};
|
|
@@ -5562,6 +5570,27 @@ var init_datastore_db = __esm({
|
|
|
5562
5570
|
async getAll() {
|
|
5563
5571
|
return this.db.systemQuery("SELECT * FROM datastores ORDER BY created_at");
|
|
5564
5572
|
}
|
|
5573
|
+
/**
|
|
5574
|
+
* Repoint every same-family datastore to the canonical plugin version —
|
|
5575
|
+
* the datastore half of latest-only consolidation (mirrors
|
|
5576
|
+
* ProviderDB.migrateToLatestPlugin; no junction table on this side).
|
|
5577
|
+
* The $2::uuid casts keep PG16/Aurora's parameter-type deduction from
|
|
5578
|
+
* failing when the parameter is reused as both assignment and comparand.
|
|
5579
|
+
*/
|
|
5580
|
+
async migrateToLatestPlugin(family, latestPluginId) {
|
|
5581
|
+
const rows = await this.db.systemQuery(`UPDATE datastores d
|
|
5582
|
+
SET plugin_id = $2::uuid,
|
|
5583
|
+
updated_at = now()
|
|
5584
|
+
FROM plugins pl
|
|
5585
|
+
WHERE d.plugin_id = pl.id
|
|
5586
|
+
AND pl.family = $1
|
|
5587
|
+
AND d.plugin_id <> $2::uuid
|
|
5588
|
+
RETURNING d.id`, [
|
|
5589
|
+
family.toUpperCase(),
|
|
5590
|
+
latestPluginId
|
|
5591
|
+
]);
|
|
5592
|
+
return rows.length;
|
|
5593
|
+
}
|
|
5565
5594
|
async getEnabled() {
|
|
5566
5595
|
return this.db.systemQuery(`SELECT d.*, ac.encrypted_value, ac.initialization_vector
|
|
5567
5596
|
FROM datastores d
|
|
@@ -6872,16 +6901,26 @@ var init_provider_plugin_service = __esm({
|
|
|
6872
6901
|
}
|
|
6873
6902
|
protocolDB;
|
|
6874
6903
|
pluginProtocols = /* @__PURE__ */ new Map();
|
|
6904
|
+
/**
|
|
6905
|
+
* Each plugin's declared `defaultProtocol`, learned at registration. Capability
|
|
6906
|
+
* lookups prefer it when it covers the capability — without this, a family with
|
|
6907
|
+
* two protocols of one capability resolves by map insertion order, which is how
|
|
6908
|
+
* OpenAI chat silently dispatched chatCompletions instead of responses.
|
|
6909
|
+
*/
|
|
6910
|
+
pluginDefaults = /* @__PURE__ */ new Map();
|
|
6875
6911
|
constructor(protocolDB) {
|
|
6876
6912
|
super(), this.protocolDB = protocolDB;
|
|
6877
6913
|
}
|
|
6878
|
-
async registerProtocols(pluginId, routes) {
|
|
6914
|
+
async registerProtocols(pluginId, routes, defaultProtocol) {
|
|
6879
6915
|
const logger2 = this.mlog(this.registerProtocols);
|
|
6880
6916
|
let protocols = this.pluginProtocols.get(pluginId);
|
|
6881
6917
|
if (!protocols) {
|
|
6882
6918
|
protocols = /* @__PURE__ */ new Map();
|
|
6883
6919
|
this.pluginProtocols.set(pluginId, protocols);
|
|
6884
6920
|
}
|
|
6921
|
+
if (defaultProtocol) {
|
|
6922
|
+
this.pluginDefaults.set(pluginId, defaultProtocol);
|
|
6923
|
+
}
|
|
6885
6924
|
for (const routeDef of routes) {
|
|
6886
6925
|
const { protocol } = routeDef;
|
|
6887
6926
|
const path5 = routeDef.paths.join(",");
|
|
@@ -6893,23 +6932,38 @@ var init_provider_plugin_service = __esm({
|
|
|
6893
6932
|
}
|
|
6894
6933
|
}
|
|
6895
6934
|
}
|
|
6896
|
-
async loadProtocolsFromDB(pluginId) {
|
|
6935
|
+
async loadProtocolsFromDB(pluginId, defaultProtocol) {
|
|
6897
6936
|
const protocols = await this.protocolDB.getByPlugin(pluginId);
|
|
6898
6937
|
const map = /* @__PURE__ */ new Map();
|
|
6899
6938
|
for (const p of protocols) {
|
|
6900
6939
|
map.set(p.name, p);
|
|
6901
6940
|
}
|
|
6902
6941
|
this.pluginProtocols.set(pluginId, map);
|
|
6942
|
+
if (defaultProtocol) {
|
|
6943
|
+
this.pluginDefaults.set(pluginId, defaultProtocol);
|
|
6944
|
+
}
|
|
6903
6945
|
}
|
|
6904
6946
|
removeProtocols(pluginId) {
|
|
6905
6947
|
this.pluginProtocols.delete(pluginId);
|
|
6948
|
+
this.pluginDefaults.delete(pluginId);
|
|
6906
6949
|
}
|
|
6907
6950
|
async getProtocol(pluginId, protocolName) {
|
|
6908
6951
|
return this.pluginProtocols.get(pluginId)?.get(protocolName);
|
|
6909
6952
|
}
|
|
6953
|
+
/**
|
|
6954
|
+
* The plugin's declared default wins when it covers the capability; otherwise
|
|
6955
|
+
* first match. Both populators (`registerProtocols`, `loadProtocolsFromDB`)
|
|
6956
|
+
* learn the default from the loaded plugin instance, so startup registration
|
|
6957
|
+
* and runtime reload/remote-enable resolve identically.
|
|
6958
|
+
*/
|
|
6910
6959
|
getProtocolByCapability(pluginId, capability) {
|
|
6911
6960
|
const protocols = this.pluginProtocols.get(pluginId);
|
|
6912
6961
|
if (!protocols) return void 0;
|
|
6962
|
+
const preferredName = this.pluginDefaults.get(pluginId);
|
|
6963
|
+
if (preferredName) {
|
|
6964
|
+
const preferred = protocols.get(preferredName);
|
|
6965
|
+
if (preferred?.capability === capability) return preferred;
|
|
6966
|
+
}
|
|
6913
6967
|
for (const protocol of protocols.values()) {
|
|
6914
6968
|
if (protocol.capability === capability) return protocol;
|
|
6915
6969
|
}
|
|
@@ -7155,6 +7209,13 @@ var init_plugin_registry = __esm({
|
|
|
7155
7209
|
pluginDB;
|
|
7156
7210
|
plugins = /* @__PURE__ */ new Map();
|
|
7157
7211
|
loading = /* @__PURE__ */ new Map();
|
|
7212
|
+
/**
|
|
7213
|
+
* The DB row's sha256 AS OF the moment each plugin was loaded. Reconcile
|
|
7214
|
+
* uses this as its baseline for plugins loaded outside the lifecycle
|
|
7215
|
+
* service (boot-time loads) — comparing against the CURRENT row hash would
|
|
7216
|
+
* miss an update that landed between load and reconcile.
|
|
7217
|
+
*/
|
|
7218
|
+
loadedShas = /* @__PURE__ */ new Map();
|
|
7158
7219
|
constructor(installer, discovery, loader, pluginDB) {
|
|
7159
7220
|
super(), this.installer = installer, this.discovery = discovery, this.loader = loader, this.pluginDB = pluginDB;
|
|
7160
7221
|
}
|
|
@@ -7178,6 +7239,10 @@ var init_plugin_registry = __esm({
|
|
|
7178
7239
|
has(pluginId) {
|
|
7179
7240
|
return this.plugins.has(pluginId);
|
|
7180
7241
|
}
|
|
7242
|
+
/** sha256 the plugin was loaded with, when it is loaded; undefined otherwise. */
|
|
7243
|
+
loadedSha(pluginId) {
|
|
7244
|
+
return this.plugins.has(pluginId) ? this.loadedShas.get(pluginId) : void 0;
|
|
7245
|
+
}
|
|
7181
7246
|
getAsProvider(pluginId) {
|
|
7182
7247
|
return this.plugins.get(pluginId);
|
|
7183
7248
|
}
|
|
@@ -7204,6 +7269,7 @@ var init_plugin_registry = __esm({
|
|
|
7204
7269
|
plugin.destroy().catch(() => {
|
|
7205
7270
|
});
|
|
7206
7271
|
this.plugins.delete(pluginId);
|
|
7272
|
+
this.loadedShas.delete(pluginId);
|
|
7207
7273
|
}
|
|
7208
7274
|
}
|
|
7209
7275
|
async loadAllActive() {
|
|
@@ -7249,6 +7315,7 @@ var init_plugin_registry = __esm({
|
|
|
7249
7315
|
}
|
|
7250
7316
|
plugin.id = pluginId;
|
|
7251
7317
|
this.plugins.set(pluginId, plugin);
|
|
7318
|
+
this.loadedShas.set(pluginId, row.sha256 ?? "");
|
|
7252
7319
|
log.info(`Plugin ${row.name}@${row.version} loaded and ready (${pluginId})`);
|
|
7253
7320
|
return plugin;
|
|
7254
7321
|
}
|
|
@@ -9253,6 +9320,7 @@ var init_token_verifier_support = __esm({
|
|
|
9253
9320
|
// src/services/auth/on-behalf-token.service.ts
|
|
9254
9321
|
import "reflect-metadata";
|
|
9255
9322
|
import { injectable as injectable34 } from "tsyringe";
|
|
9323
|
+
import { validate as isUuid } from "uuid";
|
|
9256
9324
|
function _ts_decorate34(decorators, target, key, desc) {
|
|
9257
9325
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
9258
9326
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -9289,9 +9357,9 @@ var init_on_behalf_token_service = __esm({
|
|
|
9289
9357
|
/**
|
|
9290
9358
|
* verify — verify a raw bearer token against the requested alias. Resolves to the
|
|
9291
9359
|
* request-scoped context (org + resolved selector from the token only) on success;
|
|
9292
|
-
* rejects with a status-bearing, claim-free `HoloError` on any failed claim. The
|
|
9293
|
-
*
|
|
9294
|
-
*
|
|
9360
|
+
* rejects with a status-bearing, claim-free `HoloError` on any failed claim. The `jti` is
|
|
9361
|
+
* consumed as the final step only when the token's `token_limit` selects single-use (the
|
|
9362
|
+
* default); a `token_limit=0` token consumes nothing and may be reused until `exp`.
|
|
9295
9363
|
*
|
|
9296
9364
|
* `clientSuppliedProfile` reports only WHETHER the client attempted to select a profile,
|
|
9297
9365
|
* because the sole thing done with it is refuse it — a value would imply it could
|
|
@@ -9299,46 +9367,136 @@ var init_on_behalf_token_service = __esm({
|
|
|
9299
9367
|
*/
|
|
9300
9368
|
async verify(rawToken, requestedAlias, clientSuppliedProfile2 = false) {
|
|
9301
9369
|
const logger2 = this.mlog(this.verify);
|
|
9370
|
+
const loggedAlias = requestedAlias.length > 64 ? `${requestedAlias.slice(0, 64)}\u2026` : requestedAlias;
|
|
9302
9371
|
if (!rawToken) {
|
|
9303
|
-
|
|
9372
|
+
this.deny("no_bearer", HoloError.unauthorized("Invalid or unauthorized token."), {
|
|
9373
|
+
requestedAlias: loggedAlias
|
|
9374
|
+
});
|
|
9375
|
+
}
|
|
9376
|
+
let issuer;
|
|
9377
|
+
try {
|
|
9378
|
+
issuer = readUnverifiedIssuer(rawToken);
|
|
9379
|
+
} catch (err) {
|
|
9380
|
+
return this.deny("undecodable_jwt", err, {
|
|
9381
|
+
requestedAlias: loggedAlias
|
|
9382
|
+
});
|
|
9304
9383
|
}
|
|
9305
|
-
const issuer = readUnverifiedIssuer(rawToken);
|
|
9306
9384
|
if (!env.onBehalf.trustedIssuers.includes(issuer)) {
|
|
9307
|
-
|
|
9385
|
+
this.deny("untrusted_issuer", HoloError.unauthorized("Invalid or unauthorized token."), {
|
|
9386
|
+
issuer,
|
|
9387
|
+
requestedAlias: loggedAlias
|
|
9388
|
+
});
|
|
9308
9389
|
}
|
|
9309
9390
|
const jwks = this.jwksResolver.get(issuer);
|
|
9310
9391
|
if (!jwks) {
|
|
9311
|
-
|
|
9392
|
+
this.deny("jwks_unresolved", HoloError.unauthorized("Invalid or unauthorized token."), {
|
|
9393
|
+
issuer,
|
|
9394
|
+
requestedAlias: loggedAlias
|
|
9395
|
+
});
|
|
9312
9396
|
}
|
|
9313
|
-
|
|
9397
|
+
let payload;
|
|
9398
|
+
try {
|
|
9399
|
+
payload = await verifyRs256(rawToken, jwks, env.onBehalf.audience, issuer);
|
|
9400
|
+
} catch (err) {
|
|
9401
|
+
return this.deny("signature_or_standard_claims", err, {
|
|
9402
|
+
issuer,
|
|
9403
|
+
requestedAlias: loggedAlias
|
|
9404
|
+
});
|
|
9405
|
+
}
|
|
9406
|
+
const diag = this.safeDiag(issuer, payload, loggedAlias);
|
|
9314
9407
|
if (payload.capability !== REQUIRED_CAPABILITY) {
|
|
9315
|
-
|
|
9408
|
+
this.deny("wrong_capability", HoloError.unauthorized("Invalid or unauthorized token."), diag);
|
|
9409
|
+
}
|
|
9410
|
+
try {
|
|
9411
|
+
assertWithinEffectiveWindow(payload, env.onBehalf.maxTtlMs);
|
|
9412
|
+
} catch (err) {
|
|
9413
|
+
return this.deny("effective_window", err, diag);
|
|
9316
9414
|
}
|
|
9317
|
-
assertWithinEffectiveWindow(payload, env.onBehalf.maxTtlMs);
|
|
9318
9415
|
if (!payload.org_id || !payload.sub) {
|
|
9319
|
-
|
|
9416
|
+
this.deny("missing_org_or_sub", HoloError.unauthorized("Invalid or unauthorized token."), diag);
|
|
9320
9417
|
}
|
|
9321
9418
|
if (!payload.jti) {
|
|
9322
|
-
|
|
9419
|
+
this.deny("missing_jti", HoloError.unauthorized("Invalid or unauthorized token."), diag);
|
|
9420
|
+
}
|
|
9421
|
+
const tokenLimit = payload.token_limit ?? 1;
|
|
9422
|
+
if (tokenLimit !== 0 && tokenLimit !== 1) {
|
|
9423
|
+
this.deny("token_limit_unsupported", HoloError.unauthorized("Invalid or unauthorized token."), diag);
|
|
9323
9424
|
}
|
|
9324
9425
|
const allowed = payload.allowed_aliases;
|
|
9325
9426
|
if (!Array.isArray(allowed) || !allowed.includes(requestedAlias)) {
|
|
9326
|
-
|
|
9427
|
+
this.deny("alias_not_permitted", HoloError.forbidden("Requested model is not permitted for this token."), diag);
|
|
9327
9428
|
}
|
|
9328
9429
|
if (clientSuppliedProfile2) {
|
|
9329
|
-
|
|
9430
|
+
this.deny("client_supplied_profile", HoloError.forbidden("Requested profile is not permitted for this token."), diag);
|
|
9431
|
+
}
|
|
9432
|
+
const profileClaim = payload.profile_id;
|
|
9433
|
+
if (profileClaim !== void 0 && (typeof profileClaim !== "string" || !isUuid(profileClaim))) {
|
|
9434
|
+
this.deny("malformed_profile_claim", HoloError.unauthorized("Invalid or unauthorized token."), diag);
|
|
9435
|
+
}
|
|
9436
|
+
const profileId = profileClaim ?? null;
|
|
9437
|
+
if (tokenLimit === 1) {
|
|
9438
|
+
try {
|
|
9439
|
+
await this.consumeJti(payload.jti);
|
|
9440
|
+
} catch (err) {
|
|
9441
|
+
const replay = err instanceof HoloError && err.statusCode === 401;
|
|
9442
|
+
return this.deny(replay ? "jti_replay" : "replay_store_unavailable", err, diag);
|
|
9443
|
+
}
|
|
9330
9444
|
}
|
|
9331
|
-
await this.consumeJti(payload.jti);
|
|
9332
9445
|
const selector = requestedAlias;
|
|
9333
9446
|
logger2.info("On-behalf token accepted", {
|
|
9334
9447
|
issuer,
|
|
9335
9448
|
selector,
|
|
9336
|
-
capability: payload.capability
|
|
9449
|
+
capability: payload.capability,
|
|
9450
|
+
has_profile: profileId !== null
|
|
9337
9451
|
});
|
|
9338
9452
|
return {
|
|
9339
9453
|
organizationId: payload.org_id,
|
|
9340
9454
|
sub: payload.sub,
|
|
9341
|
-
selector
|
|
9455
|
+
selector,
|
|
9456
|
+
profileId
|
|
9457
|
+
};
|
|
9458
|
+
}
|
|
9459
|
+
/**
|
|
9460
|
+
* deny — log a claim-SAFE rejection reason to the server log, then throw the fixed,
|
|
9461
|
+
* claim-free wire error unchanged (the wire message never gains a claim). Debug aid only:
|
|
9462
|
+
* the `reason` names the exact gate that rejected, and `diag` carries only safe fields.
|
|
9463
|
+
* NEVER logs the raw token (a live bearer credential) or `sub` (which may be an email);
|
|
9464
|
+
* `sub` is reported as presence only. Returns `never` so callers narrow past it.
|
|
9465
|
+
*
|
|
9466
|
+
* The HTTP status rides `http_status`, not `status`: the logger reserves `status` for the log
|
|
9467
|
+
* level and pads it as a string, so a numeric `status` field breaks the formatter.
|
|
9468
|
+
*/
|
|
9469
|
+
deny(reason, error, diag = {}) {
|
|
9470
|
+
this.mlog(this.verify).warn("On-behalf token rejected", {
|
|
9471
|
+
reason,
|
|
9472
|
+
http_status: error.statusCode,
|
|
9473
|
+
...diag
|
|
9474
|
+
});
|
|
9475
|
+
throw error;
|
|
9476
|
+
}
|
|
9477
|
+
/**
|
|
9478
|
+
* safeDiag — claim-SAFE fields for a rejection log. `iss`/`capability`/`token_limit` are fixed
|
|
9479
|
+
* scalars, `org_id`/`jti` are IDs, `iat`/`exp` are timings — all loggable. `sub` is deliberately
|
|
9480
|
+
* reported only as presence (`hasSub`) because it may be an email; the raw token and the full
|
|
9481
|
+
* payload are never dumped. `expiresInMs` is surfaced so an expiry/window rejection is obvious.
|
|
9482
|
+
*/
|
|
9483
|
+
safeDiag(issuer, payload, loggedAlias) {
|
|
9484
|
+
return {
|
|
9485
|
+
issuer,
|
|
9486
|
+
capability: payload.capability,
|
|
9487
|
+
iat: payload.iat,
|
|
9488
|
+
exp: payload.exp,
|
|
9489
|
+
expiresInMs: typeof payload.exp === "number" ? payload.exp * 1e3 - Date.now() : null,
|
|
9490
|
+
hasSub: !!payload.sub,
|
|
9491
|
+
orgId: payload.org_id,
|
|
9492
|
+
jti: payload.jti,
|
|
9493
|
+
allowedAliases: Array.isArray(payload.allowed_aliases) ? payload.allowed_aliases : null,
|
|
9494
|
+
requestedAlias: loggedAlias,
|
|
9495
|
+
// The mint's replay-posture claim as sent (null when absent) — one scalar, so it stays
|
|
9496
|
+
// claim-safe while showing why a token was single-use vs reusable. The full payload is
|
|
9497
|
+
// deliberately never dumped: it carries `sub`/`email` PII, and a blanket dump could not
|
|
9498
|
+
// stay claim-safe as the token contract grows.
|
|
9499
|
+
token_limit: payload.token_limit ?? null
|
|
9342
9500
|
};
|
|
9343
9501
|
}
|
|
9344
9502
|
/**
|
|
@@ -10136,7 +10294,7 @@ var init_plugin_service = __esm({
|
|
|
10136
10294
|
case PluginType.PROVIDER: {
|
|
10137
10295
|
const providerPlugin = plugin;
|
|
10138
10296
|
const routes = providerPlugin.getRoutes();
|
|
10139
|
-
await this.providerPluginService.registerProtocols(pluginId, routes);
|
|
10297
|
+
await this.providerPluginService.registerProtocols(pluginId, routes, providerPlugin.defaultProtocol);
|
|
10140
10298
|
const pricingResult = await this.registerDefaultPricing(providerPlugin, family, pluginId);
|
|
10141
10299
|
log.info(`${tag}: ${routes.length} route(s), ${pricingResult}`);
|
|
10142
10300
|
break;
|
|
@@ -10177,7 +10335,7 @@ var init_plugin_service = __esm({
|
|
|
10177
10335
|
case PluginType.PROVIDER: {
|
|
10178
10336
|
const providerPlugin = plugin;
|
|
10179
10337
|
const routes = providerPlugin.getRoutes();
|
|
10180
|
-
await this.providerPluginService.registerProtocols(pluginRow.id, routes);
|
|
10338
|
+
await this.providerPluginService.registerProtocols(pluginRow.id, routes, providerPlugin.defaultProtocol);
|
|
10181
10339
|
await this.registerDefaultPricing(providerPlugin, family, pluginRow.id);
|
|
10182
10340
|
break;
|
|
10183
10341
|
}
|
|
@@ -10206,7 +10364,7 @@ var init_plugin_service = __esm({
|
|
|
10206
10364
|
this.versionedPlugins.get(familyKey).set(pluginRow.version, pluginRow);
|
|
10207
10365
|
switch (plugin.type) {
|
|
10208
10366
|
case PluginType.PROVIDER: {
|
|
10209
|
-
await this.providerPluginService.loadProtocolsFromDB(pluginId);
|
|
10367
|
+
await this.providerPluginService.loadProtocolsFromDB(pluginId, plugin.defaultProtocol);
|
|
10210
10368
|
break;
|
|
10211
10369
|
}
|
|
10212
10370
|
case PluginType.DATASTORE: {
|
|
@@ -10433,6 +10591,7 @@ var init_provider_impl_service = __esm({
|
|
|
10433
10591
|
init_entities2();
|
|
10434
10592
|
init_provider_db();
|
|
10435
10593
|
init_plugin_db();
|
|
10594
|
+
init_datastore_db();
|
|
10436
10595
|
init_auth2();
|
|
10437
10596
|
init_plugin_registry();
|
|
10438
10597
|
init_plugin();
|
|
@@ -10447,30 +10606,39 @@ var init_provider_impl_service = __esm({
|
|
|
10447
10606
|
pluginRegistry;
|
|
10448
10607
|
providerDB;
|
|
10449
10608
|
pluginDB;
|
|
10609
|
+
datastoreDB;
|
|
10450
10610
|
/** One impl per provider↔plugin association, keyed `${providerId}::${pluginId}`. */
|
|
10451
10611
|
providerImpls = /* @__PURE__ */ new Map();
|
|
10452
10612
|
/** providerId → primary plugin id, so bare lookups stay map-only on the hot path. */
|
|
10453
10613
|
primaryPluginIds = /* @__PURE__ */ new Map();
|
|
10454
|
-
constructor(providerService, cryptoService, pluginRegistry, providerDB, pluginDB) {
|
|
10455
|
-
super(), this.providerService = providerService, this.cryptoService = cryptoService, this.pluginRegistry = pluginRegistry, this.providerDB = providerDB, this.pluginDB = pluginDB;
|
|
10614
|
+
constructor(providerService, cryptoService, pluginRegistry, providerDB, pluginDB, datastoreDB) {
|
|
10615
|
+
super(), this.providerService = providerService, this.cryptoService = cryptoService, this.pluginRegistry = pluginRegistry, this.providerDB = providerDB, this.pluginDB = pluginDB, this.datastoreDB = datastoreDB;
|
|
10456
10616
|
}
|
|
10457
10617
|
implKey(providerId, pluginId) {
|
|
10458
10618
|
return `${providerId}::${pluginId}`;
|
|
10459
10619
|
}
|
|
10460
10620
|
async consolidateToLatest(seeded) {
|
|
10461
10621
|
const log = this.mlog(this.consolidateToLatest);
|
|
10462
|
-
const
|
|
10622
|
+
const providersByFamily = /* @__PURE__ */ new Map();
|
|
10623
|
+
const datastoresByFamily = /* @__PURE__ */ new Map();
|
|
10463
10624
|
for (const plugin of seeded) {
|
|
10464
|
-
if (plugin.type
|
|
10465
|
-
|
|
10625
|
+
if (plugin.type === PluginType.PROVIDER) providersByFamily.set(plugin.family, plugin);
|
|
10626
|
+
if (plugin.type === PluginType.DATASTORE) datastoresByFamily.set(plugin.family, plugin);
|
|
10466
10627
|
}
|
|
10467
|
-
for (const [family, canonical] of
|
|
10628
|
+
for (const [family, canonical] of providersByFamily) {
|
|
10468
10629
|
const repointed = await this.providerDB.migrateToLatestPlugin(family, canonical.id);
|
|
10469
10630
|
const disabled = await this.pluginDB.deactivateOtherVersions(family, canonical.id);
|
|
10470
10631
|
if (repointed || disabled) {
|
|
10471
10632
|
log.info(`Consolidated ${family} to v${canonical.version}: repointed ${repointed} provider(s), disabled ${disabled} older version(s)`);
|
|
10472
10633
|
}
|
|
10473
10634
|
}
|
|
10635
|
+
for (const [family, canonical] of datastoresByFamily) {
|
|
10636
|
+
const repointed = await this.datastoreDB.migrateToLatestPlugin(family, canonical.id);
|
|
10637
|
+
const disabled = await this.pluginDB.deactivateOtherVersions(family, canonical.id);
|
|
10638
|
+
if (repointed || disabled) {
|
|
10639
|
+
log.info(`Consolidated ${family} to v${canonical.version}: repointed ${repointed} datastore(s), disabled ${disabled} older version(s)`);
|
|
10640
|
+
}
|
|
10641
|
+
}
|
|
10474
10642
|
}
|
|
10475
10643
|
/**
|
|
10476
10644
|
* Fast path first: a known (provider, plugin) key returns straight from
|
|
@@ -10674,7 +10842,8 @@ var init_provider_impl_service = __esm({
|
|
|
10674
10842
|
typeof CryptoService === "undefined" ? Object : CryptoService,
|
|
10675
10843
|
typeof PluginRegistry === "undefined" ? Object : PluginRegistry,
|
|
10676
10844
|
typeof ProviderDB === "undefined" ? Object : ProviderDB,
|
|
10677
|
-
typeof PluginDB === "undefined" ? Object : PluginDB
|
|
10845
|
+
typeof PluginDB === "undefined" ? Object : PluginDB,
|
|
10846
|
+
typeof DatastoreDB === "undefined" ? Object : DatastoreDB
|
|
10678
10847
|
])
|
|
10679
10848
|
], ProviderImplService);
|
|
10680
10849
|
}
|
|
@@ -10694,7 +10863,14 @@ function _ts_decorate40(decorators, target, key, desc) {
|
|
|
10694
10863
|
function _ts_metadata38(k, v) {
|
|
10695
10864
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10696
10865
|
}
|
|
10697
|
-
|
|
10866
|
+
function errorText(error) {
|
|
10867
|
+
const err = error;
|
|
10868
|
+
if (err?.message) return err.message;
|
|
10869
|
+
const inner = err?.errors?.map((e) => e.code ?? e.message).filter(Boolean);
|
|
10870
|
+
if (inner?.length) return inner.join(", ");
|
|
10871
|
+
return err?.code ?? String(error);
|
|
10872
|
+
}
|
|
10873
|
+
var QueueUnavailableError, MAX_RETRY_DELAY_MS, QueueService;
|
|
10698
10874
|
var init_queue_service = __esm({
|
|
10699
10875
|
"src/services/queue.service.ts"() {
|
|
10700
10876
|
"use strict";
|
|
@@ -10702,27 +10878,50 @@ var init_queue_service = __esm({
|
|
|
10702
10878
|
init_src2();
|
|
10703
10879
|
__name(_ts_decorate40, "_ts_decorate");
|
|
10704
10880
|
__name(_ts_metadata38, "_ts_metadata");
|
|
10881
|
+
QueueUnavailableError = class extends Error {
|
|
10882
|
+
static {
|
|
10883
|
+
__name(this, "QueueUnavailableError");
|
|
10884
|
+
}
|
|
10885
|
+
constructor(operation) {
|
|
10886
|
+
super(`RabbitMQ unavailable: ${operation}`);
|
|
10887
|
+
this.name = "QueueUnavailableError";
|
|
10888
|
+
}
|
|
10889
|
+
};
|
|
10890
|
+
MAX_RETRY_DELAY_MS = 6e4;
|
|
10891
|
+
__name(errorText, "errorText");
|
|
10705
10892
|
QueueService = class extends withClassLogger(EventEmitter2) {
|
|
10706
10893
|
static {
|
|
10707
10894
|
__name(this, "QueueService");
|
|
10708
10895
|
}
|
|
10709
10896
|
isConnected = false;
|
|
10897
|
+
/** Current CONSECUTIVE reconnect attempt count — reset to 0 on success, so /health shows the live outage, not a lifetime tally. */
|
|
10710
10898
|
reconnectAttempts = 0;
|
|
10899
|
+
/** Start of the current connection outage; null while connected. */
|
|
10900
|
+
disconnectedSince = null;
|
|
10901
|
+
/**
|
|
10902
|
+
* Seam for the fatal-misconfiguration path (406 argument mismatch on
|
|
10903
|
+
* declare). Overridable so tests inject a spy instead of stubbing the
|
|
10904
|
+
* global `process.exit` across concurrently running suites.
|
|
10905
|
+
*/
|
|
10906
|
+
fatalExit = /* @__PURE__ */ __name((code) => process.exit(code), "fatalExit");
|
|
10711
10907
|
config = env.queue.config;
|
|
10712
10908
|
connection = null;
|
|
10713
10909
|
channel = null;
|
|
10714
10910
|
recreatingChannel = false;
|
|
10715
10911
|
reconnectCallbacks = [];
|
|
10716
10912
|
activeConsumers = /* @__PURE__ */ new Set();
|
|
10717
|
-
//
|
|
10718
|
-
// channel rebuild can re-subscribe them.
|
|
10719
|
-
//
|
|
10913
|
+
// Legacy/dynamic consumers (notifications, ad-hoc test consumers) keyed by
|
|
10914
|
+
// queue name so a shared-channel rebuild can re-subscribe them. Static
|
|
10915
|
+
// server consumers use registerConsumer/consumerSpecs instead.
|
|
10720
10916
|
consumerRegistry = /* @__PURE__ */ new Map();
|
|
10917
|
+
/** Spec-registered consumers, keyed by ConsumerSpec.id. */
|
|
10918
|
+
consumerSpecs = /* @__PURE__ */ new Map();
|
|
10721
10919
|
closing = false;
|
|
10722
10920
|
replaying = false;
|
|
10723
10921
|
connecting = null;
|
|
10724
10922
|
hasConnectedOnce = false;
|
|
10725
10923
|
reconnectScheduled = false;
|
|
10924
|
+
reconnectTimer = null;
|
|
10726
10925
|
counters = {
|
|
10727
10926
|
channelRecreates: 0,
|
|
10728
10927
|
consumersResubscribed: 0,
|
|
@@ -10734,12 +10933,17 @@ var init_queue_service = __esm({
|
|
|
10734
10933
|
this.on("channel:lost", () => this.activeConsumers.clear());
|
|
10735
10934
|
}
|
|
10736
10935
|
/**
|
|
10737
|
-
* Consumers registered on
|
|
10936
|
+
* Consumers registered on live channels: shared-channel tags (cleared on
|
|
10937
|
+
* channel:lost) plus spec consumers with a live per-consumer channel. A
|
|
10738
10938
|
* value > 0 means this process is actively consuming — the signal the
|
|
10739
10939
|
* worker/audit /health probe reports on.
|
|
10740
10940
|
*/
|
|
10741
10941
|
get activeConsumerCount() {
|
|
10742
|
-
|
|
10942
|
+
let specCount = 0;
|
|
10943
|
+
for (const entry of this.consumerSpecs.values()) {
|
|
10944
|
+
if (entry.state.channel && entry.state.consumerTag) specCount++;
|
|
10945
|
+
}
|
|
10946
|
+
return this.activeConsumers.size + specCount;
|
|
10743
10947
|
}
|
|
10744
10948
|
/**
|
|
10745
10949
|
* Operational counters for observability (surfaced on the /health endpoint).
|
|
@@ -10782,48 +10986,67 @@ var init_queue_service = __esm({
|
|
|
10782
10986
|
await this.createChannel();
|
|
10783
10987
|
this.connection.on("error", (err) => {
|
|
10784
10988
|
logger2.error(`RabbitMQ connection error: ${err.message}`);
|
|
10785
|
-
this.
|
|
10786
|
-
this.channel = null;
|
|
10787
|
-
this.connection = null;
|
|
10989
|
+
this.markDisconnected();
|
|
10788
10990
|
this.emit("channel:lost");
|
|
10789
10991
|
this.reconnect();
|
|
10790
10992
|
});
|
|
10791
10993
|
this.connection.on("close", () => {
|
|
10792
10994
|
if (this.isConnected) {
|
|
10793
10995
|
logger2.warn("RabbitMQ connection closed unexpectedly");
|
|
10794
|
-
this.
|
|
10795
|
-
this.channel = null;
|
|
10796
|
-
this.connection = null;
|
|
10996
|
+
this.markDisconnected();
|
|
10797
10997
|
this.emit("channel:lost");
|
|
10798
10998
|
this.reconnect();
|
|
10799
10999
|
}
|
|
10800
11000
|
});
|
|
10801
11001
|
this.isConnected = true;
|
|
11002
|
+
await this.activateAllConsumers();
|
|
10802
11003
|
} catch (error) {
|
|
10803
|
-
logger2.error(`Failed to connect to RabbitMQ: ${error
|
|
11004
|
+
logger2.error(`Failed to connect to RabbitMQ: ${errorText(error)}`);
|
|
10804
11005
|
throw error;
|
|
10805
11006
|
}
|
|
10806
11007
|
}
|
|
11008
|
+
/** Connection-level loss: every channel (shared and per-consumer) died with it. */
|
|
11009
|
+
markDisconnected() {
|
|
11010
|
+
this.isConnected = false;
|
|
11011
|
+
this.channel = null;
|
|
11012
|
+
this.connection = null;
|
|
11013
|
+
if (!this.disconnectedSince) this.disconnectedSince = /* @__PURE__ */ new Date();
|
|
11014
|
+
for (const entry of this.consumerSpecs.values()) {
|
|
11015
|
+
this.teardownConsumerState(entry);
|
|
11016
|
+
}
|
|
11017
|
+
}
|
|
11018
|
+
teardownConsumerState(entry) {
|
|
11019
|
+
if (entry.state.consumerTag) this.activeConsumers.delete(entry.state.consumerTag);
|
|
11020
|
+
entry.state.channel = null;
|
|
11021
|
+
entry.state.consumerTag = null;
|
|
11022
|
+
if (entry.state.retryTimer) {
|
|
11023
|
+
clearTimeout(entry.state.retryTimer);
|
|
11024
|
+
entry.state.retryTimer = null;
|
|
11025
|
+
}
|
|
11026
|
+
}
|
|
11027
|
+
/**
|
|
11028
|
+
* Reconnect indefinitely with capped exponential backoff. The broker being
|
|
11029
|
+
* down for hours must never strand a live process — the previous bounded
|
|
11030
|
+
* give-up (5 attempts, ~2.5 min) left servers permanently disconnected,
|
|
11031
|
+
* which let idle x-expires queues be deleted underneath them.
|
|
11032
|
+
*/
|
|
10807
11033
|
async reconnect() {
|
|
10808
11034
|
const logger2 = this.mlog(this.reconnect);
|
|
10809
11035
|
if (this.closing || this.isConnected || this.reconnectScheduled) return;
|
|
10810
|
-
if (this.reconnectAttempts >= this.config.reconnectAttempts) {
|
|
10811
|
-
logger2.error(`Failed to reconnect to RabbitMQ after ${this.reconnectAttempts} attempts`);
|
|
10812
|
-
return;
|
|
10813
|
-
}
|
|
10814
11036
|
this.reconnectAttempts++;
|
|
10815
|
-
const delay = this.config.reconnectDelayMs * Math.pow(2, this.reconnectAttempts - 1);
|
|
11037
|
+
const delay = Math.min(this.config.reconnectDelayMs * Math.pow(2, this.reconnectAttempts - 1), MAX_RETRY_DELAY_MS);
|
|
10816
11038
|
logger2.info(`Attempting to reconnect to RabbitMQ in ${delay}ms (attempt ${this.reconnectAttempts})`);
|
|
10817
11039
|
this.channel = null;
|
|
10818
11040
|
this.connection = null;
|
|
10819
11041
|
this.reconnectScheduled = true;
|
|
10820
|
-
setTimeout(async () => {
|
|
11042
|
+
this.reconnectTimer = setTimeout(async () => {
|
|
11043
|
+
this.reconnectTimer = null;
|
|
10821
11044
|
this.reconnectScheduled = false;
|
|
10822
11045
|
if (this.closing) return;
|
|
10823
11046
|
try {
|
|
10824
11047
|
await this.connect();
|
|
10825
11048
|
} catch (error) {
|
|
10826
|
-
logger2.error(`Reconnect attempt ${this.reconnectAttempts} failed: ${error
|
|
11049
|
+
logger2.error(`Reconnect attempt ${this.reconnectAttempts} failed: ${errorText(error)}`);
|
|
10827
11050
|
void this.reconnect();
|
|
10828
11051
|
}
|
|
10829
11052
|
}, delay);
|
|
@@ -10840,6 +11063,18 @@ var init_queue_service = __esm({
|
|
|
10840
11063
|
async disconnect() {
|
|
10841
11064
|
const logger2 = this.mlog(this.disconnect);
|
|
10842
11065
|
this.closing = true;
|
|
11066
|
+
if (this.reconnectTimer) {
|
|
11067
|
+
clearTimeout(this.reconnectTimer);
|
|
11068
|
+
this.reconnectTimer = null;
|
|
11069
|
+
this.reconnectScheduled = false;
|
|
11070
|
+
}
|
|
11071
|
+
for (const entry of this.consumerSpecs.values()) {
|
|
11072
|
+
const ch = entry.state.channel;
|
|
11073
|
+
this.teardownConsumerState(entry);
|
|
11074
|
+
if (ch) await ch.close().catch(() => {
|
|
11075
|
+
});
|
|
11076
|
+
}
|
|
11077
|
+
this.consumerSpecs.clear();
|
|
10843
11078
|
this.consumerRegistry.clear();
|
|
10844
11079
|
if (!this.isConnected && !this.connection) return;
|
|
10845
11080
|
await this.closeChannel();
|
|
@@ -10851,6 +11086,133 @@ var init_queue_service = __esm({
|
|
|
10851
11086
|
this.isConnected = false;
|
|
10852
11087
|
logger2.info("Disconnected from RabbitMQ");
|
|
10853
11088
|
}
|
|
11089
|
+
/**
|
|
11090
|
+
* Register a static consumer: stores the spec once, then activates it
|
|
11091
|
+
* (assert → bind → consume on the consumer's own channel). The single
|
|
11092
|
+
* entry point for every server's queue topology — replay after reconnect
|
|
11093
|
+
* reactivates the same specs, so declaration can never drift from
|
|
11094
|
+
* consumption.
|
|
11095
|
+
*
|
|
11096
|
+
* Registration failures propagate (startup contract: a server that cannot
|
|
11097
|
+
* establish its consumers must fail `onInit`, not limp). Failures on later
|
|
11098
|
+
* reactivation retry with bounded backoff instead.
|
|
11099
|
+
*/
|
|
11100
|
+
async registerConsumer(spec) {
|
|
11101
|
+
const existing = this.consumerSpecs.get(spec.id);
|
|
11102
|
+
if (existing) {
|
|
11103
|
+
if (existing.spec === spec) return;
|
|
11104
|
+
throw new Error(`Consumer id '${spec.id}' is already registered with a different spec`);
|
|
11105
|
+
}
|
|
11106
|
+
const entry = {
|
|
11107
|
+
spec,
|
|
11108
|
+
state: {
|
|
11109
|
+
channel: null,
|
|
11110
|
+
consumerTag: null,
|
|
11111
|
+
activating: false,
|
|
11112
|
+
retryTimer: null,
|
|
11113
|
+
retryAttempts: 0
|
|
11114
|
+
}
|
|
11115
|
+
};
|
|
11116
|
+
this.consumerSpecs.set(spec.id, entry);
|
|
11117
|
+
if (!this.isConnected) {
|
|
11118
|
+
await this.connect();
|
|
11119
|
+
}
|
|
11120
|
+
await this.activateConsumer(entry, true);
|
|
11121
|
+
}
|
|
11122
|
+
/**
|
|
11123
|
+
* Assert → bind → consume for one spec on its own channel. Single-flight
|
|
11124
|
+
* per id: a channel-error retry racing a full connection replay can never
|
|
11125
|
+
* produce two live consumers for the same spec.
|
|
11126
|
+
*/
|
|
11127
|
+
async activateConsumer(entry, throwOnFailure) {
|
|
11128
|
+
const logger2 = this.mlog(this.activateConsumer);
|
|
11129
|
+
const { spec, state } = entry;
|
|
11130
|
+
if (this.closing || state.activating || state.channel && state.consumerTag) return;
|
|
11131
|
+
if (!this.connection) {
|
|
11132
|
+
if (throwOnFailure) throw new QueueUnavailableError(`activate consumer '${spec.id}'`);
|
|
11133
|
+
this.scheduleConsumerRetry(entry);
|
|
11134
|
+
return;
|
|
11135
|
+
}
|
|
11136
|
+
state.activating = true;
|
|
11137
|
+
let channel = null;
|
|
11138
|
+
try {
|
|
11139
|
+
channel = await this.connection.createChannel();
|
|
11140
|
+
await channel.prefetch(env.queue.prefetch);
|
|
11141
|
+
channel.on("error", (err) => {
|
|
11142
|
+
logger2.error(`Consumer '${spec.id}' channel error: ${err.message}`);
|
|
11143
|
+
if (entry.state.channel === channel) {
|
|
11144
|
+
this.teardownConsumerState(entry);
|
|
11145
|
+
if (!this.closing) this.scheduleConsumerRetry(entry);
|
|
11146
|
+
}
|
|
11147
|
+
});
|
|
11148
|
+
try {
|
|
11149
|
+
await channel.assertQueue(spec.queue, spec.assertOptions ?? {
|
|
11150
|
+
durable: true
|
|
11151
|
+
});
|
|
11152
|
+
if (spec.binding) {
|
|
11153
|
+
await channel.assertExchange(spec.binding.exchange, spec.binding.type, {
|
|
11154
|
+
durable: true
|
|
11155
|
+
});
|
|
11156
|
+
for (const pattern of spec.binding.patterns) {
|
|
11157
|
+
await channel.bindQueue(spec.queue, spec.binding.exchange, pattern);
|
|
11158
|
+
}
|
|
11159
|
+
}
|
|
11160
|
+
} catch (declareError) {
|
|
11161
|
+
if (this.isPreconditionMismatch(declareError)) {
|
|
11162
|
+
logger2.error(`FATAL: queue declaration mismatch for consumer '${spec.id}' \u2014 queue=${spec.queue} options=${JSON.stringify(spec.assertOptions ?? {
|
|
11163
|
+
durable: true
|
|
11164
|
+
})} code=${declareError.code} reply=${declareError.message}`);
|
|
11165
|
+
this.fatalExit(1);
|
|
11166
|
+
}
|
|
11167
|
+
throw declareError;
|
|
11168
|
+
}
|
|
11169
|
+
const boundChannel = channel;
|
|
11170
|
+
const result = await boundChannel.consume(spec.queue, this.buildMessageHandler(spec.queue, spec.callback, spec.ignoreErrors ?? false, spec.dropStale ?? false, boundChannel, () => entry.state.channel), spec.consumeOptions ?? {
|
|
11171
|
+
noAck: false
|
|
11172
|
+
});
|
|
11173
|
+
state.channel = channel;
|
|
11174
|
+
state.consumerTag = result.consumerTag;
|
|
11175
|
+
state.retryAttempts = 0;
|
|
11176
|
+
if (state.retryTimer) {
|
|
11177
|
+
clearTimeout(state.retryTimer);
|
|
11178
|
+
state.retryTimer = null;
|
|
11179
|
+
}
|
|
11180
|
+
this.counters.consumersResubscribed++;
|
|
11181
|
+
} catch (error) {
|
|
11182
|
+
if (channel) await channel.close().catch(() => {
|
|
11183
|
+
});
|
|
11184
|
+
state.channel = null;
|
|
11185
|
+
state.consumerTag = null;
|
|
11186
|
+
logger2.error(`Error activating consumer '${spec.id}' on queue ${spec.queue}: ${error.message}`);
|
|
11187
|
+
if (throwOnFailure) throw error;
|
|
11188
|
+
this.scheduleConsumerRetry(entry);
|
|
11189
|
+
} finally {
|
|
11190
|
+
state.activating = false;
|
|
11191
|
+
}
|
|
11192
|
+
}
|
|
11193
|
+
/** Bounded per-consumer backoff — recovery must not depend on a full connection reconnect. */
|
|
11194
|
+
scheduleConsumerRetry(entry) {
|
|
11195
|
+
if (this.closing || entry.state.retryTimer) return;
|
|
11196
|
+
entry.state.retryAttempts++;
|
|
11197
|
+
const delay = Math.min(this.config.reconnectDelayMs * Math.pow(2, entry.state.retryAttempts - 1), MAX_RETRY_DELAY_MS);
|
|
11198
|
+
this.mlog(this.scheduleConsumerRetry).info(`Retrying consumer '${entry.spec.id}' in ${delay}ms (attempt ${entry.state.retryAttempts})`);
|
|
11199
|
+
entry.state.retryTimer = setTimeout(() => {
|
|
11200
|
+
entry.state.retryTimer = null;
|
|
11201
|
+
if (this.closing) return;
|
|
11202
|
+
void this.activateConsumer(entry, false);
|
|
11203
|
+
}, delay);
|
|
11204
|
+
}
|
|
11205
|
+
/** Reactivate every registered spec — the reconnect-time replay path. */
|
|
11206
|
+
async activateAllConsumers() {
|
|
11207
|
+
for (const entry of this.consumerSpecs.values()) {
|
|
11208
|
+
await this.activateConsumer(entry, false);
|
|
11209
|
+
}
|
|
11210
|
+
}
|
|
11211
|
+
/** 406 PRECONDITION_FAILED from a declare — the args-mismatch signature. Everything else is treated as transient. */
|
|
11212
|
+
isPreconditionMismatch(error) {
|
|
11213
|
+
const err = error;
|
|
11214
|
+
return err?.code === 406 && /PRECONDITION[-_]FAILED/i.test(String(err?.message ?? ""));
|
|
11215
|
+
}
|
|
10854
11216
|
async consume(queueName, callback, ignoreErrors = false, options2 = {
|
|
10855
11217
|
noAck: false
|
|
10856
11218
|
}, replay = true, dropStale = false) {
|
|
@@ -10860,57 +11222,7 @@ var init_queue_service = __esm({
|
|
|
10860
11222
|
}
|
|
10861
11223
|
try {
|
|
10862
11224
|
const boundChannel = this.channel;
|
|
10863
|
-
const result = await boundChannel.consume(queueName,
|
|
10864
|
-
if (!message) {
|
|
10865
|
-
logger2.warn(`Received null message from queue: ${queueName}`);
|
|
10866
|
-
return;
|
|
10867
|
-
}
|
|
10868
|
-
let content;
|
|
10869
|
-
try {
|
|
10870
|
-
content = JSON.parse(message.content.toString());
|
|
10871
|
-
} catch {
|
|
10872
|
-
logger2.error(`Dropping unparseable message from ${queueName} (invalid JSON, ${message.content.length} bytes)`);
|
|
10873
|
-
this.counters.messagesDropped++;
|
|
10874
|
-
this.safeNack(boundChannel, message, false);
|
|
10875
|
-
return;
|
|
10876
|
-
}
|
|
10877
|
-
if (!content || typeof content !== "object") {
|
|
10878
|
-
logger2.error(`Dropping non-object message from ${queueName}`);
|
|
10879
|
-
this.counters.messagesDropped++;
|
|
10880
|
-
this.safeNack(boundChannel, message, false);
|
|
10881
|
-
return;
|
|
10882
|
-
}
|
|
10883
|
-
const { requestId } = content;
|
|
10884
|
-
if (dropStale && this.isOverAge(content)) {
|
|
10885
|
-
logger2.warn(`Dropping over-age message ${requestId ?? "<no-id>"} from ${queueName} (older than ${env.queue.maxMessageAgeMs}ms)`);
|
|
10886
|
-
this.counters.messagesDropped++;
|
|
10887
|
-
this.safeNack(boundChannel, message, false);
|
|
10888
|
-
return;
|
|
10889
|
-
}
|
|
10890
|
-
try {
|
|
10891
|
-
await callback(requestId, content, message);
|
|
10892
|
-
this.safeAck(boundChannel, message);
|
|
10893
|
-
} catch (error) {
|
|
10894
|
-
const err = error;
|
|
10895
|
-
logger2.error(`Error processing message from queue: ${queueName}: ${err?.message}`);
|
|
10896
|
-
const pgCode = err?.code ?? err?.error?.code ?? err?.cause?.code;
|
|
10897
|
-
const isUnrecoverable = typeof pgCode === "string" && /^(22|23)/.test(pgCode);
|
|
10898
|
-
if (ignoreErrors || isUnrecoverable) {
|
|
10899
|
-
if (isUnrecoverable) {
|
|
10900
|
-
logger2.warn(`Dropping message from ${queueName} \u2014 unrecoverable data error (code ${pgCode})`);
|
|
10901
|
-
this.counters.messagesDropped++;
|
|
10902
|
-
}
|
|
10903
|
-
this.safeAck(boundChannel, message);
|
|
10904
|
-
} else if (message.fields.redelivered) {
|
|
10905
|
-
this.counters.messagesDropped++;
|
|
10906
|
-
this.safeNack(boundChannel, message, false);
|
|
10907
|
-
logger2.warn("Rejected problematic message after redelivery");
|
|
10908
|
-
} else {
|
|
10909
|
-
this.safeNack(boundChannel, message, true);
|
|
10910
|
-
logger2.info("Message re-queued for retry");
|
|
10911
|
-
}
|
|
10912
|
-
}
|
|
10913
|
-
}, options2);
|
|
11225
|
+
const result = await boundChannel.consume(queueName, this.buildMessageHandler(queueName, callback, ignoreErrors, dropStale, boundChannel, () => this.channel), options2);
|
|
10914
11226
|
this.activeConsumers.add(result.consumerTag);
|
|
10915
11227
|
if (replay && !this.replaying) {
|
|
10916
11228
|
this.consumerRegistry.set(queueName, {
|
|
@@ -10927,6 +11239,66 @@ var init_queue_service = __esm({
|
|
|
10927
11239
|
throw error;
|
|
10928
11240
|
}
|
|
10929
11241
|
}
|
|
11242
|
+
/**
|
|
11243
|
+
* The shared delivery pipeline: parse guards, over-age drop, ack/nack
|
|
11244
|
+
* policy. `currentChannel` identifies the channel the owning consumer
|
|
11245
|
+
* currently considers live, so acks for deliveries from a rotated channel
|
|
11246
|
+
* are suppressed (a stale-tag ack is a second channel-killing 406).
|
|
11247
|
+
*/
|
|
11248
|
+
buildMessageHandler(queueName, callback, ignoreErrors, dropStale, boundChannel, currentChannel) {
|
|
11249
|
+
const logger2 = this.mlog(this.consume);
|
|
11250
|
+
return async (message) => {
|
|
11251
|
+
if (!message) {
|
|
11252
|
+
logger2.warn(`Received null message from queue: ${queueName}`);
|
|
11253
|
+
return;
|
|
11254
|
+
}
|
|
11255
|
+
let content;
|
|
11256
|
+
try {
|
|
11257
|
+
content = JSON.parse(message.content.toString());
|
|
11258
|
+
} catch {
|
|
11259
|
+
logger2.error(`Dropping unparseable message from ${queueName} (invalid JSON, ${message.content.length} bytes)`);
|
|
11260
|
+
this.counters.messagesDropped++;
|
|
11261
|
+
this.safeNack(boundChannel, message, false, currentChannel);
|
|
11262
|
+
return;
|
|
11263
|
+
}
|
|
11264
|
+
if (!content || typeof content !== "object") {
|
|
11265
|
+
logger2.error(`Dropping non-object message from ${queueName}`);
|
|
11266
|
+
this.counters.messagesDropped++;
|
|
11267
|
+
this.safeNack(boundChannel, message, false, currentChannel);
|
|
11268
|
+
return;
|
|
11269
|
+
}
|
|
11270
|
+
const { requestId } = content;
|
|
11271
|
+
if (dropStale && this.isOverAge(content)) {
|
|
11272
|
+
logger2.warn(`Dropping over-age message ${requestId ?? "<no-id>"} from ${queueName} (older than ${env.queue.maxMessageAgeMs}ms)`);
|
|
11273
|
+
this.counters.messagesDropped++;
|
|
11274
|
+
this.safeNack(boundChannel, message, false, currentChannel);
|
|
11275
|
+
return;
|
|
11276
|
+
}
|
|
11277
|
+
try {
|
|
11278
|
+
await callback(requestId, content, message);
|
|
11279
|
+
this.safeAck(boundChannel, message, currentChannel);
|
|
11280
|
+
} catch (error) {
|
|
11281
|
+
const err = error;
|
|
11282
|
+
logger2.error(`Error processing message from queue: ${queueName}: ${err?.message}`);
|
|
11283
|
+
const pgCode = err?.code ?? err?.error?.code ?? err?.cause?.code;
|
|
11284
|
+
const isUnrecoverable = typeof pgCode === "string" && /^(22|23)/.test(pgCode);
|
|
11285
|
+
if (ignoreErrors || isUnrecoverable) {
|
|
11286
|
+
if (isUnrecoverable) {
|
|
11287
|
+
logger2.warn(`Dropping message from ${queueName} \u2014 unrecoverable data error (code ${pgCode})`);
|
|
11288
|
+
this.counters.messagesDropped++;
|
|
11289
|
+
}
|
|
11290
|
+
this.safeAck(boundChannel, message, currentChannel);
|
|
11291
|
+
} else if (message.fields.redelivered) {
|
|
11292
|
+
this.counters.messagesDropped++;
|
|
11293
|
+
this.safeNack(boundChannel, message, false, currentChannel);
|
|
11294
|
+
logger2.warn("Rejected problematic message after redelivery");
|
|
11295
|
+
} else {
|
|
11296
|
+
this.safeNack(boundChannel, message, true, currentChannel);
|
|
11297
|
+
logger2.info("Message re-queued for retry");
|
|
11298
|
+
}
|
|
11299
|
+
}
|
|
11300
|
+
};
|
|
11301
|
+
}
|
|
10930
11302
|
/**
|
|
10931
11303
|
* Send a message to a queue
|
|
10932
11304
|
* @param {string} queue - Queue name
|
|
@@ -10935,9 +11307,7 @@ var init_queue_service = __esm({
|
|
|
10935
11307
|
*/
|
|
10936
11308
|
async sendToQueue(queue, message, options2 = {}) {
|
|
10937
11309
|
const logger2 = this.mlog(this.sendToQueue);
|
|
10938
|
-
|
|
10939
|
-
await this.connect();
|
|
10940
|
-
}
|
|
11310
|
+
const channel = await this.publishChannel("sendToQueue");
|
|
10941
11311
|
try {
|
|
10942
11312
|
const buffer = Buffer.from(JSON.stringify(message));
|
|
10943
11313
|
const defaultOptions = {
|
|
@@ -10948,7 +11318,7 @@ var init_queue_service = __esm({
|
|
|
10948
11318
|
...defaultOptions,
|
|
10949
11319
|
...options2
|
|
10950
11320
|
};
|
|
10951
|
-
const sent =
|
|
11321
|
+
const sent = channel.sendToQueue(queue, buffer, messageOptions);
|
|
10952
11322
|
if (sent) {
|
|
10953
11323
|
logger2.debug(`Message sent to queue ${queue} with ID: ${message.id}`);
|
|
10954
11324
|
} else {
|
|
@@ -10969,9 +11339,7 @@ var init_queue_service = __esm({
|
|
|
10969
11339
|
*/
|
|
10970
11340
|
async sendToExchange(exchange, routingKey, message, options2 = {}) {
|
|
10971
11341
|
const logger2 = this.mlog(this.sendToExchange);
|
|
10972
|
-
|
|
10973
|
-
await this.connect();
|
|
10974
|
-
}
|
|
11342
|
+
const channel = await this.publishChannel(`publish to ${exchange}`);
|
|
10975
11343
|
try {
|
|
10976
11344
|
const buffer = Buffer.from(JSON.stringify(message));
|
|
10977
11345
|
const defaultOptions = {
|
|
@@ -10982,7 +11350,7 @@ var init_queue_service = __esm({
|
|
|
10982
11350
|
...defaultOptions,
|
|
10983
11351
|
...options2
|
|
10984
11352
|
};
|
|
10985
|
-
const sent =
|
|
11353
|
+
const sent = channel.publish(exchange, routingKey, buffer, messageOptions);
|
|
10986
11354
|
if (!sent) {
|
|
10987
11355
|
logger2.warn(`Failed to publish message to exchange ${exchange}, channel back-pressured`);
|
|
10988
11356
|
}
|
|
@@ -10992,6 +11360,27 @@ var init_queue_service = __esm({
|
|
|
10992
11360
|
throw error;
|
|
10993
11361
|
}
|
|
10994
11362
|
}
|
|
11363
|
+
/**
|
|
11364
|
+
* The shared channel used for publishes and legacy/dynamic operations.
|
|
11365
|
+
* Attempts a (single-flight) lazy connect when disconnected; if the broker
|
|
11366
|
+
* is still unavailable, fails fast with a typed error rather than
|
|
11367
|
+
* null-crashing — reconnection continues in the background.
|
|
11368
|
+
*/
|
|
11369
|
+
async publishChannel(operation) {
|
|
11370
|
+
if (!this.isConnected) {
|
|
11371
|
+
try {
|
|
11372
|
+
await this.connect();
|
|
11373
|
+
} catch {
|
|
11374
|
+
void this.reconnect();
|
|
11375
|
+
throw new QueueUnavailableError(operation);
|
|
11376
|
+
}
|
|
11377
|
+
}
|
|
11378
|
+
const channel = this.channel;
|
|
11379
|
+
if (!channel) {
|
|
11380
|
+
throw new QueueUnavailableError(operation);
|
|
11381
|
+
}
|
|
11382
|
+
return channel;
|
|
11383
|
+
}
|
|
10995
11384
|
/**
|
|
10996
11385
|
* Stop consuming messages
|
|
10997
11386
|
*/
|
|
@@ -11078,8 +11467,8 @@ var init_queue_service = __esm({
|
|
|
11078
11467
|
if (Number.isNaN(published)) return false;
|
|
11079
11468
|
return Date.now() - published > maxAge;
|
|
11080
11469
|
}
|
|
11081
|
-
safeAck(channel, message) {
|
|
11082
|
-
if (channel !==
|
|
11470
|
+
safeAck(channel, message, currentChannel) {
|
|
11471
|
+
if (channel !== currentChannel()) {
|
|
11083
11472
|
this.counters.staleDeliveriesSuppressed++;
|
|
11084
11473
|
return;
|
|
11085
11474
|
}
|
|
@@ -11089,8 +11478,8 @@ var init_queue_service = __esm({
|
|
|
11089
11478
|
this.mlog(this.safeAck).warn(`Ack on closed channel ignored: ${error.message}`);
|
|
11090
11479
|
}
|
|
11091
11480
|
}
|
|
11092
|
-
safeNack(channel, message, requeue) {
|
|
11093
|
-
if (channel !==
|
|
11481
|
+
safeNack(channel, message, requeue, currentChannel) {
|
|
11482
|
+
if (channel !== currentChannel()) {
|
|
11094
11483
|
this.counters.staleDeliveriesSuppressed++;
|
|
11095
11484
|
return;
|
|
11096
11485
|
}
|
|
@@ -11101,10 +11490,10 @@ var init_queue_service = __esm({
|
|
|
11101
11490
|
}
|
|
11102
11491
|
}
|
|
11103
11492
|
/**
|
|
11104
|
-
* Re-subscribe every registered
|
|
11493
|
+
* Re-subscribe every registered legacy consumer onto the shared channel.
|
|
11105
11494
|
* Runs inside createChannel() so it covers BOTH a connection-level reconnect
|
|
11106
|
-
* and a channel-only recreate (the 406 consumer_timeout path).
|
|
11107
|
-
*
|
|
11495
|
+
* and a channel-only recreate (the 406 consumer_timeout path). Spec-based
|
|
11496
|
+
* consumers reactivate through activateAllConsumers/their own retry instead.
|
|
11108
11497
|
*/
|
|
11109
11498
|
async replayConsumers() {
|
|
11110
11499
|
if (this.closing || this.consumerRegistry.size === 0) return;
|
|
@@ -11155,6 +11544,7 @@ var init_queue_service = __esm({
|
|
|
11155
11544
|
});
|
|
11156
11545
|
this.isConnected = true;
|
|
11157
11546
|
this.reconnectAttempts = 0;
|
|
11547
|
+
this.disconnectedSince = null;
|
|
11158
11548
|
const reconnected = this.hasConnectedOnce;
|
|
11159
11549
|
this.hasConnectedOnce = true;
|
|
11160
11550
|
await this.replayConsumers();
|
|
@@ -11399,7 +11789,19 @@ var init_plugin_lifecycle_service = __esm({
|
|
|
11399
11789
|
for (const pluginRow of enabled) {
|
|
11400
11790
|
const family = pluginRow.family.toUpperCase();
|
|
11401
11791
|
enabledFamilies.add(family);
|
|
11402
|
-
|
|
11792
|
+
let current = this.loadedState.get(family);
|
|
11793
|
+
if (!current && this.pluginRegistry.has(pluginRow.id)) {
|
|
11794
|
+
current = {
|
|
11795
|
+
pluginId: pluginRow.id,
|
|
11796
|
+
family,
|
|
11797
|
+
version: pluginRow.version,
|
|
11798
|
+
sha256: this.pluginRegistry.loadedSha(pluginRow.id) ?? "",
|
|
11799
|
+
packageName: pluginRow.name,
|
|
11800
|
+
packageDir: "",
|
|
11801
|
+
loadedAt: Date.now()
|
|
11802
|
+
};
|
|
11803
|
+
this.loadedState.set(family, current);
|
|
11804
|
+
}
|
|
11403
11805
|
if (current?.sha256 === pluginRow.sha256) continue;
|
|
11404
11806
|
try {
|
|
11405
11807
|
if (current) {
|
|
@@ -13007,7 +13409,28 @@ var init_response_service = __esm({
|
|
|
13007
13409
|
async startLLMResponseConsumer(serverId) {
|
|
13008
13410
|
this.serverId = serverId || this.serverId;
|
|
13009
13411
|
const queueName = `${this.responseQueue}.${this.serverId}`;
|
|
13010
|
-
await this.queueService.
|
|
13412
|
+
await this.queueService.registerConsumer({
|
|
13413
|
+
id: "api.llm_responses",
|
|
13414
|
+
queue: queueName,
|
|
13415
|
+
// x-expires reclaims the per-instance queue when this API instance is
|
|
13416
|
+
// gone for good; registerConsumer re-asserts it on every reactivation,
|
|
13417
|
+
// so an expiry during an outage can no longer strand the consumer.
|
|
13418
|
+
assertOptions: {
|
|
13419
|
+
durable: true,
|
|
13420
|
+
arguments: {
|
|
13421
|
+
"x-expires": env.queue.queueExpiration
|
|
13422
|
+
}
|
|
13423
|
+
},
|
|
13424
|
+
binding: {
|
|
13425
|
+
exchange: env.queue.responseExchange,
|
|
13426
|
+
type: "direct",
|
|
13427
|
+
patterns: [
|
|
13428
|
+
this.serverId
|
|
13429
|
+
]
|
|
13430
|
+
},
|
|
13431
|
+
callback: /* @__PURE__ */ __name(async (id, content) => this.processWireChunk(id, content), "callback"),
|
|
13432
|
+
ignoreErrors: true
|
|
13433
|
+
});
|
|
13011
13434
|
}
|
|
13012
13435
|
async processWireChunk(_id, wire) {
|
|
13013
13436
|
const logger2 = this.mlog(this.processWireChunk);
|
|
@@ -13439,13 +13862,14 @@ var init_init_service = __esm({
|
|
|
13439
13862
|
await this.pluginWatcherService.startWatching();
|
|
13440
13863
|
}
|
|
13441
13864
|
}
|
|
13865
|
+
// Queue topology lives with each queue's consumer (QueueService.registerConsumer
|
|
13866
|
+
// asserts immediately before consuming), so this only declares the exchanges the
|
|
13867
|
+
// API publishes to. Consumer-owned queue declarations were removed with the move
|
|
13868
|
+
// to registerConsumer — a second declaration site is how declare/consume drift.
|
|
13442
13869
|
async setupQueues(serverId) {
|
|
13443
13870
|
logger_default.debug(`setupQueues called with ${serverId}`);
|
|
13444
13871
|
this.serverId = serverId || this.serverId;
|
|
13445
13872
|
await this._setupGlobalExchanges();
|
|
13446
|
-
await this._setupRequestQueues();
|
|
13447
|
-
await this._setupResponseQueues();
|
|
13448
|
-
await this._setupNotificationQueues();
|
|
13449
13873
|
}
|
|
13450
13874
|
async _setupGlobalExchanges() {
|
|
13451
13875
|
await this.queueService.assertExchange(env.queue.requestExchange, "fanout");
|
|
@@ -13455,31 +13879,6 @@ var init_init_service = __esm({
|
|
|
13455
13879
|
await this.queueService.assertExchange(env.queue.adminResponseExchange, "direct");
|
|
13456
13880
|
await this.queueService.assertExchange(env.queue.directExchange, "direct");
|
|
13457
13881
|
}
|
|
13458
|
-
async _setupRequestQueues() {
|
|
13459
|
-
await this.queueService.assertQueue(env.queue.requestQueue, {
|
|
13460
|
-
durable: true
|
|
13461
|
-
}, env.queue.requestExchange);
|
|
13462
|
-
await this.queueService.assertQueue(env.queue.auditRequestQueue, {
|
|
13463
|
-
durable: true
|
|
13464
|
-
}, env.queue.requestExchange);
|
|
13465
|
-
}
|
|
13466
|
-
async _setupResponseQueues() {
|
|
13467
|
-
const responseQueueName = `${env.queue.responseQueue}.${this.serverId}`;
|
|
13468
|
-
await this.queueService.assertQueue(responseQueueName, {
|
|
13469
|
-
durable: true,
|
|
13470
|
-
arguments: {
|
|
13471
|
-
"x-expires": env.queue.queueExpiration
|
|
13472
|
-
}
|
|
13473
|
-
}, env.queue.responseExchange, this.serverId);
|
|
13474
|
-
await this.queueService.assertQueue(env.queue.auditResponseQueue, {
|
|
13475
|
-
durable: true
|
|
13476
|
-
}, env.queue.responseExchange, env.queue.auditRoutingKey);
|
|
13477
|
-
}
|
|
13478
|
-
async _setupNotificationQueues() {
|
|
13479
|
-
await this.queueService.assertQueue(env.queue.auditNotificationQueue, {
|
|
13480
|
-
durable: true
|
|
13481
|
-
}, env.queue.notificationExchange, "#");
|
|
13482
|
-
}
|
|
13483
13882
|
};
|
|
13484
13883
|
InitService = _ts_decorate49([
|
|
13485
13884
|
injectable49(),
|
|
@@ -14310,10 +14709,11 @@ var init_system_model_binding_cache = __esm({
|
|
|
14310
14709
|
* legitimately mint, including the platform-org sentinel, which is why it asserts UUID
|
|
14311
14710
|
* *shape* rather than RFC-4122 conformance (see {@link isKeySafeUuid}).
|
|
14312
14711
|
*/
|
|
14313
|
-
key(orgId, selector) {
|
|
14712
|
+
key(orgId, selector, profileId) {
|
|
14314
14713
|
this.assertOrgId(orgId);
|
|
14315
14714
|
this.assertSelector(selector);
|
|
14316
|
-
|
|
14715
|
+
this.assertProfileId(profileId);
|
|
14716
|
+
return `${this.prefix}${orgId}:selector:${selector}:profile:${profileId ?? "none"}`;
|
|
14317
14717
|
}
|
|
14318
14718
|
assertOrgId(orgId) {
|
|
14319
14719
|
if (!isKeySafeUuid(orgId)) {
|
|
@@ -14325,16 +14725,26 @@ var init_system_model_binding_cache = __esm({
|
|
|
14325
14725
|
throw HoloError.badRequest("Invalid system-model selector.");
|
|
14326
14726
|
}
|
|
14327
14727
|
}
|
|
14328
|
-
|
|
14329
|
-
|
|
14728
|
+
/**
|
|
14729
|
+
* The profile segment admits exactly two shapes: a UUID (the token's mint-validated
|
|
14730
|
+
* claim, already shape-checked by the verifier — re-asserted here so no other caller
|
|
14731
|
+
* can widen a glob through this segment) or `null`, rendered as the literal `none`.
|
|
14732
|
+
*/
|
|
14733
|
+
assertProfileId(profileId) {
|
|
14734
|
+
if (profileId !== null && !isKeySafeUuid(profileId)) {
|
|
14735
|
+
throw HoloError.badRequest("Invalid system-model profile.");
|
|
14736
|
+
}
|
|
14737
|
+
}
|
|
14738
|
+
async get(orgId, selector, profileId) {
|
|
14739
|
+
return this.redis.get(this.key(orgId, selector, profileId));
|
|
14330
14740
|
}
|
|
14331
14741
|
/**
|
|
14332
14742
|
* Admit a binding to the cache. The parameter type is the admission predicate: only a
|
|
14333
14743
|
* row already proven servable can be passed, so admission is exactly servability
|
|
14334
14744
|
* rather than an approximation of it re-derived here.
|
|
14335
14745
|
*/
|
|
14336
|
-
async set(orgId, selector, row) {
|
|
14337
|
-
await this.redis.set(this.key(orgId, selector), row, Math.ceil(env.onBehalf.maxTtlMs / 1e3));
|
|
14746
|
+
async set(orgId, selector, profileId, row) {
|
|
14747
|
+
await this.redis.set(this.key(orgId, selector, profileId), row, Math.ceil(env.onBehalf.maxTtlMs / 1e3));
|
|
14338
14748
|
}
|
|
14339
14749
|
/**
|
|
14340
14750
|
* Evict every cached binding for an org (`binding:org:{org}:*`). Returns the count removed.
|
|
@@ -14360,7 +14770,7 @@ var init_system_model_binding_cache = __esm({
|
|
|
14360
14770
|
*/
|
|
14361
14771
|
async evictSelectorAcrossOrgs(selector) {
|
|
14362
14772
|
this.assertSelector(selector);
|
|
14363
|
-
return this.scanAndDelete(`${this.prefix}*:selector:${selector}
|
|
14773
|
+
return this.scanAndDelete(`${this.prefix}*:selector:${selector}:profile:*`);
|
|
14364
14774
|
}
|
|
14365
14775
|
/** Delete every key matching `pattern`. Returns the count removed. */
|
|
14366
14776
|
async scanAndDelete(pattern) {
|
|
@@ -14453,12 +14863,12 @@ var init_holo_request_service = __esm({
|
|
|
14453
14863
|
async resolveSystemModel(auth, request) {
|
|
14454
14864
|
const logger2 = this.mlog(this.resolveSystemModel);
|
|
14455
14865
|
const orgId = auth.organizationId;
|
|
14456
|
-
const { selector } = auth.onBehalf;
|
|
14866
|
+
const { selector, profileId } = auth.onBehalf;
|
|
14457
14867
|
if (!selector) {
|
|
14458
14868
|
throw HoloError.badRequest("A system-model alias is required.");
|
|
14459
14869
|
}
|
|
14460
|
-
const cached = await this.bindingCache.get(orgId, selector);
|
|
14461
|
-
const row = cached ?? await this.effectiveModelDB.getEffectiveModel(orgId, selector);
|
|
14870
|
+
const cached = await this.bindingCache.get(orgId, selector, profileId);
|
|
14871
|
+
const row = cached ?? await this.effectiveModelDB.getEffectiveModel(orgId, selector, profileId);
|
|
14462
14872
|
if (!row) {
|
|
14463
14873
|
this.logSystemModelDeny(logger2, "unknown_selector", auth, selector);
|
|
14464
14874
|
throw HoloError.notFound("Model");
|
|
@@ -14468,7 +14878,7 @@ var init_holo_request_service = __esm({
|
|
|
14468
14878
|
throw HoloError.conflict("The configured model binding is currently unservable.", HoloErrorCode.UNSERVABLE_BINDING);
|
|
14469
14879
|
}
|
|
14470
14880
|
if (!cached) {
|
|
14471
|
-
await this.bindingCache.set(orgId, selector, row);
|
|
14881
|
+
await this.bindingCache.set(orgId, selector, profileId, row);
|
|
14472
14882
|
}
|
|
14473
14883
|
const provider = await this.providerDB.getById(row.provider_id);
|
|
14474
14884
|
if (!provider) {
|
|
@@ -14977,7 +15387,8 @@ function makeOnBehalfPreHandler(verifier) {
|
|
|
14977
15387
|
applications: [],
|
|
14978
15388
|
clientIdentifier: verified.sub,
|
|
14979
15389
|
onBehalf: {
|
|
14980
|
-
selector: verified.selector
|
|
15390
|
+
selector: verified.selector,
|
|
15391
|
+
profileId: verified.profileId
|
|
14981
15392
|
}
|
|
14982
15393
|
};
|
|
14983
15394
|
req.auth = auth;
|
|
@@ -16918,16 +17329,6 @@ function withAdmin(Base) {
|
|
|
16918
17329
|
logger_default.debug(`${this.id} is Admin Aware`);
|
|
16919
17330
|
const commandQueue = `${env.queue.adminCommandQueue}_${this.id}`;
|
|
16920
17331
|
const exchange = env.queue.adminExchange;
|
|
16921
|
-
await this.queueService.assertExchange(exchange, "topic");
|
|
16922
|
-
await this.queueService.assertQueue(commandQueue, {
|
|
16923
|
-
exclusive: false,
|
|
16924
|
-
durable: true,
|
|
16925
|
-
autoDelete: true
|
|
16926
|
-
}, exchange, "model.#");
|
|
16927
|
-
await this.queueService.bindQueue(commandQueue, exchange, "worker.#");
|
|
16928
|
-
await this.queueService.bindQueue(commandQueue, exchange, "plugin.#");
|
|
16929
|
-
await this.queueService.bindQueue(commandQueue, exchange, "provider.#");
|
|
16930
|
-
await this.queueService.bindQueue(commandQueue, exchange, "datastore.#");
|
|
16931
17332
|
this.pluginLifecycleService.setServerId(this.id);
|
|
16932
17333
|
this.adminHandlers.set("plugin.enabled", async (_serverId, payload) => {
|
|
16933
17334
|
await this.pluginLifecycleService.handleRemoteEnable(payload);
|
|
@@ -16968,29 +17369,49 @@ function withAdmin(Base) {
|
|
|
16968
17369
|
if (env.api.pluginReconcileIntervalMs > 0) {
|
|
16969
17370
|
this.pluginLifecycleService.startPeriodicReconciliation(env.api.pluginReconcileIntervalMs);
|
|
16970
17371
|
}
|
|
16971
|
-
await this.queueService.
|
|
16972
|
-
|
|
16973
|
-
|
|
16974
|
-
|
|
16975
|
-
|
|
16976
|
-
|
|
16977
|
-
|
|
16978
|
-
|
|
16979
|
-
|
|
16980
|
-
|
|
16981
|
-
|
|
16982
|
-
|
|
16983
|
-
|
|
16984
|
-
|
|
16985
|
-
|
|
16986
|
-
|
|
17372
|
+
await this.queueService.registerConsumer({
|
|
17373
|
+
id: `admin.commands.${this.id}`,
|
|
17374
|
+
queue: commandQueue,
|
|
17375
|
+
assertOptions: {
|
|
17376
|
+
exclusive: false,
|
|
17377
|
+
durable: true,
|
|
17378
|
+
autoDelete: true
|
|
17379
|
+
},
|
|
17380
|
+
binding: {
|
|
17381
|
+
exchange,
|
|
17382
|
+
type: "topic",
|
|
17383
|
+
patterns: [
|
|
17384
|
+
"model.#",
|
|
17385
|
+
"worker.#",
|
|
17386
|
+
"plugin.#",
|
|
17387
|
+
"provider.#",
|
|
17388
|
+
"datastore.#"
|
|
17389
|
+
]
|
|
17390
|
+
},
|
|
17391
|
+
callback: /* @__PURE__ */ __name(async (messageId, content, _message) => {
|
|
17392
|
+
const { action, serverId, timestamp, ...payload } = content;
|
|
17393
|
+
logger_default.info(`Worker (${this.id}) received admin command: ${action}, messageId: ${messageId}`);
|
|
17394
|
+
const handler = this.adminHandlers.get(action);
|
|
17395
|
+
let response = {};
|
|
17396
|
+
if (!handler) {
|
|
17397
|
+
logger_default.warn(`No handler registered for admin command: ${action}`);
|
|
16987
17398
|
response = {
|
|
16988
17399
|
success: false,
|
|
16989
|
-
message:
|
|
17400
|
+
message: `No handler registered for admin command: ${action}`
|
|
16990
17401
|
};
|
|
17402
|
+
} else {
|
|
17403
|
+
try {
|
|
17404
|
+
response = await handler(this.id, payload);
|
|
17405
|
+
} catch (error) {
|
|
17406
|
+
logger_default.error(`Error handling admin command: ${action} - ${error.message}`);
|
|
17407
|
+
response = {
|
|
17408
|
+
success: false,
|
|
17409
|
+
message: error.message
|
|
17410
|
+
};
|
|
17411
|
+
}
|
|
16991
17412
|
}
|
|
16992
|
-
|
|
16993
|
-
|
|
17413
|
+
await this.adminService.sendAdminResponse(this.id, serverId, messageId, response);
|
|
17414
|
+
}, "callback")
|
|
16994
17415
|
});
|
|
16995
17416
|
}
|
|
16996
17417
|
async onShutdown() {
|
|
@@ -17124,6 +17545,10 @@ function withHealth(Base) {
|
|
|
17124
17545
|
status: healthy ? "ok" : "unhealthy",
|
|
17125
17546
|
connected,
|
|
17126
17547
|
consumers,
|
|
17548
|
+
// Current consecutive attempt count (0 while connected) +
|
|
17549
|
+
// outage start — makes a stuck-retrying instance visible.
|
|
17550
|
+
reconnectAttempts: queueService?.reconnectAttempts ?? 0,
|
|
17551
|
+
disconnectedSince: queueService?.disconnectedSince?.toISOString() ?? null,
|
|
17127
17552
|
metrics: queueService?.metrics,
|
|
17128
17553
|
uptime: process.uptime(),
|
|
17129
17554
|
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -21133,116 +21558,129 @@ var init_worker_server = __esm({
|
|
|
21133
21558
|
this.adminHandlers.set("worker.restart", this.adminService.restartWorker);
|
|
21134
21559
|
const requestQueue = env.queue.requestQueue;
|
|
21135
21560
|
const logger2 = this.mlog("workerConsume");
|
|
21136
|
-
await this.queueService.
|
|
21137
|
-
|
|
21138
|
-
|
|
21139
|
-
|
|
21140
|
-
|
|
21141
|
-
|
|
21142
|
-
|
|
21143
|
-
|
|
21144
|
-
|
|
21145
|
-
|
|
21146
|
-
|
|
21147
|
-
|
|
21148
|
-
|
|
21149
|
-
|
|
21150
|
-
|
|
21151
|
-
|
|
21152
|
-
|
|
21153
|
-
|
|
21154
|
-
|
|
21155
|
-
|
|
21156
|
-
|
|
21157
|
-
|
|
21158
|
-
|
|
21159
|
-
ts: Date.now(),
|
|
21160
|
-
status: 400,
|
|
21161
|
-
metrics: {
|
|
21162
|
-
inputTokens: 0,
|
|
21163
|
-
outputTokens: 0,
|
|
21164
|
-
timeToFirstToken: 0,
|
|
21165
|
-
totalProcessingTime: 0,
|
|
21166
|
-
totalTokens: 0
|
|
21167
|
-
},
|
|
21168
|
-
text: errorMessage,
|
|
21169
|
-
error: ai.responseFactory.createError(errorMessage)
|
|
21170
|
-
};
|
|
21171
|
-
for (const wireChunk of await wire.fromProviderEvent(evt)) {
|
|
21172
|
-
logger2.debug(`Guard failed response: ${JSON.stringify(wireChunk)}`);
|
|
21173
|
-
await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
|
|
21174
|
-
}
|
|
21175
|
-
await this.responseService.sendToAudit({
|
|
21176
|
-
...workerRequest,
|
|
21177
|
-
providerEvent: evt,
|
|
21178
|
-
workerId: this.id
|
|
21179
|
-
});
|
|
21180
|
-
await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, evt));
|
|
21181
|
-
} else {
|
|
21182
|
-
const apiCredentialId = workerRequest.provider.api_credential_id;
|
|
21183
|
-
if (apiCredentialId) {
|
|
21184
|
-
this.apiCredentialDB.updateLastUsed(apiCredentialId).catch((err) => reqLogger.error(`Failed to update api_credential last_used_at: ${err.message}`));
|
|
21185
|
-
}
|
|
21186
|
-
const wirePlugin = this.resolveWirePlugin(workerRequest, plugin);
|
|
21187
|
-
if (wirePlugin) {
|
|
21188
|
-
logger2.info(`Translating inbound: ${wirePlugin.family}/${workerRequest.wireProtocolName} \u2192 holo \u2192 ${plugin.family}/${workerRequest.protocol.name}`);
|
|
21189
|
-
await this.applyInboundTranslation(workerRequest, wirePlugin);
|
|
21190
|
-
}
|
|
21191
|
-
const wire = await this.createWireAdapter(requestId, workerRequest, wirePlugin, plugin);
|
|
21192
|
-
const rawEvents = await ai.processWorkerRequest(workerRequest, {
|
|
21193
|
-
requestTimeoutMs: env.worker.requestTimeoutMs,
|
|
21194
|
-
idleTimeoutMs: env.worker.streamIdleTimeoutMs
|
|
21195
|
-
});
|
|
21196
|
-
const events = wirePlugin ? translateProviderEvents(rawEvents, plugin.translator, wirePlugin.translator) : rawEvents;
|
|
21197
|
-
await processProviderEvents(events, wire, workerRequest, envelope, {
|
|
21198
|
-
publishChunk: this.responseService.sendResponseChunk.bind(this),
|
|
21199
|
-
publishFinal: /* @__PURE__ */ __name(async (final) => {
|
|
21200
|
-
await this.responseService.sendToAudit(final);
|
|
21201
|
-
await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, final.providerEvent));
|
|
21202
|
-
}, "publishFinal")
|
|
21203
|
-
});
|
|
21204
|
-
}
|
|
21205
|
-
} catch (error) {
|
|
21206
|
-
logger2.error(`Error handling request (${requestId}): ${error.message}`);
|
|
21561
|
+
await this.queueService.registerConsumer({
|
|
21562
|
+
id: "worker.llm_requests",
|
|
21563
|
+
queue: requestQueue,
|
|
21564
|
+
assertOptions: {
|
|
21565
|
+
durable: true
|
|
21566
|
+
},
|
|
21567
|
+
binding: {
|
|
21568
|
+
exchange: env.queue.requestExchange,
|
|
21569
|
+
type: "fanout",
|
|
21570
|
+
patterns: [
|
|
21571
|
+
""
|
|
21572
|
+
]
|
|
21573
|
+
},
|
|
21574
|
+
callback: /* @__PURE__ */ __name(async (requestId, workerRequest, message) => {
|
|
21575
|
+
this.stats.totalRequests++;
|
|
21576
|
+
const reqLogger = workerRequest.capability ? logger2.child(capabilityToMetadata(workerRequest.capability)) : logger2;
|
|
21577
|
+
const redelivered = message?.fields?.redelivered ?? false;
|
|
21578
|
+
reqLogger.info(`Worker ${this.id} handling request: ${requestId} provider: ${workerRequest.provider.name} from server ${workerRequest.sourceId} and queue ${requestQueue}${redelivered ? " (redelivered)" : ""}...`);
|
|
21579
|
+
const ai = await this.providerPluginService.getProviderImplById(workerRequest.provider.id, workerRequest.protocol?.plugin_id);
|
|
21580
|
+
const plugin = ai.plugin;
|
|
21581
|
+
logger2.info(`resolved ai provider: ${ai.name}`);
|
|
21582
|
+
const { sourceId, guardResult } = workerRequest;
|
|
21583
|
+
const envelope = await ai.auditor.createWorkerResponseEnvelope(workerRequest, this.id);
|
|
21207
21584
|
try {
|
|
21208
|
-
|
|
21209
|
-
|
|
21210
|
-
|
|
21211
|
-
|
|
21212
|
-
|
|
21213
|
-
|
|
21214
|
-
|
|
21215
|
-
|
|
21216
|
-
|
|
21217
|
-
|
|
21218
|
-
|
|
21219
|
-
|
|
21220
|
-
|
|
21221
|
-
|
|
21222
|
-
|
|
21585
|
+
if (guardResult && !guardResult.passed) {
|
|
21586
|
+
logger2.info("Guards failed. Sending back guard errors.");
|
|
21587
|
+
const wire = await plugin.createWireAdapter({
|
|
21588
|
+
requestId,
|
|
21589
|
+
isStreaming: false,
|
|
21590
|
+
protocol: ai.plugin.defaultProtocol
|
|
21591
|
+
});
|
|
21592
|
+
const errorMessage = guardResult.errors?.join("\n\n") || "Policy violation";
|
|
21593
|
+
const evt = {
|
|
21594
|
+
type: "error",
|
|
21595
|
+
requestId,
|
|
21596
|
+
seq: 0,
|
|
21597
|
+
ts: Date.now(),
|
|
21598
|
+
status: 400,
|
|
21599
|
+
metrics: {
|
|
21600
|
+
inputTokens: 0,
|
|
21601
|
+
outputTokens: 0,
|
|
21602
|
+
timeToFirstToken: 0,
|
|
21603
|
+
totalProcessingTime: 0,
|
|
21604
|
+
totalTokens: 0
|
|
21605
|
+
},
|
|
21606
|
+
text: errorMessage,
|
|
21607
|
+
error: ai.responseFactory.createError(errorMessage)
|
|
21608
|
+
};
|
|
21609
|
+
for (const wireChunk of await wire.fromProviderEvent(evt)) {
|
|
21610
|
+
logger2.debug(`Guard failed response: ${JSON.stringify(wireChunk)}`);
|
|
21611
|
+
await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
|
|
21223
21612
|
}
|
|
21224
|
-
|
|
21225
|
-
|
|
21226
|
-
|
|
21227
|
-
|
|
21228
|
-
|
|
21229
|
-
|
|
21230
|
-
|
|
21231
|
-
|
|
21613
|
+
await this.responseService.sendToAudit({
|
|
21614
|
+
...workerRequest,
|
|
21615
|
+
providerEvent: evt,
|
|
21616
|
+
workerId: this.id
|
|
21617
|
+
});
|
|
21618
|
+
await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, evt));
|
|
21619
|
+
} else {
|
|
21620
|
+
const apiCredentialId = workerRequest.provider.api_credential_id;
|
|
21621
|
+
if (apiCredentialId) {
|
|
21622
|
+
this.apiCredentialDB.updateLastUsed(apiCredentialId).catch((err) => reqLogger.error(`Failed to update api_credential last_used_at: ${err.message}`));
|
|
21623
|
+
}
|
|
21624
|
+
const wirePlugin = this.resolveWirePlugin(workerRequest, plugin);
|
|
21625
|
+
if (wirePlugin) {
|
|
21626
|
+
logger2.info(`Translating inbound: ${wirePlugin.family}/${workerRequest.wireProtocolName} \u2192 holo \u2192 ${plugin.family}/${workerRequest.protocol.name}`);
|
|
21627
|
+
await this.applyInboundTranslation(workerRequest, wirePlugin);
|
|
21628
|
+
}
|
|
21629
|
+
const wire = await this.createWireAdapter(requestId, workerRequest, wirePlugin, plugin);
|
|
21630
|
+
const rawEvents = await ai.processWorkerRequest(workerRequest, {
|
|
21631
|
+
requestTimeoutMs: env.worker.requestTimeoutMs,
|
|
21632
|
+
idleTimeoutMs: env.worker.streamIdleTimeoutMs
|
|
21633
|
+
});
|
|
21634
|
+
const events = wirePlugin ? translateProviderEvents(rawEvents, plugin.translator, wirePlugin.translator) : rawEvents;
|
|
21635
|
+
await processProviderEvents(events, wire, workerRequest, envelope, {
|
|
21636
|
+
publishChunk: this.responseService.sendResponseChunk.bind(this),
|
|
21637
|
+
publishFinal: /* @__PURE__ */ __name(async (final) => {
|
|
21638
|
+
await this.responseService.sendToAudit(final);
|
|
21639
|
+
await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, final.providerEvent));
|
|
21640
|
+
}, "publishFinal")
|
|
21641
|
+
});
|
|
21232
21642
|
}
|
|
21233
|
-
|
|
21234
|
-
|
|
21235
|
-
|
|
21236
|
-
|
|
21237
|
-
|
|
21238
|
-
|
|
21239
|
-
|
|
21643
|
+
} catch (error) {
|
|
21644
|
+
logger2.error(`Error handling request (${requestId}): ${error.message}`);
|
|
21645
|
+
try {
|
|
21646
|
+
const errorMessage = error.message;
|
|
21647
|
+
const errorEvent = {
|
|
21648
|
+
type: "error",
|
|
21649
|
+
requestId,
|
|
21650
|
+
seq: 0,
|
|
21651
|
+
ts: Date.now(),
|
|
21652
|
+
status: 500,
|
|
21653
|
+
error: ai.responseFactory.createError(errorMessage),
|
|
21654
|
+
text: errorMessage,
|
|
21655
|
+
metrics: {
|
|
21656
|
+
inputTokens: 0,
|
|
21657
|
+
outputTokens: 0,
|
|
21658
|
+
timeToFirstToken: 0,
|
|
21659
|
+
totalProcessingTime: 0,
|
|
21660
|
+
totalTokens: 0
|
|
21661
|
+
}
|
|
21662
|
+
};
|
|
21663
|
+
const wire = workerRequest.isHoloNative ? new HoloWireAdapter(requestId, false, workerRequest.payload?.model) : await plugin.createWireAdapter({
|
|
21664
|
+
requestId,
|
|
21665
|
+
isStreaming: false,
|
|
21666
|
+
protocol: workerRequest.protocol?.name ?? ai.plugin.defaultProtocol
|
|
21667
|
+
});
|
|
21668
|
+
for (const wireChunk of await wire.fromProviderEvent(errorEvent)) {
|
|
21669
|
+
await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
|
|
21670
|
+
}
|
|
21671
|
+
await this.responseService.sendToAudit({
|
|
21672
|
+
...workerRequest,
|
|
21673
|
+
providerEvent: errorEvent,
|
|
21674
|
+
workerId: this.id
|
|
21675
|
+
});
|
|
21676
|
+
} catch (auditError) {
|
|
21677
|
+
logger2.error(`Failed to send error response for (${requestId}): ${auditError.message}`);
|
|
21678
|
+
}
|
|
21679
|
+
await this.onError(error);
|
|
21240
21680
|
}
|
|
21241
|
-
|
|
21242
|
-
|
|
21243
|
-
}
|
|
21244
|
-
noAck: false
|
|
21245
|
-
}, true, true);
|
|
21681
|
+
}, "callback"),
|
|
21682
|
+
dropStale: true
|
|
21683
|
+
});
|
|
21246
21684
|
}
|
|
21247
21685
|
async onShutdown() {
|
|
21248
21686
|
await super.onShutdown();
|
|
@@ -21381,14 +21819,56 @@ var init_audit_server = __esm({
|
|
|
21381
21819
|
this.pluginLifecycleService.setPluginsDir(env.audit.pluginsDir);
|
|
21382
21820
|
await this.pluginRegistry.loadAllActive();
|
|
21383
21821
|
await this.providerImplService.warmAll();
|
|
21384
|
-
await this.queueService.
|
|
21385
|
-
|
|
21822
|
+
await this.queueService.registerConsumer({
|
|
21823
|
+
id: "audit.notifications_audit",
|
|
21824
|
+
queue: env.queue.auditNotificationQueue,
|
|
21825
|
+
assertOptions: {
|
|
21826
|
+
durable: true
|
|
21827
|
+
},
|
|
21828
|
+
binding: {
|
|
21829
|
+
exchange: env.queue.notificationExchange,
|
|
21830
|
+
type: "topic",
|
|
21831
|
+
patterns: [
|
|
21832
|
+
"#"
|
|
21833
|
+
]
|
|
21834
|
+
},
|
|
21835
|
+
callback: /* @__PURE__ */ __name(async (_id, content) => {
|
|
21836
|
+
await this.auditService.logNotification(content);
|
|
21837
|
+
}, "callback")
|
|
21386
21838
|
});
|
|
21387
|
-
await this.queueService.
|
|
21388
|
-
|
|
21839
|
+
await this.queueService.registerConsumer({
|
|
21840
|
+
id: "audit.llm_requests_audit",
|
|
21841
|
+
queue: env.queue.auditRequestQueue,
|
|
21842
|
+
assertOptions: {
|
|
21843
|
+
durable: true
|
|
21844
|
+
},
|
|
21845
|
+
binding: {
|
|
21846
|
+
exchange: env.queue.requestExchange,
|
|
21847
|
+
type: "fanout",
|
|
21848
|
+
patterns: [
|
|
21849
|
+
""
|
|
21850
|
+
]
|
|
21851
|
+
},
|
|
21852
|
+
callback: /* @__PURE__ */ __name(async (_id, content) => {
|
|
21853
|
+
await this.auditService.logRequest(content);
|
|
21854
|
+
}, "callback")
|
|
21389
21855
|
});
|
|
21390
|
-
await this.queueService.
|
|
21391
|
-
|
|
21856
|
+
await this.queueService.registerConsumer({
|
|
21857
|
+
id: "audit.llm_responses_audit",
|
|
21858
|
+
queue: env.queue.auditResponseQueue,
|
|
21859
|
+
assertOptions: {
|
|
21860
|
+
durable: true
|
|
21861
|
+
},
|
|
21862
|
+
binding: {
|
|
21863
|
+
exchange: env.queue.responseExchange,
|
|
21864
|
+
type: "direct",
|
|
21865
|
+
patterns: [
|
|
21866
|
+
env.queue.auditRoutingKey
|
|
21867
|
+
]
|
|
21868
|
+
},
|
|
21869
|
+
callback: /* @__PURE__ */ __name(async (_id, content) => {
|
|
21870
|
+
await this.auditService.logResponse(content);
|
|
21871
|
+
}, "callback")
|
|
21392
21872
|
});
|
|
21393
21873
|
}
|
|
21394
21874
|
async onShutdown() {
|