@holokai/holo 1.5.2 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/holo.js CHANGED
@@ -661,7 +661,8 @@ var init_protocol = __esm({
661
661
  EMBED: "embed",
662
662
  RERANK: "rerank",
663
663
  MODELS: "models",
664
- METRICS: "metrics"
664
+ METRICS: "metrics",
665
+ IMAGE: "image"
665
666
  };
666
667
  }
667
668
  });
@@ -760,6 +761,7 @@ var init_types4 = __esm({
760
761
  ProviderEventType = {
761
762
  STREAM_EVENT: "stream_event",
762
763
  TEXT_DELTA: "text_delta",
764
+ REASONING_DELTA: "reasoning_delta",
763
765
  TOOL_CALL_DELTA: "tool_call_delta",
764
766
  DONE: "done",
765
767
  ERROR: "error"
@@ -1275,6 +1277,11 @@ data: ${JSON.stringify(data)}
1275
1277
  chunks.push(await this.chunkify(ev, async () => this.formatWire(deltaEvent)));
1276
1278
  return chunks;
1277
1279
  }
1280
+ case "reasoning_delta": {
1281
+ const reasoningEvent = this.emitReasoningDelta(0, ev.text);
1282
+ chunks.push(await this.chunkify(ev, async () => this.formatWire(reasoningEvent)));
1283
+ return chunks;
1284
+ }
1278
1285
  case "tool_call_delta": {
1279
1286
  const tcEv = ev;
1280
1287
  const holoEvent = this.emitToolCallDelta(tcEv.index, tcEv.delta);
@@ -1755,7 +1762,6 @@ var init_env2 = __esm({
1755
1762
  })(env2.redis || (env2.redis = {}));
1756
1763
  (function(queue) {
1757
1764
  queue.url = process.env.RABBITMQ_URL || "amqp://localhost:5672";
1758
- queue.reconnectAttempts = parseNumber(process.env.RABBITMQ_RECONNECT_ATTEMPTS, 5);
1759
1765
  queue.reconnectDelayMs = parseNumber(process.env.RABBITMQ_RECONNECT_DELAY_MS, 5e3);
1760
1766
  queue.prefetch = parseNumber(process.env.RABBITMQ_PREFETCH, 10);
1761
1767
  queue.maxMessageAgeMs = parseNumber(process.env.RABBITMQ_MAX_MESSAGE_AGE_MS, 18e5);
@@ -1779,14 +1785,13 @@ var init_env2 = __esm({
1779
1785
  queue.auditRoutingKey = process.env.AUDIT_ROUTING_KEY || q("audit");
1780
1786
  queue.batchQueue = process.env.RABBITMQ_BATCH_QUEUE || q("batch-jobs");
1781
1787
  queue.batchRoutingKey = process.env.BATCH_ROUTING_KEY || q("batch-jobs");
1782
- queue.queueExpiration = process.env.QUEUE_EXPIRATION || 36e5;
1788
+ queue.queueExpiration = parseNumber(process.env.QUEUE_EXPIRATION, 36e5);
1783
1789
  queue.managementQueue = process.env.PROXY_MANAGEMENT_QUEUE || q("proxy_management");
1784
1790
  queue.notificationExchange = process.env.RABBITMQ_NOTIFICATION_EXCHANGE || q("notifications_exchange");
1785
1791
  queue.notificationQueue = process.env.RABBITMQ_NOTIFICATION_QUEUE || q("notifications");
1786
1792
  queue.auditNotificationQueue = process.env.RABBITMQ_AUDIT_NOTIFICATION_QUEUE || q("notifications_audit");
1787
1793
  queue.config = {
1788
1794
  url: queue.url,
1789
- reconnectAttempts: queue.reconnectAttempts,
1790
1795
  reconnectDelayMs: queue.reconnectDelayMs
1791
1796
  };
1792
1797
  })(env2.queue || (env2.queue = {}));
@@ -1870,7 +1875,7 @@ var VERSION;
1870
1875
  var init_version2 = __esm({
1871
1876
  "src/version.ts"() {
1872
1877
  "use strict";
1873
- VERSION = "1.5.2";
1878
+ VERSION = "1.5.3";
1874
1879
  }
1875
1880
  });
1876
1881
 
@@ -2786,8 +2791,9 @@ var init_effective_model_db = __esm({
2786
2791
  this.db = db;
2787
2792
  }
2788
2793
  /**
2789
- * Resolve the effective binding for `(orgId, selector)`. `p_org` is always the token's
2790
- * `org_id` never client-supplied and the SRF filters org internally.
2794
+ * Resolve the effective binding for `(orgId, selector, profileId)`. `p_org` is always the
2795
+ * token's `org_id`, and `profileId` the token's mint-validated `profile_id` claim (or null
2796
+ * when absent) — never client-supplied — and the SRF filters org internally.
2791
2797
  *
2792
2798
  * The SRF returns the org's set; Holo keys it by the exact selector (never another
2793
2799
  * selector's default — an unknown selector yields no row → 404). The selector is
@@ -2795,18 +2801,18 @@ var init_effective_model_db = __esm({
2795
2801
  * (`widget-jpql` vs `THINK_MAX`), so any normalization would silently break one
2796
2802
  * vocabulary or collide the two.
2797
2803
  *
2798
- * **`profile_id` is NOT a selection input** it is attribution, and only becomes a
2799
- * selector in a later phase. There is exactly one row per `(org_id, alias)`, so this takes that
2800
- * single row and reads `profile_id` off it to record which profile was active when the
2801
- * serve happened. Filtering on `profile_id IS NULL` would be actively wrong: a
2802
- * chat-tier `SYSTEM_DEFAULT` row for an org with an active override profile carries
2803
- * that concrete profile id even though the override is unapplied, so the null filter
2804
- * would miss every such org's row and 404 a whole class of orgs.
2804
+ * **`profile_id` is the third selection key** (moku P3 fan-out: one row per selectable
2805
+ * profile, unique on `(org, purpose, profile_id)`). A concrete `profileId` selects its
2806
+ * row; `null` selects the profile-less row (`profile_id IS NULL`) the shape moku
2807
+ * projects for internal purposes and for orgs with no authored profiles, the only orgs
2808
+ * whose tokens omit the claim (the mint stamps the concrete default-profile id whenever
2809
+ * any profile applies `SystemModelTokenService.decideProfile`). No row for the triple
2810
+ * fails closed (404); there is no fallback to another profile's row.
2805
2811
  *
2806
2812
  * The returned row may carry a null binding — callers must gate on
2807
2813
  * {@link isServableEffectiveModel} before any provider lookup.
2808
2814
  */
2809
- async getEffectiveModel(orgId, selector) {
2815
+ async getEffectiveModel(orgId, selector, profileId) {
2810
2816
  return this.db.queryOne(`SELECT emr.provider_id,
2811
2817
  emr.model_slug,
2812
2818
  emr.source,
@@ -2816,9 +2822,11 @@ var init_effective_model_db = __esm({
2816
2822
  emr.profile_id
2817
2823
  FROM ${SRF}($1::uuid) emr
2818
2824
  WHERE emr.${SELECTOR_COLUMN} = $2
2825
+ AND (($3::uuid IS NULL AND emr.profile_id IS NULL) OR emr.profile_id = $3::uuid)
2819
2826
  LIMIT 1`, [
2820
2827
  orgId,
2821
- selector
2828
+ selector,
2829
+ profileId
2822
2830
  ]);
2823
2831
  }
2824
2832
  };
@@ -5562,6 +5570,27 @@ var init_datastore_db = __esm({
5562
5570
  async getAll() {
5563
5571
  return this.db.systemQuery("SELECT * FROM datastores ORDER BY created_at");
5564
5572
  }
5573
+ /**
5574
+ * Repoint every same-family datastore to the canonical plugin version —
5575
+ * the datastore half of latest-only consolidation (mirrors
5576
+ * ProviderDB.migrateToLatestPlugin; no junction table on this side).
5577
+ * The $2::uuid casts keep PG16/Aurora's parameter-type deduction from
5578
+ * failing when the parameter is reused as both assignment and comparand.
5579
+ */
5580
+ async migrateToLatestPlugin(family, latestPluginId) {
5581
+ const rows = await this.db.systemQuery(`UPDATE datastores d
5582
+ SET plugin_id = $2::uuid,
5583
+ updated_at = now()
5584
+ FROM plugins pl
5585
+ WHERE d.plugin_id = pl.id
5586
+ AND pl.family = $1
5587
+ AND d.plugin_id <> $2::uuid
5588
+ RETURNING d.id`, [
5589
+ family.toUpperCase(),
5590
+ latestPluginId
5591
+ ]);
5592
+ return rows.length;
5593
+ }
5565
5594
  async getEnabled() {
5566
5595
  return this.db.systemQuery(`SELECT d.*, ac.encrypted_value, ac.initialization_vector
5567
5596
  FROM datastores d
@@ -6872,16 +6901,26 @@ var init_provider_plugin_service = __esm({
6872
6901
  }
6873
6902
  protocolDB;
6874
6903
  pluginProtocols = /* @__PURE__ */ new Map();
6904
+ /**
6905
+ * Each plugin's declared `defaultProtocol`, learned at registration. Capability
6906
+ * lookups prefer it when it covers the capability — without this, a family with
6907
+ * two protocols of one capability resolves by map insertion order, which is how
6908
+ * OpenAI chat silently dispatched chatCompletions instead of responses.
6909
+ */
6910
+ pluginDefaults = /* @__PURE__ */ new Map();
6875
6911
  constructor(protocolDB) {
6876
6912
  super(), this.protocolDB = protocolDB;
6877
6913
  }
6878
- async registerProtocols(pluginId, routes) {
6914
+ async registerProtocols(pluginId, routes, defaultProtocol) {
6879
6915
  const logger2 = this.mlog(this.registerProtocols);
6880
6916
  let protocols = this.pluginProtocols.get(pluginId);
6881
6917
  if (!protocols) {
6882
6918
  protocols = /* @__PURE__ */ new Map();
6883
6919
  this.pluginProtocols.set(pluginId, protocols);
6884
6920
  }
6921
+ if (defaultProtocol) {
6922
+ this.pluginDefaults.set(pluginId, defaultProtocol);
6923
+ }
6885
6924
  for (const routeDef of routes) {
6886
6925
  const { protocol } = routeDef;
6887
6926
  const path5 = routeDef.paths.join(",");
@@ -6893,23 +6932,38 @@ var init_provider_plugin_service = __esm({
6893
6932
  }
6894
6933
  }
6895
6934
  }
6896
- async loadProtocolsFromDB(pluginId) {
6935
+ async loadProtocolsFromDB(pluginId, defaultProtocol) {
6897
6936
  const protocols = await this.protocolDB.getByPlugin(pluginId);
6898
6937
  const map = /* @__PURE__ */ new Map();
6899
6938
  for (const p of protocols) {
6900
6939
  map.set(p.name, p);
6901
6940
  }
6902
6941
  this.pluginProtocols.set(pluginId, map);
6942
+ if (defaultProtocol) {
6943
+ this.pluginDefaults.set(pluginId, defaultProtocol);
6944
+ }
6903
6945
  }
6904
6946
  removeProtocols(pluginId) {
6905
6947
  this.pluginProtocols.delete(pluginId);
6948
+ this.pluginDefaults.delete(pluginId);
6906
6949
  }
6907
6950
  async getProtocol(pluginId, protocolName) {
6908
6951
  return this.pluginProtocols.get(pluginId)?.get(protocolName);
6909
6952
  }
6953
+ /**
6954
+ * The plugin's declared default wins when it covers the capability; otherwise
6955
+ * first match. Both populators (`registerProtocols`, `loadProtocolsFromDB`)
6956
+ * learn the default from the loaded plugin instance, so startup registration
6957
+ * and runtime reload/remote-enable resolve identically.
6958
+ */
6910
6959
  getProtocolByCapability(pluginId, capability) {
6911
6960
  const protocols = this.pluginProtocols.get(pluginId);
6912
6961
  if (!protocols) return void 0;
6962
+ const preferredName = this.pluginDefaults.get(pluginId);
6963
+ if (preferredName) {
6964
+ const preferred = protocols.get(preferredName);
6965
+ if (preferred?.capability === capability) return preferred;
6966
+ }
6913
6967
  for (const protocol of protocols.values()) {
6914
6968
  if (protocol.capability === capability) return protocol;
6915
6969
  }
@@ -7155,6 +7209,13 @@ var init_plugin_registry = __esm({
7155
7209
  pluginDB;
7156
7210
  plugins = /* @__PURE__ */ new Map();
7157
7211
  loading = /* @__PURE__ */ new Map();
7212
+ /**
7213
+ * The DB row's sha256 AS OF the moment each plugin was loaded. Reconcile
7214
+ * uses this as its baseline for plugins loaded outside the lifecycle
7215
+ * service (boot-time loads) — comparing against the CURRENT row hash would
7216
+ * miss an update that landed between load and reconcile.
7217
+ */
7218
+ loadedShas = /* @__PURE__ */ new Map();
7158
7219
  constructor(installer, discovery, loader, pluginDB) {
7159
7220
  super(), this.installer = installer, this.discovery = discovery, this.loader = loader, this.pluginDB = pluginDB;
7160
7221
  }
@@ -7178,6 +7239,10 @@ var init_plugin_registry = __esm({
7178
7239
  has(pluginId) {
7179
7240
  return this.plugins.has(pluginId);
7180
7241
  }
7242
+ /** sha256 the plugin was loaded with, when it is loaded; undefined otherwise. */
7243
+ loadedSha(pluginId) {
7244
+ return this.plugins.has(pluginId) ? this.loadedShas.get(pluginId) : void 0;
7245
+ }
7181
7246
  getAsProvider(pluginId) {
7182
7247
  return this.plugins.get(pluginId);
7183
7248
  }
@@ -7204,6 +7269,7 @@ var init_plugin_registry = __esm({
7204
7269
  plugin.destroy().catch(() => {
7205
7270
  });
7206
7271
  this.plugins.delete(pluginId);
7272
+ this.loadedShas.delete(pluginId);
7207
7273
  }
7208
7274
  }
7209
7275
  async loadAllActive() {
@@ -7249,6 +7315,7 @@ var init_plugin_registry = __esm({
7249
7315
  }
7250
7316
  plugin.id = pluginId;
7251
7317
  this.plugins.set(pluginId, plugin);
7318
+ this.loadedShas.set(pluginId, row.sha256 ?? "");
7252
7319
  log.info(`Plugin ${row.name}@${row.version} loaded and ready (${pluginId})`);
7253
7320
  return plugin;
7254
7321
  }
@@ -9253,6 +9320,7 @@ var init_token_verifier_support = __esm({
9253
9320
  // src/services/auth/on-behalf-token.service.ts
9254
9321
  import "reflect-metadata";
9255
9322
  import { injectable as injectable34 } from "tsyringe";
9323
+ import { validate as isUuid } from "uuid";
9256
9324
  function _ts_decorate34(decorators, target, key, desc) {
9257
9325
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
9258
9326
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
@@ -9289,9 +9357,9 @@ var init_on_behalf_token_service = __esm({
9289
9357
  /**
9290
9358
  * verify — verify a raw bearer token against the requested alias. Resolves to the
9291
9359
  * request-scoped context (org + resolved selector from the token only) on success;
9292
- * rejects with a status-bearing, claim-free `HoloError` on any failed claim. The
9293
- * single-use `jti` is atomically consumed as the final step, so a token is spent only
9294
- * once fully accepted.
9360
+ * rejects with a status-bearing, claim-free `HoloError` on any failed claim. The `jti` is
9361
+ * consumed as the final step only when the token's `token_limit` selects single-use (the
9362
+ * default); a `token_limit=0` token consumes nothing and may be reused until `exp`.
9295
9363
  *
9296
9364
  * `clientSuppliedProfile` reports only WHETHER the client attempted to select a profile,
9297
9365
  * because the sole thing done with it is refuse it — a value would imply it could
@@ -9299,46 +9367,136 @@ var init_on_behalf_token_service = __esm({
9299
9367
  */
9300
9368
  async verify(rawToken, requestedAlias, clientSuppliedProfile2 = false) {
9301
9369
  const logger2 = this.mlog(this.verify);
9370
+ const loggedAlias = requestedAlias.length > 64 ? `${requestedAlias.slice(0, 64)}\u2026` : requestedAlias;
9302
9371
  if (!rawToken) {
9303
- throw HoloError.unauthorized("Invalid or unauthorized token.");
9372
+ this.deny("no_bearer", HoloError.unauthorized("Invalid or unauthorized token."), {
9373
+ requestedAlias: loggedAlias
9374
+ });
9375
+ }
9376
+ let issuer;
9377
+ try {
9378
+ issuer = readUnverifiedIssuer(rawToken);
9379
+ } catch (err) {
9380
+ return this.deny("undecodable_jwt", err, {
9381
+ requestedAlias: loggedAlias
9382
+ });
9304
9383
  }
9305
- const issuer = readUnverifiedIssuer(rawToken);
9306
9384
  if (!env.onBehalf.trustedIssuers.includes(issuer)) {
9307
- throw HoloError.unauthorized("Invalid or unauthorized token.");
9385
+ this.deny("untrusted_issuer", HoloError.unauthorized("Invalid or unauthorized token."), {
9386
+ issuer,
9387
+ requestedAlias: loggedAlias
9388
+ });
9308
9389
  }
9309
9390
  const jwks = this.jwksResolver.get(issuer);
9310
9391
  if (!jwks) {
9311
- throw HoloError.unauthorized("Invalid or unauthorized token.");
9392
+ this.deny("jwks_unresolved", HoloError.unauthorized("Invalid or unauthorized token."), {
9393
+ issuer,
9394
+ requestedAlias: loggedAlias
9395
+ });
9312
9396
  }
9313
- const payload = await verifyRs256(rawToken, jwks, env.onBehalf.audience, issuer);
9397
+ let payload;
9398
+ try {
9399
+ payload = await verifyRs256(rawToken, jwks, env.onBehalf.audience, issuer);
9400
+ } catch (err) {
9401
+ return this.deny("signature_or_standard_claims", err, {
9402
+ issuer,
9403
+ requestedAlias: loggedAlias
9404
+ });
9405
+ }
9406
+ const diag = this.safeDiag(issuer, payload, loggedAlias);
9314
9407
  if (payload.capability !== REQUIRED_CAPABILITY) {
9315
- throw HoloError.unauthorized("Invalid or unauthorized token.");
9408
+ this.deny("wrong_capability", HoloError.unauthorized("Invalid or unauthorized token."), diag);
9409
+ }
9410
+ try {
9411
+ assertWithinEffectiveWindow(payload, env.onBehalf.maxTtlMs);
9412
+ } catch (err) {
9413
+ return this.deny("effective_window", err, diag);
9316
9414
  }
9317
- assertWithinEffectiveWindow(payload, env.onBehalf.maxTtlMs);
9318
9415
  if (!payload.org_id || !payload.sub) {
9319
- throw HoloError.unauthorized("Invalid or unauthorized token.");
9416
+ this.deny("missing_org_or_sub", HoloError.unauthorized("Invalid or unauthorized token."), diag);
9320
9417
  }
9321
9418
  if (!payload.jti) {
9322
- throw HoloError.unauthorized("Invalid or unauthorized token.");
9419
+ this.deny("missing_jti", HoloError.unauthorized("Invalid or unauthorized token."), diag);
9420
+ }
9421
+ const tokenLimit = payload.token_limit ?? 1;
9422
+ if (tokenLimit !== 0 && tokenLimit !== 1) {
9423
+ this.deny("token_limit_unsupported", HoloError.unauthorized("Invalid or unauthorized token."), diag);
9323
9424
  }
9324
9425
  const allowed = payload.allowed_aliases;
9325
9426
  if (!Array.isArray(allowed) || !allowed.includes(requestedAlias)) {
9326
- throw HoloError.forbidden("Requested model is not permitted for this token.");
9427
+ this.deny("alias_not_permitted", HoloError.forbidden("Requested model is not permitted for this token."), diag);
9327
9428
  }
9328
9429
  if (clientSuppliedProfile2) {
9329
- throw HoloError.forbidden("Requested profile is not permitted for this token.");
9430
+ this.deny("client_supplied_profile", HoloError.forbidden("Requested profile is not permitted for this token."), diag);
9431
+ }
9432
+ const profileClaim = payload.profile_id;
9433
+ if (profileClaim !== void 0 && (typeof profileClaim !== "string" || !isUuid(profileClaim))) {
9434
+ this.deny("malformed_profile_claim", HoloError.unauthorized("Invalid or unauthorized token."), diag);
9435
+ }
9436
+ const profileId = profileClaim ?? null;
9437
+ if (tokenLimit === 1) {
9438
+ try {
9439
+ await this.consumeJti(payload.jti);
9440
+ } catch (err) {
9441
+ const replay = err instanceof HoloError && err.statusCode === 401;
9442
+ return this.deny(replay ? "jti_replay" : "replay_store_unavailable", err, diag);
9443
+ }
9330
9444
  }
9331
- await this.consumeJti(payload.jti);
9332
9445
  const selector = requestedAlias;
9333
9446
  logger2.info("On-behalf token accepted", {
9334
9447
  issuer,
9335
9448
  selector,
9336
- capability: payload.capability
9449
+ capability: payload.capability,
9450
+ has_profile: profileId !== null
9337
9451
  });
9338
9452
  return {
9339
9453
  organizationId: payload.org_id,
9340
9454
  sub: payload.sub,
9341
- selector
9455
+ selector,
9456
+ profileId
9457
+ };
9458
+ }
9459
+ /**
9460
+ * deny — log a claim-SAFE rejection reason to the server log, then throw the fixed,
9461
+ * claim-free wire error unchanged (the wire message never gains a claim). Debug aid only:
9462
+ * the `reason` names the exact gate that rejected, and `diag` carries only safe fields.
9463
+ * NEVER logs the raw token (a live bearer credential) or `sub` (which may be an email);
9464
+ * `sub` is reported as presence only. Returns `never` so callers narrow past it.
9465
+ *
9466
+ * The HTTP status rides `http_status`, not `status`: the logger reserves `status` for the log
9467
+ * level and pads it as a string, so a numeric `status` field breaks the formatter.
9468
+ */
9469
+ deny(reason, error, diag = {}) {
9470
+ this.mlog(this.verify).warn("On-behalf token rejected", {
9471
+ reason,
9472
+ http_status: error.statusCode,
9473
+ ...diag
9474
+ });
9475
+ throw error;
9476
+ }
9477
+ /**
9478
+ * safeDiag — claim-SAFE fields for a rejection log. `iss`/`capability`/`token_limit` are fixed
9479
+ * scalars, `org_id`/`jti` are IDs, `iat`/`exp` are timings — all loggable. `sub` is deliberately
9480
+ * reported only as presence (`hasSub`) because it may be an email; the raw token and the full
9481
+ * payload are never dumped. `expiresInMs` is surfaced so an expiry/window rejection is obvious.
9482
+ */
9483
+ safeDiag(issuer, payload, loggedAlias) {
9484
+ return {
9485
+ issuer,
9486
+ capability: payload.capability,
9487
+ iat: payload.iat,
9488
+ exp: payload.exp,
9489
+ expiresInMs: typeof payload.exp === "number" ? payload.exp * 1e3 - Date.now() : null,
9490
+ hasSub: !!payload.sub,
9491
+ orgId: payload.org_id,
9492
+ jti: payload.jti,
9493
+ allowedAliases: Array.isArray(payload.allowed_aliases) ? payload.allowed_aliases : null,
9494
+ requestedAlias: loggedAlias,
9495
+ // The mint's replay-posture claim as sent (null when absent) — one scalar, so it stays
9496
+ // claim-safe while showing why a token was single-use vs reusable. The full payload is
9497
+ // deliberately never dumped: it carries `sub`/`email` PII, and a blanket dump could not
9498
+ // stay claim-safe as the token contract grows.
9499
+ token_limit: payload.token_limit ?? null
9342
9500
  };
9343
9501
  }
9344
9502
  /**
@@ -10136,7 +10294,7 @@ var init_plugin_service = __esm({
10136
10294
  case PluginType.PROVIDER: {
10137
10295
  const providerPlugin = plugin;
10138
10296
  const routes = providerPlugin.getRoutes();
10139
- await this.providerPluginService.registerProtocols(pluginId, routes);
10297
+ await this.providerPluginService.registerProtocols(pluginId, routes, providerPlugin.defaultProtocol);
10140
10298
  const pricingResult = await this.registerDefaultPricing(providerPlugin, family, pluginId);
10141
10299
  log.info(`${tag}: ${routes.length} route(s), ${pricingResult}`);
10142
10300
  break;
@@ -10177,7 +10335,7 @@ var init_plugin_service = __esm({
10177
10335
  case PluginType.PROVIDER: {
10178
10336
  const providerPlugin = plugin;
10179
10337
  const routes = providerPlugin.getRoutes();
10180
- await this.providerPluginService.registerProtocols(pluginRow.id, routes);
10338
+ await this.providerPluginService.registerProtocols(pluginRow.id, routes, providerPlugin.defaultProtocol);
10181
10339
  await this.registerDefaultPricing(providerPlugin, family, pluginRow.id);
10182
10340
  break;
10183
10341
  }
@@ -10206,7 +10364,7 @@ var init_plugin_service = __esm({
10206
10364
  this.versionedPlugins.get(familyKey).set(pluginRow.version, pluginRow);
10207
10365
  switch (plugin.type) {
10208
10366
  case PluginType.PROVIDER: {
10209
- await this.providerPluginService.loadProtocolsFromDB(pluginId);
10367
+ await this.providerPluginService.loadProtocolsFromDB(pluginId, plugin.defaultProtocol);
10210
10368
  break;
10211
10369
  }
10212
10370
  case PluginType.DATASTORE: {
@@ -10433,6 +10591,7 @@ var init_provider_impl_service = __esm({
10433
10591
  init_entities2();
10434
10592
  init_provider_db();
10435
10593
  init_plugin_db();
10594
+ init_datastore_db();
10436
10595
  init_auth2();
10437
10596
  init_plugin_registry();
10438
10597
  init_plugin();
@@ -10447,30 +10606,39 @@ var init_provider_impl_service = __esm({
10447
10606
  pluginRegistry;
10448
10607
  providerDB;
10449
10608
  pluginDB;
10609
+ datastoreDB;
10450
10610
  /** One impl per provider↔plugin association, keyed `${providerId}::${pluginId}`. */
10451
10611
  providerImpls = /* @__PURE__ */ new Map();
10452
10612
  /** providerId → primary plugin id, so bare lookups stay map-only on the hot path. */
10453
10613
  primaryPluginIds = /* @__PURE__ */ new Map();
10454
- constructor(providerService, cryptoService, pluginRegistry, providerDB, pluginDB) {
10455
- super(), this.providerService = providerService, this.cryptoService = cryptoService, this.pluginRegistry = pluginRegistry, this.providerDB = providerDB, this.pluginDB = pluginDB;
10614
+ constructor(providerService, cryptoService, pluginRegistry, providerDB, pluginDB, datastoreDB) {
10615
+ super(), this.providerService = providerService, this.cryptoService = cryptoService, this.pluginRegistry = pluginRegistry, this.providerDB = providerDB, this.pluginDB = pluginDB, this.datastoreDB = datastoreDB;
10456
10616
  }
10457
10617
  implKey(providerId, pluginId) {
10458
10618
  return `${providerId}::${pluginId}`;
10459
10619
  }
10460
10620
  async consolidateToLatest(seeded) {
10461
10621
  const log = this.mlog(this.consolidateToLatest);
10462
- const canonicalByFamily = /* @__PURE__ */ new Map();
10622
+ const providersByFamily = /* @__PURE__ */ new Map();
10623
+ const datastoresByFamily = /* @__PURE__ */ new Map();
10463
10624
  for (const plugin of seeded) {
10464
- if (plugin.type !== PluginType.PROVIDER) continue;
10465
- canonicalByFamily.set(plugin.family, plugin);
10625
+ if (plugin.type === PluginType.PROVIDER) providersByFamily.set(plugin.family, plugin);
10626
+ if (plugin.type === PluginType.DATASTORE) datastoresByFamily.set(plugin.family, plugin);
10466
10627
  }
10467
- for (const [family, canonical] of canonicalByFamily) {
10628
+ for (const [family, canonical] of providersByFamily) {
10468
10629
  const repointed = await this.providerDB.migrateToLatestPlugin(family, canonical.id);
10469
10630
  const disabled = await this.pluginDB.deactivateOtherVersions(family, canonical.id);
10470
10631
  if (repointed || disabled) {
10471
10632
  log.info(`Consolidated ${family} to v${canonical.version}: repointed ${repointed} provider(s), disabled ${disabled} older version(s)`);
10472
10633
  }
10473
10634
  }
10635
+ for (const [family, canonical] of datastoresByFamily) {
10636
+ const repointed = await this.datastoreDB.migrateToLatestPlugin(family, canonical.id);
10637
+ const disabled = await this.pluginDB.deactivateOtherVersions(family, canonical.id);
10638
+ if (repointed || disabled) {
10639
+ log.info(`Consolidated ${family} to v${canonical.version}: repointed ${repointed} datastore(s), disabled ${disabled} older version(s)`);
10640
+ }
10641
+ }
10474
10642
  }
10475
10643
  /**
10476
10644
  * Fast path first: a known (provider, plugin) key returns straight from
@@ -10674,7 +10842,8 @@ var init_provider_impl_service = __esm({
10674
10842
  typeof CryptoService === "undefined" ? Object : CryptoService,
10675
10843
  typeof PluginRegistry === "undefined" ? Object : PluginRegistry,
10676
10844
  typeof ProviderDB === "undefined" ? Object : ProviderDB,
10677
- typeof PluginDB === "undefined" ? Object : PluginDB
10845
+ typeof PluginDB === "undefined" ? Object : PluginDB,
10846
+ typeof DatastoreDB === "undefined" ? Object : DatastoreDB
10678
10847
  ])
10679
10848
  ], ProviderImplService);
10680
10849
  }
@@ -10694,7 +10863,14 @@ function _ts_decorate40(decorators, target, key, desc) {
10694
10863
  function _ts_metadata38(k, v) {
10695
10864
  if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
10696
10865
  }
10697
- var QueueService;
10866
+ function errorText(error) {
10867
+ const err = error;
10868
+ if (err?.message) return err.message;
10869
+ const inner = err?.errors?.map((e) => e.code ?? e.message).filter(Boolean);
10870
+ if (inner?.length) return inner.join(", ");
10871
+ return err?.code ?? String(error);
10872
+ }
10873
+ var QueueUnavailableError, MAX_RETRY_DELAY_MS, QueueService;
10698
10874
  var init_queue_service = __esm({
10699
10875
  "src/services/queue.service.ts"() {
10700
10876
  "use strict";
@@ -10702,27 +10878,50 @@ var init_queue_service = __esm({
10702
10878
  init_src2();
10703
10879
  __name(_ts_decorate40, "_ts_decorate");
10704
10880
  __name(_ts_metadata38, "_ts_metadata");
10881
+ QueueUnavailableError = class extends Error {
10882
+ static {
10883
+ __name(this, "QueueUnavailableError");
10884
+ }
10885
+ constructor(operation) {
10886
+ super(`RabbitMQ unavailable: ${operation}`);
10887
+ this.name = "QueueUnavailableError";
10888
+ }
10889
+ };
10890
+ MAX_RETRY_DELAY_MS = 6e4;
10891
+ __name(errorText, "errorText");
10705
10892
  QueueService = class extends withClassLogger(EventEmitter2) {
10706
10893
  static {
10707
10894
  __name(this, "QueueService");
10708
10895
  }
10709
10896
  isConnected = false;
10897
+ /** Current CONSECUTIVE reconnect attempt count — reset to 0 on success, so /health shows the live outage, not a lifetime tally. */
10710
10898
  reconnectAttempts = 0;
10899
+ /** Start of the current connection outage; null while connected. */
10900
+ disconnectedSince = null;
10901
+ /**
10902
+ * Seam for the fatal-misconfiguration path (406 argument mismatch on
10903
+ * declare). Overridable so tests inject a spy instead of stubbing the
10904
+ * global `process.exit` across concurrently running suites.
10905
+ */
10906
+ fatalExit = /* @__PURE__ */ __name((code) => process.exit(code), "fatalExit");
10711
10907
  config = env.queue.config;
10712
10908
  connection = null;
10713
10909
  channel = null;
10714
10910
  recreatingChannel = false;
10715
10911
  reconnectCallbacks = [];
10716
10912
  activeConsumers = /* @__PURE__ */ new Set();
10717
- // Static consumers (worker/audit/batch/response) keyed by queue name so a
10718
- // channel rebuild can re-subscribe them. Dynamic per-user consumers
10719
- // (notifications) pass replay=false and own their reconciliation.
10913
+ // Legacy/dynamic consumers (notifications, ad-hoc test consumers) keyed by
10914
+ // queue name so a shared-channel rebuild can re-subscribe them. Static
10915
+ // server consumers use registerConsumer/consumerSpecs instead.
10720
10916
  consumerRegistry = /* @__PURE__ */ new Map();
10917
+ /** Spec-registered consumers, keyed by ConsumerSpec.id. */
10918
+ consumerSpecs = /* @__PURE__ */ new Map();
10721
10919
  closing = false;
10722
10920
  replaying = false;
10723
10921
  connecting = null;
10724
10922
  hasConnectedOnce = false;
10725
10923
  reconnectScheduled = false;
10924
+ reconnectTimer = null;
10726
10925
  counters = {
10727
10926
  channelRecreates: 0,
10728
10927
  consumersResubscribed: 0,
@@ -10734,12 +10933,17 @@ var init_queue_service = __esm({
10734
10933
  this.on("channel:lost", () => this.activeConsumers.clear());
10735
10934
  }
10736
10935
  /**
10737
- * Consumers registered on the live channel. Cleared on channel:lost, so a
10936
+ * Consumers registered on live channels: shared-channel tags (cleared on
10937
+ * channel:lost) plus spec consumers with a live per-consumer channel. A
10738
10938
  * value > 0 means this process is actively consuming — the signal the
10739
10939
  * worker/audit /health probe reports on.
10740
10940
  */
10741
10941
  get activeConsumerCount() {
10742
- return this.activeConsumers.size;
10942
+ let specCount = 0;
10943
+ for (const entry of this.consumerSpecs.values()) {
10944
+ if (entry.state.channel && entry.state.consumerTag) specCount++;
10945
+ }
10946
+ return this.activeConsumers.size + specCount;
10743
10947
  }
10744
10948
  /**
10745
10949
  * Operational counters for observability (surfaced on the /health endpoint).
@@ -10782,48 +10986,67 @@ var init_queue_service = __esm({
10782
10986
  await this.createChannel();
10783
10987
  this.connection.on("error", (err) => {
10784
10988
  logger2.error(`RabbitMQ connection error: ${err.message}`);
10785
- this.isConnected = false;
10786
- this.channel = null;
10787
- this.connection = null;
10989
+ this.markDisconnected();
10788
10990
  this.emit("channel:lost");
10789
10991
  this.reconnect();
10790
10992
  });
10791
10993
  this.connection.on("close", () => {
10792
10994
  if (this.isConnected) {
10793
10995
  logger2.warn("RabbitMQ connection closed unexpectedly");
10794
- this.isConnected = false;
10795
- this.channel = null;
10796
- this.connection = null;
10996
+ this.markDisconnected();
10797
10997
  this.emit("channel:lost");
10798
10998
  this.reconnect();
10799
10999
  }
10800
11000
  });
10801
11001
  this.isConnected = true;
11002
+ await this.activateAllConsumers();
10802
11003
  } catch (error) {
10803
- logger2.error(`Failed to connect to RabbitMQ: ${error.message}`);
11004
+ logger2.error(`Failed to connect to RabbitMQ: ${errorText(error)}`);
10804
11005
  throw error;
10805
11006
  }
10806
11007
  }
11008
+ /** Connection-level loss: every channel (shared and per-consumer) died with it. */
11009
+ markDisconnected() {
11010
+ this.isConnected = false;
11011
+ this.channel = null;
11012
+ this.connection = null;
11013
+ if (!this.disconnectedSince) this.disconnectedSince = /* @__PURE__ */ new Date();
11014
+ for (const entry of this.consumerSpecs.values()) {
11015
+ this.teardownConsumerState(entry);
11016
+ }
11017
+ }
11018
+ teardownConsumerState(entry) {
11019
+ if (entry.state.consumerTag) this.activeConsumers.delete(entry.state.consumerTag);
11020
+ entry.state.channel = null;
11021
+ entry.state.consumerTag = null;
11022
+ if (entry.state.retryTimer) {
11023
+ clearTimeout(entry.state.retryTimer);
11024
+ entry.state.retryTimer = null;
11025
+ }
11026
+ }
11027
+ /**
11028
+ * Reconnect indefinitely with capped exponential backoff. The broker being
11029
+ * down for hours must never strand a live process — the previous bounded
11030
+ * give-up (5 attempts, ~2.5 min) left servers permanently disconnected,
11031
+ * which let idle x-expires queues be deleted underneath them.
11032
+ */
10807
11033
  async reconnect() {
10808
11034
  const logger2 = this.mlog(this.reconnect);
10809
11035
  if (this.closing || this.isConnected || this.reconnectScheduled) return;
10810
- if (this.reconnectAttempts >= this.config.reconnectAttempts) {
10811
- logger2.error(`Failed to reconnect to RabbitMQ after ${this.reconnectAttempts} attempts`);
10812
- return;
10813
- }
10814
11036
  this.reconnectAttempts++;
10815
- const delay = this.config.reconnectDelayMs * Math.pow(2, this.reconnectAttempts - 1);
11037
+ const delay = Math.min(this.config.reconnectDelayMs * Math.pow(2, this.reconnectAttempts - 1), MAX_RETRY_DELAY_MS);
10816
11038
  logger2.info(`Attempting to reconnect to RabbitMQ in ${delay}ms (attempt ${this.reconnectAttempts})`);
10817
11039
  this.channel = null;
10818
11040
  this.connection = null;
10819
11041
  this.reconnectScheduled = true;
10820
- setTimeout(async () => {
11042
+ this.reconnectTimer = setTimeout(async () => {
11043
+ this.reconnectTimer = null;
10821
11044
  this.reconnectScheduled = false;
10822
11045
  if (this.closing) return;
10823
11046
  try {
10824
11047
  await this.connect();
10825
11048
  } catch (error) {
10826
- logger2.error(`Reconnect attempt ${this.reconnectAttempts} failed: ${error.message}`);
11049
+ logger2.error(`Reconnect attempt ${this.reconnectAttempts} failed: ${errorText(error)}`);
10827
11050
  void this.reconnect();
10828
11051
  }
10829
11052
  }, delay);
@@ -10840,6 +11063,18 @@ var init_queue_service = __esm({
10840
11063
  async disconnect() {
10841
11064
  const logger2 = this.mlog(this.disconnect);
10842
11065
  this.closing = true;
11066
+ if (this.reconnectTimer) {
11067
+ clearTimeout(this.reconnectTimer);
11068
+ this.reconnectTimer = null;
11069
+ this.reconnectScheduled = false;
11070
+ }
11071
+ for (const entry of this.consumerSpecs.values()) {
11072
+ const ch = entry.state.channel;
11073
+ this.teardownConsumerState(entry);
11074
+ if (ch) await ch.close().catch(() => {
11075
+ });
11076
+ }
11077
+ this.consumerSpecs.clear();
10843
11078
  this.consumerRegistry.clear();
10844
11079
  if (!this.isConnected && !this.connection) return;
10845
11080
  await this.closeChannel();
@@ -10851,6 +11086,133 @@ var init_queue_service = __esm({
10851
11086
  this.isConnected = false;
10852
11087
  logger2.info("Disconnected from RabbitMQ");
10853
11088
  }
11089
+ /**
11090
+ * Register a static consumer: stores the spec once, then activates it
11091
+ * (assert → bind → consume on the consumer's own channel). The single
11092
+ * entry point for every server's queue topology — replay after reconnect
11093
+ * reactivates the same specs, so declaration can never drift from
11094
+ * consumption.
11095
+ *
11096
+ * Registration failures propagate (startup contract: a server that cannot
11097
+ * establish its consumers must fail `onInit`, not limp). Failures on later
11098
+ * reactivation retry with bounded backoff instead.
11099
+ */
11100
+ async registerConsumer(spec) {
11101
+ const existing = this.consumerSpecs.get(spec.id);
11102
+ if (existing) {
11103
+ if (existing.spec === spec) return;
11104
+ throw new Error(`Consumer id '${spec.id}' is already registered with a different spec`);
11105
+ }
11106
+ const entry = {
11107
+ spec,
11108
+ state: {
11109
+ channel: null,
11110
+ consumerTag: null,
11111
+ activating: false,
11112
+ retryTimer: null,
11113
+ retryAttempts: 0
11114
+ }
11115
+ };
11116
+ this.consumerSpecs.set(spec.id, entry);
11117
+ if (!this.isConnected) {
11118
+ await this.connect();
11119
+ }
11120
+ await this.activateConsumer(entry, true);
11121
+ }
11122
+ /**
11123
+ * Assert → bind → consume for one spec on its own channel. Single-flight
11124
+ * per id: a channel-error retry racing a full connection replay can never
11125
+ * produce two live consumers for the same spec.
11126
+ */
11127
+ async activateConsumer(entry, throwOnFailure) {
11128
+ const logger2 = this.mlog(this.activateConsumer);
11129
+ const { spec, state } = entry;
11130
+ if (this.closing || state.activating || state.channel && state.consumerTag) return;
11131
+ if (!this.connection) {
11132
+ if (throwOnFailure) throw new QueueUnavailableError(`activate consumer '${spec.id}'`);
11133
+ this.scheduleConsumerRetry(entry);
11134
+ return;
11135
+ }
11136
+ state.activating = true;
11137
+ let channel = null;
11138
+ try {
11139
+ channel = await this.connection.createChannel();
11140
+ await channel.prefetch(env.queue.prefetch);
11141
+ channel.on("error", (err) => {
11142
+ logger2.error(`Consumer '${spec.id}' channel error: ${err.message}`);
11143
+ if (entry.state.channel === channel) {
11144
+ this.teardownConsumerState(entry);
11145
+ if (!this.closing) this.scheduleConsumerRetry(entry);
11146
+ }
11147
+ });
11148
+ try {
11149
+ await channel.assertQueue(spec.queue, spec.assertOptions ?? {
11150
+ durable: true
11151
+ });
11152
+ if (spec.binding) {
11153
+ await channel.assertExchange(spec.binding.exchange, spec.binding.type, {
11154
+ durable: true
11155
+ });
11156
+ for (const pattern of spec.binding.patterns) {
11157
+ await channel.bindQueue(spec.queue, spec.binding.exchange, pattern);
11158
+ }
11159
+ }
11160
+ } catch (declareError) {
11161
+ if (this.isPreconditionMismatch(declareError)) {
11162
+ logger2.error(`FATAL: queue declaration mismatch for consumer '${spec.id}' \u2014 queue=${spec.queue} options=${JSON.stringify(spec.assertOptions ?? {
11163
+ durable: true
11164
+ })} code=${declareError.code} reply=${declareError.message}`);
11165
+ this.fatalExit(1);
11166
+ }
11167
+ throw declareError;
11168
+ }
11169
+ const boundChannel = channel;
11170
+ const result = await boundChannel.consume(spec.queue, this.buildMessageHandler(spec.queue, spec.callback, spec.ignoreErrors ?? false, spec.dropStale ?? false, boundChannel, () => entry.state.channel), spec.consumeOptions ?? {
11171
+ noAck: false
11172
+ });
11173
+ state.channel = channel;
11174
+ state.consumerTag = result.consumerTag;
11175
+ state.retryAttempts = 0;
11176
+ if (state.retryTimer) {
11177
+ clearTimeout(state.retryTimer);
11178
+ state.retryTimer = null;
11179
+ }
11180
+ this.counters.consumersResubscribed++;
11181
+ } catch (error) {
11182
+ if (channel) await channel.close().catch(() => {
11183
+ });
11184
+ state.channel = null;
11185
+ state.consumerTag = null;
11186
+ logger2.error(`Error activating consumer '${spec.id}' on queue ${spec.queue}: ${error.message}`);
11187
+ if (throwOnFailure) throw error;
11188
+ this.scheduleConsumerRetry(entry);
11189
+ } finally {
11190
+ state.activating = false;
11191
+ }
11192
+ }
11193
+ /** Bounded per-consumer backoff — recovery must not depend on a full connection reconnect. */
11194
+ scheduleConsumerRetry(entry) {
11195
+ if (this.closing || entry.state.retryTimer) return;
11196
+ entry.state.retryAttempts++;
11197
+ const delay = Math.min(this.config.reconnectDelayMs * Math.pow(2, entry.state.retryAttempts - 1), MAX_RETRY_DELAY_MS);
11198
+ this.mlog(this.scheduleConsumerRetry).info(`Retrying consumer '${entry.spec.id}' in ${delay}ms (attempt ${entry.state.retryAttempts})`);
11199
+ entry.state.retryTimer = setTimeout(() => {
11200
+ entry.state.retryTimer = null;
11201
+ if (this.closing) return;
11202
+ void this.activateConsumer(entry, false);
11203
+ }, delay);
11204
+ }
11205
+ /** Reactivate every registered spec — the reconnect-time replay path. */
11206
+ async activateAllConsumers() {
11207
+ for (const entry of this.consumerSpecs.values()) {
11208
+ await this.activateConsumer(entry, false);
11209
+ }
11210
+ }
11211
+ /** 406 PRECONDITION_FAILED from a declare — the args-mismatch signature. Everything else is treated as transient. */
11212
+ isPreconditionMismatch(error) {
11213
+ const err = error;
11214
+ return err?.code === 406 && /PRECONDITION[-_]FAILED/i.test(String(err?.message ?? ""));
11215
+ }
10854
11216
  async consume(queueName, callback, ignoreErrors = false, options2 = {
10855
11217
  noAck: false
10856
11218
  }, replay = true, dropStale = false) {
@@ -10860,57 +11222,7 @@ var init_queue_service = __esm({
10860
11222
  }
10861
11223
  try {
10862
11224
  const boundChannel = this.channel;
10863
- const result = await boundChannel.consume(queueName, async (message) => {
10864
- if (!message) {
10865
- logger2.warn(`Received null message from queue: ${queueName}`);
10866
- return;
10867
- }
10868
- let content;
10869
- try {
10870
- content = JSON.parse(message.content.toString());
10871
- } catch {
10872
- logger2.error(`Dropping unparseable message from ${queueName} (invalid JSON, ${message.content.length} bytes)`);
10873
- this.counters.messagesDropped++;
10874
- this.safeNack(boundChannel, message, false);
10875
- return;
10876
- }
10877
- if (!content || typeof content !== "object") {
10878
- logger2.error(`Dropping non-object message from ${queueName}`);
10879
- this.counters.messagesDropped++;
10880
- this.safeNack(boundChannel, message, false);
10881
- return;
10882
- }
10883
- const { requestId } = content;
10884
- if (dropStale && this.isOverAge(content)) {
10885
- logger2.warn(`Dropping over-age message ${requestId ?? "<no-id>"} from ${queueName} (older than ${env.queue.maxMessageAgeMs}ms)`);
10886
- this.counters.messagesDropped++;
10887
- this.safeNack(boundChannel, message, false);
10888
- return;
10889
- }
10890
- try {
10891
- await callback(requestId, content, message);
10892
- this.safeAck(boundChannel, message);
10893
- } catch (error) {
10894
- const err = error;
10895
- logger2.error(`Error processing message from queue: ${queueName}: ${err?.message}`);
10896
- const pgCode = err?.code ?? err?.error?.code ?? err?.cause?.code;
10897
- const isUnrecoverable = typeof pgCode === "string" && /^(22|23)/.test(pgCode);
10898
- if (ignoreErrors || isUnrecoverable) {
10899
- if (isUnrecoverable) {
10900
- logger2.warn(`Dropping message from ${queueName} \u2014 unrecoverable data error (code ${pgCode})`);
10901
- this.counters.messagesDropped++;
10902
- }
10903
- this.safeAck(boundChannel, message);
10904
- } else if (message.fields.redelivered) {
10905
- this.counters.messagesDropped++;
10906
- this.safeNack(boundChannel, message, false);
10907
- logger2.warn("Rejected problematic message after redelivery");
10908
- } else {
10909
- this.safeNack(boundChannel, message, true);
10910
- logger2.info("Message re-queued for retry");
10911
- }
10912
- }
10913
- }, options2);
11225
+ const result = await boundChannel.consume(queueName, this.buildMessageHandler(queueName, callback, ignoreErrors, dropStale, boundChannel, () => this.channel), options2);
10914
11226
  this.activeConsumers.add(result.consumerTag);
10915
11227
  if (replay && !this.replaying) {
10916
11228
  this.consumerRegistry.set(queueName, {
@@ -10927,6 +11239,66 @@ var init_queue_service = __esm({
10927
11239
  throw error;
10928
11240
  }
10929
11241
  }
11242
+ /**
11243
+ * The shared delivery pipeline: parse guards, over-age drop, ack/nack
11244
+ * policy. `currentChannel` identifies the channel the owning consumer
11245
+ * currently considers live, so acks for deliveries from a rotated channel
11246
+ * are suppressed (a stale-tag ack is a second channel-killing 406).
11247
+ */
11248
+ buildMessageHandler(queueName, callback, ignoreErrors, dropStale, boundChannel, currentChannel) {
11249
+ const logger2 = this.mlog(this.consume);
11250
+ return async (message) => {
11251
+ if (!message) {
11252
+ logger2.warn(`Received null message from queue: ${queueName}`);
11253
+ return;
11254
+ }
11255
+ let content;
11256
+ try {
11257
+ content = JSON.parse(message.content.toString());
11258
+ } catch {
11259
+ logger2.error(`Dropping unparseable message from ${queueName} (invalid JSON, ${message.content.length} bytes)`);
11260
+ this.counters.messagesDropped++;
11261
+ this.safeNack(boundChannel, message, false, currentChannel);
11262
+ return;
11263
+ }
11264
+ if (!content || typeof content !== "object") {
11265
+ logger2.error(`Dropping non-object message from ${queueName}`);
11266
+ this.counters.messagesDropped++;
11267
+ this.safeNack(boundChannel, message, false, currentChannel);
11268
+ return;
11269
+ }
11270
+ const { requestId } = content;
11271
+ if (dropStale && this.isOverAge(content)) {
11272
+ logger2.warn(`Dropping over-age message ${requestId ?? "<no-id>"} from ${queueName} (older than ${env.queue.maxMessageAgeMs}ms)`);
11273
+ this.counters.messagesDropped++;
11274
+ this.safeNack(boundChannel, message, false, currentChannel);
11275
+ return;
11276
+ }
11277
+ try {
11278
+ await callback(requestId, content, message);
11279
+ this.safeAck(boundChannel, message, currentChannel);
11280
+ } catch (error) {
11281
+ const err = error;
11282
+ logger2.error(`Error processing message from queue: ${queueName}: ${err?.message}`);
11283
+ const pgCode = err?.code ?? err?.error?.code ?? err?.cause?.code;
11284
+ const isUnrecoverable = typeof pgCode === "string" && /^(22|23)/.test(pgCode);
11285
+ if (ignoreErrors || isUnrecoverable) {
11286
+ if (isUnrecoverable) {
11287
+ logger2.warn(`Dropping message from ${queueName} \u2014 unrecoverable data error (code ${pgCode})`);
11288
+ this.counters.messagesDropped++;
11289
+ }
11290
+ this.safeAck(boundChannel, message, currentChannel);
11291
+ } else if (message.fields.redelivered) {
11292
+ this.counters.messagesDropped++;
11293
+ this.safeNack(boundChannel, message, false, currentChannel);
11294
+ logger2.warn("Rejected problematic message after redelivery");
11295
+ } else {
11296
+ this.safeNack(boundChannel, message, true, currentChannel);
11297
+ logger2.info("Message re-queued for retry");
11298
+ }
11299
+ }
11300
+ };
11301
+ }
10930
11302
  /**
10931
11303
  * Send a message to a queue
10932
11304
  * @param {string} queue - Queue name
@@ -10935,9 +11307,7 @@ var init_queue_service = __esm({
10935
11307
  */
10936
11308
  async sendToQueue(queue, message, options2 = {}) {
10937
11309
  const logger2 = this.mlog(this.sendToQueue);
10938
- if (!this.isConnected) {
10939
- await this.connect();
10940
- }
11310
+ const channel = await this.publishChannel("sendToQueue");
10941
11311
  try {
10942
11312
  const buffer = Buffer.from(JSON.stringify(message));
10943
11313
  const defaultOptions = {
@@ -10948,7 +11318,7 @@ var init_queue_service = __esm({
10948
11318
  ...defaultOptions,
10949
11319
  ...options2
10950
11320
  };
10951
- const sent = this.channel.sendToQueue(queue, buffer, messageOptions);
11321
+ const sent = channel.sendToQueue(queue, buffer, messageOptions);
10952
11322
  if (sent) {
10953
11323
  logger2.debug(`Message sent to queue ${queue} with ID: ${message.id}`);
10954
11324
  } else {
@@ -10969,9 +11339,7 @@ var init_queue_service = __esm({
10969
11339
  */
10970
11340
  async sendToExchange(exchange, routingKey, message, options2 = {}) {
10971
11341
  const logger2 = this.mlog(this.sendToExchange);
10972
- if (!this.isConnected) {
10973
- await this.connect();
10974
- }
11342
+ const channel = await this.publishChannel(`publish to ${exchange}`);
10975
11343
  try {
10976
11344
  const buffer = Buffer.from(JSON.stringify(message));
10977
11345
  const defaultOptions = {
@@ -10982,7 +11350,7 @@ var init_queue_service = __esm({
10982
11350
  ...defaultOptions,
10983
11351
  ...options2
10984
11352
  };
10985
- const sent = this.channel.publish(exchange, routingKey, buffer, messageOptions);
11353
+ const sent = channel.publish(exchange, routingKey, buffer, messageOptions);
10986
11354
  if (!sent) {
10987
11355
  logger2.warn(`Failed to publish message to exchange ${exchange}, channel back-pressured`);
10988
11356
  }
@@ -10992,6 +11360,27 @@ var init_queue_service = __esm({
10992
11360
  throw error;
10993
11361
  }
10994
11362
  }
11363
+ /**
11364
+ * The shared channel used for publishes and legacy/dynamic operations.
11365
+ * Attempts a (single-flight) lazy connect when disconnected; if the broker
11366
+ * is still unavailable, fails fast with a typed error rather than
11367
+ * null-crashing — reconnection continues in the background.
11368
+ */
11369
+ async publishChannel(operation) {
11370
+ if (!this.isConnected) {
11371
+ try {
11372
+ await this.connect();
11373
+ } catch {
11374
+ void this.reconnect();
11375
+ throw new QueueUnavailableError(operation);
11376
+ }
11377
+ }
11378
+ const channel = this.channel;
11379
+ if (!channel) {
11380
+ throw new QueueUnavailableError(operation);
11381
+ }
11382
+ return channel;
11383
+ }
10995
11384
  /**
10996
11385
  * Stop consuming messages
10997
11386
  */
@@ -11078,8 +11467,8 @@ var init_queue_service = __esm({
11078
11467
  if (Number.isNaN(published)) return false;
11079
11468
  return Date.now() - published > maxAge;
11080
11469
  }
11081
- safeAck(channel, message) {
11082
- if (channel !== this.channel) {
11470
+ safeAck(channel, message, currentChannel) {
11471
+ if (channel !== currentChannel()) {
11083
11472
  this.counters.staleDeliveriesSuppressed++;
11084
11473
  return;
11085
11474
  }
@@ -11089,8 +11478,8 @@ var init_queue_service = __esm({
11089
11478
  this.mlog(this.safeAck).warn(`Ack on closed channel ignored: ${error.message}`);
11090
11479
  }
11091
11480
  }
11092
- safeNack(channel, message, requeue) {
11093
- if (channel !== this.channel) {
11481
+ safeNack(channel, message, requeue, currentChannel) {
11482
+ if (channel !== currentChannel()) {
11094
11483
  this.counters.staleDeliveriesSuppressed++;
11095
11484
  return;
11096
11485
  }
@@ -11101,10 +11490,10 @@ var init_queue_service = __esm({
11101
11490
  }
11102
11491
  }
11103
11492
  /**
11104
- * Re-subscribe every registered static consumer onto the current channel.
11493
+ * Re-subscribe every registered legacy consumer onto the shared channel.
11105
11494
  * Runs inside createChannel() so it covers BOTH a connection-level reconnect
11106
- * and a channel-only recreate (the 406 consumer_timeout path). Fault-isolated:
11107
- * one queue failing to re-subscribe never aborts the others.
11495
+ * and a channel-only recreate (the 406 consumer_timeout path). Spec-based
11496
+ * consumers reactivate through activateAllConsumers/their own retry instead.
11108
11497
  */
11109
11498
  async replayConsumers() {
11110
11499
  if (this.closing || this.consumerRegistry.size === 0) return;
@@ -11155,6 +11544,7 @@ var init_queue_service = __esm({
11155
11544
  });
11156
11545
  this.isConnected = true;
11157
11546
  this.reconnectAttempts = 0;
11547
+ this.disconnectedSince = null;
11158
11548
  const reconnected = this.hasConnectedOnce;
11159
11549
  this.hasConnectedOnce = true;
11160
11550
  await this.replayConsumers();
@@ -11399,7 +11789,19 @@ var init_plugin_lifecycle_service = __esm({
11399
11789
  for (const pluginRow of enabled) {
11400
11790
  const family = pluginRow.family.toUpperCase();
11401
11791
  enabledFamilies.add(family);
11402
- const current = this.loadedState.get(family);
11792
+ let current = this.loadedState.get(family);
11793
+ if (!current && this.pluginRegistry.has(pluginRow.id)) {
11794
+ current = {
11795
+ pluginId: pluginRow.id,
11796
+ family,
11797
+ version: pluginRow.version,
11798
+ sha256: this.pluginRegistry.loadedSha(pluginRow.id) ?? "",
11799
+ packageName: pluginRow.name,
11800
+ packageDir: "",
11801
+ loadedAt: Date.now()
11802
+ };
11803
+ this.loadedState.set(family, current);
11804
+ }
11403
11805
  if (current?.sha256 === pluginRow.sha256) continue;
11404
11806
  try {
11405
11807
  if (current) {
@@ -13007,7 +13409,28 @@ var init_response_service = __esm({
13007
13409
  async startLLMResponseConsumer(serverId) {
13008
13410
  this.serverId = serverId || this.serverId;
13009
13411
  const queueName = `${this.responseQueue}.${this.serverId}`;
13010
- await this.queueService.consume(queueName, async (id, content) => this.processWireChunk(id, content), true);
13412
+ await this.queueService.registerConsumer({
13413
+ id: "api.llm_responses",
13414
+ queue: queueName,
13415
+ // x-expires reclaims the per-instance queue when this API instance is
13416
+ // gone for good; registerConsumer re-asserts it on every reactivation,
13417
+ // so an expiry during an outage can no longer strand the consumer.
13418
+ assertOptions: {
13419
+ durable: true,
13420
+ arguments: {
13421
+ "x-expires": env.queue.queueExpiration
13422
+ }
13423
+ },
13424
+ binding: {
13425
+ exchange: env.queue.responseExchange,
13426
+ type: "direct",
13427
+ patterns: [
13428
+ this.serverId
13429
+ ]
13430
+ },
13431
+ callback: /* @__PURE__ */ __name(async (id, content) => this.processWireChunk(id, content), "callback"),
13432
+ ignoreErrors: true
13433
+ });
13011
13434
  }
13012
13435
  async processWireChunk(_id, wire) {
13013
13436
  const logger2 = this.mlog(this.processWireChunk);
@@ -13439,13 +13862,14 @@ var init_init_service = __esm({
13439
13862
  await this.pluginWatcherService.startWatching();
13440
13863
  }
13441
13864
  }
13865
+ // Queue topology lives with each queue's consumer (QueueService.registerConsumer
13866
+ // asserts immediately before consuming), so this only declares the exchanges the
13867
+ // API publishes to. Consumer-owned queue declarations were removed with the move
13868
+ // to registerConsumer — a second declaration site is how declare/consume drift.
13442
13869
  async setupQueues(serverId) {
13443
13870
  logger_default.debug(`setupQueues called with ${serverId}`);
13444
13871
  this.serverId = serverId || this.serverId;
13445
13872
  await this._setupGlobalExchanges();
13446
- await this._setupRequestQueues();
13447
- await this._setupResponseQueues();
13448
- await this._setupNotificationQueues();
13449
13873
  }
13450
13874
  async _setupGlobalExchanges() {
13451
13875
  await this.queueService.assertExchange(env.queue.requestExchange, "fanout");
@@ -13455,31 +13879,6 @@ var init_init_service = __esm({
13455
13879
  await this.queueService.assertExchange(env.queue.adminResponseExchange, "direct");
13456
13880
  await this.queueService.assertExchange(env.queue.directExchange, "direct");
13457
13881
  }
13458
- async _setupRequestQueues() {
13459
- await this.queueService.assertQueue(env.queue.requestQueue, {
13460
- durable: true
13461
- }, env.queue.requestExchange);
13462
- await this.queueService.assertQueue(env.queue.auditRequestQueue, {
13463
- durable: true
13464
- }, env.queue.requestExchange);
13465
- }
13466
- async _setupResponseQueues() {
13467
- const responseQueueName = `${env.queue.responseQueue}.${this.serverId}`;
13468
- await this.queueService.assertQueue(responseQueueName, {
13469
- durable: true,
13470
- arguments: {
13471
- "x-expires": env.queue.queueExpiration
13472
- }
13473
- }, env.queue.responseExchange, this.serverId);
13474
- await this.queueService.assertQueue(env.queue.auditResponseQueue, {
13475
- durable: true
13476
- }, env.queue.responseExchange, env.queue.auditRoutingKey);
13477
- }
13478
- async _setupNotificationQueues() {
13479
- await this.queueService.assertQueue(env.queue.auditNotificationQueue, {
13480
- durable: true
13481
- }, env.queue.notificationExchange, "#");
13482
- }
13483
13882
  };
13484
13883
  InitService = _ts_decorate49([
13485
13884
  injectable49(),
@@ -14310,10 +14709,11 @@ var init_system_model_binding_cache = __esm({
14310
14709
  * legitimately mint, including the platform-org sentinel, which is why it asserts UUID
14311
14710
  * *shape* rather than RFC-4122 conformance (see {@link isKeySafeUuid}).
14312
14711
  */
14313
- key(orgId, selector) {
14712
+ key(orgId, selector, profileId) {
14314
14713
  this.assertOrgId(orgId);
14315
14714
  this.assertSelector(selector);
14316
- return `${this.prefix}${orgId}:selector:${selector}`;
14715
+ this.assertProfileId(profileId);
14716
+ return `${this.prefix}${orgId}:selector:${selector}:profile:${profileId ?? "none"}`;
14317
14717
  }
14318
14718
  assertOrgId(orgId) {
14319
14719
  if (!isKeySafeUuid(orgId)) {
@@ -14325,16 +14725,26 @@ var init_system_model_binding_cache = __esm({
14325
14725
  throw HoloError.badRequest("Invalid system-model selector.");
14326
14726
  }
14327
14727
  }
14328
- async get(orgId, selector) {
14329
- return this.redis.get(this.key(orgId, selector));
14728
+ /**
14729
+ * The profile segment admits exactly two shapes: a UUID (the token's mint-validated
14730
+ * claim, already shape-checked by the verifier — re-asserted here so no other caller
14731
+ * can widen a glob through this segment) or `null`, rendered as the literal `none`.
14732
+ */
14733
+ assertProfileId(profileId) {
14734
+ if (profileId !== null && !isKeySafeUuid(profileId)) {
14735
+ throw HoloError.badRequest("Invalid system-model profile.");
14736
+ }
14737
+ }
14738
+ async get(orgId, selector, profileId) {
14739
+ return this.redis.get(this.key(orgId, selector, profileId));
14330
14740
  }
14331
14741
  /**
14332
14742
  * Admit a binding to the cache. The parameter type is the admission predicate: only a
14333
14743
  * row already proven servable can be passed, so admission is exactly servability
14334
14744
  * rather than an approximation of it re-derived here.
14335
14745
  */
14336
- async set(orgId, selector, row) {
14337
- await this.redis.set(this.key(orgId, selector), row, Math.ceil(env.onBehalf.maxTtlMs / 1e3));
14746
+ async set(orgId, selector, profileId, row) {
14747
+ await this.redis.set(this.key(orgId, selector, profileId), row, Math.ceil(env.onBehalf.maxTtlMs / 1e3));
14338
14748
  }
14339
14749
  /**
14340
14750
  * Evict every cached binding for an org (`binding:org:{org}:*`). Returns the count removed.
@@ -14360,7 +14770,7 @@ var init_system_model_binding_cache = __esm({
14360
14770
  */
14361
14771
  async evictSelectorAcrossOrgs(selector) {
14362
14772
  this.assertSelector(selector);
14363
- return this.scanAndDelete(`${this.prefix}*:selector:${selector}`);
14773
+ return this.scanAndDelete(`${this.prefix}*:selector:${selector}:profile:*`);
14364
14774
  }
14365
14775
  /** Delete every key matching `pattern`. Returns the count removed. */
14366
14776
  async scanAndDelete(pattern) {
@@ -14453,12 +14863,12 @@ var init_holo_request_service = __esm({
14453
14863
  async resolveSystemModel(auth, request) {
14454
14864
  const logger2 = this.mlog(this.resolveSystemModel);
14455
14865
  const orgId = auth.organizationId;
14456
- const { selector } = auth.onBehalf;
14866
+ const { selector, profileId } = auth.onBehalf;
14457
14867
  if (!selector) {
14458
14868
  throw HoloError.badRequest("A system-model alias is required.");
14459
14869
  }
14460
- const cached = await this.bindingCache.get(orgId, selector);
14461
- const row = cached ?? await this.effectiveModelDB.getEffectiveModel(orgId, selector);
14870
+ const cached = await this.bindingCache.get(orgId, selector, profileId);
14871
+ const row = cached ?? await this.effectiveModelDB.getEffectiveModel(orgId, selector, profileId);
14462
14872
  if (!row) {
14463
14873
  this.logSystemModelDeny(logger2, "unknown_selector", auth, selector);
14464
14874
  throw HoloError.notFound("Model");
@@ -14468,7 +14878,7 @@ var init_holo_request_service = __esm({
14468
14878
  throw HoloError.conflict("The configured model binding is currently unservable.", HoloErrorCode.UNSERVABLE_BINDING);
14469
14879
  }
14470
14880
  if (!cached) {
14471
- await this.bindingCache.set(orgId, selector, row);
14881
+ await this.bindingCache.set(orgId, selector, profileId, row);
14472
14882
  }
14473
14883
  const provider = await this.providerDB.getById(row.provider_id);
14474
14884
  if (!provider) {
@@ -14977,7 +15387,8 @@ function makeOnBehalfPreHandler(verifier) {
14977
15387
  applications: [],
14978
15388
  clientIdentifier: verified.sub,
14979
15389
  onBehalf: {
14980
- selector: verified.selector
15390
+ selector: verified.selector,
15391
+ profileId: verified.profileId
14981
15392
  }
14982
15393
  };
14983
15394
  req.auth = auth;
@@ -16918,16 +17329,6 @@ function withAdmin(Base) {
16918
17329
  logger_default.debug(`${this.id} is Admin Aware`);
16919
17330
  const commandQueue = `${env.queue.adminCommandQueue}_${this.id}`;
16920
17331
  const exchange = env.queue.adminExchange;
16921
- await this.queueService.assertExchange(exchange, "topic");
16922
- await this.queueService.assertQueue(commandQueue, {
16923
- exclusive: false,
16924
- durable: true,
16925
- autoDelete: true
16926
- }, exchange, "model.#");
16927
- await this.queueService.bindQueue(commandQueue, exchange, "worker.#");
16928
- await this.queueService.bindQueue(commandQueue, exchange, "plugin.#");
16929
- await this.queueService.bindQueue(commandQueue, exchange, "provider.#");
16930
- await this.queueService.bindQueue(commandQueue, exchange, "datastore.#");
16931
17332
  this.pluginLifecycleService.setServerId(this.id);
16932
17333
  this.adminHandlers.set("plugin.enabled", async (_serverId, payload) => {
16933
17334
  await this.pluginLifecycleService.handleRemoteEnable(payload);
@@ -16968,29 +17369,49 @@ function withAdmin(Base) {
16968
17369
  if (env.api.pluginReconcileIntervalMs > 0) {
16969
17370
  this.pluginLifecycleService.startPeriodicReconciliation(env.api.pluginReconcileIntervalMs);
16970
17371
  }
16971
- await this.queueService.consume(commandQueue, async (messageId, content, _message) => {
16972
- const { action, serverId, timestamp, ...payload } = content;
16973
- logger_default.info(`Worker (${this.id}) received admin command: ${action}, messageId: ${messageId}`);
16974
- const handler = this.adminHandlers.get(action);
16975
- let response = {};
16976
- if (!handler) {
16977
- logger_default.warn(`No handler registered for admin command: ${action}`);
16978
- response = {
16979
- success: false,
16980
- message: `No handler registered for admin command: ${action}`
16981
- };
16982
- } else {
16983
- try {
16984
- response = await handler(this.id, payload);
16985
- } catch (error) {
16986
- logger_default.error(`Error handling admin command: ${action} - ${error.message}`);
17372
+ await this.queueService.registerConsumer({
17373
+ id: `admin.commands.${this.id}`,
17374
+ queue: commandQueue,
17375
+ assertOptions: {
17376
+ exclusive: false,
17377
+ durable: true,
17378
+ autoDelete: true
17379
+ },
17380
+ binding: {
17381
+ exchange,
17382
+ type: "topic",
17383
+ patterns: [
17384
+ "model.#",
17385
+ "worker.#",
17386
+ "plugin.#",
17387
+ "provider.#",
17388
+ "datastore.#"
17389
+ ]
17390
+ },
17391
+ callback: /* @__PURE__ */ __name(async (messageId, content, _message) => {
17392
+ const { action, serverId, timestamp, ...payload } = content;
17393
+ logger_default.info(`Worker (${this.id}) received admin command: ${action}, messageId: ${messageId}`);
17394
+ const handler = this.adminHandlers.get(action);
17395
+ let response = {};
17396
+ if (!handler) {
17397
+ logger_default.warn(`No handler registered for admin command: ${action}`);
16987
17398
  response = {
16988
17399
  success: false,
16989
- message: error.message
17400
+ message: `No handler registered for admin command: ${action}`
16990
17401
  };
17402
+ } else {
17403
+ try {
17404
+ response = await handler(this.id, payload);
17405
+ } catch (error) {
17406
+ logger_default.error(`Error handling admin command: ${action} - ${error.message}`);
17407
+ response = {
17408
+ success: false,
17409
+ message: error.message
17410
+ };
17411
+ }
16991
17412
  }
16992
- }
16993
- await this.adminService.sendAdminResponse(this.id, serverId, messageId, response);
17413
+ await this.adminService.sendAdminResponse(this.id, serverId, messageId, response);
17414
+ }, "callback")
16994
17415
  });
16995
17416
  }
16996
17417
  async onShutdown() {
@@ -17124,6 +17545,10 @@ function withHealth(Base) {
17124
17545
  status: healthy ? "ok" : "unhealthy",
17125
17546
  connected,
17126
17547
  consumers,
17548
+ // Current consecutive attempt count (0 while connected) +
17549
+ // outage start — makes a stuck-retrying instance visible.
17550
+ reconnectAttempts: queueService?.reconnectAttempts ?? 0,
17551
+ disconnectedSince: queueService?.disconnectedSince?.toISOString() ?? null,
17127
17552
  metrics: queueService?.metrics,
17128
17553
  uptime: process.uptime(),
17129
17554
  timestamp: (/* @__PURE__ */ new Date()).toISOString()
@@ -21133,116 +21558,129 @@ var init_worker_server = __esm({
21133
21558
  this.adminHandlers.set("worker.restart", this.adminService.restartWorker);
21134
21559
  const requestQueue = env.queue.requestQueue;
21135
21560
  const logger2 = this.mlog("workerConsume");
21136
- await this.queueService.consume(requestQueue, async (requestId, workerRequest, message) => {
21137
- this.stats.totalRequests++;
21138
- const reqLogger = workerRequest.capability ? logger2.child(capabilityToMetadata(workerRequest.capability)) : logger2;
21139
- const redelivered = message?.fields?.redelivered ?? false;
21140
- reqLogger.info(`Worker ${this.id} handling request: ${requestId} provider: ${workerRequest.provider.name} from server ${workerRequest.sourceId} and queue ${requestQueue}${redelivered ? " (redelivered)" : ""}...`);
21141
- const ai = await this.providerPluginService.getProviderImplById(workerRequest.provider.id, workerRequest.protocol?.plugin_id);
21142
- const plugin = ai.plugin;
21143
- logger2.info(`resolved ai provider: ${ai.name}`);
21144
- const { sourceId, guardResult } = workerRequest;
21145
- const envelope = await ai.auditor.createWorkerResponseEnvelope(workerRequest, this.id);
21146
- try {
21147
- if (guardResult && !guardResult.passed) {
21148
- logger2.info("Guards failed. Sending back guard errors.");
21149
- const wire = await plugin.createWireAdapter({
21150
- requestId,
21151
- isStreaming: false,
21152
- protocol: ai.plugin.defaultProtocol
21153
- });
21154
- const errorMessage = guardResult.errors?.join("\n\n") || "Policy violation";
21155
- const evt = {
21156
- type: "error",
21157
- requestId,
21158
- seq: 0,
21159
- ts: Date.now(),
21160
- status: 400,
21161
- metrics: {
21162
- inputTokens: 0,
21163
- outputTokens: 0,
21164
- timeToFirstToken: 0,
21165
- totalProcessingTime: 0,
21166
- totalTokens: 0
21167
- },
21168
- text: errorMessage,
21169
- error: ai.responseFactory.createError(errorMessage)
21170
- };
21171
- for (const wireChunk of await wire.fromProviderEvent(evt)) {
21172
- logger2.debug(`Guard failed response: ${JSON.stringify(wireChunk)}`);
21173
- await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
21174
- }
21175
- await this.responseService.sendToAudit({
21176
- ...workerRequest,
21177
- providerEvent: evt,
21178
- workerId: this.id
21179
- });
21180
- await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, evt));
21181
- } else {
21182
- const apiCredentialId = workerRequest.provider.api_credential_id;
21183
- if (apiCredentialId) {
21184
- this.apiCredentialDB.updateLastUsed(apiCredentialId).catch((err) => reqLogger.error(`Failed to update api_credential last_used_at: ${err.message}`));
21185
- }
21186
- const wirePlugin = this.resolveWirePlugin(workerRequest, plugin);
21187
- if (wirePlugin) {
21188
- logger2.info(`Translating inbound: ${wirePlugin.family}/${workerRequest.wireProtocolName} \u2192 holo \u2192 ${plugin.family}/${workerRequest.protocol.name}`);
21189
- await this.applyInboundTranslation(workerRequest, wirePlugin);
21190
- }
21191
- const wire = await this.createWireAdapter(requestId, workerRequest, wirePlugin, plugin);
21192
- const rawEvents = await ai.processWorkerRequest(workerRequest, {
21193
- requestTimeoutMs: env.worker.requestTimeoutMs,
21194
- idleTimeoutMs: env.worker.streamIdleTimeoutMs
21195
- });
21196
- const events = wirePlugin ? translateProviderEvents(rawEvents, plugin.translator, wirePlugin.translator) : rawEvents;
21197
- await processProviderEvents(events, wire, workerRequest, envelope, {
21198
- publishChunk: this.responseService.sendResponseChunk.bind(this),
21199
- publishFinal: /* @__PURE__ */ __name(async (final) => {
21200
- await this.responseService.sendToAudit(final);
21201
- await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, final.providerEvent));
21202
- }, "publishFinal")
21203
- });
21204
- }
21205
- } catch (error) {
21206
- logger2.error(`Error handling request (${requestId}): ${error.message}`);
21561
+ await this.queueService.registerConsumer({
21562
+ id: "worker.llm_requests",
21563
+ queue: requestQueue,
21564
+ assertOptions: {
21565
+ durable: true
21566
+ },
21567
+ binding: {
21568
+ exchange: env.queue.requestExchange,
21569
+ type: "fanout",
21570
+ patterns: [
21571
+ ""
21572
+ ]
21573
+ },
21574
+ callback: /* @__PURE__ */ __name(async (requestId, workerRequest, message) => {
21575
+ this.stats.totalRequests++;
21576
+ const reqLogger = workerRequest.capability ? logger2.child(capabilityToMetadata(workerRequest.capability)) : logger2;
21577
+ const redelivered = message?.fields?.redelivered ?? false;
21578
+ reqLogger.info(`Worker ${this.id} handling request: ${requestId} provider: ${workerRequest.provider.name} from server ${workerRequest.sourceId} and queue ${requestQueue}${redelivered ? " (redelivered)" : ""}...`);
21579
+ const ai = await this.providerPluginService.getProviderImplById(workerRequest.provider.id, workerRequest.protocol?.plugin_id);
21580
+ const plugin = ai.plugin;
21581
+ logger2.info(`resolved ai provider: ${ai.name}`);
21582
+ const { sourceId, guardResult } = workerRequest;
21583
+ const envelope = await ai.auditor.createWorkerResponseEnvelope(workerRequest, this.id);
21207
21584
  try {
21208
- const errorMessage = error.message;
21209
- const errorEvent = {
21210
- type: "error",
21211
- requestId,
21212
- seq: 0,
21213
- ts: Date.now(),
21214
- status: 500,
21215
- error: ai.responseFactory.createError(errorMessage),
21216
- text: errorMessage,
21217
- metrics: {
21218
- inputTokens: 0,
21219
- outputTokens: 0,
21220
- timeToFirstToken: 0,
21221
- totalProcessingTime: 0,
21222
- totalTokens: 0
21585
+ if (guardResult && !guardResult.passed) {
21586
+ logger2.info("Guards failed. Sending back guard errors.");
21587
+ const wire = await plugin.createWireAdapter({
21588
+ requestId,
21589
+ isStreaming: false,
21590
+ protocol: ai.plugin.defaultProtocol
21591
+ });
21592
+ const errorMessage = guardResult.errors?.join("\n\n") || "Policy violation";
21593
+ const evt = {
21594
+ type: "error",
21595
+ requestId,
21596
+ seq: 0,
21597
+ ts: Date.now(),
21598
+ status: 400,
21599
+ metrics: {
21600
+ inputTokens: 0,
21601
+ outputTokens: 0,
21602
+ timeToFirstToken: 0,
21603
+ totalProcessingTime: 0,
21604
+ totalTokens: 0
21605
+ },
21606
+ text: errorMessage,
21607
+ error: ai.responseFactory.createError(errorMessage)
21608
+ };
21609
+ for (const wireChunk of await wire.fromProviderEvent(evt)) {
21610
+ logger2.debug(`Guard failed response: ${JSON.stringify(wireChunk)}`);
21611
+ await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
21223
21612
  }
21224
- };
21225
- const wire = workerRequest.isHoloNative ? new HoloWireAdapter(requestId, false, workerRequest.payload?.model) : await plugin.createWireAdapter({
21226
- requestId,
21227
- isStreaming: false,
21228
- protocol: workerRequest.protocol?.name ?? ai.plugin.defaultProtocol
21229
- });
21230
- for (const wireChunk of await wire.fromProviderEvent(errorEvent)) {
21231
- await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
21613
+ await this.responseService.sendToAudit({
21614
+ ...workerRequest,
21615
+ providerEvent: evt,
21616
+ workerId: this.id
21617
+ });
21618
+ await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, evt));
21619
+ } else {
21620
+ const apiCredentialId = workerRequest.provider.api_credential_id;
21621
+ if (apiCredentialId) {
21622
+ this.apiCredentialDB.updateLastUsed(apiCredentialId).catch((err) => reqLogger.error(`Failed to update api_credential last_used_at: ${err.message}`));
21623
+ }
21624
+ const wirePlugin = this.resolveWirePlugin(workerRequest, plugin);
21625
+ if (wirePlugin) {
21626
+ logger2.info(`Translating inbound: ${wirePlugin.family}/${workerRequest.wireProtocolName} \u2192 holo \u2192 ${plugin.family}/${workerRequest.protocol.name}`);
21627
+ await this.applyInboundTranslation(workerRequest, wirePlugin);
21628
+ }
21629
+ const wire = await this.createWireAdapter(requestId, workerRequest, wirePlugin, plugin);
21630
+ const rawEvents = await ai.processWorkerRequest(workerRequest, {
21631
+ requestTimeoutMs: env.worker.requestTimeoutMs,
21632
+ idleTimeoutMs: env.worker.streamIdleTimeoutMs
21633
+ });
21634
+ const events = wirePlugin ? translateProviderEvents(rawEvents, plugin.translator, wirePlugin.translator) : rawEvents;
21635
+ await processProviderEvents(events, wire, workerRequest, envelope, {
21636
+ publishChunk: this.responseService.sendResponseChunk.bind(this),
21637
+ publishFinal: /* @__PURE__ */ __name(async (final) => {
21638
+ await this.responseService.sendToAudit(final);
21639
+ await this.notificationService.publish(NotificationEventFactory.fromProviderEvent(envelope, final.providerEvent));
21640
+ }, "publishFinal")
21641
+ });
21232
21642
  }
21233
- await this.responseService.sendToAudit({
21234
- ...workerRequest,
21235
- providerEvent: errorEvent,
21236
- workerId: this.id
21237
- });
21238
- } catch (auditError) {
21239
- logger2.error(`Failed to send error response for (${requestId}): ${auditError.message}`);
21643
+ } catch (error) {
21644
+ logger2.error(`Error handling request (${requestId}): ${error.message}`);
21645
+ try {
21646
+ const errorMessage = error.message;
21647
+ const errorEvent = {
21648
+ type: "error",
21649
+ requestId,
21650
+ seq: 0,
21651
+ ts: Date.now(),
21652
+ status: 500,
21653
+ error: ai.responseFactory.createError(errorMessage),
21654
+ text: errorMessage,
21655
+ metrics: {
21656
+ inputTokens: 0,
21657
+ outputTokens: 0,
21658
+ timeToFirstToken: 0,
21659
+ totalProcessingTime: 0,
21660
+ totalTokens: 0
21661
+ }
21662
+ };
21663
+ const wire = workerRequest.isHoloNative ? new HoloWireAdapter(requestId, false, workerRequest.payload?.model) : await plugin.createWireAdapter({
21664
+ requestId,
21665
+ isStreaming: false,
21666
+ protocol: workerRequest.protocol?.name ?? ai.plugin.defaultProtocol
21667
+ });
21668
+ for (const wireChunk of await wire.fromProviderEvent(errorEvent)) {
21669
+ await this.responseService.sendResponseChunk(sourceId, requestId, wireChunk);
21670
+ }
21671
+ await this.responseService.sendToAudit({
21672
+ ...workerRequest,
21673
+ providerEvent: errorEvent,
21674
+ workerId: this.id
21675
+ });
21676
+ } catch (auditError) {
21677
+ logger2.error(`Failed to send error response for (${requestId}): ${auditError.message}`);
21678
+ }
21679
+ await this.onError(error);
21240
21680
  }
21241
- await this.onError(error);
21242
- }
21243
- }, false, {
21244
- noAck: false
21245
- }, true, true);
21681
+ }, "callback"),
21682
+ dropStale: true
21683
+ });
21246
21684
  }
21247
21685
  async onShutdown() {
21248
21686
  await super.onShutdown();
@@ -21381,14 +21819,56 @@ var init_audit_server = __esm({
21381
21819
  this.pluginLifecycleService.setPluginsDir(env.audit.pluginsDir);
21382
21820
  await this.pluginRegistry.loadAllActive();
21383
21821
  await this.providerImplService.warmAll();
21384
- await this.queueService.consume(env.queue.auditNotificationQueue, async (_id, content) => {
21385
- await this.auditService.logNotification(content);
21822
+ await this.queueService.registerConsumer({
21823
+ id: "audit.notifications_audit",
21824
+ queue: env.queue.auditNotificationQueue,
21825
+ assertOptions: {
21826
+ durable: true
21827
+ },
21828
+ binding: {
21829
+ exchange: env.queue.notificationExchange,
21830
+ type: "topic",
21831
+ patterns: [
21832
+ "#"
21833
+ ]
21834
+ },
21835
+ callback: /* @__PURE__ */ __name(async (_id, content) => {
21836
+ await this.auditService.logNotification(content);
21837
+ }, "callback")
21386
21838
  });
21387
- await this.queueService.consume(env.queue.auditRequestQueue, async (_id, content) => {
21388
- await this.auditService.logRequest(content);
21839
+ await this.queueService.registerConsumer({
21840
+ id: "audit.llm_requests_audit",
21841
+ queue: env.queue.auditRequestQueue,
21842
+ assertOptions: {
21843
+ durable: true
21844
+ },
21845
+ binding: {
21846
+ exchange: env.queue.requestExchange,
21847
+ type: "fanout",
21848
+ patterns: [
21849
+ ""
21850
+ ]
21851
+ },
21852
+ callback: /* @__PURE__ */ __name(async (_id, content) => {
21853
+ await this.auditService.logRequest(content);
21854
+ }, "callback")
21389
21855
  });
21390
- await this.queueService.consume(env.queue.auditResponseQueue, async (_id, content) => {
21391
- await this.auditService.logResponse(content);
21856
+ await this.queueService.registerConsumer({
21857
+ id: "audit.llm_responses_audit",
21858
+ queue: env.queue.auditResponseQueue,
21859
+ assertOptions: {
21860
+ durable: true
21861
+ },
21862
+ binding: {
21863
+ exchange: env.queue.responseExchange,
21864
+ type: "direct",
21865
+ patterns: [
21866
+ env.queue.auditRoutingKey
21867
+ ]
21868
+ },
21869
+ callback: /* @__PURE__ */ __name(async (_id, content) => {
21870
+ await this.auditService.logResponse(content);
21871
+ }, "callback")
21392
21872
  });
21393
21873
  }
21394
21874
  async onShutdown() {