@cotal-ai/core 0.161.1 → 0.162.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/endpoint.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { requestSafeReconnect } from "./request-safe-reconnect.js";
1
2
  import { EventEmitter } from "node:events";
2
3
  import { drainAndClose } from "./connection-cleanup.js";
3
4
  import { randomUUID } from "node:crypto";
@@ -543,6 +544,8 @@ export class CotalEndpoint extends EventEmitter {
543
544
  /** Full per-endpoint-name {@link resolveService} cache for {@link describeService}. Narrow command
544
545
  * handles never enter it, so discovery cannot inherit a truncated invocation surface. */
545
546
  resolvedServices = new Map();
547
+ /** Requests still resolving, preparing observation or awaiting their command's answer. */
548
+ serviceInvocations = new WeakMap();
546
549
  /** Narrow invocation handles, keyed by endpoint and the sorted command set they verify. Kept
547
550
  * separate from full discovery handles so alternating commands neither overwrite each other nor
548
551
  * poison {@link describeService}. */
@@ -1177,6 +1180,25 @@ export class CotalEndpoint extends EventEmitter {
1177
1180
  this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
1178
1181
  }
1179
1182
  }
1183
+ /** Fetch, broker-prove and adopt the source credential, then refresh the same connection
1184
+ * once outgoing mux and named-reply requests are quiet. A busy timeout retains the connection
1185
+ * and adopted credential. Call again to retry. This does not tear down endpoint bindings. */
1186
+ async refreshCredsRequestSafe() {
1187
+ if (!this.credsSource)
1188
+ throw new Error("credential refresh requires a creds source");
1189
+ const nc = this.nc;
1190
+ if (!nc || this.stopped)
1191
+ throw new Error("credential refresh requires a running endpoint");
1192
+ const deadline = Date.now() + CotalEndpoint.RELOAD_DEADLINE_MS;
1193
+ await this.runCredsTxn(async () => {
1194
+ await this.adoptFreshCreds({ deadline });
1195
+ if (this.nc !== nc || this.stopped)
1196
+ throw new Error("endpoint connection changed during credential refresh");
1197
+ await requestSafeReconnect(nc, () => this.requestsAwaitingReply.get(nc)?.size ?? 0);
1198
+ if (this.nc === nc && !this.stopped)
1199
+ this.armAuthExpiryReconnectFence(nc);
1200
+ });
1201
+ }
1180
1202
  /** EXPLICIT credential reload — the auditable adoption step of D5 class-2 standing renewal (served
1181
1203
  * to the renewal owner via the delivery-admin rail). The PROOF-OF-RECORD is a DISPOSABLE PREFLIGHT
1182
1204
  * connection that presents exactly the candidate: it succeeds only when the BROKER accepts this
@@ -2514,6 +2536,39 @@ export class CotalEndpoint extends EventEmitter {
2514
2536
  this.resolvedServices.delete(endpoint);
2515
2537
  this.resolvedCommandServices.delete(endpoint);
2516
2538
  }
2539
+ /** Hold connection recovery through the whole pre-acceptance phase, including the progress
2540
+ * subscription flush. Release on setup failure or the command's answer, never on goal terminal. */
2541
+ beginServiceInvocation(nc) {
2542
+ let pending = this.serviceInvocations.get(nc);
2543
+ if (!pending) {
2544
+ pending = new Set();
2545
+ this.serviceInvocations.set(nc, pending);
2546
+ }
2547
+ let settle;
2548
+ const request = new Promise((resolve) => { settle = resolve; });
2549
+ pending.add(request);
2550
+ return () => { pending.delete(request); settle(); };
2551
+ }
2552
+ async waitServiceRecovery(recovery, deadline, signal, timeout) {
2553
+ let onAbort;
2554
+ const cancelled = new Promise((_, reject) => {
2555
+ onAbort = () => reject(signal?.reason);
2556
+ if (signal?.aborted)
2557
+ onAbort();
2558
+ else
2559
+ signal?.addEventListener("abort", onAbort, { once: true });
2560
+ });
2561
+ try {
2562
+ await withDeadline(Promise.race([recovery, cancelled]), deadline - Date.now(), timeout);
2563
+ }
2564
+ finally {
2565
+ if (onAbort)
2566
+ signal?.removeEventListener("abort", onAbort);
2567
+ }
2568
+ signal?.throwIfAborted();
2569
+ if (deadline <= Date.now())
2570
+ throw timeout();
2571
+ }
2517
2572
  /** The resolved, digest-verified command surface of `endpoint` as this connection's caller sees
2518
2573
  * it. Unpinned calls cache the complete discovery surface, and `refresh` drops both the full
2519
2574
  * handle and any command-specific invocation handles. `instanceId` selects one instance without
@@ -2592,15 +2647,63 @@ export class CotalEndpoint extends EventEmitter {
2592
2647
  * the bind is dropped but the call is re-issued only for a command on the
2593
2648
  * {@link isRepeatSafeCommand} allowlist; anything else surfaces, since a second attempt could
2594
2649
  * duplicate its effect.
2650
+ * A broker permission refusal marked `not-executed` rebuilds the connection and retries once
2651
+ * within the original deadline. A second refusal stands. Responder errors and calls that may
2652
+ * have run never trigger this repair.
2595
2653
  * Errors from the responder come back structurally on the attributed reply
2596
2654
  * (`reply.ok === false`); transport/validation refusals throw {@link EpEnvelopeError}. */
2597
2655
  async invokeService(endpoint, command, args, opts = {}) {
2656
+ const deadline = Date.now() + (opts.deadlineMs ?? 10_000);
2657
+ const rebuilding = this.rebuildPromise;
2658
+ if (rebuilding) {
2659
+ await this.waitServiceRecovery(rebuilding, deadline, opts.signal, () => new EpEnvelopeError("deadline-exceeded", "the invocation deadline elapsed waiting for connection recovery", undefined, "not-executed"));
2660
+ opts = { ...opts, deadlineMs: deadline - Date.now() };
2661
+ }
2662
+ const nc = this.nc;
2663
+ if (!nc)
2664
+ throw new Error(this.notLiveMsg());
2665
+ try {
2666
+ return await this.invokeServiceOnce(endpoint, command, args, opts, rebuilding ? deadline : undefined);
2667
+ }
2668
+ catch (e) {
2669
+ if (!(e instanceof EpEnvelopeError) || e.code !== "permission-denied" || e.outcome !== "not-executed"
2670
+ || deadline <= Date.now())
2671
+ throw e;
2672
+ opts.signal?.throwIfAborted();
2673
+ const recovery = (async () => {
2674
+ // Re-read the live set: calls admitted while earlier ones settle must keep their replies.
2675
+ // The empty check and rebuild admission have no await between them. Later calls wait on
2676
+ // rebuildPromise instead of publishing on the connection being replaced.
2677
+ for (;;) {
2678
+ opts.signal?.throwIfAborted();
2679
+ if (deadline <= Date.now())
2680
+ throw e;
2681
+ const pending = this.serviceInvocations.get(nc);
2682
+ if (!pending?.size)
2683
+ break;
2684
+ await Promise.allSettled([...pending]);
2685
+ }
2686
+ if (this.nc === nc)
2687
+ await this.reconnect();
2688
+ else
2689
+ await this.rebuildPromise;
2690
+ })();
2691
+ await this.waitServiceRecovery(recovery, deadline, opts.signal, () => e);
2692
+ this.clearResolvedService(endpoint);
2693
+ return this.invokeServiceOnce(endpoint, command, args, { ...opts, deadlineMs: deadline - Date.now() }, deadline);
2694
+ }
2695
+ }
2696
+ async invokeServiceOnce(endpoint, command, args, opts = {}, retryDeadline) {
2598
2697
  if (!this.nc)
2599
2698
  throw new Error(this.notLiveMsg());
2600
2699
  const nc = this.nc;
2601
2700
  const caller = this.serviceCaller();
2602
2701
  const resolve = async (signal = opts.signal, deadlineMs = opts.deadlineMs ?? 10_000) => {
2603
2702
  signal?.throwIfAborted();
2703
+ if (retryDeadline !== undefined)
2704
+ deadlineMs = Math.min(deadlineMs, retryDeadline - Date.now());
2705
+ if (deadlineMs <= 0)
2706
+ throw new EpEnvelopeError("deadline-exceeded", "the invocation deadline elapsed before resolve", undefined, "not-executed");
2604
2707
  const needed = opts.follow ? [command, "goal-result"] : [command];
2605
2708
  const classSelected = opts.instanceId === undefined && opts.classInstanceId === undefined;
2606
2709
  const full = classSelected ? this.resolvedServices.get(endpoint) : undefined;
@@ -2637,7 +2740,10 @@ export class CotalEndpoint extends EventEmitter {
2637
2740
  const unfollowable = opts.follow ? goalFollowRefusal(service) : undefined;
2638
2741
  if (unfollowable)
2639
2742
  return Promise.resolve(unfollowable);
2640
- return invokeCommand(nc, this.space, service, command, args, { ...invokeOpts, signal });
2743
+ const deadlineMs = retryDeadline === undefined ? invokeOpts.deadlineMs : retryDeadline - Date.now();
2744
+ if (deadlineMs !== undefined && deadlineMs <= 0)
2745
+ throw new EpEnvelopeError("deadline-exceeded", "the invocation deadline elapsed before publish", undefined, "not-executed");
2746
+ return invokeCommand(nc, this.space, service, command, args, { ...invokeOpts, deadlineMs, signal });
2641
2747
  };
2642
2748
  // A `failed-precondition` from the resolve is its own refusal, raised before any command was
2643
2749
  // published, so resolving once more is a repair. The repair spends what is left of the same
@@ -2771,15 +2877,28 @@ export class CotalEndpoint extends EventEmitter {
2771
2877
  };
2772
2878
  // A goal-bearing command (spawn/launch) follows its acceptance to the terminal so
2773
2879
  // the caller still returns on the real outcome (UX unchanged); every other command replies directly.
2774
- if (!opts.follow)
2775
- return doInvoke(await resolveFirst());
2776
- // Resolved before the submission starts: the follow reports an unmarked failure inside the
2777
- // submission as a command that may have run, and the resolve publishes none of this call.
2778
- let service;
2779
- return this.followServiceGoal(endpoint, (signal) => doInvoke(service, signal), opts.deadlineMs ?? 10_000, {
2780
- signal: opts.signal,
2781
- prepare: async (signal, deadline) => { service = await resolveFirst(signal, deadline); },
2782
- });
2880
+ const release = this.beginServiceInvocation(nc);
2881
+ try {
2882
+ if (!opts.follow)
2883
+ return await doInvoke(await resolveFirst());
2884
+ // Resolve and confirm observation before submission. Hold the barrier continuously across
2885
+ // both phases, then release when the command answers so accepted goals can outlive recovery.
2886
+ let service;
2887
+ return await this.followServiceGoal(endpoint, async (signal) => {
2888
+ try {
2889
+ return await doInvoke(service, signal);
2890
+ }
2891
+ finally {
2892
+ release();
2893
+ }
2894
+ }, opts.deadlineMs ?? 10_000, {
2895
+ signal: opts.signal,
2896
+ prepare: async (signal, deadline) => { service = await resolveFirst(signal, deadline); },
2897
+ });
2898
+ }
2899
+ finally {
2900
+ release();
2901
+ }
2783
2902
  }
2784
2903
  /** Send a durable-membership request to the SERVER-SIDE delivery daemon (`ctl.delivery`) and await its
2785
2904
  * reply. The reply rides a subject UNDER `ctl.delivery.<id>.>` (not the per-id `_INBOX`), so the