@milaboratories/pl-client 3.14.4 → 3.14.6

package/src/core/config.ts

@@ -90,6 +90,11 @@ export interface PlClientConfig {
 
   /** Value from 0 to 1, determine level of randomness to introduce to the backoff delays sequence. (0 meaning no randomness) */
   retryJitter: number;
+
+  /** Upper bound for a single backoff delay, in ms. Without it the exponential
+   * sequence keeps growing across all attempts: at the defaults the last sleep
+   * alone reaches ~66s (~150s once jitter compounds). */
+  retryMaxDelay: number;
 }
 
 export const DEFAULT_REQUEST_TIMEOUT = 5_000;
@@ -106,6 +111,41 @@ export const DEFAULT_RETRY_INITIAL_DELAY = 20; // 20 ms * <jitter> of sleep afte
 export const DEFAULT_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER = 1.5; // + 50% on each round
 export const DEFAULT_RETRY_LINEAR_BACKOFF_STEP = 50; // + 50 ms
 export const DEFAULT_RETRY_JITTER = 0.3; // 30%
+export const DEFAULT_RETRY_MAX_DELAY = 5_000; // 5 seconds
+
+/** Multiplier applied to the observed RTT when deriving the unary deadline. A unary call
+ * costs at least one round trip, plus connect/DNS/LB and server work on top, so the
+ * deadline needs sizeable headroom over the bare RTT. */
+export const RTT_DEADLINE_MULTIPLIER = 8;
+
+/** Ceiling for the RTT-derived unary deadline. Beyond this a call is stuck rather than
+ * slow, and waiting longer only delays the retry. */
+export const MAX_ADAPTIVE_REQUEST_TIMEOUT = 60_000;
+
+/** Weight of the newest RTT sample in the smoothed estimate. Low enough that one unlucky
+ * sample cannot swing the deadline, high enough to follow a real change. */
+export const RTT_SMOOTHING_ALPHA = 0.3;
+
+/** Login runs password hashing (and possibly an external IdP round trip) server side, so
+ * it is legitimately slow and gets its own deadline instead of the unary default. */
+export const DEFAULT_LOGIN_TIMEOUT = 120_000;
+
+/** Unary deadline for a given RTT estimate: floored by the configured timeout so a fast
+ * link behaves exactly as before, capped by {@link MAX_ADAPTIVE_REQUEST_TIMEOUT}.
+ * An undefined `rttMs` (no ping yet) yields the configured value unchanged. */
+export function deriveUnaryDeadline(configuredMs: number, rttMs: number | undefined): number {
+  if (rttMs === undefined) return configuredMs;
+  return Math.min(
+    MAX_ADAPTIVE_REQUEST_TIMEOUT,
+    Math.max(configuredMs, Math.ceil(rttMs * RTT_DEADLINE_MULTIPLIER)),
+  );
+}
+
+/** Folds a new round-trip sample into an exponentially smoothed estimate. */
+export function smoothRtt(previousMs: number | undefined, sampleMs: number): number {
+  if (previousMs === undefined) return sampleMs;
+  return previousMs * (1 - RTT_SMOOTHING_ALPHA) + sampleMs * RTT_SMOOTHING_ALPHA;
+}
 
 export const DefaultRetryOptions: ExponentialBackoffRetryOptions = {
   type: "exponentialBackoff",
@@ -113,6 +153,7 @@ export const DefaultRetryOptions: ExponentialBackoffRetryOptions = {
   initialDelay: DEFAULT_RETRY_INITIAL_DELAY,
   backoffMultiplier: DEFAULT_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER,
   jitter: DEFAULT_RETRY_JITTER,
+  maxDelay: DEFAULT_RETRY_MAX_DELAY,
 };
 
 type PlConfigOverrides = Partial<
@@ -161,6 +202,7 @@ export function plAddressToConfig(
       retryExponentialBackoffMultiplier: DEFAULT_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER,
       retryLinearBackoffStep: DEFAULT_RETRY_LINEAR_BACKOFF_STEP,
       retryJitter: DEFAULT_RETRY_JITTER,
+      retryMaxDelay: DEFAULT_RETRY_MAX_DELAY,
 
       ...overrides,
     };
@@ -234,6 +276,7 @@ export function plAddressToConfig(
       parseInt(url.searchParams.get("retry-linear-backoff-step")) ??
       DEFAULT_RETRY_LINEAR_BACKOFF_STEP,
     retryJitter: parseInt(url.searchParams.get("retry-backoff-jitter")) ?? DEFAULT_RETRY_JITTER,
+    retryMaxDelay: parseInt(url.searchParams.get("retry-max-delay")) ?? DEFAULT_RETRY_MAX_DELAY,
 
     ...overrides,
   };

package/src/core/error.test.ts

@@ -1,7 +1,29 @@
 import * as tp from "node:timers/promises";
-import { isTimeoutOrCancelError } from "./errors";
+import { isTimeoutOrCancelError, isTransientCallFailure } from "./errors";
 import { test, expect } from "vitest";
 
+/** Shapes the predicates match on: an RpcError-like from grpc, a RESTError-like from REST. */
+function rpcError(code: string) {
+  return { name: "RpcError", code };
+}
+
+test("only transport failures count as transient", () => {
+  // Worth retrying: the call never reached a verdict.
+  expect(isTransientCallFailure(rpcError("UNAVAILABLE"))).toBe(true);
+  expect(isTransientCallFailure(rpcError("DEADLINE_EXCEEDED"))).toBe(true);
+
+  // Real answers from the server. Retrying these would hide a genuine failure and,
+  // for auth, silently repeat a rejected credential.
+  expect(isTransientCallFailure(rpcError("UNAUTHENTICATED"))).toBe(false);
+  expect(isTransientCallFailure(rpcError("PERMISSION_DENIED"))).toBe(false);
+  expect(isTransientCallFailure(rpcError("NOT_FOUND"))).toBe(false);
+  expect(isTransientCallFailure(rpcError("UNIMPLEMENTED"))).toBe(false);
+
+  expect(isTransientCallFailure(undefined)).toBe(false);
+  expect(isTransientCallFailure(null)).toBe(false);
+  expect(isTransientCallFailure(new Error("plain"))).toBe(false);
+});
+
 test("timeout of sleep error type detection", async () => {
   let noError = false;
   try {

package/src/core/errors.ts

@@ -82,6 +82,13 @@ export function isTimeoutOrCancelError(err: unknown, nested: boolean = false): b
   return false;
 }
 
+/** Failures worth another attempt on an idempotent call: the peer was unreachable, or the
+ * deadline elapsed. Everything else (auth, permission, not-found, unimplemented) is a real
+ * answer from the server, and retrying it only wastes the caller's time. */
+export function isTransientCallFailure(err: unknown): boolean {
+  return isConnectionProblem(err) || isTimeoutError(err);
+}
+
 export function isUnimplementedError(err: unknown, nested: boolean = false): boolean {
   if (err === undefined || err === null) return false;
 

package/src/core/ll_client.ts

@@ -13,7 +13,14 @@ import type {
   PlConnectionStatus,
   PlConnectionStatusListener,
 } from "./config";
-import { plAddressToConfig, type wireProtocol, SUPPORTED_WIRE_PROTOCOLS } from "./config";
+import {
+  plAddressToConfig,
+  type wireProtocol,
+  SUPPORTED_WIRE_PROTOCOLS,
+  DEFAULT_LOGIN_TIMEOUT,
+  deriveUnaryDeadline,
+  smoothRtt,
+} from "./config";
 import type { GrpcOptions } from "@protobuf-ts/grpc-transport";
 import { GrpcTransport } from "@protobuf-ts/grpc-transport";
 import { LLPlTransaction } from "./ll_transaction";
@@ -42,7 +49,7 @@ import {
   TxAPI_ServerMessage,
 } from "../proto-grpc/github.com/milaboratory/pl/plapi/plapiproto/api";
 import type { MiLogger } from "@milaboratories/ts-helpers";
-import { isAbortedError, isUnauthenticated } from "./errors";
+import { isAbortedError, isTransientCallFailure, isUnauthenticated } from "./errors";
 import { Timestamp } from "../proto-grpc/google/protobuf/timestamp";
 
 export interface PlCallOps {
@@ -50,6 +57,17 @@ export interface PlCallOps {
   abortSignal?: AbortSignal;
 }
 
+/** Bounded retry for idempotent unary calls. Deliberately short: these calls sit on the
+ * connect path and gate the UI, so a few quick attempts beat a long grind. */
+const IDEMPOTENT_RETRY_OPTIONS: RetryOptions = {
+  type: "exponentialBackoff",
+  maxAttempts: 4,
+  initialDelay: 200,
+  backoffMultiplier: 2,
+  jitter: 0.3,
+  maxDelay: 2_000,
+};
+
 // Parses leading "<major>.<minor>.<patch>" from a version string like
 // "3.1.1" or "3.1.1-rc1" and returns true if the parsed version is >= target.
 // Returns false for unparseable versions (safer to assume an old backend).
@@ -128,6 +146,15 @@ export class LLPlClient implements WireClientProviderFactory {
   private _wireProto: wireProtocol = "grpc";
   private _wireConn!: WireConnection;
 
+  /** The live GrpcOptions object handed to GrpcTransport. The transport keeps it by
+   * reference and re-reads it on every call, so mutating `timeout` here retunes
+   * subsequent deadlines without tearing down the connection. */
+  private _grpcOptions?: GrpcOptions;
+
+  /** Smoothed round-trip estimate in ms, from ping timings. Undefined until the first
+   * successful ping, in which case the configured deadline is used as-is. */
+  private _rttMs?: number;
+
   private readonly _restInterceptors: Dispatcher.DispatcherComposeInterceptor[];
   private readonly _restMiddlewares: Middleware[];
   private readonly _grpcInterceptors: Interceptor[];
@@ -160,7 +187,10 @@ export class LLPlClient implements WireClientProviderFactory {
     // Guarantee a ping happened so capability-gated paths (login, refresh) can branch synchronously.
     // In the autodetect path the loop's last successful ping already populated _serverInfo via the
     // side-effect in ping(); this fallback covers the path where autodetect is disabled.
-    if (!pl._serverInfo) await pl.ping();
+    // Retried, not bare: this ping is the first contact with the server, so DNS and LB
+    // warm-up land here. A single transient failure must not fail the whole connect.
+    // (Not inside ping() itself: the autodetect path already wraps it in its own retry.)
+    if (!pl._serverInfo) await pl.withIdempotentRetry("ping", () => pl.ping());
 
     // Install the process-global signature-strictness flag based on backend version.
     setResourceSignaturesRequired(pl.supportsResourceSignatures);
@@ -266,6 +296,10 @@ export class LLPlClient implements WireClientProviderFactory {
     const clientOptions: ClientOptions = {
       "grpc.keepalive_time_ms": 30_000, // 30 seconds
       "grpc.service_config_disable_resolution": 1, // Disable DNS TXT lookups for service config
+      // Raises the per-stream and connection-level HTTP/2 windows. Node's 64 KB default
+      // caps a big single-stream read (project open) at window/RTT rather than link speed,
+      // and a lost WINDOW_UPDATE can deadlock it under packet loss.
+      "grpc-node.flow_control_window": 16 * 1024 * 1024, // 16 MiB
       interceptors: this._grpcInterceptors,
     };
 
@@ -280,7 +314,7 @@ export class LLPlClient implements WireClientProviderFactory {
     //
     const grpcOptions: GrpcOptions = {
       host: this.conf.hostAndPort,
-      timeout: this.conf.defaultRequestTimeout,
+      timeout: this.unaryDeadline(),
       channelCredentials: this.conf.ssl
         ? ChannelCredentials.createSsl()
         : ChannelCredentials.createInsecure(),
@@ -305,9 +339,48 @@ export class LLPlClient implements WireClientProviderFactory {
       delete process.env.grpc_proxy;
     }
 
+    this._grpcOptions = grpcOptions;
     this._replaceWireConnection({ type: "grpc", Transport: new GrpcTransport(grpcOptions) });
   }
 
+  /** Unary deadline derived from the observed RTT, floored by the configured value so
+   * a fast link behaves exactly as before, and capped by
+   * {@link MAX_ADAPTIVE_REQUEST_TIMEOUT}. */
+  private unaryDeadline(): number {
+    return deriveUnaryDeadline(this.conf.defaultRequestTimeout, this._rttMs);
+  }
+
+  /** Folds a fresh round-trip sample into the estimate and retunes the live unary
+   * deadline. Called after every successful ping. */
+  private recordRtt(sampleMs: number): void {
+    this._rttMs = smoothRtt(this._rttMs, sampleMs);
+
+    if (this._grpcOptions) {
+      const next = this.unaryDeadline();
+      if (next !== this._grpcOptions.timeout) {
+        this.ops.logger?.info(
+          `Unary deadline retuned to ${next}ms (rtt estimate ${Math.round(this._rttMs)}ms)`,
+        );
+        this._grpcOptions.timeout = next;
+      }
+    }
+  }
+
+  /** Smoothed round-trip estimate in ms, or undefined before the first ping. */
+  public get rttEstimateMs(): number | undefined {
+    return this._rttMs;
+  }
+
+  /** Runs an idempotent unary call with a bounded retry on transient transport failures.
+   * Safe only for calls with no side effects, since a retry may duplicate the request. */
+  private async withIdempotentRetry<T>(name: string, cb: () => Promise<T>): Promise<T> {
+    return await retry(cb, IDEMPOTENT_RETRY_OPTIONS, (e: unknown) => {
+      if (!isTransientCallFailure(e)) return false;
+      this.ops.logger?.info(`${name}: transient failure, retrying. err=${String(e)}`);
+      return true;
+    });
+  }
+
   private _replaceWireConnection(newConn: WireConnection): void {
     const oldConn = this._wireConn;
     this._wireConn = newConn;
@@ -558,7 +631,7 @@ export class LLPlClient implements WireClientProviderFactory {
             expiration: { seconds: ttlSeconds, nanos: 0 },
             requestedRole: role,
           },
-          { meta },
+          { meta, timeout: DEFAULT_LOGIN_TIMEOUT },
         ).response
       ).token;
     } else {
@@ -584,14 +657,17 @@ export class LLPlClient implements WireClientProviderFactory {
 
     if (cl instanceof GrpcPlApiClient) {
       return (
-        await cl.login({
-          credentials: {
-            oneofKind: "basic",
-            basic: { login: user, password },
+        await cl.login(
+          {
+            credentials: {
+              oneofKind: "basic",
+              basic: { login: user, password },
+            },
+            expiration: { seconds: ttl, nanos: 0 },
+            requestedRole: role,
           },
-          expiration: { seconds: ttl, nanos: 0 },
-          requestedRole: role,
-        }).response
+          { timeout: DEFAULT_LOGIN_TIMEOUT },
+        ).response
       ).token;
     } else {
       const resp = cl.POST("/v1/auth/login", {
@@ -620,14 +696,17 @@ export class LLPlClient implements WireClientProviderFactory {
 
     if (cl instanceof GrpcPlApiClient) {
       return (
-        await cl.login({
-          credentials: {
-            oneofKind: "token",
-            token: { token: bytes },
+        await cl.login(
+          {
+            credentials: {
+              oneofKind: "token",
+              token: { token: bytes },
+            },
+            expiration: { seconds: ttl, nanos: 0 },
+            requestedRole: role,
           },
-          expiration: { seconds: ttl, nanos: 0 },
-          requestedRole: role,
-        }).response
+          { timeout: DEFAULT_LOGIN_TIMEOUT },
+        ).response
       ).token;
     } else {
       const resp = cl.POST("/v1/auth/login", {
@@ -649,12 +728,15 @@ export class LLPlClient implements WireClientProviderFactory {
     const cl = this.clientProvider.get();
     if (cl instanceof GrpcPlApiClient) {
       return (
-        await cl.login({
-          credentials: {
-            oneofKind: "sso",
-            sso: { tokenResponse },
+        await cl.login(
+          {
+            credentials: {
+              oneofKind: "sso",
+              sso: { tokenResponse },
+            },
           },
-        }).response
+          { timeout: DEFAULT_LOGIN_TIMEOUT },
+        ).response
       ).token;
     } else {
       const resp = cl.POST("/v1/auth/login", {
@@ -707,10 +789,13 @@ export class LLPlClient implements WireClientProviderFactory {
 
     if (cl instanceof GrpcPlApiClient) {
       return (
-        await cl.refreshToken({
-          token: currentToken,
-          expiration: { seconds: ttl, nanos: 0 },
-        }).response
+        await cl.refreshToken(
+          {
+            token: currentToken,
+            expiration: { seconds: ttl, nanos: 0 },
+          },
+          { timeout: DEFAULT_LOGIN_TIMEOUT },
+        ).response
       ).token;
     } else {
       const resp = cl.POST("/v1/auth/refresh", {
@@ -723,6 +808,8 @@ export class LLPlClient implements WireClientProviderFactory {
   public async ping(): Promise<grpcTypes.MaintenanceAPI_Ping_Response> {
     const cl = this.clientProvider.get();
     let resp: grpcTypes.MaintenanceAPI_Ping_Response;
+    // Ping is the cheapest call we make, so its duration is our best RTT proxy.
+    const startedAt = performance.now();
     if (cl instanceof GrpcPlApiClient) {
       resp = (await cl.ping({})).response;
     } else {
@@ -737,6 +824,7 @@ export class LLPlClient implements WireClientProviderFactory {
         capabilities: (pingData as any).capabilities ?? [],
       };
     }
+    this.recordRtt(performance.now() - startedAt);
     this._serverInfo = resp;
     return resp;
   }
@@ -895,6 +983,14 @@ export class LLPlClient implements WireClientProviderFactory {
 
   public async getUserRoot(
     opts: { login?: string; createIfNotExists?: boolean } = {},
+  ): Promise<grpcTypes.AuthAPI_GetUserRoot_Response> {
+    // Retryable even with createIfNotExists: the call is get-or-create keyed on login, so a
+    // second attempt returns the existing root rather than making another one.
+    return await this.withIdempotentRetry("getUserRoot", () => this.getUserRootOnce(opts));
+  }
+
+  private async getUserRootOnce(
+    opts: { login?: string; createIfNotExists?: boolean } = {},
   ): Promise<grpcTypes.AuthAPI_GetUserRoot_Response> {
     const cl = this.clientProvider.get();
     if (cl instanceof GrpcPlApiClient) {