@adaptic/backend-legacy 0.0.960 → 0.0.962

package/client.cjs

@@ -39,7 +39,9 @@ exports.configureConnectionPool = configureConnectionPool;
 exports.setTokenProvider = setTokenProvider;
 exports.getApolloClient = getApolloClient;
 exports.getApolloModules = getApolloModules;
+exports.getPoolStats = getPoolStats;
 exports.stopClient = stopClient;
+const node_crypto_1 = require("node:crypto");
 const logger_1 = require("./utils/logger");
 const DEFAULT_POOL_CONFIG = {
     maxConcurrentOperations: 100, // Maximum concurrent operations to the database
@@ -56,6 +58,18 @@ let pendingOperations = 0;
 const operationQueue = [];
 let poolConfig = DEFAULT_POOL_CONFIG;
 let customTokenProvider;
+// === Pool observability counters ===
+// These are best-effort process-local counters. They are exposed via
+// `getPoolStats()` for engine-side Prometheus scraping. Resetting
+// categorical counters is not offered — callers that care about deltas
+// should sample-then-diff.
+const poolCounters = {
+    executedTotal: 0,
+    rejectedLoadShed: 0,
+    rejectedQueueWaitTimeout: 0,
+    rejectedNonRetryable: 0,
+    retryAttemptsTotal: 0,
+};
 /**
  * Dynamically loads the correct Apollo modules based on the runtime environment.
  */
@@ -154,15 +168,23 @@ async function getAuthToken() {
 }
 /**
  * Processes the operation queue respecting the connection pool limits.
+ *
+ * Before executing each dequeued operation, this also performs a proactive
+ * queue-wait-timeout sweep: any operation that has been waiting longer than
+ * `queueWaitTimeoutMs` on its first attempt is rejected inline by its own
+ * `executeWithRetry` (attempt === 0 branch) rather than consuming a slot.
+ * The sweep runs here (not on a separate timer) because queue pressure
+ * almost always clears at the same moment a slot opens up, so the check
+ * is cheap and avoids introducing another timer.
  */
 function processQueue() {
     // Process queue until we reach max concurrent operations or queue is empty
     while (pendingOperations < poolConfig.maxConcurrentOperations &&
         operationQueue.length > 0) {
-        const operation = operationQueue.shift();
-        if (operation) {
+        const entry = operationQueue.shift();
+        if (entry) {
             pendingOperations++;
-            void operation().finally(() => {
+            void entry.executeWithRetry().finally(() => {
                 pendingOperations--;
                 processQueue(); // Process next item after an operation completes
             });
@@ -206,6 +228,50 @@ function extractOperationName(options) {
  * timeouts are indistinguishable in logs and root-cause diagnosis is
  * impossible.
  */
+/**
+ * Classify an error as definitively non-retryable. These errors cost capacity
+ * when retried and cannot succeed without the caller changing behaviour:
+ *
+ * - **Accelerate 1102 "Worker exceeded resource limits"**: the Cloudflare
+ *   Worker backing Prisma Accelerate hit its CPU/memory budget. Cloudflare's
+ *   guidance is explicit: do not retry — retrying the same shape will hit
+ *   the same limit and only inflates worker concurrency across the fleet,
+ *   which can trigger broader throttling. Fix is at the caller (smaller
+ *   batch, narrower selection, chunked write).
+ * - **P2002 unique-constraint violation**: another writer already inserted
+ *   the conflicting row. Retrying the same mutation produces the same
+ *   failure. Fix is at the caller (idempotent find-or-create, stable
+ *   idempotency key, serialized writer).
+ * - **P2025 "record not found" race**: a concurrent delete/update removed
+ *   the target row. Retrying a delete/update against a vanished row loops.
+ * - **GraphQL schema validation / 400 Bad Request**: the query itself is
+ *   malformed; retrying guarantees the same 400.
+ *
+ * Explicit classification avoids the previous pattern where the generic
+ * `error.message.includes('Accelerate')` branch treated all Accelerate
+ * errors — including 1102 — as retryable, turning a single worker-limit
+ * breach into a retry storm that further starved the Accelerate worker
+ * pool. See Cloudflare error codes
+ * (https://developers.cloudflare.com/workers/observability/errors/#error-1102).
+ */
+function isDefinitivelyNonRetryable(message) {
+    return (
+    // Accelerate Worker resource-limit errors — never retry
+    message.includes('1102') ||
+        message.includes('Worker exceeded resource limits') ||
+        message.includes('Worker exceeded CPU') ||
+        message.includes('Worker exceeded memory') ||
+        // Prisma unique-constraint violation — retry just reproduces it
+        message.includes('P2002') ||
+        message.includes('Unique constraint failed') ||
+        // Prisma record-not-found race — deletes/updates already lost the row
+        message.includes('P2025') ||
+        message.includes('Record to update not found') ||
+        message.includes('Record to delete does not exist') ||
+        // GraphQL schema / 400 — malformed query, retry is pointless
+        message.includes('GRAPHQL_VALIDATION_FAILED') ||
+        message.includes('status code 400'));
+}
 async function enqueueOperation(operation, operationName, attempt = 0) {
     // Load shedding: reject new (first-attempt) operations when the queue is
     // saturated. This turns a cascading 30s timeout storm during backend
@@ -213,8 +279,8 @@ async function enqueueOperation(operation, operationName, attempt = 0) {
     // retry later, or skip this cycle). Retries bypass this check because
     // they represent in-flight commitments that already consumed backend
     // capacity on the first attempt.
-    if (attempt === 0 &&
-        operationQueue.length >= poolConfig.maxQueueDepth) {
+    if (attempt === 0 && operationQueue.length >= poolConfig.maxQueueDepth) {
+        poolCounters.rejectedLoadShed++;
         logger_1.logger.warn(`Apollo operation '${operationName}' rejected — queue saturated (${operationQueue.length}/${poolConfig.maxQueueDepth})`, {
             operation: operationName,
             queueDepth: operationQueue.length,
@@ -237,6 +303,7 @@ async function enqueueOperation(operation, operationName, attempt = 0) {
             if (attempt === 0) {
                 const waitMs = Date.now() - enqueuedAt;
                 if (waitMs > poolConfig.queueWaitTimeoutMs) {
+                    poolCounters.rejectedQueueWaitTimeout++;
                     logger_1.logger.warn(`Apollo operation '${operationName}' dropped — queue wait exceeded ${poolConfig.queueWaitTimeoutMs}ms (waited ${waitMs}ms)`, {
                         operation: operationName,
                         waitMs,
@@ -251,6 +318,7 @@ async function enqueueOperation(operation, operationName, attempt = 0) {
             }
             try {
                 const result = await operation();
+                poolCounters.executedTotal++;
                 resolve(result);
             }
             catch (error) {
@@ -276,10 +344,22 @@ async function enqueueOperation(operation, operationName, attempt = 0) {
                 // engine's database-operations-wrapper already handles this path with
                 // idempotent=true).
                 const isNonRetryableDelete = operationName.startsWith('deleteOne') && isTimeoutLike;
+                // Hard non-retry classes: retrying guarantees the same failure AND
+                // wastes backend capacity. Cross-check this list against caller-side
+                // handling — engine's graphql-error-handler should also treat these
+                // as non-retryable so double-wrapped callers don't re-retry.
+                const isHardNonRetryable = isDefinitivelyNonRetryable(errorMessage);
                 const isRetryable = !isNonRetryableDelete &&
+                    !isHardNonRetryable &&
                     error instanceof Error &&
-                    (error.message.includes('Accelerate') ||
-                        error.message.includes('code: 1016') ||
+                    // Narrowed from the previous broad `includes('Accelerate')`: that
+                    // matched 1102 (Worker resource-limit) and fed retry storms. Keep
+                    // only transient Accelerate classes that genuinely benefit from
+                    // retry (1016 = origin DNS temporarily unresolved, 1033 = worker
+                    // script temporarily unreachable).
+                    (error.message.includes('code: 1016') ||
+                        error.message.includes('code: 1033') ||
+                        error.message.includes('Accelerate transient') ||
                         error.message.includes('ECONNREFUSED') ||
                         error.message.includes('ECONNRESET') ||
                         error.message.includes('ETIMEDOUT') ||
@@ -304,14 +384,36 @@ async function enqueueOperation(operation, operationName, attempt = 0) {
                         category: 'SUPPRESSED_DELETE_RETRY',
                     });
                 }
+                if (isHardNonRetryable) {
+                    poolCounters.rejectedNonRetryable++;
+                    logger_1.logger.warn(`Apollo operation '${operationName}' failed with non-retryable error — NOT retrying (would reproduce same failure or hit resource limit)`, {
+                        operation: operationName,
+                        attempt: attempt + 1,
+                        error: errorMessage,
+                        category: 'NON_RETRYABLE_HARD',
+                    });
+                }
                 if (attempt < poolConfig.retryAttempts && isRetryable) {
-                    const delay = poolConfig.retryDelay * Math.pow(2, attempt);
+                    // Decorrelated jitter: delay = random_between(retryDelay, base * 3)
+                    // where base = retryDelay * 2^attempt. This breaks the thundering
+                    // herd pattern where every op enqueued at the same stall moment
+                    // retries at the same time (1.5s, 3s, 6s — all in lockstep) and
+                    // overwhelms the backend the instant it recovers. See AWS
+                    // architecture blog "Exponential Backoff And Jitter".
+                    const base = poolConfig.retryDelay * Math.pow(2, attempt);
+                    const upper = base * 3;
+                    const lower = poolConfig.retryDelay;
+                    const delay = lower >= upper
+                        ? upper
+                        : (0, node_crypto_1.randomInt)(Math.floor(lower), Math.floor(upper) + 1);
+                    poolCounters.retryAttemptsTotal++;
                     logger_1.logger.warn(`Apollo operation '${operationName}' failed, retrying in ${delay}ms (attempt ${attempt + 1}/${poolConfig.retryAttempts})`, {
                         operation: operationName,
                         attempt: attempt + 1,
                         maxAttempts: poolConfig.retryAttempts,
                         delayMs: delay,
                         error: errorMessage,
+                        category: 'RETRY',
                     });
                     setTimeout(() => {
                         enqueueOperation(operation, operationName, attempt + 1)
@@ -325,13 +427,14 @@ async function enqueueOperation(operation, operationName, attempt = 0) {
                             operation: operationName,
                             attempts: poolConfig.retryAttempts,
                             error: errorMessage,
+                            category: 'RETRY_EXHAUSTED',
                         });
                     }
                     reject(error);
                 }
             }
         };
-        operationQueue.push(executeWithRetry);
+        operationQueue.push({ executeWithRetry, enqueuedAt, operationName });
         processQueue();
     });
 }
@@ -538,6 +641,25 @@ exports.client = {
         return getApolloClient().then(onfulfilled, onrejected);
     },
 };
+/**
+ * Returns a read-only snapshot of pool state for observability.
+ * Safe to call on every scrape interval (O(1)).
+ */
+function getPoolStats() {
+    return {
+        pendingOperations,
+        queueDepth: operationQueue.length,
+        maxConcurrentOperations: poolConfig.maxConcurrentOperations,
+        maxQueueDepth: poolConfig.maxQueueDepth,
+        queueWaitTimeoutMs: poolConfig.queueWaitTimeoutMs,
+        connectionTimeoutMs: poolConfig.connectionTimeout,
+        executedTotal: poolCounters.executedTotal,
+        rejectedLoadShed: poolCounters.rejectedLoadShed,
+        rejectedQueueWaitTimeout: poolCounters.rejectedQueueWaitTimeout,
+        rejectedNonRetryable: poolCounters.rejectedNonRetryable,
+        retryAttemptsTotal: poolCounters.retryAttemptsTotal,
+    };
+}
 /**
  * Gracefully stops the singleton Apollo Client and releases its resources.
  * Call this during process shutdown to close keep-alive HTTP connections and

package/client.d.ts

@@ -100,6 +100,38 @@ export declare function getApolloModules(): Promise<ApolloModules>;
  * @deprecated Prefer calling `getApolloClient()` directly for explicit lifecycle control.
  */
 export declare const client: PromiseLike<ApolloClientType<NormalizedCacheObject>>;
+/**
+ * Snapshot of connection-pool state at the instant `getPoolStats` is called.
+ * Exposed so downstream observability (engine's Prometheus registry) can
+ * scrape pool depth, pending-op concurrency, retry counters, and rejection
+ * categories without having to reach into module-private state.
+ */
+export interface PoolStats {
+    /** Operations currently in flight (post-dequeue, pre-resolution). */
+    pendingOperations: number;
+    /** Operations waiting in the queue for a concurrency slot. */
+    queueDepth: number;
+    /** Configured limits at the time of sampling. */
+    maxConcurrentOperations: number;
+    maxQueueDepth: number;
+    queueWaitTimeoutMs: number;
+    connectionTimeoutMs: number;
+    /** Count of operations whose underlying fetch resolved (success path). */
+    executedTotal: number;
+    /** Count of first-attempt ops rejected by load-shedding admission control. */
+    rejectedLoadShed: number;
+    /** Count of ops rejected after queue-wait-timeout at dequeue. */
+    rejectedQueueWaitTimeout: number;
+    /** Count of ops that failed with a hard non-retryable error class. */
+    rejectedNonRetryable: number;
+    /** Count of retry attempts fired across all operations. */
+    retryAttemptsTotal: number;
+}
+/**
+ * Returns a read-only snapshot of pool state for observability.
+ * Safe to call on every scrape interval (O(1)).
+ */
+export declare function getPoolStats(): PoolStats;
 /**
  * Gracefully stops the singleton Apollo Client and releases its resources.
  * Call this during process shutdown to close keep-alive HTTP connections and

package/esm/ABTest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ABTest.d.ts","sourceRoot":"","sources":["../../src/ABTest.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAmB,gBAAgB,EAAE,qBAAqB,EAAoB,MAAM,UAAU,CAAC;AAsLpG,eAAO,MAAM,MAAM;IAEjB;;;;;OAKG;IAEH;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAklD9G;;;;;;;OAOG;sBACqB,UAAU,EAAE,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,YAAY;QAAE,cAAc,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IA4JxK;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAy3L5G;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IA40O5G;;;;;;OAMG;sBACqB,UAAU,EAAE,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAw3LhI;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAyI5G;;;;;;;OAOG;eACc,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAsHtJ;;;;;OAKG;0BACyB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;IAkGlG;;;;;;;OAOG;oBACmB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;CAkI9J,CAAC"}
+{"version":3,"file":"ABTest.d.ts","sourceRoot":"","sources":["../../src/ABTest.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,8CAA8C,CAAC;AACpF,OAAO,EAAmB,gBAAgB,EAAE,qBAAqB,EAAoB,MAAM,UAAU,CAAC;AAsLpG,eAAO,MAAM,MAAM;IAEjB;;;;;OAKG;IAEH;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAgmD9G;;;;;;;OAOG;sBACqB,UAAU,EAAE,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,YAAY;QAAE,cAAc,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAuKxK;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAq4L5G;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAw1O5G;;;;;;OAMG;sBACqB,UAAU,EAAE,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAm4LhI;;;;;;OAMG;kBACiB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;IAqJ5G;;;;;;;OAOG;eACc,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAiItJ;;;;;OAKG;0BACyB,gBAAgB,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;IA6GlG;;;;;;;OAOG;oBACmB,UAAU,iBAAiB,gBAAgB,CAAC,qBAAqB,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;CA6I9J,CAAC"}