@superblocksteam/gateway 2.0.165-next.0 → 2.0.166-next.1

package/dist/server/client.js

@@ -1,5 +1,5 @@
 import { uploadAttachmentToServerApi } from "@superblocksteam/shared";
-import { recordIdentityReadOutcome } from "../telemetry/metrics.js";
+import { recordFactsFlagBootstrapOutcome, recordIdentityReadOutcome, recordServerReadRetryOutcome, } from "../telemetry/metrics.js";
 import { withGuardedIdentityReads, } from "./identity-reads.js";
 const MAX_DETAIL_LENGTH = 300;
 /**
@@ -31,12 +31,98 @@ function detailSuffix(body) {
 }
 export class ServerHttpError extends Error {
     status;
-    constructor(message, status) {
+    retryAfterMs;
+    constructor(message, status, retryAfterMs) {
         super(message);
         this.status = status;
+        this.retryAfterMs = retryAfterMs;
         this.name = "ServerHttpError";
     }
 }
+/**
+ * This client gave up on a request before the server answered it.
+ *
+ * Told apart from a lost connection because the two deserve different
+ * treatment on a retry: a connection that was refused or dropped is cheap to
+ * try again, while repeating a request that already burned its whole bound
+ * multiplies the wait a caller is sitting through.
+ */
+export class ServerTimeoutError extends Error {
+    constructor(message, options) {
+        super(message, options);
+        this.name = "ServerTimeoutError";
+    }
+}
+const RETRYABLE_TRANSPORT_CODES = new Set([
+    "EAI_AGAIN",
+    "ECONNABORTED",
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "EHOSTUNREACH",
+    "ENETDOWN",
+    "ENETUNREACH",
+    "EPIPE",
+    "ETIMEDOUT",
+    "UND_ERR_BODY_TIMEOUT",
+    "UND_ERR_CONNECT_TIMEOUT",
+    "UND_ERR_HEADERS_TIMEOUT",
+    "UND_ERR_SOCKET",
+]);
+const UNSENT_TRANSPORT_CODES = new Set([
+    "EAI_AGAIN",
+    "ECONNREFUSED",
+    "EHOSTUNREACH",
+    "ENETDOWN",
+    "ENETUNREACH",
+    "UND_ERR_CONNECT_TIMEOUT",
+]);
+function errorCode(error, depth = 0) {
+    if (depth >= 4 || !error || typeof error !== "object") {
+        return undefined;
+    }
+    const { cause, code } = error;
+    if (typeof code === "string") {
+        return code;
+    }
+    const causeCode = errorCode(cause, depth + 1);
+    if (causeCode) {
+        return causeCode;
+    }
+    if (error instanceof AggregateError) {
+        for (const nested of error.errors) {
+            const nestedCode = errorCode(nested, depth + 1);
+            if (nestedCode) {
+                return nestedCode;
+            }
+        }
+    }
+    return undefined;
+}
+/**
+ * The connection failed: refused, reset, or dropped before the answer was
+ * complete. Named so a retry can cover it without also covering a response
+ * that did arrive and turned out to be unreadable.
+ *
+ * Most failures do not prove whether the request reached a server, so repeat
+ * safety remains the call site's decision. `requestNotSent` is reserved for
+ * pre-connect failures whose error code proves otherwise.
+ */
+export class ServerTransportError extends Error {
+    code;
+    requestNotSent;
+    retryable;
+    constructor(message, options) {
+        const code = errorCode(options.cause);
+        super(code ? `${message} (${code})` : message, options);
+        this.name = "ServerTransportError";
+        this.code = code;
+        this.requestNotSent =
+            options.requestNotSent === true &&
+                code !== undefined &&
+                UNSENT_TRANSPORT_CODES.has(code);
+        this.retryable = code !== undefined && RETRYABLE_TRANSPORT_CODES.has(code);
+    }
+}
 /**
  * Generous enough for a sandbox start.
  *
@@ -73,21 +159,142 @@ const DEFAULT_IDENTITY_TIMEOUT_MS = 30_000;
  * kept going straight at an endpoint already asking for less load.
  */
 function identityEndpointUnhealthy(error) {
-    return !(error instanceof ServerHttpError &&
-        error.status < 500 &&
-        error.status !== 429);
+    return !(error instanceof ServerHttpError && !isTransientHttpStatus(error.status));
+}
+function isTransientHttpStatus(status) {
+    return status >= 500 || status === 429;
 }
 /**
- * Extra attempts for reads that failed with a server-side error.
+ * How long a read waits before each re-send, by what failed it.
+ *
+ * Only reads are retried, and only because the call sites that leave `retry`
+ * on are pure lookups: a repeat of one changes nothing. A failed POST may well
+ * have taken effect, and re-sending one can deploy or commit twice; a read
+ * that provisions, or that already sits in its own poll loop, opts out.
+ *
+ * A transient status is a fast answer from a route that is up, so a short
+ * pause is enough. 429 counts as transient alongside 5xx: it is the route
+ * reporting its own capacity, and backing off is the whole point of it.
+ * Retry-After replaces that pause when it is at most five seconds; a longer
+ * server window is surfaced immediately rather than parking a tool call.
+ *
+ * A lost connection is usually a deployment rolling through the API, which
+ * refuses and resets connections for a few seconds. This schedule waits that
+ * out, and still finishes well inside the ten seconds a tool call waits on
+ * identity. A timeout is never retried: that attempt already spent its whole
+ * bound, and sending it again spends another.
  *
- * Only GETs are retried: they carry no side effects, so a repeat is free. A
- * failed POST may well have taken effect, and re-sending one can deploy or
- * commit twice.
+ * Each delay is one retry: transient statuses get two, lost connections get
+ * four.
  */
-const READ_RETRY_ATTEMPTS = 2;
-const READ_RETRY_BASE_DELAY_MS = 100;
-function delay(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+const TRANSIENT_STATUS_RETRY_DELAYS_MS = [100, 200];
+const LOST_CONNECTION_RETRY_DELAYS_MS = [250, 500, 1_000, 2_000];
+const MAX_RETRY_AFTER_MS = 5_000;
+const MIN_RETRY_ATTEMPT_MS = 50;
+function jitter(delayMs) {
+    return Math.ceil(delayMs * (0.75 + Math.random() * 0.25));
+}
+function jitterAfterMinimum(delayMs) {
+    return Math.ceil(delayMs * (1 + Math.random() * 0.25));
+}
+class ReadRetrySchedule {
+    lostConnections = 0;
+    transientStatuses = 0;
+    next(error) {
+        if (error instanceof ServerHttpError) {
+            if (!isTransientHttpStatus(error.status)) {
+                return undefined;
+            }
+            const delayMs = TRANSIENT_STATUS_RETRY_DELAYS_MS[this.transientStatuses++];
+            if (delayMs === undefined) {
+                return "exhausted";
+            }
+            if (error.retryAfterMs === undefined) {
+                return { cause: "http", waitMs: jitter(delayMs) };
+            }
+            return error.retryAfterMs > MAX_RETRY_AFTER_MS
+                ? undefined
+                : {
+                    cause: "http",
+                    waitMs: jitterAfterMinimum(Math.max(delayMs, error.retryAfterMs)),
+                };
+        }
+        if (error instanceof ServerTransportError && error.retryable) {
+            const delayMs = LOST_CONNECTION_RETRY_DELAYS_MS[this.lostConnections++];
+            return delayMs === undefined
+                ? "exhausted"
+                : { cause: "transport", waitMs: jitter(delayMs) };
+        }
+        return undefined;
+    }
+}
+function parseRetryAfterMs(value) {
+    const retryAfter = value?.trim();
+    if (!retryAfter) {
+        return undefined;
+    }
+    const seconds = Number(retryAfter);
+    if (Number.isFinite(seconds) && seconds >= 0) {
+        return Math.ceil(seconds * 1_000);
+    }
+    const atMs = Date.parse(retryAfter);
+    return Number.isNaN(atMs) ? undefined : Math.max(0, atMs - Date.now());
+}
+function withAttemptCount(error, attempts) {
+    if (attempts < 2 || !(error instanceof Error)) {
+        return error;
+    }
+    error.message = `${error.message}; ${attempts} attempts failed`;
+    return error;
+}
+function delay(ms, signal) {
+    if (signal?.aborted) {
+        return Promise.reject(signal.reason);
+    }
+    return new Promise((resolve, reject) => {
+        const onAbort = () => {
+            clearTimeout(timeout);
+            reject(signal?.reason);
+        };
+        const timeout = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+/**
+ * One attempt at one request: the signal that bounds it, and the judgement of
+ * why it failed, applied alike to the connection and to the body read so a
+ * connection lost mid-response counts the same as one refused up front.
+ */
+class ServerAttempt {
+    path;
+    boundMs;
+    callerSignal;
+    timeoutSignal;
+    signal;
+    constructor(path, boundMs, callerSignal) {
+        this.path = path;
+        this.boundMs = boundMs;
+        this.callerSignal = callerSignal;
+        this.timeoutSignal = AbortSignal.timeout(boundMs);
+        this.signal = callerSignal
+            ? AbortSignal.any([callerSignal, this.timeoutSignal])
+            : this.timeoutSignal;
+    }
+    /** What to throw for a fetch or body read that rejected with `error`. */
+    failure(error, lost, requestNotSent = false) {
+        // The caller's own abort is neither our timeout nor a lost connection,
+        // and wrapping it as one would hand it to the read retry.
+        if (this.callerSignal?.aborted) {
+            return error;
+        }
+        if (this.timeoutSignal.aborted) {
+            return new ServerTimeoutError(`Server ${this.path} timed out after ${this.boundMs}ms`, { cause: error });
+        }
+        return new ServerTransportError(`Server ${this.path} ${lost}: ${error instanceof Error ? error.message : String(error)}`, { cause: error, requestNotSent });
+    }
 }
 const DEPLOY_STATUSES = [
     "awaiting_ack",
@@ -188,67 +395,152 @@ export class HttpSuperblocksServerClient {
         return body.data;
     }
     /**
-     * Issues the request, retrying a transient status when the verb makes that
-     * safe.
-     *
-     * A transient failure on the identity lookup used to fail the whole tool call
-     * and leave the caller guessing whether their account was broken.
+     * Issues the request, sending a read again when what failed it says nothing
+     * about the request itself: a transient status, or a connection that was
+     * lost. {@link ReadRetrySchedule} decides which failures qualify and for how
+     * long.
      *
-     * 429 counts as transient alongside 5xx: it is the route reporting its own
-     * capacity, and backing off is the whole point of it.
+     * A transient failure on the identity lookup used to fail the whole tool
+     * call and leave the caller guessing whether their account was broken. A
+     * lost connection used to fail on first sight, which is how a build's poll
+     * loop lost its caller's identity while a deployment rolled through the API.
      */
     async request(path, init) {
-        const retryable = init.retry !== false && (init.method ?? "GET").toUpperCase() === "GET";
-        let lastError;
-        for (let attempt = 0; attempt <= (retryable ? READ_RETRY_ATTEMPTS : 0);) {
-            try {
-                return await this.send(path, init);
+        const method = (init.method ?? "GET").toUpperCase();
+        const repeatSafe = init.retry ?? method === "GET";
+        const retries = new ReadRetrySchedule();
+        const timeoutMs = init.timeoutMs ?? this.timeoutMs;
+        const deadlineAtMs = Date.now() + timeoutMs;
+        let attempts = 0;
+        let lastFailure;
+        let retryCause;
+        let retryExhausted = false;
+        const operation = init.operation ?? "other";
+        const timeoutError = (cause = lastFailure) => new ServerTimeoutError(`Server ${path} timed out after ${timeoutMs}ms${attempts > 0 ? `; ${attempts} ${attempts === 1 ? "attempt" : "attempts"} failed` : ""}`, cause === undefined ? undefined : { cause });
+        try {
+            for (;;) {
+                const remainingMs = deadlineAtMs - Date.now();
+                if (remainingMs <= 0) {
+                    retryExhausted = retryCause !== undefined;
+                    throw timeoutError();
+                }
+                attempts += 1;
+                try {
+                    const result = await this.send(path, {
+                        ...init,
+                        timeoutMs: remainingMs,
+                    });
+                    if (retryCause) {
+                        recordServerReadRetryOutcome({
+                            cause: retryCause,
+                            operation,
+                            outcome: "recovered",
+                        });
+                    }
+                    return result;
+                }
+                catch (error) {
+                    if (init.signal?.aborted) {
+                        throw error;
+                    }
+                    if (error instanceof ServerTimeoutError) {
+                        retryExhausted = retryCause !== undefined;
+                        throw timeoutError(lastFailure ?? error.cause);
+                    }
+                    lastFailure = error;
+                    const retry = repeatSafe ||
+                        (method === "GET" &&
+                            error instanceof ServerTransportError &&
+                            error.requestNotSent)
+                        ? retries.next(error)
+                        : undefined;
+                    if (retry === undefined || retry === "exhausted") {
+                        retryExhausted = retryCause !== undefined && retry === "exhausted";
+                        throw withAttemptCount(error, attempts);
+                    }
+                    const remainingAfterAttemptMs = deadlineAtMs - Date.now();
+                    if (remainingAfterAttemptMs <= 0) {
+                        retryExhausted = retryCause !== undefined;
+                        throw timeoutError();
+                    }
+                    if (remainingAfterAttemptMs - retry.waitMs < MIN_RETRY_ATTEMPT_MS) {
+                        retryExhausted = retryCause !== undefined;
+                        throw withAttemptCount(error, attempts);
+                    }
+                    retryCause ??= retry.cause;
+                    await delay(retry.waitMs, init.signal);
+                    if (init.signal?.aborted) {
+                        throw init.signal.reason;
+                    }
+                    if (Date.now() >= deadlineAtMs) {
+                        retryExhausted = true;
+                        throw timeoutError();
+                    }
+                }
             }
-            catch (error) {
-                lastError = error;
-                const transient = error instanceof ServerHttpError &&
-                    (error.status >= 500 || error.status === 429);
-                attempt += 1;
-                if (!retryable || !transient || attempt > READ_RETRY_ATTEMPTS) {
-                    throw error;
+        }
+        catch (error) {
+            if (retryCause && !init.signal?.aborted) {
+                if (retryExhausted) {
+                    console.error("gateway server read retries exhausted", {
+                        attempts,
+                        cause: retryCause,
+                        message: error instanceof Error ? error.message : String(error),
+                        operation,
+                        path,
+                    });
                 }
-                await delay(READ_RETRY_BASE_DELAY_MS * attempt);
+                recordServerReadRetryOutcome({
+                    cause: retryCause,
+                    operation,
+                    outcome: retryExhausted ? "exhausted" : "terminal",
+                });
             }
+            throw error;
         }
-        throw lastError;
     }
     async send(path, init) {
-        const { apiKey, headers: callerHeaders, retry: _retry, signal: callerSignal, timeoutMs, ...rest } = init;
-        const bound = timeoutMs ?? this.timeoutMs;
+        const { apiKey, headers: callerHeaders, operation: _operation, retry: _retry, signal: callerSignal, timeoutMs, ...rest } = init;
         const mergedHeaders = new Headers(callerHeaders);
         // Auth and content-type are owned by this client — never override from callers.
         mergedHeaders.set("Authorization", `Bearer ${apiKey}`);
         if (!mergedHeaders.has("Content-Type")) {
             mergedHeaders.set("Content-Type", "application/json");
         }
-        const timeoutSignal = AbortSignal.timeout(bound);
+        const attempt = new ServerAttempt(path, timeoutMs ?? this.timeoutMs, callerSignal);
         let response;
         try {
             response = await fetch(`${this.baseUrl()}${path}`, {
                 ...rest,
                 headers: mergedHeaders,
-                signal: callerSignal
-                    ? AbortSignal.any([callerSignal, timeoutSignal])
-                    : timeoutSignal,
+                signal: attempt.signal,
             });
         }
         catch (error) {
-            if (timeoutSignal.aborted &&
-                error instanceof Error &&
-                error.name === "TimeoutError") {
-                throw new Error(`Server ${path} timed out after ${bound}ms`);
+            throw attempt.failure(error, "could not be reached", true);
+        }
+        let body;
+        try {
+            body = await response.text();
+        }
+        catch (error) {
+            const failure = attempt.failure(error, "dropped the connection mid-response");
+            const retryAfterMs = parseRetryAfterMs(response.headers.get("Retry-After"));
+            if (!response.ok &&
+                !callerSignal?.aborted &&
+                (response.status === 429 ||
+                    !isTransientHttpStatus(response.status) ||
+                    (failure instanceof ServerTransportError && !failure.retryable))) {
+                throw new ServerHttpError(`Server ${path} failed: ${response.status}`, response.status, retryAfterMs);
             }
-            throw error;
+            throw failure;
         }
         if (!response.ok) {
-            throw new ServerHttpError(`Server ${path} failed: ${response.status}${detailSuffix(await response.text().catch(() => ""))}`, response.status);
+            throw new ServerHttpError(`Server ${path} failed: ${response.status}${detailSuffix(body)}`, response.status, parseRetryAfterMs(response.headers.get("Retry-After")));
         }
-        return (await response.json());
+        // A body that arrived whole but cannot be parsed is an answer, not a lost
+        // connection: the route is up and will say the same thing again.
+        return JSON.parse(body);
     }
     /**
      * The Superblocks user behind a downstream credential.
@@ -266,6 +558,7 @@ export class HttpSuperblocksServerClient {
             const data = await this.requestData("/api/v1/users/me", {
                 apiKey,
                 method: "GET",
+                operation: "identity",
                 timeoutMs: this.identityTimeoutMs,
             });
             const user = data.user;
@@ -291,6 +584,7 @@ export class HttpSuperblocksServerClient {
         const data = await this.requestData("/api/v1/users/me", {
             apiKey: input.apiKey,
             method: "GET",
+            operation: "identity",
             // Same route, so the same bound. This one runs on the build path rather
             // than per tool call, and left at the sandbox timeout a wedged
             // /users/me held a start_app open for two minutes here instead.
@@ -317,9 +611,17 @@ export class HttpSuperblocksServerClient {
                 url: raw.url,
             });
         }
+        const factsFlag = data.flagBootstrap?.["ai.facts.enabled"];
+        recordFactsFlagBootstrapOutcome(factsFlag === true
+            ? "enabled"
+            : factsFlag === false
+                ? "disabled"
+                : "invalid");
         return {
             agentType: organization.agentType,
             agents,
+            controlPlaneUrl: this.baseUrl(),
+            isFactsEnabled: factsFlag === true,
         };
     }
     async createApplication(input) {
@@ -384,6 +686,7 @@ export class HttpSuperblocksServerClient {
                 return await this.requestData(`${path}${query}`, {
                     apiKey: input.apiKey,
                     method: "GET",
+                    operation: "live_edit",
                     retry: false,
                     timeoutMs: Math.max(1, deadlineAtMs - Date.now()),
                 });
@@ -502,12 +805,48 @@ export class HttpSuperblocksServerClient {
         if (input.applicationId) {
             params.set("applicationId", input.applicationId);
         }
-        const data = await this.requestData(`/api/v1/integrations/superset?${params.toString()}`, { apiKey: input.apiKey, method: "GET" });
+        const data = await this.requestData(`/api/v1/integrations/superset?${params.toString()}`, {
+            apiKey: input.apiKey,
+            method: "GET",
+            operation: "superset_integrations",
+        });
         return Array.isArray(data) ? data : [];
     }
     async getIntegration(input) {
         try {
-            return await this.requestData(`/api/v1/integrations/${encodeURIComponent(input.integrationId)}`, { apiKey: input.apiKey, method: "GET" });
+            return await this.requestData(`/api/v1/integrations/${encodeURIComponent(input.integrationId)}`, { apiKey: input.apiKey, method: "GET", operation: "integration" });
+        }
+        catch (error) {
+            if (error instanceof ServerHttpError && error.status === 404) {
+                return undefined;
+            }
+            throw error;
+        }
+    }
+    async getApplicationEnvironment(input) {
+        try {
+            const data = await this.requestData(`/api/v2/applications/${encodeURIComponent(input.applicationId)}/environments`, { apiKey: input.apiKey, method: "GET" });
+            const editor = data?.profiles?.editor;
+            const availableProfileIds = Array.isArray(editor?.availableProfileIds)
+                ? editor.availableProfileIds.filter((id) => typeof id === "string")
+                : undefined;
+            return {
+                ...(typeof data?.needsAttention?.editor === "boolean"
+                    ? { needsAttention: { editor: data.needsAttention.editor } }
+                    : {}),
+                profiles: {
+                    ...(availableProfileIds
+                        ? {
+                            editor: {
+                                availableProfileIds,
+                                ...(typeof editor?.defaultProfileId === "string"
+                                    ? { defaultProfileId: editor.defaultProfileId }
+                                    : {}),
+                            },
+                        }
+                        : {}),
+                },
+            };
         }
         catch (error) {
             if (error instanceof ServerHttpError && error.status === 404) {
@@ -517,7 +856,11 @@ export class HttpSuperblocksServerClient {
         }
     }
     async getOrganizationProfiles(input) {
-        const data = await this.requestData(`/api/v1/organizations/${encodeURIComponent(input.organizationId)}/profiles`, { apiKey: input.apiKey, method: "GET" });
+        const data = await this.requestData(`/api/v1/organizations/${encodeURIComponent(input.organizationId)}/profiles`, {
+            apiKey: input.apiKey,
+            method: "GET",
+            operation: "organization_profiles",
+        });
         if (!Array.isArray(data)) {
             return [];
         }
@@ -527,6 +870,9 @@ export class HttpSuperblocksServerClient {
             ...(row.allowedInEditor !== undefined
                 ? { allowedInEditor: row.allowedInEditor }
                 : {}),
+            ...(typeof row.defaultInEditor === "boolean"
+                ? { defaultInEditor: row.defaultInEditor }
+                : {}),
             ...(row.displayName ? { displayName: row.displayName } : {}),
             ...(row.id ? { id: row.id } : {}),
             key: row.key,
@@ -585,6 +931,8 @@ export class HttpSuperblocksServerClient {
             apiKey: input.apiKey,
             body: JSON.stringify({ directoryHashes: [input.directoryHash] }),
             method: "POST",
+            operation: "build_progress",
+            retry: true,
             ...(input.signal ? { signal: input.signal } : {}),
             ...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}),
         });
@@ -669,7 +1017,11 @@ export class HttpSuperblocksServerClient {
      * why nothing on a hot path should call it.
      */
     async getApplicationName(input) {
-        const body = await this.request("/api/v2/applications", { apiKey: input.apiKey, method: "GET" });
+        const body = await this.request("/api/v2/applications", {
+            apiKey: input.apiKey,
+            method: "GET",
+            operation: "applications",
+        });
         const found = body?.data?.applications?.find((application) => application?.id === input.applicationId);
         return found?.name?.trim() || undefined;
     }