@turingfocus/chat-gateway-tfrobot 0.6.0 → 0.7.0

package/dist/socket.js

@@ -1,10 +1,10 @@
 import { io } from "socket.io-client";
-import { chatErrorSchema, chatUpdateSchema, createGatewayDeadlineExceededError, isGatewayDeadlineExceeded, } from "@turingfocus/chat-protocol";
+import { chatErrorSchema, chatUpdateSchema, createGatewayDeadlineExceededError, getTimelineItemKey, isGatewayDeadlineExceeded, } from "@turingfocus/chat-protocol";
 import { awaitBounded } from "./bounded.js";
 import { chatErrorEventDtoSchema, eventDtoSchema, messageDtoSchema, socketProtocolErrorDtoSchema, stateChangedDtoSchema, statusDtoSchema, } from "./dto.js";
 import { mapEventUpdate, mapMessageUpdate, mapRun, mapUnknownSocketEvent, } from "./mapper.js";
 import { sanitizeCredentialError, sanitizeCredentialRaw, sanitizeCredentialText, } from "./redaction.js";
-import { isValidTFRobotSession } from "./types.js";
+import { isValidTFRobotSession, resolveTFRobotServerProfile } from "./types.js";
 const KNOWN_EVENTS = new Set([
     "chat_error",
     "chat_event",
@@ -18,17 +18,21 @@ const KNOWN_EVENTS = new Set([
 const MAX_TIMER_DELAY = 2_147_483_647;
 const RECONNECT_AUTH_TIMEOUT_MS = 10_000;
 const RECONNECT_JOIN_TIMEOUT_MS = 10_000;
-const parseJoinAcknowledgement = (value, reconnect) => {
+const parseJoinAcknowledgement = (value, reconnect, acceptEmpty) => {
     if (value === undefined) {
         return {
-            accepted: reconnect,
+            accepted: reconnect || acceptEmpty,
+            empty: true,
             recoveryComplete: false,
-            ...(reconnect ? {} : { rejectionCode: "validation" }),
+            ...(reconnect || acceptEmpty
+                ? {}
+                : { rejectionCode: "validation" }),
         };
     }
     if (value === null) {
         return {
             accepted: false,
+            empty: false,
             recoveryComplete: false,
             rejectionCode: "validation",
         };
@@ -36,16 +40,18 @@ const parseJoinAcknowledgement = (value, reconnect) => {
     if (value === false) {
         return {
             accepted: false,
+            empty: false,
             recoveryComplete: false,
             rejectionCode: "validation",
         };
     }
     if (value === true) {
-        return { accepted: true, recoveryComplete: !reconnect };
+        return { accepted: true, empty: false, recoveryComplete: !reconnect };
     }
     if (typeof value !== "object") {
         return {
             accepted: false,
+            empty: false,
             recoveryComplete: false,
             rejectionCode: "validation",
         };
@@ -120,6 +126,7 @@ const parseJoinAcknowledgement = (value, reconnect) => {
     const authorizationRejected = statuses.some(({ normalized, numeric }) => numeric === 403 || normalized === "forbidden") || normalizedError === "forbidden";
     return {
         accepted,
+        empty: false,
         ...(authenticationRejected
             ? { rejectionCode: "authentication" }
             : authorizationRejected
@@ -218,27 +225,65 @@ export const createSocketIoFactoryWith = (connect) => ({ getAuth, namespaceUrl,
     },
 });
 export const createSocketIoFactory = createSocketIoFactoryWith(io);
+const MAX_RECOVERY_IDENTITIES = 10_000;
+const rememberBoundedSetValue = (values, value) => {
+    values.delete(value);
+    values.add(value);
+    while (values.size > MAX_RECOVERY_IDENTITIES) {
+        const oldest = values.values().next().value;
+        if (oldest === undefined)
+            break;
+        values.delete(oldest);
+    }
+};
+const rememberBoundedMapValue = (values, key, value) => {
+    values.delete(key);
+    values.set(key, value);
+    while (values.size > MAX_RECOVERY_IDENTITIES) {
+        const oldest = values.keys().next().value;
+        if (oldest === undefined)
+            break;
+        values.delete(oldest);
+    }
+};
+export const recoveryIdentityOfUpdate = (update) => {
+    if (update.kind === "timeline.upsert") {
+        return getTimelineItemKey(update.item);
+    }
+    if (update.kind === "event.transition.upsert") {
+        return `event:${update.event.id}:transition:${update.event.transition.id}`;
+    }
+    return undefined;
+};
+const recoveryIdentitiesOfSnapshot = (snapshot) => snapshot.timeline.flatMap((item) => item.kind === "agent-event"
+    ? item.transitions.map((transition) => `event:${item.id}:transition:${transition.id}`)
+    : [getTimelineItemKey(item)]);
 export class TFRobotSocketClient {
     #disposed = false;
-    #establishmentAbort;
     #establishing;
+    #pendingEstablishment;
     #factory;
     #namespaceUrl;
     #now;
     #options;
     #path;
+    #profile;
+    #currentServerRestLoader;
     #reconnectStatusLoader;
+    #recoveryCheckpoints = new Map();
     #runs = new Map();
     #subscriptions = new Set();
     #subscriptionGeneration = 0;
-    constructor(options, reconnectStatusLoader) {
+    constructor(options, reconnectStatusLoader, currentServerRestLoader) {
         this.#options = options;
         this.#factory = options.socketFactory ?? createSocketIoFactory;
         this.#path = options.socketPath ?? "/socket.io";
         this.#now = options.now ?? Date.now;
+        this.#profile = resolveTFRobotServerProfile(options.serverProfile);
         this.#namespaceUrl =
             options.socketNamespaceUrl ?? this.#defaultNamespaceUrl(options.baseUrl);
         this.#reconnectStatusLoader = reconnectStatusLoader;
+        this.#currentServerRestLoader = currentServerRestLoader;
     }
     async subscribe(conversationId, options, observer) {
         if (this.#disposed) {
@@ -257,14 +302,25 @@ export class TFRobotSocketClient {
         const subscriptionId = `tfrobot-subscription-${generation}`;
         this.#cancelEstablishment();
         const establishmentAbort = new AbortController();
-        this.#establishmentAbort = establishmentAbort;
-        const authOutcome = await awaitBounded(() => this.#getSocketAuth(conversationId, "connect"), {
+        const pendingEstablishment = {
+            abort: establishmentAbort,
+            conversationId,
+            generation,
+        };
+        this.#pendingEstablishment = pendingEstablishment;
+        const clearPendingEstablishment = () => {
+            if (this.#pendingEstablishment === pendingEstablishment) {
+                this.#pendingEstablishment = undefined;
+            }
+        };
+        const authOutcome = await awaitBounded(() => this.#getSocketSession(conversationId, "connect"), {
             deadlineAt: options.deadlineAt,
             now: this.#now,
             signal: establishmentAbort.signal,
         });
         switch (authOutcome.kind) {
             case "aborted": {
+                clearPendingEstablishment();
                 return {
                     ok: false,
                     error: this.#error("conflict", this.#disposed
@@ -273,12 +329,14 @@ export class TFRobotSocketClient {
                 };
             }
             case "deadline": {
+                clearPendingEstablishment();
                 return {
                     ok: false,
                     error: createGatewayDeadlineExceededError(),
                 };
             }
             case "error": {
+                clearPendingEstablishment();
                 return {
                     ok: false,
                     error: this.#error("authentication", "Unable to obtain a TFRobot Socket session", true, undefined),
@@ -289,14 +347,43 @@ export class TFRobotSocketClient {
             }
         }
         if (this.#disposed || establishmentAbort.signal.aborted) {
+            clearPendingEstablishment();
             return {
                 ok: false,
                 error: this.#error("conflict", "TFRobot Gateway was disposed during subscription", false, undefined),
             };
         }
-        const credentialValues = new Set(Object.values(authOutcome.value));
+        const initialSession = authOutcome.value;
+        if (this.#profile.kind === "current-server") {
+            const preflight = await this.#currentServerRestLoader({
+                checkpoint: new Set(),
+                conversationId,
+                deadlineAt: options.deadlineAt,
+                limits: {
+                    ...this.#profile.rebase,
+                    maxItems: this.#profile.rebase.pageSize,
+                    maxPages: 1,
+                },
+                session: initialSession,
+                signal: establishmentAbort.signal,
+            });
+            if (this.#disposed || establishmentAbort.signal.aborted) {
+                clearPendingEstablishment();
+                return {
+                    ok: false,
+                    error: this.#error("conflict", "TFRobot subscription was replaced during REST preflight", false, undefined),
+                };
+            }
+            if (!preflight.ok) {
+                clearPendingEstablishment();
+                return preflight;
+            }
+        }
+        const initialAuth = authOf(initialSession);
+        const credentialValues = new Set(Object.values(initialAuth));
         const errorConversationId = () => sanitizeCredentialText(conversationId, credentialValues);
-        let firstAuth = authOutcome.value;
+        let firstAuth = initialAuth;
+        let currentServerSession = initialSession;
         let authenticationFailure;
         const connectionAbort = new AbortController();
         let socket;
@@ -313,20 +400,22 @@ export class TFRobotSocketClient {
                         firstAuth = undefined;
                         return auth;
                     }
-                    const reconnectAuth = await awaitBounded(() => this.#getSocketAuth(conversationId, "reconnect"), {
+                    const reconnectSession = await awaitBounded(() => this.#getSocketSession(conversationId, "reconnect"), {
                         deadlineAt: this.#now() + RECONNECT_AUTH_TIMEOUT_MS,
                         now: this.#now,
                         signal: connectionAbort.signal,
                     });
-                    if (reconnectAuth.kind === "value") {
-                        for (const value of Object.values(reconnectAuth.value)) {
+                    if (reconnectSession.kind === "value") {
+                        currentServerSession = reconnectSession.value;
+                        const reconnectAuth = authOf(reconnectSession.value);
+                        for (const value of Object.values(reconnectAuth)) {
                             credentialValues.add(value);
                         }
-                        return reconnectAuth.value;
+                        return reconnectAuth;
                     }
-                    const reason = reconnectAuth.kind === "error"
-                        ? reconnectAuth.reason
-                        : new Error(reconnectAuth.kind === "deadline"
+                    const reason = reconnectSession.kind === "error"
+                        ? reconnectSession.reason
+                        : new Error(reconnectSession.kind === "deadline"
                             ? "TFRobot Socket session refresh exceeded its deadline"
                             : "TFRobot Socket session refresh was cancelled");
                     authenticationFailure = reason;
@@ -336,6 +425,7 @@ export class TFRobotSocketClient {
         }
         catch (reason) {
             connectionAbort.abort();
+            clearPendingEstablishment();
             return {
                 ok: false,
                 error: this.#transportError(reason, "Unable to create the TFRobot Socket transport", conversationId, credentialValues),
@@ -496,12 +586,18 @@ export class TFRobotSocketClient {
                         ? {}
                         : {
                             recoveryComplete: recovery.complete,
+                            ...(recovery.assurance === undefined
+                                ? {}
+                                : { recoveryAssurance: recovery.assurance }),
                             ...(recovery.cursor === undefined
                                 ? {}
                                 : { recoveryCursor: recovery.cursor }),
                             ...(recovery.reason === undefined
                                 ? {}
                                 : { recoveryReason: recovery.reason }),
+                            ...(recovery.source === undefined
+                                ? {}
+                                : { recoverySource: recovery.source }),
                         }),
                 })).catch(() => undefined);
             }
@@ -533,8 +629,74 @@ export class TFRobotSocketClient {
                 report(this.#error("validation", invalidMessage, false, errorConversationId()));
                 return;
             }
+            const recoveryIdentity = recoveryIdentityOfUpdate(update);
+            if (recoveryIdentity !== undefined) {
+                active.realtimeRevision += 1;
+                rememberBoundedMapValue(active.realtimeIdentityRevisions, recoveryIdentity, active.realtimeRevision);
+                const checkpoint = this.#recoveryCheckpoints.get(conversationId) ?? new Set();
+                rememberBoundedSetValue(checkpoint, recoveryIdentity);
+                this.#recoveryCheckpoints.set(conversationId, checkpoint);
+            }
             next(update);
         };
+        const rebaseCurrentServer = async (recoveryRevision, runRevision, session, checkpoint, realtimeRevision) => {
+            if (this.#profile.kind !== "current-server") {
+                return { kind: "failure" };
+            }
+            const result = await this.#currentServerRestLoader({
+                checkpoint,
+                conversationId,
+                deadlineAt: this.#now() + this.#profile.rebase.deadlineMs,
+                limits: this.#profile.rebase,
+                session,
+                signal: connectionAbort.signal,
+            });
+            if (!active.active ||
+                !this.#subscriptions.has(active) ||
+                !active.socket.connected ||
+                active.recoveryRevision !== recoveryRevision) {
+                return { kind: "failure" };
+            }
+            if (!result.ok) {
+                const error = sanitizeCredentialError(result.error, credentialValues);
+                if (error.code === "authentication" || error.code === "authorization") {
+                    report(error, "authentication");
+                    return { kind: "auth-required" };
+                }
+                report(error, "recovery");
+                return { kind: "failure" };
+            }
+            const remembered = this.#recoveryCheckpoints.get(conversationId) ?? new Set();
+            for (const update of result.value.updates) {
+                const identity = recoveryIdentityOfUpdate(update);
+                if (identity !== undefined &&
+                    (active.realtimeIdentityRevisions.get(identity) ?? 0) >
+                        realtimeRevision) {
+                    continue;
+                }
+                next(update);
+                if (identity !== undefined)
+                    rememberBoundedSetValue(remembered, identity);
+            }
+            this.#recoveryCheckpoints.set(conversationId, remembered);
+            if (active.runRevision === runRevision) {
+                const previous = this.#runs.get(conversationId);
+                this.#runs.set(conversationId, result.value.run);
+                if (!sameRun(previous, result.value.run)) {
+                    next(chatUpdateSchema.parse({
+                        kind: "run.replace",
+                        conversationId: errorConversationId(),
+                        run: result.value.run,
+                    }));
+                }
+            }
+            const reason = result.value.checkpointReached
+                ? "rest-rebase-reached-checkpoint"
+                : result.value.boundedBy === undefined
+                    ? "rest-rebase-history-exhausted"
+                    : `rest-rebase-${result.value.boundedBy}`;
+            return { kind: "success", reason };
+        };
         let settleEstablishment;
         add("connect", () => {
             if (!active.active)
@@ -563,6 +725,8 @@ export class TFRobotSocketClient {
             const reconnect = subscriptionEstablished;
             if (reconnect)
                 active.reconnectAttempt += 1;
+            const rebaseCheckpoint = new Set(this.#recoveryCheckpoints.get(conversationId) ?? []);
+            const rebaseRealtimeRevision = active.realtimeRevision;
             const recoveryRevision = ++active.recoveryRevision;
             const runRevision = active.runRevision;
             const joinStartedAt = this.#now();
@@ -589,11 +753,12 @@ export class TFRobotSocketClient {
                     const rawAcknowledgement = acknowledgementArguments[0];
                     acknowledgement = parseJoinAcknowledgement(rawAcknowledgement === undefined || rawAcknowledgement === true
                         ? rawAcknowledgement
-                        : sanitizeCredentialRaw(rawAcknowledgement, credentialValues), reconnect);
+                        : sanitizeCredentialRaw(rawAcknowledgement, credentialValues), reconnect, this.#profile.kind === "current-server");
                 }
                 catch {
                     acknowledgement = {
                         accepted: false,
+                        empty: false,
                         recoveryComplete: false,
                         rejectionCode: "validation",
                         message: "Invalid TFRobot conversation subscription acknowledgement",
@@ -661,12 +826,24 @@ export class TFRobotSocketClient {
                     });
                     resolveError("connection");
                     resolveError("authentication");
-                    lifecycle("active", {
-                        complete: true,
-                        ...(acknowledgement.cursor === undefined
-                            ? {}
-                            : { cursor: acknowledgement.cursor }),
-                    }, this.#now() - joinStartedAt);
+                    if (this.#profile.kind === "current-server" &&
+                        acknowledgement.empty) {
+                        lifecycle("degraded", {
+                            assurance: "best-effort",
+                            complete: false,
+                            reason: "initial-rest-preflight",
+                            source: "rest-rebase",
+                        }, this.#now() - joinStartedAt);
+                    }
+                    else {
+                        lifecycle("active", {
+                            complete: true,
+                            ...(acknowledgement.cursor === undefined
+                                ? {}
+                                : { cursor: acknowledgement.cursor }),
+                        }, this.#now() - joinStartedAt);
+                    }
+                    currentServerSession = undefined;
                     active.acceptingEvents = true;
                     flushRealtime();
                     return;
@@ -684,6 +861,48 @@ export class TFRobotSocketClient {
                 }, joinLatencyMs);
                 active.acceptingEvents = true;
                 flushRealtime();
+                if (this.#profile.kind === "current-server" &&
+                    !acknowledgement.recoveryComplete) {
+                    const recoverySession = currentServerSession;
+                    currentServerSession = undefined;
+                    if (recoverySession === undefined) {
+                        report(this.#error("authentication", "TFRobot reconnect did not provide a session for REST rebase", true, errorConversationId()), "recovery");
+                        return;
+                    }
+                    void rebaseCurrentServer(recoveryRevision, runRevision, recoverySession, rebaseCheckpoint, rebaseRealtimeRevision)
+                        .then((outcome) => {
+                        if (!active.active ||
+                            !this.#subscriptions.has(active) ||
+                            !active.socket.connected ||
+                            active.recoveryRevision !== recoveryRevision) {
+                            return;
+                        }
+                        if (outcome.kind === "auth-required") {
+                            enterAuthRequired();
+                            return;
+                        }
+                        if (outcome.kind !== "success")
+                            return;
+                        resolveError("connection");
+                        resolveError("recovery");
+                        resolveError("authentication");
+                        active.manualReconnectAttempted = false;
+                        active.manualReconnectPending = false;
+                        active.authRequired = false;
+                        active.terminal = false;
+                        lifecycle("degraded", {
+                            assurance: "best-effort",
+                            complete: false,
+                            reason: outcome.reason,
+                            source: "rest-rebase",
+                        }, joinLatencyMs);
+                    })
+                        .catch(() => {
+                        report(this.#error("unknown", "TFRobot REST rebase failed unexpectedly", true, errorConversationId()), "recovery");
+                    });
+                    return;
+                }
+                currentServerSession = undefined;
                 void this.#reconcileRunAfterReconnect(active, conversationId, credentialValues, recoveryRevision, runRevision, next, report)
                     .then((outcome) => {
                     if (outcome === "auth-required") {
@@ -987,6 +1206,7 @@ export class TFRobotSocketClient {
                 // Adapter ownership is cleared even if an injected transport misbehaves.
             }
             firstAuth = undefined;
+            currentServerSession = undefined;
             credentialValues.clear();
             pendingNotifications.length = 0;
             pendingRealtimeActions.length = 0;
@@ -994,6 +1214,7 @@ export class TFRobotSocketClient {
         if (setupFailure !== undefined) {
             const error = this.#transportError(setupFailure.reason, "Unable to configure the TFRobot Socket transport", conversationId, credentialValues);
             cleanup();
+            clearPendingEstablishment();
             return {
                 ok: false,
                 error,
@@ -1016,6 +1237,8 @@ export class TFRobotSocketClient {
             manualReconnectPending: false,
             observer,
             reconnectAttempt: 0,
+            realtimeIdentityRevisions: new Map(),
+            realtimeRevision: 0,
             recoveryRevision: 0,
             runRevision: 0,
             socket,
@@ -1031,9 +1254,7 @@ export class TFRobotSocketClient {
                     return;
                 settled = true;
                 settleEstablishment = undefined;
-                if (this.#establishmentAbort === establishmentAbort) {
-                    this.#establishmentAbort = undefined;
-                }
+                clearPendingEstablishment();
                 if (this.#establishing === active)
                     this.#establishing = undefined;
                 clearTimeout(timeout);
@@ -1087,16 +1308,46 @@ export class TFRobotSocketClient {
             subscription.cleanup();
         }
         this.#subscriptions.clear();
+        this.#recoveryCheckpoints.clear();
         this.#runs.clear();
     }
+    rememberSnapshot(snapshot) {
+        if (this.#disposed)
+            return;
+        const conversationId = snapshot.conversation.id;
+        this.#runs.set(conversationId, snapshot.run);
+        const checkpoint = this.#recoveryCheckpoints.get(conversationId) ?? new Set();
+        for (const identity of recoveryIdentitiesOfSnapshot(snapshot)) {
+            rememberBoundedSetValue(checkpoint, identity);
+        }
+        this.#recoveryCheckpoints.set(conversationId, checkpoint);
+    }
     rememberRun(conversationId, run) {
         if (this.#disposed)
             return;
         this.#runs.set(conversationId, run);
     }
+    forgetConversation(conversationId) {
+        if (this.#disposed)
+            return;
+        if (this.#pendingEstablishment?.conversationId === conversationId ||
+            this.#establishing?.conversationId === conversationId) {
+            this.#cancelEstablishment();
+        }
+        for (const subscription of [...this.#subscriptions]) {
+            if (subscription.conversationId !== conversationId)
+                continue;
+            subscription.active = false;
+            subscription.recoveryRevision += 1;
+            subscription.cleanup();
+            this.#subscriptions.delete(subscription);
+        }
+        this.#recoveryCheckpoints.delete(conversationId);
+        this.#runs.delete(conversationId);
+    }
     #cancelEstablishment() {
-        this.#establishmentAbort?.abort();
-        this.#establishmentAbort = undefined;
+        this.#pendingEstablishment?.abort.abort();
+        this.#pendingEstablishment = undefined;
         this.#establishing?.cancelEstablishment();
         this.#establishing = undefined;
     }
@@ -1107,7 +1358,7 @@ export class TFRobotSocketClient {
         url.hash = "";
         return url.toString().replace(/\/$/u, "");
     }
-    async #getSocketAuth(conversationId, purpose) {
+    async #getSocketSession(conversationId, purpose) {
         const session = await this.#options.sessionProvider.getSession({
             purpose,
             operation: "subscribe",
@@ -1116,7 +1367,7 @@ export class TFRobotSocketClient {
         if (!isValidTFRobotSession(session)) {
             throw new TypeError("SessionProvider returned invalid TFRobot credentials");
         }
-        return authOf(session);
+        return session;
     }
     async #reconcileRunAfterReconnect(active, conversationId, credentialValues, recoveryRevision, runRevision, next, report) {
         const result = await this.#reconnectStatusLoader(conversationId);