@xfey/tutti 0.1.29 → 0.1.30

package/dist/server-shell/cli/host-relay-connection-manager.js

@@ -0,0 +1,447 @@
+import { RelayHostControlClientError } from "@tutti/relay-client";
+import { LaunchError } from "./errors.js";
+import { inspectMachineProjectInventory, normalizeMachineWorkspacePath, } from "./machine-project-inspector.js";
+class WorkspaceUnavailableError extends Error {
+    workspaceState;
+    constructor(workspaceState) {
+        super(`Workspace is ${workspaceState}`);
+        this.workspaceState = workspaceState;
+        this.name = "WorkspaceUnavailableError";
+    }
+}
+const DEFAULT_RETRY_DELAYS_MS = [0, 1_000, 2_000, 5_000, 10_000, 30_000];
+const DEFAULT_JITTER_RATIO = 0.2;
+const DEFAULT_WORKSPACE_CHECK_INTERVAL_MS = 30_000;
+function mergeKnownJoinUrl(previous, next) {
+    if (next.join_url !== undefined ||
+        previous?.join_url === undefined ||
+        previous.join_token_expires_at !== next.join_token_expires_at) {
+        return next;
+    }
+    return {
+        ...next,
+        join_url: previous.join_url,
+        join_url_visibility: previous.join_url_visibility,
+    };
+}
+function retryableConnectionError(error) {
+    if (error instanceof RelayHostControlClientError) {
+        return error.retryable;
+    }
+    if (error instanceof LaunchError) {
+        return false;
+    }
+    return true;
+}
+function connectionErrorCode(error) {
+    if (error instanceof RelayHostControlClientError) {
+        return error.relayErrorCode ?? error.code;
+    }
+    if (error instanceof LaunchError) {
+        return error.code;
+    }
+    if (error instanceof Error && error.name.trim() !== "") {
+        return error.name.slice(0, 80);
+    }
+    return "connection_failed";
+}
+class DefaultHostRelayConnectionManager {
+    options;
+    state = {
+        status: "unavailable",
+        reason_code: "not_started",
+    };
+    retryDelaysMs;
+    jitterRatio;
+    workspaceCheckIntervalMs;
+    now;
+    random;
+    normalizedWorkspaceRoot;
+    scheduledTimer;
+    activeTunnel;
+    pendingTunnel;
+    attemptController;
+    attemptPromise;
+    startPromise;
+    reconnectPromise;
+    refreshPromise;
+    lastConnection;
+    reconnectFailureCount = 0;
+    stopping = false;
+    constructor(options) {
+        this.options = options;
+        this.retryDelaysMs =
+            options.retryDelaysMs === undefined || options.retryDelaysMs.length === 0
+                ? DEFAULT_RETRY_DELAYS_MS
+                : options.retryDelaysMs;
+        this.jitterRatio = Math.max(0, Math.min(options.jitterRatio ?? DEFAULT_JITTER_RATIO, 1));
+        this.workspaceCheckIntervalMs =
+            options.workspaceCheckIntervalMs ?? DEFAULT_WORKSPACE_CHECK_INTERVAL_MS;
+        this.now = options.now ?? (() => new Date());
+        this.random = options.random ?? Math.random;
+        this.normalizedWorkspaceRoot = normalizeMachineWorkspacePath(options.workspaceRoot);
+    }
+    getState() {
+        return this.state;
+    }
+    async start() {
+        if (this.state.status === "connected") {
+            return this.state.connection;
+        }
+        if (this.stopping || this.state.status === "stopped") {
+            throw new LaunchError("relay_registration_failed", "Relay connection manager is stopped", "Start a new Tutti host process before reconnecting.");
+        }
+        if (this.startPromise !== undefined) {
+            return await this.startPromise;
+        }
+        const promise = (async () => {
+            try {
+                return await this.connectOnce("rotate", true);
+            }
+            catch (error) {
+                this.handleInitialFailure(error);
+                throw error;
+            }
+        })();
+        this.startPromise = promise;
+        void promise.then(() => {
+            if (this.startPromise === promise) {
+                this.startPromise = undefined;
+            }
+        }, () => {
+            if (this.startPromise === promise) {
+                this.startPromise = undefined;
+            }
+        });
+        return await promise;
+    }
+    async reconnectNow() {
+        if (this.state.status === "connected" || this.state.status === "stopped") {
+            return this.state;
+        }
+        this.clearScheduledTimer();
+        await this.beginReconnect();
+        return this.state;
+    }
+    async refreshJoinUrl() {
+        if (this.refreshPromise !== undefined) {
+            return await this.refreshPromise;
+        }
+        if (this.options.refreshJoinUrl === undefined ||
+            this.state.status !== "connected" ||
+            this.activeTunnel === undefined) {
+            throw new LaunchError("relay_registration_failed", "Relay is not connected, so the join URL cannot be refreshed", "Wait for Relay reconnection, then retry `tutti invite`.");
+        }
+        const connection = this.state.connection;
+        const promise = (async () => {
+            const refreshed = await this.options.refreshJoinUrl?.(connection);
+            if (refreshed === undefined) {
+                throw new LaunchError("relay_registration_failed", "Relay join URL refresh is unavailable", "Retry after the Relay connection is restored.");
+            }
+            if (this.state.status === "connected" &&
+                this.state.connection.host_connection_ref === connection.host_connection_ref) {
+                this.lastConnection = refreshed;
+                this.setState({ status: "connected", connection: refreshed });
+            }
+            return refreshed;
+        })();
+        this.refreshPromise = promise;
+        void promise.then(() => {
+            if (this.refreshPromise === promise) {
+                this.refreshPromise = undefined;
+            }
+        }, () => {
+            if (this.refreshPromise === promise) {
+                this.refreshPromise = undefined;
+            }
+        });
+        return await promise;
+    }
+    async stop() {
+        if (this.state.status === "stopped") {
+            return;
+        }
+        this.stopping = true;
+        this.clearScheduledTimer();
+        this.attemptController?.abort();
+        const pendingTunnel = this.pendingTunnel;
+        const activeTunnel = this.activeTunnel;
+        this.pendingTunnel = undefined;
+        this.activeTunnel = undefined;
+        pendingTunnel?.close(1000, "host stopping");
+        if (activeTunnel !== pendingTunnel) {
+            activeTunnel?.close(1000, "host stopping");
+        }
+        this.setState({ status: "stopped" });
+        const pending = [];
+        if (this.startPromise !== undefined) {
+            pending.push(this.startPromise);
+        }
+        if (this.reconnectPromise !== undefined) {
+            pending.push(this.reconnectPromise);
+        }
+        if (this.attemptPromise !== undefined) {
+            pending.push(this.attemptPromise);
+        }
+        await Promise.allSettled(pending);
+    }
+    readWorkspaceAvailability() {
+        const inspection = inspectMachineProjectInventory(this.options.projectId, this.options.tuttiHome);
+        if (inspection.workspace.kind === "missing") {
+            return "missing";
+        }
+        if (inspection.normalized_workspace_root !== this.normalizedWorkspaceRoot) {
+            return "replaced";
+        }
+        if (inspection.workspace.kind === "matching") {
+            return "matching";
+        }
+        if (inspection.workspace.kind === "replaced" ||
+            inspection.workspace.kind === "not_a_tutti_project") {
+            return "replaced";
+        }
+        return "unreadable";
+    }
+    connectOnce(joinTokenMode, initial) {
+        if (this.attemptPromise !== undefined) {
+            return this.attemptPromise;
+        }
+        const promise = this.performConnectionAttempt(joinTokenMode, initial);
+        this.attemptPromise = promise;
+        void promise.then(() => {
+            if (this.attemptPromise === promise) {
+                this.attemptPromise = undefined;
+            }
+        }, () => {
+            if (this.attemptPromise === promise) {
+                this.attemptPromise = undefined;
+            }
+        });
+        return promise;
+    }
+    async performConnectionAttempt(joinTokenMode, initial) {
+        const workspaceAvailability = this.readWorkspaceAvailability();
+        if (workspaceAvailability !== "matching") {
+            throw new WorkspaceUnavailableError(workspaceAvailability);
+        }
+        this.clearScheduledTimer();
+        if (initial) {
+            this.setState({ status: "connecting", attempt: 0 });
+        }
+        else {
+            this.setState(this.reconnectingState());
+        }
+        const controller = new AbortController();
+        this.attemptController = controller;
+        let tunnel;
+        try {
+            const registered = await this.options.registerConnection({
+                joinTokenMode,
+                signal: controller.signal,
+            });
+            if (this.stopping || controller.signal.aborted) {
+                throw new Error("Relay connection attempt was cancelled");
+            }
+            tunnel = this.options.connectTunnel(registered);
+            this.pendingTunnel = tunnel;
+            await tunnel.connected;
+            if (this.stopping || controller.signal.aborted) {
+                tunnel.close(1000, "host stopping");
+                throw new Error("Relay connection attempt was cancelled");
+            }
+            const connection = mergeKnownJoinUrl(this.lastConnection, registered.summary);
+            const previousTunnel = this.activeTunnel;
+            this.pendingTunnel = undefined;
+            this.activeTunnel = tunnel;
+            this.lastConnection = connection;
+            this.reconnectFailureCount = 0;
+            this.setState({ status: "connected", connection });
+            this.watchTunnel(tunnel);
+            this.scheduleWorkspaceGuard();
+            if (previousTunnel !== undefined && previousTunnel !== tunnel) {
+                previousTunnel.close(1000, "connection replaced");
+            }
+            return connection;
+        }
+        catch (error) {
+            if (tunnel !== undefined && this.activeTunnel !== tunnel) {
+                tunnel.close(1000, "connection attempt failed");
+            }
+            throw error;
+        }
+        finally {
+            if (this.pendingTunnel === tunnel) {
+                this.pendingTunnel = undefined;
+            }
+            if (this.attemptController === controller) {
+                this.attemptController = undefined;
+            }
+        }
+    }
+    watchTunnel(tunnel) {
+        void tunnel.closed.then(() => {
+            if (this.stopping || this.activeTunnel !== tunnel) {
+                return;
+            }
+            this.activeTunnel = undefined;
+            this.clearScheduledTimer();
+            this.scheduleReconnect();
+        });
+    }
+    beginReconnect() {
+        if (this.reconnectPromise !== undefined) {
+            return this.reconnectPromise;
+        }
+        if (this.stopping || this.state.status === "stopped" || this.state.status === "connected") {
+            return Promise.resolve();
+        }
+        const promise = (async () => {
+            try {
+                await this.connectOnce("reuse_active", false);
+            }
+            catch (error) {
+                this.handleReconnectFailure(error);
+            }
+        })();
+        this.reconnectPromise = promise;
+        void promise.finally(() => {
+            if (this.reconnectPromise === promise) {
+                this.reconnectPromise = undefined;
+            }
+        });
+        return promise;
+    }
+    handleInitialFailure(error) {
+        if (this.stopping) {
+            return;
+        }
+        if (error instanceof WorkspaceUnavailableError) {
+            this.setWorkspaceUnavailable(error.workspaceState, false);
+            return;
+        }
+        this.setState({
+            status: "unavailable",
+            reason_code: retryableConnectionError(error) ? "relay_unavailable" : "fatal_error",
+            error_code: connectionErrorCode(error),
+        });
+    }
+    handleReconnectFailure(error) {
+        if (this.stopping) {
+            return;
+        }
+        if (error instanceof WorkspaceUnavailableError) {
+            this.setWorkspaceUnavailable(error.workspaceState, true);
+            return;
+        }
+        if (!retryableConnectionError(error)) {
+            this.setState({
+                status: "unavailable",
+                reason_code: "fatal_error",
+                error_code: connectionErrorCode(error),
+                ...(this.lastConnection === undefined ? {} : { last_connection: this.lastConnection }),
+            });
+            return;
+        }
+        this.reconnectFailureCount += 1;
+        this.scheduleReconnect();
+    }
+    setWorkspaceUnavailable(workspaceState, allowMissingRecheck) {
+        const reasonCode = workspaceState === "missing"
+            ? "workspace_missing"
+            : workspaceState === "replaced"
+                ? "workspace_replaced"
+                : "workspace_unreadable";
+        if (workspaceState === "missing" && allowMissingRecheck) {
+            const nextRetryAt = new Date(this.now().getTime() + this.workspaceCheckIntervalMs).toISOString();
+            this.setState({
+                status: "unavailable",
+                reason_code: reasonCode,
+                next_retry_at: nextRetryAt,
+                ...(this.lastConnection === undefined ? {} : { last_connection: this.lastConnection }),
+            });
+            this.scheduleTimer(this.workspaceCheckIntervalMs, () => {
+                void this.beginReconnect();
+            });
+            return;
+        }
+        this.setState({
+            status: "unavailable",
+            reason_code: reasonCode,
+            ...(this.lastConnection === undefined ? {} : { last_connection: this.lastConnection }),
+        });
+    }
+    scheduleReconnect() {
+        if (this.stopping) {
+            return;
+        }
+        const index = Math.min(this.reconnectFailureCount, this.retryDelaysMs.length - 1);
+        const baseDelay = this.retryDelaysMs[index] ?? 0;
+        const jitter = baseDelay === 0 ? 0 : Math.round(baseDelay * this.jitterRatio * (this.random() * 2 - 1));
+        const delay = Math.max(0, baseDelay + jitter);
+        const nextRetryAt = new Date(this.now().getTime() + delay).toISOString();
+        this.setState({
+            ...this.reconnectingState(),
+            next_retry_at: nextRetryAt,
+        });
+        this.scheduleTimer(delay, () => {
+            void this.beginReconnect();
+        });
+    }
+    reconnectingState() {
+        return {
+            status: "reconnecting",
+            attempt: this.reconnectFailureCount + 1,
+            ...(this.lastConnection === undefined
+                ? {}
+                : {
+                    last_connected_at: this.lastConnection.connected_at,
+                    last_connection: this.lastConnection,
+                }),
+        };
+    }
+    scheduleWorkspaceGuard() {
+        this.scheduleTimer(this.workspaceCheckIntervalMs, () => {
+            this.runWorkspaceGuard();
+        });
+    }
+    runWorkspaceGuard() {
+        if (this.stopping || this.state.status !== "connected") {
+            return;
+        }
+        const workspaceAvailability = this.readWorkspaceAvailability();
+        if (workspaceAvailability === "matching") {
+            this.scheduleWorkspaceGuard();
+            return;
+        }
+        const tunnel = this.activeTunnel;
+        this.activeTunnel = undefined;
+        tunnel?.close(1000, "workspace unavailable");
+        this.setWorkspaceUnavailable(workspaceAvailability, true);
+    }
+    scheduleTimer(delayMs, callback) {
+        this.clearScheduledTimer();
+        this.scheduledTimer = setTimeout(() => {
+            this.scheduledTimer = undefined;
+            callback();
+        }, delayMs);
+    }
+    clearScheduledTimer() {
+        if (this.scheduledTimer !== undefined) {
+            clearTimeout(this.scheduledTimer);
+            this.scheduledTimer = undefined;
+        }
+    }
+    setState(state) {
+        this.state = state;
+        try {
+            this.options.onStateChange?.(state);
+        }
+        catch {
+            // State observation must not interrupt the connection lifecycle.
+        }
+    }
+}
+export function createHostRelayConnectionManager(options) {
+    return new DefaultHostRelayConnectionManager(options);
+}
+//# sourceMappingURL=host-relay-connection-manager.js.map

package/dist/server-shell/cli/host-relay-status.d.ts

@@ -1,6 +1,10 @@
+import { type ProjectId } from "@tutti/shared/ids";
 import type { HostLocalLaunchStatus } from "../http/routes/local-control.js";
 import type { HostWorkspaceRelayState } from "../http/routes/project-api.js";
+import type { HostRelayConnectionManagerState } from "./host-relay-connection-manager.js";
 import type { HostLaunchRelaySummary } from "./relay-registration.js";
 export declare function relayStateFromLaunchStatus(relayUrl: string, status: HostLocalLaunchStatus | undefined): HostWorkspaceRelayState;
+export declare function relayStateFromConnectionManagerState(relayUrl: string, state: HostRelayConnectionManagerState): HostWorkspaceRelayState;
+export declare function hostLocalLaunchStatusFromConnectionManagerState(projectId: ProjectId, state: HostRelayConnectionManagerState): HostLocalLaunchStatus;
 export declare function relayStatusFromLocalStatus(status: HostLocalLaunchStatus): HostLaunchRelaySummary | undefined;
 //# sourceMappingURL=host-relay-status.d.ts.map

package/dist/server-shell/cli/host-relay-status.js

@@ -1,22 +1,81 @@
-import { ID_PREFIXES, isPrefixedId, } from "@tutti/shared/ids";
+import { ID_PREFIXES, isPrefixedId } from "@tutti/shared/ids";
+function relayProjection(relayUrl, status, relay) {
+    return {
+        status,
+        relay_url: relayUrl,
+        ...(relay === undefined
+            ? {}
+            : {
+                relay_project_ref: relay.relay_project_ref,
+                ...(relay.join_url === undefined ? {} : { join_url: relay.join_url }),
+                join_url_visibility: relay.join_url_visibility,
+                ...(relay.join_token_expires_at === undefined
+                    ? {}
+                    : { join_token_expires_at: relay.join_token_expires_at }),
+            }),
+        join_token_reusable: relay?.join_token_reusable ?? true,
+    };
+}
 export function relayStateFromLaunchStatus(relayUrl, status) {
+    const connectionStatus = status?.relay_connection?.status;
+    if (connectionStatus === "connecting" || connectionStatus === "reconnecting") {
+        return relayProjection(relayUrl, "reconnecting", status?.relay);
+    }
+    if (connectionStatus === "unavailable" || connectionStatus === "stopped") {
+        return relayProjection(relayUrl, "unavailable", status?.relay);
+    }
     if (status?.relay === undefined) {
-        return {
-            status: "unavailable",
-            relay_url: relayUrl,
-            join_token_reusable: true,
-        };
+        return relayProjection(relayUrl, "unavailable", undefined);
+    }
+    return relayProjection(relayUrl, "connected", status.relay);
+}
+export function relayStateFromConnectionManagerState(relayUrl, state) {
+    if (state.status === "connected") {
+        return relayProjection(relayUrl, "connected", state.connection);
+    }
+    if (state.status === "connecting" || state.status === "reconnecting") {
+        return relayProjection(relayUrl, "reconnecting", state.status === "reconnecting" ? state.last_connection : undefined);
     }
+    return relayProjection(relayUrl, "unavailable", state.status === "unavailable" ? state.last_connection : undefined);
+}
+export function hostLocalLaunchStatusFromConnectionManagerState(projectId, state) {
+    const relay = state.status === "connected"
+        ? state.connection
+        : state.status === "reconnecting" || state.status === "unavailable"
+            ? state.last_connection
+            : undefined;
+    const connection = state.status === "connected"
+        ? {
+            status: "connected",
+            last_connected_at: state.connection.connected_at,
+        }
+        : state.status === "connecting"
+            ? { status: "connecting", attempt: state.attempt }
+            : state.status === "reconnecting"
+                ? {
+                    status: "reconnecting",
+                    attempt: state.attempt,
+                    ...(state.next_retry_at === undefined ? {} : { next_retry_at: state.next_retry_at }),
+                    ...(state.last_connected_at === undefined
+                        ? {}
+                        : { last_connected_at: state.last_connected_at }),
+                }
+                : state.status === "unavailable"
+                    ? {
+                        status: "unavailable",
+                        reason_code: state.reason_code,
+                        ...(state.next_retry_at === undefined
+                            ? {}
+                            : { next_retry_at: state.next_retry_at }),
+                        ...(state.last_connection === undefined
+                            ? {}
+                            : { last_connected_at: state.last_connection.connected_at }),
+                    }
+                    : { status: "stopped" };
     return {
-        status: "connected",
-        relay_url: relayUrl,
-        relay_project_ref: status.relay.relay_project_ref,
-        ...(status.relay.join_url === undefined ? {} : { join_url: status.relay.join_url }),
-        join_url_visibility: status.relay.join_url_visibility,
-        ...(status.relay.join_token_expires_at === undefined
-            ? {}
-            : { join_token_expires_at: status.relay.join_token_expires_at }),
-        join_token_reusable: status.relay.join_token_reusable ?? true,
+        project_id: projectId,
+        relay_connection: connection,
+        ...(relay === undefined ? {} : { relay }),
     };
 }
 export function relayStatusFromLocalStatus(status) {

package/dist/server-shell/cli/host-runtime-endpoint.js

@@ -95,6 +95,32 @@ function isLaunchStatusPayload(value) {
         return false;
     }
     const record = value;
+    if (record.project_id !== undefined && typeof record.project_id !== "string") {
+        return false;
+    }
+    if (record.relay_connection !== undefined) {
+        if (typeof record.relay_connection !== "object" ||
+            record.relay_connection === null ||
+            Array.isArray(record.relay_connection)) {
+            return false;
+        }
+        const connection = record.relay_connection;
+        if (connection.status !== "connecting" &&
+            connection.status !== "connected" &&
+            connection.status !== "reconnecting" &&
+            connection.status !== "unavailable" &&
+            connection.status !== "stopped") {
+            return false;
+        }
+        if ((connection.attempt !== undefined &&
+            (!Number.isInteger(connection.attempt) || connection.attempt < 0)) ||
+            (connection.next_retry_at !== undefined && typeof connection.next_retry_at !== "string") ||
+            (connection.last_connected_at !== undefined &&
+                typeof connection.last_connected_at !== "string") ||
+            (connection.reason_code !== undefined && typeof connection.reason_code !== "string")) {
+            return false;
+        }
+    }
     if (record.relay === undefined) {
         return true;
     }

package/dist/server-shell/cli/host-server-runtime.d.ts

@@ -1,10 +1,10 @@
 import type { FastifyInstance } from "fastify";
-import { type RelayHostTunnelHandle } from "@tutti/relay-client";
 import { type HostProjectStore } from "../../store/index.js";
 import type { createHostServer } from "../http/create-server.js";
 import type { HostLocalLaunchStatus } from "../http/routes/local-control.js";
 import type { HostWorkspaceRelayState } from "../http/routes/project-api/types.js";
 import { type TrustedRelaySessionMetadataStore } from "../session/relay-session-context.js";
+import type { HostRelayConnectionManager } from "./host-relay-connection-manager.js";
 import type { LaunchPreparationResult } from "./launch.js";
 import { type MachineRuntimeEndpointRecord } from "./machine-local.js";
 export declare const HOST_SERVER_VERSION = "0.0.0";
@@ -15,7 +15,7 @@ export type HostServerHandle = {
     log_file_path: string;
     runtime_endpoint: MachineRuntimeEndpointRecord;
     trusted_relay_metadata_store: TrustedRelaySessionMetadataStore;
-    attachRelayTunnel: (tunnel: RelayHostTunnelHandle) => void;
+    attachRelayConnectionManager: (manager: HostRelayConnectionManager) => void;
     closed: Promise<void>;
     close: () => Promise<void>;
 };