@xfey/tutti 0.1.28 → 0.1.30

package/dist/server-shell/cli/host-lifecycle.js

@@ -4,15 +4,19 @@ import { createHostServer } from "../http/create-server.js";
 import { createHostProjectApiTunnelRequestHandler, createHostProjectApiTunnelStreamRequestHandler, } from "../http/host-tunnel.js";
 import { LaunchError } from "./errors.js";
 import { TUTTI_MAINLINE_BRANCH } from "./git-bootstrap.js";
-import { relayStateFromLaunchStatus, relayStatusFromLocalStatus } from "./host-relay-status.js";
+import { hostLocalLaunchStatusFromConnectionManagerState, relayStateFromConnectionManagerState, relayStatusFromLocalStatus, } from "./host-relay-status.js";
+import { createHostRelayConnectionManager, } from "./host-relay-connection-manager.js";
 import { createRuntimeEndpointProbe, readExistingHostLaunchStatus, requestExistingHostShutdown, } from "./host-runtime-endpoint.js";
+import { reconnectHostLocalRelay } from "./local-control-client.js";
 import { HOST_SERVER_VERSION, startForegroundHostServer, } from "./host-server-runtime.js";
 import { deleteMachineRuntimeEndpoint, readHostRegistrationSecret, readMachineProjectBinding, readMachineRuntimeEndpoint, } from "./machine-local.js";
+import { inspectMachineProject, inspectMachineProjectInventories, normalizeMachineWorkspacePath, } from "./machine-project-inspector.js";
 import { prepareLaunchProject, resolveLaunchLocalContext, } from "./launch.js";
 import { registerHostWithRelay, relayErrorToLaunchError, } from "./relay-registration.js";
 import { formatJoinLinkValidity, formatTerminalLink } from "./terminal-qr.js";
 export { createRuntimeEndpointProbe } from "./host-runtime-endpoint.js";
 const DEFAULT_SHUTDOWN_WAIT_MS = 2_000;
+const DEFAULT_RECONNECT_WAIT_MS = 8_000;
 function urlLine(label, value, options) {
     return `${label}: ${formatTerminalLink(value, options)}`;
 }
@@ -72,21 +76,106 @@ function createSummary(options) {
     }
     return summary;
 }
+async function reconcileReplacedWorkspaceHosts(options) {
+    const normalizedWorkspaceRoot = normalizeMachineWorkspacePath(options.workspaceRoot);
+    const replacedProjects = inspectMachineProjectInventories(options.tuttiHome).filter((inspection) => inspection.project_id !== options.projectId &&
+        inspection.normalized_workspace_root === normalizedWorkspaceRoot &&
+        inspection.workspace.kind === "replaced" &&
+        inspection.workspace.actual_project_id === options.projectId &&
+        inspection.endpoint.kind === "valid" &&
+        normalizeMachineWorkspacePath(inspection.endpoint.record.workspace_root) ===
+            normalizedWorkspaceRoot);
+    for (const replacedProject of replacedProjects) {
+        const inspection = await inspectMachineProject(replacedProject.project_id, {
+            tuttiHome: options.tuttiHome,
+            fetchImpl: options.fetchImpl,
+            probeRuntimeEndpoint: options.probeRuntimeEndpoint,
+        });
+        if (inspection.endpoint.kind !== "valid") {
+            continue;
+        }
+        if ((inspection.host.kind === "running" && !inspection.host.identity_verified) ||
+            (inspection.host.kind === "stale" &&
+                inspection.host.reason_code === "local_control_unavailable")) {
+            throw new LaunchError("takeover_failed", "The replaced project host identity could not be verified before shutdown", "Stop the old host manually, then run `tutti launch` again.");
+        }
+        const shouldRequestShutdown = inspection.host.kind === "running";
+        if (shouldRequestShutdown) {
+            await requestExistingHostShutdown({
+                endpoint: inspection.endpoint.record,
+                fetchImpl: options.fetchImpl,
+                probeRuntimeEndpoint: options.probeRuntimeEndpoint,
+                shutdownWaitMs: options.shutdownWaitMs,
+            });
+        }
+        if (shouldRequestShutdown ||
+            inspection.host.kind === "stale" ||
+            inspection.host.kind === "identity_mismatch") {
+            deleteMachineRuntimeEndpoint({
+                tuttiHome: options.tuttiHome,
+                projectId: inspection.project_id,
+                expectedToken: inspection.endpoint.record.token,
+            });
+        }
+    }
+}
 export async function startLaunchProject(options) {
     const context = resolveLaunchLocalContext(options);
     const fetchImpl = options.fetch ?? fetch;
     const probeRuntimeEndpoint = options.probeRuntimeEndpoint ?? createRuntimeEndpointProbe(fetchImpl);
     const shutdownWaitMs = options.shutdownWaitMs ?? DEFAULT_SHUTDOWN_WAIT_MS;
+    const reconnectWaitMs = options.reconnectWaitMs ?? DEFAULT_RECONNECT_WAIT_MS;
     let startupStatus = context.existing_identity.kind === "missing" ? "created" : "recovered";
     let allowWorkspaceRootTakeover = false;
     if (context.existing_identity.kind === "present") {
         const projectId = context.existing_identity.project_id;
+        await reconcileReplacedWorkspaceHosts({
+            tuttiHome: context.tutti_home,
+            workspaceRoot: context.workspace_root,
+            projectId,
+            fetchImpl,
+            probeRuntimeEndpoint,
+            shutdownWaitMs,
+        });
         const endpoint = readMachineRuntimeEndpoint(context.tutti_home, projectId);
         if (endpoint !== null) {
             const sameWorkspace = resolve(endpoint.workspace_root) === context.workspace_root;
             const probe = await probeRuntimeEndpoint(endpoint);
             if (probe.kind === "alive" && sameWorkspace) {
-                const launchStatus = await readExistingHostLaunchStatus({ endpoint, fetchImpl });
+                let launchStatus = await readExistingHostLaunchStatus({ endpoint, fetchImpl });
+                if (launchStatus.project_id !== projectId) {
+                    throw new LaunchError("existing_server_unhealthy", "The existing Tutti host identity could not be verified for this project", "Stop the existing host manually, then run `tutti launch` again.");
+                }
+                const relayConnected = launchStatus.relay_connection?.status === "connected" ||
+                    (launchStatus.relay_connection === undefined && probe.readiness?.relay === "connected");
+                if (!relayConnected) {
+                    try {
+                        launchStatus = await reconnectHostLocalRelay({
+                            endpoint,
+                            fetchImpl,
+                            timeoutMs: reconnectWaitMs,
+                        });
+                    }
+                    catch {
+                        throw new LaunchError("relay_unavailable", "The existing Tutti host could not complete a bounded Relay reconnect", "The host is still running. Check Relay connectivity, then run `tutti launch` again.");
+                    }
+                    if (launchStatus.project_id !== projectId) {
+                        throw new LaunchError("existing_server_unhealthy", "The reconnected Tutti host identity does not match this project", "The existing host was preserved. Stop it manually, then run `tutti launch` again.");
+                    }
+                    const reconnectedProbe = await probeRuntimeEndpoint(endpoint);
+                    const reconnected = launchStatus.relay_connection?.status === "connected" ||
+                        (launchStatus.relay_connection === undefined &&
+                            reconnectedProbe.kind === "alive" &&
+                            reconnectedProbe.readiness?.relay === "connected");
+                    if (!reconnected) {
+                        throw new LaunchError("relay_unavailable", "The existing Tutti host is running but Relay reconnection is not complete", "The host was preserved. Check `tutti ps` or `tutti doctor`, then retry launch.", {
+                            relay_status: launchStatus.relay_connection?.status ?? "not_connected",
+                            ...(launchStatus.relay_connection?.reason_code === undefined
+                                ? {}
+                                : { reason_code: launchStatus.relay_connection.reason_code }),
+                        });
+                    }
+                }
                 const relay = relayStatusFromLocalStatus(launchStatus);
                 return {
                     kind: "connected",
@@ -154,58 +243,55 @@ export async function startLaunchProject(options) {
         ...options,
         allowWorkspaceRootTakeover,
     });
-    const launchStatus = {};
-    const refreshRelayJoinUrl = async () => {
-        if (launchStatus.relay === undefined) {
-            return relayStateFromLaunchStatus(preparation.relay_url, launchStatus);
-        }
-        const hostSecret = readHostRegistrationSecret(preparation.tutti_home, preparation.project_id);
-        const response = await refreshRelayJoinTokenWithRelay({
-            relayUrl: preparation.relay_url,
+    if (context.existing_identity.kind === "missing") {
+        await reconcileReplacedWorkspaceHosts({
+            tuttiHome: preparation.tutti_home,
+            workspaceRoot: preparation.workspace_root,
             projectId: preparation.project_id,
-            relayProjectRef: launchStatus.relay.relay_project_ref,
-            hostConnectionRef: launchStatus.relay.host_connection_ref,
-            hostRegistrationSecret: hostSecret.secret,
-            joinToken: { mode: "rotate" },
-            fetch: fetchImpl,
+            fetchImpl,
+            probeRuntimeEndpoint,
+            shutdownWaitMs,
         });
-        const relayWithoutJoinUrl = { ...launchStatus.relay };
-        delete relayWithoutJoinUrl.join_url;
-        launchStatus.relay = {
-            ...relayWithoutJoinUrl,
-            ...(response.join_token.join_url === undefined
-                ? {}
-                : { join_url: response.join_token.join_url }),
-            join_url_visibility: response.join_token.visibility,
-            join_token_expires_at: response.join_token.expires_at,
-            join_token_reusable: response.join_token.reusable,
-        };
-        return relayStateFromLaunchStatus(preparation.relay_url, launchStatus);
+    }
+    const relayRuntime = {};
+    const readConnectionState = () => relayRuntime.connectionManager?.getState() ?? {
+        status: "unavailable",
+        reason_code: "not_started",
+    };
+    const getLaunchStatus = () => hostLocalLaunchStatusFromConnectionManagerState(preparation.project_id, readConnectionState());
+    const refreshRelayJoinUrl = async () => {
+        if (relayRuntime.connectionManager === undefined) {
+            return relayStateFromConnectionManagerState(preparation.relay_url, readConnectionState());
+        }
+        await relayRuntime.connectionManager.refreshJoinUrl();
+        return relayStateFromConnectionManagerState(preparation.relay_url, relayRuntime.connectionManager.getState());
     };
     const host = await startForegroundHostServer({
         preparation,
         endpointHost: context.local_diagnostic_endpoint.host,
         endpointPort: context.local_diagnostic_endpoint.port,
         createServer: options.createServer ?? createHostServer,
-        getLaunchStatus: () => launchStatus,
+        getLaunchStatus,
         refreshRelayJoinUrl,
         ...(options.now === undefined ? {} : { now: options.now }),
     });
-    let relay;
-    try {
-        const registered = await registerHostWithRelay({
+    const connectionManager = createHostRelayConnectionManager({
+        tuttiHome: preparation.tutti_home,
+        projectId: preparation.project_id,
+        workspaceRoot: preparation.workspace_root,
+        registerConnection: async ({ joinTokenMode, signal }) => await registerHostWithRelay({
             preparation,
             fetchImpl,
             registerRelayHostConnection: options.registerRelayHostConnection ?? registerRelayHostConnectionWithRelay,
             serverVersion: HOST_SERVER_VERSION,
+            joinTokenMode,
+            signal,
             ...(options.now === undefined ? {} : { now: options.now }),
-        });
-        relay = registered.summary;
-        launchStatus.relay = relay;
-        const tunnel = (options.connectRelayHostTunnel ?? connectRelayHostTunnel)({
-            tunnelUrl: relay.tunnel_url,
-            relayProjectRef: relay.relay_project_ref,
-            hostConnectionRef: relay.host_connection_ref,
+        }),
+        connectTunnel: (registered) => (options.connectRelayHostTunnel ?? connectRelayHostTunnel)({
+            tunnelUrl: registered.summary.tunnel_url,
+            relayProjectRef: registered.summary.relay_project_ref,
+            hostConnectionRef: registered.summary.host_connection_ref,
             hostConnectionToken: registered.hostConnectionToken,
             handleRequest: createHostProjectApiTunnelRequestHandler({
                 app: host.app,
@@ -215,9 +301,37 @@ export async function startLaunchProject(options) {
                 baseUrl: host.runtime_endpoint.base_url,
                 trustedMetadataStore: host.trusted_relay_metadata_store,
             }),
-        });
-        await tunnel.connected;
-        host.attachRelayTunnel(tunnel);
+        }),
+        refreshJoinUrl: async (current) => {
+            const hostSecret = readHostRegistrationSecret(preparation.tutti_home, preparation.project_id);
+            const response = await refreshRelayJoinTokenWithRelay({
+                relayUrl: preparation.relay_url,
+                projectId: preparation.project_id,
+                relayProjectRef: current.relay_project_ref,
+                hostConnectionRef: current.host_connection_ref,
+                hostRegistrationSecret: hostSecret.secret,
+                joinToken: { mode: "rotate" },
+                fetch: fetchImpl,
+            });
+            const withoutJoinUrl = { ...current };
+            delete withoutJoinUrl.join_url;
+            return {
+                ...withoutJoinUrl,
+                ...(response.join_token.join_url === undefined
+                    ? {}
+                    : { join_url: response.join_token.join_url }),
+                join_url_visibility: response.join_token.visibility,
+                join_token_expires_at: response.join_token.expires_at,
+                join_token_reusable: response.join_token.reusable,
+            };
+        },
+        ...(options.now === undefined ? {} : { now: options.now }),
+    });
+    relayRuntime.connectionManager = connectionManager;
+    host.attachRelayConnectionManager(connectionManager);
+    let relay;
+    try {
+        relay = await connectionManager.start();
     }
     catch (error) {
         const launchError = error instanceof LaunchError || error instanceof RelayHostControlClientError

package/dist/server-shell/cli/host-relay-connection-manager.d.ts

@@ -0,0 +1,50 @@
+import { type RelayHostTunnelHandle } from "@tutti/relay-client";
+import type { ProjectId } from "@tutti/shared/ids";
+import type { HostLaunchRelaySummary, RegisteredRelayHostConnection } from "./relay-registration.js";
+export type HostRelayConnectionManagerState = {
+    status: "unavailable";
+    reason_code: "not_started" | "relay_unavailable" | "workspace_missing" | "workspace_replaced" | "workspace_unreadable" | "fatal_error";
+    error_code?: string;
+    next_retry_at?: string;
+    last_connection?: HostLaunchRelaySummary;
+} | {
+    status: "connecting";
+    attempt: 0;
+} | {
+    status: "connected";
+    connection: HostLaunchRelaySummary;
+} | {
+    status: "reconnecting";
+    attempt: number;
+    next_retry_at?: string;
+    last_connected_at?: string;
+    last_connection?: HostLaunchRelaySummary;
+} | {
+    status: "stopped";
+};
+export type HostRelayConnectionManager = {
+    start: () => Promise<HostLaunchRelaySummary>;
+    reconnectNow: () => Promise<HostRelayConnectionManagerState>;
+    refreshJoinUrl: () => Promise<HostLaunchRelaySummary>;
+    getState: () => HostRelayConnectionManagerState;
+    stop: () => Promise<void>;
+};
+export type CreateHostRelayConnectionManagerOptions = {
+    tuttiHome: string;
+    projectId: ProjectId;
+    workspaceRoot: string;
+    registerConnection: (options: {
+        joinTokenMode: "reuse_active" | "rotate";
+        signal: AbortSignal;
+    }) => Promise<RegisteredRelayHostConnection>;
+    connectTunnel: (registered: RegisteredRelayHostConnection) => RelayHostTunnelHandle;
+    refreshJoinUrl?: (connection: HostLaunchRelaySummary) => Promise<HostLaunchRelaySummary>;
+    onStateChange?: (state: HostRelayConnectionManagerState) => void;
+    retryDelaysMs?: readonly number[];
+    jitterRatio?: number;
+    workspaceCheckIntervalMs?: number;
+    now?: () => Date;
+    random?: () => number;
+};
+export declare function createHostRelayConnectionManager(options: CreateHostRelayConnectionManagerOptions): HostRelayConnectionManager;
+//# sourceMappingURL=host-relay-connection-manager.d.ts.map