@agentlang/agentmanager 0.0.3

package/apps/fabric/src/employees/employees.service.ts

@@ -0,0 +1,1333 @@
+import {
+  Injectable,
+  Inject,
+  Logger,
+  OnModuleDestroy,
+  Optional,
+} from '@nestjs/common';
+import { join } from 'node:path';
+import type { ConfigType } from '@nestjs/config';
+import fabricConfig from '../config/configuration';
+import { RuntimeStore } from './runtime.store';
+import { VersionsService } from '../versions/versions.service';
+import {
+  DEPLOYMENT_PROVIDER,
+  type DeploymentProvider,
+} from '../providers/provider.interface';
+import {
+  REGISTRY_PROVIDER,
+  REGISTRY_PATH,
+  type RegistryProvider,
+} from '@app/project-provider';
+import { ProviderFactory } from '../providers/provider-factory';
+import { PortAllocator } from '../providers/port-allocator';
+import { EnvironmentsService } from '../environments/environments.service';
+import { ProjectSourceService } from '../project-source/project-source.service';
+import { DeploymentEventsService } from '../deployment-events/deployment-events.service';
+import { LogsService } from '../logs/logs.service';
+import { NotificationsService } from '../notifications/notifications.service';
+import {
+  PostgresProvisionerService,
+  type PostgresCredentials,
+} from '../provisioners/postgres-provisioner.service';
+import { CognitoProvisionerService } from '../provisioners/cognito-provisioner.service';
+import { ProjectsDataService } from '../projects/projects-data.service';
+import type { ProjectSourceConfig } from '../project-source/project-source.types';
+import type { ProviderType } from '../environments/environments.types';
+import {
+  NoLiveVersionError,
+  AlreadyInStateError,
+  EmployeeNotFoundError,
+  InvalidStateTransitionError,
+  CannotRemoveError,
+  DeploymentEnvironmentMismatchError,
+  UndeployEnvironmentUnresolvedError,
+} from './errors/employee-deployment.errors';
+import type {
+  Env,
+  EmployeeStatus,
+  DeployResult,
+  ResolveResult,
+} from './employees.types';
+import { ACTIVE_DEPLOYMENT_STATUSES } from './employees.types';
+import type { LogType } from '../logs/logs.service';
+
+@Injectable()
+export class EmployeesService implements OnModuleDestroy {
+  private readonly logger = new Logger(EmployeesService.name);
+  private readonly deploying = new Set<string>();
+
+  constructor(
+    private readonly runtimeStore: RuntimeStore,
+    private readonly versionsService: VersionsService,
+    @Inject(DEPLOYMENT_PROVIDER)
+    private readonly provider: DeploymentProvider,
+    private readonly providerFactory: ProviderFactory,
+    private readonly portAllocator: PortAllocator,
+    private readonly environmentsService: EnvironmentsService,
+    private readonly projectSource: ProjectSourceService,
+    private readonly eventsService: DeploymentEventsService,
+    private readonly logsService: LogsService,
+    private readonly notificationsService: NotificationsService,
+    private readonly postgresProvisioner: PostgresProvisionerService,
+    private readonly cognitoProvisioner: CognitoProvisionerService,
+    @Inject(fabricConfig.KEY)
+    private readonly config: ConfigType<typeof fabricConfig>,
+    private readonly projectsDataService: ProjectsDataService,
+    @Optional()
+    @Inject(REGISTRY_PROVIDER)
+    private readonly registry: RegistryProvider | null,
+    @Optional()
+    @Inject(REGISTRY_PATH)
+    private readonly registryPath: string | null,
+  ) {}
+
+  async onModuleDestroy() {
+    const entries = [...this.runtimeStore.listAll()];
+    const running = entries.filter(
+      (e) =>
+        e.entry.containerId &&
+        (e.entry.runtime === 'running' || e.entry.runtime === 'paused'),
+    );
+
+    if (running.length === 0) return;
+
+    this.logger.log(
+      `Graceful shutdown: stopping ${running.length} running process(es)`,
+    );
+
+    await Promise.allSettled(
+      running.map(async (e) => {
+        try {
+          const resolved = this.resolveEnvironment(
+            e.entry.environmentId ?? undefined,
+          );
+          if (e.entry.projectPath) {
+            await this.logsService.saveStdoutLogsBeforeStop(
+              e.entry.containerId!,
+              e.entry.projectPath,
+              resolved.providerType,
+            );
+          }
+          await resolved.provider.stop(e.entry.containerId!);
+          this.logger.log(`Stopped ${e.projectId}/${e.employeeId} (${e.env})`);
+        } catch (err) {
+          this.logger.warn(
+            `Failed to stop ${e.projectId}/${e.employeeId} (${e.env}): ${err}`,
+          );
+        }
+      }),
+    );
+  }
+
+  private processName(projectId: string, employeeId: string, env: Env): string {
+    return `fabric-${projectId}-${employeeId}-${env}`;
+  }
+
+  private resolveEnvironment(environmentId?: string): {
+    provider: DeploymentProvider;
+    providerType: ProviderType;
+    envVars: Record<string, string>;
+    jsonConfig: Record<string, unknown>;
+    environmentId: string | null;
+  } {
+    if (!environmentId) {
+      return {
+        provider: this.provider,
+        providerType: this.config.provider,
+        envVars: {},
+        jsonConfig: {},
+        environmentId: null,
+      };
+    }
+
+    const env = this.environmentsService.get(environmentId);
+    return {
+      provider: this.providerFactory.get(env.providerType),
+      providerType: env.providerType,
+      envVars: env.envVars,
+      jsonConfig: env.jsonConfig,
+      environmentId: env.id,
+    };
+  }
+
+  async deploy(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+    environmentId?: string,
+    source?: ProjectSourceConfig,
+    userId?: string | null,
+  ): Promise<DeployResult> {
+    const deployKey = `${projectId}/${employeeId}/${env}`;
+    if (this.deploying.has(deployKey)) {
+      throw new AlreadyInStateError(employeeId, env, 'deploying');
+    }
+
+    const now = new Date().toISOString();
+    let versionId: string | undefined;
+
+    if (env === 'live') {
+      const latest = this.versionsService.getLatest(projectId, employeeId);
+      if (!latest) {
+        throw new NoLiveVersionError(projectId, employeeId);
+      }
+      versionId = latest.versionId;
+    }
+
+    // Resolve project path from source, or reuse existing
+    let projectPath: string;
+    let storedSource: unknown = source ?? null;
+
+    if (source) {
+      projectPath = await this.projectSource.resolve(source);
+    } else {
+      const existing = this.runtimeStore.get(projectId, employeeId, env);
+      if (existing?.projectPath) {
+        projectPath = existing.projectPath;
+        storedSource = existing.source;
+      } else {
+        const otherEnv = env === 'draft' ? 'live' : 'draft';
+        const other = this.runtimeStore.get(projectId, employeeId, otherEnv);
+        if (other?.projectPath) {
+          projectPath = other.projectPath;
+          storedSource = other.source;
+        } else if (this.registry && this.registryPath) {
+          // Fall back to agentmanager registry
+          try {
+            await this.registry.getEmployee(projectId, employeeId);
+            projectPath = join(this.registryPath, projectId, employeeId);
+            storedSource = { type: 'local', path: projectPath };
+          } catch {
+            throw new EmployeeNotFoundError(projectId, employeeId);
+          }
+        } else {
+          throw new EmployeeNotFoundError(projectId, employeeId);
+        }
+      }
+    }
+
+    // If no explicit environmentId, try to use the project's environmentId
+    let resolvedEnvironmentId = environmentId;
+    if (!resolvedEnvironmentId) {
+      const project = await this.projectsDataService.get(projectId);
+      if (project) {
+        resolvedEnvironmentId = project.environmentId;
+      }
+    }
+
+    const resolved = this.resolveEnvironment(resolvedEnvironmentId);
+    const port = await this.portAllocator.allocateVerified();
+
+    const resolvedUserId = userId ?? null;
+
+    // Set initial pending state and return immediately
+    this.runtimeStore.set(projectId, employeeId, env, {
+      userId: resolvedUserId,
+      runtime: 'pending',
+      deployedAt: now,
+      lastError: null,
+      versionId: versionId ?? null,
+      publishedAt: versionId ? now : null,
+      moduleCount: 0,
+      containerId: null,
+      port,
+      environmentId: resolved.environmentId,
+      projectPath,
+      source: storedSource,
+      healthcheckRetryCount: 0,
+      envVars: null,
+    });
+
+    // Fire-and-forget: run the actual deployment in background
+    this.runDeploy(
+      projectId,
+      employeeId,
+      env,
+      projectPath,
+      port,
+      versionId,
+      storedSource,
+      resolved,
+      resolvedUserId,
+    ).catch((err) => {
+      this.logger.error(
+        `Unexpected error in runDeploy for ${deployKey}: ${err}`,
+      );
+    });
+
+    return {
+      projectId,
+      employeeId,
+      env,
+      runtime: 'pending',
+      deployedAt: now,
+      versionId,
+      port,
+      environmentId: resolved.environmentId ?? undefined,
+    };
+  }
+
+  private async runDeploy(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+    projectPath: string,
+    port: number,
+    versionId: string | undefined,
+    storedSource: unknown,
+    resolved: ReturnType<typeof this.resolveEnvironment>,
+    userId: string | null,
+  ): Promise<void> {
+    const deployKey = `${projectId}/${employeeId}/${env}`;
+    this.deploying.add(deployKey);
+    const now = new Date().toISOString();
+    const name = this.processName(projectId, employeeId, env);
+
+    try {
+      // ── Cloning phase ───────────────────────────────────────────────
+      this.runtimeStore.updateField(projectId, employeeId, env, {
+        runtime: 'cloning',
+      });
+
+      // ── Building phase ──────────────────────────────────────────────
+      this.runtimeStore.updateField(projectId, employeeId, env, {
+        runtime: 'building',
+      });
+
+      // Build merged config and env vars
+      const baseAppConfig = { service: { port } };
+      let mergedConfig: Record<string, unknown> = {
+        ...resolved.jsonConfig,
+        ...baseAppConfig,
+      };
+      const envVars: Record<string, string> = {
+        ...resolved.envVars,
+        PORT: String(port),
+      };
+
+      // Postgres provisioning
+      const storeConfig = mergedConfig.store as
+        | Record<string, unknown>
+        | undefined;
+      if (storeConfig?.type === 'postgres') {
+        try {
+          // Check if project has a shared database
+          let creds: PostgresCredentials | undefined;
+          try {
+            const project = await this.projectsDataService.get(projectId);
+            if (project?.dbName) {
+              creds = this.postgresProvisioner.getCredentialsForExistingDb(
+                project.dbName,
+              );
+            }
+          } catch {
+            // Ignore — will create new database below
+          }
+
+          if (!creds) {
+            creds = await this.postgresProvisioner.createDatabase(
+              `${projectId}_${employeeId}_${env}`,
+            );
+          }
+
+          mergedConfig = {
+            ...mergedConfig,
+            store: {
+              ...storeConfig,
+              type: 'postgres',
+              dbname: creds.dbname,
+              host: creds.host,
+              port: creds.port,
+              username: creds.username,
+              password: creds.password,
+            },
+          };
+        } catch (err) {
+          this.logger.warn(`Postgres provisioning failed: ${err}`);
+        }
+      }
+
+      // Cognito provisioning
+      const authConfig = mergedConfig.auth as
+        | Record<string, unknown>
+        | undefined;
+      if (authConfig?.enabled === true) {
+        try {
+          const cognito = await this.cognitoProvisioner.provision(
+            `${projectId}-${employeeId}-${env}`,
+            '',
+            '',
+          );
+          envVars.COGNITO_USER_POOL_ID = cognito.userPoolId;
+          envVars.COGNITO_CLIENT_ID = cognito.clientId;
+          envVars.AWS_REGION = cognito.region;
+          envVars.COGNITO_DEFAULT_USER_PASSWORD = cognito.defaultPassword;
+          const awsKey = this.config.aws.accessKeyId;
+          const awsSecret = this.config.aws.secretAccessKey;
+          if (awsKey) envVars.AWS_ACCESS_KEY_ID = awsKey;
+          if (awsSecret) envVars.AWS_SECRET_ACCESS_KEY = awsSecret;
+        } catch (err) {
+          this.logger.warn(`Cognito provisioning failed: ${err}`);
+        }
+      }
+
+      // Set APP_CONFIG env var with merged config
+      envVars.APP_CONFIG = JSON.stringify(mergedConfig);
+
+      // ── Deploying phase ─────────────────────────────────────────────
+      this.runtimeStore.updateField(projectId, employeeId, env, {
+        runtime: 'deploying',
+        envVars,
+      });
+
+      const result = await resolved.provider.deploy(
+        name,
+        projectPath,
+        envVars,
+        port,
+        { projectId, employeeId },
+      );
+
+      if (!result.success) {
+        const errMsg = result.error ?? 'Deploy failed';
+        this.runtimeStore.set(projectId, employeeId, env, {
+          userId,
+          runtime: 'failed',
+          deployedAt: now,
+          lastError: errMsg,
+          versionId: versionId ?? null,
+          publishedAt: versionId ? now : null,
+          moduleCount: 0,
+          containerId: null,
+          port: null,
+          environmentId: resolved.environmentId,
+          projectPath,
+          source: storedSource,
+          healthcheckRetryCount: 0,
+          envVars,
+        });
+
+        this.logger.error(
+          `Failed to deploy ${projectId}/${employeeId} (${env}): ${errMsg}`,
+        );
+
+        this.eventsService.create({
+          userId,
+          projectId,
+          employeeId,
+          env,
+          type: 'error',
+          message: errMsg,
+        });
+
+        this.notificationsService
+          .notifyOnFailure(userId ?? '__anonymous__', {
+            projectId,
+            employeeId,
+            env,
+            type: 'error',
+            message: errMsg,
+          })
+          .catch((notifyErr) => {
+            this.logger.warn(
+              `Failed to send failure notification: ${notifyErr}`,
+            );
+          });
+
+        return;
+      }
+
+      // ── Running ─────────────────────────────────────────────────────
+      this.runtimeStore.set(projectId, employeeId, env, {
+        userId,
+        runtime: 'running',
+        deployedAt: now,
+        lastError: null,
+        versionId: versionId ?? null,
+        publishedAt: versionId ? now : null,
+        moduleCount: 0,
+        containerId: result.containerId ?? null,
+        port,
+        environmentId: resolved.environmentId,
+        projectPath,
+        source: storedSource,
+        healthcheckRetryCount: 0,
+        envVars,
+      });
+
+      this.logger.log(
+        `Deployed ${projectId}/${employeeId} (${env}) → ${result.containerId} on port ${port}`,
+      );
+
+      this.eventsService.create({
+        userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'start',
+        message: 'Deployment started successfully',
+      });
+    } catch (err) {
+      const errMsg = err instanceof Error ? err.message : String(err);
+      this.runtimeStore.updateField(projectId, employeeId, env, {
+        runtime: 'failed',
+        lastError: errMsg,
+      });
+
+      this.eventsService.create({
+        userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'error',
+        message: errMsg,
+      });
+
+      this.notificationsService
+        .notifyOnFailure(userId ?? '__anonymous__', {
+          projectId,
+          employeeId,
+          env,
+          type: 'error',
+          message: errMsg,
+        })
+        .catch((notifyErr) => {
+          this.logger.warn(`Failed to send failure notification: ${notifyErr}`);
+        });
+    } finally {
+      this.deploying.delete(deployKey);
+    }
+  }
+
+  async stop(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<DeployResult> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+    if (existing?.runtime === 'stopped') {
+      throw new AlreadyInStateError(employeeId, env, 'stopped');
+    }
+
+    if (existing?.containerId) {
+      const resolved = this.resolveEnvironment(
+        existing.environmentId ?? undefined,
+      );
+
+      // Save logs before stopping
+      if (existing.projectPath) {
+        await this.logsService.saveStdoutLogsBeforeStop(
+          existing.containerId,
+          existing.projectPath,
+          resolved.providerType,
+        );
+      }
+
+      await resolved.provider.stop(existing.containerId);
+
+      // Clean up Cognito pool if provisioned
+      if (existing.envVars?.COGNITO_USER_POOL_ID) {
+        try {
+          await this.cognitoProvisioner.deletePool(
+            existing.envVars.COGNITO_USER_POOL_ID,
+          );
+        } catch (err) {
+          this.logger.error(`Failed to delete Cognito pool: ${err}`);
+        }
+      }
+
+      // Clean up per-deployment database (skip if it's the shared project DB)
+      const appConfigStr = existing.envVars?.APP_CONFIG;
+      if (appConfigStr) {
+        try {
+          const appConfig = JSON.parse(appConfigStr) as Record<string, unknown>;
+          const storeConfig = appConfig.store as
+            | Record<string, unknown>
+            | undefined;
+          const dbname = storeConfig?.dbname as string | undefined;
+          if (storeConfig?.type === 'postgres' && dbname) {
+            let isSharedDb = false;
+            try {
+              const project = await this.projectsDataService.get(projectId);
+              isSharedDb = project?.dbName === dbname;
+            } catch {
+              // Project may not exist
+            }
+
+            if (!isSharedDb) {
+              await this.postgresProvisioner.dropDatabase(dbname);
+            }
+          }
+        } catch (err) {
+          this.logger.error(`Failed to drop database: ${err}`);
+        }
+      }
+    }
+
+    this.runtimeStore.set(projectId, employeeId, env, {
+      userId: existing?.userId ?? null,
+      runtime: 'stopped',
+      deployedAt: existing?.deployedAt ?? null,
+      lastError: null,
+      versionId: existing?.versionId ?? null,
+      publishedAt: existing?.publishedAt ?? null,
+      moduleCount: existing?.moduleCount ?? 0,
+      containerId: null,
+      port: null,
+      environmentId: existing?.environmentId ?? null,
+      projectPath: existing?.projectPath ?? null,
+      source: existing?.source ?? null,
+      healthcheckRetryCount: 0,
+      envVars: existing?.envVars ?? null,
+    });
+
+    this.logger.log(`Stopped ${projectId}/${employeeId} (${env})`);
+
+    this.eventsService.create({
+      userId: existing?.userId,
+      projectId,
+      employeeId,
+      env,
+      type: 'stop',
+      message: 'Deployment stopped',
+    });
+
+    return {
+      projectId,
+      employeeId,
+      env,
+      runtime: 'stopped',
+      deployedAt: existing?.deployedAt ?? new Date().toISOString(),
+    };
+  }
+
+  async pause(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<DeployResult> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+
+    if (!existing || existing.runtime !== 'running') {
+      throw new InvalidStateTransitionError(
+        employeeId,
+        env,
+        existing?.runtime ?? 'unknown',
+        'paused',
+      );
+    }
+
+    if (existing.containerId) {
+      const resolved = this.resolveEnvironment(
+        existing.environmentId ?? undefined,
+      );
+
+      // Save logs before pausing
+      if (existing.projectPath) {
+        await this.logsService.saveStdoutLogsBeforeStop(
+          existing.containerId,
+          existing.projectPath,
+          resolved.providerType,
+        );
+      }
+
+      await resolved.provider.stop(existing.containerId);
+    }
+
+    this.runtimeStore.set(projectId, employeeId, env, {
+      ...existing,
+      runtime: 'paused',
+      lastError: null,
+      containerId: null,
+    });
+
+    this.logger.log(`Paused ${projectId}/${employeeId} (${env})`);
+
+    this.eventsService.create({
+      userId: existing.userId,
+      projectId,
+      employeeId,
+      env,
+      type: 'stop',
+      message: 'Deployment paused',
+    });
+
+    return {
+      projectId,
+      employeeId,
+      env,
+      runtime: 'paused',
+      deployedAt: existing.deployedAt ?? new Date().toISOString(),
+      versionId: existing.versionId ?? undefined,
+      containerId: existing.containerId ?? undefined,
+      port: existing.port ?? undefined,
+    };
+  }
+
+  async resume(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<DeployResult> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+
+    if (
+      !existing ||
+      (existing.runtime !== 'paused' && existing.runtime !== 'stopped')
+    ) {
+      throw new InvalidStateTransitionError(
+        employeeId,
+        env,
+        existing?.runtime ?? 'unknown',
+        'running',
+      );
+    }
+
+    if (existing.runtime === 'stopped') {
+      return this.deploy(
+        projectId,
+        employeeId,
+        env,
+        existing.environmentId ?? undefined,
+      );
+    }
+
+    // Paused — re-deploy with preserved envVars and config
+    const resolved = this.resolveEnvironment(
+      existing.environmentId ?? undefined,
+    );
+    const name = this.processName(projectId, employeeId, env);
+    const projectPath = existing.projectPath!;
+    const port = existing.port ?? this.portAllocator.allocate();
+    const now = new Date().toISOString();
+    const envVars = existing.envVars ?? resolved.envVars;
+
+    const result = await resolved.provider.deploy(
+      name,
+      projectPath,
+      envVars,
+      port,
+      { projectId, employeeId },
+    );
+
+    if (!result.success) {
+      this.runtimeStore.set(projectId, employeeId, env, {
+        ...existing,
+        runtime: 'crashed',
+        lastError: result.error ?? 'Resume failed',
+      });
+
+      this.logger.error(
+        `Failed to resume ${projectId}/${employeeId} (${env}): ${result.error}`,
+      );
+
+      this.eventsService.create({
+        userId: existing.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'error',
+        message: result.error ?? 'Resume failed',
+      });
+
+      return {
+        projectId,
+        employeeId,
+        env,
+        runtime: 'crashed',
+        deployedAt: existing.deployedAt ?? now,
+        versionId: existing.versionId ?? undefined,
+      };
+    }
+
+    this.runtimeStore.set(projectId, employeeId, env, {
+      ...existing,
+      runtime: 'running',
+      deployedAt: now,
+      lastError: null,
+      containerId: result.containerId ?? existing.containerId,
+      port,
+    });
+
+    this.logger.log(`Resumed ${projectId}/${employeeId} (${env})`);
+
+    this.eventsService.create({
+      userId: existing.userId,
+      projectId,
+      employeeId,
+      env,
+      type: 'start',
+      message: 'Deployment resumed',
+    });
+
+    return {
+      projectId,
+      employeeId,
+      env,
+      runtime: 'running',
+      deployedAt: now,
+      versionId: existing.versionId ?? undefined,
+      containerId: result.containerId ?? existing.containerId ?? undefined,
+      port,
+      environmentId: existing.environmentId ?? undefined,
+    };
+  }
+
+  async restart(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<DeployResult> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+
+    if (!existing?.containerId) {
+      throw new EmployeeNotFoundError(projectId, employeeId);
+    }
+
+    const resolved = this.resolveEnvironment(
+      existing.environmentId ?? undefined,
+    );
+    await resolved.provider.restart(existing.containerId);
+    const now = new Date().toISOString();
+
+    this.runtimeStore.set(projectId, employeeId, env, {
+      ...existing,
+      runtime: 'running',
+      deployedAt: now,
+      lastError: null,
+    });
+
+    this.logger.log(`Restarted ${projectId}/${employeeId} (${env})`);
+
+    this.eventsService.create({
+      userId: existing.userId,
+      projectId,
+      employeeId,
+      env,
+      type: 'restart',
+      message: 'Deployment restarted',
+    });
+
+    return {
+      projectId,
+      employeeId,
+      env,
+      runtime: 'running',
+      deployedAt: now,
+      versionId: existing.versionId ?? undefined,
+      containerId: existing.containerId,
+      port: existing.port ?? undefined,
+    };
+  }
+
+  async publish(projectId: string, employeeId: string): Promise<DeployResult> {
+    const version = this.versionsService.create(projectId, employeeId);
+
+    this.logger.log(
+      `Published ${projectId}/${employeeId} as ${version.versionId}`,
+    );
+
+    const existing = this.runtimeStore.get(projectId, employeeId, 'live');
+    return this.deploy(
+      projectId,
+      employeeId,
+      'live',
+      existing?.environmentId ?? undefined,
+    );
+  }
+
+  async unpublish(
+    projectId: string,
+    employeeId: string,
+  ): Promise<DeployResult> {
+    const result = await this.stop(projectId, employeeId, 'live');
+    this.runtimeStore.delete(projectId, employeeId, 'live');
+
+    this.logger.log(`Unpublished ${projectId}/${employeeId}`);
+
+    return result;
+  }
+
+  async rollback(
+    projectId: string,
+    employeeId: string,
+    versionId?: string,
+  ): Promise<DeployResult> {
+    const version = versionId
+      ? this.versionsService.get(projectId, employeeId, versionId)
+      : this.versionsService.getPrevious(projectId, employeeId);
+
+    if (!version) {
+      throw new NoLiveVersionError(projectId, employeeId);
+    }
+
+    const existing = this.runtimeStore.get(projectId, employeeId, 'live');
+    if (existing?.containerId) {
+      try {
+        const resolved = this.resolveEnvironment(
+          existing.environmentId ?? undefined,
+        );
+        await resolved.provider.stop(existing.containerId);
+      } catch (err) {
+        this.logger.warn(
+          `Best-effort stop before rollback failed for ${projectId}/${employeeId}: ${err instanceof Error ? err.message : err}`,
+        );
+      }
+    }
+
+    return this.deploy(
+      projectId,
+      employeeId,
+      'live',
+      existing?.environmentId ?? undefined,
+    );
+  }
+
+  getStatus(projectId: string, employeeId: string): EmployeeStatus {
+    const draft = this.runtimeStore.get(projectId, employeeId, 'draft');
+    const live = this.runtimeStore.get(projectId, employeeId, 'live');
+
+    return {
+      projectId,
+      employeeId,
+      draft: this.runtimeStore.toDraftInfo(draft),
+      live: this.runtimeStore.toLiveInfo(live),
+      draftAheadOfLive:
+        !!draft?.deployedAt &&
+        (!live?.deployedAt || draft.deployedAt > live.deployedAt),
+    };
+  }
+
+  getBulkStatus(projectId: string): EmployeeStatus[] {
+    const entries = this.runtimeStore.listByProject(projectId);
+    const employeeIds = [...new Set(entries.map((e) => e.employeeId))];
+    return employeeIds.map((id) => this.getStatus(projectId, id));
+  }
+
+  resolve(projectId: string, employeeId: string, env: Env): ResolveResult {
+    const entry = this.runtimeStore.get(projectId, employeeId, env);
+
+    return {
+      projectId,
+      employeeId,
+      env,
+      runtime: entry?.runtime ?? 'stopped',
+      endpoint:
+        entry?.runtime === 'running' && entry.port
+          ? `http://localhost:${entry.port}`
+          : null,
+      port: entry?.port ?? null,
+      containerId: entry?.containerId ?? null,
+      capabilities: [],
+    };
+  }
+
+  /**
+   * When Fabric still marks a deployment as running but PM2/Docker reports the
+   * process/container stopped or missing, restart it or trigger a fresh deploy.
+   * Invoked periodically by `DeploymentWatcherService`.
+   */
+  async recoverIfProcessMissing(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<void> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+    if (!existing || existing.runtime !== 'running' || !existing.containerId) {
+      return;
+    }
+
+    const deployKey = `${projectId}/${employeeId}/${env}`;
+    if (this.deploying.has(deployKey)) {
+      return;
+    }
+
+    const resolved = this.resolveEnvironment(
+      existing.environmentId ?? undefined,
+    );
+    const provider = resolved.provider;
+
+    let status: Awaited<ReturnType<DeploymentProvider['getStatus']>>;
+    try {
+      status = await provider.getStatus(existing.containerId);
+    } catch (err) {
+      this.logger.warn(
+        `Deployment watcher: getStatus failed for ${projectId}/${employeeId} (${env}): ${err instanceof Error ? err.message : err}`,
+      );
+      return;
+    }
+
+    if (status === 'running') {
+      return;
+    }
+
+    this.logger.warn(
+      `Deployment watcher: ${projectId}/${employeeId} (${env}) provider status is "${status}" but Fabric state is running — recovering`,
+    );
+
+    if (status === 'stopped') {
+      try {
+        await provider.restart(existing.containerId);
+        const after = await provider.getStatus(existing.containerId);
+        if (after === 'running') {
+          this.eventsService.create({
+            userId: existing.userId,
+            projectId,
+            employeeId,
+            env,
+            type: 'watcher_recover',
+            message:
+              'Process was stopped externally; provider restart succeeded.',
+          });
+          return;
+        }
+      } catch (err) {
+        this.logger.warn(
+          `Deployment watcher: restart failed for ${projectId}/${employeeId} (${env}): ${err instanceof Error ? err.message : err}`,
+        );
+      }
+    }
+
+    try {
+      await this.deploy(
+        projectId,
+        employeeId,
+        env,
+        existing.environmentId ?? undefined,
+        undefined,
+        existing.userId,
+      );
+      this.eventsService.create({
+        userId: existing.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'watcher_recover',
+        message:
+          status === 'not_found'
+            ? 'Process no longer present in provider; redeploy started.'
+            : 'Restart failed or process still down; redeploy started.',
+      });
+    } catch (err) {
+      if (err instanceof AlreadyInStateError) {
+        return;
+      }
+      this.logger.error(
+        `Deployment watcher: redeploy failed for ${projectId}/${employeeId} (${env}): ${err instanceof Error ? err.message : err}`,
+      );
+    }
+  }
+
+  async getLogs(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+    options?: { type?: LogType; date?: string; tail?: number },
+  ): Promise<{ logs: string; error?: string }> {
+    const entry = this.runtimeStore.get(projectId, employeeId, env);
+
+    // Pass null providerType when no environment is set so LogsService
+    // uses the injected default provider (important for testability).
+    const providerType = entry?.environmentId
+      ? this.resolveEnvironment(entry.environmentId).providerType
+      : null;
+
+    return this.logsService.getDeploymentLogs(
+      entry?.containerId ?? null,
+      entry?.projectPath ?? null,
+      providerType,
+      {
+        type: options?.type ?? 'stdout',
+        date: options?.date,
+        tail: options?.tail,
+      },
+    );
+  }
+
+  /**
+   * Stops the deployment if it is active, then deletes the runtime record for this track.
+   * `fabricEnvironmentId` is the Fabric environment id (same as POST …/deploy `env`);
+   * `track` is the draft vs live slot (same as deploy `track`).
+   */
+  async undeploy(
+    projectId: string,
+    employeeId: string,
+    track: Env,
+    fabricEnvironmentId?: string,
+  ): Promise<{
+    removed: boolean;
+    projectId: string;
+    employeeId: string;
+    env: Env;
+  }> {
+    const existing = this.runtimeStore.get(projectId, employeeId, track);
+    if (!existing) {
+      throw new EmployeeNotFoundError(projectId, employeeId);
+    }
+
+    const project = await this.projectsDataService.get(projectId);
+    const expectedEnvId =
+      existing.environmentId ?? project?.environmentId ?? null;
+
+    const resolvedFabricEnvId =
+      fabricEnvironmentId ?? (expectedEnvId || undefined);
+
+    if (!resolvedFabricEnvId) {
+      throw new UndeployEnvironmentUnresolvedError(
+        projectId,
+        employeeId,
+        track,
+      );
+    }
+
+    if (expectedEnvId && resolvedFabricEnvId !== expectedEnvId) {
+      throw new DeploymentEnvironmentMismatchError(
+        projectId,
+        employeeId,
+        track,
+        expectedEnvId,
+        resolvedFabricEnvId,
+      );
+    }
+
+    if (!expectedEnvId) {
+      this.environmentsService.get(resolvedFabricEnvId);
+    }
+
+    if (ACTIVE_DEPLOYMENT_STATUSES.includes(existing.runtime)) {
+      await this.stop(projectId, employeeId, track);
+    }
+
+    const { removed } = await this.remove(projectId, employeeId, track);
+    return { removed, projectId, employeeId, env: track };
+  }
+
+  async remove(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<{ removed: boolean }> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+    if (!existing) {
+      throw new EmployeeNotFoundError(projectId, employeeId);
+    }
+
+    if (
+      existing.runtime !== 'stopped' &&
+      existing.runtime !== 'failed' &&
+      existing.runtime !== 'crashed'
+    ) {
+      throw new CannotRemoveError(employeeId, env, existing.runtime);
+    }
+
+    this.runtimeStore.delete(projectId, employeeId, env);
+
+    this.logger.log(`Removed ${projectId}/${employeeId} (${env})`);
+
+    this.eventsService.create({
+      userId: existing.userId,
+      projectId,
+      employeeId,
+      env,
+      type: 'remove',
+      message: 'Deployment removed',
+    });
+
+    await Promise.resolve();
+    return { removed: true };
+  }
+
+  async reconfigureDbAndRedeploy(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<DeployResult | null> {
+    const existing = this.runtimeStore.get(projectId, employeeId, env);
+    if (!existing?.projectPath || !existing.envVars) return null;
+
+    const appConfigStr = existing.envVars.APP_CONFIG ?? '{}';
+    let appConfig: Record<string, unknown>;
+    try {
+      appConfig = JSON.parse(appConfigStr) as Record<string, unknown>;
+    } catch {
+      return null;
+    }
+
+    const storeConfig = appConfig.store as Record<string, unknown> | undefined;
+    if (storeConfig?.type !== 'postgres') return null;
+
+    const resolved = this.resolveEnvironment(
+      existing.environmentId ?? undefined,
+    );
+    const name = this.processName(projectId, employeeId, env);
+
+    try {
+      const creds = await this.postgresProvisioner.createNewDatabase(
+        `${projectId}_${employeeId}_${env}`,
+      );
+
+      const newStoreConfig = {
+        ...storeConfig,
+        dbname: creds.dbname,
+        host: creds.host,
+        port: creds.port,
+        username: creds.username,
+        password: creds.password,
+      };
+      const newAppConfig = { ...appConfig, store: newStoreConfig };
+      const newEnvVars = {
+        ...existing.envVars,
+        APP_CONFIG: JSON.stringify(newAppConfig),
+      };
+
+      if (existing.containerId) {
+        if (existing.projectPath) {
+          await this.logsService.saveStdoutLogsBeforeStop(
+            existing.containerId,
+            existing.projectPath,
+            resolved.providerType,
+          );
+        }
+        await resolved.provider.stop(existing.containerId);
+      }
+
+      const now = new Date().toISOString();
+      this.runtimeStore.set(projectId, employeeId, env, {
+        ...existing,
+        envVars: newEnvVars,
+        containerId: null,
+        runtime: 'deploying',
+        lastError: null,
+        healthcheckRetryCount: 0,
+      });
+
+      const result = await resolved.provider.deploy(
+        name,
+        existing.projectPath,
+        newEnvVars,
+        existing.port!,
+        { projectId, employeeId },
+      );
+
+      if (!result.success) {
+        this.runtimeStore.updateField(projectId, employeeId, env, {
+          runtime: 'failed',
+          lastError: result.error ?? 'Reconfigure deploy failed',
+        });
+        this.eventsService.create({
+          userId: existing.userId,
+          projectId,
+          employeeId,
+          env,
+          type: 'error',
+          message: result.error ?? 'Reconfigure deploy failed',
+        });
+        return {
+          projectId,
+          employeeId,
+          env,
+          runtime: 'failed',
+          deployedAt: existing.deployedAt ?? now,
+        };
+      }
+
+      this.runtimeStore.set(projectId, employeeId, env, {
+        ...existing,
+        runtime: 'running',
+        containerId: result.containerId ?? null,
+        envVars: newEnvVars,
+        lastError: null,
+        healthcheckRetryCount: 0,
+      });
+
+      this.eventsService.create({
+        userId: existing.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'restart',
+        message: 'Database reconfigured and deployment redeployed',
+      });
+
+      this.logger.log(
+        `Reconfigured DB and redeployed ${projectId}/${employeeId} (${env})`,
+      );
+
+      return {
+        projectId,
+        employeeId,
+        env,
+        runtime: 'running',
+        deployedAt: now,
+        containerId: result.containerId ?? undefined,
+        port: existing.port ?? undefined,
+      };
+    } catch (err) {
+      const errMsg = err instanceof Error ? err.message : String(err);
+      this.runtimeStore.updateField(projectId, employeeId, env, {
+        runtime: 'failed',
+        lastError: errMsg,
+      });
+      this.eventsService.create({
+        userId: existing.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'error',
+        message: errMsg,
+      });
+      return null;
+    }
+  }
+
+  async checkHealth(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<{
+    ok: boolean;
+    status?: number;
+    error?: string;
+    healthData?: Record<string, unknown>;
+  }> {
+    const entry = this.runtimeStore.get(projectId, employeeId, env);
+    if (!entry?.port || entry.runtime !== 'running') {
+      return { ok: false, error: 'Deployment not running' };
+    }
+
+    const url = `http://localhost:${entry.port}/health`;
+    try {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), 5000);
+      const res = await fetch(url, {
+        method: 'GET',
+        signal: controller.signal,
+      });
+      clearTimeout(timeout);
+
+      let healthData: Record<string, unknown> | undefined;
+      try {
+        const text = await res.text();
+        healthData = text
+          ? (JSON.parse(text) as Record<string, unknown>)
+          : undefined;
+      } catch {
+        //
+      }
+
+      if (res.ok) {
+        return { ok: true, status: res.status, healthData };
+      }
+      return {
+        ok: false,
+        status: res.status,
+        error: `HTTP ${res.status}`,
+        healthData,
+      };
+    } catch (err) {
+      const error = err instanceof Error ? err.message : String(err);
+      return { ok: false, error };
+    }
+  }
+}