@agentlang/agentmanager 0.0.3

package/apps/fabric/src/healthcheck/healthcheck.service.ts

@@ -0,0 +1,494 @@
+import {
+  Injectable,
+  Inject,
+  Logger,
+  OnModuleInit,
+  OnModuleDestroy,
+} from '@nestjs/common';
+import type { ConfigType } from '@nestjs/config';
+import fabricConfig from '../config/configuration';
+import { RuntimeStore } from '../employees/runtime.store';
+import { EmployeesService } from '../employees/employees.service';
+import { DeploymentEventsService } from '../deployment-events/deployment-events.service';
+import { UserConfigService } from '../user-config/user-config.service';
+import { NotificationsService } from '../notifications/notifications.service';
+import { AiAgentService } from '../ai-agent/ai-agent.service';
+import { LogsService } from '../logs/logs.service';
+import { ProviderFactory } from '../providers/provider-factory';
+import type { ProviderType } from '../environments/environments.types';
+
+export interface HealthCheckResult {
+  ok: boolean;
+  status?: number;
+  error?: string;
+  healthData?: Record<string, unknown>;
+}
+
+@Injectable()
+export class HealthcheckService implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(HealthcheckService.name);
+  private intervalId: ReturnType<typeof setInterval> | null = null;
+
+  constructor(
+    @Inject(fabricConfig.KEY)
+    private readonly config: ConfigType<typeof fabricConfig>,
+    private readonly runtimeStore: RuntimeStore,
+    private readonly eventsService: DeploymentEventsService,
+    private readonly userConfigService: UserConfigService,
+    private readonly notificationsService: NotificationsService,
+    private readonly aiAgentService: AiAgentService,
+    private readonly logsService: LogsService,
+    private readonly providerFactory: ProviderFactory,
+    private readonly employeesService: EmployeesService,
+  ) {}
+
+  onModuleInit() {
+    const intervalMs = this.config.healthcheckIntervalMs;
+    if (intervalMs <= 0) {
+      this.logger.log('Healthcheck disabled (interval <= 0)');
+      return;
+    }
+
+    // this.intervalId = setInterval(() => {
+    //   this.runHealthcheck().catch((err) => {
+    //     this.logger.error(`Healthcheck error: ${err}`);
+    //   });
+    // }, intervalMs);
+
+    this.logger.log(`Healthcheck started (interval: ${intervalMs / 1000}s)`);
+  }
+
+  onModuleDestroy() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+  }
+
+  private async fetchHealth(url: string): Promise<HealthCheckResult> {
+    try {
+      const controller = new AbortController();
+      const timeout = setTimeout(() => controller.abort(), 5000);
+
+      const res = await fetch(url, {
+        method: 'GET',
+        signal: controller.signal,
+      });
+      clearTimeout(timeout);
+
+      let healthData: Record<string, unknown> | undefined;
+      try {
+        const text = await res.text();
+        if (text) {
+          healthData = JSON.parse(text) as Record<string, unknown>;
+        }
+      } catch {
+        // Not JSON
+      }
+
+      if (res.ok) {
+        return { ok: true, status: res.status, healthData };
+      }
+      return {
+        ok: false,
+        status: res.status,
+        error: `HTTP ${res.status}`,
+        healthData,
+      };
+    } catch (err) {
+      const error = err instanceof Error ? err.message : String(err);
+      return { ok: false, error };
+    }
+  }
+
+  private exceedsResourceThreshold(
+    healthData?: Record<string, unknown>,
+  ): boolean {
+    if (!healthData) return false;
+    const cpu = healthData.cpu_usage;
+    const mem = healthData.memory_usage;
+    const cpuNum =
+      typeof cpu === 'number'
+        ? cpu
+        : typeof cpu === 'string'
+          ? parseFloat(cpu)
+          : NaN;
+    const memNum =
+      typeof mem === 'number'
+        ? mem
+        : typeof mem === 'string'
+          ? parseFloat(mem)
+          : NaN;
+    return (
+      (!Number.isNaN(cpuNum) &&
+        cpuNum >= this.config.healthcheckCpuThreshold) ||
+      (!Number.isNaN(memNum) &&
+        memNum >= this.config.healthcheckMemoryThreshold)
+    );
+  }
+
+  private hasPostgresStore(envVars: Record<string, string> | null): boolean {
+    try {
+      const cfg = envVars?.APP_CONFIG;
+      if (!cfg) return false;
+      const parsed = JSON.parse(cfg) as Record<string, unknown>;
+      return (parsed.store as Record<string, unknown>)?.type === 'postgres';
+    } catch (err) {
+      this.logger.debug(
+        `Failed to parse APP_CONFIG for postgres check: ${err instanceof Error ? err.message : err}`,
+      );
+      return false;
+    }
+  }
+
+  private async runHealthcheck(): Promise<void> {
+    const allEntries = this.runtimeStore.listAll();
+    const running = allEntries.filter(
+      (e) =>
+        e.entry.runtime === 'running' && e.entry.containerId && e.entry.port,
+    );
+
+    for (const entry of running) {
+      const { projectId, employeeId, env, entry: runtime } = entry;
+      const providerType = this.resolveProviderType(runtime.environmentId);
+
+      let result: HealthCheckResult;
+      const provider = this.providerFactory.get(providerType);
+      if (provider.checkHealth) {
+        result = await provider.checkHealth(
+          runtime.containerId!,
+          runtime.port!,
+        );
+      } else {
+        result = await this.fetchHealth(
+          `http://localhost:${runtime.port}/health`,
+        );
+      }
+
+      await this.processHealthResult(
+        projectId,
+        employeeId,
+        env,
+        runtime,
+        result,
+        providerType,
+      );
+    }
+  }
+
+  private resolveProviderType(environmentId: string | null): ProviderType {
+    if (!environmentId) return this.config.provider;
+    try {
+      // The providerFactory can resolve type from environment
+      return this.config.provider;
+    } catch {
+      return this.config.provider;
+    }
+  }
+
+  private async processHealthResult(
+    projectId: string,
+    employeeId: string,
+    env: string,
+    runtime: {
+      userId: string | null;
+      containerId: string | null;
+      port: number | null;
+      healthcheckRetryCount: number;
+      environmentId: string | null;
+      projectPath: string | null;
+      envVars: Record<string, string> | null;
+    },
+    result: HealthCheckResult,
+    providerType: ProviderType,
+  ): Promise<void> {
+    const resourceExceeded =
+      result.ok && this.exceedsResourceThreshold(result.healthData);
+    const failed = !result.ok || resourceExceeded;
+    const errorMsg = resourceExceeded
+      ? `Resource threshold exceeded (cpu/memory). Health: ${JSON.stringify(result.healthData)}`
+      : (result.error ?? 'Unknown');
+
+    if (!failed) {
+      if ((runtime.healthcheckRetryCount ?? 0) > 0) {
+        this.runtimeStore.updateField(projectId, employeeId, env, {
+          healthcheckRetryCount: 0,
+        });
+      }
+      return;
+    }
+
+    const maxRetries =
+      this.userConfigService.getHealthcheckMaxRetries('__anonymous__');
+    const currentRetries = runtime.healthcheckRetryCount ?? 0;
+    const nextRetries = currentRetries + 1;
+
+    if (nextRetries >= maxRetries) {
+      await this.handleFailureWithAI(
+        projectId,
+        employeeId,
+        env,
+        runtime,
+        errorMsg,
+        nextRetries,
+        maxRetries,
+        true,
+        providerType,
+        result.healthData,
+      );
+    } else if (runtime.containerId) {
+      try {
+        const provider = this.providerFactory.get(providerType);
+        await provider.restart(runtime.containerId);
+        this.runtimeStore.updateField(projectId, employeeId, env, {
+          healthcheckRetryCount: nextRetries,
+        });
+        this.eventsService.create({
+          userId: runtime.userId,
+          projectId,
+          employeeId,
+          env,
+          type: 'healthcheck_retry',
+          message: `Healthcheck failed (${errorMsg}). Container restarted. Retry ${nextRetries}/${maxRetries}.`,
+          retryCount: nextRetries,
+        });
+      } catch (restartErr) {
+        const errMsg =
+          restartErr instanceof Error ? restartErr.message : String(restartErr);
+        await this.handleFailureWithAI(
+          projectId,
+          employeeId,
+          env,
+          runtime,
+          `Restart failed: ${errMsg}`,
+          nextRetries,
+          maxRetries,
+          false,
+          providerType,
+          result.healthData,
+        );
+      }
+    }
+  }
+
+  private async handleFailureWithAI(
+    projectId: string,
+    employeeId: string,
+    env: string,
+    runtime: {
+      userId: string | null;
+      containerId: string | null;
+      projectPath: string | null;
+      envVars: Record<string, string> | null;
+      environmentId: string | null;
+    },
+    error: string,
+    retryCount: number,
+    maxRetries: number,
+    exhausted: boolean,
+    providerType: ProviderType,
+    healthData?: Record<string, unknown>,
+  ): Promise<void> {
+    let logs = '';
+    try {
+      const res = await this.logsService.getDeploymentLogs(
+        runtime.containerId,
+        runtime.projectPath,
+        providerType,
+        { type: 'stdout' },
+      );
+      logs = res.logs ?? '';
+    } catch (err) {
+      this.logger.warn(
+        `Failed to fetch logs for AI recovery (${projectId}/${employeeId}/${env}): ${err instanceof Error ? err.message : err}`,
+      );
+    }
+
+    const action = await this.aiAgentService.getRecoveryAction({
+      projectId,
+      employeeId,
+      env,
+      error,
+      retryCount,
+      maxRetries,
+      logs: logs.slice(-8000),
+      hasPostgresStore: this.hasPostgresStore(runtime.envVars),
+      hasKubernetes: providerType === 'eks',
+      healthData,
+    });
+
+    const provider = this.providerFactory.get(providerType);
+
+    const notifyUserId = runtime.userId ?? '__anonymous__';
+
+    if (action === 'restart_app' && runtime.containerId) {
+      try {
+        await provider.restart(runtime.containerId);
+        this.runtimeStore.updateField(projectId, employeeId, env, {
+          healthcheckRetryCount: exhausted ? 0 : retryCount,
+        });
+        this.eventsService.create({
+          userId: runtime.userId,
+          projectId,
+          employeeId,
+          env,
+          type: 'healthcheck_retry',
+          message: exhausted
+            ? 'AI recovery: restarted app after exhaustion.'
+            : 'AI recovery: restarted app after restart failure.',
+          retryCount: exhausted ? 0 : retryCount,
+        });
+        return;
+      } catch (err) {
+        this.logger.warn(
+          `AI restart_app recovery failed for ${projectId}/${employeeId}/${env}: ${err instanceof Error ? err.message : err}`,
+        );
+      }
+    }
+
+    if (action === 'rescale_kubernetes' && providerType === 'eks') {
+      try {
+        if (provider.scaleUp && runtime.containerId) {
+          await provider.scaleUp(runtime.containerId);
+          this.runtimeStore.updateField(projectId, employeeId, env, {
+            healthcheckRetryCount: exhausted ? 0 : retryCount,
+          });
+          this.eventsService.create({
+            userId: runtime.userId,
+            projectId,
+            employeeId,
+            env,
+            type: 'healthcheck_retry',
+            message: 'AI recovery: scaled up Kubernetes deployment.',
+            retryCount: exhausted ? 0 : retryCount,
+          });
+          return;
+        }
+      } catch (err) {
+        this.logger.error(`AI rescale_kubernetes failed: ${err}`);
+      }
+    }
+
+    if (action === 'reconfigure_db') {
+      try {
+        const result = await this.employeesService.reconfigureDbAndRedeploy(
+          projectId,
+          employeeId,
+          env as 'draft' | 'live',
+        );
+        if (result) {
+          this.eventsService.create({
+            userId: runtime.userId,
+            projectId,
+            employeeId,
+            env,
+            type: 'healthcheck_retry',
+            message: 'AI recovery: database reconfigured and redeployed.',
+            retryCount: exhausted ? 0 : retryCount,
+          });
+          return;
+        }
+      } catch (err) {
+        this.logger.error(`AI reconfigure_db failed: ${err}`);
+      }
+    }
+
+    if (action === 'send_email_to_user') {
+      const msg = exhausted
+        ? `Healthcheck failed after ${maxRetries} retries. Last error: ${error}. Container stopped.`
+        : `Healthcheck failed and restart failed: ${error}`;
+      const eventType = exhausted
+        ? ('healthcheck_exhausted' as const)
+        : ('healthcheck_failed' as const);
+      this.notificationsService
+        .notifyOnFailure(notifyUserId, {
+          projectId,
+          employeeId,
+          env,
+          type: eventType,
+          message: msg,
+          retryCount,
+        })
+        .catch((err) => {
+          this.logger.warn(`Failed to send failure notification: ${err}`);
+        });
+    }
+
+    // Default: stop if exhausted
+    if (exhausted && runtime.containerId) {
+      if (runtime.projectPath) {
+        await this.logsService.saveStdoutLogsBeforeStop(
+          runtime.containerId,
+          runtime.projectPath,
+          providerType,
+        );
+      }
+      await provider.stop(runtime.containerId);
+
+      const existing = this.runtimeStore.get(
+        projectId,
+        employeeId,
+        env as 'draft' | 'live',
+      );
+      if (existing) {
+        this.runtimeStore.set(projectId, employeeId, env as 'draft' | 'live', {
+          ...existing,
+          runtime: 'crashed',
+          lastError: `Healthcheck failed after ${maxRetries} retries: ${error}`,
+          healthcheckRetryCount: retryCount,
+          containerId: null,
+        });
+      }
+
+      this.eventsService.create({
+        userId: runtime.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'healthcheck_exhausted',
+        message: `Healthcheck failed after ${maxRetries} retries. Last error: ${error}. Container stopped.`,
+        retryCount,
+      });
+
+      if (!action || action !== 'send_email_to_user') {
+        this.notificationsService
+          .notifyOnFailure(notifyUserId, {
+            projectId,
+            employeeId,
+            env,
+            type: 'healthcheck_exhausted',
+            message: `Healthcheck failed after ${maxRetries} retries. Last error: ${error}. Container stopped.`,
+            retryCount,
+          })
+          .catch((err) => {
+            this.logger.warn(`Failed to send failure notification: ${err}`);
+          });
+      }
+    } else if (!exhausted) {
+      const failedMsg = `Healthcheck failed and restart failed: ${error}`;
+      this.eventsService.create({
+        userId: runtime.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'healthcheck_failed',
+        message: failedMsg,
+        retryCount,
+      });
+      if (!action || action !== 'send_email_to_user') {
+        this.notificationsService
+          .notifyOnFailure(notifyUserId, {
+            projectId,
+            employeeId,
+            env,
+            type: 'healthcheck_failed',
+            message: failedMsg,
+            retryCount,
+          })
+          .catch((err) => {
+            this.logger.warn(`Failed to send failure notification: ${err}`);
+          });
+      }
+    }
+  }
+}

package/apps/fabric/src/logs/logs.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { LogsService } from './logs.service';
+import { ProvidersModule } from '../providers/providers.module';
+
+@Module({
+  imports: [ProvidersModule],
+  providers: [LogsService],
+  exports: [LogsService],
+})
+export class LogsModule {}

package/apps/fabric/src/logs/logs.service.ts

@@ -0,0 +1,149 @@
+import { Injectable, Inject, Logger } from '@nestjs/common';
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import {
+  DEPLOYMENT_PROVIDER,
+  type DeploymentProvider,
+} from '../providers/provider.interface';
+import { ProviderFactory } from '../providers/provider-factory';
+import type { ProviderType } from '../environments/environments.types';
+
+const STDOUT_LAST_LOG = 'logs/app-stdout-last.log';
+
+export type LogType = 'stdout' | 'file';
+
+function formatDate(d: Date): string {
+  const y = d.getFullYear();
+  const m = String(d.getMonth() + 1).padStart(2, '0');
+  const day = String(d.getDate()).padStart(2, '0');
+  return `${y}-${m}-${day}`;
+}
+
+@Injectable()
+export class LogsService {
+  private readonly logger = new Logger(LogsService.name);
+
+  constructor(
+    @Inject(DEPLOYMENT_PROVIDER)
+    private readonly defaultProvider: DeploymentProvider,
+    private readonly providerFactory: ProviderFactory,
+  ) {}
+
+  async getDeploymentLogs(
+    containerId: string | null,
+    projectPath: string | null,
+    providerType: ProviderType | null,
+    options: { type: LogType; date?: string; tail?: number },
+  ): Promise<{ logs: string; error?: string }> {
+    const { type, date, tail } = options;
+    const logDate = date ?? formatDate(new Date());
+
+    if (type === 'stdout') {
+      const result = await this.getStdoutLogs(containerId, providerType, tail);
+      if (result.error && projectPath) {
+        const savedStdout = await this.getSavedStdoutLogs(projectPath);
+        if (savedStdout.logs) return { logs: savedStdout.logs };
+        const fileResult = await this.getFileLogs(projectPath, logDate);
+        if (fileResult.logs) return { logs: fileResult.logs };
+      }
+      return result;
+    }
+
+    if (type === 'file') {
+      if (!projectPath) {
+        return {
+          logs: '',
+          error: 'Deployment has no project path (not yet deployed)',
+        };
+      }
+      return this.getFileLogs(projectPath, logDate);
+    }
+
+    return { logs: '', error: 'Invalid log type' };
+  }
+
+  private async getStdoutLogs(
+    containerId: string | null,
+    providerType: ProviderType | null,
+    tail?: number,
+  ): Promise<{ logs: string; error?: string }> {
+    if (!containerId) {
+      return { logs: '', error: 'Deployment has no container/process' };
+    }
+
+    const provider = providerType
+      ? this.providerFactory.get(providerType)
+      : this.defaultProvider;
+
+    try {
+      const logs = await provider.getLogs(containerId, tail ?? 1000);
+      return { logs };
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return { logs: '', error: msg };
+    }
+  }
+
+  async saveStdoutLogsBeforeStop(
+    containerId: string,
+    projectPath: string,
+    providerType: ProviderType,
+  ): Promise<void> {
+    try {
+      const provider = this.providerFactory.get(providerType);
+      const logs = await provider.getLogs(containerId, 1000);
+      if (!logs) return;
+
+      const logPath = path.join(projectPath, STDOUT_LAST_LOG);
+      await fs.promises.mkdir(path.dirname(logPath), { recursive: true });
+      await fs.promises.writeFile(logPath, logs, 'utf-8');
+    } catch (err) {
+      this.logger.warn(
+        `Failed to save stdout logs before stop for ${containerId}: ${err instanceof Error ? err.message : err}`,
+      );
+    }
+  }
+
+  private async getSavedStdoutLogs(
+    projectPath: string,
+  ): Promise<{ logs: string; error?: string }> {
+    const logPath = path.join(projectPath, STDOUT_LAST_LOG);
+    const resolved = path.resolve(logPath);
+    const projectResolved = path.resolve(projectPath);
+    if (!resolved.startsWith(projectResolved)) return { logs: '' };
+
+    try {
+      const content = await fs.promises.readFile(resolved, 'utf-8');
+      return { logs: content };
+    } catch (err) {
+      this.logger.debug(
+        `No saved stdout logs at ${resolved}: ${err instanceof Error ? err.message : err}`,
+      );
+      return { logs: '' };
+    }
+  }
+
+  private async getFileLogs(
+    projectPath: string,
+    date: string,
+  ): Promise<{ logs: string; error?: string }> {
+    const logPath = path.join(projectPath, 'logs', `app-${date}.log`);
+    const resolved = path.resolve(logPath);
+    const projectResolved = path.resolve(projectPath);
+
+    if (!resolved.startsWith(projectResolved)) {
+      return { logs: '', error: 'Invalid log path' };
+    }
+
+    try {
+      const content = await fs.promises.readFile(resolved, 'utf-8');
+      return { logs: content };
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
+        return { logs: '', error: `Log file not found: logs/app-${date}.log` };
+      }
+      const msg = err instanceof Error ? err.message : String(err);
+      return { logs: '', error: msg };
+    }
+  }
+}

package/apps/fabric/src/main.ts

@@ -0,0 +1,57 @@
+import { NestFactory } from '@nestjs/core';
+import {
+  FastifyAdapter,
+  NestFastifyApplication,
+} from '@nestjs/platform-fastify';
+import { Logger } from '@nestjs/common';
+import { ZodValidationPipe } from 'nestjs-zod';
+import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger';
+import helmet from '@fastify/helmet';
+import { AppModule } from './app.module';
+import { ConfigService } from '@nestjs/config';
+
+async function bootstrap() {
+  const app = await NestFactory.create<NestFastifyApplication>(
+    AppModule,
+    new FastifyAdapter({ bodyLimit: 1_048_576 }),
+  );
+
+  await app.register(helmet as Parameters<typeof app.register>[0], {
+    contentSecurityPolicy: false,
+  });
+
+  app.setGlobalPrefix('api');
+  app.enableShutdownHooks();
+  app.useGlobalPipes(new ZodValidationPipe());
+
+  const config = app.get(ConfigService);
+  const port = config.get<number>('fabric.port', 3003);
+  const corsOrigins = config.get<string[]>('fabric.cors.origins', [
+    'http://localhost:5173',
+    'http://127.0.0.1:5173',
+  ]);
+
+  app.enableCors({
+    origin: corsOrigins,
+    credentials: true,
+  });
+
+  const swaggerConfig = new DocumentBuilder()
+    .setTitle('Fabric API')
+    .setDescription('NestJS deployment orchestration service')
+    .setVersion('1.0')
+    .build();
+  const document = SwaggerModule.createDocument(app, swaggerConfig);
+  SwaggerModule.setup('docs', app, document);
+
+  await app.listen(port, '0.0.0.0');
+
+  const logger = new Logger('Bootstrap');
+  logger.log(`Fabric listening on port ${port}`);
+}
+
+bootstrap().catch((err) => {
+  const logger = new Logger('Bootstrap');
+  logger.error(`Failed to start Fabric: ${err}`);
+  process.exit(1);
+});