@agentlang/agentmanager 0.0.3

package/apps/fabric/src/versions/versions.store.ts

@@ -0,0 +1,160 @@
+import {
+  Injectable,
+  OnModuleInit,
+  OnModuleDestroy,
+  Logger,
+} from '@nestjs/common';
+import type { Repository } from 'typeorm';
+import { FileStoreFactory, FileStore, DataSourceService } from '@app/common';
+import { VersionEntity } from '../database/entities';
+import type { VersionSnapshot } from './versions.types';
+
+interface PersistedData {
+  counter: number;
+  versions: Record<string, VersionSnapshot[]>;
+}
+
+@Injectable()
+export class VersionsStore implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(VersionsStore.name);
+  private counter = 0;
+  private readonly versions = new Map<string, VersionSnapshot[]>();
+  private readonly fileStore: FileStore<PersistedData>;
+  private repo: Repository<VersionEntity> | null = null;
+
+  constructor(
+    fileStoreFactory: FileStoreFactory,
+    private readonly dataSourceService: DataSourceService,
+  ) {
+    this.fileStore = fileStoreFactory.create<PersistedData>('versions.json');
+  }
+
+  async onModuleInit() {
+    this.repo = this.dataSourceService.getRepository(VersionEntity);
+
+    if (this.repo) {
+      const rows = await this.repo.find({ order: { rowId: 'ASC' } });
+      for (const row of rows) {
+        const k = this.key(row.projectId, row.employeeId);
+        const list = this.versions.get(k) ?? [];
+        list.push(this.entityToSnapshot(row));
+        this.versions.set(k, list);
+      }
+      // Derive counter from existing version IDs
+      for (const list of this.versions.values()) {
+        for (const v of list) {
+          const match = v.versionId.match(/^v(\d+)$/);
+          if (match) {
+            this.counter = Math.max(this.counter, parseInt(match[1], 10));
+          }
+        }
+      }
+      this.logger.log(
+        `Loaded ${this.versions.size} version histories (counter=${this.counter}) from database`,
+      );
+    } else {
+      const data = await this.fileStore.load();
+      if (data) {
+        this.counter = data.counter ?? 0;
+        if (data.versions) {
+          for (const [key, snapshots] of Object.entries(data.versions)) {
+            this.versions.set(key, snapshots);
+          }
+        }
+        this.logger.log(
+          `Loaded ${this.versions.size} version histories (counter=${this.counter}) from disk`,
+        );
+      }
+    }
+  }
+
+  private entityToSnapshot(entity: VersionEntity): VersionSnapshot {
+    return {
+      versionId: entity.versionId,
+      projectId: entity.projectId,
+      employeeId: entity.employeeId,
+      createdAt: entity.createdAt,
+      snapshot: entity.snapshot,
+    };
+  }
+
+  private snapshotToEntity(snapshot: VersionSnapshot): VersionEntity {
+    const entity = new VersionEntity();
+    entity.versionId = snapshot.versionId;
+    entity.projectId = snapshot.projectId;
+    entity.employeeId = snapshot.employeeId;
+    entity.createdAt = snapshot.createdAt;
+    entity.snapshot = snapshot.snapshot;
+    return entity;
+  }
+
+  private key(projectId: string, employeeId: string): string {
+    return `${projectId}/${employeeId}`;
+  }
+
+  generateId(): string {
+    return `v${++this.counter}`;
+  }
+
+  append(snapshot: VersionSnapshot): void {
+    const k = this.key(snapshot.projectId, snapshot.employeeId);
+    const list = this.versions.get(k) ?? [];
+    list.push(snapshot);
+    this.versions.set(k, list);
+
+    if (this.repo) {
+      this.repo
+        .save(this.snapshotToEntity(snapshot))
+        .catch((err) => this.logger.error(`Failed to persist version: ${err}`));
+    } else {
+      this.persistFile();
+    }
+  }
+
+  private persistFile(): void {
+    const versions: Record<string, VersionSnapshot[]> = {};
+    for (const [key, snapshots] of this.versions) {
+      versions[key] = snapshots;
+    }
+    this.fileStore.save({ counter: this.counter, versions });
+  }
+
+  list(projectId: string, employeeId: string): VersionSnapshot[] {
+    return this.versions.get(this.key(projectId, employeeId)) ?? [];
+  }
+
+  get(
+    projectId: string,
+    employeeId: string,
+    versionId: string,
+  ): VersionSnapshot | undefined {
+    return this.list(projectId, employeeId).find(
+      (v) => v.versionId === versionId,
+    );
+  }
+
+  getLatest(
+    projectId: string,
+    employeeId: string,
+  ): VersionSnapshot | undefined {
+    const list = this.list(projectId, employeeId);
+    return list[list.length - 1];
+  }
+
+  getPrevious(
+    projectId: string,
+    employeeId: string,
+  ): VersionSnapshot | undefined {
+    const list = this.list(projectId, employeeId);
+    return list.length >= 2 ? list[list.length - 2] : undefined;
+  }
+
+  async onModuleDestroy() {
+    if (!this.repo) {
+      await this.fileStore.flush();
+    }
+    this.logger.log(
+      `VersionsStore shutting down with ${this.versions.size} version histories`,
+    );
+  }
+}

package/apps/fabric/src/versions/versions.types.ts

@@ -0,0 +1,12 @@
+export interface VersionSnapshot {
+  versionId: string;
+  projectId: string;
+  employeeId: string;
+  createdAt: string;
+  snapshot: Record<string, unknown>;
+}
+
+export interface VersionSummary {
+  versionId: string;
+  createdAt: string;
+}

package/apps/fabric/src/watcher/deployment-watcher.module.ts

@@ -0,0 +1,22 @@
+import { Module } from '@nestjs/common';
+import { EmployeesModule } from '../employees/employees.module';
+import { DeploymentWatcherService } from './deployment-watcher.service';
+import { PodProbeService } from './pod-probe.service';
+import { LogsModule } from '../logs/logs.module';
+import { AiAgentModule } from '../ai-agent/ai-agent.module';
+import { NotificationsModule } from '../notifications/notifications.module';
+import { DeploymentEventsModule } from '../deployment-events/deployment-events.module';
+import { EnvironmentsModule } from '../environments/environments.module';
+
+@Module({
+  imports: [
+    EmployeesModule,
+    LogsModule,
+    AiAgentModule,
+    NotificationsModule,
+    DeploymentEventsModule,
+    EnvironmentsModule,
+  ],
+  providers: [DeploymentWatcherService, PodProbeService],
+})
+export class DeploymentWatcherModule {}

package/apps/fabric/src/watcher/deployment-watcher.service.ts

@@ -0,0 +1,92 @@
+import {
+  Inject,
+  Injectable,
+  Logger,
+  OnModuleDestroy,
+  OnModuleInit,
+} from '@nestjs/common';
+import type { ConfigType } from '@nestjs/config';
+import fabricConfig from '../config/configuration';
+import { RuntimeStore } from '../employees/runtime.store';
+import { EmployeesService } from '../employees/employees.service';
+import { PodProbeService } from './pod-probe.service';
+
+/**
+ * Polls PM2/Docker via `EmployeesService.recoverIfProcessMissing` on a fixed
+ * interval (`healthcheckIntervalMs` in Fabric config, default 15s).
+ * Disabled when that interval is `<= 0` (e.g. tests set `HEALTHCHECK_INTERVAL_MS=0`).
+ */
+@Injectable()
+export class DeploymentWatcherService implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(DeploymentWatcherService.name);
+  private intervalId: ReturnType<typeof setInterval> | null = null;
+
+  constructor(
+    @Inject(fabricConfig.KEY)
+    private readonly config: ConfigType<typeof fabricConfig>,
+    private readonly runtimeStore: RuntimeStore,
+    private readonly employeesService: EmployeesService,
+    private readonly podProbeService: PodProbeService,
+  ) {}
+
+  onModuleInit() {
+    const intervalMs = this.config.healthcheckIntervalMs;
+    if (intervalMs <= 0) {
+      this.logger.log(
+        'Deployment watcher disabled (healthcheckIntervalMs <= 0)',
+      );
+      return;
+    }
+
+    this.intervalId = setInterval(() => {
+      this.runWatchCycle().catch((err) => {
+        this.logger.error(
+          `Deployment watcher cycle failed: ${err instanceof Error ? err.message : err}`,
+        );
+      });
+    }, intervalMs);
+
+    this.logger.log(
+      `Deployment watcher started (interval: ${intervalMs / 1000}s)`,
+    );
+  }
+
+  onModuleDestroy() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+      this.intervalId = null;
+    }
+  }
+
+  private async runWatchCycle(): Promise<void> {
+    const all = this.runtimeStore.listAll();
+    const running = all.filter(
+      (e) => e.entry.runtime === 'running' && e.entry.containerId,
+    );
+
+    for (const { projectId, employeeId, env } of running) {
+      try {
+        await this.employeesService.recoverIfProcessMissing(
+          projectId,
+          employeeId,
+          env,
+        );
+      } catch (err) {
+        this.logger.warn(
+          `Deployment watcher: recovery failed for ${projectId}/${employeeId}/${env}: ${err instanceof Error ? err.message : err}`,
+        );
+      }
+      try {
+        await this.podProbeService.probeRunningDeployment(
+          projectId,
+          employeeId,
+          env,
+        );
+      } catch (err) {
+        this.logger.warn(
+          `Deployment watcher: pod probe failed for ${projectId}/${employeeId}/${env}: ${err instanceof Error ? err.message : err}`,
+        );
+      }
+    }
+  }
+}

package/apps/fabric/src/watcher/pod-probe.service.ts

@@ -0,0 +1,231 @@
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import type { ConfigType } from '@nestjs/config';
+import fabricConfig from '../config/configuration';
+import { RuntimeStore } from '../employees/runtime.store';
+import { EmployeesService } from '../employees/employees.service';
+import type { Env } from '../employees/employees.types';
+import { LogsService } from '../logs/logs.service';
+import { AiAgentService } from '../ai-agent/ai-agent.service';
+import type {
+  ProbeAiPreviousAction,
+  ProbeInvestigationResult,
+} from '../ai-agent/probe-investigation.types';
+import { NotificationsService } from '../notifications/notifications.service';
+import { DeploymentEventsService } from '../deployment-events/deployment-events.service';
+import type { ProviderType } from '../environments/environments.types';
+import { EnvironmentsService } from '../environments/environments.service';
+
+const MAX_PROBE_AI_HISTORY = 12;
+
+/**
+ * After the process exists (PM2/Docker), POST `http://127.0.0.1:{port}/` with `{}`.
+ * If the response is not HTTP 200, fetch stdout logs and ask an LLM whether to restart
+ * or escalate (SES email to `FABRIC_ESCALATION_TO_EMAIL`).
+ */
+@Injectable()
+export class PodProbeService {
+  private readonly logger = new Logger(PodProbeService.name);
+  /** deployKey → last time we ran an AI investigation for a failed probe */
+  private readonly lastAiInvestigationAt = new Map<string, number>();
+  /** deployKey → prior AI probe decisions (for model context) */
+  private readonly probeAiHistory = new Map<string, ProbeAiPreviousAction[]>();
+
+  constructor(
+    @Inject(fabricConfig.KEY)
+    private readonly config: ConfigType<typeof fabricConfig>,
+    private readonly runtimeStore: RuntimeStore,
+    private readonly employeesService: EmployeesService,
+    private readonly logsService: LogsService,
+    private readonly aiAgentService: AiAgentService,
+    private readonly notificationsService: NotificationsService,
+    private readonly eventsService: DeploymentEventsService,
+    private readonly environmentsService: EnvironmentsService,
+  ) {}
+
+  async probeRunningDeployment(
+    projectId: string,
+    employeeId: string,
+    env: Env,
+  ): Promise<void> {
+    const entry = this.runtimeStore.get(projectId, employeeId, env);
+    if (!entry || entry.runtime !== 'running' || entry.port == null) {
+      return;
+    }
+
+    const port = entry.port;
+    const probe = await this.postProbe(port);
+    if (probe.ok) {
+      return;
+    }
+
+    const deployKey = `${projectId}/${employeeId}/${env}`;
+    const now = Date.now();
+    const cooldown = this.config.probeAiCooldownMs;
+    const lastAi = this.lastAiInvestigationAt.get(deployKey) ?? 0;
+    if (now - lastAi < cooldown) {
+      this.logger.debug(
+        `Pod probe: skip AI for ${deployKey} (cooldown ${cooldown}ms)`,
+      );
+      return;
+    }
+    this.lastAiInvestigationAt.set(deployKey, now);
+
+    const providerType = this.resolveProviderType(entry.environmentId);
+    let logs = '';
+    try {
+      const logRes = await this.logsService.getDeploymentLogs(
+        entry.containerId,
+        entry.projectPath,
+        providerType,
+        { type: 'stdout', tail: 2000 },
+      );
+      logs = logRes.logs ?? '';
+      if (logRes.error) {
+        logs = `${logs}\n(log fetch note: ${logRes.error})`;
+      }
+    } catch (err) {
+      this.logger.warn(
+        `Pod probe: could not load logs for ${deployKey}: ${err instanceof Error ? err.message : err}`,
+      );
+    }
+
+    const previousActions = this.probeAiHistory.get(deployKey) ?? [];
+    const investigation: ProbeInvestigationResult | null =
+      await this.aiAgentService.getProbeInvestigationDecision({
+        projectId,
+        employeeId,
+        env,
+        httpStatus: probe.status,
+        fetchError: probe.error,
+        logs: logs.slice(-24_000),
+        previousActions,
+      });
+
+    if (!investigation) {
+      this.logger.warn(
+        `Pod probe: no AI decision for ${deployKey} (configure ANTHROPIC_API_KEY or OPENAI_API_KEY, or FABRIC_AI_PROVIDER)`,
+      );
+      return;
+    }
+
+    this.logger.log(
+      `Pod probe ${deployKey}: AI action=${investigation.action} reasoning=${investigation.reasoning}`,
+    );
+
+    if (investigation.action === 'restart') {
+      this.recordProbeHistory(deployKey, investigation);
+      try {
+        await this.employeesService.restart(projectId, employeeId, env);
+        this.eventsService.create({
+          userId: entry.userId,
+          projectId,
+          employeeId,
+          env,
+          type: 'probe_restart',
+          message: eventMessage(probe, 'restart', investigation.reasoning),
+        });
+      } catch (err) {
+        this.logger.error(
+          `Pod probe: restart failed for ${deployKey}: ${err instanceof Error ? err.message : err}`,
+        );
+      }
+      return;
+    }
+
+    if (investigation.action === 'escalate') {
+      this.recordProbeHistory(deployKey, investigation);
+      await this.notificationsService.sendEscalationEmail({
+        subject: `[Fabric] Escalation: ${projectId}/${employeeId} (${env}) — probe not 200`,
+        body: [
+          `Deployment: ${projectId} / ${employeeId} / ${env}`,
+          `Port: ${port}`,
+          `Probe: ${probeSummary(probe)}`,
+          '',
+          '--- AI reasoning ---',
+          investigation.reasoning,
+          '',
+          '--- Recent stdout logs ---',
+          logs.slice(-12_000) || '(none)',
+        ].join('\n'),
+      });
+      this.eventsService.create({
+        userId: entry.userId,
+        projectId,
+        employeeId,
+        env,
+        type: 'probe_escalate',
+        message: eventMessage(probe, 'escalate', investigation.reasoning),
+      });
+      return;
+    }
+  }
+
+  private recordProbeHistory(
+    deployKey: string,
+    result: ProbeInvestigationResult,
+  ): void {
+    const prior = this.probeAiHistory.get(deployKey);
+    const list: ProbeAiPreviousAction[] = prior ? prior.slice() : [];
+    const action = result.action;
+    const reasoning = result.reasoning;
+    list.push({
+      atIso: new Date().toISOString(),
+      action,
+      reasoning,
+    });
+    while (list.length > MAX_PROBE_AI_HISTORY) {
+      list.shift();
+    }
+    this.probeAiHistory.set(deployKey, list);
+  }
+
+  private resolveProviderType(
+    environmentId: string | null,
+  ): ProviderType | null {
+    if (!environmentId) {
+      return this.config.provider;
+    }
+    try {
+      return this.environmentsService.get(environmentId).providerType;
+    } catch {
+      return this.config.provider;
+    }
+  }
+
+  private async postProbe(port: number): Promise<{
+    ok: boolean;
+    status?: number;
+    error?: string;
+  }> {
+    const url = `http://127.0.0.1:${port}/`;
+    try {
+      const controller = new AbortController();
+      const t = setTimeout(() => controller.abort(), 8000);
+      const res = await fetch(url, {
+        method: 'GET',
+        signal: controller.signal,
+      });
+      clearTimeout(t);
+      const ok = res.status === 200;
+      return { ok, status: res.status };
+    } catch (err) {
+      const error = err instanceof Error ? err.message : String(err);
+      return { ok: false, error };
+    }
+  }
+}
+
+function probeSummary(probe: { status?: number; error?: string }): string {
+  if (probe.status != null) return `HTTP ${probe.status}`;
+  if (probe.error) return probe.error;
+  return 'unknown';
+}
+
+function eventMessage(
+  probe: { status?: number; error?: string },
+  action: 'restart' | 'escalate',
+  reasoning: string,
+): string {
+  const r = reasoning.replace(/\s+/g, ' ').trim();
+  return `POST probe did not return 200 (${probeSummary(probe)}). AI chose ${action}. Reasoning: ${r}`;
+}

package/apps/fabric/tsconfig.app.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/apps/fabric"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "test", "**/*spec.ts"]
+}

package/apps/integrations/prisma/prisma.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig, env } from 'prisma/config';
+
+export default defineConfig({
+  schema: './schema.postgres.prisma',
+  datasource: {
+    url: env('INTEGRATIONS_DATABASE_URL_POSTGRES'),
+  },
+});

package/apps/integrations/prisma/prisma.sqlite.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig, env } from 'prisma/config';
+
+export default defineConfig({
+  schema: './schema.sqlite.prisma',
+  datasource: {
+    url: env('INTEGRATIONS_DATABASE_URL_SQLITE'),
+  },
+});

package/apps/integrations/prisma/schema.postgres.prisma

@@ -0,0 +1,27 @@
+generator client {
+  provider = "prisma-client"
+  output   = "../src/generated/prisma-postgres"
+}
+
+datasource db {
+  provider = "postgresql"
+}
+
+model IntegrationEntity {
+  entity       String
+  id           String
+  tenantId     String?
+  providerId   String?
+  connectionId String?
+  status       String?
+  data         Json
+  createdAt    DateTime @default(now())
+  updatedAt    DateTime @updatedAt
+
+  @@id([entity, id])
+  @@index([entity])
+  @@index([entity, tenantId])
+  @@index([entity, providerId])
+  @@index([entity, connectionId])
+  @@index([entity, status])
+}

package/apps/integrations/prisma/schema.sqlite.prisma

@@ -0,0 +1,27 @@
+generator client {
+  provider = "prisma-client"
+  output   = "../src/generated/prisma-sqlite"
+}
+
+datasource db {
+  provider = "sqlite"
+}
+
+model IntegrationEntity {
+  entity       String
+  id           String
+  tenantId     String?
+  providerId   String?
+  connectionId String?
+  status       String?
+  data         Json
+  createdAt    DateTime @default(now())
+  updatedAt    DateTime @updatedAt
+
+  @@id([entity, id])
+  @@index([entity])
+  @@index([entity, tenantId])
+  @@index([entity, providerId])
+  @@index([entity, connectionId])
+  @@index([entity, status])
+}

package/apps/integrations/scripts/prepare-db.cjs

@@ -0,0 +1,92 @@
+const { spawnSync } = require('node:child_process');
+const { resolve } = require('node:path');
+
+function asStoreType(raw) {
+  return raw === 'postgres' ? 'postgres' : 'sqlite';
+}
+
+function buildPostgresUrl(env) {
+  if (env.INTEGRATIONS_DATABASE_URL_POSTGRES) {
+    return env.INTEGRATIONS_DATABASE_URL_POSTGRES;
+  }
+
+  if (env.POSTGRES_URL) {
+    return env.POSTGRES_URL;
+  }
+
+  const user = env.POSTGRES_USER || 'postgres';
+  const password = env.POSTGRES_PASSWORD || '';
+  const host = env.POSTGRES_HOST || 'localhost';
+  const port = env.POSTGRES_PORT || '5432';
+  const database = env.POSTGRES_DATABASE || 'integrations';
+
+  const encodedUser = encodeURIComponent(user);
+  const encodedPassword = password ? `:${encodeURIComponent(password)}` : '';
+
+  return `postgresql://${encodedUser}${encodedPassword}@${host}:${port}/${database}`;
+}
+
+function buildSqliteUrl(env) {
+  if (env.INTEGRATIONS_DATABASE_URL_SQLITE) {
+    const configured = env.INTEGRATIONS_DATABASE_URL_SQLITE;
+    if (configured.startsWith('file:')) {
+      const pathWithoutScheme = configured.slice(5);
+      if (pathWithoutScheme.startsWith('/')) {
+        return configured;
+      }
+      return `file:${resolve(pathWithoutScheme)}`;
+    }
+    return `file:${resolve(configured)}`;
+  }
+
+  const sqlitePath = env.SQLITE_PATH || './.integrations/integrations.sqlite';
+  if (sqlitePath.startsWith('file:')) {
+    const pathWithoutScheme = sqlitePath.slice(5);
+    if (pathWithoutScheme.startsWith('/')) {
+      return sqlitePath;
+    }
+    return `file:${resolve(pathWithoutScheme)}`;
+  }
+  return `file:${resolve(sqlitePath)}`;
+}
+
+function run() {
+  const storeType = asStoreType(process.env.STORE_TYPE);
+  const env = { ...process.env };
+
+  if (storeType === 'postgres') {
+    env.INTEGRATIONS_DATABASE_URL_POSTGRES = buildPostgresUrl(env);
+  } else {
+    env.INTEGRATIONS_DATABASE_URL_SQLITE = buildSqliteUrl(env);
+  }
+
+  const postgresArgs = [
+    'prisma',
+    'db',
+    'push',
+    '--config',
+    'apps/integrations/prisma/prisma.config.ts',
+  ];
+  const sqliteArgs = [
+    'prisma',
+    'db',
+    'push',
+    '--config',
+    'apps/integrations/prisma/prisma.sqlite.config.ts',
+  ];
+
+  const pushed = spawnSync(
+    'npx',
+    storeType === 'postgres' ? postgresArgs : sqliteArgs,
+    {
+      env,
+      stdio: 'inherit',
+    },
+  );
+
+  if (pushed.status !== 0) {
+    process.exit(pushed.status || 1);
+  }
+}
+
+run();