@agentlang/agentmanager 0.0.3

package/apps/knowledge/src/modules/graph/graph.service.ts

@@ -0,0 +1,223 @@
+import { Injectable, OnModuleDestroy } from '@nestjs/common';
+import neo4j, { Driver } from 'neo4j-driver';
+import { Neo4jConfiguration } from '../../config/configuration';
+import { MetadataStoreService } from '../platform/metadata-store.service';
+
+interface GraphNode {
+  id: string;
+  name: string;
+  type: string;
+  topicId?: string;
+  documentId?: string;
+  documentVersionId?: string;
+}
+
+interface GraphRelation {
+  id: string;
+  fromNodeId: string;
+  toNodeId: string;
+  type: string;
+  topicId?: string;
+}
+
+@Injectable()
+export class GraphService implements OnModuleDestroy {
+  private readonly neo4jConfig?: Neo4jConfiguration;
+  private readonly neo4jDriver: Driver | null;
+
+  public constructor(private readonly metadataStore: MetadataStoreService) {
+    this.neo4jConfig = this.loadNeo4jConfig();
+    this.neo4jDriver = this.neo4jConfig
+      ? neo4j.driver(
+          this.neo4jConfig.uri,
+          neo4j.auth.basic(this.neo4jConfig.user, this.neo4jConfig.password),
+        )
+      : null;
+  }
+
+  public async onModuleDestroy() {
+    await this.neo4jDriver?.close();
+  }
+
+  public async health() {
+    if (!this.neo4jDriver || !this.neo4jConfig) {
+      return { graph: 'disabled', status: 'ok' };
+    }
+
+    const session = this.neo4jDriver.session({
+      database: this.neo4jConfig.database,
+    });
+    try {
+      await session.run('RETURN 1 as ok');
+      return { graph: 'neo4j', status: 'ok' };
+    } finally {
+      await session.close();
+    }
+  }
+
+  public async deleteDocumentGraph(documentId: string) {
+    if (!this.neo4jDriver || !this.neo4jConfig) {
+      return;
+    }
+
+    const session = this.neo4jDriver.session({
+      database: this.neo4jConfig.database,
+    });
+    try {
+      await session.run(
+        'MATCH (n:KnowledgeNode {documentId: $documentId}) DETACH DELETE n',
+        { documentId },
+      );
+    } finally {
+      await session.close();
+    }
+  }
+
+  public async syncDocumentGraph(params: {
+    topicId?: string;
+    documentId: string;
+    documentVersionId: string;
+    chunks: Array<{ id: string; text: string }>;
+  }) {
+    const entities = this.extractEntities(
+      params.chunks.map((chunk) => chunk.text),
+    );
+
+    const nodes: GraphNode[] = entities.map((entity) => ({
+      id: `${params.documentVersionId}:${entity}`,
+      name: entity,
+      type: 'Entity',
+      topicId: params.topicId,
+      documentId: params.documentId,
+      documentVersionId: params.documentVersionId,
+    }));
+
+    const relations: GraphRelation[] = [];
+    for (let index = 0; index < nodes.length - 1; index += 1) {
+      relations.push({
+        id: `${nodes[index].id}->${nodes[index + 1].id}`,
+        fromNodeId: nodes[index].id,
+        toNodeId: nodes[index + 1].id,
+        type: 'RELATED_TO',
+        topicId: params.topicId,
+      });
+    }
+
+    for (const node of nodes) {
+      await this.metadataStore.upsert('KnowledgeNode', {
+        id: node.id,
+        name: node.name,
+        type: node.type,
+        topicId: node.topicId,
+        documentId: params.documentId,
+        documentVersionId: params.documentVersionId,
+        updatedAt: new Date().toISOString(),
+      });
+    }
+
+    for (const relation of relations) {
+      await this.metadataStore.upsert('KnowledgeRelation', {
+        id: relation.id,
+        fromNodeId: relation.fromNodeId,
+        toNodeId: relation.toNodeId,
+        type: relation.type,
+        topicId: relation.topicId,
+        documentVersionId: params.documentVersionId,
+        updatedAt: new Date().toISOString(),
+      });
+    }
+
+    if (this.neo4jDriver) {
+      await this.syncNeo4j(nodes, relations);
+    }
+
+    return {
+      nodesCreated: nodes.length,
+      relationsCreated: relations.length,
+    };
+  }
+
+  private async syncNeo4j(nodes: GraphNode[], relations: GraphRelation[]) {
+    if (!this.neo4jDriver || !this.neo4jConfig) {
+      return;
+    }
+
+    const session = this.neo4jDriver.session({
+      database: this.neo4jConfig.database,
+    });
+
+    try {
+      for (const node of nodes) {
+        await session.run(
+          `MERGE (n:KnowledgeNode {id: $id})
+           SET n.name = $name,
+                 n.type = $type,
+                 n.topicId = $topicId,
+                 n.documentId = $documentId,
+                 n.documentVersionId = $documentVersionId`,
+          {
+            id: node.id,
+            name: node.name,
+            type: node.type,
+            topicId: node.topicId,
+            documentId: node.documentId,
+            documentVersionId: node.documentVersionId,
+          },
+        );
+      }
+
+      for (const relation of relations) {
+        await session.run(
+          `MATCH (a:KnowledgeNode {id: $fromId}), (b:KnowledgeNode {id: $toId})
+           MERGE (a)-[r:RELATED_TO {id: $id}]->(b)
+           SET r.topicId = $topicId, r.type = $type`,
+          {
+            id: relation.id,
+            fromId: relation.fromNodeId,
+            toId: relation.toNodeId,
+            topicId: relation.topicId,
+            type: relation.type,
+          },
+        );
+      }
+    } finally {
+      await session.close();
+    }
+  }
+
+  private extractEntities(texts: string[]) {
+    const tokens = texts
+      .join(' ')
+      .split(/[^A-Za-z0-9_]+/)
+      .map((token) => token.trim())
+      .filter((token) => token.length >= 5)
+      .map((token) => token.toLowerCase());
+
+    const counts = new Map<string, number>();
+    for (const token of tokens) {
+      counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+
+    return [...counts.entries()]
+      .sort((left, right) => right[1] - left[1])
+      .slice(0, 25)
+      .map(([token]) => token);
+  }
+
+  private loadNeo4jConfig(): Neo4jConfiguration | undefined {
+    const uri = process.env.GRAPH_DB_URI;
+    const user = process.env.GRAPH_DB_USER;
+    const password = process.env.GRAPH_DB_PASSWORD;
+
+    if (!uri || !user || !password) {
+      return undefined;
+    }
+
+    return {
+      uri,
+      user,
+      password,
+      database: process.env.GRAPH_DB_DATABASE,
+    };
+  }
+}

package/apps/knowledge/src/modules/ingestion/dto/ingestion.dto.ts

@@ -0,0 +1,58 @@
+import { createZodDto } from 'nestjs-zod';
+import { z } from 'zod';
+
+export const UploadDocumentVersionSchema = z.object({
+  tenantId: z.string().min(1),
+  appId: z.string().min(1),
+  topicId: z.string().min(1),
+  topicName: z.string().optional(),
+  documentId: z.string().uuid().optional(),
+  connectionId: z.string().min(1),
+  logicalName: z.string().min(1),
+  fileName: z.string().min(1),
+  mimeType: z.string().min(1),
+  contentBase64: z.string().min(1),
+  containerTag: z.string().min(1),
+  createdBy: z.string().min(1),
+});
+
+export const ReIngestDocumentVersionSchema = z.object({
+  topicId: z.string().optional(),
+  documentVersionId: z.string().min(1),
+  containerTag: z.string().optional(),
+});
+
+export const ProcessQueuesSchema = z.object({
+  limit: z.coerce.number().int().min(1).max(500).optional(),
+});
+
+export const QueueListQuerySchema = z.object({
+  topicId: z.string().optional(),
+  documentId: z.string().optional(),
+  status: z
+    .enum(['queued', 'processing', 'completed', 'failed', 'dead_letter'])
+    .optional(),
+  limit: z.coerce.number().int().min(1).max(1000).optional(),
+  offset: z.coerce.number().int().min(0).optional(),
+});
+
+export class UploadDocumentVersionDto extends createZodDto(
+  UploadDocumentVersionSchema,
+) {}
+
+export class ReIngestDocumentVersionDto extends createZodDto(
+  ReIngestDocumentVersionSchema,
+) {}
+
+export class ProcessQueuesDto extends createZodDto(ProcessQueuesSchema) {}
+
+export class QueueListQueryDto extends createZodDto(QueueListQuerySchema) {}
+
+export type UploadDocumentVersionInput = z.infer<
+  typeof UploadDocumentVersionSchema
+>;
+export type ReIngestDocumentVersionInput = z.infer<
+  typeof ReIngestDocumentVersionSchema
+>;
+export type ProcessQueuesInput = z.infer<typeof ProcessQueuesSchema>;
+export type QueueListQueryInput = z.infer<typeof QueueListQuerySchema>;

package/apps/knowledge/src/modules/ingestion/ingestion-workflows.service.ts

@@ -0,0 +1,246 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+import { randomUUID } from 'node:crypto';
+import { MetadataStoreService } from '../platform/metadata-store.service';
+import { StorageService } from '../storage/storage.service';
+import {
+  QueueListQueryInput,
+  ReIngestDocumentVersionInput,
+  UploadDocumentVersionInput,
+} from './dto/ingestion.dto';
+import { IngestionService } from './ingestion.service';
+
+@Injectable()
+export class IngestionWorkflowsService {
+  public constructor(
+    private readonly metadataStore: MetadataStoreService,
+    private readonly storageService: StorageService,
+    private readonly ingestionService: IngestionService,
+  ) {}
+
+  public async uploadDocumentVersion(payload: UploadDocumentVersionInput) {
+    const now = new Date().toISOString();
+
+    const documentId = payload.documentId ?? randomUUID();
+    const versionId = randomUUID();
+    const objectKey =
+      `documents/${payload.topicId}/${documentId}/` +
+      `${versionId}/${encodeURIComponent(payload.fileName)}`;
+
+    const content = Buffer.from(payload.contentBase64, 'base64');
+    await this.storageService.writeObject({
+      key: objectKey,
+      content,
+      contentType: payload.mimeType,
+    });
+
+    const existing = await this.metadataStore.getById(
+      'KnowledgeDocument',
+      documentId,
+    );
+    const previousVersion = this.readNumber(existing, 'currentVersion') ?? 0;
+    const nextVersion = previousVersion + 1;
+
+    const document = await this.metadataStore.upsert('KnowledgeDocument', {
+      ...(existing ?? {}),
+      id: documentId,
+      tenantId: payload.tenantId,
+      appId: payload.appId,
+      topicId: payload.topicId,
+      connectionId: payload.connectionId,
+      title: payload.logicalName,
+      remotePath: `${payload.topicId}/${payload.logicalName}`,
+      fileName: payload.fileName,
+      fileType: this.inferFileType(payload.fileName),
+      sizeBytes: content.byteLength,
+      currentVersion: nextVersion,
+      status: 'processing',
+      isDeleted: false,
+      lastSyncedAt: now,
+      createdBy: payload.createdBy,
+      updatedBy: payload.createdBy,
+      createdAt: this.readString(existing, 'createdAt') ?? now,
+      updatedAt: now,
+    });
+
+    const version = await this.metadataStore.upsert('DocumentVersion', {
+      id: versionId,
+      tenantId: payload.tenantId,
+      documentId,
+      topicId: payload.topicId,
+      version: nextVersion,
+      sizeBytes: content.byteLength,
+      objectKey,
+      storageKey: objectKey,
+      mimeType: payload.mimeType,
+      originalFileName: payload.fileName,
+      isCurrent: true,
+      status: 'queued',
+      ingestStatus: 'queued',
+      changeType: existing ? 'modified' : 'added',
+      syncedAt: now,
+      uploadedBy: payload.createdBy,
+      createdAt: now,
+      updatedAt: now,
+    });
+
+    if (payload.topicId) {
+      const topic = await this.metadataStore.getById('Topic', payload.topicId);
+      if (topic) {
+        const currentCount = this.readNumber(topic, 'documentCount') ?? 0;
+        await this.metadataStore.upsert('Topic', {
+          ...topic,
+          status: 'ingesting',
+          documentCount: existing ? currentCount : currentCount + 1,
+          updatedAt: now,
+        } as { id: string });
+      }
+    }
+
+    const queueItem = await this.ingestionService.queueDocumentVersion({
+      tenantId: payload.tenantId,
+      appId: payload.appId,
+      connectionId: payload.connectionId,
+      topicId: payload.topicId,
+      documentId,
+      versionId,
+      storageKey: objectKey,
+      containerTag: payload.containerTag,
+      queuedAt: now,
+      progress: 0,
+      progressStage: 'queued',
+    });
+
+    return {
+      status: 'queued',
+      document,
+      version,
+      queueItem,
+    };
+  }
+
+  public async reIngestDocumentVersion(payload: ReIngestDocumentVersionInput) {
+    const version = await this.metadataStore.getById(
+      'DocumentVersion',
+      payload.documentVersionId,
+    );
+    if (!version) {
+      throw new NotFoundException(
+        `DocumentVersion/${payload.documentVersionId} not found`,
+      );
+    }
+
+    await this.metadataStore.upsert('DocumentVersion', {
+      ...version,
+      id: payload.documentVersionId,
+      status: 'queued',
+      ingestStatus: 'queued',
+      updatedAt: new Date().toISOString(),
+    } as { id: string });
+
+    const queueItem = await this.ingestionService.queueDocumentVersion({
+      topicId:
+        payload.topicId ?? this.readString(version, 'topicId') ?? undefined,
+      documentId: this.readString(version, 'documentId') ?? '',
+      versionId: payload.documentVersionId,
+      containerTag:
+        payload.containerTag ??
+        this.readString(version, 'containerTag') ??
+        undefined,
+      storageKey: this.readString(version, 'storageKey') ?? undefined,
+      queuedAt: new Date().toISOString(),
+      progress: 0,
+      progressStage: 'queued',
+    });
+
+    return {
+      status: 'queued',
+      queueItem,
+    };
+  }
+
+  public async processQueuesNow(limit?: number) {
+    const safeLimit = limit ?? 50;
+
+    const [vector, graph, recommendations, automation] = await Promise.all([
+      this.ingestionService.processVectorQueue(safeLimit),
+      this.ingestionService.processGraphSyncQueue(safeLimit),
+      this.ingestionService.processAgentRecommendationQueue(safeLimit),
+      this.ingestionService.processAutomationOpportunityQueue(safeLimit),
+    ]);
+
+    return {
+      vector,
+      graph,
+      recommendations,
+      automation,
+      processedAt: new Date().toISOString(),
+    };
+  }
+
+  public async recoverQueuesAfterRestart() {
+    return this.ingestionService.recoverQueuesAfterRestart();
+  }
+
+  public async listVectorQueue(query: QueueListQueryInput) {
+    return this.listQueue('VectorIngestionQueueItem', query);
+  }
+
+  public async listGraphQueue(query: QueueListQueryInput) {
+    return this.listQueue('GraphSyncQueueItem', query);
+  }
+
+  private async listQueue(entity: string, query: QueueListQueryInput) {
+    const filters: Record<string, string> = {};
+    if (query.topicId) filters.topicId = query.topicId;
+    if (query.documentId) filters.documentId = query.documentId;
+    if (query.status) filters.status = query.status;
+
+    return this.metadataStore.list(entity, {
+      filters: Object.keys(filters).length > 0 ? filters : undefined,
+      limit: query.limit,
+      offset: query.offset,
+    });
+  }
+
+  private inferFileType(fileName: string) {
+    const extension = fileName.split('.').pop()?.toLowerCase();
+    switch (extension) {
+      case 'pdf':
+      case 'markdown':
+      case 'text':
+      case 'docx':
+      case 'html':
+      case 'json':
+      case 'csv':
+      case 'xlsx':
+        return extension;
+      case 'md':
+        return 'markdown';
+      case 'txt':
+        return 'text';
+      default:
+        return 'unknown';
+    }
+  }
+
+  private readString(
+    payload: Record<string, unknown> | null,
+    key: string,
+  ): string | undefined {
+    if (!payload) {
+      return undefined;
+    }
+    const value = payload[key];
+    return typeof value === 'string' ? value : undefined;
+  }
+
+  private readNumber(payload: Record<string, unknown> | null, key: string) {
+    if (!payload) {
+      return undefined;
+    }
+    const value = payload[key];
+    return typeof value === 'number' && Number.isFinite(value)
+      ? value
+      : undefined;
+  }
+}

package/apps/knowledge/src/modules/ingestion/ingestion.module.ts

@@ -0,0 +1,14 @@
+import { Module } from '@nestjs/common';
+import { GraphModule } from '../graph/graph.module';
+import { PlatformModule } from '../platform/platform.module';
+import { RecommendationsModule } from '../recommendations/recommendations.module';
+import { StorageModule } from '../storage/storage.module';
+import { IngestionWorkflowsService } from './ingestion-workflows.service';
+import { IngestionService } from './ingestion.service';
+
+@Module({
+  imports: [PlatformModule, StorageModule, GraphModule, RecommendationsModule],
+  providers: [IngestionService, IngestionWorkflowsService],
+  exports: [IngestionService, IngestionWorkflowsService],
+})
+export class IngestionModule {}