@agentlang/agentmanager 0.0.3

package/apps/knowledge/src/modules/ingestion/ingestion.service.ts

@@ -0,0 +1,619 @@
+import { Injectable } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { createHash, randomUUID } from 'node:crypto';
+import { AppConfiguration } from '../../config/configuration';
+import { GraphService } from '../graph/graph.service';
+import { MetadataStoreService } from '../platform/metadata-store.service';
+import { VectorStoreService } from '../platform/vector-store.service';
+import { RecommendationsService } from '../recommendations/recommendations.service';
+import { StorageService } from '../storage/storage.service';
+
+type QueueStatus =
+  | 'queued'
+  | 'processing'
+  | 'completed'
+  | 'failed'
+  | 'dead_letter';
+
+type QueueEntityName =
+  | 'VectorIngestionQueueItem'
+  | 'GraphSyncQueueItem'
+  | 'AgentRecommendationQueueItem'
+  | 'AutomationOpportunityQueueItem';
+
+export interface QueueItem {
+  id: string;
+  status: QueueStatus;
+  attempts: number;
+  retryCount?: number;
+  maxAttempts: number;
+  queuedAt?: string;
+  startedAt?: string;
+  completedAt?: string;
+  progress?: number;
+  progressStage?: string;
+  nextAttemptAt?: string;
+  lastError?: string;
+  createdAt: string;
+  updatedAt: string;
+  [key: string]: unknown;
+}
+
+@Injectable()
+export class IngestionService {
+  public constructor(
+    private readonly configService: ConfigService<AppConfiguration>,
+    private readonly metadataStore: MetadataStoreService,
+    private readonly vectorStore: VectorStoreService,
+    private readonly storageService: StorageService,
+    private readonly graphService: GraphService,
+    private readonly recommendationsService: RecommendationsService,
+  ) {}
+
+  public async query(payload: Record<string, unknown>) {
+    const topicId = this.readString(payload, 'topicId');
+    const queryText =
+      this.readString(payload, 'queryText') ??
+      this.readString(payload, 'query') ??
+      this.readString(payload, 'question');
+    const limit = this.readNumber(payload, 'limit') ?? 5;
+
+    const candidates = await this.metadataStore.list('KnowledgeChunk', {
+      filters: topicId ? { topicId } : undefined,
+      limit,
+      offset: 0,
+    });
+
+    const embedding: number[] = Array.isArray(payload.embedding)
+      ? payload.embedding.map((value) => Number(value) || 0)
+      : await this.embedText(queryText ?? '');
+
+    const vectorMatches = await this.vectorStore.search(embedding, {
+      entity: 'KnowledgeChunk',
+      limit,
+    });
+
+    return {
+      response:
+        queryText && candidates.length > 0
+          ? `Found ${candidates.length} relevant chunks for query: ${queryText}`
+          : 'No indexed knowledge available for this query yet.',
+      chunksJson: JSON.stringify(candidates),
+      entitiesJson: JSON.stringify([]),
+      edgesJson: JSON.stringify([]),
+      vectorMatches,
+    };
+  }
+
+  public async queueDocumentVersion(
+    payload: {
+      topicId?: string;
+      documentId: string;
+      versionId: string;
+    } & Record<string, unknown>,
+  ) {
+    const { versionId, ...queueMetadata } = payload;
+
+    return this.enqueue('VectorIngestionQueueItem', {
+      ...queueMetadata,
+      documentVersionId: versionId,
+    });
+  }
+
+  public async queueGraphSync(payload: Record<string, unknown>) {
+    return this.enqueue('GraphSyncQueueItem', payload);
+  }
+
+  public async queueAgentRecommendation(payload: Record<string, unknown>) {
+    return this.enqueue('AgentRecommendationQueueItem', payload);
+  }
+
+  public async queueAutomationOpportunity(payload: Record<string, unknown>) {
+    return this.enqueue('AutomationOpportunityQueueItem', payload);
+  }
+
+  public async processVectorQueue(limit = 50) {
+    return this.processQueue('VectorIngestionQueueItem', limit);
+  }
+
+  public async processGraphSyncQueue(limit = 50) {
+    return this.processQueue('GraphSyncQueueItem', limit);
+  }
+
+  public async processAgentRecommendationQueue(limit = 50) {
+    return this.processQueue('AgentRecommendationQueueItem', limit);
+  }
+
+  public async processAutomationOpportunityQueue(limit = 50) {
+    return this.processQueue('AutomationOpportunityQueueItem', limit);
+  }
+
+  public async recoverQueuesAfterRestart() {
+    const queueEntities: QueueEntityName[] = [
+      'VectorIngestionQueueItem',
+      'GraphSyncQueueItem',
+      'AgentRecommendationQueueItem',
+      'AutomationOpportunityQueueItem',
+    ];
+
+    const recoveredByQueue: Record<string, number> = {};
+
+    for (const queue of queueEntities) {
+      const processingItems = await this.metadataStore.list(queue, {
+        filters: { status: 'processing' },
+        limit: 500,
+        offset: 0,
+      });
+
+      let recovered = 0;
+      for (const item of processingItems) {
+        const queueItem = item as QueueItem;
+        await this.metadataStore.upsert(queue, {
+          ...queueItem,
+          status: 'queued',
+          progress: 0,
+          progressStage: 'queued',
+          updatedAt: new Date().toISOString(),
+        });
+        recovered += 1;
+      }
+
+      recoveredByQueue[queue] = recovered;
+    }
+
+    return {
+      recoveredByQueue,
+      recoveredTotal: Object.values(recoveredByQueue).reduce(
+        (sum, count) => sum + count,
+        0,
+      ),
+    };
+  }
+
+  private async enqueue(
+    queueEntity: QueueEntityName,
+    payload: Record<string, unknown>,
+  ) {
+    const now = new Date().toISOString();
+    const queueItem: QueueItem = {
+      id: randomUUID(),
+      status: 'queued',
+      attempts: 0,
+      retryCount: 0,
+      maxAttempts: this.readNumber(payload, 'maxAttempts') ?? 5,
+      queuedAt: now,
+      progress: this.readNumber(payload, 'progress') ?? 0,
+      progressStage: this.readString(payload, 'progressStage') ?? 'queued',
+      createdAt: now,
+      updatedAt: now,
+      ...payload,
+    };
+
+    await this.metadataStore.upsert(queueEntity, queueItem);
+    return queueItem;
+  }
+
+  private async processQueue(queueEntity: QueueEntityName, limit: number) {
+    const [queuedItems, failedItems] = await Promise.all([
+      this.metadataStore.list(queueEntity, {
+        filters: { status: 'queued' },
+        limit,
+        offset: 0,
+      }),
+      this.metadataStore.list(queueEntity, {
+        filters: { status: 'failed' },
+        limit,
+        offset: 0,
+      }),
+    ]);
+
+    const now = Date.now();
+    const readyFailedItems = failedItems.filter((item) => {
+      const queueItem = item as QueueItem;
+      if (
+        !queueItem.nextAttemptAt ||
+        typeof queueItem.nextAttemptAt !== 'string'
+      ) {
+        return true;
+      }
+
+      return new Date(queueItem.nextAttemptAt).getTime() <= now;
+    });
+
+    const candidates = [...queuedItems, ...readyFailedItems].slice(0, limit);
+
+    let completed = 0;
+    let failed = 0;
+    let deadLettered = 0;
+
+    for (const item of candidates) {
+      const queueItem = item as QueueItem;
+      const processing: QueueItem = {
+        ...queueItem,
+        status: 'processing',
+        startedAt: queueItem.startedAt ?? new Date().toISOString(),
+        progress: Math.max(10, Number(queueItem.progress ?? 0)),
+        progressStage: 'processing',
+        updatedAt: new Date().toISOString(),
+      };
+
+      await this.metadataStore.upsert(queueEntity, processing);
+
+      try {
+        await this.runQueueJob(queueEntity, processing);
+
+        await this.metadataStore.upsert(queueEntity, {
+          ...processing,
+          status: 'completed',
+          retryCount: processing.attempts,
+          progress: 100,
+          progressStage: 'completed',
+          completedAt: new Date().toISOString(),
+          updatedAt: new Date().toISOString(),
+          lastError: undefined,
+          nextAttemptAt: undefined,
+        });
+
+        completed += 1;
+      } catch (error) {
+        const nextAttempts = (processing.attempts ?? 0) + 1;
+        const maxAttempts = processing.maxAttempts ?? 3;
+        const nowIso = new Date().toISOString();
+
+        if (nextAttempts >= maxAttempts) {
+          await this.metadataStore.upsert(queueEntity, {
+            ...processing,
+            attempts: nextAttempts,
+            retryCount: nextAttempts,
+            status: 'dead_letter',
+            progress: 0,
+            progressStage: 'dead_letter',
+            updatedAt: nowIso,
+            lastError: (error as Error).message,
+            nextAttemptAt: undefined,
+          });
+
+          deadLettered += 1;
+          continue;
+        }
+
+        const backoffMs = this.calculateBackoffMs(nextAttempts);
+        await this.metadataStore.upsert(queueEntity, {
+          ...processing,
+          attempts: nextAttempts,
+          retryCount: nextAttempts,
+          status: 'failed',
+          progress: 0,
+          progressStage: 'failed',
+          updatedAt: nowIso,
+          lastError: (error as Error).message,
+          nextAttemptAt: new Date(Date.now() + backoffMs).toISOString(),
+        });
+
+        failed += 1;
+      }
+    }
+
+    return {
+      queue: queueEntity,
+      attempted: candidates.length,
+      completed,
+      failed,
+      deadLettered,
+    };
+  }
+
+  private async runQueueJob(queueEntity: QueueEntityName, item: QueueItem) {
+    const shouldFail = item.forceFail === true || item.forceFail === 'true';
+    if (shouldFail) {
+      throw new Error('Forced queue failure requested by payload');
+    }
+
+    switch (queueEntity) {
+      case 'VectorIngestionQueueItem':
+        await this.ingestDocumentVersion(item);
+        return;
+      case 'GraphSyncQueueItem':
+        await this.syncGraph(item);
+        return;
+      case 'AgentRecommendationQueueItem':
+        await this.generateRecommendations(item);
+        return;
+      case 'AutomationOpportunityQueueItem':
+        await this.generateAutomationOpportunities(item);
+        return;
+      default:
+        return;
+    }
+  }
+
+  private async ingestDocumentVersion(item: QueueItem) {
+    const documentVersionId = this.readUnknownString(item, 'documentVersionId');
+    if (!documentVersionId) {
+      throw new Error(
+        'documentVersionId is required for vector ingestion queue',
+      );
+    }
+
+    const version = await this.metadataStore.getById(
+      'DocumentVersion',
+      documentVersionId,
+    );
+    if (!version) {
+      throw new Error(`DocumentVersion/${documentVersionId} not found`);
+    }
+
+    const objectKey = this.readUnknownString(version, 'objectKey');
+    if (!objectKey) {
+      throw new Error('DocumentVersion.objectKey is missing');
+    }
+
+    const object = await this.storageService.readObject(objectKey);
+    const content = object.content.toString('utf8');
+    const chunks = this.chunkText(content);
+
+    const topicId = this.readUnknownString(version, 'topicId');
+    const documentId = this.readUnknownString(version, 'documentId') ?? '';
+
+    let index = 0;
+    for (const chunkText of chunks) {
+      index += 1;
+      const chunkId = `${documentVersionId}:chunk:${index}`;
+      const embedding = await this.embedText(chunkText);
+
+      await this.metadataStore.upsert('KnowledgeChunk', {
+        id: chunkId,
+        topicId,
+        documentId,
+        documentVersionId,
+        chunkIndex: index,
+        text: chunkText,
+        updatedAt: new Date().toISOString(),
+      });
+
+      await this.vectorStore.upsert({
+        id: chunkId,
+        entity: 'KnowledgeChunk',
+        embedding,
+        metadata: {
+          topicId,
+          documentId,
+          documentVersionId,
+          chunkIndex: index,
+          text: chunkText,
+        },
+      });
+    }
+
+    await this.metadataStore.upsert('DocumentVersion', {
+      ...version,
+      status: 'completed',
+      ingestStatus: 'ingested',
+      chunkCount: chunks.length,
+      updatedAt: new Date().toISOString(),
+    } as { id: string });
+
+    if (topicId) {
+      const topic = await this.metadataStore.getById('Topic', topicId);
+      if (topic) {
+        await this.metadataStore.upsert('Topic', {
+          ...topic,
+          status: 'ready',
+          readyAt: new Date().toISOString(),
+          updatedAt: new Date().toISOString(),
+        } as { id: string });
+      }
+    }
+
+    await this.queueGraphSync({ topicId, documentId, documentVersionId });
+    await this.queueAgentRecommendation({ topicId, documentVersionId });
+    await this.queueAutomationOpportunity({ topicId, documentVersionId });
+  }
+
+  private async syncGraph(item: QueueItem) {
+    const documentVersionId = this.readUnknownString(item, 'documentVersionId');
+    const topicId = this.readUnknownString(item, 'topicId');
+    const documentId = this.readUnknownString(item, 'documentId') ?? '';
+
+    if (!documentVersionId) {
+      return;
+    }
+
+    const chunks = await this.metadataStore.list('KnowledgeChunk', {
+      filters: { documentVersionId },
+      limit: 2000,
+      offset: 0,
+    });
+
+    await this.graphService.syncDocumentGraph({
+      topicId,
+      documentId,
+      documentVersionId,
+      chunks: chunks.map((chunk) => ({
+        id: String(chunk.id),
+        text: this.asText(chunk.text),
+      })),
+    });
+  }
+
+  private async generateRecommendations(item: QueueItem) {
+    await this.recommendationsService.generateAgentRecommendations({
+      topicId: this.readUnknownString(item, 'topicId'),
+      documentVersionId: this.readUnknownString(item, 'documentVersionId'),
+    });
+  }
+
+  private async generateAutomationOpportunities(item: QueueItem) {
+    await this.recommendationsService.generateAutomationOpportunities({
+      topicId: this.readUnknownString(item, 'topicId'),
+      documentVersionId: this.readUnknownString(item, 'documentVersionId'),
+    });
+  }
+
+  private chunkText(content: string) {
+    const clean = content.trim();
+    if (!clean) {
+      return [];
+    }
+
+    const paragraphs = clean
+      .split(/\n{2,}/)
+      .map((paragraph) => paragraph.trim())
+      .filter((paragraph) => paragraph.length > 0);
+
+    const chunks: string[] = [];
+    let current = '';
+
+    for (const paragraph of paragraphs) {
+      if ((current + '\n\n' + paragraph).length > 1500 && current.length > 0) {
+        chunks.push(current);
+        current = paragraph;
+      } else {
+        current = current.length > 0 ? `${current}\n\n${paragraph}` : paragraph;
+      }
+    }
+
+    if (current.length > 0) {
+      chunks.push(current);
+    }
+
+    return chunks.length > 0 ? chunks : [clean.slice(0, 1500)];
+  }
+
+  private async embedText(text: string): Promise<number[]> {
+    const configuredDimension =
+      this.configService.get('vector', { infer: true })?.dimension ?? 1536;
+    const normalized = text.trim();
+
+    if (normalized.length === 0) {
+      return Array<number>(configuredDimension).fill(0);
+    }
+
+    const apiKey = this.configService.get<string>('openAiApiKey', {
+      infer: true,
+    });
+    if (apiKey) {
+      try {
+        const response = await fetch('https://api.openai.com/v1/embeddings', {
+          method: 'POST',
+          headers: {
+            authorization: `Bearer ${apiKey}`,
+            'content-type': 'application/json',
+          },
+          body: JSON.stringify({
+            model: 'text-embedding-3-small',
+            input: normalized,
+          }),
+        });
+
+        if (response.ok) {
+          const json = (await response.json()) as {
+            data?: Array<{ embedding?: number[] }>;
+          };
+
+          const embedding = json.data?.[0]?.embedding;
+          if (embedding && embedding.length > 0) {
+            return this.normalizeEmbedding(embedding, configuredDimension);
+          }
+        }
+      } catch {
+        // Fall through to deterministic local embedding when API is unavailable.
+      }
+    }
+
+    return this.fallbackEmbedding(normalized, configuredDimension);
+  }
+
+  private fallbackEmbedding(text: string, dimension: number): number[] {
+    const vector = new Array<number>(dimension).fill(0);
+    const bytes = createHash('sha256').update(text).digest();
+
+    for (let index = 0; index < dimension; index += 1) {
+      const byte = bytes[index % bytes.length];
+      vector[index] = (byte / 255) * 2 - 1;
+    }
+
+    return vector;
+  }
+
+  private normalizeEmbedding(embedding: number[], dimension: number): number[] {
+    if (embedding.length === dimension) {
+      return embedding;
+    }
+
+    if (embedding.length > dimension) {
+      return embedding.slice(0, dimension);
+    }
+
+    return [
+      ...embedding,
+      ...Array<number>(dimension - embedding.length).fill(0),
+    ];
+  }
+
+  private calculateBackoffMs(attempt: number) {
+    return Math.min(120000, 1000 * Math.pow(2, Math.max(0, attempt - 1)));
+  }
+
+  private readString(payload: Record<string, unknown>, key: string) {
+    const value = payload[key];
+    return typeof value === 'string' ? value : undefined;
+  }
+
+  private readNumber(payload: Record<string, unknown>, key: string) {
+    const value = payload[key];
+    return typeof value === 'number' && Number.isFinite(value)
+      ? value
+      : undefined;
+  }
+
+  private readUnknownString(payload: Record<string, unknown>, key: string) {
+    const value = payload[key];
+    return typeof value === 'string' ? value : undefined;
+  }
+
+  private asText(value: unknown) {
+    return typeof value === 'string' ? value : '';
+  }
+
+  public async recoverStuckItems(): Promise<number> {
+    const stuckStatuses: QueueStatus[] = ['processing'];
+    let recoveredCount = 0;
+
+    for (const status of stuckStatuses) {
+      const items = await this.metadataStore.list('IngestionQueue', {
+        filters: { status },
+        limit: 1000,
+        offset: 0,
+      });
+
+      for (const item of items) {
+        const id = typeof item.id === 'string' ? item.id : undefined;
+        if (!id) continue;
+
+        // Check if item has been processing for too long (> 5 minutes)
+        const startedAt =
+          typeof item.startedAt === 'string' ? item.startedAt : undefined;
+        if (startedAt) {
+          const startTime = new Date(startedAt).getTime();
+          const now = Date.now();
+          const processingTime = now - startTime;
+
+          if (processingTime > 5 * 60 * 1000) {
+            // Reset to pending
+            await this.metadataStore.upsert('IngestionQueue', {
+              ...item,
+              id,
+              status: 'pending',
+              lastError: 'Reset due to timeout during recovery',
+              updatedAt: new Date().toISOString(),
+            });
+            recoveredCount++;
+          }
+        }
+      }
+    }
+
+    return recoveredCount;
+  }
+}

package/apps/knowledge/src/modules/oauth-bridge/oauth-bridge.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { OauthBridgeService } from './oauth-bridge.service';
+
+@Module({
+  providers: [OauthBridgeService],
+  exports: [OauthBridgeService],
+})
+export class OauthBridgeModule {}