@agentlang/agentmanager 0.0.3

package/apps/knowledge/src/modules/recommendations/recommendations.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { PlatformModule } from '../platform/platform.module';
+import { RecommendationsService } from './recommendations.service';
+
+@Module({
+  imports: [PlatformModule],
+  providers: [RecommendationsService],
+  exports: [RecommendationsService],
+})
+export class RecommendationsModule {}

package/apps/knowledge/src/modules/recommendations/recommendations.service.ts

@@ -0,0 +1,121 @@
+import { Injectable } from '@nestjs/common';
+import { MetadataStoreService } from '../platform/metadata-store.service';
+
+@Injectable()
+export class RecommendationsService {
+  public constructor(private readonly metadataStore: MetadataStoreService) {}
+
+  public async generateAgentRecommendations(input: {
+    topicId?: string;
+    documentVersionId?: string;
+  }) {
+    const topicId = input.topicId;
+    const chunks = await this.metadataStore.list('KnowledgeChunk', {
+      filters: topicId ? { topicId } : undefined,
+      limit: 200,
+      offset: 0,
+    });
+
+    const topTerms = this.extractTopTerms(
+      chunks.map((chunk) => this.asText(chunk.text)),
+    );
+
+    const createdIds: string[] = [];
+    for (const term of topTerms.slice(0, 10)) {
+      const id = `${input.documentVersionId ?? 'global'}:agent:${term}`;
+      await this.metadataStore.upsert('AgentRecommendation', {
+        id,
+        topicId,
+        title: `Investigate ${term}`,
+        rationale: `Frequent entity in knowledge base: ${term}`,
+        confidence: 0.6,
+        source: 'knowledge-ingestion',
+        updatedAt: new Date().toISOString(),
+      });
+      createdIds.push(id);
+    }
+
+    return {
+      created: createdIds.length,
+      recommendationIds: createdIds,
+    };
+  }
+
+  public async generateAutomationOpportunities(input: {
+    topicId?: string;
+    documentVersionId?: string;
+  }) {
+    const topicId = input.topicId;
+    const chunks = await this.metadataStore.list('KnowledgeChunk', {
+      filters: topicId ? { topicId } : undefined,
+      limit: 200,
+      offset: 0,
+    });
+
+    const opportunities = this.extractTopTerms(
+      chunks.map((chunk) => this.asText(chunk.text)),
+    ).slice(0, 10);
+
+    const createdIds: string[] = [];
+    for (const term of opportunities) {
+      const id = `${input.documentVersionId ?? 'global'}:automation:${term}`;
+      await this.metadataStore.upsert('AutomationOpportunity', {
+        id,
+        topicId,
+        title: `Automate workflows for ${term}`,
+        summary: `Detected repeated process context around ${term}`,
+        impactScore: 0.5,
+        source: 'knowledge-ingestion',
+        updatedAt: new Date().toISOString(),
+      });
+      createdIds.push(id);
+    }
+
+    return {
+      created: createdIds.length,
+      opportunityIds: createdIds,
+    };
+  }
+
+  private extractTopTerms(texts: string[]) {
+    const counts = new Map<string, number>();
+    const stopWords = new Set([
+      'about',
+      'after',
+      'again',
+      'because',
+      'before',
+      'their',
+      'there',
+      'these',
+      'those',
+      'where',
+      'which',
+      'would',
+      'could',
+      'should',
+      'topic',
+      'document',
+      'knowledge',
+    ]);
+
+    for (const text of texts) {
+      for (const token of text.split(/[^A-Za-z0-9_]+/)) {
+        const term = token.toLowerCase().trim();
+        if (term.length < 5 || stopWords.has(term)) {
+          continue;
+        }
+
+        counts.set(term, (counts.get(term) ?? 0) + 1);
+      }
+    }
+
+    return [...counts.entries()]
+      .sort((left, right) => right[1] - left[1])
+      .map(([term]) => term);
+  }
+
+  private asText(value: unknown) {
+    return typeof value === 'string' ? value : '';
+  }
+}

package/apps/knowledge/src/modules/storage/storage.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { StorageService } from './storage.service';
+
+@Module({
+  providers: [StorageService],
+  exports: [StorageService],
+})
+export class StorageModule {}

package/apps/knowledge/src/modules/storage/storage.service.ts

@@ -0,0 +1,150 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { Readable } from 'node:stream';
+import { AppConfiguration } from '../../config/configuration';
+
+@Injectable()
+export class StorageService {
+  private readonly storeType: 'local' | 's3';
+  private readonly bucket?: string;
+  private readonly localPath: string;
+  private readonly s3Client?: S3Client;
+
+  public constructor(
+    private readonly configService: ConfigService<AppConfiguration>,
+  ) {
+    const objectStore = this.configService.get('objectStore', { infer: true });
+
+    this.storeType = objectStore?.type ?? 'local';
+    this.bucket = objectStore?.bucket;
+    this.localPath = objectStore?.localPath ?? './.knowledge/object-store';
+
+    if (this.storeType === 's3') {
+      this.s3Client = new S3Client({ region: objectStore?.region });
+    }
+  }
+
+  public async health() {
+    if (this.storeType === 's3') {
+      if (!this.bucket || !this.s3Client) {
+        throw new Error('S3 bucket is required when OBJECT_STORE_TYPE=s3');
+      }
+
+      await this.s3Client
+        .send(
+          new GetObjectCommand({
+            Bucket: this.bucket,
+            Key: '__knowledge_healthcheck__',
+          }),
+        )
+        .catch(() => null);
+
+      return { objectStoreType: this.storeType, status: 'ok' };
+    }
+
+    await mkdir(this.localPath, { recursive: true });
+    return { objectStoreType: this.storeType, status: 'ok' };
+  }
+
+  public async writeObject(params: {
+    key: string;
+    content: Buffer;
+    contentType?: string;
+  }) {
+    if (this.storeType === 's3') {
+      if (!this.bucket || !this.s3Client) {
+        throw new Error('S3 bucket is required when OBJECT_STORE_TYPE=s3');
+      }
+
+      await this.s3Client.send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: params.key,
+          Body: params.content,
+          ContentType: params.contentType,
+        }),
+      );
+
+      return;
+    }
+
+    const target = join(this.localPath, params.key);
+    await mkdir(dirname(target), { recursive: true });
+    await writeFile(target, params.content);
+  }
+
+  public async readObject(key: string) {
+    if (this.storeType === 's3') {
+      if (!this.bucket || !this.s3Client) {
+        throw new Error('S3 bucket is required when OBJECT_STORE_TYPE=s3');
+      }
+
+      const response = await this.s3Client.send(
+        new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        }),
+      );
+
+      if (!response.Body) {
+        throw new NotFoundException(`Object not found: ${key}`);
+      }
+
+      const content = await this.readS3Body(response.Body as Readable);
+      return {
+        key,
+        content,
+        contentType: response.ContentType,
+      };
+    }
+
+    const target = join(this.localPath, key);
+    const content = await readFile(target);
+
+    return {
+      key,
+      content,
+      contentType: 'application/octet-stream',
+    };
+  }
+
+  public async deleteObject(key: string) {
+    if (this.storeType === 's3') {
+      if (!this.bucket || !this.s3Client) {
+        throw new Error('S3 bucket is required when OBJECT_STORE_TYPE=s3');
+      }
+
+      await this.s3Client.send(
+        new DeleteObjectCommand({
+          Bucket: this.bucket,
+          Key: key,
+        }),
+      );
+      return;
+    }
+
+    const target = join(this.localPath, key);
+    await rm(target, { force: true });
+  }
+
+  private async readS3Body(stream: Readable): Promise<Buffer> {
+    const chunks: Uint8Array[] = [];
+    for await (const chunk of stream) {
+      if (Buffer.isBuffer(chunk)) {
+        chunks.push(new Uint8Array(chunk));
+      } else {
+        chunks.push(new Uint8Array(Buffer.from(chunk)));
+      }
+    }
+
+    return Buffer.concat(chunks);
+  }
+}

package/apps/knowledge/src/modules/sync/scheduler.service.ts

@@ -0,0 +1,175 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { SchedulerRegistry } from '@nestjs/schedule';
+import { CronJob } from 'cron';
+import { IngestionService } from '../ingestion/ingestion.service';
+
+interface SchedulerConfig {
+  enabled: boolean;
+  cronExpression: string;
+  queueProcessingEnabled: boolean;
+  queueProcessingIntervalMs: number;
+  recoveryEnabled: boolean;
+}
+
+@Injectable()
+export class SchedulerService implements OnModuleInit {
+  private readonly logger = new Logger(SchedulerService.name);
+  private readonly config: SchedulerConfig;
+  private isRunning = false;
+
+  public constructor(
+    private readonly configService: ConfigService,
+    private readonly schedulerRegistry: SchedulerRegistry,
+    private readonly ingestionService: IngestionService,
+  ) {
+    this.config = {
+      enabled: this.configService.get<boolean>('SCHEDULER_ENABLED', true),
+      cronExpression: this.configService.get<string>(
+        'SCHEDULER_CRON',
+        '*/5 * * * *',
+      ),
+      queueProcessingEnabled: this.configService.get<boolean>(
+        'QUEUE_PROCESSING_ENABLED',
+        true,
+      ),
+      queueProcessingIntervalMs: this.configService.get<number>(
+        'QUEUE_PROCESSING_INTERVAL_MS',
+        30000,
+      ),
+      recoveryEnabled: this.configService.get<boolean>(
+        'RECOVERY_ENABLED',
+        true,
+      ),
+    };
+  }
+
+  public async onModuleInit(): Promise<void> {
+    this.logger.log('Scheduler service initializing...');
+
+    if (this.config.recoveryEnabled) {
+      await this.recoverStuckQueues();
+    }
+
+    if (this.config.enabled) {
+      this.start();
+    }
+  }
+
+  public start(): void {
+    if (this.isRunning) {
+      this.logger.warn('Scheduler is already running');
+      return;
+    }
+
+    this.logger.log('Starting scheduler...');
+    this.isRunning = true;
+
+    // Schedule cron job for periodic processing
+    const job = new CronJob(
+      this.config.cronExpression,
+      async () => {
+        await this.runScheduledTasks();
+      },
+      null,
+      true,
+      'UTC',
+    );
+
+    this.schedulerRegistry.addCronJob('knowledge-scheduler', job);
+    job.start();
+
+    this.logger.log(
+      `Scheduler started with cron: ${this.config.cronExpression}`,
+    );
+  }
+
+  public stop(): void {
+    if (!this.isRunning) {
+      this.logger.warn('Scheduler is not running');
+      return;
+    }
+
+    this.logger.log('Stopping scheduler...');
+    this.isRunning = false;
+
+    try {
+      const job = this.schedulerRegistry.getCronJob('knowledge-scheduler');
+      void job.stop();
+      this.schedulerRegistry.deleteCronJob('knowledge-scheduler');
+      this.logger.log('Scheduler stopped');
+    } catch (error: unknown) {
+      this.logger.error('Error stopping scheduler:', error);
+    }
+  }
+
+  public restart(): void {
+    this.logger.log('Restarting scheduler...');
+    this.stop();
+    this.start();
+  }
+
+  public isSchedulerRunning(): boolean {
+    return this.isRunning;
+  }
+
+  private async runScheduledTasks(): Promise<void> {
+    this.logger.debug('Running scheduled tasks...');
+
+    if (this.config.queueProcessingEnabled) {
+      try {
+        await this.processQueues();
+      } catch (error) {
+        this.logger.error('Error processing queues:', error);
+      }
+    }
+  }
+
+  private async processQueues(): Promise<void> {
+    this.logger.debug('Processing queues...');
+
+    try {
+      const vectorResult = await this.ingestionService.processVectorQueue(10);
+      const graphResult = await this.ingestionService.processGraphSyncQueue(10);
+      const agentResult =
+        await this.ingestionService.processAgentRecommendationQueue(10);
+      const automationResult =
+        await this.ingestionService.processAutomationOpportunityQueue(10);
+
+      const totalProcessed =
+        vectorResult.completed +
+        graphResult.completed +
+        agentResult.completed +
+        automationResult.completed;
+
+      if (totalProcessed > 0) {
+        this.logger.log(
+          `Processed ${totalProcessed} queue items (vector: ${vectorResult.completed}, graph: ${graphResult.completed}, agent: ${agentResult.completed}, automation: ${automationResult.completed})`,
+        );
+      }
+    } catch (error) {
+      this.logger.error('Failed to process queues:', error);
+      throw error;
+    }
+  }
+
+  private async recoverStuckQueues(): Promise<void> {
+    this.logger.log('Recovering stuck queues...');
+
+    try {
+      // Use the dedicated recovery method that handles all queue types
+      await this.ingestionService.recoverQueuesAfterRestart();
+
+      // Also check for any stuck items in the legacy queue
+      const recovered = await this.ingestionService.recoverStuckItems();
+
+      if (recovered > 0) {
+        this.logger.log(`Recovered ${recovered} stuck queue items`);
+      } else {
+        this.logger.log('No stuck queue items found');
+      }
+    } catch (error) {
+      this.logger.error('Error recovering stuck queues:', error);
+    }
+  }
+}

package/apps/knowledge/src/modules/sync/sync.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { ScheduleModule } from '@nestjs/schedule';
+import { IngestionModule } from '../ingestion/ingestion.module';
+import { SchedulerService } from './scheduler.service';
+
+@Module({
+  imports: [ScheduleModule.forRoot(), IngestionModule],
+  providers: [SchedulerService],
+  exports: [SchedulerService],
+})
+export class SyncModule {}

package/apps/knowledge/src/modules/topics/dto/topic.dto.ts

@@ -0,0 +1,50 @@
+import { createZodDto } from 'nestjs-zod';
+import { z } from 'zod';
+
+export const TopicTypeSchema = z.enum(['manual', 'auto']);
+export const TopicStatusSchema = z.enum([
+  'empty',
+  'ingesting',
+  'ready',
+  'error',
+]);
+
+export const CreateTopicSchema = z.object({
+  id: z.string().uuid().optional(),
+  tenantId: z.string().min(1),
+  appId: z.string().min(1),
+  name: z.string().min(1),
+  containerTag: z.string().min(1),
+  description: z.string().optional(),
+  type: TopicTypeSchema.optional(),
+  status: TopicStatusSchema.optional(),
+  documentCount: z.number().int().min(0).optional(),
+  createdBy: z.string().min(1),
+});
+
+export const UpdateTopicSchema = CreateTopicSchema.omit({
+  id: true,
+  tenantId: true,
+  appId: true,
+  createdBy: true,
+  containerTag: true,
+}).partial();
+
+export const ListTopicsQuerySchema = z.object({
+  tenantId: z.string().optional(),
+  appId: z.string().optional(),
+  type: TopicTypeSchema.optional(),
+  status: TopicStatusSchema.optional(),
+  limit: z.coerce.number().int().min(1).max(200).optional(),
+  offset: z.coerce.number().int().min(0).optional(),
+});
+
+export class CreateTopicDto extends createZodDto(CreateTopicSchema) {}
+
+export class UpdateTopicDto extends createZodDto(UpdateTopicSchema) {}
+
+export class ListTopicsQueryDto extends createZodDto(ListTopicsQuerySchema) {}
+
+export type CreateTopicInput = z.infer<typeof CreateTopicSchema>;
+export type UpdateTopicInput = z.infer<typeof UpdateTopicSchema>;
+export type ListTopicsQueryInput = z.infer<typeof ListTopicsQuerySchema>;

package/apps/knowledge/src/modules/topics/topics.module.ts

@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+import { PlatformModule } from '../platform/platform.module';
+import { TopicsService } from './topics.service';
+
+@Module({
+  imports: [PlatformModule],
+  providers: [TopicsService],
+  exports: [TopicsService],
+})
+export class TopicsModule {}

package/apps/knowledge/src/modules/topics/topics.service.ts

@@ -0,0 +1,86 @@
+import { Injectable, NotFoundException } from '@nestjs/common';
+import { randomUUID } from 'node:crypto';
+import { MetadataStoreService } from '../platform/metadata-store.service';
+import {
+  CreateTopicInput,
+  ListTopicsQueryInput,
+  UpdateTopicInput,
+} from './dto/topic.dto';
+
+export interface TopicRecord extends Record<string, unknown> {
+  id: string;
+}
+
+@Injectable()
+export class TopicsService {
+  public constructor(private readonly metadataStore: MetadataStoreService) {}
+
+  public async list(query: ListTopicsQueryInput): Promise<TopicRecord[]> {
+    const filters: Record<string, string> = {};
+
+    if (query.tenantId) {
+      filters.tenantId = query.tenantId;
+    }
+    if (query.appId) {
+      filters.appId = query.appId;
+    }
+    if (query.type) {
+      filters.type = query.type;
+    }
+    if (query.status) {
+      filters.status = query.status;
+    }
+
+    return this.metadataStore.list('Topic', {
+      filters: Object.keys(filters).length > 0 ? filters : undefined,
+      limit: query.limit,
+      offset: query.offset,
+    });
+  }
+
+  public async getById(id: string): Promise<TopicRecord> {
+    const topic = await this.metadataStore.getById('Topic', id);
+    if (!topic) {
+      throw new NotFoundException(`Topic/${id} not found`);
+    }
+
+    return topic;
+  }
+
+  public async create(payload: CreateTopicInput): Promise<TopicRecord> {
+    const now = new Date().toISOString();
+
+    return this.metadataStore.upsert('Topic', {
+      ...payload,
+      id: payload.id ?? randomUUID(),
+      type: payload.type ?? 'manual',
+      status: payload.status ?? 'empty',
+      documentCount: payload.documentCount ?? 0,
+      createdAt: now,
+      updatedAt: now,
+    });
+  }
+
+  public async update(
+    id: string,
+    payload: UpdateTopicInput,
+  ): Promise<TopicRecord> {
+    const current = await this.getById(id);
+
+    return this.metadataStore.upsert('Topic', {
+      ...current,
+      ...payload,
+      id,
+      updatedAt: new Date().toISOString(),
+    });
+  }
+
+  public async remove(id: string): Promise<{ success: boolean; id: string }> {
+    const deleted = await this.metadataStore.remove('Topic', id);
+
+    return {
+      success: deleted,
+      id,
+    };
+  }
+}

package/apps/knowledge/tsconfig.app.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/apps/knowledge"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "test", "**/*spec.ts"]
+}