@louloulinx/metagpt 0.1.3

package/src/types/document.ts

@@ -0,0 +1,354 @@
+import { z } from 'zod';
+import path from 'path';
+import fs from 'fs/promises';
+
+/**
+ * Document status enum, similar to RFC/PEP
+ */
+export enum DocumentStatus {
+  DRAFT = 'draft',
+  UNDERREVIEW = 'underreview',
+  APPROVED = 'approved',
+  DONE = 'done',
+}
+
+/**
+ * Base document schema
+ */
+export const DocumentSchema = z.object({
+  path: z.string().optional(),
+  name: z.string().default(''),
+  content: z.string().default(''),
+  author: z.string().default(''),
+  status: z.nativeEnum(DocumentStatus).default(DocumentStatus.DRAFT),
+  reviews: z.array(z.any()).default([]),
+});
+
+/**
+ * Indexable document schema for vector databases or search engines
+ */
+export const IndexableDocumentSchema = DocumentSchema.extend({
+  data: z.union([z.array(z.any()), z.record(z.string(), z.any())]),
+  contentCol: z.string().default(''),
+  metaCol: z.string().default(''),
+});
+
+/**
+ * Repository metadata schema
+ */
+export const RepoMetadataSchema = z.object({
+  name: z.string().default(''),
+  nDocs: z.number().default(0),
+  nChars: z.number().default(0),
+  symbols: z.array(z.any()).default([]),
+});
+
+/**
+ * Repository schema
+ */
+export const RepoSchema = z.object({
+  name: z.string().default(''),
+  docs: z.record(z.string(), DocumentSchema).default({}),
+  codes: z.record(z.string(), DocumentSchema).default({}),
+  assets: z.record(z.string(), DocumentSchema).default({}),
+  path: z.string().optional(),
+});
+
+/**
+ * Document types
+ */
+export type Document = z.infer<typeof DocumentSchema> & {
+  toPath?: (path?: string) => Promise<void>;
+};
+export type IndexableDocument = z.infer<typeof IndexableDocumentSchema>;
+export type RepoMetadata = z.infer<typeof RepoMetadataSchema>;
+export type Repo = z.infer<typeof RepoSchema>;
+
+/**
+ * Document class implementation
+ */
+export class DocumentImpl implements Document {
+  public path?: string;
+  public name: string = '';
+  public content: string = '';
+  public author: string = '';
+  public status: DocumentStatus = DocumentStatus.DRAFT;
+  public reviews: any[] = [];
+
+  constructor(data: Partial<Document> = {}) {
+    Object.assign(this, DocumentSchema.parse(data));
+  }
+
+  /**
+   * Create a Document instance from a file path
+   */
+  public static async fromPath(filePath: string): Promise<DocumentImpl> {
+    try {
+      const content = await fs.readFile(filePath, 'utf-8');
+      return new DocumentImpl({
+        content,
+        path: filePath,
+        name: path.basename(filePath),
+      });
+    } catch (error: unknown) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Failed to read file ${filePath}: ${message}`);
+    }
+  }
+
+  /**
+   * Create a Document from a text string
+   */
+  public static fromText(text: string, filePath?: string): DocumentImpl {
+    return new DocumentImpl({
+      content: text,
+      path: filePath,
+      name: filePath ? path.basename(filePath) : '',
+    });
+  }
+
+  /**
+   * Save content to the specified file path
+   */
+  public async toPath(filePath?: string): Promise<void> {
+    const targetPath = filePath || this.path;
+    if (!targetPath) {
+      throw new Error('File path is not set.');
+    }
+
+    try {
+      await fs.mkdir(path.dirname(targetPath), { recursive: true });
+      await fs.writeFile(targetPath, this.content, 'utf-8');
+      this.path = targetPath;
+    } catch (error: unknown) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Failed to write file ${targetPath}: ${message}`);
+    }
+  }
+
+  /**
+   * Persist document to disk
+   */
+  public async persist(): Promise<void> {
+    return this.toPath();
+  }
+}
+
+/**
+ * Indexable document class implementation
+ */
+export class IndexableDocumentImpl extends DocumentImpl implements IndexableDocument {
+  public data: any[] | Record<string, any>;
+  public contentCol: string = '';
+  public metaCol: string = '';
+
+  constructor(data: Partial<IndexableDocument> = {}) {
+    super(data);
+    const parsed = IndexableDocumentSchema.parse(data);
+    this.data = parsed.data;
+    this.contentCol = parsed.contentCol;
+    this.metaCol = parsed.metaCol;
+  }
+
+  /**
+   * Create an IndexableDocument instance from a file path
+   */
+  public static async fromPath(
+    filePath: string,
+    contentCol: string = 'content',
+    metaCol: string = 'metadata'
+  ): Promise<IndexableDocumentImpl> {
+    try {
+      const content = await fs.readFile(filePath, 'utf-8');
+      let data: any[] | Record<string, any>;
+
+      // Parse data based on file extension
+      const ext = path.extname(filePath).toLowerCase();
+      switch (ext) {
+        case '.json':
+          data = JSON.parse(content);
+          break;
+        case '.csv':
+          // Simple CSV parsing (for more complex needs, use a CSV library)
+          data = content.split('\n').map(line => line.split(','));
+          break;
+        default:
+          data = [{ content }];
+      }
+
+      return new IndexableDocumentImpl({
+        content,
+        path: filePath,
+        name: path.basename(filePath),
+        data,
+        contentCol,
+        metaCol,
+      });
+    } catch (error: unknown) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Failed to read file ${filePath}: ${message}`);
+    }
+  }
+
+  /**
+   * Get documents and metadata from the data
+   */
+  public getDocsAndMetadata(): [string[], Record<string, any>[]] {
+    const docs: string[] = [];
+    const metadata: Record<string, any>[] = [];
+
+    if (Array.isArray(this.data)) {
+      // Handle array data
+      this.data.forEach(item => {
+        const content = typeof item === 'string' ? item : item[this.contentCol] || '';
+        docs.push(content);
+        metadata.push(this.metaCol && typeof item === 'object' ? { [this.metaCol]: item[this.metaCol] } : {});
+      });
+    } else {
+      // Handle record data
+      Object.entries(this.data).forEach(([key, value]) => {
+        const content = typeof value === 'string' ? value : value[this.contentCol] || '';
+        docs.push(content);
+        metadata.push(this.metaCol && typeof value === 'object' ? { [this.metaCol]: value[this.metaCol] } : {});
+      });
+    }
+
+    return [docs, metadata];
+  }
+}
+
+/**
+ * Repository class implementation
+ */
+export class RepoImpl implements Repo {
+  public name: string = '';
+  public docs: Record<string, Document> = {};
+  public codes: Record<string, Document> = {};
+  public assets: Record<string, Document> = {};
+  public path?: string;
+
+  constructor(data: Partial<Repo> = {}) {
+    Object.assign(this, RepoSchema.parse(data));
+  }
+
+  private _path(filename: string): string {
+    if (!this.path) {
+      throw new Error('Repository path is not set.');
+    }
+    return path.join(this.path, filename);
+  }
+
+  /**
+   * Create a Repo instance from a directory path
+   */
+  public static async fromPath(dirPath: string): Promise<RepoImpl> {
+    try {
+      await fs.mkdir(dirPath, { recursive: true });
+      const repo = new RepoImpl({
+        path: dirPath,
+        name: path.basename(dirPath),
+      });
+
+      // Read all files recursively
+      const readDir = async (dir: string) => {
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+          const fullPath = path.join(dir, entry.name);
+          if (entry.isDirectory()) {
+            await readDir(fullPath);
+          } else {
+            const ext = path.extname(entry.name).toLowerCase();
+            if (['.json', '.txt', '.md', '.py', '.js', '.css', '.html'].includes(ext)) {
+              const content = await fs.readFile(fullPath, 'utf-8');
+              await repo._set(content, fullPath);
+            }
+          }
+        }
+      };
+
+      await readDir(dirPath);
+      return repo;
+    } catch (error: unknown) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`Failed to create repository from ${dirPath}: ${message}`);
+    }
+  }
+
+  /**
+   * Save all documents to disk
+   */
+  public async toPath(): Promise<void> {
+    const saveAll = async (docs: Record<string, Document>) => {
+      for (const doc of Object.values(docs)) {
+        if (doc instanceof DocumentImpl && doc.toPath) {
+          await doc.toPath();
+        }
+      }
+    };
+
+    await Promise.all([
+      saveAll(this.docs),
+      saveAll(this.codes),
+      saveAll(this.assets),
+    ]);
+  }
+
+  private async _set(content: string, filePath: string): Promise<Document> {
+    const ext = path.extname(filePath).toLowerCase();
+    const doc = new DocumentImpl({
+      content,
+      path: filePath,
+      name: path.relative(this.path || '', filePath),
+    });
+
+    if (ext === '.md') {
+      this.docs[filePath] = doc;
+    } else if (['.py', '.js', '.css', '.html'].includes(ext)) {
+      this.codes[filePath] = doc;
+    } else {
+      this.assets[filePath] = doc;
+    }
+
+    return doc;
+  }
+
+  /**
+   * Set a document and persist it to disk
+   */
+  public async set(filename: string, content: string): Promise<void> {
+    const filePath = this._path(filename);
+    const doc = await this._set(content, filePath);
+    await doc.toPath();
+  }
+
+  /**
+   * Get a document by its filename
+   */
+  public get(filename: string): Document | undefined {
+    const filePath = this._path(filename);
+    return this.docs[filePath] || this.codes[filePath] || this.assets[filePath];
+  }
+
+  /**
+   * Get all text documents
+   */
+  public getTextDocuments(): Document[] {
+    return [...Object.values(this.docs), ...Object.values(this.codes)];
+  }
+
+  /**
+   * Get repository metadata
+   */
+  public getMetadata(): RepoMetadata {
+    const nDocs = Object.keys(this.docs).length + Object.keys(this.codes).length + Object.keys(this.assets).length;
+    const nChars = [...Object.values(this.docs), ...Object.values(this.codes), ...Object.values(this.assets)]
+      .reduce((sum, doc) => sum + doc.content.length, 0);
+
+    return {
+      name: this.name,
+      nDocs,
+      nChars,
+      symbols: [], // TODO: Implement symbol extraction
+    };
+  }
+} 

package/src/types/llm.ts

@@ -0,0 +1,64 @@
+/**
+ * @module LLM
+ * @category Core
+ */
+
+import { z } from 'zod';
+
+/**
+ * LLM configuration schema
+ * Defines settings for language model operations
+ */
+export const LLMConfigSchema = z.object({
+  /** API key for authentication */
+  apiKey: z.string(),
+  /** Model name (e.g., 'gpt-4') */
+  model: z.string().default('gpt-4'),
+  /** Temperature for response generation */
+  temperature: z.number().min(0).max(2).default(0.7),
+  /** Maximum tokens in response */
+  maxTokens: z.number().min(1).default(2000),
+  /** Top P sampling */
+  topP: z.number().min(0).max(1).default(1),
+  /** Frequency penalty */
+  frequencyPenalty: z.number().min(-2).max(2).default(0),
+  /** Presence penalty */
+  presencePenalty: z.number().min(-2).max(2).default(0),
+  /** Base URL for API */
+  baseURL: z.string().optional(),
+  /** Organization ID */
+  organization: z.string().optional(),
+  /** Proxy URL */
+  proxy: z.string().optional(),
+});
+
+export type LLMConfig = z.infer<typeof LLMConfigSchema>;
+
+/**
+ * LLM provider interface
+ * Defines methods that must be implemented by LLM providers
+ */
+export interface LLMProvider {
+  /**
+   * Generate text completion
+   * @param prompt - Input prompt
+   * @param config - Optional configuration overrides
+   * @returns Generated text
+   */
+  generate(prompt: string, config?: Partial<LLMConfig>): Promise<string>;
+
+  /**
+   * Generate text completion as a stream
+   * @param prompt - Input prompt
+   * @param config - Optional configuration overrides
+   * @returns Generated text stream
+   */
+  generateStream(prompt: string, config?: Partial<LLMConfig>): AsyncIterable<string>;
+
+  /**
+   * Create text embeddings
+   * @param text - Input text
+   * @returns Embedding vector
+   */
+  embed(text: string): Promise<number[]>;
+} 

package/src/types/memory.ts

@@ -0,0 +1,36 @@
+import { z } from 'zod';
+import type { Message } from './message';
+
+export const MemoryConfigSchema = z.object({
+  maxSize: z.number().optional(),
+});
+
+export type MemoryConfig = z.infer<typeof MemoryConfigSchema>;
+
+export class ArrayMemory {
+  private messages: Message[] = [];
+  private config: MemoryConfig;
+
+  constructor(config: MemoryConfig = {}) {
+    this.config = MemoryConfigSchema.parse(config);
+  }
+
+  add(message: Message): void {
+    this.messages.push(message);
+    if (this.config.maxSize && this.messages.length > this.config.maxSize) {
+      this.messages.shift();
+    }
+  }
+
+  get(): Message[] {
+    return [...this.messages];
+  }
+
+  getByActions(actions: Set<string>): Message[] {
+    return this.messages.filter(msg => actions.has(msg.causedBy));
+  }
+
+  clear(): void {
+    this.messages = [];
+  }
+} 

package/src/types/message.ts

@@ -0,0 +1,193 @@
+import { z } from 'zod';
+import { v4 as uuidv4 } from 'uuid';
+
+/**
+ * Message route constants
+ */
+export const MESSAGE_ROUTE = {
+  CAUSE_BY: 'UserRequirement',
+  FROM: '',
+  TO: '*',
+  TO_ALL: '*',
+} as const;
+
+/**
+ * Simple message schema without routing information
+ */
+export const SimpleMessageSchema = z.object({
+  content: z.string(),
+  role: z.string(),
+});
+
+/**
+ * Base message schema with full routing and content information
+ */
+export const MessageSchema = z.object({
+  id: z.string().default('').transform(id => id || uuidv4()),
+  content: z.string(),
+  instructContent: z.any().optional().nullable(),
+  role: z.string().default('user'),
+  causedBy: z.string().default(MESSAGE_ROUTE.CAUSE_BY),
+  sentFrom: z.string().default(MESSAGE_ROUTE.FROM),
+  sendTo: z.union([
+    z.string(),
+    z.array(z.string()),
+    z.instanceof(Set),
+  ]).default(MESSAGE_ROUTE.TO_ALL).transform(data => {
+    if (typeof data === 'string') {
+      return new Set([data]);
+    }
+    if (Array.isArray(data)) {
+      return new Set(data);
+    }
+    if (data instanceof Set) {
+      return data;
+    }
+    return new Set([MESSAGE_ROUTE.TO_ALL]);
+  }),
+});
+
+/**
+ * Message types
+ */
+export type SimpleMessage = z.infer<typeof SimpleMessageSchema>;
+export type Message = z.infer<typeof MessageSchema>;
+
+/**
+ * Message queue interface for handling asynchronous message updates
+ */
+export interface MessageQueue {
+  push(msg: Message): void;
+  pop(): Promise<Message | null>;
+  popAll(): Promise<Message[]>;
+  empty(): boolean;
+}
+
+/**
+ * Message queue implementation using async queue
+ */
+export class AsyncMessageQueue implements MessageQueue {
+  private queue: Message[] = [];
+
+  /**
+   * Push a message to the queue
+   * @param msg - Message to push
+   */
+  public push(msg: Message): void {
+    this.queue.push(msg);
+  }
+
+  /**
+   * Pop a message from the queue
+   * @returns Promise resolving to the next message or null if queue is empty
+   */
+  public async pop(): Promise<Message | null> {
+    if (this.queue.length === 0) {
+      return null;
+    }
+    return this.queue.shift() || null;
+  }
+
+  /**
+   * Pop all messages from the queue
+   * @returns Promise resolving to array of all messages
+   */
+  public async popAll(): Promise<Message[]> {
+    const messages = [...this.queue];
+    this.queue = [];
+    return messages;
+  }
+
+  /**
+   * Check if queue is empty
+   * @returns True if queue is empty
+   */
+  public empty(): boolean {
+    return this.queue.length === 0;
+  }
+
+  /**
+   * Convert queue to JSON string
+   * @returns JSON string representation of queue
+   */
+  public async dump(): Promise<string> {
+    return JSON.stringify({
+      messages: this.queue.map(msg => ({
+        ...msg,
+        sendTo: Array.from(msg.sendTo),
+      })),
+    });
+  }
+
+  /**
+   * Create queue from JSON string
+   * @param data - JSON string representation of queue
+   * @returns New AsyncMessageQueue instance
+   */
+  public static load(data: string): AsyncMessageQueue {
+    const queue = new AsyncMessageQueue();
+    try {
+      const parsed = JSON.parse(data);
+      if (Array.isArray(parsed.messages)) {
+        queue.queue = parsed.messages.map((msg: any) => ({
+          ...msg,
+          sendTo: new Set(msg.sendTo),
+        }));
+      }
+    } catch (error) {
+      console.error('Failed to load message queue:', error);
+    }
+    return queue;
+  }
+}
+
+/**
+ * User message with predefined role
+ */
+export class UserMessage implements Message {
+  public id: string = uuidv4();
+  public content: string;
+  public instructContent: any = null;
+  public role: string = 'user';
+  public causedBy: string = MESSAGE_ROUTE.CAUSE_BY;
+  public sentFrom: string = MESSAGE_ROUTE.FROM;
+  public sendTo: Set<string> = new Set([MESSAGE_ROUTE.TO_ALL]);
+
+  constructor(content: string) {
+    this.content = content;
+  }
+}
+
+/**
+ * System message with predefined role
+ */
+export class SystemMessage implements Message {
+  public id: string = uuidv4();
+  public content: string;
+  public instructContent: any = null;
+  public role: string = 'system';
+  public causedBy: string = MESSAGE_ROUTE.CAUSE_BY;
+  public sentFrom: string = MESSAGE_ROUTE.FROM;
+  public sendTo: Set<string> = new Set([MESSAGE_ROUTE.TO_ALL]);
+
+  constructor(content: string) {
+    this.content = content;
+  }
+}
+
+/**
+ * AI message with predefined role
+ */
+export class AIMessage implements Message {
+  public id: string = uuidv4();
+  public content: string;
+  public instructContent: any = null;
+  public role: string = 'assistant';
+  public causedBy: string = MESSAGE_ROUTE.CAUSE_BY;
+  public sentFrom: string = MESSAGE_ROUTE.FROM;
+  public sendTo: Set<string> = new Set([MESSAGE_ROUTE.TO_ALL]);
+
+  constructor(content: string) {
+    this.content = content;
+  }
+}