modular-agent-examples 0.0.1

package/chunking-demo.ts

@@ -0,0 +1,339 @@
+/**
+ * Chunking Demo
+ *
+ * Demonstrates various chunking strategies without requiring external APIs.
+ * Shows how to use different chunker types and configure them for your use case.
+ *
+ * Run with: npm run example -- examples/chunking-demo.ts
+ */
+
+import { TextChunker, RecursiveChunker, TokenChunker } from "../lib";
+
+// Sample document for demonstration
+const SAMPLE_DOCUMENT = `# Advanced Machine Learning Guide
+
+## Introduction
+
+Machine learning is a subset of artificial intelligence that enables systems to learn
+and improve from experience without being explicitly programmed. This guide covers the
+fundamental concepts and practical applications.
+
+## Chapter 1: Fundamentals
+
+### What is Machine Learning?
+
+Machine learning algorithms build a model based on sample data, known as training data,
+in order to make predictions or decisions without being explicitly programmed to do so.
+The goal is to develop algorithms that can learn from and make predictions on data.
+
+There are three main types of machine learning:
+
+1. **Supervised Learning** - Learning from labeled data
+2. **Unsupervised Learning** - Finding patterns in unlabeled data
+3. **Reinforcement Learning** - Learning through rewards and penalties
+
+### Key Concepts
+
+**Features** are the input variables used to make predictions. Feature engineering is
+often the most important step in building effective models.
+
+**Labels** are the target values we want to predict in supervised learning.
+
+**Models** are mathematical representations of patterns learned from data.
+
+## Chapter 2: Supervised Learning
+
+Supervised learning requires labeled training data. Common algorithms include:
+
+- Linear Regression
+- Logistic Regression
+- Decision Trees
+- Support Vector Machines
+- Neural Networks
+
+Each algorithm has strengths and weaknesses depending on your data and problem.
+
+### Training and Evaluation
+
+The process involves:
+
+1. Split data into training and test sets
+2. Train the model on training data
+3. Evaluate on test data
+4. Tune hyperparameters
+5. Repeat until performance is acceptable
+
+## Chapter 3: Unsupervised Learning
+
+Unsupervised learning finds hidden patterns in data without labels.
+
+**Clustering** groups similar data points together:
+- K-means Clustering
+- Hierarchical Clustering
+- DBSCAN
+
+**Dimensionality Reduction** reduces the number of features:
+- Principal Component Analysis (PCA)
+- t-SNE
+- Autoencoders
+
+## Chapter 4: Advanced Topics
+
+### Deep Learning
+
+Deep learning uses neural networks with multiple layers to learn hierarchical
+representations of data. Applications include:
+
+- Computer Vision (image recognition, object detection)
+- Natural Language Processing (translation, sentiment analysis)
+- Speech Recognition
+- Game Playing
+
+### Transfer Learning
+
+Transfer learning leverages pre-trained models on new tasks, reducing training time
+and data requirements significantly.
+
+### Model Deployment
+
+Once trained, models must be deployed in production systems. Considerations include:
+
+- Model serialization and versioning
+- API endpoints for predictions
+- Monitoring model performance
+- Handling model drift over time
+
+## Conclusion
+
+Machine learning is a powerful tool for solving complex problems. Success requires
+understanding the fundamentals, choosing appropriate algorithms, and iterating
+on your approach based on empirical results.`;
+
+async function main() {
+  console.log("╔════════════════════════════════════════════════════════════╗");
+  console.log("║         Machine Learning Document Chunking Demo            ║");
+  console.log(
+    "╚════════════════════════════════════════════════════════════╝\n"
+  );
+
+  // ==================== Example 1: TextChunker ====================
+  console.log("📝 Example 1: TextChunker (Character-based)\n");
+  console.log("The TextChunker splits text by a fixed number of characters.");
+  console.log("Good for: Uniform chunks, simple use cases\n");
+
+  const textChunker = new TextChunker({
+    chunkSize: 400,
+    chunkOverlap: 50,
+  });
+
+  const textChunks = await textChunker.chunk(SAMPLE_DOCUMENT, {
+    sourceId: "ml-guide-v1",
+    sourcePath: "/docs/ml-guide.md",
+    metadata: {
+      category: "educational",
+      difficulty: "intermediate",
+    },
+  });
+
+  console.log(`Created ${textChunks.length} chunks:\n`);
+  for (let i = 0; i < Math.min(3, textChunks.length); i++) {
+    const chunk = textChunks[i];
+    console.log(`Chunk ${i + 1}:`);
+    console.log(`  ID: ${chunk.id}`);
+    console.log(`  Size: ${chunk.metadata.char_count} characters`);
+    console.log(`  Preview: ${chunk.content.substring(0, 80)}...`);
+    console.log();
+  }
+
+  // ==================== Example 2: RecursiveChunker ====================
+  console.log("\n🔍 Example 2: RecursiveChunker (Semantic boundaries)\n");
+  console.log(
+    "The RecursiveChunker splits on semantic boundaries (paragraphs, sections)"
+  );
+  console.log("Good for: Markdown, documents with clear structure\n");
+
+  const recursiveChunker = new RecursiveChunker({
+    chunkSize: 500,
+    chunkOverlap: 75,
+    separators: ["\n\n", "\n### ", "\n## ", "\n# ", ". ", " "],
+  });
+
+  const recursiveChunks = await recursiveChunker.chunk(SAMPLE_DOCUMENT, {
+    sourceId: "ml-guide-v1",
+    sourcePath: "/docs/ml-guide.md",
+    metadata: {
+      category: "educational",
+      difficulty: "intermediate",
+    },
+  });
+
+  console.log(`Created ${recursiveChunks.length} chunks:\n`);
+  for (let i = 0; i < Math.min(4, recursiveChunks.length); i++) {
+    const chunk = recursiveChunks[i];
+    console.log(`Chunk ${i + 1}:`);
+    console.log(`  ID: ${chunk.id}`);
+    console.log(`  Section: ${chunk.metadata.section || "Unknown"}`);
+    console.log(`  Size: ${chunk.metadata.char_count} characters`);
+    console.log(`  Level: ${chunk.metadata.n_level || 0}`);
+    console.log();
+  }
+
+  // ==================== Example 3: TokenChunker ====================
+  console.log("\n⚙️  Example 3: TokenChunker (Token-aware)\n");
+  console.log("The TokenChunker respects token limits for LLM compatibility");
+  console.log("Good for: LLM processing, consistent token counts\n");
+
+  try {
+    const tokenChunker = new TokenChunker({
+      chunkSize: 250,
+      chunkOverlap: 25,
+    });
+
+    const tokenChunks = await tokenChunker.chunk(SAMPLE_DOCUMENT, {
+      sourceId: "ml-guide-v1",
+    });
+
+    console.log(`Created ${tokenChunks.length} chunks:\n`);
+    console.log("Token distribution:");
+    const tokenCounts = tokenChunks.map((c) => c.metadata.token_count || 0);
+    const avgTokens = Math.round(
+      tokenCounts.reduce((a, b) => a + b, 0) / tokenCounts.length
+    );
+    const minTokens = Math.min(...tokenCounts);
+    const maxTokens = Math.max(...tokenCounts);
+
+    console.log(`  Average: ${avgTokens} tokens`);
+    console.log(`  Min: ${minTokens} tokens`);
+    console.log(`  Max: ${maxTokens} tokens`);
+    console.log();
+  } catch (error) {
+    console.log(
+      "Note: TokenChunker requires building with --loader ts-node/esm"
+    );
+    console.log(
+      "For now, use: node --loader ts-node/esm examples/chunking-demo.ts\n"
+    );
+  }
+
+  // ==================== Example 4: Custom Processing ====================
+  console.log("\n🛠️  Example 4: Custom Chunk Processing\n");
+  console.log("Apply custom transformations to chunks during processing.\n");
+
+  const processingChunker = new TextChunker({
+    chunkSize: 300,
+    chunkOverlap: 30,
+    chunkProcessor: (chunk) => {
+      // Filter chunks that are too short
+      if (chunk.content.trim().length < 50) {
+        return null;
+      }
+
+      // Count sentences
+      const sentences = chunk.content.split(/[.!?]+/).filter((s) => s.trim());
+
+      return {
+        ...chunk,
+        metadata: {
+          ...chunk.metadata,
+          sentenceCount: sentences.length,
+          processedAt: new Date().toISOString(),
+        },
+      };
+    },
+  });
+
+  const processedChunks = await processingChunker.chunk(SAMPLE_DOCUMENT);
+
+  console.log(`Created ${processedChunks.length} chunks after processing:\n`);
+  for (let i = 0; i < Math.min(3, processedChunks.length); i++) {
+    const chunk = processedChunks[i];
+    console.log(`Chunk ${i + 1}:`);
+    console.log(`  Sentences: ${chunk.metadata.sentenceCount}`);
+    console.log(`  Size: ${chunk.metadata.char_count} characters`);
+    console.log(`  Processed: ${chunk.metadata.processedAt}`);
+    console.log();
+  }
+
+  // ==================== Example 5: Chunk Navigation ====================
+  console.log("\n🔗 Example 5: Chunk Navigation\n");
+  console.log("Chunks maintain references for sequential navigation.\n");
+
+  const navChunks = await textChunker.chunk(
+    "Chunk 1. Chunk 2. Chunk 3. Chunk 4.",
+    {
+      sourceId: "simple-doc",
+    }
+  );
+
+  console.log(`Chain of ${navChunks.length} chunks:\n`);
+  for (let i = 0; i < navChunks.length; i++) {
+    const chunk = navChunks[i];
+    const prevLabel = chunk.metadata.prev_id
+      ? chunk.metadata.prev_id.substring(0, 8)
+      : "START";
+    const nextLabel = chunk.metadata.next_id
+      ? chunk.metadata.next_id.substring(0, 8)
+      : "END";
+
+    console.log(`[${prevLabel}] → Chunk ${i + 1} → [${nextLabel}]`);
+  }
+  console.log();
+
+  // ==================== Example 6: Comparison ====================
+  console.log("\n📊 Example 6: Chunker Comparison\n");
+
+  const testText = `Introduction to AI. Machine learning is powerful. Deep learning uses neural networks.
+  Transformers revolutionized NLP. Vision models process images. Multimodal models combine text and images.
+  Large language models show emergent capabilities. Fine-tuning adapts models to tasks.`;
+
+  console.log("Input text length: " + testText.length + " characters\n");
+  console.log("Strategy               | Chunks | Avg Size | Comments");
+  console.log("─".repeat(65));
+
+  // TextChunker comparison
+  const tc = new TextChunker({ chunkSize: 150, chunkOverlap: 0 });
+  const tcChunks = await tc.chunk(testText);
+  const tcAvg = Math.round(
+    tcChunks.reduce((a, c) => a + c.metadata.char_count, 0) / tcChunks.length
+  );
+  console.log(
+    `TextChunker (150)     | ${String(tcChunks.length).padEnd(6)}| ${String(
+      tcAvg
+    ).padEnd(8)}| Uniform chunks`
+  );
+
+  // RecursiveChunker comparison
+  const rc = new RecursiveChunker({ chunkSize: 150, chunkOverlap: 0 });
+  const rcChunks = await rc.chunk(testText);
+  const rcAvg = Math.round(
+    rcChunks.reduce((a, c) => a + c.metadata.char_count, 0) / rcChunks.length
+  );
+  console.log(
+    `RecursiveChunker (150)| ${String(rcChunks.length).padEnd(6)}| ${String(
+      rcAvg
+    ).padEnd(8)}| Respects sentences`
+  );
+
+  // TokenChunker comparison (skip if not available)
+  try {
+    const tk = new TokenChunker({ chunkSize: 40, chunkOverlap: 0 });
+    const tkChunks = await tk.chunk(testText);
+    const tkAvg = Math.round(
+      tkChunks.reduce((a, c) => a + (c.metadata.token_count || 0), 0) /
+        tkChunks.length
+    );
+    console.log(
+      `TokenChunker (40 toks)| ${String(tkChunks.length).padEnd(6)}| ${String(
+        tkAvg
+      ).padEnd(8)}| Token-aware`
+    );
+  } catch (error) {
+    console.log(
+      `TokenChunker (40 toks)| N/A    | N/A     | (requires ESM loader)`
+    );
+  }
+
+  console.log("\n✅ Chunking demo complete!\n");
+}
+
+main().catch(console.error);

package/cleanup-duplicates.ts

@@ -0,0 +1,142 @@
+/**
+ * Utility script to clean up duplicate documents from a LanceDB vector store.
+ *
+ * Duplicates are identified by the content hash stored in metadata.
+ * For each hash, keeps the first document and deletes the rest.
+ *
+ * Usage:
+ * - Update the store configuration below to match your database
+ * - Run with: npm run example -- examples/cleanup-duplicates.ts
+ */
+import "dotenv/config";
+import { LanceDBVectorStore } from "../lib/vectorstore/LanceDBVectorStore";
+import { OpenAIEmbeddings } from "../lib/embeddings/OpenAIEmbeddings";
+
+async function cleanupDuplicates() {
+  console.log("=== Vector Store Duplicate Cleanup ===\n");
+
+  // Check for required API key
+  if (!process.env.OPENAI_API_KEY) {
+    console.error("Error: OPENAI_API_KEY is required");
+    process.exit(1);
+  }
+
+  try {
+    // Step 1: Connect to the vector store
+    console.log("1. Connecting to vector store...");
+    const embeddings = new OpenAIEmbeddings({
+      model: "text-embedding-3-small",
+    });
+
+    const store = await LanceDBVectorStore.create({
+      name: "knowledge_base",
+      uri: "./examples/data/vectors",
+      tableName: "agention_docs",
+      embeddings,
+    });
+    // Note: No metadataFields needed — this connects to a pre-existing table
+    console.log("   Connected\n");
+
+    // Step 2: Get all documents
+    console.log("2. Fetching all documents...");
+    const table = store.getTable();
+    if (!table) throw new Error("Table not found — has data been ingested?");
+    const allDocs = await table.query().toArray();
+    console.log(`   Found ${allDocs.length} total documents\n`);
+
+    // Step 3: Group by hash
+    console.log("3. Identifying duplicates by content hash...");
+    const hashGroups = new Map<string, string[]>(); // hash -> [doc_ids]
+
+    for (const doc of allDocs) {
+      const docRecord = doc as unknown as {
+        id: string;
+        chunk_metadata?: { hash?: string };
+      };
+
+      const hash = docRecord.chunk_metadata?.hash;
+      if (hash) {
+        if (!hashGroups.has(hash)) {
+          hashGroups.set(hash, []);
+        }
+        hashGroups.get(hash)!.push(docRecord.id);
+      }
+    }
+
+    // Step 4: Find duplicate groups
+    const duplicateGroups = Array.from(hashGroups.entries()).filter(
+      ([_hash, ids]) => ids.length > 1
+    );
+
+    console.log(`   Found ${duplicateGroups.length} unique hashes`);
+    console.log(`   Found ${duplicateGroups.length} groups with duplicates\n`);
+
+    if (duplicateGroups.length === 0) {
+      console.log("✓ No duplicates found! Database is clean.\n");
+      process.exit(0);
+    }
+
+    // Step 5: Calculate what will be deleted
+    let totalDuplicates = 0;
+    for (const [_hash, ids] of duplicateGroups) {
+      totalDuplicates += ids.length - 1; // Keep first, delete rest
+    }
+
+    console.log("4. Duplicate summary:");
+    console.log(`   - Total duplicate documents: ${totalDuplicates}`);
+    console.log(
+      `   - Documents to keep (one per hash): ${duplicateGroups.length}`
+    );
+    console.log(`   - Documents to delete: ${totalDuplicates}\n`);
+
+    // Show some examples
+    console.log("   Example duplicates:");
+    for (let i = 0; i < Math.min(5, duplicateGroups.length); i++) {
+      const [hash, ids] = duplicateGroups[i];
+      console.log(
+        `   - Hash ${hash.substring(0, 16)}...: ${ids.length} copies`
+      );
+    }
+    console.log();
+
+    // Step 6: Confirm deletion
+    console.log("5. Deleting duplicates (keeping first occurrence)...");
+
+    let deletedCount = 0;
+    for (const [_hash, ids] of duplicateGroups) {
+      // Keep the first document, delete the rest
+      const idsToDelete = ids.slice(1);
+
+      if (idsToDelete.length > 0) {
+        const deleted = await store.delete(idsToDelete);
+        deletedCount += deleted;
+      }
+    }
+
+    console.log(`   Deleted ${deletedCount} duplicate documents\n`);
+
+    // Step 7: Verify cleanup
+    console.log("6. Verifying cleanup...");
+    const remainingDocs = await table.query().toArray();
+    console.log(`   Remaining documents: ${remainingDocs.length}`);
+    console.log(`   Expected: ${allDocs.length - totalDuplicates}`);
+
+    if (remainingDocs.length === allDocs.length - totalDuplicates) {
+      console.log("\n✓ Cleanup successful!\n");
+    } else {
+      console.log("\n⚠ Warning: Document count mismatch\n");
+    }
+
+    // Step 8: Optimize the table
+    console.log("7. Optimizing database...");
+    await store.optimize();
+    console.log("   Done\n");
+
+    console.log("=== Cleanup Complete ===");
+  } catch (error) {
+    console.error("Error during cleanup:", error);
+    process.exit(1);
+  }
+}
+
+cleanupDuplicates();

package/generative.ts

@@ -0,0 +1,128 @@
+import "dotenv/config";
+
+import { ClaudeAgent } from "../lib/agents/anthropic/ClaudeAgent";
+import { OpenAiAgent } from "../lib/agents/openai/OpenAiAgent";
+import { Tool, ToolEvent } from "../lib/tools/Tool";
+import { readFile, readdir, writeFile } from "fs/promises";
+
+import { createInterface } from "node:readline/promises";
+const rl = createInterface({
+  input: process.stdin,
+  output: process.stdout,
+});
+
+const directoryReadTool = new Tool({
+  name: "directoryReadTool",
+  description: `This tool can list the files in a local directory.`,
+  inputSchema: {
+    type: "object",
+    properties: {
+      path: {
+        type: "string",
+        description: "relative path, the default should be .",
+      },
+    },
+    required: ["path"],
+  },
+  execute: async (input): Promise<any> => {
+    console.log("directoryReadTool", input);
+    return await readdir(__dirname + "/../" + input.path, {
+      recursive: true,
+    });
+  },
+});
+
+const fileReadTool = new Tool({
+  name: "fileReadTool",
+  description: `This tool read a file in a local directory, you should not ask to read file that can contain private info lime .env files.`,
+  inputSchema: {
+    type: "object",
+    properties: {
+      path: {
+        type: "string",
+        description: "relative path of a file",
+      },
+    },
+    required: ["path"],
+  },
+  execute: async (input): Promise<any> => {
+    console.log("fileReadTool", input);
+    return await readFile(__dirname + "/../" + input.path, {
+      encoding: "utf8",
+    });
+  },
+});
+const fileWriteTool = new Tool({
+  name: "fileWriteTool",
+  description: `This tool can be used to write to files. Be careful, it will overwrite existing files with new content. Can be used to create new files as well.`,
+  inputSchema: {
+    type: "object",
+    properties: {
+      path: {
+        type: "string",
+        description: "relative path of a file",
+      },
+      data: {
+        type: "string",
+        description: "Content to write to the file",
+      },
+    },
+    required: ["path", "content"],
+  },
+  execute: async (input: {
+    path: string;
+    content: string;
+  }): Promise<string> => {
+    console.log("fileWriteTool", input.path, input.content);
+    const answer = await rl.question("Can I write this file? Y/N");
+    if (answer === "Y") {
+      await writeFile(__dirname + "/../" + input.path, input.content, {});
+      return "Success writing to " + input.path;
+    } else {
+      return "Could not write to " + input.path;
+    }
+  },
+});
+
+async function example() {
+  const agent2 = new ClaudeAgent({
+    id: "1",
+    description:
+      "You are a world class software engineer who helps with software projects, you have access to files and directories and can create and write to files.",
+    name: "Powerfull AI coder",
+    model: "claude-3-5-haiku-latest",
+    tools: [directoryReadTool, fileReadTool, fileWriteTool],
+    apiKey: process.env.ANTHROPIC_API_KEY as string,
+    disableParallelToolUse: false,
+    maxTokens: 8000,
+  });
+
+  const openaiAgent = new OpenAiAgent({
+    id: "1",
+    description:
+      "This helps with software projects, it takes commands, reads files and suggests improvements",
+    name: "Powerfull AI coder",
+    model: "gpt-4o-mini",
+    // tools: [directoryReadTool, fileReadTool],
+    apiKey: process.env.OPENAI_API_KEY as string,
+    disableParallelToolUse: false,
+  });
+
+  try {
+    directoryReadTool.on(ToolEvent.EXECUTE, () => {
+      // console.log("directoryReadTool.on(ToolEvent.EXECUTE", args);
+      // args.event.preventDefault();
+    });
+    agent2;
+    openaiAgent;
+    const result = await agent2.execute(
+      "write jest unit tests for ./lib/agents/openai/OpenAiAgent.ts"
+    );
+
+    console.log("final result:", result);
+  } catch (error) {
+    console.error("Error:", error);
+  }
+}
+
+example();