@stackmemoryai/stackmemory 0.5.58 → 0.5.61

package/dist/src/skills/repo-ingestion-skill.js

@@ -0,0 +1,632 @@
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname as __pathDirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import { Logger } from "../core/monitoring/logger.js";
+import {
+  isChromaDBEnabled,
+  getChromaDBConfig
+} from "../core/config/storage-config.js";
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+import { execSync } from "child_process";
+import ignore from "ignore";
+async function getChromaDBAdapter() {
+  try {
+    return await import("../core/storage/chromadb-adapter.js");
+  } catch {
+    throw new Error(
+      "chromadb is not installed. Install it with: npm install chromadb"
+    );
+  }
+}
+class RepoIngestionSkill {
+  constructor(config, userId, teamId) {
+    this.config = config;
+    this.userId = userId;
+    this.teamId = teamId;
+    this.logger = new Logger("RepoIngestionSkill");
+    this.chromaEnabled = isChromaDBEnabled();
+    if (this.chromaEnabled) {
+      const chromaConfig = getChromaDBConfig();
+      if (chromaConfig && chromaConfig.apiKey) {
+        this.adapterConfig = {
+          apiKey: config?.apiKey || chromaConfig.apiKey,
+          tenant: config?.tenant || chromaConfig.tenant || "default_tenant",
+          database: config?.database || chromaConfig.database || "default_database",
+          collectionName: config?.collectionName || "stackmemory_repos"
+        };
+      }
+    }
+  }
+  logger;
+  adapter = null;
+  metadataCache = /* @__PURE__ */ new Map();
+  fileHashCache = /* @__PURE__ */ new Map();
+  chromaEnabled = false;
+  adapterConfig = null;
+  /**
+   * Check if ChromaDB is available for use
+   */
+  isAvailable() {
+    return this.chromaEnabled && this.adapter !== null;
+  }
+  async initialize() {
+    if (!this.chromaEnabled || !this.adapterConfig) {
+      this.logger.warn(
+        "ChromaDB not enabled. Repository ingestion features are unavailable."
+      );
+      this.logger.warn('Run "stackmemory init --chromadb" to enable ChromaDB.');
+      return;
+    }
+    if (!this.adapter && this.adapterConfig) {
+      try {
+        const { ChromaDBAdapter } = await getChromaDBAdapter();
+        this.adapter = await ChromaDBAdapter.create(
+          this.adapterConfig,
+          this.userId,
+          this.teamId
+        );
+      } catch (error) {
+        this.logger.warn(
+          `Failed to initialize ChromaDB: ${error instanceof Error ? error.message : String(error)}`
+        );
+        this.logger.warn(
+          "chromadb is optional. Install it with: npm install chromadb"
+        );
+        return;
+      }
+    }
+    if (this.adapter) {
+      await this.adapter.initialize();
+    }
+    await this.loadMetadataCache();
+  }
+  /**
+   * Ingest a repository into ChromaDB
+   */
+  async ingestRepository(repoPath, repoName, options = {}) {
+    if (!this.isAvailable()) {
+      return {
+        success: false,
+        message: 'ChromaDB not enabled. Run "stackmemory init --chromadb" to enable semantic search features.'
+      };
+    }
+    const startTime = Date.now();
+    try {
+      this.logger.info(`Starting repository ingestion for ${repoName}`);
+      if (!fs.existsSync(repoPath)) {
+        throw new Error(`Repository path not found: ${repoPath}`);
+      }
+      const metadata = await this.getRepoMetadata(repoPath, repoName);
+      const existingMetadata = this.metadataCache.get(metadata.repoId);
+      if (options.incremental && existingMetadata && !options.forceUpdate) {
+        const changedFiles = await this.getChangedFiles(
+          repoPath,
+          existingMetadata.lastCommit,
+          metadata.lastCommit
+        );
+        if (changedFiles.length === 0) {
+          return {
+            success: true,
+            message: "No changes detected since last ingestion"
+          };
+        }
+        this.logger.info(
+          `Incremental update: ${changedFiles.length} files changed`
+        );
+      }
+      const files = await this.getRepoFiles(repoPath, options);
+      this.logger.info(`Found ${files.length} files to process`);
+      let filesProcessed = 0;
+      let chunksCreated = 0;
+      let totalSize = 0;
+      for (const file of files) {
+        try {
+          const chunks = await this.processFile(
+            file,
+            repoPath,
+            repoName,
+            metadata,
+            options
+          );
+          for (const chunk of chunks) {
+            await this.storeChunk(chunk, metadata);
+            chunksCreated++;
+          }
+          filesProcessed++;
+          totalSize += fs.statSync(file).size;
+          if (filesProcessed % 100 === 0) {
+            this.logger.info(
+              `Processed ${filesProcessed}/${files.length} files`
+            );
+          }
+        } catch (error) {
+          this.logger.warn(`Failed to process file ${file}:`, error);
+        }
+      }
+      metadata.filesCount = filesProcessed;
+      metadata.totalSize = totalSize;
+      metadata.lastIngested = Date.now();
+      await this.saveMetadata(metadata);
+      const timeElapsed = Date.now() - startTime;
+      this.logger.info(
+        `Repository ingestion complete: ${filesProcessed} files, ${chunksCreated} chunks in ${timeElapsed}ms`
+      );
+      return {
+        success: true,
+        message: `Successfully ingested ${repoName}`,
+        stats: {
+          filesProcessed,
+          chunksCreated,
+          timeElapsed,
+          totalSize
+        }
+      };
+    } catch (error) {
+      this.logger.error("Repository ingestion failed:", error);
+      return {
+        success: false,
+        message: `Failed to ingest repository: ${error instanceof Error ? error.message : "Unknown error"}`
+      };
+    }
+  }
+  /**
+   * Update an existing repository in ChromaDB
+   */
+  async updateRepository(repoPath, repoName, options = {}) {
+    const startTime = Date.now();
+    try {
+      const metadata = await this.getRepoMetadata(repoPath, repoName);
+      const existingMetadata = this.metadataCache.get(metadata.repoId);
+      if (!existingMetadata) {
+        return this.ingestRepository(repoPath, repoName, options);
+      }
+      const changedFiles = await this.getChangedFiles(
+        repoPath,
+        existingMetadata.lastCommit,
+        metadata.lastCommit
+      );
+      if (changedFiles.length === 0) {
+        return {
+          success: true,
+          message: "No changes detected",
+          stats: {
+            filesUpdated: 0,
+            filesAdded: 0,
+            filesRemoved: 0,
+            timeElapsed: Date.now() - startTime
+          }
+        };
+      }
+      let filesUpdated = 0;
+      let filesAdded = 0;
+      let filesRemoved = 0;
+      for (const change of changedFiles) {
+        const filePath = path.join(repoPath, change.path);
+        if (change.status === "deleted") {
+          await this.removeFileChunks(change.path, metadata.repoId);
+          filesRemoved++;
+        } else if (change.status === "added") {
+          const chunks = await this.processFile(
+            filePath,
+            repoPath,
+            repoName,
+            metadata,
+            options
+          );
+          for (const chunk of chunks) {
+            await this.storeChunk(chunk, metadata);
+          }
+          filesAdded++;
+        } else if (change.status === "modified") {
+          await this.removeFileChunks(change.path, metadata.repoId);
+          const chunks = await this.processFile(
+            filePath,
+            repoPath,
+            repoName,
+            metadata,
+            options
+          );
+          for (const chunk of chunks) {
+            await this.storeChunk(chunk, metadata);
+          }
+          filesUpdated++;
+        }
+      }
+      metadata.lastIngested = Date.now();
+      await this.saveMetadata(metadata);
+      const timeElapsed = Date.now() - startTime;
+      return {
+        success: true,
+        message: `Successfully updated ${repoName}`,
+        stats: {
+          filesUpdated,
+          filesAdded,
+          filesRemoved,
+          timeElapsed
+        }
+      };
+    } catch (error) {
+      this.logger.error("Repository update failed:", error);
+      return {
+        success: false,
+        message: `Failed to update repository: ${error instanceof Error ? error.message : "Unknown error"}`
+      };
+    }
+  }
+  /**
+   * Search code in ingested repositories
+   */
+  async searchCode(query, options) {
+    if (!this.isAvailable() || !this.adapter) {
+      this.logger.warn("ChromaDB not enabled. Code search unavailable.");
+      return [];
+    }
+    try {
+      const filters = {
+        type: ["code_chunk"]
+      };
+      if (options?.repoName) {
+        filters.repo_name = options.repoName;
+      }
+      if (options?.language) {
+        filters.language = options.language;
+      }
+      const results = await this.adapter.queryContexts(
+        query,
+        options?.limit || 20,
+        filters
+      );
+      return results.map((result) => ({
+        filePath: result.metadata.file_path,
+        content: result.content,
+        score: 1 - result.distance,
+        // Convert distance to similarity score
+        startLine: result.metadata.start_line,
+        endLine: result.metadata.end_line,
+        repoName: result.metadata.repo_name
+      }));
+    } catch (error) {
+      this.logger.error("Code search failed:", error);
+      return [];
+    }
+  }
+  /**
+   * Get repository metadata
+   */
+  async getRepoMetadata(repoPath, repoName) {
+    const branch = this.getCurrentBranch(repoPath);
+    const lastCommit = this.getLastCommit(repoPath);
+    const repoId = `${repoName}_${branch}`.replace(/[^a-zA-Z0-9_-]/g, "_");
+    const { language, framework } = await this.detectLanguageAndFramework(repoPath);
+    return {
+      repoId,
+      repoName,
+      branch,
+      lastCommit,
+      lastIngested: Date.now(),
+      filesCount: 0,
+      totalSize: 0,
+      language,
+      framework
+    };
+  }
+  /**
+   * Get current git branch
+   */
+  getCurrentBranch(repoPath) {
+    try {
+      return execSync("git rev-parse --abbrev-ref HEAD", {
+        cwd: repoPath,
+        encoding: "utf8"
+      }).trim();
+    } catch {
+      return "main";
+    }
+  }
+  /**
+   * Get last commit hash
+   */
+  getLastCommit(repoPath) {
+    try {
+      return execSync("git rev-parse HEAD", {
+        cwd: repoPath,
+        encoding: "utf8"
+      }).trim();
+    } catch {
+      return "unknown";
+    }
+  }
+  /**
+   * Get changed files between commits
+   */
+  async getChangedFiles(repoPath, fromCommit, toCommit) {
+    try {
+      const diff = execSync(
+        `git diff --name-status ${fromCommit}..${toCommit}`,
+        {
+          cwd: repoPath,
+          encoding: "utf8"
+        }
+      );
+      return diff.split("\n").filter((line) => line.trim()).map((line) => {
+        const [status, ...pathParts] = line.split("	");
+        return {
+          path: pathParts.join("	"),
+          status: status === "A" ? "added" : status === "D" ? "deleted" : "modified"
+        };
+      });
+    } catch {
+      return [];
+    }
+  }
+  /**
+   * Get repository files to process
+   */
+  async getRepoFiles(repoPath, options) {
+    const files = [];
+    const ig = ignore();
+    const gitignorePath = path.join(repoPath, ".gitignore");
+    if (fs.existsSync(gitignorePath)) {
+      ig.add(fs.readFileSync(gitignorePath, "utf8"));
+    }
+    const defaultExcludes = [
+      "node_modules",
+      ".git",
+      "dist",
+      "build",
+      "coverage",
+      ".env",
+      "*.log",
+      ...options.excludePatterns || []
+    ];
+    ig.add(defaultExcludes);
+    const extensions = options.extensions || [
+      ".ts",
+      ".tsx",
+      ".js",
+      ".jsx",
+      ".py",
+      ".java",
+      ".go",
+      ".rs",
+      ".c",
+      ".cpp",
+      ".h",
+      ".hpp",
+      ".cs",
+      ".rb",
+      ".php",
+      ".swift",
+      ".kt",
+      ".scala",
+      ".r",
+      ".m",
+      ".sql",
+      ".yaml",
+      ".yml",
+      ".json"
+    ];
+    if (options.includeDocs) {
+      extensions.push(".md", ".rst", ".txt");
+    }
+    const maxFileSize = options.maxFileSize || 1024 * 1024;
+    const walkDir = (dir, baseDir = repoPath) => {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        const relativePath = path.relative(baseDir, fullPath);
+        if (ig.ignores(relativePath)) {
+          continue;
+        }
+        if (entry.isDirectory()) {
+          walkDir(fullPath, baseDir);
+        } else if (entry.isFile()) {
+          const ext = path.extname(entry.name);
+          if (!extensions.includes(ext)) {
+            continue;
+          }
+          if (!options.includeTests && (entry.name.includes(".test.") || entry.name.includes(".spec.") || relativePath.includes("__tests__") || relativePath.includes("test/") || relativePath.includes("tests/"))) {
+            continue;
+          }
+          const stats = fs.statSync(fullPath);
+          if (stats.size > maxFileSize) {
+            this.logger.debug(`Skipping large file: ${relativePath}`);
+            continue;
+          }
+          files.push(fullPath);
+        }
+      }
+    };
+    walkDir(repoPath);
+    return files;
+  }
+  /**
+   * Process a file into chunks
+   */
+  async processFile(filePath, repoPath, repoName, metadata, options) {
+    const relativePath = path.relative(repoPath, filePath);
+    const content = fs.readFileSync(filePath, "utf8");
+    const lines = content.split("\n");
+    const language = this.detectFileLanguage(filePath);
+    const chunkSize = options.chunkSize || 100;
+    const chunks = [];
+    const fileHash = crypto.createHash("md5").update(content).digest("hex");
+    const cachedHash = this.fileHashCache.get(relativePath);
+    if (cachedHash === fileHash && !options.forceUpdate) {
+      return [];
+    }
+    this.fileHashCache.set(relativePath, fileHash);
+    for (let i = 0; i < lines.length; i += chunkSize) {
+      const chunkLines = lines.slice(i, Math.min(i + chunkSize, lines.length));
+      const chunkContent = chunkLines.join("\n");
+      if (chunkContent.trim().length === 0) {
+        continue;
+      }
+      const chunkId = `${metadata.repoId}_${relativePath}_${i}`;
+      const chunkHash = crypto.createHash("md5").update(chunkContent).digest("hex");
+      chunks.push({
+        id: chunkId,
+        filePath: relativePath,
+        content: chunkContent,
+        startLine: i + 1,
+        endLine: Math.min(i + chunkSize, lines.length),
+        hash: chunkHash,
+        language
+      });
+    }
+    return chunks;
+  }
+  /**
+   * Store a chunk in ChromaDB
+   */
+  async storeChunk(chunk, metadata) {
+    if (!this.adapter) {
+      throw new Error("ChromaDB adapter not available");
+    }
+    const documentContent = `File: ${chunk.filePath} (Lines ${chunk.startLine}-${chunk.endLine})
+Language: ${chunk.language}
+Repository: ${metadata.repoName}/${metadata.branch}
+
+${chunk.content}`;
+    if (!this.adapter) {
+      throw new Error("ChromaDB adapter not initialized");
+    }
+    await this.adapter.storeContext("observation", documentContent, {
+      type: "code_chunk",
+      repo_id: metadata.repoId,
+      repo_name: metadata.repoName,
+      branch: metadata.branch,
+      file_path: chunk.filePath,
+      start_line: chunk.startLine,
+      end_line: chunk.endLine,
+      language: chunk.language,
+      framework: metadata.framework,
+      chunk_hash: chunk.hash,
+      last_commit: metadata.lastCommit
+    });
+  }
+  /**
+   * Remove file chunks from ChromaDB
+   */
+  async removeFileChunks(filePath, repoId) {
+    this.logger.debug(
+      `Would remove chunks for file: ${filePath} from repo: ${repoId}`
+    );
+  }
+  /**
+   * Detect file language
+   */
+  detectFileLanguage(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    const languageMap = {
+      ".ts": "typescript",
+      ".tsx": "typescript",
+      ".js": "javascript",
+      ".jsx": "javascript",
+      ".py": "python",
+      ".java": "java",
+      ".go": "go",
+      ".rs": "rust",
+      ".c": "c",
+      ".cpp": "cpp",
+      ".cs": "csharp",
+      ".rb": "ruby",
+      ".php": "php",
+      ".swift": "swift",
+      ".kt": "kotlin",
+      ".scala": "scala",
+      ".r": "r",
+      ".sql": "sql",
+      ".yaml": "yaml",
+      ".yml": "yaml",
+      ".json": "json",
+      ".md": "markdown"
+    };
+    return languageMap[ext] || "unknown";
+  }
+  /**
+   * Detect language and framework
+   */
+  async detectLanguageAndFramework(repoPath) {
+    const packageJsonPath = path.join(repoPath, "package.json");
+    if (fs.existsSync(packageJsonPath)) {
+      try {
+        const packageJson = JSON.parse(
+          fs.readFileSync(packageJsonPath, "utf8")
+        );
+        const deps = {
+          ...packageJson.dependencies,
+          ...packageJson.devDependencies
+        };
+        let framework;
+        if (deps.react) framework = "react";
+        else if (deps.vue) framework = "vue";
+        else if (deps.angular) framework = "angular";
+        else if (deps.express) framework = "express";
+        else if (deps.next) framework = "nextjs";
+        else if (deps.svelte) framework = "svelte";
+        return {
+          language: deps.typescript ? "typescript" : "javascript",
+          framework
+        };
+      } catch {
+      }
+    }
+    if (fs.existsSync(path.join(repoPath, "requirements.txt")) || fs.existsSync(path.join(repoPath, "setup.py"))) {
+      return { language: "python" };
+    }
+    if (fs.existsSync(path.join(repoPath, "go.mod"))) {
+      return { language: "go" };
+    }
+    if (fs.existsSync(path.join(repoPath, "Cargo.toml"))) {
+      return { language: "rust" };
+    }
+    if (fs.existsSync(path.join(repoPath, "pom.xml")) || fs.existsSync(path.join(repoPath, "build.gradle"))) {
+      return { language: "java" };
+    }
+    return { language: "unknown" };
+  }
+  /**
+   * Load metadata cache
+   */
+  async loadMetadataCache() {
+    this.metadataCache.clear();
+  }
+  /**
+   * Save metadata
+   */
+  async saveMetadata(metadata) {
+    this.metadataCache.set(metadata.repoId, metadata);
+  }
+  /**
+   * Get repository statistics
+   */
+  async getRepoStats(repoName) {
+    const stats = {
+      totalRepos: this.metadataCache.size,
+      totalFiles: 0,
+      totalChunks: 0,
+      languages: {},
+      frameworks: {}
+    };
+    for (const metadata of this.metadataCache.values()) {
+      if (!repoName || metadata.repoName === repoName) {
+        stats.totalFiles += metadata.filesCount;
+        if (metadata.language) {
+          stats.languages[metadata.language] = (stats.languages[metadata.language] || 0) + 1;
+        }
+        if (metadata.framework) {
+          stats.frameworks[metadata.framework] = (stats.frameworks[metadata.framework] || 0) + 1;
+        }
+      }
+    }
+    return stats;
+  }
+}
+export {
+  RepoIngestionSkill
+};
+//# sourceMappingURL=repo-ingestion-skill.js.map