agentic-knowledge-mcp 0.0.1

package/packages/core/dist/content/documentation-site-loader.js

@@ -0,0 +1,45 @@
+/**
+ * Documentation site content loader (STUB - not implemented yet)
+ */
+import { ContentLoader } from "./loader.js";
+import { WebSourceType, KnowledgeError, ErrorType } from "../types.js";
+/**
+ * Content loader for documentation websites (STUB IMPLEMENTATION)
+ */
+export class DocumentationSiteLoader extends ContentLoader {
+  /**
+   * Check if this loader can handle the given web source type
+   */
+  canHandle(webSource) {
+    return webSource.type === WebSourceType.DOCUMENTATION_SITE;
+  }
+  /**
+   * Validate the web source configuration
+   */
+  validateConfig(webSource) {
+    if (!webSource.url) {
+      return "Documentation site URL is required";
+    }
+    return true;
+  }
+  /**
+   * Load content from a documentation site (NOT IMPLEMENTED)
+   */
+  async load(webSource, targetPath) {
+    throw new KnowledgeError(
+      ErrorType.NOT_IMPLEMENTED,
+      "Documentation site loading is not yet implemented. Use git_repo type for repositories with documentation.",
+      { webSource: webSource.url, targetPath },
+    );
+  }
+  /**
+   * Get content identifier (NOT IMPLEMENTED)
+   */
+  async getContentId(webSource) {
+    throw new KnowledgeError(
+      ErrorType.NOT_IMPLEMENTED,
+      "Documentation site content ID generation is not yet implemented.",
+      { webSource: webSource.url },
+    );
+  }
+}

package/packages/core/dist/content/git-repo-loader.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Git repository content loader
+ */
+import { ContentLoader, type LoadResult } from "./loader.js";
+import { WebSourceConfig } from "../types.js";
+/**
+ * Content loader for Git repositories (GitHub, GitLab, any Git repo)
+ */
+export declare class GitRepoLoader extends ContentLoader {
+  /**
+   * Check if this loader can handle the given web source type
+   */
+  canHandle(webSource: WebSourceConfig): boolean;
+  /**
+   * Validate the web source configuration
+   */
+  validateConfig(webSource: WebSourceConfig): true | string;
+  /**
+   * Load content from a Git repository
+   */
+  load(webSource: WebSourceConfig, targetPath: string): Promise<LoadResult>;
+  /**
+   * Get content identifier for change detection
+   */
+  getContentId(webSource: WebSourceConfig): Promise<string>;
+  /**
+   * Validate if URL is a valid Git repository URL
+   */
+  private isValidGitUrl;
+  /**
+   * Create a temporary directory for cloning
+   */
+  private createTempDirectory;
+  /**
+   * Clone the Git repository
+   */
+  private cloneRepository;
+  /**
+   * Extract content from cloned repository to target directory
+   */
+  private extractContent;
+  /**
+   * Copy directory recursively
+   */
+  private copyDirectory;
+  /**
+   * Generate content hash for change detection
+   */
+  private generateContentHash;
+  /**
+   * Clean up temporary directory
+   */
+  private cleanupTempDirectory;
+}

package/packages/core/dist/content/git-repo-loader.js

@@ -0,0 +1,264 @@
+/**
+ * Git repository content loader
+ */
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { execSync } from "node:child_process";
+import { ContentLoader } from "./loader.js";
+import { WebSourceType, KnowledgeError, ErrorType } from "../types.js";
+import * as crypto from "node:crypto";
+/**
+ * Content loader for Git repositories (GitHub, GitLab, any Git repo)
+ */
+export class GitRepoLoader extends ContentLoader {
+  /**
+   * Check if this loader can handle the given web source type
+   */
+  canHandle(webSource) {
+    return webSource.type === WebSourceType.GIT_REPO;
+  }
+  /**
+   * Validate the web source configuration
+   */
+  validateConfig(webSource) {
+    if (!webSource.url) {
+      return "Git repository URL is required";
+    }
+    // Basic URL validation for Git repos
+    if (!this.isValidGitUrl(webSource.url)) {
+      return "Invalid Git repository URL";
+    }
+    return true;
+  }
+  /**
+   * Load content from a Git repository
+   */
+  async load(webSource, targetPath) {
+    try {
+      const options = webSource.options;
+      const tempDir = await this.createTempDirectory();
+      try {
+        // Clone the repository
+        await this.cloneRepository(webSource.url, tempDir, options);
+        // Extract specified paths or all content
+        const extractedFiles = await this.extractContent(
+          tempDir,
+          targetPath,
+          options?.paths,
+        );
+        // Generate content hash
+        const contentHash = await this.generateContentHash(
+          targetPath,
+          extractedFiles,
+        );
+        return {
+          success: true,
+          files: extractedFiles,
+          contentHash,
+        };
+      } finally {
+        // Clean up temp directory
+        await this.cleanupTempDirectory(tempDir);
+      }
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return {
+        success: false,
+        files: [],
+        contentHash: "",
+        error: `Git repository loading failed: ${errorMessage}`,
+      };
+    }
+  }
+  /**
+   * Get content identifier for change detection
+   */
+  async getContentId(webSource) {
+    try {
+      const options = webSource.options;
+      const branch = options?.branch || "HEAD";
+      // Get the latest commit hash from the remote repository
+      const command = `git ls-remote ${webSource.url} ${branch}`;
+      const output = execSync(command, { encoding: "utf8", timeout: 30000 });
+      const commitHash = output.trim().split("\t")[0];
+      // Combine with URL and paths for a unique identifier
+      const paths = options?.paths ? options.paths.sort().join(",") : "all";
+      return crypto
+        .createHash("sha256")
+        .update(`${webSource.url}:${commitHash}:${paths}`)
+        .digest("hex");
+    } catch (error) {
+      // Fallback to URL-based hash if remote access fails
+      const options = webSource.options;
+      const paths = options?.paths ? options.paths.sort().join(",") : "all";
+      return crypto
+        .createHash("sha256")
+        .update(`${webSource.url}:${paths}`)
+        .digest("hex");
+    }
+  }
+  /**
+   * Validate if URL is a valid Git repository URL
+   */
+  isValidGitUrl(url) {
+    const gitUrlPatterns = [
+      /^https:\/\/github\.com\/[\w\-._]+\/[\w\-._]+(?:\.git)?$/,
+      /^https:\/\/gitlab\.com\/[\w\-._\/]+(?:\.git)?$/,
+      /^https:\/\/[\w\-._]+\/[\w\-._\/]+\.git$/,
+      /^git@[\w\-._]+:[\w\-._\/]+\.git$/,
+    ];
+    return gitUrlPatterns.some((pattern) => pattern.test(url));
+  }
+  /**
+   * Create a temporary directory for cloning
+   */
+  async createTempDirectory() {
+    const tempDir = path.join(
+      process.cwd(),
+      ".tmp",
+      `git-clone-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+    );
+    await fs.mkdir(tempDir, { recursive: true });
+    return tempDir;
+  }
+  /**
+   * Clone the Git repository
+   */
+  async cloneRepository(url, targetDir, options) {
+    const branch = options?.branch || "main";
+    const depth = "--depth 1"; // Shallow clone for efficiency
+    let gitCommand = `git clone ${depth} --branch ${branch} ${url} ${targetDir}`;
+    // Add authentication if token is provided
+    if (options?.token) {
+      // For HTTPS URLs, inject token
+      if (url.startsWith("https://")) {
+        const urlWithToken = url.replace(
+          "https://",
+          `https://${options.token}@`,
+        );
+        gitCommand = `git clone ${depth} --branch ${branch} ${urlWithToken} ${targetDir}`;
+      }
+    }
+    try {
+      execSync(gitCommand, {
+        stdio: "pipe",
+        timeout: 120000, // 2 minutes timeout
+        cwd: process.cwd(),
+      });
+    } catch (error) {
+      // Try with master branch if main fails
+      if (branch === "main") {
+        const masterCommand = gitCommand.replace(
+          "--branch main",
+          "--branch master",
+        );
+        try {
+          execSync(masterCommand, {
+            stdio: "pipe",
+            timeout: 120000,
+            cwd: process.cwd(),
+          });
+          return;
+        } catch (masterError) {
+          // If both fail, throw the original error
+        }
+      }
+      throw new KnowledgeError(
+        ErrorType.GIT_REPO_ERROR,
+        `Failed to clone repository: ${error instanceof Error ? error.message : String(error)}`,
+        { url, branch, command: gitCommand },
+      );
+    }
+  }
+  /**
+   * Extract content from cloned repository to target directory
+   */
+  async extractContent(sourceDir, targetDir, paths) {
+    await fs.mkdir(targetDir, { recursive: true });
+    const extractedFiles = [];
+    if (paths && paths.length > 0) {
+      // Extract only specified paths
+      for (const relPath of paths) {
+        const sourcePath = path.join(sourceDir, relPath);
+        const targetPath = path.join(targetDir, relPath);
+        try {
+          const stats = await fs.stat(sourcePath);
+          if (stats.isDirectory()) {
+            await this.copyDirectory(sourcePath, targetPath, extractedFiles);
+          } else if (stats.isFile()) {
+            await fs.mkdir(path.dirname(targetPath), { recursive: true });
+            await fs.copyFile(sourcePath, targetPath);
+            extractedFiles.push(relPath);
+          }
+        } catch (error) {
+          // Skip files that don't exist or can't be accessed
+          console.warn(
+            `Warning: Could not extract ${relPath}: ${error instanceof Error ? error.message : String(error)}`,
+          );
+        }
+      }
+    } else {
+      // Extract all content (excluding .git directory)
+      await this.copyDirectory(sourceDir, targetDir, extractedFiles, [".git"]);
+    }
+    return extractedFiles;
+  }
+  /**
+   * Copy directory recursively
+   */
+  async copyDirectory(source, target, fileList, excludeDirs = []) {
+    await fs.mkdir(target, { recursive: true });
+    const items = await fs.readdir(source);
+    for (const item of items) {
+      if (excludeDirs.includes(item)) {
+        continue;
+      }
+      const sourcePath = path.join(source, item);
+      const targetPath = path.join(target, item);
+      const stats = await fs.stat(sourcePath);
+      if (stats.isDirectory()) {
+        await this.copyDirectory(sourcePath, targetPath, fileList, excludeDirs);
+      } else {
+        await fs.copyFile(sourcePath, targetPath);
+        // Calculate relative path from the initial target directory
+        const relativePath = path.relative(target, targetPath);
+        fileList.push(relativePath);
+      }
+    }
+  }
+  /**
+   * Generate content hash for change detection
+   */
+  async generateContentHash(targetDir, files) {
+    const hash = crypto.createHash("sha256");
+    // Sort files for consistent hashing
+    const sortedFiles = files.slice().sort();
+    for (const file of sortedFiles) {
+      const filePath = path.join(targetDir, file);
+      try {
+        const content = await fs.readFile(filePath);
+        hash.update(file); // Include filename
+        hash.update(content); // Include content
+      } catch (error) {
+        // Skip files that can't be read
+        console.warn(
+          `Warning: Could not hash ${file}: ${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+    }
+    return hash.digest("hex");
+  }
+  /**
+   * Clean up temporary directory
+   */
+  async cleanupTempDirectory(tempDir) {
+    try {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    } catch (error) {
+      console.warn(
+        `Warning: Could not clean up temp directory ${tempDir}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}

package/packages/core/dist/content/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Content loading and processing exports
+ */
+export { ContentLoader } from "./loader.js";
+export { GitRepoLoader } from "./git-repo-loader.js";
+export { DocumentationSiteLoader } from "./documentation-site-loader.js";
+export { ApiDocumentationLoader } from "./api-documentation-loader.js";
+export { ContentProcessor } from "./content-processor.js";
+export { MetadataManager } from "./metadata-manager.js";

package/packages/core/dist/content/index.js

@@ -0,0 +1,9 @@
+/**
+ * Content loading and processing exports
+ */
+export { ContentLoader } from "./loader.js";
+export { GitRepoLoader } from "./git-repo-loader.js";
+export { DocumentationSiteLoader } from "./documentation-site-loader.js";
+export { ApiDocumentationLoader } from "./api-documentation-loader.js";
+export { ContentProcessor } from "./content-processor.js";
+export { MetadataManager } from "./metadata-manager.js";

package/packages/core/dist/content/loader.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Abstract base class for content loaders
+ */
+import type { WebSourceConfig } from "../types.js";
+/**
+ * Result of a content loading operation
+ */
+export interface LoadResult {
+  /** Whether the operation was successful */
+  success: boolean;
+  /** Files that were created or updated */
+  files: string[];
+  /** Hash of the loaded content for change detection */
+  contentHash: string;
+  /** Error message if success is false */
+  error?: string;
+}
+/**
+ * Abstract base class for loading content from different web sources
+ */
+export declare abstract class ContentLoader {
+  /**
+   * Load content from a web source to the specified target directory
+   * @param webSource - Configuration for the web source
+   * @param targetPath - Local directory to store the content
+   * @returns Promise with load result
+   */
+  abstract load(
+    webSource: WebSourceConfig,
+    targetPath: string,
+  ): Promise<LoadResult>;
+  /**
+   * Check if this loader can handle the given web source type
+   * @param webSource - Web source configuration
+   * @returns True if this loader can handle the source type
+   */
+  abstract canHandle(webSource: WebSourceConfig): boolean;
+  /**
+   * Validate the web source configuration for this loader
+   * @param webSource - Web source configuration
+   * @returns True if configuration is valid, error message if not
+   */
+  abstract validateConfig(webSource: WebSourceConfig): true | string;
+  /**
+   * Get a unique identifier for the content (used for change detection)
+   * @param webSource - Web source configuration
+   * @returns Promise with content identifier
+   */
+  abstract getContentId(webSource: WebSourceConfig): Promise<string>;
+}

package/packages/core/dist/content/loader.js

@@ -0,0 +1,7 @@
+/**
+ * Abstract base class for content loaders
+ */
+/**
+ * Abstract base class for loading content from different web sources
+ */
+export class ContentLoader {}

package/packages/core/dist/content/metadata-manager.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Metadata manager for tracking web source downloads
+ */
+import { DocsetMetadata, WebSourceMetadata } from "../types.js";
+/**
+ * Manager for docset metadata files
+ */
+export declare class MetadataManager {
+  /**
+   * Load metadata from a docset directory
+   * @param docsetPath - Path to the docset directory
+   * @returns Metadata object or null if not found
+   */
+  loadMetadata(docsetPath: string): Promise<DocsetMetadata | null>;
+  /**
+   * Save metadata to a docset directory
+   * @param docsetPath - Path to the docset directory
+   * @param metadata - Metadata to save
+   */
+  saveMetadata(docsetPath: string, metadata: DocsetMetadata): Promise<void>;
+  /**
+   * Create initial metadata for a docset
+   * @param docsetId - ID of the docset
+   * @returns Initial metadata structure
+   */
+  createInitialMetadata(docsetId: string): DocsetMetadata;
+  /**
+   * Update metadata for a specific web source
+   * @param metadata - Current metadata
+   * @param sourceUrl - URL of the web source
+   * @param sourceMetadata - New metadata for the source
+   * @returns Updated metadata
+   */
+  updateSourceMetadata(
+    metadata: DocsetMetadata,
+    sourceUrl: string,
+    sourceMetadata: Partial<WebSourceMetadata>,
+  ): DocsetMetadata;
+  /**
+   * Get metadata for a specific web source
+   * @param metadata - Metadata to search
+   * @param sourceUrl - URL of the web source
+   * @returns Source metadata or null if not found
+   */
+  getSourceMetadata(
+    metadata: DocsetMetadata,
+    sourceUrl: string,
+  ): WebSourceMetadata | null;
+  /**
+   * Check if metadata file exists
+   * @param docsetPath - Path to the docset directory
+   * @returns True if metadata file exists
+   */
+  metadataExists(docsetPath: string): Promise<boolean>;
+  /**
+   * Remove metadata for a specific web source
+   * @param metadata - Current metadata
+   * @param sourceUrl - URL of the web source to remove
+   * @returns Updated metadata
+   */
+  removeSourceMetadata(
+    metadata: DocsetMetadata,
+    sourceUrl: string,
+  ): DocsetMetadata;
+}

package/packages/core/dist/content/metadata-manager.js

@@ -0,0 +1,160 @@
+/**
+ * Metadata manager for tracking web source downloads
+ */
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { METADATA_FILENAME } from "../types.js";
+/**
+ * Manager for docset metadata files
+ */
+export class MetadataManager {
+  /**
+   * Load metadata from a docset directory
+   * @param docsetPath - Path to the docset directory
+   * @returns Metadata object or null if not found
+   */
+  async loadMetadata(docsetPath) {
+    const metadataPath = path.join(docsetPath, METADATA_FILENAME);
+    try {
+      const content = await fs.readFile(metadataPath, "utf-8");
+      return JSON.parse(content);
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return null; // File doesn't exist
+      }
+      throw error; // Other errors should be propagated
+    }
+  }
+  /**
+   * Save metadata to a docset directory
+   * @param docsetPath - Path to the docset directory
+   * @param metadata - Metadata to save
+   */
+  async saveMetadata(docsetPath, metadata) {
+    await fs.mkdir(docsetPath, { recursive: true });
+    const metadataPath = path.join(docsetPath, METADATA_FILENAME);
+    const content = JSON.stringify(metadata, null, 2);
+    await fs.writeFile(metadataPath, content, "utf-8");
+  }
+  /**
+   * Create initial metadata for a docset
+   * @param docsetId - ID of the docset
+   * @returns Initial metadata structure
+   */
+  createInitialMetadata(docsetId) {
+    return {
+      version: "1.0",
+      docset_id: docsetId,
+      sources: [],
+      last_refresh: new Date().toISOString(),
+    };
+  }
+  /**
+   * Update metadata for a specific web source
+   * @param metadata - Current metadata
+   * @param sourceUrl - URL of the web source
+   * @param sourceMetadata - New metadata for the source
+   * @returns Updated metadata
+   */
+  updateSourceMetadata(metadata, sourceUrl, sourceMetadata) {
+    const updatedMetadata = { ...metadata };
+    updatedMetadata.last_refresh = new Date().toISOString();
+    // Find existing source or create new one
+    const existingIndex = updatedMetadata.sources.findIndex(
+      (s) => s.url === sourceUrl,
+    );
+    if (existingIndex >= 0) {
+      // Update existing source - TypeScript knows it exists here
+      const existing = updatedMetadata.sources[existingIndex];
+      updatedMetadata.sources[existingIndex] = {
+        url: existing.url,
+        type: existing.type,
+        status: sourceMetadata.status ?? existing.status,
+      };
+      // Update optional fields if provided, preserve existing if not
+      const updated = updatedMetadata.sources[existingIndex];
+      if (sourceMetadata.last_fetched !== undefined) {
+        updated.last_fetched = sourceMetadata.last_fetched;
+      } else if (existing.last_fetched !== undefined) {
+        updated.last_fetched = existing.last_fetched;
+      }
+      if (sourceMetadata.content_hash !== undefined) {
+        updated.content_hash = sourceMetadata.content_hash;
+      } else if (existing.content_hash !== undefined) {
+        updated.content_hash = existing.content_hash;
+      }
+      if (sourceMetadata.files_created !== undefined) {
+        updated.files_created = sourceMetadata.files_created;
+      } else if (existing.files_created !== undefined) {
+        updated.files_created = existing.files_created;
+      }
+      if (sourceMetadata.error_message !== undefined) {
+        updated.error_message = sourceMetadata.error_message;
+      } else if (existing.error_message !== undefined) {
+        updated.error_message = existing.error_message;
+      }
+    } else {
+      // Add new source - ensure required fields are present
+      if (!sourceMetadata.type) {
+        throw new Error("Source type is required for new metadata entries");
+      }
+      const newSource = {
+        url: sourceUrl,
+        type: sourceMetadata.type,
+        status: sourceMetadata.status || "pending",
+      };
+      // Add optional fields if provided
+      if (sourceMetadata.last_fetched !== undefined) {
+        newSource.last_fetched = sourceMetadata.last_fetched;
+      }
+      if (sourceMetadata.content_hash !== undefined) {
+        newSource.content_hash = sourceMetadata.content_hash;
+      }
+      if (sourceMetadata.files_created !== undefined) {
+        newSource.files_created = sourceMetadata.files_created;
+      }
+      if (sourceMetadata.error_message !== undefined) {
+        newSource.error_message = sourceMetadata.error_message;
+      }
+      updatedMetadata.sources.push(newSource);
+    }
+    return updatedMetadata;
+  }
+  /**
+   * Get metadata for a specific web source
+   * @param metadata - Metadata to search
+   * @param sourceUrl - URL of the web source
+   * @returns Source metadata or null if not found
+   */
+  getSourceMetadata(metadata, sourceUrl) {
+    return metadata.sources.find((s) => s.url === sourceUrl) || null;
+  }
+  /**
+   * Check if metadata file exists
+   * @param docsetPath - Path to the docset directory
+   * @returns True if metadata file exists
+   */
+  async metadataExists(docsetPath) {
+    const metadataPath = path.join(docsetPath, METADATA_FILENAME);
+    try {
+      await fs.access(metadataPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Remove metadata for a specific web source
+   * @param metadata - Current metadata
+   * @param sourceUrl - URL of the web source to remove
+   * @returns Updated metadata
+   */
+  removeSourceMetadata(metadata, sourceUrl) {
+    const updatedMetadata = { ...metadata };
+    updatedMetadata.last_refresh = new Date().toISOString();
+    updatedMetadata.sources = updatedMetadata.sources.filter(
+      (s) => s.url !== sourceUrl,
+    );
+    return updatedMetadata;
+  }
+}

package/packages/core/dist/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * @codemcp/knowledge-core
+ *
+ * Core functionality for the agentic knowledge guidance system.
+ * Provides configuration discovery, path calculation, and template processing.
+ */
+export * from "./types.js";
+export { findConfigPath, findConfigPathSync } from "./config/discovery.js";
+export { loadConfig, loadConfigSync, validateConfig } from "./config/loader.js";
+export { ConfigManager } from "./config/manager.js";
+export { calculateLocalPath, formatPath, validatePath, validatePathSync, getRelativePath, ensureKnowledgeGitignore, ensureKnowledgeGitignoreSync, } from "./paths/calculator.js";
+export { processTemplate, getEffectiveTemplate, validateTemplate, extractVariables, createTemplateContext, } from "./templates/processor.js";