agentic-knowledge-mcp 0.0.1

package/packages/content-loader/dist/content/metadata-manager.js

@@ -0,0 +1,160 @@
+/**
+ * Metadata manager for tracking web source downloads
+ */
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { METADATA_FILENAME } from "../types.js";
+/**
+ * Manager for docset metadata files
+ */
+export class MetadataManager {
+  /**
+   * Load metadata from a docset directory
+   * @param docsetPath - Path to the docset directory
+   * @returns Metadata object or null if not found
+   */
+  async loadMetadata(docsetPath) {
+    const metadataPath = path.join(docsetPath, METADATA_FILENAME);
+    try {
+      const content = await fs.readFile(metadataPath, "utf-8");
+      return JSON.parse(content);
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return null; // File doesn't exist
+      }
+      throw error; // Other errors should be propagated
+    }
+  }
+  /**
+   * Save metadata to a docset directory
+   * @param docsetPath - Path to the docset directory
+   * @param metadata - Metadata to save
+   */
+  async saveMetadata(docsetPath, metadata) {
+    await fs.mkdir(docsetPath, { recursive: true });
+    const metadataPath = path.join(docsetPath, METADATA_FILENAME);
+    const content = JSON.stringify(metadata, null, 2);
+    await fs.writeFile(metadataPath, content, "utf-8");
+  }
+  /**
+   * Create initial metadata for a docset
+   * @param docsetId - ID of the docset
+   * @returns Initial metadata structure
+   */
+  createInitialMetadata(docsetId) {
+    return {
+      version: "1.0",
+      docset_id: docsetId,
+      sources: [],
+      last_refresh: new Date().toISOString(),
+    };
+  }
+  /**
+   * Update metadata for a specific web source
+   * @param metadata - Current metadata
+   * @param sourceUrl - URL of the web source
+   * @param sourceMetadata - New metadata for the source
+   * @returns Updated metadata
+   */
+  updateSourceMetadata(metadata, sourceUrl, sourceMetadata) {
+    const updatedMetadata = { ...metadata };
+    updatedMetadata.last_refresh = new Date().toISOString();
+    // Find existing source or create new one
+    const existingIndex = updatedMetadata.sources.findIndex(
+      (s) => s.url === sourceUrl,
+    );
+    if (existingIndex >= 0) {
+      // Update existing source - TypeScript knows it exists here
+      const existing = updatedMetadata.sources[existingIndex];
+      updatedMetadata.sources[existingIndex] = {
+        url: existing.url,
+        type: existing.type,
+        status: sourceMetadata.status ?? existing.status,
+      };
+      // Update optional fields if provided, preserve existing if not
+      const updated = updatedMetadata.sources[existingIndex];
+      if (sourceMetadata.last_fetched !== undefined) {
+        updated.last_fetched = sourceMetadata.last_fetched;
+      } else if (existing.last_fetched !== undefined) {
+        updated.last_fetched = existing.last_fetched;
+      }
+      if (sourceMetadata.content_hash !== undefined) {
+        updated.content_hash = sourceMetadata.content_hash;
+      } else if (existing.content_hash !== undefined) {
+        updated.content_hash = existing.content_hash;
+      }
+      if (sourceMetadata.files_created !== undefined) {
+        updated.files_created = sourceMetadata.files_created;
+      } else if (existing.files_created !== undefined) {
+        updated.files_created = existing.files_created;
+      }
+      if (sourceMetadata.error_message !== undefined) {
+        updated.error_message = sourceMetadata.error_message;
+      } else if (existing.error_message !== undefined) {
+        updated.error_message = existing.error_message;
+      }
+    } else {
+      // Add new source - ensure required fields are present
+      if (!sourceMetadata.type) {
+        throw new Error("Source type is required for new metadata entries");
+      }
+      const newSource = {
+        url: sourceUrl,
+        type: sourceMetadata.type,
+        status: sourceMetadata.status || "pending",
+      };
+      // Add optional fields if provided
+      if (sourceMetadata.last_fetched !== undefined) {
+        newSource.last_fetched = sourceMetadata.last_fetched;
+      }
+      if (sourceMetadata.content_hash !== undefined) {
+        newSource.content_hash = sourceMetadata.content_hash;
+      }
+      if (sourceMetadata.files_created !== undefined) {
+        newSource.files_created = sourceMetadata.files_created;
+      }
+      if (sourceMetadata.error_message !== undefined) {
+        newSource.error_message = sourceMetadata.error_message;
+      }
+      updatedMetadata.sources.push(newSource);
+    }
+    return updatedMetadata;
+  }
+  /**
+   * Get metadata for a specific web source
+   * @param metadata - Metadata to search
+   * @param sourceUrl - URL of the web source
+   * @returns Source metadata or null if not found
+   */
+  getSourceMetadata(metadata, sourceUrl) {
+    return metadata.sources.find((s) => s.url === sourceUrl) || null;
+  }
+  /**
+   * Check if metadata file exists
+   * @param docsetPath - Path to the docset directory
+   * @returns True if metadata file exists
+   */
+  async metadataExists(docsetPath) {
+    const metadataPath = path.join(docsetPath, METADATA_FILENAME);
+    try {
+      await fs.access(metadataPath);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Remove metadata for a specific web source
+   * @param metadata - Current metadata
+   * @param sourceUrl - URL of the web source to remove
+   * @returns Updated metadata
+   */
+  removeSourceMetadata(metadata, sourceUrl) {
+    const updatedMetadata = { ...metadata };
+    updatedMetadata.last_refresh = new Date().toISOString();
+    updatedMetadata.sources = updatedMetadata.sources.filter(
+      (s) => s.url !== sourceUrl,
+    );
+    return updatedMetadata;
+  }
+}

package/packages/content-loader/dist/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Content loading and metadata management for agentic knowledge system
+ */
+export * from "./types.js";
+export * from "./content/index.js";

package/packages/content-loader/dist/index.js

@@ -0,0 +1,5 @@
+/**
+ * Content loading and metadata management for agentic knowledge system
+ */
+export * from "./types.js";
+export * from "./content/index.js";

package/packages/content-loader/dist/types.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Types and interfaces for web content loading and management
+ */
+/**
+ * Basic docset configuration (subset needed for web loading)
+ */
+export interface DocsetConfig {
+    /** Unique identifier for the docset */
+    id: string;
+    /** Human-readable name of the docset */
+    name: string;
+    /** Description of what this docset contains */
+    description?: string;
+    /** Path to the documentation (relative to .knowledge folder or absolute) */
+    local_path: string;
+    /** Optional custom instruction template for this docset */
+    template?: string;
+}
+/**
+ * Types of web sources supported
+ */
+export declare enum WebSourceType {
+    GIT_REPO = "git_repo",
+    DOCUMENTATION_SITE = "documentation_site",
+    API_DOCUMENTATION = "api_documentation"
+}
+/**
+ * Configuration for Git repository web sources
+ */
+export interface GitRepoOptions {
+    /** Specific paths to extract from the repository */
+    paths?: string[];
+    /** Branch or tag to checkout (defaults to main/master) */
+    branch?: string;
+    /** Optional authentication token for private repos */
+    token?: string;
+}
+/**
+ * Configuration for documentation site web sources (STUB - not implemented yet)
+ */
+export interface DocumentationSiteOptions {
+    /** Maximum depth to crawl */
+    max_depth?: number;
+    /** File patterns to include */
+    include_patterns?: string[];
+    /** File patterns to exclude */
+    exclude_patterns?: string[];
+}
+/**
+ * Configuration for API documentation web sources (STUB - not implemented yet)
+ */
+export interface ApiDocumentationOptions {
+    /** Format of the API documentation */
+    doc_format?: string;
+    /** Packages or modules to include */
+    include_packages?: string[];
+}
+/**
+ * Configuration for a single web source
+ */
+export interface WebSourceConfig {
+    /** URL of the web source */
+    url: string;
+    /** Type of web source */
+    type: WebSourceType;
+    /** Type-specific options */
+    options?: GitRepoOptions | DocumentationSiteOptions | ApiDocumentationOptions;
+}
+/**
+ * Metadata for a single web source download
+ */
+export interface WebSourceMetadata {
+    /** URL of the web source */
+    url: string;
+    /** Type of web source */
+    type: WebSourceType;
+    /** Timestamp of last successful fetch */
+    last_fetched?: string;
+    /** Hash of content for change detection */
+    content_hash?: string;
+    /** Files created during this fetch */
+    files_created?: string[];
+    /** Status of last fetch attempt */
+    status: "success" | "error" | "pending";
+    /** Error message if status is error */
+    error_message?: string;
+}
+/**
+ * Metadata file stored in each docset's local_path
+ */
+export interface DocsetMetadata {
+    /** Version of metadata format */
+    version: string;
+    /** Docset identifier */
+    docset_id: string;
+    /** Metadata for each web source */
+    sources: WebSourceMetadata[];
+    /** Timestamp of last refresh attempt */
+    last_refresh: string;
+}
+/**
+ * Metadata file name pattern
+ */
+export declare const METADATA_FILENAME = ".agentic-metadata.json";
+/**
+ * Web source specific error types
+ */
+export declare enum WebSourceErrorType {
+    WEB_SOURCE_ERROR = "WEB_SOURCE_ERROR",
+    GIT_REPO_ERROR = "GIT_REPO_ERROR",
+    NOT_IMPLEMENTED = "NOT_IMPLEMENTED"
+}
+/**
+ * Custom error class for web source errors
+ */
+export declare class WebSourceError extends Error {
+    type: WebSourceErrorType;
+    context?: Record<string, unknown> | undefined;
+    constructor(type: WebSourceErrorType, message: string, context?: Record<string, unknown> | undefined);
+}
+/**
+ * Extended docset config that includes web sources
+ */
+export interface DocsetConfigWithWebSources extends DocsetConfig {
+    /** Optional web sources to load content from */
+    web_sources?: WebSourceConfig[];
+}

package/packages/content-loader/dist/types.js

@@ -0,0 +1,48 @@
+/**
+ * Types and interfaces for web content loading and management
+ */
+/**
+ * Types of web sources supported
+ */
+export var WebSourceType;
+(function (WebSourceType) {
+    // eslint-disable-next-line no-unused-vars
+    WebSourceType["GIT_REPO"] = "git_repo";
+    // eslint-disable-next-line no-unused-vars
+    WebSourceType["DOCUMENTATION_SITE"] = "documentation_site";
+    // eslint-disable-next-line no-unused-vars
+    WebSourceType["API_DOCUMENTATION"] = "api_documentation";
+})(WebSourceType || (WebSourceType = {}));
+/**
+ * Metadata file name pattern
+ */
+export const METADATA_FILENAME = ".agentic-metadata.json";
+/**
+ * Web source specific error types
+ */
+export var WebSourceErrorType;
+(function (WebSourceErrorType) {
+    // eslint-disable-next-line no-unused-vars
+    WebSourceErrorType["WEB_SOURCE_ERROR"] = "WEB_SOURCE_ERROR";
+    // eslint-disable-next-line no-unused-vars
+    WebSourceErrorType["GIT_REPO_ERROR"] = "GIT_REPO_ERROR";
+    // eslint-disable-next-line no-unused-vars
+    WebSourceErrorType["NOT_IMPLEMENTED"] = "NOT_IMPLEMENTED";
+})(WebSourceErrorType || (WebSourceErrorType = {}));
+/**
+ * Custom error class for web source errors
+ */
+export class WebSourceError extends Error {
+    type;
+    context;
+    constructor(
+    // eslint-disable-next-line no-unused-vars
+    type, message, 
+    // eslint-disable-next-line no-unused-vars
+    context) {
+        super(message);
+        this.type = type;
+        this.context = context;
+        this.name = "WebSourceError";
+    }
+}

package/packages/content-loader/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@codemcp/knowledge-content-loader",
+  "version": "0.0.1",
+  "description": "Web content loading and metadata management for agentic knowledge system",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "build:watch": "tsc -p tsconfig.build.json --watch",
+    "clean": "rimraf dist",
+    "dev": "tsc -p tsconfig.build.json --watch",
+    "lint": "oxlint && eslint .",
+    "lint:fix": "oxlint --fix && eslint . --fix",
+    "format:check": "prettier --check .",
+    "format:fix": "prettier --write .",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "simple-git": "^3.22.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.3.0",
+    "rimraf": "^6.0.1",
+    "typescript": "^5.9.2",
+    "vitest": "^3.2.4"
+  },
+  "keywords": [
+    "agentic",
+    "knowledge",
+    "content-loading",
+    "web-sources",
+    "git",
+    "metadata"
+  ],
+  "author": "Oliver Jägle <github@beimir.net>",
+  "license": "MIT"
+}

package/packages/core/dist/config/discovery.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Configuration discovery functionality
+ */
+/**
+ * Find the configuration path by walking up the directory tree
+ * @param startPath - Starting directory path (defaults to current working directory)
+ * @returns Path to config file or null if not found
+ */
+export declare function findConfigPath(startPath?: string): Promise<string | null>;
+/**
+ * Synchronous version of findConfigPath for cases where async is not suitable
+ * @param startPath - Starting directory path (defaults to current working directory)
+ * @returns Path to config file or null if not found
+ */
+export declare function findConfigPathSync(startPath?: string): string | null;

package/packages/core/dist/config/discovery.js

@@ -0,0 +1,65 @@
+/**
+ * Configuration discovery functionality
+ */
+import { promises as fs } from "node:fs";
+import * as fsSync from "node:fs";
+import { resolve, dirname, join } from "node:path";
+import { CONFIG_DIR, CONFIG_FILENAME } from "../types.js";
+/**
+ * Find the configuration path by walking up the directory tree
+ * @param startPath - Starting directory path (defaults to current working directory)
+ * @returns Path to config file or null if not found
+ */
+export async function findConfigPath(startPath = process.cwd()) {
+    let currentDir = resolve(startPath);
+    while (true) {
+        const configDir = join(currentDir, CONFIG_DIR);
+        const configPath = join(configDir, CONFIG_FILENAME);
+        try {
+            const stats = await fs.stat(configPath);
+            if (stats.isFile()) {
+                return configPath;
+            }
+        }
+        catch {
+            // File doesn't exist, continue searching
+        }
+        // Move up one directory
+        const parentDir = dirname(currentDir);
+        if (parentDir === currentDir) {
+            // Reached filesystem root
+            break;
+        }
+        currentDir = parentDir;
+    }
+    return null;
+}
+/**
+ * Synchronous version of findConfigPath for cases where async is not suitable
+ * @param startPath - Starting directory path (defaults to current working directory)
+ * @returns Path to config file or null if not found
+ */
+export function findConfigPathSync(startPath = process.cwd()) {
+    let currentDir = resolve(startPath);
+    while (true) {
+        const configDir = join(currentDir, CONFIG_DIR);
+        const configPath = join(configDir, CONFIG_FILENAME);
+        try {
+            const stats = fsSync.statSync(configPath);
+            if (stats.isFile()) {
+                return configPath;
+            }
+        }
+        catch {
+            // File doesn't exist, continue searching
+        }
+        // Move up one directory
+        const parentDir = dirname(currentDir);
+        if (parentDir === currentDir) {
+            // Reached filesystem root
+            break;
+        }
+        currentDir = parentDir;
+    }
+    return null;
+}

package/packages/core/dist/config/loader.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Configuration loading and validation
+ */
+import type { KnowledgeConfig } from "../types.js";
+/**
+ * Load configuration from a YAML file with strict template validation
+ * @param configPath - Path to the configuration file
+ * @returns Parsed configuration object
+ */
+export declare function loadConfig(configPath: string): Promise<KnowledgeConfig>;
+/**
+ * Synchronous version of loadConfig with strict template validation
+ * @param configPath - Path to the configuration file
+ * @returns Parsed configuration object
+ */
+export declare function loadConfigSync(configPath: string): KnowledgeConfig;
+/**
+ * Validate a configuration object
+ * @param config - Configuration to validate
+ * @returns True if valid, false if invalid
+ */
+export declare function validateConfig(config: unknown): config is KnowledgeConfig;