agentic-knowledge-mcp 0.0.1

package/packages/cli/dist/commands/create.js

@@ -0,0 +1,90 @@
+/**
+ * Create command - add new docsets using presets
+ */
+import { Command } from "commander";
+import chalk from "chalk";
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { ConfigManager } from "@codemcp/knowledge-core";
+export const createCommand = new Command("create")
+    .description("Create a new docset using presets")
+    .requiredOption("--preset <type>", "Preset type: git-repo or local-folder")
+    .requiredOption("--id <id>", "Unique docset ID")
+    .requiredOption("--name <name>", "Human-readable docset name")
+    .option("--description <desc>", "Docset description")
+    .option("--url <url>", "Git repository URL (required for git-repo preset)")
+    .option("--path <path>", "Local folder path (required for local-folder preset)")
+    .option("--branch <branch>", "Git branch (default: main)", "main")
+    .action(async (options) => {
+    try {
+        console.log(chalk.blue("🚀 Creating new docset..."));
+        const configManager = new ConfigManager();
+        const { config, configPath } = await configManager.loadConfig(process.cwd());
+        // Check if docset ID already exists
+        if (config.docsets.find(d => d.id === options.id)) {
+            throw new Error(`Docset with ID '${options.id}' already exists`);
+        }
+        let newDocset;
+        if (options.preset === "git-repo") {
+            newDocset = await createGitRepoDocset(options);
+        }
+        else if (options.preset === "local-folder") {
+            newDocset = await createLocalFolderDocset(options);
+        }
+        else {
+            throw new Error(`Unknown preset: ${options.preset}. Use 'git-repo' or 'local-folder'`);
+        }
+        // Add to config
+        config.docsets.push(newDocset);
+        await configManager.saveConfig(config, configPath);
+        console.log(chalk.green(`✅ Created docset '${options.id}' successfully`));
+        console.log(chalk.gray(`   Config saved to: ${configPath}`));
+    }
+    catch (error) {
+        console.error(chalk.red("❌ Error creating docset:"), error.message);
+        process.exit(1);
+    }
+});
+async function createGitRepoDocset(options) {
+    if (!options.url) {
+        throw new Error("--url is required for git-repo preset");
+    }
+    // Basic URL validation
+    if (!options.url.match(/^https?:\/\/.*\.git$|^git@.*\.git$/)) {
+        throw new Error("Invalid git URL format. Expected .git URL");
+    }
+    return {
+        id: options.id,
+        name: options.name,
+        description: options.description || `Git repository: ${options.url}`,
+        web_sources: [{
+                url: options.url,
+                type: "git_repo",
+                options: {
+                    branch: options.branch
+                }
+            }]
+    };
+}
+async function createLocalFolderDocset(options) {
+    if (!options.path) {
+        throw new Error("--path is required for local-folder preset");
+    }
+    // Validate path exists
+    const fullPath = path.resolve(options.path);
+    try {
+        const stat = await fs.stat(fullPath);
+        if (!stat.isDirectory()) {
+            throw new Error(`Path is not a directory: ${options.path}`);
+        }
+    }
+    catch (error) {
+        throw new Error(`Path does not exist: ${options.path}`);
+    }
+    return {
+        id: options.id,
+        name: options.name,
+        description: options.description || `Local documentation: ${options.path}`,
+        local_path: options.path
+    };
+}

package/packages/cli/dist/commands/init.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Initialize command - set up web sources for a docset using GitRepoLoader
+ */
+import { Command } from "commander";
+export declare const initCommand: Command;

package/packages/cli/dist/commands/init.js

@@ -0,0 +1,182 @@
+/**
+ * Initialize command - set up web sources for a docset using GitRepoLoader
+ */
+import { Command } from "commander";
+import chalk from "chalk";
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import {
+  ConfigManager,
+  calculateLocalPath,
+  ensureKnowledgeGitignoreSync,
+} from "@codemcp/knowledge-core";
+import {
+  GitRepoLoader,
+  WebSourceType,
+} from "@codemcp/knowledge-content-loader";
+export const initCommand = new Command("init")
+  .description("Initialize web sources for a docset from configuration")
+  .argument("<docset-id>", "ID of the docset to initialize")
+  .option("-c, --config <path>", "Path to configuration file")
+  .option("--force", "Force re-initialization even if already exists", false)
+  .action(async (docsetId, options) => {
+    console.log(chalk.blue("🚀 Agentic Knowledge Integration Test"));
+    try {
+      // Use ConfigManager for all config operations
+      const configManager = new ConfigManager();
+      const { config, configPath } = await configManager.loadConfig(
+        process.cwd(),
+      );
+      // Ensure .knowledge/.gitignore exists and contains docsets/ ignore rule
+      ensureKnowledgeGitignoreSync(configPath);
+      const docset = config.docsets.find((d) => d.id === docsetId);
+      if (!docset) {
+        throw new Error(
+          `Docset '${docsetId}' not found in configuration. Available: ${config.docsets.map((d) => d.id).join(", ")}`,
+        );
+      }
+      if (!docset.web_sources || docset.web_sources.length === 0) {
+        throw new Error(`Docset '${docsetId}' has no web sources configured`);
+      }
+      console.log(chalk.green(`✅ Found docset: ${docset.name}`));
+      console.log(chalk.gray(`📝 Description: ${docset.description}`));
+      console.log(chalk.gray(`🔗 Web sources: ${docset.web_sources.length}`));
+      // Calculate the local path for this docset
+      const localPath = calculateLocalPath(docset, configPath);
+      console.log(chalk.yellow(`\n📁 Target directory: ${localPath}`));
+      // Check if already exists
+      let existsAlready = false;
+      try {
+        const stat = await fs.stat(localPath);
+        if (stat.isDirectory()) {
+          existsAlready = true;
+        }
+      } catch {
+        // Directory doesn't exist, which is fine
+      }
+      if (existsAlready && !options.force) {
+        console.log(
+          chalk.yellow(
+            "⚠️  Directory already exists. Use --force to overwrite.",
+          ),
+        );
+        const files = await fs.readdir(localPath);
+        console.log(
+          chalk.gray(
+            `Existing files: ${files.slice(0, 5).join(", ")}${files.length > 5 ? "..." : ""}`,
+          ),
+        );
+        return;
+      }
+      // Create target directory
+      await fs.mkdir(localPath, { recursive: true });
+      let totalFiles = 0;
+      const allDiscoveredPaths = [];
+      // Process each web source
+      for (const [index, webSource] of docset.web_sources.entries()) {
+        console.log(
+          chalk.yellow(
+            `\n🔄 Loading source ${index + 1}/${docset.web_sources.length}: ${webSource.url}`,
+          ),
+        );
+        if (webSource.type === "git_repo") {
+          // Use GitRepoLoader for all Git operations (REQ-19)
+          const loader = new GitRepoLoader();
+          console.log(
+            chalk.gray(`  Using GitRepoLoader for smart content filtering`),
+          );
+          const webSourceConfig = {
+            url: webSource.url,
+            type: WebSourceType.GIT_REPO,
+            options: webSource.options || {},
+          };
+          // Validate configuration
+          const validation = loader.validateConfig(webSourceConfig);
+          if (validation !== true) {
+            throw new Error(
+              `Invalid Git repository configuration: ${validation}`,
+            );
+          }
+          // Load content using GitRepoLoader
+          const result = await loader.load(webSourceConfig, localPath);
+          if (!result.success) {
+            throw new Error(`Git repository loading failed: ${result.error}`);
+          }
+          // Collect discovered paths for config update
+          allDiscoveredPaths.push(...result.files);
+          totalFiles += result.files.length;
+          console.log(
+            chalk.green(
+              `    ✅ Copied ${result.files.length} files using smart filtering`,
+            ),
+          );
+          // Create source metadata
+          const metadata = {
+            source_url: webSource.url,
+            source_type: webSource.type,
+            downloaded_at: new Date().toISOString(),
+            files_count: result.files.length,
+            files: result.files,
+            docset_id: docsetId,
+            content_hash: result.contentHash,
+          };
+          await fs.writeFile(
+            path.join(localPath, `.agentic-source-${index}.json`),
+            JSON.stringify(metadata, null, 2),
+          );
+        } else {
+          console.log(
+            chalk.red(
+              `    ❌ Web source type '${webSource.type}' not yet supported`,
+            ),
+          );
+        }
+      }
+      // Create overall metadata
+      const overallMetadata = {
+        docset_id: docsetId,
+        docset_name: docset.name,
+        initialized_at: new Date().toISOString(),
+        total_files: totalFiles,
+        web_sources_count: docset.web_sources.length,
+      };
+      await fs.writeFile(
+        path.join(localPath, ".agentic-metadata.json"),
+        JSON.stringify(overallMetadata, null, 2),
+      );
+      // Update configuration with discovered paths (only if paths were discovered and force flag used)
+      if (allDiscoveredPaths.length > 0 && options.force) {
+        console.log(
+          chalk.yellow(`\n📝 Updating configuration with discovered paths...`),
+        );
+        try {
+          await configManager.updateDocsetPaths(docsetId, allDiscoveredPaths);
+          console.log(
+            chalk.green(
+              `    ✅ Updated config with ${allDiscoveredPaths.length} discovered paths`,
+            ),
+          );
+        } catch (configError) {
+          console.log(
+            chalk.yellow(
+              `    ⚠️  Could not update config: ${configError instanceof Error ? configError.message : String(configError)}`,
+            ),
+          );
+        }
+      }
+      console.log(
+        chalk.green(`\n🎉 Successfully initialized docset '${docsetId}'`),
+      );
+      console.log(chalk.gray(`📁 Location: ${localPath}`));
+      console.log(chalk.gray(`📄 Total files: ${totalFiles}`));
+      console.log(
+        chalk.gray(`🔗 Sources processed: ${docset.web_sources.length}`),
+      );
+    } catch (error) {
+      console.error(chalk.red("\n❌ Error:"));
+      console.error(
+        chalk.red(error instanceof Error ? error.message : String(error)),
+      );
+      process.exit(1);
+    }
+  });

package/packages/cli/dist/commands/refresh.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Refresh command - update web sources for docsets
+ */
+import { Command } from "commander";
+export declare const refreshCommand: Command;

package/packages/cli/dist/commands/refresh.js

@@ -0,0 +1,322 @@
+/**
+ * Refresh command - update web sources for docsets
+ */
+import { Command } from "commander";
+import chalk from "chalk";
+import ora from "ora";
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { execSync } from "node:child_process";
+import {
+  findConfigPathSync,
+  loadConfigSync,
+  calculateLocalPath,
+  ensureKnowledgeGitignoreSync,
+} from "@codemcp/knowledge-core";
+export const refreshCommand = new Command("refresh")
+  .description("Refresh web sources for docsets")
+  .argument(
+    "[docset-id]",
+    "ID of specific docset to refresh (refresh all if not specified)",
+  )
+  .option("-c, --config <path>", "Path to configuration file")
+  .option("-f, --force", "Force refresh even if content unchanged", false)
+  .action(async (docsetId, options) => {
+    console.log(chalk.blue("🔄 Agentic Knowledge Refresh"));
+    try {
+      // Find and load configuration
+      const configPath = options.config || findConfigPathSync(process.cwd());
+      if (!configPath) {
+        throw new Error(
+          "No configuration file found. Run this command from a directory with .knowledge/config.yaml",
+        );
+      }
+      console.log(chalk.gray(`📄 Loading config: ${configPath}`));
+      const config = loadConfigSync(configPath);
+      // Ensure .knowledge/.gitignore exists
+      ensureKnowledgeGitignoreSync(configPath);
+      // Determine which docsets to refresh
+      const docsetsToRefresh = docsetId
+        ? config.docsets.filter((d) => d.id === docsetId)
+        : config.docsets.filter(
+            (d) => d.web_sources && d.web_sources.length > 0,
+          );
+      if (docsetsToRefresh.length === 0) {
+        if (docsetId) {
+          throw new Error(
+            `Docset '${docsetId}' not found or has no web sources. Available docsets with web sources: ${
+              config.docsets
+                .filter((d) => d.web_sources && d.web_sources.length > 0)
+                .map((d) => d.id)
+                .join(", ") || "none"
+            }`,
+          );
+        } else {
+          console.log(chalk.yellow("⚠️  No docsets with web sources found."));
+          return;
+        }
+      }
+      console.log(
+        chalk.green(
+          `✅ Found ${docsetsToRefresh.length} docset(s) to refresh: ${docsetsToRefresh.map((d) => d.id).join(", ")}`,
+        ),
+      );
+      // Refresh each docset
+      for (const docset of docsetsToRefresh) {
+        await refreshDocset(docset, configPath, options.force);
+      }
+      console.log(chalk.green("\n🎉 All docsets refreshed successfully!"));
+    } catch (error) {
+      console.error(chalk.red("\n❌ Error:"));
+      console.error(
+        chalk.red(error instanceof Error ? error.message : String(error)),
+      );
+      process.exit(1);
+    }
+  });
+async function refreshDocset(docset, configPath, force) {
+  const spinner = ora(`Refreshing ${docset.id}...`).start();
+  try {
+    const localPath = calculateLocalPath(docset, configPath);
+    spinner.text = `Checking ${docset.id} metadata...`;
+    // Check if docset has been initialized
+    const metadataPath = path.join(localPath, ".agentic-metadata.json");
+    let metadata = null;
+    try {
+      const metadataContent = await fs.readFile(metadataPath, "utf8");
+      metadata = JSON.parse(metadataContent);
+    } catch {
+      spinner.warn(`${docset.id}: Not initialized, use 'init' command first`);
+      return;
+    }
+    // Check if forced or if we should check for updates
+    if (!force && metadata) {
+      const lastRefresh = metadata.last_refreshed || metadata.initialized_at;
+      const lastRefreshTime = new Date(lastRefresh);
+      const timeSinceRefresh = Date.now() - lastRefreshTime.getTime();
+      const hoursSince = timeSinceRefresh / (1000 * 60 * 60);
+      if (hoursSince < 1) {
+        spinner.succeed(
+          `${docset.id}: Recently refreshed (${Math.round(hoursSince * 60)} minutes ago), skipping`,
+        );
+        return;
+      }
+    }
+    spinner.text = `Refreshing ${docset.id} web sources...`;
+    // Create backup of current metadata
+    const backupPath = path.join(localPath, `.agentic-metadata.backup.json`);
+    await fs.copyFile(metadataPath, backupPath);
+    let totalFiles = 0;
+    const refreshedSources = [];
+    // Process each web source
+    for (const [index, webSource] of (docset.web_sources || []).entries()) {
+      spinner.text = `${docset.id}: Refreshing source ${index + 1}/${docset.web_sources.length}...`;
+      if (webSource.type === "git_repo") {
+        const sourceFiles = await refreshGitSource(
+          webSource,
+          localPath,
+          index,
+          docset.id,
+          force,
+        );
+        totalFiles += sourceFiles.files_count;
+        refreshedSources.push(sourceFiles);
+      } else {
+        console.log(
+          chalk.yellow(
+            `    ⚠️  Web source type '${webSource.type}' not yet supported, skipping`,
+          ),
+        );
+      }
+    }
+    // Update metadata
+    if (!metadata) {
+      throw new Error("Metadata is null - this should not happen");
+    }
+    const updatedMetadata = {
+      docset_id: metadata.docset_id,
+      docset_name: metadata.docset_name,
+      initialized_at: metadata.initialized_at,
+      last_refreshed: new Date().toISOString(),
+      total_files: totalFiles,
+      web_sources_count: docset.web_sources?.length || 0,
+    };
+    await fs.writeFile(metadataPath, JSON.stringify(updatedMetadata, null, 2));
+    // Remove backup if successful
+    await fs.unlink(backupPath);
+    spinner.succeed(
+      `${docset.id}: Refreshed successfully (${totalFiles} files from ${refreshedSources.length} sources)`,
+    );
+  } catch (error) {
+    spinner.fail(
+      `${docset.id}: Failed to refresh - ${error instanceof Error ? error.message : String(error)}`,
+    );
+    // Try to restore from backup
+    const backupPath = path.join(
+      calculateLocalPath(docset, configPath),
+      `.agentic-metadata.backup.json`,
+    );
+    try {
+      const metadataPath = path.join(
+        calculateLocalPath(docset, configPath),
+        ".agentic-metadata.json",
+      );
+      await fs.copyFile(backupPath, metadataPath);
+      await fs.unlink(backupPath);
+      console.log(chalk.gray(`    Restored metadata from backup`));
+    } catch {
+      // Backup restore failed, but don't throw
+    }
+    throw error;
+  }
+}
+async function refreshGitSource(webSource, localPath, index, docsetId, force) {
+  // Check existing source metadata
+  const sourceMetadataPath = path.join(
+    localPath,
+    `.agentic-source-${index}.json`,
+  );
+  let existingSourceMetadata = null;
+  try {
+    const content = await fs.readFile(sourceMetadataPath, "utf8");
+    existingSourceMetadata = JSON.parse(content);
+  } catch {
+    // No existing metadata, will do full refresh
+  }
+  // Create temp directory for cloning
+  const tempDir = path.join(localPath, ".tmp", `git-refresh-${Date.now()}`);
+  await fs.mkdir(tempDir, { recursive: true });
+  try {
+    // Clone repository
+    const options = webSource.options || {};
+    const branch = options.branch || "main";
+    const paths = options.paths || [];
+    execSync(
+      `git clone --depth 1 --branch ${branch} ${webSource.url} ${tempDir}`,
+      {
+        stdio: "pipe",
+        timeout: 60000,
+      },
+    );
+    // Get latest commit hash for change detection
+    const latestCommit = execSync("git rev-parse HEAD", {
+      cwd: tempDir,
+      encoding: "utf8",
+    }).trim();
+    // Check if we need to update (compare with last known commit if available)
+    if (!force && existingSourceMetadata) {
+      const lastCommit = existingSourceMetadata.last_commit;
+      if (lastCommit === latestCommit) {
+        // No changes, update timestamp only
+        const updatedMetadata = {
+          ...existingSourceMetadata,
+          downloaded_at: new Date().toISOString(),
+        };
+        await fs.writeFile(
+          sourceMetadataPath,
+          JSON.stringify(updatedMetadata, null, 2),
+        );
+        return updatedMetadata;
+      }
+    }
+    // Remove old files from this source (if we have metadata)
+    if (existingSourceMetadata) {
+      for (const file of existingSourceMetadata.files) {
+        const filePath = path.join(localPath, file);
+        try {
+          await fs.unlink(filePath);
+        } catch {
+          // File might already be deleted, ignore
+        }
+      }
+    }
+    // Copy new files
+    const filesToCopy = [];
+    if (paths.length > 0) {
+      // Copy specified paths
+      for (const relPath of paths) {
+        const sourcePath = path.join(tempDir, relPath);
+        const targetPath = path.join(localPath, relPath);
+        try {
+          const stat = await fs.stat(sourcePath);
+          if (stat.isDirectory()) {
+            const dirFiles = await copyDirectory(sourcePath, targetPath);
+            filesToCopy.push(...dirFiles);
+          } else {
+            await fs.mkdir(path.dirname(targetPath), { recursive: true });
+            await fs.copyFile(sourcePath, targetPath);
+            filesToCopy.push(relPath);
+          }
+        } catch (error) {
+          console.log(
+            chalk.yellow(
+              `    ⚠️  Skipping ${relPath}: ${error instanceof Error ? error.message : String(error)}`,
+            ),
+          );
+        }
+      }
+    } else {
+      // Copy all markdown files
+      const allFiles = await findMarkdownFiles(tempDir);
+      for (const file of allFiles) {
+        const relativePath = path.relative(tempDir, file);
+        const targetPath = path.join(localPath, relativePath);
+        await fs.mkdir(path.dirname(targetPath), { recursive: true });
+        await fs.copyFile(file, targetPath);
+        filesToCopy.push(relativePath);
+      }
+    }
+    // Create updated source metadata
+    const metadata = {
+      source_url: webSource.url,
+      source_type: webSource.type,
+      downloaded_at: new Date().toISOString(),
+      files_count: filesToCopy.length,
+      files: filesToCopy,
+      docset_id: docsetId,
+      last_commit: latestCommit,
+    };
+    await fs.writeFile(sourceMetadataPath, JSON.stringify(metadata, null, 2));
+    return metadata;
+  } finally {
+    // Cleanup temp directory
+    await fs.rm(tempDir, { recursive: true, force: true });
+  }
+}
+// Reuse utility functions from init.ts
+async function findMarkdownFiles(dir) {
+  const files = [];
+  async function scan(currentDir) {
+    const items = await fs.readdir(currentDir);
+    for (const item of items) {
+      if (item.startsWith(".git")) continue;
+      const fullPath = path.join(currentDir, item);
+      const stat = await fs.stat(fullPath);
+      if (stat.isDirectory()) {
+        await scan(fullPath);
+      } else if (item.endsWith(".md") || item.endsWith(".mdx")) {
+        files.push(fullPath);
+      }
+    }
+  }
+  await scan(dir);
+  return files;
+}
+async function copyDirectory(source, target) {
+  const files = [];
+  await fs.mkdir(target, { recursive: true });
+  const items = await fs.readdir(source);
+  for (const item of items) {
+    const sourcePath = path.join(source, item);
+    const targetPath = path.join(target, item);
+    const stat = await fs.stat(sourcePath);
+    if (stat.isDirectory()) {
+      const subFiles = await copyDirectory(sourcePath, targetPath);
+      files.push(...subFiles.map((f) => path.join(item, f)));
+    } else {
+      await fs.copyFile(sourcePath, targetPath);
+      files.push(item);
+    }
+  }
+  return files;
+}

package/packages/cli/dist/commands/status.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Status command - show status of web sources
+ */
+import { Command } from "commander";
+export declare const statusCommand: Command;