orquesta-cli 0.1.12

package/src/core/docs-manager.ts

@@ -0,0 +1,678 @@
+/**
+ * Docs Manager
+ *
+ * llms.txt-based document download and management
+ * - Parse llms.txt to extract document URLs
+ * - Auto-categorize by category
+ * - Parallel download (20 concurrent)
+ * - Auto-retry (up to 3 times)
+ */
+
+import * as fs from 'fs';
+import * as fsPromises from 'fs/promises';
+import * as path from 'path';
+import * as https from 'https';
+import * as http from 'http';
+import * as os from 'os';
+import { logger } from '../utils/logger.js';
+
+const DOCS_DIR = path.join(os.homedir(), '.local-cli', 'docs');
+const CONCURRENT_DOWNLOADS = 20;
+const MAX_RETRIES = 3;
+
+/**
+ * Document source definition
+ */
+export interface DocsSource {
+  id: string;
+  name: string;
+  description: string;
+  llmsTxtUrl: string;
+  baseUrl?: string;  // URL prefix (for agno)
+  targetDir: string;
+  categoryMapping: Record<string, string>;
+  urlFilter?: (url: string) => boolean;  // URL filtering (for adk)
+  urlConverter?: (url: string) => string;  // URL conversion (for adk raw URL conversion)
+  filenameGenerator?: (url: string, title: string) => string;  // Custom filename generation
+}
+
+/**
+ * ADK category mapping
+ */
+const ADK_CATEGORY_MAPPING: Record<string, string> = {
+  'docs/agents/': 'agents',
+  'docs/tools/': 'tools',
+  'docs/sessions/': 'sessions',
+  'docs/deploy/': 'deploy',
+  'docs/streaming/': 'streaming',
+  'docs/callbacks/': 'callbacks',
+  'docs/mcp/': 'mcp',
+  'docs/observability/': 'observability',
+  'docs/get-started/': 'get-started',
+  'docs/tutorials/': 'tutorials',
+  'docs/runtime/': 'runtime',
+  'docs/events/': 'events',
+  'docs/context/': 'context',
+  'docs/artifacts/': 'artifacts',
+  'docs/evaluate/': 'evaluate',
+  'docs/safety/': 'safety',
+  'docs/api-reference/': 'api-reference',
+  'docs/community': 'community',
+  'docs/contributing': 'community',
+};
+
+/**
+ * AGNO category mapping
+ */
+const AGNO_CATEGORY_MAPPING: Record<string, string> = {
+  'agent-os/': 'agent',
+  'concepts/agents/': 'agent',
+  'concepts/agents/memory.md': 'memory',
+  'agent-os/features/memories.md': 'memory',
+  'concepts/agents/sessions.md': 'memory',
+  'reference/memory/': 'memory',
+  'concepts/agents/knowledge.md': 'rag',
+  'agent-os/features/knowledge-management.md': 'rag',
+  'reference/knowledge/': 'rag',
+  'agent-os/mcp/': 'mcp',
+  'concepts/agents/tools.md': 'tools',
+  'reference/tools/': 'tools',
+  'concepts/db/': 'database',
+  'reference/storage/': 'database',
+  'reference/vector_db/': 'vector_db',
+  'reference/workflows/': 'workflows',
+  'concepts/workflows/': 'workflows',
+  'reference/models/': 'models',
+  'concepts/agents/storage.md': 'storage',
+  'reference/teams/': 'teams',
+  'concepts/agents/guardrails/': 'guardrails',
+  'templates/': 'templates',
+  'tutorials/': 'tutorials',
+};
+
+/**
+ * Convert GitHub URL to raw content URL
+ */
+function convertToRawUrl(url: string): string {
+  if (url.includes('github.com') && url.includes('/blob/')) {
+    return url
+      .replace('github.com', 'raw.githubusercontent.com')
+      .replace('/blob/', '/');
+  }
+  return url;
+}
+
+/**
+ * Pre-defined document sources by development team
+ */
+export const AVAILABLE_SOURCES: DocsSource[] = [
+  {
+    id: 'agno',
+    name: 'Agno Framework',
+    description: 'Agno AI Agent Framework Documentation',
+    llmsTxtUrl: 'https://docs.agno.com/llms.txt',
+    baseUrl: 'https://docs.agno.com',
+    targetDir: 'agent_framework/agno',
+    categoryMapping: AGNO_CATEGORY_MAPPING,
+  },
+  {
+    id: 'adk',
+    name: 'Google ADK',
+    description: 'Google Agent Development Kit Documentation',
+    llmsTxtUrl: 'https://raw.githubusercontent.com/google/adk-python/main/llms.txt',
+    targetDir: 'agent_framework/adk',
+    categoryMapping: ADK_CATEGORY_MAPPING,
+    urlFilter: (url: string) => url.includes('github.com/google/adk-docs') && url.endsWith('.md'),
+    urlConverter: convertToRawUrl,
+  },
+];
+
+/**
+ * Document directory information
+ */
+export interface DocsInfo {
+  path: string;
+  exists: boolean;
+  totalFiles: number;
+  totalSize: string;
+  installedSources: string[];
+}
+
+/**
+ * Document entry
+ */
+interface DocEntry {
+  title: string;
+  url: string;
+  rawUrl: string;
+  category: string;
+}
+
+/**
+ * Download file via HTTP request
+ */
+function downloadFile(url: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const client = url.startsWith('https') ? https : http;
+
+    client.get(url, (res) => {
+      if (res.statusCode === 301 || res.statusCode === 302) {
+        const location = res.headers.location;
+        if (!location) {
+          return reject(new Error(`Redirect without Location header: ${url}`));
+        }
+        return downloadFile(location).then(resolve).catch(reject);
+      }
+
+      if (res.statusCode !== 200) {
+        reject(new Error(`HTTP ${res.statusCode}: ${url}`));
+        return;
+      }
+
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => { resolve(data); });
+    }).on('error', reject);
+  });
+}
+
+/**
+ * Determine category from URL
+ */
+function getCategory(url: string, categoryMapping: Record<string, string>): string {
+  const lowerUrl = url.toLowerCase();
+
+  // Path-based category mapping
+  for (const [pattern, category] of Object.entries(categoryMapping)) {
+    if (lowerUrl.includes(pattern.toLowerCase())) {
+      return category;
+    }
+  }
+
+  // Keyword-based classification
+  if (lowerUrl.includes('agent')) return 'agents';
+  if (lowerUrl.includes('tool')) return 'tools';
+  if (lowerUrl.includes('memory') || lowerUrl.includes('session')) return 'memory';
+  if (lowerUrl.includes('deploy')) return 'deploy';
+  if (lowerUrl.includes('stream')) return 'streaming';
+  if (lowerUrl.includes('tutorial')) return 'tutorials';
+  if (lowerUrl.includes('model')) return 'models';
+
+  return 'other';
+}
+
+/**
+ * Generate filename
+ */
+function getFilename(url: string, _title: string): string {
+  const parts = url.split('/');
+  const lastPart = parts[parts.length - 1] || 'index';
+
+  // Extract prefix from path (e.g., workflow-agents)
+  const docsIndex = parts.findIndex(part => part === 'docs');
+  if (docsIndex !== -1 && docsIndex < parts.length - 2) {
+    const subPath = parts.slice(docsIndex + 1, -1);
+    if (subPath.length > 1) {
+      const prefix = subPath.slice(1).join('_');
+      const baseFilename = lastPart.endsWith('.md')
+        ? lastPart.replace(/\.md$/, '')
+        : lastPart;
+
+      if (baseFilename === 'index') {
+        return `${prefix}.md`;
+      }
+      return `${prefix}_${baseFilename}.md`;
+    }
+  }
+
+  // Handle models/{model_name}/*
+  const modelsIndex = parts.findIndex(part => part === 'models');
+  if (modelsIndex !== -1 && modelsIndex < parts.length - 2) {
+    const modelName = parts[modelsIndex + 1];
+    if (modelName && modelName !== 'models') {
+      const baseFilename = lastPart.endsWith('.md')
+        ? lastPart.replace(/\.md$/, '')
+        : lastPart;
+      return `${modelName}_${baseFilename}.md`;
+    }
+  }
+
+  if (!lastPart.endsWith('.md')) {
+    return `${lastPart}.md`;
+  }
+
+  return lastPart;
+}
+
+/**
+ * Parse llms.txt
+ */
+async function parseLlmsTxt(content: string, source: DocsSource): Promise<DocEntry[]> {
+  const entries: DocEntry[] = [];
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    // Markdown link format: - [Title](URL) or [Title](URL)
+    const match = line.match(/^-?\s*\[([^\]]+)\]\(([^)]+)\)/);
+    if (match) {
+      const title = match[1]!;
+      let url = match[2]!;
+
+      // URL filtering
+      if (source.urlFilter && !source.urlFilter(url)) {
+        continue;
+      }
+
+      // Apply baseUrl
+      if (source.baseUrl && !url.startsWith('http')) {
+        url = `${source.baseUrl}/${url.replace(/^\//, '')}`;
+      }
+
+      // URL conversion (raw URL, etc.)
+      const rawUrl = source.urlConverter ? source.urlConverter(url) : url;
+
+      // Determine category
+      const category = getCategory(url, source.categoryMapping);
+
+      entries.push({ title, url, rawUrl, category });
+    }
+  }
+
+  return entries;
+}
+
+/**
+ * Query document directory information
+ */
+export async function getDocsInfo(): Promise<DocsInfo> {
+  logger.enter('getDocsInfo');
+
+  try {
+    const exists = fs.existsSync(DOCS_DIR);
+
+    if (!exists) {
+      logger.exit('getDocsInfo', { exists: false });
+      return {
+        path: DOCS_DIR,
+        exists: false,
+        totalFiles: 0,
+        totalSize: '0 B',
+        installedSources: [],
+      };
+    }
+
+    let totalFiles = 0;
+    let totalBytes = 0;
+    const installedSources: string[] = [];
+
+    for (const source of AVAILABLE_SOURCES) {
+      const sourcePath = path.join(DOCS_DIR, source.targetDir);
+      if (fs.existsSync(sourcePath)) {
+        installedSources.push(source.id);
+      }
+    }
+
+    const scanDirectory = (dir: string) => {
+      if (!fs.existsSync(dir)) return;
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isFile()) {
+          totalFiles++;
+          totalBytes += fs.statSync(fullPath).size;
+        } else if (entry.isDirectory()) {
+          scanDirectory(fullPath);
+        }
+      }
+    };
+
+    scanDirectory(DOCS_DIR);
+
+    const formatSize = (bytes: number): string => {
+      if (bytes < 1024) return `${bytes} B`;
+      if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+      return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    };
+
+    const result = {
+      path: DOCS_DIR,
+      exists: true,
+      totalFiles,
+      totalSize: formatSize(totalBytes),
+      installedSources,
+    };
+
+    logger.exit('getDocsInfo', result);
+    return result;
+  } catch (error) {
+    logger.error('Failed to get docs info', error as Error);
+    return {
+      path: DOCS_DIR,
+      exists: false,
+      totalFiles: 0,
+      totalSize: '0 B',
+      installedSources: [],
+    };
+  }
+}
+
+/**
+ * Return list of available document sources
+ */
+export function getAvailableSources(): DocsSource[] {
+  return AVAILABLE_SOURCES;
+}
+
+/**
+ * Document download result
+ */
+export interface DownloadResult {
+  success: boolean;
+  message: string;
+  downloadedFiles?: number;
+  skippedFiles?: number;
+  failedFiles?: number;
+  targetPath?: string;
+}
+
+/**
+ * Progress callback
+ */
+export type ProgressCallback = (progress: {
+  current: number;
+  total: number;
+  status: 'success' | 'fail' | 'skip';
+  filename: string;
+  category: string;
+}) => void;
+
+/**
+ * Download documents from predefined source
+ */
+export async function downloadDocsFromSource(
+  sourceId: string,
+  onProgress?: ProgressCallback
+): Promise<DownloadResult> {
+  logger.enter('downloadDocsFromSource', { sourceId });
+
+  const source = AVAILABLE_SOURCES.find(s => s.id === sourceId);
+  if (!source) {
+    const availableIds = AVAILABLE_SOURCES.map(s => s.id).join(', ');
+    logger.exit('downloadDocsFromSource', { success: false, reason: 'unknown source' });
+    return {
+      success: false,
+      message: `Unknown source: ${sourceId}\nAvailable sources: ${availableIds}`,
+    };
+  }
+
+  try {
+    // 1. Download llms.txt
+    logger.flow('Downloading llms.txt');
+    const llmsContent = await downloadFile(source.llmsTxtUrl);
+
+    // 2. Parse document list
+    logger.flow('Parsing llms.txt');
+    const entries = await parseLlmsTxt(llmsContent, source);
+
+    if (entries.length === 0) {
+      return {
+        success: false,
+        message: 'No documents found in llms.txt.',
+      };
+    }
+
+    // 3. Create directory
+    const targetPath = path.join(DOCS_DIR, source.targetDir);
+    const categories = new Set(entries.map(e => e.category));
+
+    for (const category of categories) {
+      const dir = path.join(targetPath, category);
+      await fsPromises.mkdir(dir, { recursive: true });
+    }
+
+    // 4. Parallel download
+    let successCount = 0;
+    let failCount = 0;
+    let skipCount = 0;
+    let completedCount = 0;
+
+    const activeDownloads = new Set<Promise<void>>();
+
+    const downloadSingleFile = async (entry: DocEntry): Promise<void> => {
+      const filename = source.filenameGenerator
+        ? source.filenameGenerator(entry.url, entry.title)
+        : getFilename(entry.url, entry.title);
+      const filePath = path.join(targetPath, entry.category, filename);
+
+      try {
+        // Skip if file already exists
+        try {
+          await fsPromises.access(filePath);
+          skipCount++;
+          completedCount++;
+          onProgress?.({
+            current: completedCount,
+            total: entries.length,
+            status: 'skip',
+            filename,
+            category: entry.category,
+          });
+          return;
+        } catch {
+          // Proceed with download if file doesn't exist
+        }
+
+        // Retry logic
+        let retries = MAX_RETRIES;
+        let content: string | null = null;
+
+        while (retries > 0) {
+          try {
+            content = await downloadFile(entry.rawUrl);
+            break;
+          } catch {
+            retries--;
+            if (retries > 0) {
+              await new Promise(resolve => setTimeout(resolve, 500));
+            }
+          }
+        }
+
+        if (!content) {
+          throw new Error('Failed after retries');
+        }
+
+        // Add metadata
+        const markdownContent = `# ${entry.title}
+
+> Original: [${entry.title}](${entry.url})
+> Category: ${entry.category}
+> Downloaded: ${new Date().toISOString()}
+
+---
+
+${content}`;
+
+        await fsPromises.writeFile(filePath, markdownContent, 'utf-8');
+        successCount++;
+        completedCount++;
+        onProgress?.({
+          current: completedCount,
+          total: entries.length,
+          status: 'success',
+          filename,
+          category: entry.category,
+        });
+      } catch {
+        failCount++;
+        completedCount++;
+        onProgress?.({
+          current: completedCount,
+          total: entries.length,
+          status: 'fail',
+          filename,
+          category: entry.category,
+        });
+      }
+    };
+
+    // Execute parallel download
+    let currentIndex = 0;
+
+    while (currentIndex < entries.length) {
+      while (activeDownloads.size < CONCURRENT_DOWNLOADS && currentIndex < entries.length) {
+        const entry = entries[currentIndex]!;
+        currentIndex++;
+
+        const downloadPromise = downloadSingleFile(entry).finally(() => {
+          activeDownloads.delete(downloadPromise);
+        });
+
+        activeDownloads.add(downloadPromise);
+      }
+
+      if (activeDownloads.size > 0) {
+        await Promise.race(activeDownloads);
+      }
+    }
+
+    await Promise.all(activeDownloads);
+
+    // 5. Generate README
+    await generateReadme(targetPath, entries, source, categories);
+
+    const result: DownloadResult = {
+      success: true,
+      message: `${source.name} documentation download complete`,
+      downloadedFiles: successCount,
+      skippedFiles: skipCount,
+      failedFiles: failCount,
+      targetPath,
+    };
+
+    logger.exit('downloadDocsFromSource', result);
+    return result;
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+    logger.error('Download failed', error as Error);
+    return {
+      success: false,
+      message: `Download failed: ${errorMessage}`,
+    };
+  }
+}
+
+/**
+ * Generate README file
+ */
+async function generateReadme(
+  targetPath: string,
+  entries: DocEntry[],
+  source: DocsSource,
+  categories: Set<string>
+): Promise<void> {
+  // README per category
+  for (const category of categories) {
+    const categoryEntries = entries.filter(e => e.category === category);
+    const readmePath = path.join(targetPath, category, 'README.md');
+
+    const readmeContent = `# ${category.toUpperCase()} Documentation
+
+${source.name} ${category} related documentation.
+
+## Document List
+
+${categoryEntries.map((e, idx) => `${idx + 1}. [${e.title}](${getFilename(e.url, e.title)})`).join('\n')}
+
+Total ${categoryEntries.length} documents
+
+---
+> Auto-generated: ${new Date().toISOString()}
+`;
+
+    await fsPromises.writeFile(readmePath, readmeContent, 'utf-8');
+  }
+
+  // Main README
+  const mainReadmePath = path.join(targetPath, 'README.md');
+  const mainReadmeContent = `# ${source.name} Documentation
+
+${source.name} documentation organized by category.
+
+## Categories
+
+${Array.from(categories).sort().map(category => {
+  const count = entries.filter(e => e.category === category).length;
+  return `- [${category}/](./${category}/) - ${count} documents`;
+}).join('\n')}
+
+## Statistics
+
+- Total documents: ${entries.length}
+- Categories: ${categories.size}
+
+---
+> Auto-generated: ${new Date().toISOString()}
+`;
+
+  await fsPromises.writeFile(mainReadmePath, mainReadmeContent, 'utf-8');
+}
+
+/**
+ * Query directory list (for DocsBrowser)
+ */
+export interface DocsListItem {
+  name: string;
+  type: 'file' | 'directory';
+  size?: string;
+}
+
+export async function listDocsDirectory(subPath?: string): Promise<DocsListItem[]> {
+  logger.enter('listDocsDirectory', { subPath });
+
+  try {
+    const targetPath = subPath ? path.join(DOCS_DIR, subPath) : DOCS_DIR;
+
+    if (!fs.existsSync(targetPath)) {
+      logger.exit('listDocsDirectory', { exists: false });
+      return [];
+    }
+
+    const entries = fs.readdirSync(targetPath, { withFileTypes: true });
+    const items: DocsListItem[] = [];
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        items.push({ name: entry.name, type: 'directory' });
+      } else {
+        const fullPath = path.join(targetPath, entry.name);
+        const stats = fs.statSync(fullPath);
+
+        const formatSize = (bytes: number): string => {
+          if (bytes < 1024) return `${bytes} B`;
+          if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+          return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+        };
+
+        items.push({
+          name: entry.name,
+          type: 'file',
+          size: formatSize(stats.size),
+        });
+      }
+    }
+
+    logger.exit('listDocsDirectory', { itemCount: items.length });
+    return items;
+  } catch (error) {
+    logger.error('Failed to list docs directory', error as Error);
+    return [];
+  }
+}
+
+/**
+ * Return document path
+ */
+export function getDocsPath(): string {
+  return DOCS_DIR;
+}

package/src/core/llm/index.ts

@@ -0,0 +1,7 @@
+/**
+ * LLM module barrel export
+ */
+
+export * from './llm-client.js';
+
+// Note: PlanningLLM is in src/agents/planner/