orquesta-cli 0.1.13 → 0.1.14

package/src/core/docs-manager.ts

@@ -1,678 +0,0 @@
-/**
- * Docs Manager
- *
- * llms.txt-based document download and management
- * - Parse llms.txt to extract document URLs
- * - Auto-categorize by category
- * - Parallel download (20 concurrent)
- * - Auto-retry (up to 3 times)
- */
-
-import * as fs from 'fs';
-import * as fsPromises from 'fs/promises';
-import * as path from 'path';
-import * as https from 'https';
-import * as http from 'http';
-import * as os from 'os';
-import { logger } from '../utils/logger.js';
-
-const DOCS_DIR = path.join(os.homedir(), '.local-cli', 'docs');
-const CONCURRENT_DOWNLOADS = 20;
-const MAX_RETRIES = 3;
-
-/**
- * Document source definition
- */
-export interface DocsSource {
-  id: string;
-  name: string;
-  description: string;
-  llmsTxtUrl: string;
-  baseUrl?: string;  // URL prefix (for agno)
-  targetDir: string;
-  categoryMapping: Record<string, string>;
-  urlFilter?: (url: string) => boolean;  // URL filtering (for adk)
-  urlConverter?: (url: string) => string;  // URL conversion (for adk raw URL conversion)
-  filenameGenerator?: (url: string, title: string) => string;  // Custom filename generation
-}
-
-/**
- * ADK category mapping
- */
-const ADK_CATEGORY_MAPPING: Record<string, string> = {
-  'docs/agents/': 'agents',
-  'docs/tools/': 'tools',
-  'docs/sessions/': 'sessions',
-  'docs/deploy/': 'deploy',
-  'docs/streaming/': 'streaming',
-  'docs/callbacks/': 'callbacks',
-  'docs/mcp/': 'mcp',
-  'docs/observability/': 'observability',
-  'docs/get-started/': 'get-started',
-  'docs/tutorials/': 'tutorials',
-  'docs/runtime/': 'runtime',
-  'docs/events/': 'events',
-  'docs/context/': 'context',
-  'docs/artifacts/': 'artifacts',
-  'docs/evaluate/': 'evaluate',
-  'docs/safety/': 'safety',
-  'docs/api-reference/': 'api-reference',
-  'docs/community': 'community',
-  'docs/contributing': 'community',
-};
-
-/**
- * AGNO category mapping
- */
-const AGNO_CATEGORY_MAPPING: Record<string, string> = {
-  'agent-os/': 'agent',
-  'concepts/agents/': 'agent',
-  'concepts/agents/memory.md': 'memory',
-  'agent-os/features/memories.md': 'memory',
-  'concepts/agents/sessions.md': 'memory',
-  'reference/memory/': 'memory',
-  'concepts/agents/knowledge.md': 'rag',
-  'agent-os/features/knowledge-management.md': 'rag',
-  'reference/knowledge/': 'rag',
-  'agent-os/mcp/': 'mcp',
-  'concepts/agents/tools.md': 'tools',
-  'reference/tools/': 'tools',
-  'concepts/db/': 'database',
-  'reference/storage/': 'database',
-  'reference/vector_db/': 'vector_db',
-  'reference/workflows/': 'workflows',
-  'concepts/workflows/': 'workflows',
-  'reference/models/': 'models',
-  'concepts/agents/storage.md': 'storage',
-  'reference/teams/': 'teams',
-  'concepts/agents/guardrails/': 'guardrails',
-  'templates/': 'templates',
-  'tutorials/': 'tutorials',
-};
-
-/**
- * Convert GitHub URL to raw content URL
- */
-function convertToRawUrl(url: string): string {
-  if (url.includes('github.com') && url.includes('/blob/')) {
-    return url
-      .replace('github.com', 'raw.githubusercontent.com')
-      .replace('/blob/', '/');
-  }
-  return url;
-}
-
-/**
- * Pre-defined document sources by development team
- */
-export const AVAILABLE_SOURCES: DocsSource[] = [
-  {
-    id: 'agno',
-    name: 'Agno Framework',
-    description: 'Agno AI Agent Framework Documentation',
-    llmsTxtUrl: 'https://docs.agno.com/llms.txt',
-    baseUrl: 'https://docs.agno.com',
-    targetDir: 'agent_framework/agno',
-    categoryMapping: AGNO_CATEGORY_MAPPING,
-  },
-  {
-    id: 'adk',
-    name: 'Google ADK',
-    description: 'Google Agent Development Kit Documentation',
-    llmsTxtUrl: 'https://raw.githubusercontent.com/google/adk-python/main/llms.txt',
-    targetDir: 'agent_framework/adk',
-    categoryMapping: ADK_CATEGORY_MAPPING,
-    urlFilter: (url: string) => url.includes('github.com/google/adk-docs') && url.endsWith('.md'),
-    urlConverter: convertToRawUrl,
-  },
-];
-
-/**
- * Document directory information
- */
-export interface DocsInfo {
-  path: string;
-  exists: boolean;
-  totalFiles: number;
-  totalSize: string;
-  installedSources: string[];
-}
-
-/**
- * Document entry
- */
-interface DocEntry {
-  title: string;
-  url: string;
-  rawUrl: string;
-  category: string;
-}
-
-/**
- * Download file via HTTP request
- */
-function downloadFile(url: string): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const client = url.startsWith('https') ? https : http;
-
-    client.get(url, (res) => {
-      if (res.statusCode === 301 || res.statusCode === 302) {
-        const location = res.headers.location;
-        if (!location) {
-          return reject(new Error(`Redirect without Location header: ${url}`));
-        }
-        return downloadFile(location).then(resolve).catch(reject);
-      }
-
-      if (res.statusCode !== 200) {
-        reject(new Error(`HTTP ${res.statusCode}: ${url}`));
-        return;
-      }
-
-      let data = '';
-      res.on('data', (chunk) => { data += chunk; });
-      res.on('end', () => { resolve(data); });
-    }).on('error', reject);
-  });
-}
-
-/**
- * Determine category from URL
- */
-function getCategory(url: string, categoryMapping: Record<string, string>): string {
-  const lowerUrl = url.toLowerCase();
-
-  // Path-based category mapping
-  for (const [pattern, category] of Object.entries(categoryMapping)) {
-    if (lowerUrl.includes(pattern.toLowerCase())) {
-      return category;
-    }
-  }
-
-  // Keyword-based classification
-  if (lowerUrl.includes('agent')) return 'agents';
-  if (lowerUrl.includes('tool')) return 'tools';
-  if (lowerUrl.includes('memory') || lowerUrl.includes('session')) return 'memory';
-  if (lowerUrl.includes('deploy')) return 'deploy';
-  if (lowerUrl.includes('stream')) return 'streaming';
-  if (lowerUrl.includes('tutorial')) return 'tutorials';
-  if (lowerUrl.includes('model')) return 'models';
-
-  return 'other';
-}
-
-/**
- * Generate filename
- */
-function getFilename(url: string, _title: string): string {
-  const parts = url.split('/');
-  const lastPart = parts[parts.length - 1] || 'index';
-
-  // Extract prefix from path (e.g., workflow-agents)
-  const docsIndex = parts.findIndex(part => part === 'docs');
-  if (docsIndex !== -1 && docsIndex < parts.length - 2) {
-    const subPath = parts.slice(docsIndex + 1, -1);
-    if (subPath.length > 1) {
-      const prefix = subPath.slice(1).join('_');
-      const baseFilename = lastPart.endsWith('.md')
-        ? lastPart.replace(/\.md$/, '')
-        : lastPart;
-
-      if (baseFilename === 'index') {
-        return `${prefix}.md`;
-      }
-      return `${prefix}_${baseFilename}.md`;
-    }
-  }
-
-  // Handle models/{model_name}/*
-  const modelsIndex = parts.findIndex(part => part === 'models');
-  if (modelsIndex !== -1 && modelsIndex < parts.length - 2) {
-    const modelName = parts[modelsIndex + 1];
-    if (modelName && modelName !== 'models') {
-      const baseFilename = lastPart.endsWith('.md')
-        ? lastPart.replace(/\.md$/, '')
-        : lastPart;
-      return `${modelName}_${baseFilename}.md`;
-    }
-  }
-
-  if (!lastPart.endsWith('.md')) {
-    return `${lastPart}.md`;
-  }
-
-  return lastPart;
-}
-
-/**
- * Parse llms.txt
- */
-async function parseLlmsTxt(content: string, source: DocsSource): Promise<DocEntry[]> {
-  const entries: DocEntry[] = [];
-  const lines = content.split('\n');
-
-  for (const line of lines) {
-    // Markdown link format: - [Title](URL) or [Title](URL)
-    const match = line.match(/^-?\s*\[([^\]]+)\]\(([^)]+)\)/);
-    if (match) {
-      const title = match[1]!;
-      let url = match[2]!;
-
-      // URL filtering
-      if (source.urlFilter && !source.urlFilter(url)) {
-        continue;
-      }
-
-      // Apply baseUrl
-      if (source.baseUrl && !url.startsWith('http')) {
-        url = `${source.baseUrl}/${url.replace(/^\//, '')}`;
-      }
-
-      // URL conversion (raw URL, etc.)
-      const rawUrl = source.urlConverter ? source.urlConverter(url) : url;
-
-      // Determine category
-      const category = getCategory(url, source.categoryMapping);
-
-      entries.push({ title, url, rawUrl, category });
-    }
-  }
-
-  return entries;
-}
-
-/**
- * Query document directory information
- */
-export async function getDocsInfo(): Promise<DocsInfo> {
-  logger.enter('getDocsInfo');
-
-  try {
-    const exists = fs.existsSync(DOCS_DIR);
-
-    if (!exists) {
-      logger.exit('getDocsInfo', { exists: false });
-      return {
-        path: DOCS_DIR,
-        exists: false,
-        totalFiles: 0,
-        totalSize: '0 B',
-        installedSources: [],
-      };
-    }
-
-    let totalFiles = 0;
-    let totalBytes = 0;
-    const installedSources: string[] = [];
-
-    for (const source of AVAILABLE_SOURCES) {
-      const sourcePath = path.join(DOCS_DIR, source.targetDir);
-      if (fs.existsSync(sourcePath)) {
-        installedSources.push(source.id);
-      }
-    }
-
-    const scanDirectory = (dir: string) => {
-      if (!fs.existsSync(dir)) return;
-      const entries = fs.readdirSync(dir, { withFileTypes: true });
-      for (const entry of entries) {
-        const fullPath = path.join(dir, entry.name);
-        if (entry.isFile()) {
-          totalFiles++;
-          totalBytes += fs.statSync(fullPath).size;
-        } else if (entry.isDirectory()) {
-          scanDirectory(fullPath);
-        }
-      }
-    };
-
-    scanDirectory(DOCS_DIR);
-
-    const formatSize = (bytes: number): string => {
-      if (bytes < 1024) return `${bytes} B`;
-      if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
-      return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
-    };
-
-    const result = {
-      path: DOCS_DIR,
-      exists: true,
-      totalFiles,
-      totalSize: formatSize(totalBytes),
-      installedSources,
-    };
-
-    logger.exit('getDocsInfo', result);
-    return result;
-  } catch (error) {
-    logger.error('Failed to get docs info', error as Error);
-    return {
-      path: DOCS_DIR,
-      exists: false,
-      totalFiles: 0,
-      totalSize: '0 B',
-      installedSources: [],
-    };
-  }
-}
-
-/**
- * Return list of available document sources
- */
-export function getAvailableSources(): DocsSource[] {
-  return AVAILABLE_SOURCES;
-}
-
-/**
- * Document download result
- */
-export interface DownloadResult {
-  success: boolean;
-  message: string;
-  downloadedFiles?: number;
-  skippedFiles?: number;
-  failedFiles?: number;
-  targetPath?: string;
-}
-
-/**
- * Progress callback
- */
-export type ProgressCallback = (progress: {
-  current: number;
-  total: number;
-  status: 'success' | 'fail' | 'skip';
-  filename: string;
-  category: string;
-}) => void;
-
-/**
- * Download documents from predefined source
- */
-export async function downloadDocsFromSource(
-  sourceId: string,
-  onProgress?: ProgressCallback
-): Promise<DownloadResult> {
-  logger.enter('downloadDocsFromSource', { sourceId });
-
-  const source = AVAILABLE_SOURCES.find(s => s.id === sourceId);
-  if (!source) {
-    const availableIds = AVAILABLE_SOURCES.map(s => s.id).join(', ');
-    logger.exit('downloadDocsFromSource', { success: false, reason: 'unknown source' });
-    return {
-      success: false,
-      message: `Unknown source: ${sourceId}\nAvailable sources: ${availableIds}`,
-    };
-  }
-
-  try {
-    // 1. Download llms.txt
-    logger.flow('Downloading llms.txt');
-    const llmsContent = await downloadFile(source.llmsTxtUrl);
-
-    // 2. Parse document list
-    logger.flow('Parsing llms.txt');
-    const entries = await parseLlmsTxt(llmsContent, source);
-
-    if (entries.length === 0) {
-      return {
-        success: false,
-        message: 'No documents found in llms.txt.',
-      };
-    }
-
-    // 3. Create directory
-    const targetPath = path.join(DOCS_DIR, source.targetDir);
-    const categories = new Set(entries.map(e => e.category));
-
-    for (const category of categories) {
-      const dir = path.join(targetPath, category);
-      await fsPromises.mkdir(dir, { recursive: true });
-    }
-
-    // 4. Parallel download
-    let successCount = 0;
-    let failCount = 0;
-    let skipCount = 0;
-    let completedCount = 0;
-
-    const activeDownloads = new Set<Promise<void>>();
-
-    const downloadSingleFile = async (entry: DocEntry): Promise<void> => {
-      const filename = source.filenameGenerator
-        ? source.filenameGenerator(entry.url, entry.title)
-        : getFilename(entry.url, entry.title);
-      const filePath = path.join(targetPath, entry.category, filename);
-
-      try {
-        // Skip if file already exists
-        try {
-          await fsPromises.access(filePath);
-          skipCount++;
-          completedCount++;
-          onProgress?.({
-            current: completedCount,
-            total: entries.length,
-            status: 'skip',
-            filename,
-            category: entry.category,
-          });
-          return;
-        } catch {
-          // Proceed with download if file doesn't exist
-        }
-
-        // Retry logic
-        let retries = MAX_RETRIES;
-        let content: string | null = null;
-
-        while (retries > 0) {
-          try {
-            content = await downloadFile(entry.rawUrl);
-            break;
-          } catch {
-            retries--;
-            if (retries > 0) {
-              await new Promise(resolve => setTimeout(resolve, 500));
-            }
-          }
-        }
-
-        if (!content) {
-          throw new Error('Failed after retries');
-        }
-
-        // Add metadata
-        const markdownContent = `# ${entry.title}
-
-> Original: [${entry.title}](${entry.url})
-> Category: ${entry.category}
-> Downloaded: ${new Date().toISOString()}
-
----
-
-${content}`;
-
-        await fsPromises.writeFile(filePath, markdownContent, 'utf-8');
-        successCount++;
-        completedCount++;
-        onProgress?.({
-          current: completedCount,
-          total: entries.length,
-          status: 'success',
-          filename,
-          category: entry.category,
-        });
-      } catch {
-        failCount++;
-        completedCount++;
-        onProgress?.({
-          current: completedCount,
-          total: entries.length,
-          status: 'fail',
-          filename,
-          category: entry.category,
-        });
-      }
-    };
-
-    // Execute parallel download
-    let currentIndex = 0;
-
-    while (currentIndex < entries.length) {
-      while (activeDownloads.size < CONCURRENT_DOWNLOADS && currentIndex < entries.length) {
-        const entry = entries[currentIndex]!;
-        currentIndex++;
-
-        const downloadPromise = downloadSingleFile(entry).finally(() => {
-          activeDownloads.delete(downloadPromise);
-        });
-
-        activeDownloads.add(downloadPromise);
-      }
-
-      if (activeDownloads.size > 0) {
-        await Promise.race(activeDownloads);
-      }
-    }
-
-    await Promise.all(activeDownloads);
-
-    // 5. Generate README
-    await generateReadme(targetPath, entries, source, categories);
-
-    const result: DownloadResult = {
-      success: true,
-      message: `${source.name} documentation download complete`,
-      downloadedFiles: successCount,
-      skippedFiles: skipCount,
-      failedFiles: failCount,
-      targetPath,
-    };
-
-    logger.exit('downloadDocsFromSource', result);
-    return result;
-  } catch (error) {
-    const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-    logger.error('Download failed', error as Error);
-    return {
-      success: false,
-      message: `Download failed: ${errorMessage}`,
-    };
-  }
-}
-
-/**
- * Generate README file
- */
-async function generateReadme(
-  targetPath: string,
-  entries: DocEntry[],
-  source: DocsSource,
-  categories: Set<string>
-): Promise<void> {
-  // README per category
-  for (const category of categories) {
-    const categoryEntries = entries.filter(e => e.category === category);
-    const readmePath = path.join(targetPath, category, 'README.md');
-
-    const readmeContent = `# ${category.toUpperCase()} Documentation
-
-${source.name} ${category} related documentation.
-
-## Document List
-
-${categoryEntries.map((e, idx) => `${idx + 1}. [${e.title}](${getFilename(e.url, e.title)})`).join('\n')}
-
-Total ${categoryEntries.length} documents
-
----
-> Auto-generated: ${new Date().toISOString()}
-`;
-
-    await fsPromises.writeFile(readmePath, readmeContent, 'utf-8');
-  }
-
-  // Main README
-  const mainReadmePath = path.join(targetPath, 'README.md');
-  const mainReadmeContent = `# ${source.name} Documentation
-
-${source.name} documentation organized by category.
-
-## Categories
-
-${Array.from(categories).sort().map(category => {
-  const count = entries.filter(e => e.category === category).length;
-  return `- [${category}/](./${category}/) - ${count} documents`;
-}).join('\n')}
-
-## Statistics
-
-- Total documents: ${entries.length}
-- Categories: ${categories.size}
-
----
-> Auto-generated: ${new Date().toISOString()}
-`;
-
-  await fsPromises.writeFile(mainReadmePath, mainReadmeContent, 'utf-8');
-}
-
-/**
- * Query directory list (for DocsBrowser)
- */
-export interface DocsListItem {
-  name: string;
-  type: 'file' | 'directory';
-  size?: string;
-}
-
-export async function listDocsDirectory(subPath?: string): Promise<DocsListItem[]> {
-  logger.enter('listDocsDirectory', { subPath });
-
-  try {
-    const targetPath = subPath ? path.join(DOCS_DIR, subPath) : DOCS_DIR;
-
-    if (!fs.existsSync(targetPath)) {
-      logger.exit('listDocsDirectory', { exists: false });
-      return [];
-    }
-
-    const entries = fs.readdirSync(targetPath, { withFileTypes: true });
-    const items: DocsListItem[] = [];
-
-    for (const entry of entries) {
-      if (entry.isDirectory()) {
-        items.push({ name: entry.name, type: 'directory' });
-      } else {
-        const fullPath = path.join(targetPath, entry.name);
-        const stats = fs.statSync(fullPath);
-
-        const formatSize = (bytes: number): string => {
-          if (bytes < 1024) return `${bytes} B`;
-          if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
-          return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
-        };
-
-        items.push({
-          name: entry.name,
-          type: 'file',
-          size: formatSize(stats.size),
-        });
-      }
-    }
-
-    logger.exit('listDocsDirectory', { itemCount: items.length });
-    return items;
-  } catch (error) {
-    logger.error('Failed to list docs directory', error as Error);
-    return [];
-  }
-}
-
-/**
- * Return document path
- */
-export function getDocsPath(): string {
-  return DOCS_DIR;
-}

package/src/core/llm/index.ts

@@ -1,7 +0,0 @@
-/**
- * LLM module barrel export
- */
-
-export * from './llm-client.js';
-
-// Note: PlanningLLM is in src/agents/planner/