@agenticmail/enterprise 0.5.326 → 0.5.328

package/src/engine/knowledge-import/processors/index.ts

@@ -1,18 +0,0 @@
-/**
- * Processing Pipeline — Barrel Export
- *
- * Files:
- *   types.ts             — Shared types for all layers
- *   extract-web.ts       — Layer 1: Web/URL extractor (Intercom, Zendesk, GitBook, etc.)
- *   extract-github.ts    — Layer 1: GitHub content extractor
- *   extract-sharepoint.ts— Layer 1: SharePoint/OneDrive extractor
- *   extract-gdrive.ts    — Layer 1: Google Drive/Sites extractor
- *   clean.ts             — Layer 2: 3-pass deep cleaning & normalization
- *   validate.ts          — Layer 3: Quality gates & scoring (6 checks)
- *   pipeline.ts          — Orchestrator: chains all 3 layers
- */
-
-export { processDocument } from './pipeline.js';
-export { cleanContent } from './clean.js';
-export { validateContent } from './validate.js';
-export type { ProcessedDocument, DocumentSection, QualityReport, QualityCheck, LayerReport } from './types.js';

package/src/engine/knowledge-import/processors/pipeline.ts

@@ -1,171 +0,0 @@
-/**
- * Processing Pipeline — Orchestrator
- *
- * Chains the three processing layers:
- *   Layer 1: EXTRACT — Source-specific content extraction
- *   Layer 2: CLEAN   — Deep cleaning & normalization (3 passes)
- *   Layer 3: VALIDATE — Quality gates & scoring
- *
- * Then splits into sections for chunking.
- */
-
-import type { ImportDocument } from '../types.js';
-import type { ProcessedDocument, DocumentSection, LayerReport, ContentExtractor } from './types.js';
-import { WebContentExtractor } from './extract-web.js';
-import { GitHubContentExtractor } from './extract-github.js';
-import { SharePointContentExtractor } from './extract-sharepoint.js';
-import { GoogleDriveContentExtractor } from './extract-gdrive.js';
-import { cleanContent } from './clean.js';
-import { validateContent } from './validate.js';
-
-// ─── Extractor Registry ──────────────────────────────
-
-const EXTRACTORS: Record<string, ContentExtractor> = {
-  'github': new GitHubContentExtractor(),
-  'sharepoint': new SharePointContentExtractor(),
-  'google-sites': new GoogleDriveContentExtractor(),
-  'url': new WebContentExtractor(),
-  'file-upload': new GitHubContentExtractor(),  // markdown/text files use same extractor
-  'confluence': new WebContentExtractor(),
-  'notion': new WebContentExtractor(),
-};
-
-// ─── Pipeline ────────────────────────────────────────
-
-export function processDocument(doc: ImportDocument): ProcessedDocument | null {
-  const pipelineStart = Date.now();
-  const layers: LayerReport[] = [];
-
-  // ─── Layer 1: EXTRACT ──────────────────────────
-
-  const extractStart = Date.now();
-  const extractor = EXTRACTORS[doc.sourceType] || new WebContentExtractor();
-
-  const extracted = extractor.extract(doc.content, doc.sourceUrl);
-  const extractOps: string[] = [`extractor: ${doc.sourceType}`, `contentType: ${extracted.contentType}`];
-
-  if (extracted.contentType === 'html' || doc.contentType === 'html') {
-    extractOps.push('converted HTML to text');
-  }
-
-  layers.push({
-    name: 'extract',
-    inputSize: doc.content.length,
-    outputSize: extracted.content.length,
-    removedBytes: doc.content.length - extracted.content.length,
-    operations: extractOps,
-    durationMs: Date.now() - extractStart,
-  });
-
-  // ─── Layer 2: CLEAN ───────────────────────────
-
-  const { content: cleaned, report: cleanReport } = cleanContent(extracted.content);
-  layers.push(cleanReport);
-
-  // ─── Layer 3: VALIDATE ────────────────────────
-
-  const validateStart = Date.now();
-  const quality = validateContent(cleaned, extracted.title || doc.title);
-
-  layers.push({
-    name: 'validate',
-    inputSize: cleaned.length,
-    outputSize: cleaned.length,
-    removedBytes: 0,
-    operations: quality.checks.map(c => `${c.name}: ${c.passed ? 'PASS' : 'FAIL'} (${c.score})`),
-    durationMs: Date.now() - validateStart,
-  });
-
-  // Reject if quality gate fails
-  if (!quality.passed) {
-    console.log(`[knowledge-import] Rejected "${extracted.title || doc.title}": score ${quality.score}/100, warnings: ${quality.warnings.join('; ')}`);
-    return null;
-  }
-
-  // ─── Split into sections ──────────────────────
-
-  const sections = splitIntoSections(cleaned);
-
-  const totalMs = Date.now() - pipelineStart;
-  const compressionRatio = doc.content.length > 0 ? 1 - (cleaned.length / doc.content.length) : 0;
-
-  return {
-    id: doc.id,
-    title: extracted.title || doc.title,
-    content: cleaned,
-    contentType: 'text',
-    sections,
-    metadata: {
-      sourceType: doc.sourceType,
-      sourcePath: doc.sourcePath,
-      sourceUrl: doc.sourceUrl,
-      author: extracted.author,
-      lastModified: doc.lastModified,
-      extractedTitle: extracted.title,
-      originalSize: doc.content.length,
-      cleanedSize: cleaned.length,
-      compressionRatio,
-      processingMs: totalMs,
-      layers,
-    },
-    quality,
-  };
-}
-
-// ─── Section Splitter ────────────────────────────────
-
-function splitIntoSections(content: string): DocumentSection[] {
-  const lines = content.split('\n');
-  const sections: DocumentSection[] = [];
-  let currentTitle = '';
-  let currentLevel = 0;
-  let currentLines: string[] = [];
-
-  for (const line of lines) {
-    const headingMatch = line.match(/^(#{1,6})\s+(.+)/);
-    if (headingMatch) {
-      // Flush current section
-      if (currentLines.length > 0) {
-        const text = currentLines.join('\n').trim();
-        if (text.length > 0) {
-          sections.push({
-            title: currentTitle || 'Introduction',
-            content: text,
-            level: currentLevel,
-            wordCount: text.split(/\s+/).length,
-          });
-        }
-      }
-      currentTitle = headingMatch[2].trim();
-      currentLevel = headingMatch[1].length;
-      currentLines = [];
-    } else {
-      currentLines.push(line);
-    }
-  }
-
-  // Flush last section
-  if (currentLines.length > 0) {
-    const text = currentLines.join('\n').trim();
-    if (text.length > 0) {
-      sections.push({
-        title: currentTitle || 'Content',
-        content: text,
-        level: currentLevel,
-        wordCount: text.split(/\s+/).length,
-      });
-    }
-  }
-
-  // If no sections found, treat entire content as one section
-  if (sections.length === 0 && content.trim().length > 0) {
-    sections.push({
-      title: 'Content',
-      content: content.trim(),
-      level: 0,
-      wordCount: content.trim().split(/\s+/).length,
-    });
-  }
-
-  return sections;
-}

package/src/engine/knowledge-import/processors/types.ts

@@ -1,78 +0,0 @@
-/**
- * Processing Pipeline — Types
- *
- * Three-layer processing:
- *   Layer 1: EXTRACT — Source-specific raw content extraction
- *   Layer 2: CLEAN   — Deep cleaning, normalization, deduplication
- *   Layer 3: VALIDATE — Quality gates, scoring, rejection
- */
-
-export interface ProcessedDocument {
-  id: string;
-  title: string;
-  content: string;           // fully cleaned content
-  contentType: 'markdown' | 'text';
-  sections: DocumentSection[];
-  metadata: DocumentMetadata;
-  quality: QualityReport;
-}
-
-export interface DocumentSection {
-  title: string;
-  content: string;
-  level: number;              // heading level (1-6, 0 = no heading)
-  wordCount: number;
-  language?: string;
-}
-
-export interface DocumentMetadata {
-  sourceType: string;
-  sourcePath: string;
-  sourceUrl?: string;
-  author?: string;
-  lastModified?: string;
-  extractedTitle: string;     // title as extracted (before cleaning)
-  originalSize: number;
-  cleanedSize: number;
-  compressionRatio: number;   // how much junk was removed
-  processingMs: number;
-  layers: LayerReport[];
-}
-
-export interface LayerReport {
-  name: string;
-  inputSize: number;
-  outputSize: number;
-  removedBytes: number;
-  operations: string[];       // what this layer did
-  durationMs: number;
-}
-
-export interface QualityReport {
-  score: number;              // 0-100
-  passed: boolean;
-  checks: QualityCheck[];
-  warnings: string[];
-}
-
-export interface QualityCheck {
-  name: string;
-  passed: boolean;
-  score: number;              // 0-100
-  detail?: string;
-}
-
-/** Source-specific extractor interface. */
-export interface ContentExtractor {
-  /** Extract clean text from raw source content. */
-  extract(raw: string, sourceUrl?: string): ExtractResult;
-}
-
-export interface ExtractResult {
-  title: string;
-  content: string;
-  contentType: 'markdown' | 'html' | 'text';
-  author?: string;
-  publishedDate?: string;
-  sections?: string[];        // detected section titles
-}

package/src/engine/knowledge-import/processors/validate.ts

@@ -1,150 +0,0 @@
-/**
- * Layer 3: VALIDATE — Quality Gates & Scoring
- *
- * Runs quality checks on cleaned content before it enters the knowledge base.
- * Rejects junk, scores quality, flags warnings.
- *
- * Quality checks:
- *   1. Minimum content length (reject if too short)
- *   2. Language coherence (reject gibberish)
- *   3. Information density (reject navigation-only pages)
- *   4. Duplication ratio (reject if mostly repeated)
- *   5. Code-to-text ratio (flag if mostly code)
- *   6. Readability score
- */
-
-import type { QualityReport, QualityCheck } from './types.js';
-
-const MIN_CONTENT_LENGTH = 50;    // chars
-const MIN_WORD_COUNT = 15;
-const MIN_SENTENCE_COUNT = 2;
-const MAX_DUPLICATE_RATIO = 0.4;
-const MIN_QUALITY_SCORE = 30;     // out of 100
-
-export function validateContent(content: string, _title?: string): QualityReport {
-  const checks: QualityCheck[] = [];
-  const warnings: string[] = [];
-
-  // ─── Check 1: Content Length ──────────────────────
-
-  const length = content.length;
-  const lengthPassed = length >= MIN_CONTENT_LENGTH;
-  checks.push({
-    name: 'content_length',
-    passed: lengthPassed,
-    score: Math.min(100, Math.round((length / 200) * 100)),
-    detail: `${length} chars (min ${MIN_CONTENT_LENGTH})`,
-  });
-
-  // ─── Check 2: Word Count & Coherence ─────────────
-
-  const words = content.split(/\s+/).filter(w => w.length > 0);
-  const wordCount = words.length;
-  const avgWordLength = wordCount > 0 ? words.reduce((s, w) => s + w.length, 0) / wordCount : 0;
-  const wordsPassed = wordCount >= MIN_WORD_COUNT;
-
-  // Detect gibberish: too many very short or very long words
-  const shortWords = words.filter(w => w.length <= 1).length;
-  const longWords = words.filter(w => w.length > 25).length;
-  const gibberishRatio = (shortWords + longWords) / Math.max(wordCount, 1);
-  const coherent = gibberishRatio < 0.3 && avgWordLength > 2 && avgWordLength < 20;
-
-  checks.push({
-    name: 'word_count',
-    passed: wordsPassed,
-    score: Math.min(100, Math.round((wordCount / 50) * 100)),
-    detail: `${wordCount} words (min ${MIN_WORD_COUNT})`,
-  });
-
-  checks.push({
-    name: 'coherence',
-    passed: coherent,
-    score: coherent ? Math.round((1 - gibberishRatio) * 100) : 10,
-    detail: `avg word length ${avgWordLength.toFixed(1)}, gibberish ratio ${(gibberishRatio * 100).toFixed(0)}%`,
-  });
-  if (!coherent) warnings.push('Content may contain gibberish or encoded data');
-
-  // ─── Check 3: Information Density ────────────────
-
-  const sentences = content.split(/[.!?]+/).filter(s => s.trim().length > 10);
-  const sentenceCount = sentences.length;
-  const hasSentences = sentenceCount >= MIN_SENTENCE_COUNT;
-
-  // Check if it's mostly links/navigation (high ratio of [] or http://)
-  const linkMatches = content.match(/\[|\]|\(http|https?:\/\//g) || [];
-  const linkDensity = linkMatches.length / Math.max(wordCount, 1);
-  const notJustLinks = linkDensity < 0.15;
-
-  // Check for actual informational content (has verbs, explanations)
-  const hasSubstance = /\b(?:is|are|was|were|will|can|should|must|how|when|where|what|why|because|therefore|means|allows|enables|provides|requires|includes)\b/i.test(content);
-
-  const densityScore = Math.round(
-    ((hasSentences ? 40 : 0) + (notJustLinks ? 30 : 0) + (hasSubstance ? 30 : 0))
-  );
-
-  checks.push({
-    name: 'information_density',
-    passed: densityScore >= 40,
-    score: densityScore,
-    detail: `${sentenceCount} sentences, link density ${(linkDensity * 100).toFixed(0)}%, substance: ${hasSubstance}`,
-  });
-  if (linkDensity >= 0.15) warnings.push('High link density — may be a navigation/index page');
-  if (!hasSubstance) warnings.push('Content may lack substantive information');
-
-  // ─── Check 4: Duplication Ratio ──────────────────
-
-  const lines = content.split('\n').filter(l => l.trim().length > 0);
-  const uniqueLines = new Set(lines.map(l => l.trim().toLowerCase()));
-  const dupRatio = lines.length > 0 ? 1 - (uniqueLines.size / lines.length) : 0;
-  const noDuplication = dupRatio <= MAX_DUPLICATE_RATIO;
-
-  checks.push({
-    name: 'duplication',
-    passed: noDuplication,
-    score: Math.round((1 - dupRatio) * 100),
-    detail: `${(dupRatio * 100).toFixed(0)}% duplicate lines (max ${MAX_DUPLICATE_RATIO * 100}%)`,
-  });
-  if (!noDuplication) warnings.push('High content duplication detected');
-
-  // ─── Check 5: Code-to-Text Ratio ─────────────────
-
-  const codeBlocks = content.match(/```[\s\S]*?```/g) || [];
-  const codeLength = codeBlocks.reduce((s, b) => s + b.length, 0);
-  const codeRatio = length > 0 ? codeLength / length : 0;
-  const notMostlyCode = codeRatio < 0.8;
-
-  checks.push({
-    name: 'code_ratio',
-    passed: notMostlyCode,
-    score: Math.round((1 - codeRatio) * 100),
-    detail: `${(codeRatio * 100).toFixed(0)}% code blocks`,
-  });
-  if (codeRatio >= 0.5) warnings.push('Content is mostly code — may need context text');
-
-  // ─── Check 6: Readability ────────────────────────
-
-  // Simplified readability: based on sentence length and word length
-  const avgSentenceLength = sentenceCount > 0 ? wordCount / sentenceCount : wordCount;
-  const readabilityScore = Math.min(100, Math.max(0,
-    100 - Math.abs(avgSentenceLength - 15) * 3 - Math.abs(avgWordLength - 5) * 5
-  ));
-
-  checks.push({
-    name: 'readability',
-    passed: readabilityScore >= 30,
-    score: Math.round(readabilityScore),
-    detail: `avg sentence ${avgSentenceLength.toFixed(0)} words, avg word ${avgWordLength.toFixed(1)} chars`,
-  });
-
-  // ─── Overall Score ───────────────────────────────
-
-  const totalScore = Math.round(checks.reduce((s, c) => s + c.score, 0) / checks.length);
-  const allCriticalPassed = checks.filter(c => ['content_length', 'word_count', 'coherence'].includes(c.name)).every(c => c.passed);
-  const passed = allCriticalPassed && totalScore >= MIN_QUALITY_SCORE;
-
-  if (!passed && totalScore < MIN_QUALITY_SCORE) {
-    warnings.unshift(`Quality score ${totalScore}/100 below threshold ${MIN_QUALITY_SCORE}`);
-  }
-
-  return { score: totalScore, passed, checks, warnings };
-}

package/src/engine/knowledge-import/provider-file-upload.ts

@@ -1,95 +0,0 @@
-/**
- * Knowledge Import — File Upload Provider
- *
- * Imports documentation from directly uploaded files.
- * Supports: Markdown, HTML, TXT, PDF, DOCX, JSON, CSV.
- *
- * Files are received via multipart upload and processed in-memory.
- */
-
-import type { ImportProvider, ImportDocument, ImportConfigField } from './types.js';
-
-const SUPPORTED_EXTENSIONS: Record<string, 'markdown' | 'html' | 'text' | 'pdf'> = {
-  'md': 'markdown',
-  'mdx': 'markdown',
-  'markdown': 'markdown',
-  'html': 'html',
-  'htm': 'html',
-  'txt': 'text',
-  'text': 'text',
-  'rst': 'text',
-  'adoc': 'text',
-  'pdf': 'pdf',
-  'json': 'text',
-  'csv': 'text',
-  'yaml': 'text',
-  'yml': 'text',
-};
-
-const MAX_FILE_SIZE = 10_000_000; // 10MB
-
-export class FileUploadImportProvider implements ImportProvider {
-  type = 'file-upload' as const;
-
-  getConfigFields(): ImportConfigField[] {
-    return [
-      { name: 'files', label: 'Files', type: 'text', required: true, helpText: 'Upload markdown, HTML, TXT, PDF, or DOCX files. Max 10MB per file.' },
-    ];
-  }
-
-  async validate(config: Record<string, any>): Promise<{ valid: boolean; error?: string }> {
-    const files = config.files;
-    if (!files || !Array.isArray(files) || files.length === 0) {
-      return { valid: false, error: 'At least one file is required' };
-    }
-
-    for (const file of files) {
-      if (!file.name) return { valid: false, error: 'Each file must have a name' };
-      const ext = file.name.split('.').pop()?.toLowerCase();
-      if (!ext || !SUPPORTED_EXTENSIONS[ext]) {
-        return { valid: false, error: `Unsupported file type: .${ext}. Supported: ${Object.keys(SUPPORTED_EXTENSIONS).join(', ')}` };
-      }
-      if (file.size && file.size > MAX_FILE_SIZE) {
-        return { valid: false, error: `File "${file.name}" exceeds 10MB limit` };
-      }
-    }
-
-    return { valid: true };
-  }
-
-  async *discover(config: Record<string, any>): AsyncGenerator<ImportDocument> {
-    const files: Array<{ name: string; content: string; size?: number }> = config.files || [];
-
-    for (const file of files) {
-      const ext = file.name.split('.').pop()?.toLowerCase() || '';
-      const contentType = SUPPORTED_EXTENSIONS[ext] || 'text';
-      const title = file.name.replace(/\.[^.]+$/, '').replace(/[-_]/g, ' ');
-
-      yield {
-        id: `upload-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`,
-        sourceType: 'file-upload',
-        sourcePath: file.name,
-        title,
-        content: file.content,
-        contentType,
-        metadata: { filename: file.name, uploadedAt: new Date().toISOString() },
-        size: file.size || file.content.length,
-      };
-    }
-  }
-}
-
-/**
- * Parse uploaded multipart file data into the format expected by the provider.
- * Called from the route handler before passing to the import manager.
- */
-export function parseUploadedFiles(formData: Array<{ filename: string; data: Buffer | string; mimeType?: string }>): Array<{ name: string; content: string; size: number }> {
-  return formData.map(f => {
-    const content = typeof f.data === 'string' ? f.data : f.data.toString('utf-8');
-    return {
-      name: f.filename,
-      content,
-      size: content.length,
-    };
-  });
-}

package/src/engine/knowledge-import/provider-github.ts

@@ -1,144 +0,0 @@
-/**
- * Knowledge Import — GitHub Provider
- *
- * Imports documentation from public (or private with token) GitHub repositories.
- * Supports: README, docs folders, wiki, markdown/text files.
- */
-
-import type { ImportProvider, ImportDocument, ImportConfigField } from './types.js';
-
-const IMPORTABLE_EXTENSIONS = ['.md', '.mdx', '.txt', '.rst', '.adoc', '.html', '.htm'];
-const SKIP_PATHS = ['node_modules', '.git', 'vendor', 'dist', 'build', '__pycache__', '.next', '.nuxt'];
-const MAX_FILE_SIZE = 500_000; // 500KB per file
-
-export class GitHubImportProvider implements ImportProvider {
-  type = 'github' as const;
-
-  getConfigFields(): ImportConfigField[] {
-    return [
-      { name: 'repoUrl', label: 'Repository URL', type: 'url', placeholder: 'https://github.com/owner/repo', required: true, helpText: 'Full URL to the GitHub repository' },
-      { name: 'branch', label: 'Branch', type: 'text', placeholder: 'main', helpText: 'Branch to import from (defaults to main)' },
-      { name: 'docsPath', label: 'Docs Path', type: 'text', placeholder: 'docs/', helpText: 'Only import files from this directory (leave empty for entire repo)' },
-      { name: 'token', label: 'GitHub Token', type: 'password', placeholder: 'ghp_...', helpText: 'Required for private repos. Use a fine-grained personal access token with "Contents" read access.' },
-      { name: 'includeReadme', label: 'Include README files', type: 'checkbox', helpText: 'Import README.md files from the root and subdirectories' },
-      { name: 'maxDepth', label: 'Max Directory Depth', type: 'text', placeholder: '5', helpText: 'Maximum folder depth to scan (default 5)' },
-    ];
-  }
-
-  async validate(config: Record<string, any>): Promise<{ valid: boolean; error?: string }> {
-    const { repoUrl } = config;
-    if (!repoUrl) return { valid: false, error: 'Repository URL is required' };
-
-    const parsed = parseGitHubUrl(repoUrl);
-    if (!parsed) return { valid: false, error: 'Invalid GitHub URL. Use format: https://github.com/owner/repo' };
-
-    // Check repo accessibility
-    try {
-      const headers: Record<string, string> = { 'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'AgenticMail-KnowledgeImport' };
-      if (config.token) headers['Authorization'] = `Bearer ${config.token}`;
-
-      const resp = await fetch(`https://api.github.com/repos/${parsed.owner}/${parsed.repo}`, { headers });
-      if (resp.status === 404) return { valid: false, error: 'Repository not found. Check the URL or provide a token for private repos.' };
-      if (resp.status === 401) return { valid: false, error: 'Authentication failed. Check your GitHub token.' };
-      if (!resp.ok) return { valid: false, error: `GitHub API error: ${resp.status} ${resp.statusText}` };
-
-      return { valid: true };
-    } catch (e: any) {
-      return { valid: false, error: `Failed to reach GitHub: ${e.message}` };
-    }
-  }
-
-  async *discover(config: Record<string, any>): AsyncGenerator<ImportDocument> {
-    const parsed = parseGitHubUrl(config.repoUrl);
-    if (!parsed) return;
-
-    const branch = config.branch || 'main';
-    const docsPath = (config.docsPath || '').replace(/^\/|\/$/g, '');
-    const maxDepth = parseInt(config.maxDepth) || 5;
-    const includeReadme = config.includeReadme !== false;
-    const headers: Record<string, string> = { 'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'AgenticMail-KnowledgeImport' };
-    if (config.token) headers['Authorization'] = `Bearer ${config.token}`;
-
-    // Use the Git Trees API for efficient full-tree listing
-    const treeUrl = `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/git/trees/${branch}?recursive=1`;
-    const treeResp = await fetch(treeUrl, { headers });
-    if (!treeResp.ok) {
-      throw new Error(`Failed to fetch repo tree: ${treeResp.status} ${treeResp.statusText}`);
-    }
-
-    const treeData = await treeResp.json() as { tree: Array<{ path: string; type: string; size?: number; sha: string }> };
-
-    for (const item of treeData.tree) {
-      if (item.type !== 'blob') continue;
-
-      // Check path filters
-      if (docsPath && !item.path.startsWith(docsPath + '/') && item.path !== docsPath) continue;
-
-      // Check depth
-      const depth = item.path.split('/').length;
-      if (depth > maxDepth) continue;
-
-      // Skip common non-doc directories
-      if (SKIP_PATHS.some(skip => item.path.includes('/' + skip + '/') || item.path.startsWith(skip + '/'))) continue;
-
-      // Check extension
-      const ext = '.' + item.path.split('.').pop()?.toLowerCase();
-      const isReadme = /readme/i.test(item.path.split('/').pop() || '');
-
-      if (!IMPORTABLE_EXTENSIONS.includes(ext) && !isReadme) continue;
-      if (isReadme && !includeReadme) continue;
-
-      // Skip large files
-      if (item.size && item.size > MAX_FILE_SIZE) continue;
-
-      // Fetch file content
-      try {
-        const contentUrl = `https://api.github.com/repos/${parsed.owner}/${parsed.repo}/contents/${item.path}?ref=${branch}`;
-        const contentResp = await fetch(contentUrl, { headers });
-        if (!contentResp.ok) continue;
-
-        const fileData = await contentResp.json() as { content?: string; encoding?: string; html_url?: string; size: number };
-        if (!fileData.content) continue;
-
-        // Decode base64 content
-        const rawContent = Buffer.from(fileData.content, 'base64').toString('utf-8');
-        const filename = item.path.split('/').pop() || item.path;
-        const title = filename.replace(/\.[^.]+$/, '').replace(/[-_]/g, ' ');
-
-        yield {
-          id: `github-${parsed.owner}-${parsed.repo}-${item.sha.slice(0, 8)}`,
-          sourceType: 'github',
-          sourcePath: item.path,
-          sourceUrl: fileData.html_url || `https://github.com/${parsed.owner}/${parsed.repo}/blob/${branch}/${item.path}`,
-          title: isReadme ? `${item.path.split('/').slice(0, -1).join('/') || parsed.repo} README` : title,
-          content: rawContent,
-          contentType: ext === '.html' || ext === '.htm' ? 'html' : 'markdown',
-          metadata: {
-            repo: `${parsed.owner}/${parsed.repo}`,
-            branch,
-            sha: item.sha,
-            filename,
-          },
-          size: fileData.size,
-        };
-
-        // Rate limit: small delay between file fetches
-        await sleep(100);
-      } catch {
-        // Skip files that fail to fetch
-        continue;
-      }
-    }
-  }
-}
-
-function parseGitHubUrl(url: string): { owner: string; repo: string } | null {
-  // Match: https://github.com/owner/repo(.git)
-  const match = url.match(/github\.com[/:]([^/]+)\/([^/.]+)/);
-  if (!match) return null;
-  return { owner: match[1], repo: match[2] };
-}
-
-function sleep(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}