@wenathlan/saddle 1.8.17 → 2.0.2

package/docs/plans/23.research.ai.integration.md

@@ -1,1465 +0,0 @@
-# Pesquisa: AI/LLM Integration para Web Scraping (2026)
-
-> Como integrar scraping web com LLMs: formatação para consumo de tokens, RAG pipelines, extração inteligente, e spec llms.txt.
-
----
-
-## 1. Formatting Scraped Content for LLMs
-
-### Markdown como ponte universal
-
-Markdown é o formato ideal para LLMs porque:
-
-1. **Estrutura sem overhead** — headings, lists, code blocks são compactos
-2. **60-80% reduction de tokens** vs HTML puro
-3. **Preserva semântica** — bold, italic, links, tabelas mantêm significado
-4. **Fácil de parsear** — qualquer LLM entende Markdown nativamente
-
-### Comparação de formatos
-
-| Formato | Tokens (p/ 1KB de conteúdo) | Preservação semântica | Facilidade de parse |
-|---------|----------------------------|----------------------|-------------------|
-| HTML puro | ~300-400 | Alta | Alta |
-| HTML limpo | ~200-250 | Média | Média |
-| **Markdown** | **~80-120** | **Alta** | **Alta** |
-| Texto puro | ~60-80 | Baixa | Baixa |
-| JSON | ~150-200 | Alta | Alta |
-
-### Estratégia de conversão HTML → Markdown
-
-```typescript
-import TurndownService from 'turndown';
-
-function htmlToLLMContent(html: string, options?: {
-  preserveTables?: boolean;
-  preserveCodeBlocks?: boolean;
-  maxLineLength?: number;
-}): string {
-  const turndown = new TurndownService({
-    headingStyle: 'atx',
-    codeBlockStyle: 'fenced',
-    bulletListMarker: '-',
-  });
-
-  // Remover elementos irrelevantes para LLMs
-  turndown.remove([
-    'script', 'style', 'noscript',
-    'nav', 'footer', 'header', // Navigation boilerplate
-    'aside', // Ads, sidebars
-    '[role="navigation"]',
-    '[role="banner"]',
-    '.ad', '.advertisement', '.sidebar',
-    '.cookie-banner', '.popup',
-  ]);
-
-  // Preservar tabelas como Markdown
-  if (options?.preserveTables !== false) {
-    turndown.addRule('tables', {
-      filter: 'table',
-      replacement: (_content, node) => {
-        const table = node as HTMLTableElement;
-        const rows = Array.from(table.rows);
-        if (rows.length === 0) return '';
-
-        const header = Array.from(rows[0].cells)
-          .map(c => c.textContent?.trim() ?? '')
-          .join(' | ');
-
-        const separator = Array.from(rows[0].cells)
-          .map(() => '---')
-          .join(' | ');
-
-        const body = rows.slice(1).map(row =>
-          Array.from(row.cells)
-            .map(c => c.textContent?.trim() ?? '')
-            .join(' | ')
-        ).join('\n');
-
-        return `\n\n${header}\n${separator}\n${body}\n\n`;
-      },
-    });
-  }
-
-  let markdown = turndown.turndown(html);
-
-  // Limpar whitespace excessivo
-  markdown = markdown
-    .replace(/\n{3,}/g, '\n\n')  // Max 2 newlines
-    .replace(/[ \t]+$/gm, '')    // Trailing spaces
-    .trim();
-
-  // Truncar se muito longo (opcional)
-  if (options?.maxLineLength && markdown.length > options.maxLineLength) {
-    markdown = markdown.slice(0, options.maxLineLength) + '\n\n[...truncated]';
-  }
-
-  return markdown;
-}
-```
-
-### Metadata para LLM context
-
-```typescript
-interface LLMFormattedContent {
-  content: string;          // Markdown limpo
-  metadata: {
-    sourceUrl: string;
-    title: string;
-    description: string;
-    scrapedAt: string;
-    contentHash: string;    // SHA-256 para dedup
-    tokenEstimate: number;  // Estimativa de tokens
-    headingPath: string[];  // Hierarquia de headings
-  };
-}
-
-function formatForLLM(scrapeResult: ScrapeResult): LLMFormattedContent {
-  const content = htmlToLLMContent(scrapeResult.content);
-
-  return {
-    content,
-    metadata: {
-      sourceUrl: scrapeResult.url,
-      title: scrapeResult.metadata.title,
-      description: scrapeResult.metadata.description,
-      scrapedAt: new Date().toISOString(),
-      contentHash: hashContent(content),
-      tokenEstimate: estimateTokens(content),
-      headingPath: extractHeadingPath(content),
-    },
-  };
-}
-
-function hashContent(content: string): string {
-  return crypto.subtle
-    .digest('SHA-256', new TextEncoder().encode(content))
-    .then(buf => Array.from(new Uint8Array(buf))
-      .map(b => b.toString(16).padStart(2, '0'))
-      .join(''));
-}
-
-function estimateTokens(content: string): number {
-  // Estimativa: ~4 chars por token (média para inglês)
-  return Math.ceil(content.length / 4);
-}
-
-function extractHeadingPath(content: string): string[] {
-  const headings = content.match(/^#{1,6}\s+(.+)$/gm) ?? [];
-  return headings.map(h => {
-    const level = h.match(/^(#+)/)?.[1].length ?? 1;
-    const text = h.replace(/^#+\s+/, '');
-    return `${level}:${text}`;
-  });
-}
-```
-
----
-
-## 2. Token Limits e Chunking Strategies
-
-### Limites de contexto por modelo (2026)
-
-| Modelo | Context Window | Sweet Spot (eficiente) | Max Input Tokens |
-|--------|---------------|----------------------|-----------------|
-| GPT-4o | 128K | 8K-32K | ~100K |
-| GPT-4o-mini | 128K | 4K-16K | ~100K |
-| Claude 3.5 Sonnet | 200K | 8K-32K | ~180K |
-| Claude 3 Opus | 200K | 8K-32K | ~180K |
-| Gemini 1.5 Pro | 2M | 16K-128K | ~1.8M |
-| Gemini 1.5 Flash | 1M | 8K-64K | ~900K |
-| Llama 3.1 405B | 128K | 4K-16K | ~100K |
-| Mistral Large | 128K | 4K-16K | ~100K |
-
-### Sweet spot: 400-512 tokens por chunk
-
-**Por que 400-512 tokens?**
-
-1. **Embedding models** — a maioria treina com chunks de 512 tokens
-2. **Retrieval quality** — chunks muito pequenos perdem contexto, muito grandes diluem
-3. **Memory efficiency** — vetores de 512 tokens são eficientes em storage
-4. **Overlap** — 10-20% de overlap (40-100 tokens) garante continuidade
-
-### Estratégias de chunking
-
-#### 2.1. Fixed-size chunks (simples, não recomendado)
-
-```typescript
-function fixedChunk(text: string, size = 500, overlap = 50): string[] {
-  const chunks: string[] = [];
-  const words = text.split(/\s+/);
-
-  for (let i = 0; i < words.length; i += size - overlap) {
-    chunks.push(words.slice(i, i + size).join(' '));
-  }
-
-  return chunks;
-}
-
-// ❌ Problema: corta no meio de frases e parágrafos
-```
-
-#### 2.2. Recursive character splitting (69% accuracy)
-
-```typescript
-function recursiveChunk(
-  text: string,
-  maxTokens = 512,
-  overlap = 0.1
-): string[] {
-  const separators = ['\n\n', '\n', '. ', '! ', '? ', ', ', ' '];
-
-  return chunkRecursive(text, maxTokens, overlap, separators);
-}
-
-function chunkRecursive(
-  text: string,
-  maxTokens: number,
-  overlap: number,
-  separators: string[]
-): string[] {
-  if (estimateTokens(text) <= maxTokens) {
-    return [text];
-  }
-
-  const separator = separators[0];
-  const remainingSeparators = separators.slice(1);
-
-  if (!separator) {
-    // Fallback: cortar por caracteres
-    return hardChunk(text, maxTokens);
-  }
-
-  const parts = text.split(separator);
-  const chunks: string[] = [];
-  let current = '';
-
-  for (const part of parts) {
-    const test = current ? current + separator + part : part;
-
-    if (estimateTokens(test) > maxTokens && current) {
-      chunks.push(current);
-      // Overlap: manter últimos N tokens
-      const overlapTokens = Math.floor(maxTokens * overlap);
-      current = keepLastTokens(current, overlapTokens) + separator + part;
-    } else {
-      current = test;
-    }
-  }
-
-  if (current) {
-    chunks.push(current);
-  }
-
-  // Recursivamente quebrar chunks que ainda são grandes
-  return chunks.flatMap(chunk =>
-    estimateTokens(chunk) > maxTokens
-      ? chunkRecursive(chunk, maxTokens, overlap, remainingSeparators)
-      : [chunk]
-  );
-}
-
-function hardChunk(text: string, maxTokens: number): string[] {
-  const chars = text.split('');
-  const chunks: string[] = [];
-  let current = '';
-
-  for (const char of chars) {
-    current += char;
-    if (estimateTokens(current) >= maxTokens) {
-      chunks.push(current);
-      current = '';
-    }
-  }
-
-  if (current) chunks.push(current);
-  return chunks;
-}
-
-function keepLastTokens(text: string, tokens: number): string {
-  const words = text.split(/\s+/);
-  const keepWords = Math.floor(tokens * 1.3); // ~1.3 words per token
-  return words.slice(-keepWords).join(' ');
-}
-```
-
-#### 2.3. Hierarchical parent-child (padrão production)
-
-```typescript
-interface HierarchicalChunk {
-  id: string;
-  content: string;
-  parent?: string;        // heading pai
-  children?: string[];    // sub-chunks
-  metadata: {
-    heading: string;
-    level: number;
-    tokenCount: number;
-    position: number;
-  };
-}
-
-function hierarchicalChunk(markdown: string, maxTokens = 512): HierarchicalChunk[] {
-  const sections = splitByHeadings(markdown);
-  const chunks: HierarchicalChunk[] = [];
-
-  for (let i = 0; i < sections.length; i++) {
-    const section = sections[i];
-    const tokens = estimateTokens(section.content);
-
-    if (tokens <= maxTokens) {
-      // Seção cabe em um chunk
-      chunks.push({
-        id: `chunk-${i}`,
-        content: section.content,
-        parent: section.parentHeading,
-        metadata: {
-          heading: section.heading,
-          level: section.level,
-          tokenCount: tokens,
-          position: i,
-        },
-      });
-    } else {
-      // Seção é grande — subdividir
-      const subChunks = recursiveChunk(section.content, maxTokens, 0.1);
-      for (let j = 0; j < subChunks.length; j++) {
-        chunks.push({
-          id: `chunk-${i}-${j}`,
-          content: subChunks[j],
-          parent: section.heading,
-          metadata: {
-            heading: `${section.heading} (part ${j + 1})`,
-            level: section.level,
-            tokenCount: estimateTokens(subChunks[j]),
-            position: i * 100 + j,
-          },
-        });
-      }
-    }
-  }
-
-  // Link parent-child
-  for (const chunk of chunks) {
-    const parent = chunks.find(c =>
-      c.metadata.heading === chunk.parent &&
-      c.metadata.level < chunk.metadata.level
-    );
-    if (parent) {
-      if (!parent.children) parent.children = [];
-      parent.children.push(chunk.id);
-    }
-  }
-
-  return chunks;
-}
-
-interface Section {
-  heading: string;
-  level: number;
-  parentHeading: string;
-  content: string;
-}
-
-function splitByHeadings(markdown: string): Section[] {
-  const lines = markdown.split('\n');
-  const sections: Section[] = [];
-  let current: Section | null = null;
-  let headingStack: { heading: string; level: number }[] = [];
-
-  for (const line of lines) {
-    const match = line.match(/^(#{1,6})\s+(.+)/);
-    if (match) {
-      const level = match[1].length;
-      const heading = match[2];
-
-      // Atualizar stack
-      while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= level) {
-        headingStack.pop();
-      }
-
-      const parentHeading = headingStack.length > 0
-        ? headingStack[headingStack.length - 1].heading
-        : '';
-
-      if (current) sections.push(current);
-
-      current = {
-        heading,
-        level,
-        parentHeading,
-        content: line + '\n',
-      };
-
-      headingStack.push({ heading, level });
-    } else if (current) {
-      current.content += line + '\n';
-    }
-  }
-
-  if (current) sections.push(current);
-  return sections;
-}
-```
-
----
-
-## 3. RAG Pipeline Completo
-
-### Pipeline: Fetch → Extract → Normalize → Markdown → Chunk → Embed → Store
-
-```
-┌─────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐
-│  Fetch   │───▶│ Extract  │───▶│ Normalize│───▶│ Markdown │
-│ (fetch)  │    │ (cheerio)│    │ (clean)  │    │(turndown)│
-└─────────┘    └──────────┘    └──────────┘    └──────────┘
-                                                       │
-                                                       ▼
-┌─────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐
-│  Store   │◀───│  Embed   │◀───│  Chunk   │◀───│ Metadata │
-│ (vector) │    │(embeddings│   │(hierar.) │    │ (hash)   │
-└─────────┘    └──────────┘    └──────────┘    └──────────┘
-```
-
-### Implementação completa
-
-```typescript
-// src/core/rag-pipeline.ts
-import TurndownService from 'turndown';
-import { createHash } from 'node:crypto';
-
-interface RAGConfig {
-  maxChunkTokens?: number;
-  chunkOverlap?: number;
-  embeddingModel?: string;
-  vectorStore?: 'memory' | 'sqlite' | 'redis';
-}
-
-interface ProcessedDocument {
-  id: string;
-  url: string;
-  title: string;
-  content: string;
-  chunks: ProcessedChunk[];
-  metadata: DocumentMetadata;
-}
-
-interface ProcessedChunk {
-  id: string;
-  documentId: string;
-  content: string;
-  embedding?: number[];
-  metadata: ChunkMetadata;
-}
-
-interface DocumentMetadata {
-  url: string;
-  title: string;
-  description: string;
-  scrapedAt: string;
-  contentHash: string;
-  totalTokens: number;
-  language: string;
-}
-
-interface ChunkMetadata {
-  sourceUrl: string;
-  headingPath: string[];
-  contentHash: string;
-  tokenCount: number;
-  position: number;
-  parentChunkId?: string;
-}
-
-class RAGPipeline {
-  private config: RAGConfig;
-
-  constructor(config: RAGConfig = {}) {
-    this.config = {
-      maxChunkTokens: 512,
-      chunkOverlap: 0.1,
-      embeddingModel: 'text-embedding-3-small',
-      vectorStore: 'memory',
-      ...config,
-    };
-  }
-
-  // Pipeline completo: URL → Document processado
-  async processUrl(url: string): Promise<ProcessedDocument> {
-    // 1. Fetch
-    const response = await fetch(url);
-    const html = await response.text();
-
-    // 2. Extract + Normalize
-    const normalized = this.extractAndNormalize(html, url);
-
-    // 3. Convert to Markdown
-    const markdown = this.htmlToMarkdown(normalized.content);
-
-    // 4. Add metadata header
-    const fullContent = this.addMetadataHeader(markdown, normalized.metadata);
-
-    // 5. Chunk
-    const chunks = this.hierarchicalChunk(fullContent, normalized.metadata);
-
-    return {
-      id: this.generateId(url),
-      url,
-      title: normalized.metadata.title,
-      content: fullContent,
-      chunks,
-      metadata: normalized.metadata,
-    };
-  }
-
-  // Processar múltiplas URLs com deduplicação
-  async processUrls(
-    urls: string[],
-    onProgress?: (current: number, total: number) => void
-  ): Promise<ProcessedDocument[]> {
-    const documents: ProcessedDocument[] = [];
-    const seen = new Set<string>();
-
-    for (let i = 0; i < urls.length; i++) {
-      const url = urls[i];
-
-      // Deduplicação por URL
-      if (seen.has(url)) continue;
-      seen.add(url);
-
-      try {
-        const doc = await this.processUrl(url);
-        documents.push(doc);
-        onProgress?.(i + 1, urls.length);
-      } catch (error) {
-        console.error(`Failed to process ${url}:`, error);
-      }
-    }
-
-    return documents;
-  }
-
-  private extractAndNormalize(html: string, url: string) {
-    // Usar cheerio para extrair conteúdo limpo
-    // Similar ao que já existe no projeto
-    const title = '';
-    const description = '';
-    const content = html;
-
-    const contentHash = createHash('sha256')
-      .update(content)
-      .digest('hex');
-
-    return {
-      content,
-      metadata: {
-        url,
-        title,
-        description,
-        scrapedAt: new Date().toISOString(),
-        contentHash,
-        totalTokens: estimateTokens(content),
-        language: 'en',
-      },
-    };
-  }
-
-  private htmlToMarkdown(html: string): string {
-    const turndown = new TurndownService({
-      headingStyle: 'atx',
-      codeBlockStyle: 'fenced',
-    });
-
-    turndown.remove(['script', 'style', 'nav', 'footer', 'aside']);
-    return turndown.turndown(html);
-  }
-
-  private addMetadataHeader(content: string, metadata: DocumentMetadata): string {
-    return `---
-source: ${metadata.url}
-title: "${metadata.title}"
-description: "${metadata.description}"
-scraped_at: ${metadata.scrapedAt}
-content_hash: ${metadata.contentHash}
-language: ${metadata.language}
----
-
-${content}`;
-  }
-
-  private hierarchicalChunk(
-    content: string,
-    docMetadata: DocumentMetadata
-  ): ProcessedChunk[] {
-    const sections = splitByHeadings(content);
-    const chunks: ProcessedChunk[] = [];
-
-    for (let i = 0; i < sections.length; i++) {
-      const section = sections[i];
-      const tokens = estimateTokens(section.content);
-
-      if (tokens <= this.config.maxChunkTokens!) {
-        chunks.push(this.createChunk(
-          section.content,
-          docMetadata,
-          i,
-          section.headingPath
-        ));
-      } else {
-        const subChunks = recursiveChunk(
-          section.content,
-          this.config.maxChunkTokens!,
-          this.config.chunkOverlap!
-        );
-        for (let j = 0; j < subChunks.length; j++) {
-          chunks.push(this.createChunk(
-            subChunks[j],
-            docMetadata,
-            i * 100 + j,
-            [...section.headingPath, `${section.heading} (part ${j + 1})`]
-          ));
-        }
-      }
-    }
-
-    return chunks;
-  }
-
-  private createChunk(
-    content: string,
-    docMetadata: DocumentMetadata,
-    position: number,
-    headingPath: string[]
-  ): ProcessedChunk {
-    const contentHash = createHash('sha256')
-      .update(content)
-      .digest('hex');
-
-    return {
-      id: `${docMetadata.contentHash.slice(0, 8)}-${position}`,
-      documentId: docMetadata.contentHash.slice(0, 16),
-      content,
-      metadata: {
-        sourceUrl: docMetadata.url,
-        headingPath,
-        contentHash,
-        tokenCount: estimateTokens(content),
-        position,
-      },
-    };
-  }
-
-  private generateId(url: string): string {
-    return createHash('sha256')
-      .update(url)
-      .digest('hex')
-      .slice(0, 16);
-  }
-}
-```
-
----
-
-## 4. AI-Powered Content Extraction
-
-### Padrão: Extract Once, Scrape Millions
-
-```typescript
-// src/core/ai-extraction.ts
-
-interface ExtractionSchema {
-  name: string;
-  fields: {
-    name: string;
-    type: 'string' | 'number' | 'boolean' | 'array';
-    description: string;
-    required: boolean;
-  }[];
-}
-
-interface ExtractedResult {
-  schema: string;
-  data: Record<string, unknown>;
-  confidence: number;
-  sourceUrl: string;
-}
-
-// Schema para extrair produtos
-const ProductSchema: ExtractionSchema = {
-  name: 'product',
-  fields: [
-    { name: 'name', type: 'string', description: 'Product name', required: true },
-    { name: 'price', type: 'number', description: 'Price in USD', required: true },
-    { name: 'currency', type: 'string', description: 'Currency code', required: true },
-    { name: 'description', type: 'string', description: 'Product description', required: false },
-    { name: 'rating', type: 'number', description: 'Average rating 0-5', required: false },
-    { name: 'reviewCount', type: 'number', description: 'Number of reviews', required: false },
-    { name: 'inStock', type: 'boolean', description: 'Available in stock', required: true },
-    { name: 'images', type: 'array', description: 'Image URLs', required: false },
-  ],
-};
-
-async function extractWithLLM(
-  content: string,
-  schema: ExtractionSchema,
-  llmConfig: { apiKey: string; model: string; baseUrl: string }
-): Promise<ExtractedResult> {
-  const prompt = `Extract structured data from the following content according to this schema:
-
-Schema: ${JSON.stringify(schema, null, 2)}
-
-Content:
-${content}
-
-Return ONLY valid JSON matching the schema. No explanation.`;
-
-  const response = await fetch(`${llmConfig.baseUrl}/chat/completions`, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${llmConfig.apiKey}`,
-    },
-    body: JSON.stringify({
-      model: llmConfig.model,
-      messages: [
-        {
-          role: 'system',
-          content: 'You are a data extraction assistant. Return only valid JSON.',
-        },
-        { role: 'user', content: prompt },
-      ],
-      temperature: 0,
-      response_format: { type: 'json_object' },
-    }),
-  });
-
-  const result = await response.json();
-  const data = JSON.parse(result.choices[0].message.content);
-
-  return {
-    schema: schema.name,
-    data,
-    confidence: calculateConfidence(data, schema),
-    sourceUrl: '',
-  };
-}
-
-function calculateConfidence(
-  data: Record<string, unknown>,
-  schema: ExtractionSchema
-): number {
-  const requiredFields = schema.fields.filter(f => f.required);
-  const presentFields = requiredFields.filter(f => data[f.name] !== undefined);
-  return presentFields.length / requiredFields.length;
-}
-```
-
-### Self-Healing Selectors
-
-```typescript
-// src/core/self-healing.ts
-
-interface SelectorStrategy {
-  name: string;
-  priority: number;
-  test: (url: string, html: string) => Promise<boolean>;
-  extract: (html: string) => string;
-}
-
-class SelfHealingExtractor {
-  private strategies: SelectorStrategy[] = [];
-
-  addStrategy(strategy: SelectorStrategy) {
-    this.strategies.push(strategy);
-    this.strategies.sort((a, b) => b.priority - a.priority);
-  }
-
-  async extract(url: string, html: string): Promise<string | null> {
-    for (const strategy of this.strategies) {
-      try {
-        const works = await strategy.test(url, html);
-        if (works) {
-          return strategy.extract(html);
-        }
-      } catch {
-        continue;
-      }
-    }
-    return null;
-  }
-
-  // Auto-heal: quando um selector quebra, tentar alternativas
-  async heal(
-    url: string,
-    html: string,
-    failedSelector: string
-  ): Promise<string | null> {
-    // 1. Tentar variações do selector quebrado
-    const variations = this.generateSelectorVariations(failedSelector);
-    for (const variation of variations) {
-      const result = this.trySelector(html, variation);
-      if (result) return result;
-    }
-
-    // 2. Usar estratégias ML-based (AI extraction)
-    const aiResult = await this.extract(url, html);
-    if (aiResult) return aiResult;
-
-    // 3. Fallback: extrair todo texto
-    return this.extractAllText(html);
-  }
-
-  private generateSelectorVariations(selector: string): string[] {
-    const variations: string[] = [];
-
-    // Adicionar/remover classes
-    const classMatch = selector.match(/\.([a-zA-Z0-9_-]+)/);
-    if (classMatch) {
-      variations.push(selector.replace(classMatch[0], ''));
-      variations.push(selector.replace(classMatch[0], `.${classMatch[1]}-*`));
-    }
-
-    // Tentar IDs vs classes
-    const idMatch = selector.match(/#([a-zA-Z0-9_-]+)/);
-    if (idMatch) {
-      variations.push(`[id*="${idMatch[1]}"]`);
-    }
-
-    // Adicionar nth-child
-    variations.push(`${selector}:nth-child(1)`);
-
-    // Usar aria-label
-    variations.push(`${selector.split('.')[0]}[aria-label]`);
-
-    return variations;
-  }
-
-  private trySelector(html: string, selector: string): string | null {
-    // Usar cheerio
-    try {
-      const cheerio = require('cheerio');
-      const $ = cheerio.load(html);
-      const text = $(selector).text().trim();
-      return text || null;
-    } catch {
-      return null;
-    }
-  }
-
-  private extractAllText(html: string): string {
-    const cheerio = require('cheerio');
-    const $ = cheerio.load(html);
-    $('script, style, noscript').remove();
-    return $.text().trim();
-  }
-}
-```
-
----
-
-## 5. llms.txt Specification
-
-### Estrutura
-
-```txt
-# Site Name
-
-> Brief description of what this site does
-
-## Section Name
-
-- [Page Title](https://example.com/page): Description of the page
-- [Another Page](https://example.com/another): Another description
-
-## API Reference
-
-- [GET /api/users](https://api.example.com/docs/users): Returns all users
-- [POST /api/users](https://api.example.com/docs/users-create): Create a new user
-
-## Blog
-
-- [Latest Post](https://blog.example.com/latest): Summary of the latest blog post
-```
-
-### Regras
-
-1. **Primeira linha:** Nome do site (em `#`)
-2. **Segunda linha:** Descrição (em `>`)
-3. **Seções:** Organizadas por `##`
-4. **Links:** Formato `- [Title](URL): Description`
-5. **Máximo:** 100 links por arquivo
-6. **Profundidade:** Máximo 2 níveis de seções
-7. **Descrições:** Máximo 100 caracteres
-8. **URLs:** Absolutas, com HTTPS
-
-### Exemplo real
-
-```txt
-# DevThink WebScrape
-
-> AI-powered web scraping toolkit with Playwright, multi-format serialization, and LLM-friendly output
-
-## Getting Started
-
-- [Installation](https://github.com/iNathlan/devthink#installation): How to install and set up
-- [Quick Start](https://github.com/iNathlan/devthink#quick-start): Get running in 5 minutes
-
-## API Reference
-
-- [scrape()](https://github.com/iNathlan/devthink/blob/main/docs/API.md#scrape): Main scraping function
-- [Agent Browser](https://github.com/iNathlan/devthink/blob/main/docs/API.md#agent-browser): Playwright-based browser automation
-- [Serializers](https://github.com/iNathlan/devthink/blob/main/docs/API.md#serializers): Output format conversion
-
-## Examples
-
-- [Basic Scraping](https://github.com/iNathlan/devthink/blob/main/examples/basic.ts): Simple URL scraping
-- [AI Agent](https://github.com/iNathlan/devthink/blob/main/examples/agent.ts): Browser agent with AI
-```
-
-### Adoção em 2026
-
-- **840+ sites** já publicam llms.txt
-- **Vercel, Anthropic, OpenAI, Stripe, Supabase** entre os adotantes
-- **Ferramentas:** `llmstxt` CLI para gerar, validadores automáticos
-- **Automação:** Many sites auto-generate via CI/CD
-
-### Como consumir llms.txt
-
-```typescript
-async function fetchLLMsTxt(baseUrl: string): Promise<string> {
-  // Tentar llms.txt primeiro
-  const res = await fetch(`${baseUrl}/llms.txt`);
-  if (res.ok) return res.text();
-
-  // Fallback: robots.txt parseado
-  const robotsRes = await fetch(`${baseUrl}/robots.txt`);
-  if (robotsRes.ok) return parseRobotsTxt(await robotsRes.text());
-
-  return '';
-}
-
-function parseRobotsTxt(robots: string): string {
-  return robots
-    .split('\n')
-    .filter(line => line.startsWith('Allow: ') || line.startsWith('Disallow: '))
-    .map(line => line.replace(/^(Allow|Disallow): /, ''))
-    .join('\n');
-}
-```
-
----
-
-## 6. Token Counting
-
-### 6.1. js-tiktoken (OpenAI)
-
-```typescript
-import { encoding_for_model } from 'js-tiktoken';
-
-const enc = encoding_for_model('gpt-4o');
-
-function countTokens(text: string): number {
-  return enc.encode(text).length;
-}
-
-function chunkByTokens(text: string, maxTokens: number): string[] {
-  const tokens = enc.encode(text);
-  const chunks: string[] = [];
-
-  for (let i = 0; i < tokens.length; i += maxTokens) {
-    const chunk = tokens.slice(i, i + maxTokens);
-    chunks.push(enc.decode(chunk));
-  }
-
-  return chunks;
-}
-
-// Cleanup
-enc.free();
-```
-
-### 6.2. gpt-tokenizer (multi-model)
-
-```typescript
-import { encode, decode } from 'gpt-tokenizer';
-
-function countTokensGPT(text: string): number {
-  return encode(text).length;
-}
-
-// Suporta: gpt-4, gpt-4o, gpt-3.5-turbo
-// Mais leve que js-tiktoken
-```
-
-### 6.3. Claude API
-
-```typescript
-// Anthropic não tem lib oficial de token counting
-// Usar estimativa: ~4 chars por token para inglês
-// Ou usar a API de contagem
-
-async function countClaudeTokens(
-  text: string,
-  apiKey: string
-): Promise<number> {
-  const response = await fetch('https://api.anthropic.com/v1/messages', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'x-api-key': apiKey,
-      'anthropic-version': '2023-06-01',
-    },
-    body: JSON.stringify({
-      model: 'claude-3-5-sonnet-20241022',
-      max_tokens: 1,
-      messages: [{ role: 'user', content: text }],
-    }),
-  });
-
-  const result = await response.json();
-  return result.usage?.input_tokens ?? Math.ceil(text.length / 4);
-}
-```
-
-### 6.4. bpe-lite (multi-model, lightweight)
-
-```typescript
-import { BpeTokenizer } from 'bpe-lite';
-
-// Suporta múltiplos modelos
-const tokenizer = new BpeTokenizer('gpt-4');
-
-function countTokensLite(text: string): number {
-  return tokenizer.tokenize(text).length;
-}
-```
-
-### Comparação
-
-| Package | Size | Speed | Modelos | Precisão |
-|---------|------|-------|---------|----------|
-| js-tiktoken | ~5MB | Rápido | OpenAI | Alta |
-| gpt-tokenizer | ~1MB | Muito rápido | OpenAI | Alta |
-| bpe-lite | ~2MB | Rápido | Multi | Média |
-| Estimativa (chars/4) | 0 | Instantâneo | Qualquer | Baixa (~70%) |
-
----
-
-## 7. Vector Database Formatting
-
-### Metadata fields essenciais
-
-```typescript
-interface VectorMetadata {
-  // Identificação
-  id: string;
-  contentHash: string;        // SHA-256 para dedup
-
-  // Fonte
-  sourceUrl: string;
-  sourceTitle: string;
-  sourceDomain: string;
-
-  // Hierarquia
-  headingPath: string[];      // ["## Section", "### Subsection"]
-  parentChunkId?: string;     // Para hierarchical retrieval
-
-  // Conteúdo
-  tokenCount: number;
-  contentType: 'text' | 'code' | 'table' | 'list';
-  language: string;
-
-  // Temporal
-  scrapedAt: string;
-  expiresAt?: string;         // Para cache TTL
-
-  // Embedding metadata
-  embeddingModel: string;
-  embeddingDimensions: number;
-}
-```
-
-### Formatação para diferentes vector DBs
-
-```typescript
-// Pinecone
-interface PineconeVector {
-  id: string;
-  values: number[];
-  metadata: Record<string, string | number | string[]>;
-}
-
-// Weaviate
-interface WeaviateVector {
-  class: string;
-  id: string;
-  vector: number[];
-  properties: Record<string, unknown>;
-}
-
-// Qdrant
-interface QdrantVector {
-  id: string;
-  vector: number[];
-  payload: Record<string, unknown>;
-}
-
-// Universal format
-function formatForVectorDB(
-  chunk: ProcessedChunk,
-  embedding: number[],
-  dbType: 'pinecone' | 'weaviate' | 'qdrant' | 'memory'
-): PineconeVector | WeaviateVector | QdrantVector | Record<string, unknown> {
-  const metadata = {
-    source_url: chunk.metadata.sourceUrl,
-    heading_path: chunk.metadata.headingPath.join(' > '),
-    content_hash: chunk.metadata.contentHash,
-    token_count: chunk.metadata.tokenCount,
-    position: chunk.metadata.position,
-    scraped_at: new Date().toISOString(),
-  };
-
-  switch (dbType) {
-    case 'pinecone':
-      return {
-        id: chunk.id,
-        values: embedding,
-        metadata,
-      };
-
-    case 'weaviate':
-      return {
-        class: 'DocumentChunk',
-        id: chunk.id,
-        vector: embedding,
-        properties: metadata,
-      };
-
-    case 'qdrant':
-      return {
-        id: chunk.id,
-        vector: embedding,
-        payload: metadata,
-      };
-
-    case 'memory':
-    default:
-      return {
-        id: chunk.id,
-        vector: embedding,
-        ...metadata,
-      };
-  }
-}
-```
-
----
-
-## 8. npm Packages
-
-### Instalação
-
-```bash
-# Scraping
-npm install @mendable/firecrawl-js    # Firecrawl SDK
-npm install llm-scraper                # LLM-powered scraping
-
-# Token counting
-npm install js-tiktoken                # OpenAI tokenizer
-npm install gpt-tokenizer              # Lightweight GPT tokenizer
-npm install bpe-lite                   # Multi-model tokenizer
-
-# Processing
-npm install turndown                   # HTML → Markdown
-npm install cheerio                    # Server-side DOM
-npm install marked                     # Markdown parser
-
-# Storage
-npm install @qdrant/js-client-rest     # Qdrant vector DB
-npm install @pinecone-database/pinecone # Pinecone
-npm install ioredis                    # Redis client
-npm install better-sqlite3             # SQLite
-npm install keyv                       # Universal KV store
-```
-
-### Uso de principais packages
-
-```typescript
-// Firecrawl
-import FirecrawlApp from '@mendable/firecrawl-js';
-
-const app = new FirecrawlApp({ apiKey: 'your-key' });
-
-const result = await app.scrapeUrl('https://example.com', {
-  formats: ['markdown'],
-  onlyMainContent: true,
-});
-console.log(result.markdown);
-
-// llm-scraper
-import { LLMScraper } from 'llm-scraper';
-
-const scraper = new LLMScraper('gpt-4');
-const data = await scraper.scrape('https://example.com', {
-  type: 'object',
-  properties: {
-    title: { type: 'string' },
-    description: { type: 'string' },
-    links: { type: 'array', items: { type: 'string' } },
-  },
-});
-```
-
----
-
-## 9. Content Deduplication
-
-### SHA-256 hashing
-
-```typescript
-async function contentHash(content: string): Promise<string> {
-  const encoder = new TextEncoder();
-  const data = encoder.encode(content);
-  const hashBuffer = await crypto.subtle.digest('SHA-256', data);
-  const hashArray = Array.from(new Uint8Array(hashBuffer));
-  return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
-}
-
-// Dedup no pipeline
-class ContentDeduplicator {
-  private seen = new Map<string, string>(); // hash → url
-
-  async isDuplicate(content: string, url: string): Promise<boolean> {
-    const hash = await contentHash(content);
-
-    if (this.seen.has(hash)) {
-      console.log(`Duplicate detected: ${url} ≈ ${this.seen.get(hash)}`);
-      return true;
-    }
-
-    this.seen.set(hash, url);
-    return false;
-  }
-
-  // Dedup com tolerance para pequenas variações
-  async isNearDuplicate(
-    content: string,
-    url: string,
-    threshold = 0.95
-  ): Promise<boolean> {
-    const hash = await contentHash(content);
-
-    // Check exato primeiro
-    if (this.seen.has(hash)) return true;
-
-    // Check similar (minhash ou simhash — simplificado aqui)
-    for (const [existingHash, existingUrl] of this.seen) {
-      const similarity = this.compareHashes(hash, existingHash);
-      if (similarity > threshold) {
-        console.log(`Near-duplicate: ${url} ≈ ${existingUrl} (${(similarity * 100).toFixed(1)}%)`);
-        return true;
-      }
-    }
-
-    this.seen.set(hash, url);
-    return false;
-  }
-
-  private compareHashes(a: string, b: string): number {
-    // SimHash simplificado
-    let matches = 0;
-    for (let i = 0; i < Math.min(a.length, b.length); i++) {
-      if (a[i] === b[i]) matches++;
-    }
-    return matches / Math.max(a.length, b.length);
-  }
-}
-```
-
----
-
-## 10. Exemplos Completos
-
-### 10.1. Scraping para RAG (end-to-end)
-
-```typescript
-import TurndownService from 'turndown';
-
-interface RAGScraperConfig {
-  maxChunkTokens?: number;
-  embeddingApiKey?: string;
-  vectorDbUrl?: string;
-}
-
-async function scrapeForRAG(
-  urls: string[],
-  config: RAGScraperConfig
-): Promise<void> {
-  const pipeline = new RAGPipeline({
-    maxChunkTokens: config.maxChunkTokens ?? 512,
-  });
-
-  // Process all URLs
-  const documents = await pipeline.processUrls(urls, (current, total) => {
-    console.log(`Processing ${current}/${total}...`);
-  });
-
-  // Generate embeddings
-  for (const doc of documents) {
-    for (const chunk of doc.chunks) {
-      chunk.embedding = await generateEmbedding(
-        chunk.content,
-        config.embeddingApiKey!
-      );
-    }
-  }
-
-  // Store in vector DB
-  for (const doc of documents) {
-    for (const chunk of doc.chunks) {
-      await storeVector(chunk, config.vectorDbUrl!);
-    }
-  }
-
-  console.log(`Processed ${documents.length} documents, ${documents.reduce((sum, d) => sum + d.chunks.length, 0)} chunks`);
-}
-
-async function generateEmbedding(
-  text: string,
-  apiKey: string
-): Promise<number[]> {
-  const response = await fetch('https://api.openai.com/v1/embeddings', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${apiKey}`,
-    },
-    body: JSON.stringify({
-      model: 'text-embedding-3-small',
-      input: text,
-    }),
-  });
-
-  const result = await response.json();
-  return result.data[0].embedding;
-}
-
-async function storeVector(
-  chunk: ProcessedChunk,
-  vectorDbUrl: string
-): Promise<void> {
-  await fetch(`${vectorDbUrl}/vectors`, {
-    method: 'PUT',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      id: chunk.id,
-      vector: chunk.embedding,
-      payload: {
-        content: chunk.content,
-        source_url: chunk.metadata.sourceUrl,
-        heading_path: chunk.metadata.headingPath.join(' > '),
-        token_count: chunk.metadata.tokenCount,
-      },
-    }),
-  });
-}
-```
-
-### 10.2. Query com RAG
-
-```typescript
-async function queryRAG(
-  query: string,
-  config: RAGScraperConfig
-): Promise<string> {
-  // 1. Embed query
-  const queryEmbedding = await generateEmbedding(query, config.embeddingApiKey!);
-
-  // 2. Search vector DB
-  const results = await fetch(`${config.vectorDbUrl}/vectors/search`, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({
-      vector: queryEmbedding,
-      limit: 5,
-      with_payload: true,
-    }),
-  });
-
-  const matches = await results.json();
-
-  // 3. Build context
-  const context = matches
-    .map((m: { payload: { content: string; source_url: string } }) =>
-      `Source: ${m.payload.source_url}\nContent: ${m.payload.content}`
-    )
-    .join('\n\n---\n\n');
-
-  // 4. Query LLM
-  const response = await fetch('https://api.openai.com/v1/chat/completions', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${config.embeddingApiKey}`,
-    },
-    body: JSON.stringify({
-      model: 'gpt-4o',
-      messages: [
-        {
-          role: 'system',
-          content: `Answer based on the context. Cite sources.\n\nContext:\n${context}`,
-        },
-        { role: 'user', content: query },
-      ],
-    }),
-  });
-
-  const result = await response.json();
-  return result.choices[0].message.content;
-}
-```
-
-### 10.3. Batch processing com progress
-
-```typescript
-async function batchScrape(urls: string[], concurrency = 5): Promise<ProcessedDocument[]> {
-  const results: ProcessedDocument[] = [];
-  const queue = [...urls];
-  const processing = new Set<Promise<void>>();
-
-  while (queue.length > 0 || processing.size > 0) {
-    while (processing.size < concurrency && queue.length > 0) {
-      const url = queue.shift()!;
-      const promise = (async () => {
-        try {
-          const doc = await pipeline.processUrl(url);
-          results.push(doc);
-          console.log(`✓ ${url} (${doc.chunks.length} chunks)`);
-        } catch (err) {
-          console.error(`✗ ${url}: ${err}`);
-        }
-      })();
-      processing.add(promise);
-      promise.finally(() => processing.delete(promise));
-    }
-
-    await Promise.race(processing);
-  }
-
-  return results;
-}
-```
-
----
-
-## 11. Checklist
-
-- [ ] Converter HTML para Markdown antes de enviar ao LLM
-- [ ] Usar chunks de 400-512 tokens com 10-20% overlap
-- [ ] Implementar RAG pipeline: Fetch → Extract → Markdown → Chunk → Embed → Store
-- [ ] Adicionar metadata (source_url, heading_path, content_hash) a cada chunk
-- [ ] Deduplicar com SHA-256 antes de armazenar
-- [ ] Tokenizar com js-tiktoken ou gpt-tokenizer
-- [ ] Formatar adequadamente para a vector DB escolhida
-- [ ] Suportar llms.txt para descoberta de conteúdo
-- [ ] Self-healing selectors para scraping resiliente
-- [ ] Testar extração com schemas definidos