@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/change-tracking.js

@@ -0,0 +1,276 @@
+/**
+ * Local-first content change tracking
+ * Stores snapshots in ~/.webpeel/snapshots/ and provides diffing
+ */
+import { createHash } from 'crypto';
+import { promises as fs } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+// Snapshot storage directory
+const SNAPSHOTS_DIR = join(homedir(), '.webpeel', 'snapshots');
+/**
+ * Get storage path for a URL
+ */
+function getSnapshotPath(url) {
+    const hash = createHash('sha256').update(url).digest('hex');
+    return join(SNAPSHOTS_DIR, `${hash}.json`);
+}
+/**
+ * Ensure snapshots directory exists
+ */
+async function ensureSnapshotsDir() {
+    try {
+        await fs.mkdir(SNAPSHOTS_DIR, { recursive: true });
+    }
+    catch (error) {
+        // Ignore if already exists
+    }
+}
+/**
+ * Get a snapshot for a URL
+ *
+ * @param url - URL to get snapshot for
+ * @returns Snapshot if exists, null otherwise
+ *
+ * @example
+ * ```typescript
+ * const snapshot = await getSnapshot('https://example.com');
+ * if (snapshot) {
+ *   console.log('Last scraped:', new Date(snapshot.timestamp));
+ * }
+ * ```
+ */
+export async function getSnapshot(url) {
+    try {
+        const path = getSnapshotPath(url);
+        const data = await fs.readFile(path, 'utf-8');
+        return JSON.parse(data);
+    }
+    catch (error) {
+        return null;
+    }
+}
+/**
+ * Save a snapshot for a URL
+ */
+async function saveSnapshot(snapshot) {
+    await ensureSnapshotsDir();
+    const path = getSnapshotPath(snapshot.url);
+    await fs.writeFile(path, JSON.stringify(snapshot, null, 2), 'utf-8');
+}
+/**
+ * Simple LCS-based unified diff implementation
+ * Returns unified diff format and change statistics
+ */
+function computeDiff(oldContent, newContent) {
+    const oldLines = oldContent.split('\n');
+    const newLines = newContent.split('\n');
+    // Compute LCS (Longest Common Subsequence) using dynamic programming
+    const m = oldLines.length;
+    const n = newLines.length;
+    const lcs = Array(m + 1).fill(null).map(() => Array(n + 1).fill(0));
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            if (oldLines[i - 1] === newLines[j - 1]) {
+                lcs[i][j] = lcs[i - 1][j - 1] + 1;
+            }
+            else {
+                lcs[i][j] = Math.max(lcs[i - 1][j], lcs[i][j - 1]);
+            }
+        }
+    }
+    // Backtrack to build diff
+    const changes = [];
+    let i = m;
+    let j = n;
+    while (i > 0 || j > 0) {
+        if (i > 0 && j > 0 && oldLines[i - 1] === newLines[j - 1]) {
+            changes.unshift({ type: 'normal', line: j, content: newLines[j - 1] });
+            i--;
+            j--;
+        }
+        else if (j > 0 && (i === 0 || lcs[i][j - 1] >= lcs[i - 1][j])) {
+            changes.unshift({ type: 'add', line: j, content: newLines[j - 1] });
+            j--;
+        }
+        else if (i > 0) {
+            changes.unshift({ type: 'del', line: i, content: oldLines[i - 1] });
+            i--;
+        }
+    }
+    // Count additions and deletions
+    let additions = 0;
+    let deletions = 0;
+    for (const change of changes) {
+        if (change.type === 'add')
+            additions++;
+        if (change.type === 'del')
+            deletions++;
+    }
+    // Build unified diff text
+    const diffLines = [];
+    let contextStart = 0;
+    for (let idx = 0; idx < changes.length; idx++) {
+        const change = changes[idx];
+        // Find chunks of changes
+        if (change.type !== 'normal') {
+            // Add context header
+            const chunkStart = Math.max(0, idx - 3);
+            const chunkEnd = Math.min(changes.length, idx + 10);
+            // Skip if we're continuing from previous chunk
+            if (idx > contextStart) {
+                diffLines.push(`@@ -${chunkStart + 1},${chunkEnd - chunkStart} +${chunkStart + 1},${chunkEnd - chunkStart} @@`);
+            }
+            // Add changes
+            for (let k = chunkStart; k < chunkEnd; k++) {
+                const c = changes[k];
+                const prefix = c.type === 'add' ? '+' : c.type === 'del' ? '-' : ' ';
+                diffLines.push(`${prefix}${c.content}`);
+            }
+            contextStart = chunkEnd;
+            idx = chunkEnd - 1;
+        }
+    }
+    return {
+        text: diffLines.join('\n'),
+        additions,
+        deletions,
+        changes,
+    };
+}
+/**
+ * Track content changes for a URL
+ * Compares with previous snapshot and saves new one
+ *
+ * @param url - URL being tracked
+ * @param content - Current content
+ * @param fingerprint - Content fingerprint (SHA256 hash)
+ * @returns Change detection result
+ *
+ * @example
+ * ```typescript
+ * const result = await trackChange('https://example.com', content, fingerprint);
+ * if (result.changeStatus === 'changed') {
+ *   console.log('Content changed!');
+ *   console.log(`+${result.diff.additions} -${result.diff.deletions}`);
+ * }
+ * ```
+ */
+export async function trackChange(url, content, fingerprint) {
+    try {
+        const previous = await getSnapshot(url);
+        if (!previous) {
+            // First time seeing this URL
+            await saveSnapshot({
+                url,
+                fingerprint,
+                content,
+                timestamp: Date.now(),
+            });
+            return {
+                changeStatus: 'new',
+                previousScrapeAt: null,
+            };
+        }
+        // Compare fingerprints
+        if (previous.fingerprint === fingerprint) {
+            // Content unchanged, just update timestamp
+            await saveSnapshot({
+                ...previous,
+                timestamp: Date.now(),
+            });
+            return {
+                changeStatus: 'same',
+                previousScrapeAt: new Date(previous.timestamp).toISOString(),
+            };
+        }
+        // Content changed - compute diff
+        const diff = computeDiff(previous.content, content);
+        // Save new snapshot
+        await saveSnapshot({
+            url,
+            fingerprint,
+            content,
+            timestamp: Date.now(),
+            metadata: {
+                previousFingerprint: previous.fingerprint,
+                previousTimestamp: previous.timestamp,
+            },
+        });
+        return {
+            changeStatus: 'changed',
+            previousScrapeAt: new Date(previous.timestamp).toISOString(),
+            diff,
+        };
+    }
+    catch (error) {
+        console.error('Change tracking error:', error);
+        // On error, treat as new
+        return {
+            changeStatus: 'new',
+            previousScrapeAt: null,
+        };
+    }
+}
+/**
+ * Clear snapshots matching a URL pattern
+ *
+ * @param urlPattern - Optional regex pattern to match URLs (if not provided, clears all)
+ * @returns Number of snapshots cleared
+ *
+ * @example
+ * ```typescript
+ * // Clear all snapshots
+ * const count = await clearSnapshots();
+ *
+ * // Clear specific domain
+ * const count = await clearSnapshots('example\\.com');
+ * ```
+ */
+export async function clearSnapshots(urlPattern) {
+    try {
+        await ensureSnapshotsDir();
+        const files = await fs.readdir(SNAPSHOTS_DIR);
+        let cleared = 0;
+        const pattern = urlPattern ? (() => {
+            if (urlPattern.length > 200)
+                throw new Error('URL pattern too long (max 200 chars)');
+            try {
+                return new RegExp(urlPattern);
+            }
+            catch {
+                throw new Error(`Invalid regex: ${urlPattern}`);
+            }
+        })() : null;
+        for (const file of files) {
+            if (!file.endsWith('.json'))
+                continue;
+            const path = join(SNAPSHOTS_DIR, file);
+            if (pattern) {
+                // Check if URL matches pattern
+                try {
+                    const data = await fs.readFile(path, 'utf-8');
+                    const snapshot = JSON.parse(data);
+                    if (pattern.test(snapshot.url)) {
+                        await fs.unlink(path);
+                        cleared++;
+                    }
+                }
+                catch (e) {
+                    if (process.env.DEBUG)
+                        console.debug('[webpeel]', 'snapshot parse failed:', e instanceof Error ? e.message : e);
+                }
+            }
+            else {
+                // Clear all
+                await fs.unlink(path);
+                cleared++;
+            }
+        }
+        return cleared;
+    }
+    catch (error) {
+        console.error('Clear snapshots error:', error);
+        return 0;
+    }
+}

package/dist/core/chunker.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Content chunker for RAG pipelines.
+ * Splits markdown content into overlapping chunks with rich metadata.
+ */
+export interface ChunkOptions {
+    /** Max tokens per chunk (approximate, using ~4 chars/token) */
+    maxTokens?: number;
+    /** Overlap tokens between chunks */
+    overlap?: number;
+    /** Chunking strategy */
+    strategy?: 'section' | 'paragraph' | 'fixed';
+}
+export interface ContentChunk {
+    /** Chunk index (0-based) */
+    index: number;
+    /** The chunk text content */
+    text: string;
+    /** Approximate token count (~4 chars per token) */
+    tokenCount: number;
+    /** Word count */
+    wordCount: number;
+    /** Section heading this chunk belongs to (if any) */
+    section: string | null;
+    /** Section depth (1=h1, 2=h2, etc.) */
+    sectionDepth: number | null;
+    /** Character offset in original content */
+    startOffset: number;
+    /** Character end offset */
+    endOffset: number;
+}
+export interface ChunkResult {
+    /** Array of content chunks */
+    chunks: ContentChunk[];
+    /** Total chunks */
+    totalChunks: number;
+    /** Original content length (chars) */
+    originalLength: number;
+    /** Chunking strategy used */
+    strategy: string;
+    /** Options used */
+    options: Required<ChunkOptions>;
+}
+/**
+ * Split content into RAG-ready chunks with metadata.
+ */
+export declare function chunkContent(content: string, options?: ChunkOptions): ChunkResult;

package/dist/core/chunker.js

@@ -0,0 +1,249 @@
+/**
+ * Content chunker for RAG pipelines.
+ * Splits markdown content into overlapping chunks with rich metadata.
+ */
+const DEFAULT_MAX_TOKENS = 512;
+const DEFAULT_OVERLAP = 50;
+const CHARS_PER_TOKEN = 4; // rough approximation
+/**
+ * Split content into RAG-ready chunks with metadata.
+ */
+export function chunkContent(content, options = {}) {
+    const maxTokens = options.maxTokens || DEFAULT_MAX_TOKENS;
+    const overlap = options.overlap || DEFAULT_OVERLAP;
+    const strategy = options.strategy || 'section';
+    const opts = { maxTokens, overlap, strategy };
+    let chunks;
+    switch (strategy) {
+        case 'section':
+            chunks = chunkBySection(content, maxTokens, overlap);
+            break;
+        case 'paragraph':
+            chunks = chunkByParagraph(content, maxTokens, overlap);
+            break;
+        case 'fixed':
+            chunks = chunkByFixed(content, maxTokens, overlap);
+            break;
+        default:
+            chunks = chunkBySection(content, maxTokens, overlap);
+    }
+    return {
+        chunks,
+        totalChunks: chunks.length,
+        originalLength: content.length,
+        strategy,
+        options: opts,
+    };
+}
+/**
+ * Section-based chunking (recommended for RAG).
+ * Splits on markdown headings (## / ### etc.), then splits large sections by paragraph.
+ * Each chunk includes its section heading for context.
+ */
+function chunkBySection(content, maxTokens, overlap) {
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = overlap * CHARS_PER_TOKEN;
+    const chunks = [];
+    // Split content into sections by headings
+    const sections = splitByHeadings(content);
+    let chunkIndex = 0;
+    for (const section of sections) {
+        const { heading, depth, body, startOffset } = section;
+        if (!body.trim())
+            continue;
+        // If section fits in one chunk, use it directly
+        if (body.length <= maxChars) {
+            const text = heading ? `${heading}\n\n${body.trim()}` : body.trim();
+            chunks.push({
+                index: chunkIndex++,
+                text,
+                tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+                wordCount: text.split(/\s+/).filter(Boolean).length,
+                section: heading ? heading.replace(/^#+\s*/, '') : null,
+                sectionDepth: depth,
+                startOffset,
+                endOffset: startOffset + body.length,
+            });
+        }
+        else {
+            // Large section — split by paragraphs with overlap
+            const paragraphs = body.split(/\n\n+/).filter(p => p.trim());
+            let currentText = '';
+            let currentStart = startOffset;
+            for (const para of paragraphs) {
+                const candidate = currentText ? `${currentText}\n\n${para}` : para;
+                if (candidate.length > maxChars && currentText) {
+                    // Emit current chunk
+                    const text = heading ? `${heading}\n\n${currentText.trim()}` : currentText.trim();
+                    chunks.push({
+                        index: chunkIndex++,
+                        text,
+                        tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+                        wordCount: text.split(/\s+/).filter(Boolean).length,
+                        section: heading ? heading.replace(/^#+\s*/, '') : null,
+                        sectionDepth: depth,
+                        startOffset: currentStart,
+                        endOffset: currentStart + currentText.length,
+                    });
+                    // Start new chunk with overlap from end of previous
+                    if (overlapChars > 0 && currentText.length > overlapChars) {
+                        currentText = currentText.slice(-overlapChars) + '\n\n' + para;
+                    }
+                    else {
+                        currentText = para;
+                    }
+                    currentStart = startOffset + body.indexOf(para);
+                }
+                else {
+                    currentText = candidate;
+                }
+            }
+            // Emit remaining
+            if (currentText.trim()) {
+                const text = heading ? `${heading}\n\n${currentText.trim()}` : currentText.trim();
+                chunks.push({
+                    index: chunkIndex++,
+                    text,
+                    tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+                    wordCount: text.split(/\s+/).filter(Boolean).length,
+                    section: heading ? heading.replace(/^#+\s*/, '') : null,
+                    sectionDepth: depth,
+                    startOffset: currentStart,
+                    endOffset: currentStart + currentText.length,
+                });
+            }
+        }
+    }
+    return chunks;
+}
+/**
+ * Paragraph-based chunking.
+ * Groups paragraphs together up to maxTokens, with overlap.
+ */
+function chunkByParagraph(content, maxTokens, overlap) {
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = overlap * CHARS_PER_TOKEN;
+    const chunks = [];
+    const paragraphs = content.split(/\n\n+/).filter(p => p.trim());
+    let currentText = '';
+    let currentStart = 0;
+    let chunkIndex = 0;
+    // Track current section heading
+    let currentHeading = null;
+    let currentDepth = null;
+    for (const para of paragraphs) {
+        // Check if paragraph is a heading
+        const headingMatch = para.match(/^(#{1,6})\s+(.+)/);
+        if (headingMatch) {
+            currentHeading = headingMatch[2];
+            currentDepth = headingMatch[1].length;
+        }
+        const candidate = currentText ? `${currentText}\n\n${para}` : para;
+        if (candidate.length > maxChars && currentText) {
+            chunks.push({
+                index: chunkIndex++,
+                text: currentText.trim(),
+                tokenCount: Math.ceil(currentText.length / CHARS_PER_TOKEN),
+                wordCount: currentText.split(/\s+/).filter(Boolean).length,
+                section: currentHeading,
+                sectionDepth: currentDepth,
+                startOffset: currentStart,
+                endOffset: currentStart + currentText.length,
+            });
+            if (overlapChars > 0 && currentText.length > overlapChars) {
+                currentText = currentText.slice(-overlapChars) + '\n\n' + para;
+            }
+            else {
+                currentText = para;
+            }
+            currentStart = content.indexOf(para, currentStart);
+        }
+        else {
+            currentText = candidate;
+        }
+    }
+    if (currentText.trim()) {
+        chunks.push({
+            index: chunkIndex++,
+            text: currentText.trim(),
+            tokenCount: Math.ceil(currentText.length / CHARS_PER_TOKEN),
+            wordCount: currentText.split(/\s+/).filter(Boolean).length,
+            section: currentHeading,
+            sectionDepth: currentDepth,
+            startOffset: currentStart,
+            endOffset: currentStart + currentText.length,
+        });
+    }
+    return chunks;
+}
+/**
+ * Fixed-size chunking with overlap.
+ * Simple character-based splitting for predictable chunk sizes.
+ */
+function chunkByFixed(content, maxTokens, overlap) {
+    const maxChars = maxTokens * CHARS_PER_TOKEN;
+    const overlapChars = overlap * CHARS_PER_TOKEN;
+    const step = Math.max(maxChars - overlapChars, 100);
+    const chunks = [];
+    let chunkIndex = 0;
+    for (let i = 0; i < content.length; i += step) {
+        const text = content.slice(i, i + maxChars).trim();
+        if (!text)
+            continue;
+        // Try to find section heading within this chunk
+        const headingMatch = text.match(/^(#{1,6})\s+(.+)/m);
+        chunks.push({
+            index: chunkIndex++,
+            text,
+            tokenCount: Math.ceil(text.length / CHARS_PER_TOKEN),
+            wordCount: text.split(/\s+/).filter(Boolean).length,
+            section: headingMatch ? headingMatch[2] : null,
+            sectionDepth: headingMatch ? headingMatch[1].length : null,
+            startOffset: i,
+            endOffset: Math.min(i + maxChars, content.length),
+        });
+    }
+    return chunks;
+}
+/** Split content into sections based on markdown headings */
+function splitByHeadings(content) {
+    const lines = content.split('\n');
+    const sections = [];
+    let currentHeading = null;
+    let currentDepth = null;
+    let currentBody = [];
+    let currentStart = 0;
+    let offset = 0;
+    for (const line of lines) {
+        const headingMatch = line.match(/^(#{1,6})\s+(.+)/);
+        if (headingMatch) {
+            // Save previous section
+            if (currentBody.length > 0 || currentHeading) {
+                sections.push({
+                    heading: currentHeading,
+                    depth: currentDepth,
+                    body: currentBody.join('\n'),
+                    startOffset: currentStart,
+                });
+            }
+            currentHeading = line;
+            currentDepth = headingMatch[1].length;
+            currentBody = [];
+            currentStart = offset;
+        }
+        else {
+            currentBody.push(line);
+        }
+        offset += line.length + 1; // +1 for newline
+    }
+    // Don't forget last section
+    if (currentBody.length > 0 || currentHeading) {
+        sections.push({
+            heading: currentHeading,
+            depth: currentDepth,
+            body: currentBody.join('\n'),
+            startOffset: currentStart,
+        });
+    }
+    return sections;
+}

package/dist/core/chunking.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Smart content chunking for LLM processing.
+ *
+ * Splits content into manageable pieces with configurable overlap and strategy.
+ * Zero external dependencies; target <1ms for typical page content.
+ */
+export interface ChunkOptions {
+    /** Target tokens per chunk. Default: 4000 */
+    chunkSize?: number;
+    /** Overlap tokens between chunks. Default: 200 */
+    overlap?: number;
+    /** Chunking strategy. Default: 'semantic' */
+    strategy?: 'fixed' | 'semantic' | 'paragraph';
+}
+export interface Chunk {
+    /** Chunk index (0-based) */
+    index: number;
+    /** Chunk content */
+    content: string;
+    /** Estimated tokens in this chunk */
+    tokens: number;
+    /** Character offset in original content */
+    startOffset: number;
+    /** Whether this is the last chunk */
+    isLast: boolean;
+}
+export interface ChunkResult {
+    /** Array of content chunks */
+    chunks: Chunk[];
+    /** Total chunks */
+    totalChunks: number;
+    /** Total tokens across all chunks */
+    totalTokens: number;
+    /** Original content tokens */
+    originalTokens: number;
+}
+/** Estimate token count using chars/4 heuristic. Accurate within ±10%. */
+export declare function estimateTokens(text: string): number;
+/**
+ * Split content into chunks suitable for LLM processing.
+ */
+export declare function chunkContent(content: string, options?: ChunkOptions): ChunkResult;