@memvid/maw 1.0.0

package/dist/src/ingestor/index.js

@@ -0,0 +1,471 @@
+/**
+ * MV2 ingestor - saves crawled content to .mv2 files
+ */
+import { existsSync } from 'fs';
+import { stat } from 'fs/promises';
+import { createLogger } from '../utils/logger.js';
+const log = createLogger();
+// Dynamic import for @memvid/sdk
+let sdkModule = null;
+async function getSDK() {
+    if (!sdkModule) {
+        sdkModule = await import('@memvid/sdk');
+    }
+    return sdkModule;
+}
+/**
+ * Ingest crawled results into an MV2 file
+ */
+export async function ingestToMv2(results, options) {
+    const sdk = await getSDK();
+    const { create, use, configure, createMemory } = sdk;
+    const startTime = Date.now();
+    const apiKey = process.env.MEMVID_API_KEY;
+    const dashboardUrl = process.env.MEMVID_DASHBOARD_URL || 'https://memvid.com';
+    let memoryId = options.memoryId;
+    // Configure SDK if API key is set
+    if (apiKey) {
+        configure({
+            apiKey,
+            dashboardUrl,
+        });
+        // Auto-create memory if API key is set but no memoryId provided
+        if (!memoryId && !existsSync(options.output)) {
+            try {
+                const memoryName = options.memoryName || `maw-${Date.now()}`;
+                log.dim(`  Creating cloud memory: ${memoryName}`);
+                const memory = await createMemory({
+                    name: memoryName,
+                    description: `Created by maw CLI`,
+                });
+                memoryId = memory.id;
+                log.dim(`  Memory ID: ${memoryId}`);
+            }
+            catch (err) {
+                // Failed to create memory, continue without cloud binding
+                log.warn(`  Could not create cloud memory: ${err.message}`);
+            }
+        }
+    }
+    // Create or open the MV2 file with optional cloud binding
+    let mem;
+    if (existsSync(options.output)) {
+        mem = await use('basic', options.output);
+        // Sync tickets if we have API key and memory ID
+        if (apiKey && memoryId) {
+            try {
+                await mem.syncTickets(memoryId, apiKey, dashboardUrl);
+            }
+            catch {
+                // Sync failed, continue with local
+            }
+        }
+    }
+    else {
+        // Create new file, optionally bound to cloud memory
+        const createOpts = {};
+        if (apiKey && memoryId) {
+            createOpts.memoryId = memoryId;
+            createOpts.memvidApiKey = apiKey;
+        }
+        mem = await create(options.output, 'basic', createOpts);
+    }
+    let pages = 0;
+    let totalBytes = 0;
+    let skippedDupes = 0;
+    let stoppedAtLimit = false;
+    let estimatedFileSize = 0;
+    // Smaller batch size to work around SDK putMany hang at ~30 docs
+    const batchSize = options.batchSize || 3;
+    const batch = [];
+    // Size limit (default 40MB to stay safely under 50MB free tier with buffer)
+    const maxSizeBytes = (options.maxSizeMB || 40) * 1024 * 1024;
+    const hasApiKey = !!process.env.MEMVID_API_KEY;
+    // Content fingerprints to skip near-duplicates
+    const contentHashes = new Set();
+    for await (const result of results) {
+        const markdown = result.extracted.markdown;
+        // Simple content fingerprint to skip near-duplicates (first 2000 chars)
+        const fingerprint = markdown.slice(0, 2000).replace(/\s+/g, ' ').trim();
+        if (contentHashes.has(fingerprint)) {
+            skippedDupes++;
+            continue;
+        }
+        contentHashes.add(fingerprint);
+        const doc = {
+            title: result.extracted.title,
+            label: options.label || 'web',
+            text: markdown,
+            metadata: {
+                url: result.url,
+                finalUrl: result.finalUrl,
+                description: result.extracted.description,
+                author: result.extracted.author,
+                publishedDate: result.extracted.publishedDate,
+                wordCount: result.extracted.wordCount,
+                crawlDepth: result.depth,
+                engine: result.engine,
+                crawledAt: new Date().toISOString(),
+            },
+        };
+        // Estimate document size (text + metadata + embeddings overhead ~3x)
+        const docSize = markdown.length * 3;
+        // Check if we'd exceed the limit (only if no API key)
+        if (!hasApiKey && estimatedFileSize + docSize > maxSizeBytes) {
+            stoppedAtLimit = true;
+            log.warn(`  Reached ~${Math.round(estimatedFileSize / 1024 / 1024)}MB limit. Set MEMVID_API_KEY for unlimited.`);
+            break;
+        }
+        batch.push(doc);
+        pages++;
+        totalBytes += result.extracted.byteSize;
+        estimatedFileSize += docSize;
+        // Show progress
+        log.progress(pages, pages, result.extracted.title.slice(0, 40));
+        // Flush batch
+        if (batch.length >= batchSize) {
+            try {
+                if (options.enableEmbedding) {
+                    log.status(`Embedding batch ${Math.ceil(pages / batchSize)} (${options.embeddingModel || 'bge-small'})...`);
+                }
+                await mem.putMany(batch, options.enableEmbedding ? {
+                    enableEmbedding: true,
+                    embeddingModel: options.embeddingModel || 'openai',
+                } : undefined);
+                batch.length = 0;
+                // Check actual file size after flush (only if no API key)
+                if (!hasApiKey) {
+                    const currentSize = await getFileSize(options.output);
+                    if (currentSize > maxSizeBytes) {
+                        stoppedAtLimit = true;
+                        break;
+                    }
+                }
+            }
+            catch (err) {
+                // Handle SDK size limit error gracefully
+                if (err.message?.includes('exceeds') && err.message?.includes('limit')) {
+                    stoppedAtLimit = true;
+                    break;
+                }
+                throw err;
+            }
+        }
+    }
+    // Flush remaining (with error handling)
+    if (batch.length > 0 && !stoppedAtLimit) {
+        try {
+            if (options.enableEmbedding) {
+                log.status(`Embedding final batch (${options.embeddingModel || 'bge-small'})...`);
+            }
+            await mem.putMany(batch, options.enableEmbedding ? {
+                enableEmbedding: true,
+                embeddingModel: options.embeddingModel || 'openai',
+            } : undefined);
+        }
+        catch (err) {
+            if (err.message?.includes('exceeds') && err.message?.includes('limit')) {
+                stoppedAtLimit = true;
+            }
+            else {
+                throw err;
+            }
+        }
+    }
+    const duration = Date.now() - startTime;
+    return {
+        pages,
+        bytes: totalBytes,
+        duration,
+        stoppedAtLimit,
+        skippedDupes,
+        memoryId,
+    };
+}
+/**
+ * Ingest git repo files into an MV2 file
+ */
+export async function ingestGitToMv2(files, options) {
+    const sdk = await getSDK();
+    const { create, use, configure, createMemory } = sdk;
+    const startTime = Date.now();
+    const apiKey = process.env.MEMVID_API_KEY;
+    const dashboardUrl = process.env.MEMVID_DASHBOARD_URL || 'https://memvid.com';
+    let memoryId = options.memoryId;
+    // Configure SDK if API key is set
+    if (apiKey) {
+        configure({ apiKey, dashboardUrl });
+        // Auto-create memory if API key is set but no memoryId provided
+        if (!memoryId && !existsSync(options.output)) {
+            try {
+                const memoryName = options.memoryName || `maw-repo-${Date.now()}`;
+                log.dim(`  Creating cloud memory: ${memoryName}`);
+                const memory = await createMemory({
+                    name: memoryName,
+                    description: `Git repo ingested by maw CLI`,
+                });
+                memoryId = memory.id;
+                log.dim(`  Memory ID: ${memoryId}`);
+            }
+            catch (err) {
+                log.warn(`  Could not create cloud memory: ${err.message}`);
+            }
+        }
+    }
+    // Create or open the MV2 file
+    let mem;
+    if (existsSync(options.output)) {
+        mem = await use('basic', options.output);
+        if (apiKey && memoryId) {
+            try {
+                await mem.syncTickets(memoryId, apiKey, dashboardUrl);
+            }
+            catch { }
+        }
+    }
+    else {
+        const createOpts = {};
+        if (apiKey && memoryId) {
+            createOpts.memoryId = memoryId;
+            createOpts.memvidApiKey = apiKey;
+        }
+        mem = await create(options.output, 'basic', createOpts);
+    }
+    let fileCount = 0;
+    let totalBytes = 0;
+    let stoppedAtLimit = false;
+    let estimatedFileSize = 0;
+    // Smaller batch size to work around SDK putMany hang
+    const batchSize = 3;
+    const batch = [];
+    // Size limit (default 40MB to stay safely under 50MB free tier)
+    const maxSizeBytes = (options.maxSizeMB || 40) * 1024 * 1024;
+    const hasApiKey = !!process.env.MEMVID_API_KEY;
+    // Embedding overhead: ~6KB per chunk for 1536-dim vectors (OpenAI), less for smaller models
+    const embeddingOverhead = options.enableEmbedding ? 6000 : 0;
+    for await (const file of files) {
+        // Check if this is a README or documentation file
+        const isReadme = /readme\.md$/i.test(file.path);
+        const isDocs = /^(docs|documentation)\//i.test(file.path) || /\.(md|mdx|rst)$/i.test(file.path);
+        // Build enhanced text for README files to improve retrieval
+        let text = `File: ${file.path}\nLanguage: ${file.language}\n\n${file.content}`;
+        // For README files, prepend searchable context
+        if (isReadme) {
+            const projectName = file.path.includes('/') ? '' : file.content.match(/^#\s+(.+)/m)?.[1] || '';
+            text = `Project Overview: ${projectName}\nThis is the main README documentation.\nIntroduction and description of the project.\n\n${text}`;
+        }
+        // Create document with code content
+        // Format: include file path as context, then the actual code
+        // This helps the LLM understand what file it's looking at
+        const doc = {
+            title: `${file.path} (${file.language})`,
+            label: options.label || 'code',
+            text,
+            uri: `file://${file.path}`, // Use file path as URI for scope filtering
+            metadata: {
+                path: file.path,
+                language: file.language,
+                size: file.size,
+                type: isReadme ? 'readme' : (isDocs ? 'docs' : 'code'),
+                isReadme,
+                ingestedAt: new Date().toISOString(),
+            },
+        };
+        // Add labels for better categorization
+        if (isReadme) {
+            doc.labels = ['README', 'Documentation', 'Overview', 'Introduction'];
+        }
+        else if (isDocs) {
+            doc.labels = ['Documentation'];
+        }
+        // Estimate document size (text + metadata + embedding vectors)
+        const docSize = text.length * 2 + embeddingOverhead;
+        // Check if we'd exceed the limit (only if no API key)
+        if (!hasApiKey && estimatedFileSize + docSize > maxSizeBytes) {
+            stoppedAtLimit = true;
+            log.warn(`  Reached ~${Math.round(estimatedFileSize / 1024 / 1024)}MB limit. Set MEMVID_API_KEY for unlimited.`);
+            break;
+        }
+        batch.push(doc);
+        fileCount++;
+        totalBytes += file.size;
+        estimatedFileSize += docSize;
+        // Show progress
+        log.progress(fileCount, fileCount, file.path.slice(-40));
+        // Flush batch
+        if (batch.length >= batchSize) {
+            try {
+                if (options.enableEmbedding) {
+                    log.status(`Embedding ${fileCount} files (${options.embeddingModel || 'bge-small'})...`);
+                }
+                await mem.putMany(batch, options.enableEmbedding ? {
+                    enableEmbedding: true,
+                    embeddingModel: options.embeddingModel || 'openai',
+                } : undefined);
+                batch.length = 0;
+                // Check actual file size after flush (only if no API key)
+                if (!hasApiKey) {
+                    const currentSize = await getFileSize(options.output);
+                    if (currentSize > maxSizeBytes) {
+                        stoppedAtLimit = true;
+                        log.warn(`  Reached ${Math.round(currentSize / 1024 / 1024)}MB limit. Set MEMVID_API_KEY for unlimited.`);
+                        break;
+                    }
+                }
+            }
+            catch (err) {
+                // Handle SDK size limit error gracefully
+                if (err.message?.includes('exceeds') && err.message?.includes('limit')) {
+                    stoppedAtLimit = true;
+                    break;
+                }
+                throw err;
+            }
+        }
+    }
+    // Flush remaining (with error handling)
+    if (batch.length > 0 && !stoppedAtLimit) {
+        try {
+            if (options.enableEmbedding) {
+                log.status(`Embedding final ${batch.length} files (${options.embeddingModel || 'bge-small'})...`);
+            }
+            await mem.putMany(batch, options.enableEmbedding ? {
+                enableEmbedding: true,
+                embeddingModel: options.embeddingModel || 'openai',
+            } : undefined);
+        }
+        catch (err) {
+            if (err.message?.includes('exceeds') && err.message?.includes('limit')) {
+                stoppedAtLimit = true;
+            }
+            else {
+                throw err;
+            }
+        }
+    }
+    const duration = Date.now() - startTime;
+    return {
+        files: fileCount,
+        bytes: totalBytes,
+        duration,
+        memoryId,
+        stoppedAtLimit,
+    };
+}
+/**
+ * Get file size
+ */
+export async function getFileSize(path) {
+    try {
+        const stats = await stat(path);
+        return stats.size;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Open an existing MV2 file for querying
+ */
+export async function openMv2(path) {
+    const { use } = await getSDK();
+    return use('basic', path);
+}
+/**
+ * Search in an MV2 file
+ * Uses semantic search when OPENAI_API_KEY is set
+ */
+export async function searchMv2(path, query, options = {}) {
+    const mem = await openMv2(path);
+    // Determine search mode: use semantic if we have an API key for query embeddings
+    const queryEmbeddingModel = options.embeddingModel || (process.env.OPENAI_API_KEY ? 'openai' : undefined);
+    const mode = queryEmbeddingModel ? 'auto' : 'lex';
+    return mem.find(query, {
+        k: options.k || 10,
+        mode,
+        queryEmbeddingModel,
+    });
+}
+/**
+ * Detect if question is asking about what something is/does
+ */
+function isOverviewQuestion(question) {
+    const lowerQ = question.toLowerCase();
+    return (/^what (is|does|are)\b/.test(lowerQ) ||
+        /^(explain|describe|tell me about)\b/.test(lowerQ) ||
+        /^how does .+ work/.test(lowerQ) ||
+        /overview|introduction|getting started/i.test(lowerQ));
+}
+/**
+ * Ask a question using an MV2 file
+ * Uses semantic search when embeddings are available and OPENAI_API_KEY is set
+ */
+export async function askMv2(path, question, options = {}) {
+    const mem = await openMv2(path);
+    // For overview questions, use higher k to get more diverse context including README chunks
+    const isOverview = isOverviewQuestion(question);
+    const effectiveK = options.k || (isOverview ? 15 : 8);
+    // Determine search mode: use semantic/auto if we have an API key for query embeddings
+    const queryEmbeddingModel = options.embeddingModel || (process.env.OPENAI_API_KEY ? 'openai' : undefined);
+    const mode = queryEmbeddingModel ? 'auto' : 'lex'; // auto = hybrid (semantic + lexical), lex = BM25 only
+    return mem.ask(question, {
+        model: options.model || 'gpt-4o-mini',
+        modelApiKey: options.apiKey || process.env.OPENAI_API_KEY,
+        k: effectiveK,
+        llmContextChars: isOverview ? 15000 : 8000, // More context for overview questions
+        mode,
+        queryEmbeddingModel,
+    });
+}
+/**
+ * List documents in an MV2 file
+ */
+export async function listMv2(path, options = {}) {
+    const mem = await openMv2(path);
+    // Use timeline or list method if available
+    if (typeof mem.timeline === 'function') {
+        return mem.timeline({ limit: options.limit || 100 });
+    }
+    // Fallback to find with empty query
+    return mem.find('', { k: options.limit || 100 });
+}
+/**
+ * Export documents from an MV2 file with full content
+ */
+export async function exportMv2(path, options = {}) {
+    const mem = await openMv2(path);
+    // Get frame list
+    const timeline = await mem.timeline({ limit: options.limit || 10000 });
+    const frames = timeline.frames || timeline;
+    // Get full content for each frame
+    const results = [];
+    for (const frame of frames) {
+        // Skip child frames (they're included in parent)
+        if (frame.child_frames && frame.child_frames.length > 0) {
+            // This is a parent frame - get its full content
+            try {
+                const content = await mem.view(frame.frame_id);
+                const uri = frame.uri || '';
+                const title = uri.replace('file://', '').replace(/^https?:\/\//, '') || `Frame ${frame.frame_id}`;
+                results.push({ title, uri, content });
+            }
+            catch {
+                // Frame might not exist, skip
+            }
+        }
+        else if (!frames.some((f) => f.child_frames?.includes(frame.frame_id))) {
+            // This is a standalone frame (not a child of another)
+            try {
+                const content = await mem.view(frame.frame_id);
+                const uri = frame.uri || '';
+                const title = uri.replace('file://', '').replace(/^https?:\/\//, '') || `Frame ${frame.frame_id}`;
+                results.push({ title, uri, content });
+            }
+            catch {
+                // Frame might not exist, skip
+            }
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=index.js.map

package/dist/src/ingestor/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/ingestor/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;AAE3B,iCAAiC;AACjC,IAAI,SAAS,GAAQ,IAAI,CAAC;AAE1B,KAAK,UAAU,MAAM;IACnB,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAsBD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAmC,EACnC,OAAsB;IAEtB,MAAM,GAAG,GAAG,MAAM,MAAM,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,oBAAoB,CAAC;IAE9E,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAEhC,kCAAkC;IAClC,IAAI,MAAM,EAAE,CAAC;QACX,SAAS,CAAC;YACR,MAAM;YACN,YAAY;SACb,CAAC,CAAC;QAEH,gEAAgE;QAChE,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC7D,GAAG,CAAC,GAAG,CAAC,4BAA4B,UAAU,EAAE,CAAC,CAAC;gBAClD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;oBAChC,IAAI,EAAE,UAAU;oBAChB,WAAW,EAAE,oBAAoB;iBAClC,CAAC,CAAC;gBACH,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;gBACrB,GAAG,CAAC,GAAG,CAAC,gBAAgB,QAAQ,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,0DAA0D;gBAC1D,GAAG,CAAC,IAAI,CAAC,oCAAoC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED,0DAA0D;IAC1D,IAAI,GAAG,CAAC;IACR,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/B,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,gDAAgD;QAChD,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC;gBACP,mCAAmC;YACrC,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,oDAAoD;QACpD,MAAM,UAAU,GAAQ,EAAE,CAAC;QAC3B,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;YACvB,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAC;YAC/B,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC;QACnC,CAAC;QACD,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,iEAAiE;IACjE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;IACzC,MAAM,KAAK,GAAyF,EAAE,CAAC;IAEvG,4EAA4E;IAC5E,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAC7D,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAE/C,+CAA+C;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAExC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC;QAE3C,wEAAwE;QACxE,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxE,IAAI,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,YAAY,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QACD,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAE/B,MAAM,GAAG,GAAG;YACV,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK;YAC7B,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK;YAC7B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE;gBACR,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW;gBACzC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM;gBAC/B,aAAa,EAAE,MAAM,CAAC,SAAS,CAAC,aAAa;gBAC7C,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS;gBACrC,UAAU,EAAE,MAAM,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC;SACF,CAAC;QAEF,qEAAqE;QACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QAEpC,sDAAsD;QACtD,IAAI,CAAC,SAAS,IAAI,iBAAiB,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7D,cAAc,GAAG,IAAI,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,GAAG,IAAI,CAAC,6CAA6C,CAAC,CAAC;YACjH,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,KAAK,EAAE,CAAC;QACR,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC;QACxC,iBAAiB,IAAI,OAAO,CAAC;QAE7B,gBAAgB;QAChB,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAEhE,cAAc;QACd,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC5B,GAAG,CAAC,MAAM,CAAC,mBAAmB,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,OAAO,CAAC,cAAc,IAAI,WAAW,MAAM,CAAC,CAAC;gBAC9G,CAAC;gBACD,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;oBACjD,eAAe,EAAE,IAAI;oBACrB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ;iBACnD,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACf,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;gBAEjB,0DAA0D;gBAC1D,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACtD,IAAI,WAAW,GAAG,YAAY,EAAE,CAAC;wBAC/B,cAAc,GAAG,IAAI,CAAC;wBACtB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,yCAAyC;gBACzC,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvE,cAAc,GAAG,IAAI,CAAC;oBACtB,MAAM;gBACR,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,GAAG,CAAC,MAAM,CAAC,0BAA0B,OAAO,CAAC,cAAc,IAAI,WAAW,MAAM,CAAC,CAAC;YACpF,CAAC;YACD,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;gBACjD,eAAe,EAAE,IAAI;gBACrB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ;aACnD,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,cAAc,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAExC,OAAO;QACL,KAAK;QACL,KAAK,EAAE,UAAU;QACjB,QAAQ;QACR,cAAc;QACd,YAAY;QACZ,QAAQ;KACT,CAAC;AACJ,CAAC;AAoBD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAuF,EACvF,OAAyB;IAEzB,MAAM,GAAG,GAAG,MAAM,MAAM,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,GAAG,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,oBAAoB,CAAC;IAE9E,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAEhC,kCAAkC;IAClC,IAAI,MAAM,EAAE,CAAC;QACX,SAAS,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAEpC,gEAAgE;QAChE,IAAI,CAAC,QAAQ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,YAAY,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;gBAClE,GAAG,CAAC,GAAG,CAAC,4BAA4B,UAAU,EAAE,CAAC,CAAC;gBAClD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;oBAChC,IAAI,EAAE,UAAU;oBAChB,WAAW,EAAE,8BAA8B;iBAC5C,CAAC,CAAC;gBACH,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC;gBACrB,GAAG,CAAC,GAAG,CAAC,gBAAgB,QAAQ,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,GAAG,CAAC,IAAI,CAAC,oCAAoC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,IAAI,GAAG,CAAC;IACR,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/B,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;YACxD,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,UAAU,GAAQ,EAAE,CAAC;QAC3B,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC;YACvB,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAC;YAC/B,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC;QACnC,CAAC;QACD,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,qDAAqD;IACrD,MAAM,SAAS,GAAG,CAAC,CAAC;IACpB,MAAM,KAAK,GAAyF,EAAE,CAAC;IAEvG,gEAAgE;IAChE,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IAC7D,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAE/C,4FAA4F;IAC5F,MAAM,iBAAiB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,kDAAkD;QAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEhG,4DAA4D;QAC5D,IAAI,IAAI,GAAG,SAAS,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,QAAQ,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;QAE/E,+CAA+C;QAC/C,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/F,IAAI,GAAG,qBAAqB,WAAW,6FAA6F,IAAI,EAAE,CAAC;QAC7I,CAAC;QAED,oCAAoC;QACpC,6DAA6D;QAC7D,0DAA0D;QAC1D,MAAM,GAAG,GAAQ;YACf,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,GAAG;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM;YAC9B,IAAI;YACJ,GAAG,EAAE,UAAU,IAAI,CAAC,IAAI,EAAE,EAAE,2CAA2C;YACvE,QAAQ,EAAE;gBACR,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBACtD,QAAQ;gBACR,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACrC;SACF,CAAC;QAEF,uCAAuC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,GAAG,CAAC,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;QACvE,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,GAAG,CAAC,MAAM,GAAG,CAAC,eAAe,CAAC,CAAC;QACjC,CAAC;QAED,+DAA+D;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,iBAAiB,CAAC;QAEpD,sDAAsD;QACtD,IAAI,CAAC,SAAS,IAAI,iBAAiB,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7D,cAAc,GAAG,IAAI,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,GAAG,IAAI,CAAC,6CAA6C,CAAC,CAAC;YACjH,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,SAAS,EAAE,CAAC;QACZ,UAAU,IAAI,IAAI,CAAC,IAAI,CAAC;QACxB,iBAAiB,IAAI,OAAO,CAAC;QAE7B,gBAAgB;QAChB,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEzD,cAAc;QACd,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC5B,GAAG,CAAC,MAAM,CAAC,aAAa,SAAS,WAAW,OAAO,CAAC,cAAc,IAAI,WAAW,MAAM,CAAC,CAAC;gBAC3F,CAAC;gBACD,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;oBACjD,eAAe,EAAE,IAAI;oBACrB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ;iBACnD,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACf,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;gBAEjB,0DAA0D;gBAC1D,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACtD,IAAI,WAAW,GAAG,YAAY,EAAE,CAAC;wBAC/B,cAAc,GAAG,IAAI,CAAC;wBACtB,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,6CAA6C,CAAC,CAAC;wBAC1G,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,yCAAyC;gBACzC,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACvE,cAAc,GAAG,IAAI,CAAC;oBACtB,MAAM;gBACR,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,GAAG,CAAC,MAAM,CAAC,mBAAmB,KAAK,CAAC,MAAM,WAAW,OAAO,CAAC,cAAc,IAAI,WAAW,MAAM,CAAC,CAAC;YACpG,CAAC;YACD,MAAM,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;gBACjD,eAAe,EAAE,IAAI;gBACrB,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,QAAQ;aACnD,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,cAAc,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAExC,OAAO;QACL,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,UAAU;QACjB,QAAQ;QACR,QAAQ;QACR,cAAc;KACf,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY;IAC5C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACxC,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,EAAE,CAAC;IAC/B,OAAO,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,KAAa,EACb,UAAmD,EAAE;IAErD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC,iFAAiF;IACjF,MAAM,mBAAmB,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1G,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;IAElD,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE;QAClB,IAAI;QACJ,mBAAmB;KACpB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACtC,OAAO,CACL,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC;QACpC,qCAAqC,CAAC,IAAI,CAAC,MAAM,CAAC;QAClD,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC;QAChC,wCAAwC,CAAC,IAAI,CAAC,MAAM,CAAC,CACtD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,IAAY,EACZ,QAAgB,EAChB,UAAoF,EAAE;IAEtF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC,2FAA2F;IAC3F,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtD,sFAAsF;IACtF,MAAM,mBAAmB,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1G,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,sDAAsD;IAEzG,OAAO,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE;QACvB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,aAAa;QACrC,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc;QACzD,CAAC,EAAE,UAAU;QACb,eAAe,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,sCAAsC;QAClF,IAAI;QACJ,mBAAmB;KACpB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAY,EACZ,UAA+C,EAAE;IAEjD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,2CAA2C;IAC3C,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QACvC,OAAO,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,oCAAoC;IACpC,OAAO,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,UAA8B,EAAE;IAEhC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAEhC,iBAAiB;IACjB,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC;IAE3C,kCAAkC;IAClC,MAAM,OAAO,GAA2D,EAAE,CAAC;IAE3E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,iDAAiD;QACjD,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,gDAAgD;YAChD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC/C,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,SAAS,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClG,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACP,8BAA8B;YAChC,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC9E,sDAAsD;YACtD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC/C,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,IAAI,SAAS,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAClG,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACP,8BAA8B;YAChC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/utils/dedup.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Smart duplicate detection for web crawling
+ * - Detects localized URLs (e.g., /en-us/, /de-de/, /fr-fr/)
+ * - Content similarity fingerprinting
+ * - URL path normalization
+ */
+export interface LocaleInfo {
+    hasLocale: boolean;
+    locale?: string;
+    language?: string;
+    country?: string;
+    canonicalPath: string;
+}
+/**
+ * Extract locale information from a URL
+ */
+export declare function extractLocale(url: string): LocaleInfo;
+/**
+ * Generate a content fingerprint for similarity detection
+ * Uses multiple techniques for robust matching
+ */
+export declare function generateFingerprint(text: string): string;
+/**
+ * Calculate similarity between two fingerprints (0-1)
+ */
+export declare function calculateSimilarity(fp1: string, fp2: string): number;
+/**
+ * Smart deduplication tracker
+ */
+export declare class DedupTracker {
+    private canonicalPaths;
+    private fingerprints;
+    stats: {
+        localeSkipped: number;
+        similarSkipped: number;
+        total: number;
+    };
+    private preferredLanguage;
+    constructor(preferredLanguage?: string);
+    /**
+     * Check if URL should be skipped (is duplicate)
+     * Returns reason if should skip, undefined if should crawl
+     */
+    shouldSkip(url: string): {
+        skip: boolean;
+        reason?: string;
+    };
+    /**
+     * Check content similarity against previously seen content
+     */
+    checkContentSimilarity(url: string, content: string, threshold?: number): {
+        skip: boolean;
+        reason?: string;
+    };
+    /**
+     * Get dedup statistics
+     */
+    getStats(): {
+        uniquePaths: number;
+        uniqueContent: number;
+        localeSkipped: number;
+        similarSkipped: number;
+        total: number;
+    };
+}
+//# sourceMappingURL=dedup.d.ts.map

package/dist/src/utils/dedup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../../src/utils/dedup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAmDH,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAsHrD;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAoBxD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAiBpE;AAED;;GAEG;AACH,qBAAa,YAAY;IAEvB,OAAO,CAAC,cAAc,CAAuD;IAG7E,OAAO,CAAC,YAAY,CAA6B;IAG1C,KAAK;;;;MAIV;IAGF,OAAO,CAAC,iBAAiB,CAAS;gBAEtB,iBAAiB,SAAO;IAIpC;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;IAqC3D;;OAEG;IACH,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,SAAO,GAAG;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;IAiB1G;;OAEG;IACH,QAAQ;;;;;;;CAOT"}