@memvid/maw 1.0.0

package/dist/src/utils/ui.js

@@ -0,0 +1,357 @@
+/**
+ * Beautiful terminal UI for maw
+ * Clean, minimal, production-ready
+ */
+import chalk from 'chalk';
+// Theme colors - refined palette
+const theme = {
+    primary: chalk.hex('#a78bfa'), // Soft violet
+    success: chalk.hex('#34d399'), // Mint green
+    warning: chalk.hex('#fbbf24'), // Golden
+    error: chalk.hex('#f87171'), // Soft red
+    muted: chalk.hex('#9ca3af'), // Gray
+    accent: chalk.hex('#60a5fa'), // Sky blue
+    info: chalk.hex('#38bdf8'), // Cyan
+    dim: chalk.dim,
+    bold: chalk.bold,
+    white: chalk.white,
+};
+// Icons
+const icons = {
+    success: '✓',
+    error: '✗',
+    warning: '⚠',
+    arrow: '→',
+    bullet: '•',
+    page: '◉',
+    folder: '◈',
+};
+/**
+ * Format bytes to human readable
+ */
+export function formatBytes(bytes) {
+    if (bytes < 1024)
+        return `${bytes} B`;
+    if (bytes < 1024 * 1024)
+        return `${(bytes / 1024).toFixed(1)} KB`;
+    if (bytes < 1024 * 1024 * 1024)
+        return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+    return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
+}
+/**
+ * Format duration to human readable
+ */
+export function formatDuration(ms) {
+    if (ms < 1000)
+        return `${ms}ms`;
+    if (ms < 60000)
+        return `${(ms / 1000).toFixed(1)}s`;
+    const mins = Math.floor(ms / 60000);
+    const secs = Math.floor((ms % 60000) / 1000);
+    return `${mins}m ${secs}s`;
+}
+/**
+ * Truncate URL for display
+ */
+function truncateUrl(url, maxLen = 60) {
+    if (url.length <= maxLen)
+        return url;
+    try {
+        const parsed = new URL(url);
+        const host = parsed.hostname;
+        const path = parsed.pathname;
+        // Show host + truncated path
+        const availableForPath = maxLen - host.length - 5;
+        if (availableForPath > 10) {
+            return `${host}/...${path.slice(-availableForPath)}`;
+        }
+        return url.slice(0, maxLen - 3) + '...';
+    }
+    catch {
+        return url.slice(0, maxLen - 3) + '...';
+    }
+}
+/**
+ * Create a smooth progress bar
+ */
+export function progressBar(current, total, width = 24) {
+    const ratio = Math.min(current / total, 1);
+    const filled = Math.round(ratio * width);
+    const empty = width - filled;
+    const filledBar = theme.primary('━'.repeat(filled));
+    const emptyBar = theme.muted('─'.repeat(empty));
+    return filledBar + emptyBar;
+}
+/**
+ * Clean header - no box, just styled text
+ */
+export function header(title, subtitle) {
+    const lines = [''];
+    lines.push(`  ${theme.primary.bold(title)}`);
+    if (subtitle) {
+        // Truncate long URLs
+        const display = truncateUrl(subtitle, 70);
+        lines.push(`  ${theme.muted(display)}`);
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Crawl progress line - clean and minimal
+ */
+export function crawlProgress(current, total, title, status) {
+    const bar = progressBar(current, Math.max(total, current), 20);
+    const count = theme.muted(`${current}/${total}`);
+    const titleTrunc = title.length > 50 ? title.slice(0, 47) + '...' : title;
+    return `\n  ${bar} ${count}\n  ${theme.white(titleTrunc)}`;
+}
+/**
+ * Success message - clean layout
+ */
+export function successMessage(output, size, pages, duration) {
+    const lines = [''];
+    // Success line
+    lines.push(`  ${theme.success(icons.success)} ${theme.success.bold(output)}`);
+    lines.push('');
+    // Stats in a clean row
+    const stats = [
+        `${theme.muted('Size')} ${theme.accent.bold(formatBytes(size))}`,
+        `${theme.muted('Pages')} ${theme.accent.bold(pages.toString())}`,
+        `${theme.muted('Time')} ${theme.accent.bold(formatDuration(duration))}`,
+    ].join('    ');
+    lines.push(`  ${stats}`);
+    return lines.join('\n');
+}
+/**
+ * Engine stats display
+ */
+export function engineStats(stats) {
+    const parts = [
+        stats.fetch > 0 ? theme.success(`fetch ${stats.fetch}`) : null,
+        stats.playwright > 0 ? theme.info(`browser ${stats.playwright}`) : null,
+        stats.rebrowser > 0 ? theme.warning(`stealth ${stats.rebrowser}`) : null,
+        stats.blocked > 0 ? theme.error(`blocked ${stats.blocked}`) : null,
+    ].filter(Boolean);
+    if (parts.length === 0)
+        return '';
+    return `\n  ${theme.muted('Engines:')} ${parts.join(theme.muted(' · '))}`;
+}
+/**
+ * Dedup stats display
+ */
+export function dedupStats(stats) {
+    if (stats.localeSkipped === 0 && stats.similarSkipped === 0)
+        return '';
+    const parts = [];
+    if (stats.localeSkipped > 0) {
+        parts.push(`${stats.localeSkipped} locales`);
+    }
+    if (stats.similarSkipped > 0) {
+        parts.push(`${stats.similarSkipped} similar`);
+    }
+    return `\n  ${theme.muted('Skipped:')} ${theme.info(parts.join(', '))}`;
+}
+/**
+ * Warning message for size limit
+ */
+export function limitWarning() {
+    const lines = [''];
+    lines.push(`  ${theme.warning(icons.warning)} ${theme.warning('Stopped at free tier limit (~50MB)')}`);
+    lines.push(`  ${theme.muted('Set MEMVID_API_KEY for unlimited crawling')}`);
+    lines.push(`  ${theme.muted('Get your key:')} ${theme.info('https://memvid.com/dashboard/api')}`);
+    return lines.join('\n');
+}
+/**
+ * Cloud sync message
+ */
+export function cloudSyncMessage(memoryId) {
+    const dashboardUrl = process.env.MEMVID_DASHBOARD_URL || 'https://memvid.com';
+    return `\n  ${theme.muted('Synced:')} ${theme.info(`${dashboardUrl}/memories/${memoryId}`)}`;
+}
+/**
+ * Error message
+ */
+export function errorMessage(message) {
+    return `\n  ${theme.error(icons.error)} ${theme.error(message)}\n`;
+}
+/**
+ * Maw banner - minimal and clean
+ */
+export function banner() {
+    return `
+  ${theme.primary.bold('maw')} ${theme.muted('— Feed the maw. It never forgets.')}
+`;
+}
+/**
+ * Preview results display
+ */
+export function previewResults(result) {
+    const lines = [''];
+    lines.push(`  ${theme.primary.bold(result.domain)}`);
+    lines.push(`  ${theme.muted(`${result.totalPages.toLocaleString()} pages in sitemap`)}`);
+    if (result.estimatedSize) {
+        lines.push(`  ${theme.muted('Est. size:')} ${theme.accent(result.estimatedSize)}`);
+    }
+    if (result.recentPages.length > 0) {
+        lines.push('');
+        lines.push(`  ${theme.muted('Recent:')}`);
+        for (const page of result.recentPages.slice(0, 8)) {
+            const date = page.lastmod ? theme.warning(page.lastmod) + ' ' : '';
+            const url = truncateUrl(page.url, 55);
+            lines.push(`  ${date}${theme.info(url)}`);
+        }
+        if (result.recentPages.length > 8) {
+            lines.push(`  ${theme.muted(`... +${result.recentPages.length - 8} more`)}`);
+        }
+    }
+    if (!result.hasSitemap) {
+        lines.push('');
+        lines.push(`  ${theme.muted('No sitemap. Will discover by crawling.')}`);
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Search results display
+ */
+export function searchResults(hits) {
+    const lines = [''];
+    if (hits.length === 0) {
+        lines.push(`  ${theme.muted('No results found.')}`);
+        return lines.join('\n');
+    }
+    // Deduplicate by base title (remove page info, keep highest score)
+    const seen = new Map();
+    for (const hit of hits) {
+        // Extract base title (remove " (page X/Y)" suffix)
+        const baseTitle = hit.title.replace(/\s*\(page \d+\/\d+\)\s*$/, '');
+        const existing = seen.get(baseTitle);
+        if (!existing || (hit.score || 0) > (existing.score || 0)) {
+            seen.set(baseTitle, { ...hit, title: baseTitle });
+        }
+    }
+    const uniqueHits = Array.from(seen.values());
+    for (let i = 0; i < uniqueHits.length; i++) {
+        const hit = uniqueHits[i];
+        // Normalize score: if > 1, it's probably raw similarity, cap at 99
+        let scoreDisplay = '';
+        if (hit.score !== undefined) {
+            const normalizedScore = hit.score > 1 ? Math.min(hit.score / 2, 0.99) : hit.score;
+            scoreDisplay = theme.accent(`${(normalizedScore * 100).toFixed(0)}%`) + ' ';
+        }
+        // Show full title (no truncation)
+        lines.push(`  ${theme.muted(`${i + 1}.`)} ${scoreDisplay}${theme.white(hit.title)}`);
+        if (hit.metadata?.url && !hit.metadata.url.startsWith('mv2://')) {
+            lines.push(`     ${theme.muted(hit.metadata.url)}`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+/**
+ * Ask result display
+ */
+export function askResult(answer, sources) {
+    const lines = [''];
+    // Wrap answer text nicely
+    const maxWidth = 75;
+    const words = answer.split(' ');
+    let currentLine = '  ';
+    for (const word of words) {
+        if (currentLine.length + word.length + 1 > maxWidth) {
+            lines.push(currentLine);
+            currentLine = '  ' + word;
+        }
+        else {
+            currentLine += (currentLine === '  ' ? '' : ' ') + word;
+        }
+    }
+    if (currentLine.trim()) {
+        lines.push(currentLine);
+    }
+    lines.push('');
+    if (sources && sources.length > 0) {
+        // Deduplicate sources by base title (remove page info)
+        const seen = new Set();
+        const uniqueSources = sources.filter(s => {
+            // Extract base title (remove " (page X/Y)" suffix)
+            let key = (s.title || s.url || '').replace(/\s*\(page \d+\/\d+\)\s*$/, '');
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+        lines.push(`  ${theme.muted('Sources:')}`);
+        for (const source of uniqueSources.slice(0, 3)) {
+            // Clean up title - remove page info for cleaner display
+            let display = (source.title || source.url || '').replace(/\s*\(page \d+\/\d+\)\s*$/, '');
+            lines.push(`  ${theme.muted('›')} ${theme.info(display)}`);
+        }
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * List documents display
+ */
+export function listDocuments(items) {
+    const lines = [''];
+    if (items.length === 0) {
+        lines.push(`  ${theme.muted('No documents found.')}`);
+        return lines.join('\n');
+    }
+    // Deduplicate by base title (remove page info)
+    const seen = new Map();
+    for (const item of items) {
+        // Extract base title (remove " (page X/Y)" suffix)
+        const baseTitle = (item.title || item.preview || '').replace(/\s*\(page \d+\/\d+\)\s*$/, '');
+        const key = baseTitle || item.url || '';
+        const existing = seen.get(key);
+        if (existing) {
+            existing.pages++;
+        }
+        else {
+            seen.set(key, { item: { ...item, title: baseTitle || item.title }, pages: 1 });
+        }
+    }
+    const uniqueItems = Array.from(seen.values());
+    lines.push(`  ${theme.muted(`${uniqueItems.length} document${uniqueItems.length !== 1 ? 's' : ''}`)}`);
+    lines.push('');
+    for (let i = 0; i < uniqueItems.length; i++) {
+        const { item, pages } = uniqueItems[i];
+        const title = item.title || item.preview || 'Untitled';
+        const pageInfo = pages > 1 ? theme.muted(` (${pages} pages)`) : '';
+        lines.push(`  ${theme.muted(`${i + 1}.`)} ${theme.white(title)}${pageInfo}`);
+        if (item.url && !item.url.startsWith('mv2://')) {
+            lines.push(`     ${theme.muted(item.url)}`);
+        }
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Stats panel for detailed view
+ */
+export function statsPanel(stats) {
+    const lines = [''];
+    for (const [key, value] of Object.entries(stats)) {
+        const val = typeof value === 'number' ? value.toLocaleString() : value;
+        lines.push(`  ${theme.muted(key)} ${theme.accent(val)}`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Summary table
+ */
+export function summaryTable(rows) {
+    const lines = [''];
+    for (const row of rows) {
+        const colorFn = row.color ? theme[row.color] : theme.info;
+        const value = typeof row.value === 'number' ? row.value.toLocaleString() : row.value;
+        lines.push(`  ${theme.muted(row.label.padEnd(20))} ${colorFn.bold(value)}`);
+    }
+    return lines.join('\n');
+}
+// Export theme for custom usage
+export { theme };
+//# sourceMappingURL=ui.js.map

package/dist/src/utils/ui.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ui.js","sourceRoot":"","sources":["../../../src/utils/ui.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,iCAAiC;AACjC,MAAM,KAAK,GAAG;IACZ,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAM,cAAc;IACjD,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAM,aAAa;IAChD,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAM,SAAS;IAC5C,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAQ,WAAW;IAC9C,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAQ,OAAO;IAC1C,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAO,WAAW;IAC9C,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAS,OAAO;IAC1C,GAAG,EAAE,KAAK,CAAC,GAAG;IACd,IAAI,EAAE,KAAK,CAAC,IAAI;IAChB,KAAK,EAAE,KAAK,CAAC,KAAK;CACnB,CAAC;AAEF,QAAQ;AACR,MAAM,KAAK,GAAG;IACZ,OAAO,EAAE,GAAG;IACZ,KAAK,EAAE,GAAG;IACV,OAAO,EAAE,GAAG;IACZ,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,GAAG;CACZ,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,GAAG,KAAK,IAAI,CAAC;IACtC,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAClE,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAClF,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,EAAU;IACvC,IAAI,EAAE,GAAG,IAAI;QAAE,OAAO,GAAG,EAAE,IAAI,CAAC;IAChC,IAAI,EAAE,GAAG,KAAK;QAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACpD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7C,OAAO,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW,EAAE,MAAM,GAAG,EAAE;IAC3C,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,6BAA6B;QAC7B,MAAM,gBAAgB,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAClD,IAAI,gBAAgB,GAAG,EAAE,EAAE,CAAC;YAC1B,OAAO,GAAG,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC1C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe,EAAE,KAAa,EAAE,KAAK,GAAG,EAAE;IACpE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IAE7B,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAEhD,OAAO,SAAS,GAAG,QAAQ,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,MAAM,CAAC,KAAa,EAAE,QAAiB;IACrD,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAE7C,IAAI,QAAQ,EAAE,CAAC;QACb,qBAAqB;QACrB,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,KAAa,EAAE,KAAa,EAAE,MAAe;IAC1F,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;IAE1E,OAAO,OAAO,GAAG,IAAI,KAAK,OAAO,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;AAC7D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,IAAY,EAAE,KAAa,EAAE,QAAgB;IAC1F,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,eAAe;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC9E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,uBAAuB;IACvB,MAAM,KAAK,GAAG;QACZ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,EAAE;QAChE,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE;QAChE,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAAE;KACxE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;IAEzB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,KAAgF;IAC1G,MAAM,KAAK,GAAG;QACZ,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QAC9D,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QACvE,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;QACxE,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;KACnE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAElB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClC,OAAO,OAAO,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,KAAuE;IAChG,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,IAAI,KAAK,CAAC,cAAc,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvE,MAAM,KAAK,GAAG,EAAE,CAAC;IACjB,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,aAAa,UAAU,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,cAAc,UAAU,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,OAAO,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY;IAC1B,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,oCAAoC,CAAC,EAAE,CAAC,CAAC;IACvG,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,2CAA2C,CAAC,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,EAAE,CAAC,CAAC;IAClG,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,oBAAoB,CAAC;IAC9E,OAAO,OAAO,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,aAAa,QAAQ,EAAE,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe;IAC1C,OAAO,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;AACrE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,MAAM;IACpB,OAAO;IACL,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,mCAAmC,CAAC;CAChF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAM9B;IACC,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,EAAE,mBAAmB,CAAC,EAAE,CAAC,CAAC;IAEzF,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAE1C,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,wCAAwC,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,IAA2E;IACvG,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;QACpD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,mEAAmE;IACnE,MAAM,IAAI,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,mDAAmD;QACnD,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,mEAAmE;QACnE,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,eAAe,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;YAClF,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QAC9E,CAAC;QAED,kCAAkC;QAClC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAErF,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChE,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,MAAc,EAAE,OAAiD;IACzF,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,0BAA0B;IAC1B,MAAM,QAAQ,GAAG,EAAE,CAAC;IACpB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,WAAW,GAAG,IAAI,CAAC;IAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxB,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,WAAW,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;QAC1D,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,uDAAuD;QACvD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;YACvC,mDAAmD;YACnD,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;YAC3E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC/C,wDAAwD;YACxD,IAAI,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;YACzF,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,KAAwF;IACpH,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,+CAA+C;IAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoD,CAAC;IACzE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,mDAAmD;QACnD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC;QAC7F,MAAM,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,IAAI,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9C,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC,MAAM,YAAY,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACvG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC;QACvD,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEnE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,QAAQ,EAAE,CAAC,CAAC;QAE7E,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,KAAsC;IAC/D,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACvE,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAwG;IACnI,MAAM,KAAK,GAAa,CAAC,EAAE,CAAC,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAC1D,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;QACrF,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,gCAAgC;AAChC,OAAO,EAAE,KAAK,EAAE,CAAC"}

package/dist/src/utils/url.d.ts

@@ -0,0 +1,21 @@
+/**
+ * URL utilities for maw
+ */
+/**
+ * Normalize a URL for deduplication
+ */
+export declare function normalizeUrl(url: string): string;
+/**
+ * Extract the base domain from a URL
+ */
+export declare function getBaseDomain(url: string): string;
+export declare function shouldSkipUrl(url: string): boolean;
+/**
+ * Resolve a relative URL against a base URL
+ */
+export declare function resolveUrl(href: string, baseUrl: string): string | null;
+/**
+ * Check if two URLs are on the same domain
+ */
+export declare function isSameDomain(url1: string, url2: string): boolean;
+//# sourceMappingURL=url.d.ts.map

package/dist/src/utils/url.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.d.ts","sourceRoot":"","sources":["../../../src/utils/url.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA2BhD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOjD;AAgBD,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAqBlD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAYvE;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAQhE"}

package/dist/src/utils/url.js

@@ -0,0 +1,107 @@
+/**
+ * URL utilities for maw
+ */
+/**
+ * Normalize a URL for deduplication
+ */
+export function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url.startsWith('http') ? url : `https://${url}`);
+        // Remove trailing slash
+        let path = parsed.pathname;
+        if (path.endsWith('/') && path !== '/') {
+            path = path.slice(0, -1);
+        }
+        // Remove common tracking params
+        parsed.searchParams.delete('utm_source');
+        parsed.searchParams.delete('utm_medium');
+        parsed.searchParams.delete('utm_campaign');
+        parsed.searchParams.delete('utm_content');
+        parsed.searchParams.delete('utm_term');
+        parsed.searchParams.delete('ref');
+        parsed.searchParams.delete('fbclid');
+        parsed.searchParams.delete('gclid');
+        // Remove hash
+        parsed.hash = '';
+        return `${parsed.origin}${path}${parsed.search}`;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Extract the base domain from a URL
+ */
+export function getBaseDomain(url) {
+    try {
+        const parsed = new URL(url.startsWith('http') ? url : `https://${url}`);
+        return parsed.hostname.replace(/^www\./, '');
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Check if a URL should be skipped (non-HTML resources)
+ */
+const SKIP_EXTENSIONS = new Set([
+    '.pdf', '.jpg', '.jpeg', '.png', '.gif', '.svg', '.ico', '.webp', '.avif',
+    '.css', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx',
+    '.woff', '.woff2', '.ttf', '.eot', '.otf',
+    '.mp4', '.mp3', '.wav', '.ogg', '.webm', '.avi', '.mov',
+    '.zip', '.tar', '.gz', '.rar', '.7z',
+    '.exe', '.dmg', '.pkg', '.deb', '.rpm',
+    '.json', '.xml', '.yaml', '.yml', '.toml',
+    '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
+]);
+export function shouldSkipUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const pathname = parsed.pathname.toLowerCase();
+        // Skip non-HTML extensions
+        for (const ext of SKIP_EXTENSIONS) {
+            if (pathname.endsWith(ext)) {
+                return true;
+            }
+        }
+        // Skip mailto and javascript links
+        if (url.startsWith('mailto:') || url.startsWith('javascript:') || url.startsWith('tel:')) {
+            return true;
+        }
+        return false;
+    }
+    catch {
+        return true;
+    }
+}
+/**
+ * Resolve a relative URL against a base URL
+ */
+export function resolveUrl(href, baseUrl) {
+    try {
+        if (href.startsWith('#'))
+            return null;
+        if (href.startsWith('mailto:') || href.startsWith('javascript:') || href.startsWith('tel:')) {
+            return null;
+        }
+        const base = new URL(baseUrl);
+        return new URL(href, base.origin).href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Check if two URLs are on the same domain
+ */
+export function isSameDomain(url1, url2) {
+    try {
+        const domain1 = getBaseDomain(url1);
+        const domain2 = getBaseDomain(url2);
+        return domain1 === domain2;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=url.js.map

package/dist/src/utils/url.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url.js","sourceRoot":"","sources":["../../../src/utils/url.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;QAExE,wBAAwB;QACxB,IAAI,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;QAED,gCAAgC;QAChC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACzC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACzC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC3C,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEpC,cAAc;QACd,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QAEjB,OAAO,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACzE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACzC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IACvD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;IACpC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACtC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IACzC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;CAClD,CAAC,CAAC;AAEH,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAE/C,2BAA2B;QAC3B,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;YAClC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACzF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe;IACtD,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5F,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,IAAY;IACrD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,KAAK,OAAO,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "@memvid/maw",
+  "version": "1.0.0",
+  "description": "Crawl any site. Search it forever.",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "maw": "dist/bin/maw.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/memvid/maw.git"
+  },
+  "homepage": "https://github.com/memvid/maw#readme",
+  "bugs": {
+    "url": "https://github.com/memvid/maw/issues"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx watch src/index.ts",
+    "test": "vitest",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "web-scraper",
+    "markdown",
+    "ai",
+    "memory",
+    "rag",
+    "offline",
+    "documentation",
+    "crawler"
+  ],
+  "author": "Memvid",
+  "license": "MIT",
+  "dependencies": {
+    "@memvid/sdk": "^2.0.148",
+    "@mozilla/readability": "^0.5.0",
+    "chalk": "^5.3.0",
+    "cheerio": "^1.0.0",
+    "cliui": "^9.0.1",
+    "commander": "^12.0.0",
+    "jsdom": "^24.0.0",
+    "node-html-markdown": "^1.3.0",
+    "p-queue": "^8.0.0",
+    "sitemapper": "^4.0.2"
+  },
+  "optionalDependencies": {
+    "playwright": "^1.47.0",
+    "rebrowser-playwright": "^1.47.0"
+  },
+  "devDependencies": {
+    "@types/jsdom": "^21.0.0",
+    "@types/node": "^20.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0",
+    "vitest": "^1.0.0"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}