@memvid/maw 1.0.0

package/dist/src/extractor/links.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Link extraction utilities
+ */
+import type { CheerioAPI } from 'cheerio';
+export interface ExtractedLink {
+    href: string;
+    text: string;
+    isInternal: boolean;
+}
+/**
+ * Extract all links from HTML
+ */
+export declare function extractLinks($: CheerioAPI, baseUrl: string): string[];
+/**
+ * Extract links with additional metadata
+ */
+export declare function extractLinksWithMeta($: CheerioAPI, baseUrl: string): ExtractedLink[];
+/**
+ * Extract only internal links (same domain)
+ */
+export declare function extractInternalLinks($: CheerioAPI, baseUrl: string): string[];
+//# sourceMappingURL=links.d.ts.map

package/dist/src/extractor/links.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"links.d.ts","sourceRoot":"","sources":["../../../src/extractor/links.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BrE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CAoCpF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAa7E"}

package/dist/src/extractor/links.js

@@ -0,0 +1,92 @@
+/**
+ * Link extraction utilities
+ */
+import { resolveUrl, shouldSkipUrl } from '../utils/url.js';
+/**
+ * Extract all links from HTML
+ */
+export function extractLinks($, baseUrl) {
+    const links = [];
+    const seen = new Set();
+    try {
+        const baseHost = new URL(baseUrl).hostname;
+        $('a[href]').each((_, el) => {
+            const href = $(el).attr('href');
+            if (!href)
+                return;
+            const resolved = resolveUrl(href, baseUrl);
+            if (!resolved)
+                return;
+            // Skip non-HTML resources
+            if (shouldSkipUrl(resolved))
+                return;
+            // Deduplicate
+            if (seen.has(resolved))
+                return;
+            seen.add(resolved);
+            links.push(resolved);
+        });
+    }
+    catch {
+        // Invalid base URL
+    }
+    return links;
+}
+/**
+ * Extract links with additional metadata
+ */
+export function extractLinksWithMeta($, baseUrl) {
+    const links = [];
+    const seen = new Set();
+    try {
+        const baseHost = new URL(baseUrl).hostname;
+        $('a[href]').each((_, el) => {
+            const $el = $(el);
+            const href = $el.attr('href');
+            if (!href)
+                return;
+            const resolved = resolveUrl(href, baseUrl);
+            if (!resolved)
+                return;
+            if (shouldSkipUrl(resolved))
+                return;
+            if (seen.has(resolved))
+                return;
+            seen.add(resolved);
+            let isInternal = false;
+            try {
+                isInternal = new URL(resolved).hostname === baseHost;
+            }
+            catch { }
+            links.push({
+                href: resolved,
+                text: $el.text().trim().slice(0, 100),
+                isInternal,
+            });
+        });
+    }
+    catch {
+        // Invalid base URL
+    }
+    return links;
+}
+/**
+ * Extract only internal links (same domain)
+ */
+export function extractInternalLinks($, baseUrl) {
+    try {
+        const baseHost = new URL(baseUrl).hostname;
+        return extractLinks($, baseUrl).filter(link => {
+            try {
+                return new URL(link).hostname === baseHost;
+            }
+            catch {
+                return false;
+            }
+        });
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=links.js.map

package/dist/src/extractor/links.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"links.js","sourceRoot":"","sources":["../../../src/extractor/links.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAQ5D;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,CAAa,EAAE,OAAe;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;QAE3C,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAEtB,0BAA0B;YAC1B,IAAI,aAAa,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEpC,cAAc;YACd,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,mBAAmB;IACrB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,CAAa,EAAE,OAAe;IACjE,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;QAE3C,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC1B,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAEtB,IAAI,aAAa,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAEpC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,OAAO;YAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEnB,IAAI,UAAU,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC;gBACH,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;YAEV,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACrC,UAAU;aACX,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,mBAAmB;IACrB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,CAAa,EAAE,OAAe;IACjE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;QAC3C,OAAO,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAC5C,IAAI,CAAC;gBACH,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;YAC7C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/src/extractor/markdown.d.ts

@@ -0,0 +1,13 @@
+/**
+ * HTML to Markdown conversion using node-html-markdown
+ * 1.57x faster than Turndown
+ */
+/**
+ * Convert HTML to Markdown
+ */
+export declare function htmlToMarkdown(html: string): string;
+/**
+ * Extract plain text from markdown (remove formatting)
+ */
+export declare function markdownToPlainText(markdown: string): string;
+//# sourceMappingURL=markdown.d.ts.map

package/dist/src/extractor/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/extractor/markdown.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYnD;AA4CD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmC5D"}

package/dist/src/extractor/markdown.js

@@ -0,0 +1,94 @@
+/**
+ * HTML to Markdown conversion using node-html-markdown
+ * 1.57x faster than Turndown
+ */
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+// Configure node-html-markdown for optimal output
+const nhm = new NodeHtmlMarkdown({
+    codeFence: '```',
+    bulletMarker: '-',
+    codeBlockStyle: 'fenced',
+    emDelimiter: '*',
+    strongDelimiter: '**',
+    maxConsecutiveNewlines: 2,
+    useLinkReferenceDefinitions: false,
+    useInlineLinks: true,
+});
+/**
+ * Convert HTML to Markdown
+ */
+export function htmlToMarkdown(html) {
+    if (!html || !html.trim()) {
+        return '';
+    }
+    try {
+        const markdown = nhm.translate(html);
+        return postProcess(markdown);
+    }
+    catch (error) {
+        // Fallback: strip HTML tags if conversion fails
+        return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+    }
+}
+/**
+ * Post-process markdown for cleaner output
+ */
+function postProcess(markdown) {
+    return markdown
+        // Normalize line endings
+        .replace(/\r\n/g, '\n')
+        // Max 2 consecutive newlines
+        .replace(/\n{3,}/g, '\n\n')
+        // Trim trailing whitespace on each line
+        .replace(/[ \t]+$/gm, '')
+        // Fix code block spacing
+        .replace(/```(\w*)\n\n+/g, '```$1\n')
+        .replace(/\n\n+```/g, '\n```')
+        // Remove empty code blocks
+        .replace(/```\w*\n\s*\n?```/g, '')
+        // Collapse multiple spaces (but not in code blocks)
+        .replace(/(?<!```)[ ]{2,}(?!```)/g, ' ')
+        // Remove leading/trailing empty lines in blockquotes
+        .replace(/^>\s*\n/gm, '> ')
+        .replace(/\n>\s*$/gm, '')
+        // Fix list formatting
+        .replace(/^(-|\*|\d+\.)\s+\n/gm, '')
+        // Remove excessive horizontal rules
+        .replace(/(---\n){2,}/g, '---\n')
+        // Clean up link formatting
+        .replace(/\[([^\]]*)\]\(\s+/g, '[$1](')
+        .replace(/\s+\)/g, ')')
+        // Final trim
+        .trim() + '\n';
+}
+/**
+ * Extract plain text from markdown (remove formatting)
+ */
+export function markdownToPlainText(markdown) {
+    return markdown
+        // Remove code blocks
+        .replace(/```[\s\S]*?```/g, '')
+        .replace(/`[^`]+`/g, '')
+        // Remove images
+        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '$1')
+        // Convert links to text
+        .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+        // Remove headers
+        .replace(/^#{1,6}\s+/gm, '')
+        // Remove emphasis
+        .replace(/\*\*([^*]+)\*\*/g, '$1')
+        .replace(/\*([^*]+)\*/g, '$1')
+        .replace(/__([^_]+)__/g, '$1')
+        .replace(/_([^_]+)_/g, '$1')
+        // Remove blockquotes
+        .replace(/^>\s*/gm, '')
+        // Remove horizontal rules
+        .replace(/^---$/gm, '')
+        // Remove list markers
+        .replace(/^[-*+]\s+/gm, '')
+        .replace(/^\d+\.\s+/gm, '')
+        // Collapse whitespace
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+//# sourceMappingURL=markdown.js.map

package/dist/src/extractor/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../../src/extractor/markdown.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,kDAAkD;AAClD,MAAM,GAAG,GAAG,IAAI,gBAAgB,CAAC;IAC/B,SAAS,EAAE,KAAK;IAChB,YAAY,EAAE,GAAG;IACjB,cAAc,EAAE,QAAQ;IACxB,WAAW,EAAE,GAAG;IAChB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,CAAC;IACzB,2BAA2B,EAAE,KAAK;IAClC,cAAc,EAAE,IAAI;CACrB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,gDAAgD;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,QAAgB;IACnC,OAAO,QAAQ;QACb,yBAAyB;SACxB,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;QAEvB,6BAA6B;SAC5B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;QAE3B,wCAAwC;SACvC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;QAEzB,yBAAyB;SACxB,OAAO,CAAC,gBAAgB,EAAE,SAAS,CAAC;SACpC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC;QAE9B,2BAA2B;SAC1B,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAElC,oDAAoD;SACnD,OAAO,CAAC,yBAAyB,EAAE,GAAG,CAAC;QAExC,qDAAqD;SACpD,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;QAEzB,sBAAsB;SACrB,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC;QAEpC,oCAAoC;SACnC,OAAO,CAAC,cAAc,EAAE,OAAO,CAAC;QAEjC,2BAA2B;SAC1B,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC;SACtC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;QAEvB,aAAa;SACZ,IAAI,EAAE,GAAG,IAAI,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,OAAO,QAAQ;QACb,qBAAqB;SACpB,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QAExB,gBAAgB;SACf,OAAO,CAAC,yBAAyB,EAAE,IAAI,CAAC;QAEzC,wBAAwB;SACvB,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC;QAExC,iBAAiB;SAChB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;QAE5B,kBAAkB;SACjB,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC;SACjC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;SAC7B,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;SAC7B,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;QAE5B,qBAAqB;SACpB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;QAEvB,0BAA0B;SACzB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;QAEvB,sBAAsB;SACrB,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;QAE3B,sBAAsB;SACrB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SAEpB,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/dist/src/git/index.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Git repository ingestion
+ * Clone and read files from git repos
+ */
+export interface GitFile {
+    path: string;
+    content: string;
+    language: string;
+    size: number;
+}
+export interface GitRepoInfo {
+    name: string;
+    url?: string;
+    localPath: string;
+    isTemp: boolean;
+}
+/**
+ * Check if a string is a git URL
+ */
+export declare function isGitUrl(input: string): boolean;
+/**
+ * Check if a path is a local git repo
+ */
+export declare function isLocalGitRepo(path: string): boolean;
+/**
+ * Clone a git repo to temp directory
+ */
+export declare function cloneRepo(url: string): Promise<GitRepoInfo>;
+/**
+ * Read all files from a git repo
+ */
+export declare function readGitRepo(input: string): AsyncGenerator<GitFile>;
+/**
+ * Get repo info without reading files
+ */
+export declare function getRepoInfo(input: string): Promise<{
+    name: string;
+    fileCount: number;
+}>;
+//# sourceMappingURL=index.d.ts.map

package/dist/src/git/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/git/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA4FH,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAS/C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAMpD;AAcD;;GAEG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAsBjE;AAsFD;;GAEG;AACH,wBAAuB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAwCzE;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CAyB7F"}

package/dist/src/git/index.js

@@ -0,0 +1,303 @@
+/**
+ * Git repository ingestion
+ * Clone and read files from git repos
+ */
+import { execSync } from 'child_process';
+import { existsSync, readdirSync, readFileSync, statSync, rmSync } from 'fs';
+import { join, extname, relative } from 'path';
+import { tmpdir } from 'os';
+import { createLogger } from '../utils/logger.js';
+const log = createLogger();
+// File extensions to include
+const CODE_EXTENSIONS = new Set([
+    // JavaScript/TypeScript
+    '.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs',
+    // Python
+    '.py', '.pyi',
+    // Rust
+    '.rs',
+    // Go
+    '.go',
+    // Java/Kotlin
+    '.java', '.kt', '.kts',
+    // C/C++
+    '.c', '.h', '.cpp', '.hpp', '.cc', '.cxx',
+    // C#
+    '.cs',
+    // Ruby
+    '.rb',
+    // PHP
+    '.php',
+    // Swift
+    '.swift',
+    // Shell
+    '.sh', '.bash', '.zsh',
+    // Web
+    '.html', '.css', '.scss', '.sass', '.less', '.vue', '.svelte',
+    // Data/Config
+    '.json', '.yaml', '.yml', '.toml', '.xml', '.ini', '.env.example',
+    // Documentation
+    '.md', '.mdx', '.rst', '.txt', '.adoc',
+    // SQL
+    '.sql',
+    // GraphQL
+    '.graphql', '.gql',
+    // Solidity
+    '.sol',
+]);
+// Directories to skip
+const SKIP_DIRS = new Set([
+    'node_modules',
+    '.git',
+    '.svn',
+    '.hg',
+    'vendor',
+    'target',
+    'dist',
+    'build',
+    'out',
+    '.next',
+    '.nuxt',
+    '__pycache__',
+    '.pytest_cache',
+    '.mypy_cache',
+    'venv',
+    '.venv',
+    'env',
+    '.tox',
+    'coverage',
+    '.coverage',
+    '.nyc_output',
+    'logs',
+    'tmp',
+    '.cache',
+    '.parcel-cache',
+    '.turbo',
+]);
+// Files to skip
+const SKIP_FILES = new Set([
+    'package-lock.json',
+    'yarn.lock',
+    'pnpm-lock.yaml',
+    'Cargo.lock',
+    'Gemfile.lock',
+    'poetry.lock',
+    'composer.lock',
+]);
+// Max file size to read (500KB)
+const MAX_FILE_SIZE = 500 * 1024;
+/**
+ * Check if a string is a git URL
+ */
+export function isGitUrl(input) {
+    return (input.startsWith('https://github.com/') ||
+        input.startsWith('https://gitlab.com/') ||
+        input.startsWith('https://bitbucket.org/') ||
+        input.startsWith('git@github.com:') ||
+        input.startsWith('git@gitlab.com:') ||
+        input.endsWith('.git'));
+}
+/**
+ * Check if a path is a local git repo
+ */
+export function isLocalGitRepo(path) {
+    try {
+        return existsSync(join(path, '.git'));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Extract repo name from URL or path
+ */
+function getRepoName(input) {
+    // GitHub URL: https://github.com/user/repo or https://github.com/user/repo.git
+    const match = input.match(/\/([^/]+?)(\.git)?$/);
+    if (match)
+        return match[1];
+    // Local path
+    return input.split('/').pop() || 'repo';
+}
+/**
+ * Clone a git repo to temp directory
+ */
+export async function cloneRepo(url) {
+    const repoName = getRepoName(url);
+    const tempPath = join(tmpdir(), `maw-${repoName}-${Date.now()}`);
+    log.info(`  Cloning ${repoName}...`);
+    try {
+        // Shallow clone for speed
+        execSync(`git clone --depth 1 "${url}" "${tempPath}"`, {
+            stdio: 'pipe',
+            timeout: 60000, // 1 minute timeout
+        });
+        return {
+            name: repoName,
+            url,
+            localPath: tempPath,
+            isTemp: true,
+        };
+    }
+    catch (error) {
+        throw new Error(`Failed to clone ${url}: ${error.message}`);
+    }
+}
+/**
+ * Get language from file extension
+ */
+function getLanguage(ext) {
+    const langMap = {
+        '.js': 'javascript',
+        '.jsx': 'javascript',
+        '.ts': 'typescript',
+        '.tsx': 'typescript',
+        '.py': 'python',
+        '.rs': 'rust',
+        '.go': 'go',
+        '.java': 'java',
+        '.kt': 'kotlin',
+        '.c': 'c',
+        '.cpp': 'cpp',
+        '.cs': 'csharp',
+        '.rb': 'ruby',
+        '.php': 'php',
+        '.swift': 'swift',
+        '.sh': 'shell',
+        '.md': 'markdown',
+        '.json': 'json',
+        '.yaml': 'yaml',
+        '.yml': 'yaml',
+        '.sql': 'sql',
+        '.html': 'html',
+        '.css': 'css',
+        '.vue': 'vue',
+        '.svelte': 'svelte',
+    };
+    return langMap[ext] || ext.slice(1) || 'text';
+}
+/**
+ * Walk directory and collect files
+ */
+function* walkDir(dir, basePath = dir) {
+    const entries = readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = join(dir, entry.name);
+        const relativePath = relative(basePath, fullPath);
+        if (entry.isDirectory()) {
+            // Skip ignored directories
+            if (SKIP_DIRS.has(entry.name) || entry.name.startsWith('.')) {
+                continue;
+            }
+            yield* walkDir(fullPath, basePath);
+        }
+        else if (entry.isFile()) {
+            // Skip ignored files
+            if (SKIP_FILES.has(entry.name))
+                continue;
+            const ext = extname(entry.name).toLowerCase();
+            // Only include known extensions
+            if (!CODE_EXTENSIONS.has(ext))
+                continue;
+            // Check file size
+            const stats = statSync(fullPath);
+            if (stats.size > MAX_FILE_SIZE)
+                continue;
+            if (stats.size === 0)
+                continue;
+            try {
+                const content = readFileSync(fullPath, 'utf-8');
+                // Skip binary files (files with null bytes)
+                if (content.includes('\0'))
+                    continue;
+                yield {
+                    path: relativePath,
+                    content,
+                    language: getLanguage(ext),
+                    size: stats.size,
+                };
+            }
+            catch {
+                // Skip files that can't be read
+                continue;
+            }
+        }
+    }
+}
+/**
+ * Read all files from a git repo
+ */
+export async function* readGitRepo(input) {
+    let repoInfo;
+    if (isGitUrl(input)) {
+        repoInfo = await cloneRepo(input);
+    }
+    else if (isLocalGitRepo(input)) {
+        repoInfo = {
+            name: getRepoName(input),
+            localPath: input,
+            isTemp: false,
+        };
+    }
+    else if (existsSync(input)) {
+        // Treat as local directory even if not a git repo
+        repoInfo = {
+            name: getRepoName(input),
+            localPath: input,
+            isTemp: false,
+        };
+    }
+    else {
+        throw new Error(`Not a valid git URL or local path: ${input}`);
+    }
+    try {
+        let fileCount = 0;
+        for (const file of walkDir(repoInfo.localPath)) {
+            fileCount++;
+            yield file;
+        }
+        log.info(`  Found ${fileCount} files`);
+    }
+    finally {
+        // Clean up temp directory
+        if (repoInfo.isTemp) {
+            try {
+                rmSync(repoInfo.localPath, { recursive: true, force: true });
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+        }
+    }
+}
+/**
+ * Get repo info without reading files
+ */
+export async function getRepoInfo(input) {
+    let localPath;
+    let isTemp = false;
+    if (isGitUrl(input)) {
+        const info = await cloneRepo(input);
+        localPath = info.localPath;
+        isTemp = true;
+    }
+    else {
+        localPath = input;
+    }
+    try {
+        let count = 0;
+        for (const _ of walkDir(localPath)) {
+            count++;
+        }
+        return { name: getRepoName(input), fileCount: count };
+    }
+    finally {
+        if (isTemp) {
+            try {
+                rmSync(localPath, { recursive: true, force: true });
+            }
+            catch { }
+        }
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/src/git/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/git/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;AAE3B,6BAA6B;AAC7B,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,wBAAwB;IACxB,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC5C,SAAS;IACT,KAAK,EAAE,MAAM;IACb,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,cAAc;IACd,OAAO,EAAE,KAAK,EAAE,MAAM;IACtB,QAAQ;IACR,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IACzC,KAAK;IACL,KAAK;IACL,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,KAAK,EAAE,OAAO,EAAE,MAAM;IACtB,MAAM;IACN,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAC7D,cAAc;IACd,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc;IACjE,gBAAgB;IAChB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACtC,MAAM;IACN,MAAM;IACN,UAAU;IACV,UAAU,EAAE,MAAM;IAClB,WAAW;IACX,MAAM;CACP,CAAC,CAAC;AAEH,sBAAsB;AACtB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,cAAc;IACd,MAAM;IACN,MAAM;IACN,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,OAAO;IACP,KAAK;IACL,OAAO;IACP,OAAO;IACP,aAAa;IACb,eAAe;IACf,aAAa;IACb,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,UAAU;IACV,WAAW;IACX,aAAa;IACb,MAAM;IACN,KAAK;IACL,QAAQ;IACR,eAAe;IACf,QAAQ;CACT,CAAC,CAAC;AAEH,gBAAgB;AAChB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,mBAAmB;IACnB,WAAW;IACX,gBAAgB;IAChB,YAAY;IACZ,cAAc;IACd,aAAa;IACb,eAAe;CAChB,CAAC,CAAC;AAEH,gCAAgC;AAChC,MAAM,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC;AAgBjC;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,OAAO,CACL,KAAK,CAAC,UAAU,CAAC,qBAAqB,CAAC;QACvC,KAAK,CAAC,UAAU,CAAC,qBAAqB,CAAC;QACvC,KAAK,CAAC,UAAU,CAAC,wBAAwB,CAAC;QAC1C,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC;QACnC,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC;QACnC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CACvB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC;QACH,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa;IAChC,+EAA+E;IAC/E,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACjD,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;IAE3B,aAAa;IACb,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW;IACzC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,OAAO,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAEjE,GAAG,CAAC,IAAI,CAAC,aAAa,QAAQ,KAAK,CAAC,CAAC;IAErC,IAAI,CAAC;QACH,0BAA0B;QAC1B,QAAQ,CAAC,wBAAwB,GAAG,MAAM,QAAQ,GAAG,EAAE;YACrD,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,KAAK,EAAE,mBAAmB;SACpC,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,GAAG;YACH,SAAS,EAAE,QAAQ;YACnB,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,OAAO,GAA2B;QACtC,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,YAAY;QACpB,KAAK,EAAE,QAAQ;QACf,KAAK,EAAE,MAAM;QACb,KAAK,EAAE,IAAI;QACX,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,QAAQ;QACf,IAAI,EAAE,GAAG;QACT,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,QAAQ;QACf,KAAK,EAAE,MAAM;QACb,MAAM,EAAE,KAAK;QACb,QAAQ,EAAE,OAAO;QACjB,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,MAAM;QACf,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,QAAQ;KACpB,CAAC;IACF,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,QAAQ,CAAC,CAAC,OAAO,CAAC,GAAW,EAAE,WAAmB,GAAG;IACnD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAE1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAElD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,2BAA2B;YAC3B,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5D,SAAS;YACX,CAAC;YACD,KAAK,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1B,qBAAqB;YACrB,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEzC,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAE9C,gCAAgC;YAChC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAExC,kBAAkB;YAClB,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,KAAK,CAAC,IAAI,GAAG,aAAa;gBAAE,SAAS;YACzC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;gBAAE,SAAS;YAE/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAEhD,4CAA4C;gBAC5C,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAErC,MAAM;oBACJ,IAAI,EAAE,YAAY;oBAClB,OAAO;oBACP,QAAQ,EAAE,WAAW,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;gBAChC,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa;IAC9C,IAAI,QAAqB,CAAC;IAE1B,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,QAAQ,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;SAAM,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,QAAQ,GAAG;YACT,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC;YACxB,SAAS,EAAE,KAAK;YAChB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;SAAM,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,kDAAkD;QAClD,QAAQ,GAAG;YACT,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC;YACxB,SAAS,EAAE,KAAK;YAChB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,CAAC;QACH,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/C,SAAS,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC;QACb,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,WAAW,SAAS,QAAQ,CAAC,CAAC;IACzC,CAAC;YAAS,CAAC;QACT,0BAA0B;QAC1B,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAa;IAC7C,IAAI,SAAiB,CAAC;IACtB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;QACpC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAC3B,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,KAAK,CAAC;IACpB,CAAC;IAED,IAAI,CAAC;QACH,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,KAAK,EAAE,CAAC;QACV,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACxD,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC"}