@memvid/maw 1.0.0

package/dist/src/index.d.ts

@@ -0,0 +1,103 @@
+/**
+ * maw - Feed the maw. It never forgets.
+ *
+ * One command to consume entire websites, git repos, and files into searchable .mv2 files.
+ */
+export interface MawOptions {
+    output: string;
+    depth?: number;
+    concurrency?: number;
+    maxPages?: number;
+    rateLimit?: number;
+    timeout?: number;
+    includePattern?: RegExp;
+    excludePattern?: RegExp;
+    useSitemap?: boolean;
+    respectRobots?: boolean;
+    forceEngine?: 'fetch' | 'playwright' | 'rebrowser';
+    label?: string;
+    memoryId?: string;
+    memoryName?: string;
+    enableEmbedding?: boolean;
+    embeddingModel?: string;
+    quiet?: boolean;
+    verbose?: boolean;
+}
+export interface MawResult {
+    output: string;
+    pages: number;
+    size: number;
+    duration: number;
+    stoppedAtLimit?: boolean;
+    skippedDupes?: number;
+    memoryId?: string;
+    stats: {
+        fetch: number;
+        playwright: number;
+        rebrowser: number;
+        blocked: number;
+        dedup: {
+            localeSkipped: number;
+            similarSkipped: number;
+            total: number;
+        };
+    };
+}
+/**
+ * Main maw function - crawl URLs, git repos, or files and save to .mv2 file
+ */
+export declare function maw(urls: string[], options: MawOptions): Promise<MawResult>;
+/**
+ * Search in an .mv2 file
+ */
+export declare function find(path: string, query: string, options?: {
+    k?: number;
+}): Promise<any>;
+/**
+ * Ask a question using an .mv2 file
+ */
+export declare function ask(path: string, question: string, options?: {
+    model?: string;
+    apiKey?: string;
+    k?: number;
+}): Promise<any>;
+/**
+ * List documents in an .mv2 file
+ */
+export declare function list(path: string, options?: {
+    limit?: number;
+}): Promise<any>;
+/**
+ * Export documents from an .mv2 file with full content
+ */
+export declare function exportDocs(path: string, options?: {
+    limit?: number;
+}): Promise<{
+    title: string;
+    uri: string;
+    content: string;
+}[]>;
+export interface PreviewResult {
+    domain: string;
+    totalPages: number;
+    hasSitemap: boolean;
+    estimatedSize?: string;
+    recentPages: Array<{
+        url: string;
+        lastmod?: string;
+    }>;
+}
+/**
+ * Preview available pages on a site (sitemap discovery)
+ */
+export declare function preview(url: string, options?: {
+    limit?: number;
+}): Promise<PreviewResult>;
+export type { CrawlOptions, CrawlResult } from './crawler/index.js';
+export type { ExtractResult } from './extractor/index.js';
+export type { EngineResult, EngineOptions, EngineStats } from './engine/index.js';
+export { Crawler } from './crawler/index.js';
+export { Extractor } from './extractor/index.js';
+export { EngineWaterfall } from './engine/index.js';
+export { createLogger, setLogMode } from './utils/logger.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE;YACL,aAAa,EAAE,MAAM,CAAC;YACtB,cAAc,EAAE,MAAM,CAAC;YACvB,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;KACH,CAAC;CACH;AAgCD;;GAEG;AACH,wBAAsB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAgFjF;AAkDD;;GAEG;AACH,wBAAsB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,CAAC,CAAC,EAAE,MAAM,CAAA;CAAO,gBAEnF;AAED;;GAEG;AACH,wBAAsB,GAAG,CACvB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAA;CAAO,gBAG9D;AAED;;GAEG;AACH,wBAAsB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,gBAExE;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO;;;;KAE9E;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,OAAO,CAAC,aAAa,CAAC,CAoCnG;AAGD,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACpE,YAAY,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAClF,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/src/index.js

@@ -0,0 +1,229 @@
+/**
+ * maw - Feed the maw. It never forgets.
+ *
+ * One command to consume entire websites, git repos, and files into searchable .mv2 files.
+ */
+import { Crawler, SitemapParser } from './crawler/index.js';
+import { ingestToMv2, ingestGitToMv2, getFileSize, searchMv2, askMv2, listMv2, exportMv2 } from './ingestor/index.js';
+import { createLogger, setLogMode } from './utils/logger.js';
+import { normalizeUrl } from './utils/url.js';
+import { isGitUrl, isLocalGitRepo, readGitRepo } from './git/index.js';
+const log = createLogger();
+/**
+ * Check if URL is a specific page (not a domain root)
+ * e.g., https://stripe.com/docs/api -> true (specific page)
+ *       https://stripe.com/ -> false (domain root)
+ *       https://stripe.com -> false (domain root)
+ */
+function isSpecificPage(url) {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname;
+        // It's a specific page if path has content beyond just /
+        // Ignore common index patterns like /index.html
+        if (path === '/' || path === '')
+            return false;
+        if (path.match(/^\/(index\.(html?|php|aspx?)|default\.(html?|aspx?))$/i))
+            return false;
+        // Has a meaningful path
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Check if input is a git repo (URL or local path)
+ */
+function isGitInput(input) {
+    return isGitUrl(input) || isLocalGitRepo(input);
+}
+/**
+ * Main maw function - crawl URLs, git repos, or files and save to .mv2 file
+ */
+export async function maw(urls, options) {
+    setLogMode(options.quiet || false, options.verbose || false);
+    // Check if any input is a git repo
+    const gitInputs = urls.filter(u => isGitInput(u));
+    const webInputs = urls.filter(u => !isGitInput(u));
+    // If we have git repos, handle them
+    if (gitInputs.length > 0) {
+        return mawGit(gitInputs, options);
+    }
+    // Auto-detect single page mode: if ALL urls are specific pages, use single-page mode
+    const allSpecificPages = webInputs.every(u => isSpecificPage(u));
+    const singlePageMode = allSpecificPages && options.depth === undefined;
+    // Show mode indicator (CLI shows header, this adds context)
+    if (!options.quiet) {
+        if (singlePageMode) {
+            log.info(`  Fetching ${webInputs.length} page${webInputs.length > 1 ? 's' : ''}...`);
+        }
+        else {
+            log.info(`  Crawling (depth ${options.depth ?? 2}, max ${options.maxPages ?? 150} pages)...`);
+        }
+    }
+    const crawler = new Crawler({
+        // Single page mode: depth=0, maxPages=urls.length, no sitemap
+        depth: singlePageMode ? 0 : (options.depth ?? 2),
+        concurrency: options.concurrency ?? 10,
+        maxPages: singlePageMode ? webInputs.length : (options.maxPages ?? 150),
+        rateLimit: options.rateLimit ?? 10,
+        timeout: options.timeout ?? 10000,
+        includePattern: options.includePattern,
+        excludePattern: options.excludePattern,
+        useSitemap: singlePageMode ? false : (options.useSitemap ?? true),
+        respectRobots: options.respectRobots ?? true,
+        forceEngine: options.forceEngine,
+    });
+    try {
+        // Crawl and ingest
+        const crawlResults = crawler.crawl(webInputs);
+        // Generate memory name from URLs if not provided
+        const memoryName = options.memoryName || webInputs.map(u => {
+            try {
+                return new URL(u).hostname.replace('www.', '');
+            }
+            catch {
+                return u;
+            }
+        }).join('-');
+        const ingestStats = await ingestToMv2(crawlResults, {
+            output: options.output,
+            label: options.label,
+            memoryId: options.memoryId,
+            memoryName,
+            enableEmbedding: options.enableEmbedding,
+            embeddingModel: options.embeddingModel,
+        });
+        // Get final stats
+        const engineStats = crawler.getStats();
+        const fileSize = await getFileSize(options.output);
+        return {
+            output: options.output,
+            pages: ingestStats.pages,
+            size: fileSize,
+            duration: ingestStats.duration,
+            stoppedAtLimit: ingestStats.stoppedAtLimit,
+            skippedDupes: ingestStats.skippedDupes,
+            memoryId: ingestStats.memoryId,
+            stats: {
+                fetch: engineStats.fetch,
+                playwright: engineStats.playwright,
+                rebrowser: engineStats.rebrowser,
+                blocked: engineStats.blocked,
+                dedup: engineStats.dedup || { localeSkipped: 0, similarSkipped: 0, total: 0 },
+            },
+        };
+    }
+    finally {
+        await crawler.close();
+    }
+}
+/**
+ * Ingest git repos into .mv2 file
+ */
+async function mawGit(repos, options) {
+    const startTime = Date.now();
+    if (!options.quiet) {
+        log.info(`  Reading ${repos.length} repo${repos.length > 1 ? 's' : ''}...`);
+    }
+    // Generate memory name from repo names
+    const memoryName = options.memoryName || repos.map(r => {
+        const match = r.match(/\/([^/]+?)(\.git)?$/);
+        return match ? match[1] : r.split('/').pop() || 'repo';
+    }).join('-');
+    // Read all git repos
+    const allFiles = readGitRepo(repos[0]); // For now, handle one repo at a time
+    const ingestStats = await ingestGitToMv2(allFiles, {
+        output: options.output,
+        label: options.label || 'code',
+        memoryId: options.memoryId,
+        memoryName,
+        enableEmbedding: options.enableEmbedding,
+        embeddingModel: options.embeddingModel,
+    });
+    const fileSize = await getFileSize(options.output);
+    const duration = Date.now() - startTime;
+    return {
+        output: options.output,
+        pages: ingestStats.files,
+        size: fileSize,
+        duration,
+        stoppedAtLimit: ingestStats.stoppedAtLimit,
+        memoryId: ingestStats.memoryId,
+        stats: {
+            fetch: ingestStats.files,
+            playwright: 0,
+            rebrowser: 0,
+            blocked: 0,
+            dedup: { localeSkipped: 0, similarSkipped: 0, total: ingestStats.files },
+        },
+    };
+}
+/**
+ * Search in an .mv2 file
+ */
+export async function find(path, query, options = {}) {
+    return searchMv2(path, query, options);
+}
+/**
+ * Ask a question using an .mv2 file
+ */
+export async function ask(path, question, options = {}) {
+    return askMv2(path, question, options);
+}
+/**
+ * List documents in an .mv2 file
+ */
+export async function list(path, options = {}) {
+    return listMv2(path, options);
+}
+/**
+ * Export documents from an .mv2 file with full content
+ */
+export async function exportDocs(path, options = {}) {
+    return exportMv2(path, options);
+}
+/**
+ * Preview available pages on a site (sitemap discovery)
+ */
+export async function preview(url, options = {}) {
+    const normalized = normalizeUrl(url);
+    const parsedUrl = new URL(normalized);
+    const domain = parsedUrl.hostname;
+    const sitemap = new SitemapParser();
+    const pages = await sitemap.parseWithMetadata(normalized);
+    // Sort by lastmod (most recent first)
+    const sortedPages = pages.sort((a, b) => {
+        if (!a.lastmod && !b.lastmod)
+            return 0;
+        if (!a.lastmod)
+            return 1;
+        if (!b.lastmod)
+            return -1;
+        return new Date(b.lastmod).getTime() - new Date(a.lastmod).getTime();
+    });
+    const limit = options.limit || 20;
+    const recentPages = sortedPages.slice(0, limit).map(p => ({
+        url: p.loc,
+        lastmod: p.lastmod,
+    }));
+    // Estimate size (~300KB average per page for news sites, ~100KB for docs)
+    const avgPageSize = domain.includes('cnn') || domain.includes('news') ? 300 : 100;
+    const estimatedMB = (pages.length * avgPageSize) / 1024;
+    const estimatedSize = estimatedMB < 50
+        ? `${estimatedMB.toFixed(0)}MB (fits in free tier)`
+        : `${estimatedMB.toFixed(0)}MB (needs API key for full crawl)`;
+    return {
+        domain,
+        totalPages: pages.length,
+        hasSitemap: pages.length > 0,
+        estimatedSize: pages.length > 0 ? estimatedSize : undefined,
+        recentPages,
+    };
+}
+export { Crawler } from './crawler/index.js';
+export { Extractor } from './extractor/index.js';
+export { EngineWaterfall } from './engine/index.js';
+export { createLogger, setLogMode } from './utils/logger.js';
+//# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAuC,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACjG,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACtH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AA4CvE,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;AAE3B;;;;;GAKG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,yDAAyD;QACzD,gDAAgD;QAChD,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,KAAK,CAAC;QAC9C,IAAI,IAAI,CAAC,KAAK,CAAC,wDAAwD,CAAC;YAAE,OAAO,KAAK,CAAC;QACvF,wBAAwB;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,cAAc,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAc,EAAE,OAAmB;IAC3D,UAAU,CAAC,OAAO,CAAC,KAAK,IAAI,KAAK,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;IAE7D,mCAAmC;IACnC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,oCAAoC;IACpC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,qFAAqF;IACrF,MAAM,gBAAgB,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,cAAc,GAAG,gBAAgB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC;IAEvE,4DAA4D;IAC5D,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,cAAc,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,cAAc,SAAS,CAAC,MAAM,QAAQ,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACvF,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,OAAO,CAAC,QAAQ,IAAI,GAAG,YAAY,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC;QAC1B,8DAA8D;QAC9D,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;QAChD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,EAAE;QACtC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;QACvE,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE;QAClC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;QACjC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;QACjE,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;QAC5C,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,mBAAmB;QACnB,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAE9C,iDAAiD;QACjD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACzD,IAAI,CAAC;gBAAC,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,CAAC,CAAC;YAAC,CAAC;QAC7E,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,YAAY,EAAE;YAClD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU;YACV,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC,CAAC;QAEH,kBAAkB;QAClB,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAEnD,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,WAAW,CAAC,QAAQ;YAC9B,cAAc,EAAE,WAAW,CAAC,cAAc;YAC1C,YAAY,EAAE,WAAW,CAAC,YAAY;YACtC,QAAQ,EAAE,WAAW,CAAC,QAAQ;YAC9B,KAAK,EAAE;gBACL,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,SAAS,EAAE,WAAW,CAAC,SAAS;gBAChC,OAAO,EAAE,WAAW,CAAC,OAAO;gBAC5B,KAAK,EAAE,WAAW,CAAC,KAAK,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;aAC9E;SACF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,MAAM,CAAC,KAAe,EAAE,OAAmB;IACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,GAAG,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,MAAM,QAAQ,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9E,CAAC;IAED,uCAAuC;IACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;QACrD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC7C,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;IACzD,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEb,qBAAqB;IACrB,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,qCAAqC;IAE7E,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE;QACjD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM;QAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,UAAU;QACV,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,cAAc,EAAE,OAAO,CAAC,cAAc;KACvC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAExC,OAAO;QACL,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,IAAI,EAAE,QAAQ;QACd,QAAQ;QACR,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,QAAQ,EAAE,WAAW,CAAC,QAAQ;QAC9B,KAAK,EAAE;YACL,KAAK,EAAE,WAAW,CAAC,KAAK;YACxB,UAAU,EAAE,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE;SACzE;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAY,EAAE,KAAa,EAAE,UAA0B,EAAE;IAClF,OAAO,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,GAAG,CACvB,IAAY,EACZ,QAAgB,EAChB,UAA2D,EAAE;IAE7D,OAAO,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAY,EAAE,UAA8B,EAAE;IACvE,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,UAA8B,EAAE;IAC7E,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAUD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,GAAW,EAAE,UAA8B,EAAE;IACzE,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC;IAElC,MAAM,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;IAE1D,sCAAsC;IACtC,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtC,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,OAAO,CAAC,CAAC;QACzB,IAAI,CAAC,CAAC,CAAC,OAAO;YAAE,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;IACvE,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxD,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC,CAAC,CAAC;IAEJ,0EAA0E;IAC1E,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClF,MAAM,WAAW,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC;IACxD,MAAM,aAAa,GAAG,WAAW,GAAG,EAAE;QACpC,CAAC,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;QACnD,CAAC,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,mCAAmC,CAAC;IAEjE,OAAO;QACL,MAAM;QACN,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,UAAU,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC;QAC5B,aAAa,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QAC3D,WAAW;KACZ,CAAC;AACJ,CAAC;AAMD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/src/ingestor/index.d.ts

@@ -0,0 +1,95 @@
+/**
+ * MV2 ingestor - saves crawled content to .mv2 files
+ */
+import type { CrawlResult } from '../crawler/index.js';
+export interface IngestOptions {
+    output: string;
+    label?: string;
+    batchSize?: number;
+    maxSizeMB?: number;
+    memoryId?: string;
+    memoryName?: string;
+    enableEmbedding?: boolean;
+    embeddingModel?: string;
+}
+export interface IngestStats {
+    pages: number;
+    bytes: number;
+    duration: number;
+    stoppedAtLimit?: boolean;
+    skippedDupes?: number;
+    memoryId?: string;
+}
+/**
+ * Ingest crawled results into an MV2 file
+ */
+export declare function ingestToMv2(results: AsyncIterable<CrawlResult>, options: IngestOptions): Promise<IngestStats>;
+export interface GitIngestOptions {
+    output: string;
+    label?: string;
+    memoryId?: string;
+    memoryName?: string;
+    enableEmbedding?: boolean;
+    embeddingModel?: string;
+    maxSizeMB?: number;
+}
+export interface GitIngestStats {
+    files: number;
+    bytes: number;
+    duration: number;
+    memoryId?: string;
+    stoppedAtLimit?: boolean;
+}
+/**
+ * Ingest git repo files into an MV2 file
+ */
+export declare function ingestGitToMv2(files: AsyncIterable<{
+    path: string;
+    content: string;
+    language: string;
+    size: number;
+}>, options: GitIngestOptions): Promise<GitIngestStats>;
+/**
+ * Get file size
+ */
+export declare function getFileSize(path: string): Promise<number>;
+/**
+ * Open an existing MV2 file for querying
+ */
+export declare function openMv2(path: string): Promise<any>;
+/**
+ * Search in an MV2 file
+ * Uses semantic search when OPENAI_API_KEY is set
+ */
+export declare function searchMv2(path: string, query: string, options?: {
+    k?: number;
+    embeddingModel?: string;
+}): Promise<any>;
+/**
+ * Ask a question using an MV2 file
+ * Uses semantic search when embeddings are available and OPENAI_API_KEY is set
+ */
+export declare function askMv2(path: string, question: string, options?: {
+    model?: string;
+    apiKey?: string;
+    k?: number;
+    embeddingModel?: string;
+}): Promise<any>;
+/**
+ * List documents in an MV2 file
+ */
+export declare function listMv2(path: string, options?: {
+    limit?: number;
+    offset?: number;
+}): Promise<any>;
+/**
+ * Export documents from an MV2 file with full content
+ */
+export declare function exportMv2(path: string, options?: {
+    limit?: number;
+}): Promise<Array<{
+    title: string;
+    uri: string;
+    content: string;
+}>>;
+//# sourceMappingURL=index.d.ts.map

package/dist/src/ingestor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/ingestor/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAevD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,aAAa,CAAC,WAAW,CAAC,EACnC,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,WAAW,CAAC,CAmLtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,aAAa,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EACvF,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,cAAc,CAAC,CAqLzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO/D;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,gBAGzC;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAO,gBAatD;AAeD;;;GAGG;AACH,wBAAsB,MAAM,CAC1B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAO,gBAoBvF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,gBASlD;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/B,OAAO,CAAC,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAoCjE"}