hoolix 0.0.1-beta.19

package/dist/ingestion/github.d.ts

@@ -0,0 +1,60 @@
+import type { FetchResult } from './fetchers.js';
+import { fetchTextWithFallback as defaultFetchTextWithFallback, fetchWithRetry as defaultFetchWithRetry } from './fetchers.js';
+interface GitHubFetchers {
+    fetchTextWithFallback: typeof defaultFetchTextWithFallback;
+    fetchWithRetry: typeof defaultFetchWithRetry;
+}
+export declare function setGitHubFetchersForTests(fetchers: Partial<GitHubFetchers> | null): void;
+export interface GitHubRepoInfo {
+    owner: string;
+    repo: string;
+    ref: string;
+    subpath?: string;
+    originalUrl: string;
+}
+export declare function getGitHubToken(): string | undefined;
+export declare function isGitHubRepoUrl(url: string): boolean;
+/**
+ * Parse GitHub repo URL into owner/repo/ref/subpath.
+ * Handles /tree/REF , /blob/ , trailing .git, subdirs.
+ * Defaults ref to 'HEAD' (GitHub resolves to default branch).
+ */
+export declare function parseGitHubRepoUrl(url: string): GitHubRepoInfo | null;
+/** Try common llms + README locations (raw) for a GitHub repo. Returns first usable.
+ * Now supports private repos by passing Authorization header when token is provided.
+ */
+export declare function fetchPrimaryGitHubContent(info: GitHubRepoInfo, opts?: {
+    token?: string;
+    timeout?: number;
+}): Promise<FetchResult | null>;
+/**
+ * Discover candidate doc files for a GitHub repo.
+ * - If token: use /git/trees?recursive=1 (rich, respects .gitignore if present).
+ * - Else: return a small set of likely README + docs/ .md + llms candidates (no API call).
+ * Returns list of {rawUrl, blobUrl, path} sorted with llms/README first.
+ */
+export declare function discoverGitHubDocFiles(info: GitHubRepoInfo, opts?: {
+    token?: string;
+    maxPages?: number;
+    onProgress?: (c: number, t: number) => void;
+}): Promise<Array<{
+    rawUrl: string;
+    blobUrl: string;
+    path: string;
+}>>;
+/**
+ * High-level: given a github URL, return primary (best llms/README) + additional pages for multi-file ingest.
+ * Reuses existing fetch* helpers + parse for manifests.
+ * Always falls back safely.
+ */
+export declare function fetchGitHubRepoDocumentation(url: string, opts?: {
+    discoverLlms?: boolean;
+    maxPages?: number;
+    onProgress?: (completed: number, total: number) => void;
+    token?: string;
+}): Promise<{
+    primary: FetchResult;
+    pages?: FetchResult[];
+} | null>;
+export {};
+//# sourceMappingURL=github.d.ts.map

package/dist/ingestion/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/ingestion/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACL,qBAAqB,IAAI,4BAA4B,EACrD,cAAc,IAAI,qBAAqB,EACxC,MAAM,eAAe,CAAC;AAEvB,UAAU,cAAc;IACtB,qBAAqB,EAAE,OAAO,4BAA4B,CAAC;IAC3D,cAAc,EAAE,OAAO,qBAAqB,CAAC;CAC9C;AAOD,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,IAAI,GAAG,IAAI,CAKxF;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAgB,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnD;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAOpD;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CA0BrE;AAwBD;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,cAAc,EACpB,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAC9C,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAqD7B;AAwBD;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,cAAc,EACpB,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,IAAI,CAAA;CAAO,GAC5F,OAAO,CAAC,KAAK,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAwFnE;AAED;;;;GAIG;AACH,wBAAsB,4BAA4B,CAChD,GAAG,EAAE,MAAM,EACX,IAAI,GAAE;IACJ,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC;CACX,GACL,OAAO,CAAC;IAAE,OAAO,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,WAAW,EAAE,CAAA;CAAE,GAAG,IAAI,CAAC,CAuEjE"}

package/dist/ingestion/github.js

@@ -0,0 +1,314 @@
+import { logger } from '../core/logger.js';
+import { fetchTextWithFallback as defaultFetchTextWithFallback, fetchWithRetry as defaultFetchWithRetry, } from './fetchers.js';
+let githubFetchers = {
+    fetchTextWithFallback: defaultFetchTextWithFallback,
+    fetchWithRetry: defaultFetchWithRetry,
+};
+export function setGitHubFetchersForTests(fetchers) {
+    githubFetchers = {
+        fetchTextWithFallback: fetchers?.fetchTextWithFallback || defaultFetchTextWithFallback,
+        fetchWithRetry: fetchers?.fetchWithRetry || defaultFetchWithRetry,
+    };
+}
+export function getGitHubToken() {
+    return process.env.GITHUB_TOKEN || process.env.GH_TOKEN || undefined;
+}
+export function isGitHubRepoUrl(url) {
+    try {
+        const u = new URL(url);
+        return u.hostname === 'github.com' || u.hostname === 'www.github.com';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Parse GitHub repo URL into owner/repo/ref/subpath.
+ * Handles /tree/REF , /blob/ , trailing .git, subdirs.
+ * Defaults ref to 'HEAD' (GitHub resolves to default branch).
+ */
+export function parseGitHubRepoUrl(url) {
+    try {
+        const u = new URL(url.replace(/\.git$/, ''));
+        if (u.hostname !== 'github.com' && u.hostname !== 'www.github.com')
+            return null;
+        const parts = u.pathname.replace(/^\/+|\/+$/g, '').split('/');
+        if (parts.length < 2)
+            return null;
+        const owner = parts[0];
+        const repo = parts[1];
+        let ref = 'HEAD';
+        let subpath;
+        // /owner/repo/tree/REF[/sub]
+        // /owner/repo[/sub]
+        if (parts[2] === 'tree' || parts[2] === 'blob') {
+            ref = parts[3] || 'HEAD';
+            if (parts.length > 4)
+                subpath = parts.slice(4).join('/');
+        }
+        else if (parts.length > 2) {
+            subpath = parts.slice(2).join('/');
+        }
+        return { owner, repo, ref, subpath, originalUrl: url };
+    }
+    catch {
+        return null;
+    }
+}
+function rawBase(info) {
+    return `https://raw.githubusercontent.com/${info.owner}/${info.repo}/${info.ref}`;
+}
+function blobBase(info) {
+    const ref = info.ref === 'HEAD' ? 'main' : info.ref; // best-effort for human urls
+    return `https://github.com/${info.owner}/${info.repo}/blob/${ref}`;
+}
+function apiBase(info) {
+    return `https://api.github.com/repos/${info.owner}/${info.repo}`;
+}
+function authHeaders(token) {
+    const h = {
+        'User-Agent': 'hoolix/0.2 (https://github.com/JayLLM/hoolix)',
+        Accept: 'application/vnd.github+json',
+    };
+    if (token)
+        h['Authorization'] = `token ${token}`;
+    return h;
+}
+/** Try common llms + README locations (raw) for a GitHub repo. Returns first usable.
+ * Now supports private repos by passing Authorization header when token is provided.
+ */
+export async function fetchPrimaryGitHubContent(info, opts = {}) {
+    const token = opts.token || getGitHubToken();
+    const candidates = [];
+    const base = rawBase(info);
+    const sub = info.subpath ? `/${info.subpath.replace(/^\//, '')}` : '';
+    // llms priority (full then txt) at root + /docs + subpath
+    candidates.push(`${base}${sub}/llms-full.txt`, `${base}${sub}/llms.txt`, `${base}/llms-full.txt`, `${base}/llms.txt`, `${base}/docs/llms-full.txt`, `${base}/docs/llms.txt`, `${base}/README.md`, `${base}/docs/README.md`, `${base}/doc/README.md`, `${base}/README`);
+    const headers = { Accept: 'text/plain' };
+    if (token) {
+        headers['Authorization'] = `token ${token}`;
+    }
+    for (const cand of candidates) {
+        try {
+            const { text, contentType } = await githubFetchers.fetchTextWithFallback(cand, {
+                timeout: opts.timeout || 15000,
+                headers,
+            });
+            if (text && text.length > 200) {
+                // Return a "blob" style url for grounding (human friendly) when possible
+                const blobUrl = cand.replace(rawBase(info), blobBase(info)).replace(/\/raw\/[^/]+\//, `/blob/${info.ref}/`);
+                return {
+                    content: text,
+                    contentType: contentType || 'text/markdown',
+                    url: blobUrl,
+                };
+            }
+        }
+        catch (err) {
+            // For private repos, 401/403 usually means missing/invalid token or no access.
+            // 404 is common while probing optional llms/README candidates, so keep it quiet.
+            if (token && (err?.message?.includes('401') || err?.message?.includes('403'))) {
+                logger.warn(`GitHub raw fetch failed for private repo candidate ${cand}. Check GITHUB_TOKEN has 'repo' scope (classic) or appropriate fine-grained permissions.`);
+            }
+            else if (!token && (err?.message?.includes('401') || err?.message?.includes('403') || err?.message?.includes('404'))) {
+                // Common for private repo without token; only warn once-ish via debug to avoid spam, user sees via higher level
+                logger.debug(`GitHub candidate ${cand} failed (likely private; set GITHUB_TOKEN for access).`);
+            }
+        }
+    }
+    return null;
+}
+/** Crude .gitignore aware filter + hardcoded ignores. */
+function shouldIgnore(path, gitignoreText) {
+    const p = path.toLowerCase();
+    const hard = [
+        'node_modules/', '.git/', 'dist/', 'build/', 'out/', '.next/', 'target/', 'vendor/',
+        'coverage/', 'tmp/', 'temp/', '.cache/', '__pycache__/', '.venv/', 'site-packages/',
+        '.DS_Store', '.exe', '.dll', '.so', '.dylib', '.bin', '.o', '.a', '.zip', '.tar', '.gz', '.7z',
+        '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.pdf', '.mp4', '.mov',
+    ];
+    if (hard.some(h => p.includes(h)))
+        return true;
+    if (!gitignoreText)
+        return false;
+    const lines = gitignoreText.split(/\r?\n/).map(l => l.trim()).filter(l => l && !l.startsWith('#'));
+    for (const line of lines) {
+        const pat = line.replace(/^\//, '').replace(/\*/g, '.*');
+        try {
+            if (new RegExp(pat, 'i').test(path))
+                return true;
+        }
+        catch { }
+    }
+    return false;
+}
+/**
+ * Discover candidate doc files for a GitHub repo.
+ * - If token: use /git/trees?recursive=1 (rich, respects .gitignore if present).
+ * - Else: return a small set of likely README + docs/ .md + llms candidates (no API call).
+ * Returns list of {rawUrl, blobUrl, path} sorted with llms/README first.
+ */
+export async function discoverGitHubDocFiles(info, opts = {}) {
+    const token = opts.token || getGitHubToken();
+    const max = opts.maxPages || 80;
+    const results = [];
+    const rBase = rawBase(info);
+    const bBase = blobBase(info);
+    const sub = info.subpath ? `/${info.subpath.replace(/^\//, '')}` : '';
+    if (token) {
+        try {
+            const treeUrl = `${apiBase(info)}/git/trees/${encodeURIComponent(info.ref)}?recursive=1`;
+            const res = await githubFetchers.fetchWithRetry(treeUrl, { headers: authHeaders(token), timeout: 15000 });
+            if (res.status === 403 || res.status === 429) {
+                const rem = res.headers.get('x-ratelimit-remaining');
+                logger.warn(`GitHub API rate limited (remaining=${rem ?? '?'}) — falling back to limited discovery (~12 files). ` +
+                    `Set GITHUB_TOKEN for 5,000 req/hr vs 60 req/hr unauthenticated: export GITHUB_TOKEN=<token>`);
+                // fall through to limited direct
+            }
+            else if (res.ok) {
+                const data = await res.json();
+                let gitignoreText;
+                const tree = data.tree || [];
+                // fetch .gitignore if present (best effort) — pass token for private repos
+                const gi = tree.find((e) => e.path === '.gitignore' && e.type === 'blob');
+                if (gi) {
+                    try {
+                        const giRaw = `${rBase}/.gitignore`;
+                        const giHeaders = { Accept: 'text/plain' };
+                        if (token)
+                            giHeaders['Authorization'] = `token ${token}`;
+                        const { text } = await githubFetchers.fetchTextWithFallback(giRaw, { timeout: 8000, headers: giHeaders });
+                        gitignoreText = text;
+                    }
+                    catch { }
+                }
+                const candidates = tree
+                    .filter((e) => e.type === 'blob' && /\.(md|mdx|txt|rst)$/i.test(e.path))
+                    .map((e) => e.path)
+                    .filter((p) => {
+                    if (sub && !p.startsWith(sub.replace(/^\//, '')))
+                        return false;
+                    return !shouldIgnore(p, gitignoreText);
+                })
+                    .sort((a, b) => {
+                    // prioritize llms + readme + docs/
+                    const score = (p) => {
+                        const l = p.toLowerCase();
+                        if (l.includes('llms-full'))
+                            return 0;
+                        if (l.endsWith('llms.txt'))
+                            return 1;
+                        if (l.includes('readme'))
+                            return 2;
+                        if (l.startsWith('docs/') || l.startsWith('doc/'))
+                            return 3;
+                        return 10;
+                    };
+                    return score(a) - score(b) || a.localeCompare(b);
+                })
+                    .slice(0, max);
+                for (const p of candidates) {
+                    const rawUrl = `${rBase}/${p}`;
+                    const blobUrl = `${bBase}/${p}`;
+                    results.push({ rawUrl, blobUrl, path: p });
+                }
+                return results;
+            }
+        }
+        catch (e) {
+            logger.debug(`GitHub tree discovery failed, using limited candidates: ${e?.message || e}`);
+        }
+    }
+    // Limited direct candidates (no or failed token; always works for public)
+    const limited = [
+        `${sub}/llms-full.txt`, `${sub}/llms.txt`,
+        'llms-full.txt', 'llms.txt',
+        'README.md', 'docs/README.md', 'doc/README.md',
+        'docs/index.md', 'README',
+    ].map(p => p.replace(/^\//, ''));
+    for (const p of limited) {
+        if (results.length >= 12)
+            break;
+        if (sub && !p.startsWith(sub.replace(/^\//, '')) && !p.startsWith('llms'))
+            continue;
+        results.push({
+            rawUrl: `${rBase}/${p}`,
+            blobUrl: `${bBase}/${p}`,
+            path: p,
+        });
+    }
+    return results.slice(0, max);
+}
+/**
+ * High-level: given a github URL, return primary (best llms/README) + additional pages for multi-file ingest.
+ * Reuses existing fetch* helpers + parse for manifests.
+ * Always falls back safely.
+ */
+export async function fetchGitHubRepoDocumentation(url, opts = {}) {
+    const info = parseGitHubRepoUrl(url);
+    if (!info)
+        return null;
+    const token = opts.token || getGitHubToken();
+    const maxPages = opts.maxPages || 80;
+    if (!token) {
+        logger.warn('No GITHUB_TOKEN set — using limited file discovery (~12 files max). ' +
+            'For full repo indexing set: export GITHUB_TOKEN=<token>  (needs "repo" scope or "Contents: Read" fine-grained permission).');
+    }
+    // Primary (llms or README)
+    let primary = await fetchPrimaryGitHubContent(info, { token });
+    if (!primary) {
+        // fallback to raw README at least (pass auth for private repos)
+        try {
+            const fallback = `${rawBase(info)}/README.md`;
+            const headers = { Accept: 'text/plain' };
+            if (token)
+                headers['Authorization'] = `token ${token}`;
+            const { text } = await githubFetchers.fetchTextWithFallback(fallback, { timeout: 12000, headers });
+            if (text)
+                primary = { content: text, contentType: 'text/markdown', url: `${blobBase(info)}/README.md` };
+        }
+        catch (e) {
+            if (!token && (e?.message?.includes('401') || e?.message?.includes('403') || e?.message?.includes('404'))) {
+                logger.warn('GitHub raw fetch failed. For private repos set GITHUB_TOKEN (with repo scope) and re-run.');
+            }
+        }
+    }
+    if (!primary)
+        return null;
+    // Additional pages (for non-manifest github docs)
+    let pages = [];
+    if (opts.discoverLlms !== false) {
+        const discovered = await discoverGitHubDocFiles(info, {
+            token,
+            maxPages,
+            onProgress: opts.onProgress,
+        });
+        // Skip the primary if it was one of them
+        const toFetch = discovered.filter(d => d.rawUrl !== primary.url && d.blobUrl !== primary.url);
+        if (toFetch.length > 0) {
+            // Reuse concurrent fetch (but map to raw for content, keep blob for metadata.url)
+            // Thread Authorization for private repo raw fetches when GITHUB_TOKEN present (fixes full private support)
+            const pageHeaders = { Accept: 'text/plain' };
+            if (token)
+                pageHeaders['Authorization'] = `token ${token}`;
+            const fetchResults = await Promise.all(toFetch.slice(0, maxPages - 1).map(async (d) => {
+                try {
+                    const { text } = await githubFetchers.fetchTextWithFallback(d.rawUrl, { timeout: 15000, headers: pageHeaders });
+                    if (text && text.length > 80) {
+                        return { content: text, contentType: 'text/markdown', url: d.blobUrl };
+                    }
+                }
+                catch (e) {
+                    if (!token && (e?.message?.includes('401') || e?.message?.includes('403'))) {
+                        logger.debug(`Private GitHub page fetch likely requires GITHUB_TOKEN: ${d.path}`);
+                    }
+                }
+                return null;
+            }));
+            pages = fetchResults.filter(Boolean);
+        }
+        // If primary was llms manifest, let normal pipeline expand it (parseLlmsManifestUrls will resolve relatives correctly against blob/raw urls)
+    }
+    return { primary, pages: pages.length ? pages : undefined };
+}
+//# sourceMappingURL=github.js.map

package/dist/ingestion/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/ingestion/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EACL,qBAAqB,IAAI,4BAA4B,EACrD,cAAc,IAAI,qBAAqB,GACxC,MAAM,eAAe,CAAC;AAOvB,IAAI,cAAc,GAAmB;IACnC,qBAAqB,EAAE,4BAA4B;IACnD,cAAc,EAAE,qBAAqB;CACtC,CAAC;AAEF,MAAM,UAAU,yBAAyB,CAAC,QAAwC;IAChF,cAAc,GAAG;QACf,qBAAqB,EAAE,QAAQ,EAAE,qBAAqB,IAAI,4BAA4B;QACtF,cAAc,EAAE,QAAQ,EAAE,cAAc,IAAI,qBAAqB;KAClE,CAAC;AACJ,CAAC;AAUD,MAAM,UAAU,cAAc;IAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,SAAS,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,CAAC,CAAC,QAAQ,KAAK,YAAY,IAAI,CAAC,CAAC,QAAQ,KAAK,gBAAgB,CAAC;IACxE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,CAAC,QAAQ,KAAK,YAAY,IAAI,CAAC,CAAC,QAAQ,KAAK,gBAAgB;YAAE,OAAO,IAAI,CAAC;QAEhF,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAElC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,GAAG,GAAG,MAAM,CAAC;QACjB,IAAI,OAA2B,CAAC;QAEhC,6BAA6B;QAC7B,oBAAoB;QACpB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;YAC/C,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;YACzB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3D,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,IAAoB;IACnC,OAAO,qCAAqC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AACpF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAoB;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,6BAA6B;IAClF,OAAO,sBAAsB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,SAAS,GAAG,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAoB;IACnC,OAAO,gCAAgC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,MAAM,CAAC,GAA2B;QAChC,YAAY,EAAE,+CAA+C;QAC7D,MAAM,EAAE,6BAA6B;KACtC,CAAC;IACF,IAAI,KAAK;QAAE,CAAC,CAAC,eAAe,CAAC,GAAG,SAAS,KAAK,EAAE,CAAC;IACjD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAoB,EACpB,OAA6C,EAAE;IAE/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,cAAc,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEtE,0DAA0D;IAC1D,UAAU,CAAC,IAAI,CACb,GAAG,IAAI,GAAG,GAAG,gBAAgB,EAC7B,GAAG,IAAI,GAAG,GAAG,WAAW,EACxB,GAAG,IAAI,gBAAgB,EACvB,GAAG,IAAI,WAAW,EAClB,GAAG,IAAI,qBAAqB,EAC5B,GAAG,IAAI,gBAAgB,EACvB,GAAG,IAAI,YAAY,EACnB,GAAG,IAAI,iBAAiB,EACxB,GAAG,IAAI,gBAAgB,EACvB,GAAG,IAAI,SAAS,CACjB,CAAC;IAEF,MAAM,OAAO,GAA2B,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IACjE,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,KAAK,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,IAAI,EAAE;gBAC7E,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,KAAK;gBAC9B,OAAO;aACR,CAAC,CAAC;YACH,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC9B,yEAAyE;gBACzE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,SAAS,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;gBAC5G,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,WAAW,IAAI,eAAe;oBAC3C,GAAG,EAAE,OAAO;iBACb,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,+EAA+E;YAC/E,iFAAiF;YACjF,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC9E,MAAM,CAAC,IAAI,CAAC,sDAAsD,IAAI,0FAA0F,CAAC,CAAC;YACpK,CAAC;iBAAM,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvH,gHAAgH;gBAChH,MAAM,CAAC,KAAK,CAAC,oBAAoB,IAAI,wDAAwD,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yDAAyD;AACzD,SAAS,YAAY,CAAC,IAAY,EAAE,aAAsB;IACxD,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG;QACX,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS;QACnF,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,gBAAgB;QACnF,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;QAC9F,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;KACxE,CAAC;IACF,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/C,IAAI,CAAC,aAAa;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACnG,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,IAAI,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,IAAI,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IACZ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAoB,EACpB,OAA2F,EAAE;IAE7F,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,cAAc,EAAE,CAAC;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAChC,MAAM,OAAO,GAA6D,EAAE,CAAC;IAE7E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEtE,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,cAAc,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC;YACzF,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1G,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC7C,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;gBACrD,MAAM,CAAC,IAAI,CACT,sCAAsC,GAAG,IAAI,GAAG,qDAAqD;oBACrG,6FAA6F,CAC9F,CAAC;gBACF,iCAAiC;YACnC,CAAC;iBAAM,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,GAAQ,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,aAAiC,CAAC;gBACtC,MAAM,IAAI,GAAU,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;gBAEpC,2EAA2E;gBAC3E,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;gBAC/E,IAAI,EAAE,EAAE,CAAC;oBACP,IAAI,CAAC;wBACH,MAAM,KAAK,GAAG,GAAG,KAAK,aAAa,CAAC;wBACpC,MAAM,SAAS,GAA2B,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;wBACnE,IAAI,KAAK;4BAAE,SAAS,CAAC,eAAe,CAAC,GAAG,SAAS,KAAK,EAAE,CAAC;wBACzD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;wBAC1G,aAAa,GAAG,IAAI,CAAC;oBACvB,CAAC;oBAAC,MAAM,CAAC,CAAA,CAAC;gBACZ,CAAC;gBAED,MAAM,UAAU,GAAG,IAAI;qBACpB,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;qBAC5E,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;qBACvB,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE;oBACpB,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAC/D,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;gBACzC,CAAC,CAAC;qBACD,IAAI,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;oBAC7B,mCAAmC;oBACnC,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE;wBAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;wBAC1B,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;4BAAE,OAAO,CAAC,CAAC;wBACtC,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;4BAAE,OAAO,CAAC,CAAC;wBACrC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;4BAAE,OAAO,CAAC,CAAC;wBACnC,IAAI,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;4BAAE,OAAO,CAAC,CAAC;wBAC5D,OAAO,EAAE,CAAC;oBACZ,CAAC,CAAC;oBACF,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAEjB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;oBAC3B,MAAM,MAAM,GAAG,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC/B,MAAM,OAAO,GAAG,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC;oBAChC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,OAAO,OAAO,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,2DAA2D,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,MAAM,OAAO,GAAG;QACd,GAAG,GAAG,gBAAgB,EAAE,GAAG,GAAG,WAAW;QACzC,eAAe,EAAE,UAAU;QAC3B,WAAW,EAAE,gBAAgB,EAAE,eAAe;QAC9C,eAAe,EAAE,QAAQ;KAC1B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAEjC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE;YAAE,MAAM;QAChC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACpF,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,GAAG,KAAK,IAAI,CAAC,EAAE;YACvB,OAAO,EAAE,GAAG,KAAK,IAAI,CAAC,EAAE;YACxB,IAAI,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,GAAW,EACX,OAKI,EAAE;IAEN,MAAM,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,cAAc,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAErC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CACT,sEAAsE;YACtE,4HAA4H,CAC7H,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,IAAI,OAAO,GAAG,MAAM,yBAAyB,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/D,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,gEAAgE;QAChE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC;YAC9C,MAAM,OAAO,GAA2B,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;YACjE,IAAI,KAAK;gBAAE,OAAO,CAAC,eAAe,CAAC,GAAG,SAAS,KAAK,EAAE,CAAC;YACvD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YACnG,IAAI,IAAI;gBAAE,OAAO,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1G,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC1G,MAAM,CAAC,IAAI,CAAC,2FAA2F,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,kDAAkD;IAClD,IAAI,KAAK,GAAkB,EAAE,CAAC;IAC9B,IAAI,IAAI,CAAC,YAAY,KAAK,KAAK,EAAE,CAAC;QAChC,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,IAAI,EAAE;YACpD,KAAK;YACL,QAAQ;YACR,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;QAEH,yCAAyC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,OAAQ,CAAC,GAAG,CAAC,CAAC;QAEhG,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,kFAAkF;YAClF,2GAA2G;YAC3G,MAAM,WAAW,GAA2B,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;YACrE,IAAI,KAAK;gBAAE,WAAW,CAAC,eAAe,CAAC,GAAG,SAAS,KAAK,EAAE,CAAC;YAC3D,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBAC7C,IAAI,CAAC;oBACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;oBAChH,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;wBAC7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,EAAiB,CAAC;oBACxF,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAM,EAAE,CAAC;oBAChB,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;wBAC3E,MAAM,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;oBACpF,CAAC;gBACH,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CACH,CAAC;YACF,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAkB,CAAC;QACxD,CAAC;QAED,6IAA6I;IAC/I,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AAC9D,CAAC"}

package/dist/ingestion/pipeline.d.ts

@@ -0,0 +1,3 @@
+import type { IngestionOptions, IngestionResult } from './types.js';
+export declare function ingestDocumentation(url: string, options?: IngestionOptions): Promise<IngestionResult>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/ingestion/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAmC,MAAM,YAAY,CAAC;AAWrG,wBAAsB,mBAAmB,CACvC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CA4K1B"}

package/dist/ingestion/pipeline.js

@@ -0,0 +1,160 @@
+import { logger } from '../core/logger.js';
+import { fetchDocumentation, parseLlmsManifestUrls, fetchPagesConcurrently, } from './fetchers.js';
+import { detectSourceType, isLikelyMarkdown, isGitHubRepoUrl } from './detectors.js';
+import { htmlToMarkdown, normalizeMarkdown } from './cleaners.js';
+import { chunkMarkdown } from './chunker.js';
+import { fetchGitHubRepoDocumentation, getGitHubToken } from './github.js';
+export async function ingestDocumentation(url, options = {}) {
+    const start = Date.now();
+    const { onProgress, maxChunks = 8000, chunkSize = 1100, chunkOverlap = 180, maxPages = 80, } = options;
+    const emit = (p) => onProgress?.(p);
+    emit({ stage: 'detect', message: 'Detecting source type...' });
+    // Primary fetch (prefers llms-full.txt sibling when input is llms.txt)
+    emit({ stage: 'fetch', message: `Fetching ${url}...` });
+    let fetched = await fetchDocumentation(url, { headers: options.headers || {} });
+    emit({ stage: 'fetch', message: `Fetched ${fetched.content.length.toLocaleString()} chars` });
+    const sourceType = detectSourceType(fetched.url, fetched.content);
+    // pagesToProcess starts with primary; may expand for manifests below.
+    let pagesToProcess = [fetched];
+    let pagesProcessed = 1;
+    let pagesDiscovered = 1;
+    let truncated = false;
+    // Manifest expansion (only for llms.txt that is not -full and contains links/URLs)
+    const isLlmsManifest = fetched.url.includes('llms.txt') &&
+        !fetched.url.includes('llms-full.txt') &&
+        (fetched.content.includes('[') || /^- /m.test(fetched.content) || /https?:\/\//.test(fetched.content));
+    if (isLlmsManifest) {
+        emit({ stage: 'manifest', message: 'Parsing llms.txt manifest for page links...' });
+        const manifestUrls = parseLlmsManifestUrls(fetched.content, fetched.url);
+        pagesDiscovered = Math.max(pagesDiscovered, 1 + manifestUrls.length);
+        const toFetch = manifestUrls.slice(0, Math.max(0, maxPages - 1));
+        if (manifestUrls.length > toFetch.length)
+            truncated = true;
+        if (toFetch.length > 0) {
+            emit({
+                stage: 'pages',
+                message: `Fetching ${toFetch.length} pages from manifest (concurrency 3-4)...`,
+                current: 0,
+                total: toFetch.length,
+            });
+            const morePages = await fetchPagesConcurrently(toFetch, 3, (completed, total) => {
+                emit({
+                    stage: 'pages',
+                    message: `Fetching pages (${completed}/${total})...`,
+                    current: completed,
+                    total,
+                });
+            }, options.headers || {});
+            if (morePages.length > 0) {
+                pagesToProcess = [fetched, ...morePages];
+                pagesProcessed = pagesToProcess.length;
+                emit({
+                    stage: 'pages',
+                    message: `Fetched ${morePages.length} additional pages`,
+                    current: pagesProcessed,
+                    total: pagesProcessed,
+                });
+            }
+        }
+    }
+    // GitHub expansion (when not already handled by llms manifest and source looks like github repo)
+    // Uses dedicated discovery (tree if token, limited candidates otherwise) so we get multiple .md/README/llms files
+    // with correct per-file metadata.url (blob form for grounding).
+    const isGh = sourceType === 'github' || isGitHubRepoUrl(fetched.url);
+    if (isGh && !isLlmsManifest && pagesToProcess.length === 1) {
+        try {
+            emit({ stage: 'manifest', message: 'Discovering GitHub documentation files (README, docs/, llms)...' });
+            const ghToken = getGitHubToken();
+            const ghRes = await fetchGitHubRepoDocumentation(fetched.url, {
+                discoverLlms: true,
+                maxPages: maxPages || 80,
+                onProgress: (c, t) => emit({ stage: 'pages', message: `GitHub files (${c}/${t})...`, current: c, total: t }),
+                token: ghToken,
+            });
+            if (ghRes && ghRes.pages && ghRes.pages.length > 0) {
+                pagesToProcess = [ghRes.primary, ...ghRes.pages];
+                pagesProcessed = pagesToProcess.length;
+                pagesDiscovered = Math.max(pagesDiscovered, pagesProcessed);
+                emit({ stage: 'pages', message: `Fetched ${ghRes.pages.length} additional GitHub files`, current: pagesProcessed, total: pagesProcessed });
+            }
+        }
+        catch (e) {
+            logger.debug(`GitHub multi-file expansion failed, using primary only: ${e?.message || e}`);
+            // keep the single primary; normal flow continues
+        }
+    }
+    // Per-page clean+chunk so metadata.url is the real page (not root manifest) for grounding.
+    emit({ stage: 'clean', message: `Converting ${pagesToProcess.length} page(s) to Markdown...` });
+    let chunks = [];
+    for (const page of pagesToProcess) {
+        let markdown;
+        if (page.contentType.includes('html') && !isLikelyMarkdown(page.contentType, page.content)) {
+            markdown = htmlToMarkdown(page.content, page.url);
+        }
+        else {
+            markdown = normalizeMarkdown(page.content);
+        }
+        if (!markdown || markdown.length < 100)
+            continue;
+        const pageTitle = extractTitle(markdown, page.url);
+        const pageChunks = chunkMarkdown(markdown, page.url, pageTitle, {
+            targetSize: chunkSize,
+            overlap: chunkOverlap,
+        });
+        chunks.push(...pageChunks);
+        if (chunks.length > maxChunks) {
+            logger.warn(`Truncating from ${chunks.length} to ${maxChunks} chunks`);
+            chunks = chunks.slice(0, maxChunks);
+            truncated = true;
+            break;
+        }
+    }
+    if (chunks.length === 0) {
+        throw new Error('Could not extract meaningful content from the provided URL(s).');
+    }
+    const totalChars = chunks.reduce((sum, c) => sum + c.content.length, 0);
+    emit({
+        stage: 'chunk',
+        message: 'Chunking documentation...',
+    });
+    const isLlmsFull = fetched.url.includes('llms-full.txt');
+    const pagesInfo = isLlmsFull
+        ? 'llms-full.txt (concatenated documentation)'
+        : `${pagesProcessed} page(s)`;
+    emit({
+        stage: 'done',
+        message: `Ingestion complete: ${chunks.length} chunks, ${totalChars.toLocaleString()} chars from ${pagesInfo}`,
+    });
+    const baseTitle = extractTitle(pagesToProcess[0]?.content || '', pagesToProcess[0]?.url || url);
+    const result = {
+        sourceUrl: fetched.url,
+        sourceType,
+        title: baseTitle,
+        chunks,
+        stats: {
+            totalChunks: chunks.length,
+            totalChars,
+            pagesProcessed,
+            pagesDiscovered,
+            durationMs: Date.now() - start,
+            truncated,
+            maxChunks,
+            maxPages,
+        },
+        rawMarkdown: (pagesToProcess[0]?.content || '').slice(0, 50_000),
+    };
+    return result;
+}
+function extractTitle(md, fallbackUrl) {
+    const h1 = md.match(/^#\s+(.+)$/m);
+    if (h1)
+        return h1[1].trim();
+    try {
+        const u = new URL(fallbackUrl);
+        return u.hostname + u.pathname;
+    }
+    catch {
+        return 'Documentation';
+    }
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/ingestion/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingestion/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,4BAA4B,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE3E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,GAAW,EACX,UAA4B,EAAE;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EACJ,UAAU,EACV,SAAS,GAAG,IAAI,EAChB,SAAS,GAAG,IAAI,EAChB,YAAY,GAAG,GAAG,EAClB,QAAQ,GAAG,EAAE,GACd,GAAG,OAAO,CAAC;IAEZ,MAAM,IAAI,GAAqB,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtD,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAE/D,uEAAuE;IACvE,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,GAAG,KAAK,EAAE,CAAC,CAAC;IACxD,IAAI,OAAO,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC;IAEhF,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE9F,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAElE,sEAAsE;IACtE,IAAI,cAAc,GAAiE,CAAC,OAAO,CAAC,CAAC;IAC7F,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,mFAAmF;IACnF,MAAM,cAAc,GAClB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;QAChC,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC;QACtC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAEzG,IAAI,cAAc,EAAE,CAAC;QACnB,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,6CAA6C,EAAE,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QACzE,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACrE,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;YAAE,SAAS,GAAG,IAAI,CAAC;QAE3D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,YAAY,OAAO,CAAC,MAAM,2CAA2C;gBAC9E,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,OAAO,CAAC,MAAM;aACtB,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;gBAC9E,IAAI,CAAC;oBACH,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,mBAAmB,SAAS,IAAI,KAAK,MAAM;oBACpD,OAAO,EAAE,SAAS;oBAClB,KAAK;iBACN,CAAC,CAAC;YACL,CAAC,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;YAE1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,cAAc,GAAG,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,CAAC;gBACzC,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC;gBACvC,IAAI,CAAC;oBACH,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,WAAW,SAAS,CAAC,MAAM,mBAAmB;oBACvD,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,cAAc;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,iGAAiG;IACjG,kHAAkH;IAClH,gEAAgE;IAChE,MAAM,IAAI,GAAG,UAAU,KAAK,QAAQ,IAAI,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACrE,IAAI,IAAI,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,iEAAiE,EAAE,CAAC,CAAC;YACxG,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,4BAA4B,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC5D,YAAY,EAAE,IAAI;gBAClB,QAAQ,EAAE,QAAQ,IAAI,EAAE;gBACxB,UAAU,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBAC5G,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;YACH,IAAI,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnD,cAAc,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;gBACjD,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC;gBACvC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;gBAC5D,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,KAAK,CAAC,KAAK,CAAC,MAAM,0BAA0B,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;YAC7I,CAAC;QACH,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,2DAA2D,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3F,iDAAiD;QACnD,CAAC;IACH,CAAC;IAED,2FAA2F;IAC3F,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,cAAc,CAAC,MAAM,yBAAyB,EAAE,CAAC,CAAC;IAEhG,IAAI,MAAM,GAAoB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,IAAI,QAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3F,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC;QAED,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG;YAAE,SAAS;QAEjD,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE;YAC9D,UAAU,EAAE,SAAS;YACrB,OAAO,EAAE,YAAY;SACtB,CAAC,CAAC;QAEH,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;QAE3B,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,MAAM,OAAO,SAAS,SAAS,CAAC,CAAC;YACvE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;YACpC,SAAS,GAAG,IAAI,CAAC;YACjB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAExE,IAAI,CAAC;QACH,KAAK,EAAE,OAAO;QACd,OAAO,EAAE,2BAA2B;KACrC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IACzD,MAAM,SAAS,GAAG,UAAU;QAC1B,CAAC,CAAC,4CAA4C;QAC9C,CAAC,CAAC,GAAG,cAAc,UAAU,CAAC;IAEhC,IAAI,CAAC;QACH,KAAK,EAAE,MAAM;QACb,OAAO,EAAE,uBAAuB,MAAM,CAAC,MAAM,YAAY,UAAU,CAAC,cAAc,EAAE,eAAe,SAAS,EAAE;KAC/G,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,YAAY,CAC5B,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,EAChC,cAAc,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,CAC9B,CAAC;IAEF,MAAM,MAAM,GAAoB;QAC9B,SAAS,EAAE,OAAO,CAAC,GAAG;QACtB,UAAU;QACV,KAAK,EAAE,SAAS;QAChB,MAAM;QACN,KAAK,EAAE;YACL,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,UAAU;YACV,cAAc;YACd,eAAe;YACf,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,SAAS;YACT,SAAS;YACT,QAAQ;SACT;QACD,WAAW,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;KACjE,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,EAAU,EAAE,WAAmB;IACnD,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACnC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,eAAe,CAAC;IACzB,CAAC;AACH,CAAC"}

package/dist/ingestion/types.d.ts

@@ -0,0 +1,51 @@
+export type SourceType = 'llms.txt' | 'github' | 'generic' | 'manual';
+export interface IngestedChunk {
+    id: string;
+    content: string;
+    metadata: {
+        url: string;
+        title: string;
+        sectionPath?: string;
+        headings?: string[];
+        charCount: number;
+        order: number;
+        sourceId?: string;
+        sourceType?: string;
+        sourceLabel?: string;
+    };
+}
+export interface IngestionResult {
+    sourceUrl: string;
+    sourceType: SourceType;
+    title: string;
+    chunks: IngestedChunk[];
+    stats: {
+        totalChunks: number;
+        totalChars: number;
+        pagesProcessed: number;
+        pagesDiscovered?: number;
+        durationMs: number;
+        truncated: boolean;
+        maxChunks: number;
+        maxPages: number;
+    };
+    rawMarkdown?: string;
+}
+export interface IngestionProgress {
+    stage: 'detect' | 'fetch' | 'manifest' | 'pages' | 'clean' | 'chunk' | 'embed' | 'done';
+    message: string;
+    current?: number;
+    total?: number;
+    percent?: number;
+}
+export type ProgressCallback = (progress: IngestionProgress) => void;
+export interface IngestionOptions {
+    maxPages?: number;
+    maxChunks?: number;
+    chunkSize?: number;
+    chunkOverlap?: number;
+    onProgress?: ProgressCallback;
+    signal?: AbortSignal;
+    headers?: Record<string, string>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/ingestion/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEtE,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,UAAU,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,KAAK,EAAE;QACL,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,OAAO,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,QAAQ,GAAG,OAAO,GAAG,UAAU,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;IACxF,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,CAAC;AAErE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC"}

package/dist/ingestion/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/ingestion/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":""}

package/dist/lib/auth.d.ts

@@ -0,0 +1,2 @@
+export declare function generateAuthKey(): string;
+//# sourceMappingURL=auth.d.ts.map