docorbit 0.1.0 → 0.1.2

package/dist/packages/core/src/pipeline.js

@@ -0,0 +1,246 @@
+import { validateTargetUrl } from "../../security/src/index.js";
+import { SecureFetcher, DEFAULT_CRAWLER_CONFIG } from "../../crawler/src/index.js";
+import { createDefaultDiscoveryCoordinator, rankSources } from "../../discovery/src/index.js";
+import { buildNormalizedPage, parseLlmsTxt, isValidLlmsTxt, slicePageIntoChunks, parseOpenApiEndpoints, detectOpenApiSpec, extractIndexedExamples, extractPitfalls, } from "../../normalizer/src/index.js";
+export class IngestionPipeline {
+    repository;
+    fetcher;
+    config;
+    policy;
+    allowLocalhostForTesting;
+    constructor(repository, options = {}) {
+        this.repository = repository;
+        this.config = {
+            ...DEFAULT_CRAWLER_CONFIG,
+            ...options.crawlerConfig,
+        };
+        this.allowLocalhostForTesting = options.allowLocalhostForTesting ?? false;
+        this.policy = {
+            purpose: 'conceptual',
+            maxPages: this.config.maxPages,
+            maxDepth: this.config.maxDepth || 2,
+            followLlmsReferences: true,
+            followExternalDomains: false,
+            ...options.crawlPolicy,
+        };
+        this.fetcher = new SecureFetcher({
+            timeoutMs: this.config.timeoutMs,
+            maxBytes: this.config.maxBytesPerResponse,
+            maxRedirects: this.config.maxRedirects,
+            userAgent: this.config.userAgent,
+            allowLocalhostForTesting: this.allowLocalhostForTesting,
+        });
+    }
+    async ingest(targetUrl) {
+        const startTime = Date.now();
+        // 1. SSRF & Protocol validation before making any discovery probes
+        await validateTargetUrl(targetUrl, {
+            allowLocalhostForTesting: this.allowLocalhostForTesting,
+        });
+        const coordinator = createDefaultDiscoveryCoordinator();
+        const discovered = await coordinator.discoverAll(targetUrl, this.fetcher);
+        const sourceIds = {};
+        for (const s of discovered) {
+            const id = this.repository.saveSource(s);
+            sourceIds[s.url] = id;
+        }
+        const conceptualRank = rankSources(discovered, 'conceptual');
+        const apiRank = rankSources(discovered, 'api');
+        const primarySource = conceptualRank.recommended || discovered[0];
+        const primarySourceId = primarySource
+            ? (sourceIds[primarySource.url] || this.repository.saveSource(primarySource))
+            : this.repository.saveSource({
+                url: targetUrl,
+                type: 'web',
+                discoveredBy: 'direct',
+                status: 'valid',
+                confidence: 1.0,
+                authority: 'official',
+                machineReadable: false,
+            });
+        const purposes = ['navigation', 'conceptual', 'api', 'examples', 'implementation'];
+        const selectedSources = purposes.map(p => {
+            const ranked = rankSources(discovered, p);
+            return {
+                purpose: p,
+                sourceId: ranked.recommended ? (sourceIds[ranked.recommended.url] || primarySourceId) : primarySourceId,
+            };
+        });
+        const queue = [];
+        const visited = new Set();
+        const ingestedPages = [];
+        const warnings = [];
+        const errors = [];
+        let pagesDiscovered = 0;
+        const enqueue = (url, depth) => {
+            try {
+                const parsed = new URL(url);
+                const normalized = `${parsed.origin}${parsed.pathname}`;
+                pagesDiscovered++;
+                if (depth > this.policy.maxDepth)
+                    return;
+                if (!visited.has(normalized) && queue.length < this.policy.maxPages) {
+                    queue.push({ url: normalized, depth });
+                }
+            }
+            catch {
+                // Ignore
+            }
+        };
+        // 1. Prioritize targetUrl as the primary entry point (depth: 0)
+        enqueue(targetUrl, 0);
+        // 2. If an OpenAPI spec is available, enqueue it (depth: 0)
+        if (apiRank.recommended && apiRank.recommended.type === 'openapi' && apiRank.recommended.url !== targetUrl) {
+            enqueue(apiRank.recommended.url, 0);
+        }
+        // 3. If llms.txt or llms-full.txt is available, enqueue it (depth: 0)
+        const llmsFull = discovered.find(s => s.type === 'llms_full_txt' && s.status === 'valid');
+        const llms = discovered.find(s => s.type === 'llms_txt' && s.status === 'valid');
+        if (llmsFull && llmsFull.url !== targetUrl) {
+            enqueue(llmsFull.url, 0);
+        }
+        else if (llms && llms.url !== targetUrl) {
+            enqueue(llms.url, 0);
+        }
+        if (primarySource && primarySource.url !== targetUrl) {
+            enqueue(primarySource.url, 0);
+        }
+        const targetOrigin = new URL(targetUrl).origin;
+        while (queue.length > 0 && ingestedPages.length < this.policy.maxPages) {
+            const item = queue.shift();
+            const currentUrl = item.url;
+            const currentDepth = item.depth;
+            if (visited.has(currentUrl))
+                continue;
+            visited.add(currentUrl);
+            try {
+                const res = await this.fetcher.fetch(currentUrl);
+                if (res.status < 200 || res.status >= 300) {
+                    errors.push({ url: currentUrl, error: `HTTP status ${res.status}` });
+                    continue;
+                }
+                const page = buildNormalizedPage({
+                    sourceId: primarySourceId,
+                    url: res.finalUrl || currentUrl,
+                    rawContent: res.body,
+                    contentType: res.contentType,
+                    sourceUrl: primarySource?.url || targetUrl,
+                    targetUrl,
+                    discoveredBy: primarySource?.discoveredBy || 'direct',
+                    fetchedAt: new Date().toISOString(),
+                });
+                this.repository.savePage(page);
+                ingestedPages.push(page);
+                // Subpage link discovery
+                if (isValidLlmsTxt(res.body)) {
+                    if (this.policy.followLlmsReferences && currentDepth < this.policy.maxDepth) {
+                        const llmsDoc = parseLlmsTxt(res.body, currentUrl);
+                        for (const sec of llmsDoc.sections) {
+                            for (const lnk of sec.links) {
+                                if (ingestedPages.length + queue.length < this.policy.maxPages) {
+                                    try {
+                                        const parsedLnk = new URL(lnk.url);
+                                        if (this.policy.followExternalDomains || parsedLnk.origin === targetOrigin) {
+                                            enqueue(lnk.url, currentDepth + 1);
+                                        }
+                                    }
+                                    catch {
+                                        // Ignore
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                else {
+                    if (currentDepth < this.policy.maxDepth) {
+                        for (const lnk of page.links) {
+                            if (ingestedPages.length + queue.length < this.policy.maxPages) {
+                                try {
+                                    const linkUrl = new URL(lnk.url);
+                                    if (this.policy.followExternalDomains || linkUrl.origin === targetOrigin) {
+                                        enqueue(lnk.url, currentDepth + 1);
+                                    }
+                                }
+                                catch {
+                                    // Ignore
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                errors.push({ url: currentUrl, error: err instanceof Error ? err.message : String(err) });
+            }
+        }
+        const snapshotId = this.repository.createSnapshot(primarySourceId, {
+            targetUrl,
+            pageCount: ingestedPages.length,
+            ingestedAt: new Date().toISOString(),
+        });
+        // Milestone 2: Semantic slicing of pages into standalone DocumentChunks
+        let totalChunks = 0;
+        for (const p of ingestedPages) {
+            const slicingResult = slicePageIntoChunks(p, snapshotId);
+            this.repository.saveChunks(slicingResult.chunks, slicingResult.relationships, slicingResult.codeSnippets, slicingResult.symbols);
+            totalChunks += slicingResult.chunks.length;
+            // Milestone 4: API Endpoints
+            if (p.openApiSummary || detectOpenApiSpec(p.content, p.url)) {
+                const endpoints = parseOpenApiEndpoints(p.content, p.id, snapshotId, p.url, p.docVersion);
+                if (endpoints.length > 0) {
+                    this.repository.saveApiEndpoints(endpoints);
+                }
+            }
+            // Milestone 4: Indexed Examples
+            const examples = extractIndexedExamples(p, slicingResult.chunks, primarySource?.authority || 'official', p.docVersion, snapshotId);
+            if (examples.length > 0) {
+                this.repository.saveIndexedExamples(examples);
+            }
+            // Milestone 4: Pitfalls
+            const pitfalls = extractPitfalls(p, slicingResult.chunks, p.docVersion, snapshotId);
+            if (pitfalls.length > 0) {
+                this.repository.savePitfalls(pitfalls);
+            }
+        }
+        const durationMs = Date.now() - startTime;
+        let totalBytes = 0;
+        let totalEstimatedTokens = 0;
+        let totalCodeExamples = 0;
+        for (const p of ingestedPages) {
+            totalBytes += p.rawBytes;
+            totalEstimatedTokens += p.estimatedTokens;
+            totalCodeExamples += p.codeExamples.length;
+            if (p.securityAnnotations.length > 0) {
+                warnings.push(`Page "${p.url}" generated ${p.securityAnnotations.length} security alert(s).`);
+            }
+        }
+        return {
+            targetUrl,
+            target: {
+                type: 'url',
+                value: targetUrl,
+                normalizedUrl: targetUrl,
+            },
+            primarySourceId,
+            sourcesDiscovered: discovered,
+            selectedSources,
+            pages: ingestedPages,
+            pagesDiscovered,
+            pagesFetched: visited.size,
+            pagesStored: ingestedPages.length,
+            snapshotId,
+            warnings,
+            errors,
+            durationMs,
+            stats: {
+                totalPages: ingestedPages.length,
+                totalBytes,
+                totalEstimatedTokens,
+                totalCodeExamples,
+                machineReadableSources: discovered.filter(s => s.status === 'valid' && s.machineReadable).length,
+                totalChunks,
+            },
+        };
+    }
+}

package/dist/packages/crawler/src/config.d.ts

@@ -0,0 +1,2 @@
+import type { CrawlerConfig } from '../../shared/src/index.ts';
+export declare const DEFAULT_CRAWLER_CONFIG: CrawlerConfig;

package/dist/packages/crawler/src/config.js

@@ -0,0 +1,10 @@
+export const DEFAULT_CRAWLER_CONFIG = {
+    maxPages: 50,
+    maxDepth: 3,
+    maxBytesPerResponse: 10 * 1024 * 1024, // 10MB
+    timeoutMs: 10000, // 10 seconds
+    maxRedirects: 5,
+    concurrency: 5,
+    allowedProtocols: ['http:', 'https:'],
+    userAgent: 'DocOrbit/1.0 (+https://github.com/docorbit/docorbit)',
+};

package/dist/packages/crawler/src/fetcher.d.ts

@@ -0,0 +1,31 @@
+import type { SsrfValidationOptions } from '../../security/src/index.ts';
+export interface FetchResult {
+    url: string;
+    finalUrl: string;
+    status: number;
+    statusText: string;
+    headers: Record<string, string>;
+    body: string;
+    contentType: string;
+    bytesRead: number;
+}
+export interface FetchOptions extends SsrfValidationOptions {
+    timeoutMs?: number;
+    maxBytes?: number;
+    maxRedirects?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    method?: string;
+    fetchFn?: typeof fetch;
+    allowCrossDomainRedirects?: boolean;
+}
+export declare class SecureFetcher {
+    private timeoutMs;
+    private maxBytes;
+    private maxRedirects;
+    private userAgent;
+    private allowLocalhostForTesting;
+    private fetchFn;
+    constructor(defaults?: Partial<FetchOptions>);
+    fetch(urlStr: string, options?: FetchOptions): Promise<FetchResult>;
+}

package/dist/packages/crawler/src/fetcher.js

@@ -0,0 +1,139 @@
+import { FetchTimeoutError, PayloadTooLargeError, DocOrbitError, } from "../../shared/src/index.js";
+import { validateTargetUrl } from "../../security/src/index.js";
+import { DEFAULT_CRAWLER_CONFIG } from "./config.js";
+export class SecureFetcher {
+    timeoutMs;
+    maxBytes;
+    maxRedirects;
+    userAgent;
+    allowLocalhostForTesting;
+    fetchFn;
+    constructor(defaults = {}) {
+        this.timeoutMs = defaults.timeoutMs ?? DEFAULT_CRAWLER_CONFIG.timeoutMs;
+        this.maxBytes = defaults.maxBytes ?? DEFAULT_CRAWLER_CONFIG.maxBytesPerResponse;
+        this.maxRedirects = defaults.maxRedirects ?? DEFAULT_CRAWLER_CONFIG.maxRedirects;
+        this.userAgent = defaults.userAgent ?? DEFAULT_CRAWLER_CONFIG.userAgent;
+        this.allowLocalhostForTesting = defaults.allowLocalhostForTesting ?? false;
+        this.fetchFn = defaults.fetchFn ?? globalThis.fetch;
+    }
+    async fetch(urlStr, options = {}) {
+        const timeoutMs = options.timeoutMs ?? this.timeoutMs;
+        const maxBytes = options.maxBytes ?? this.maxBytes;
+        const maxRedirects = options.maxRedirects ?? this.maxRedirects;
+        const userAgent = options.userAgent ?? this.userAgent;
+        const method = options.method ?? 'GET';
+        const allowLocalhost = options.allowLocalhostForTesting ?? this.allowLocalhostForTesting;
+        const activeFetch = options.fetchFn ?? this.fetchFn;
+        let currentUrl = urlStr;
+        let redirectCount = 0;
+        const seenRedirects = new Set();
+        seenRedirects.add(currentUrl);
+        while (redirectCount <= maxRedirects) {
+            // 1. SSRF & Protocol validation before making the hop
+            await validateTargetUrl(currentUrl, {
+                allowLocalhostForTesting: allowLocalhost,
+            });
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => {
+                controller.abort(new FetchTimeoutError(`Request timed out after ${timeoutMs}ms`, timeoutMs));
+            }, timeoutMs);
+            try {
+                const reqHeaders = {
+                    'User-Agent': userAgent,
+                    Accept: 'text/html,application/xhtml+xml,text/plain,text/markdown,application/json,*/*',
+                    ...options.headers,
+                };
+                const res = await activeFetch(currentUrl, {
+                    method,
+                    headers: reqHeaders,
+                    redirect: 'manual',
+                    signal: controller.signal,
+                });
+                // 2. Handle HTTP redirects (301, 302, 303, 307, 308)
+                if ([301, 302, 303, 307, 308].includes(res.status)) {
+                    const location = res.headers.get('location');
+                    if (!location) {
+                        throw new DocOrbitError(`Redirect response (${res.status}) missing Location header.`);
+                    }
+                    redirectCount++;
+                    if (redirectCount > maxRedirects) {
+                        throw new DocOrbitError(`Exceeded maximum allowed redirects (${maxRedirects}).`);
+                    }
+                    const nextUrl = new URL(location, currentUrl).href;
+                    // Check for redirect loop
+                    if (seenRedirects.has(nextUrl)) {
+                        throw new DocOrbitError(`Redirect loop detected: "${nextUrl}" already visited in redirect chain.`);
+                    }
+                    seenRedirects.add(nextUrl);
+                    // Enforce domain/origin policy
+                    if (options.allowCrossDomainRedirects === false) {
+                        const origOrigin = new URL(urlStr).origin;
+                        const nextOrigin = new URL(nextUrl).origin;
+                        if (origOrigin !== nextOrigin) {
+                            throw new DocOrbitError(`Cross-origin redirect from "${origOrigin}" to "${nextOrigin}" prohibited by policy.`);
+                        }
+                    }
+                    currentUrl = nextUrl;
+                    continue;
+                }
+                // 3. Extract response headers
+                const headers = {};
+                res.headers.forEach((val, key) => {
+                    headers[key.toLowerCase()] = val;
+                });
+                const contentType = headers['content-type'] || 'application/octet-stream';
+                // 4. Stream response body with byte counting
+                let bytesRead = 0;
+                const chunks = [];
+                if (res.body) {
+                    const reader = res.body.getReader();
+                    try {
+                        while (true) {
+                            const { done, value } = await reader.read();
+                            if (done)
+                                break;
+                            if (value) {
+                                bytesRead += value.length;
+                                if (bytesRead > maxBytes) {
+                                    controller.abort();
+                                    throw new PayloadTooLargeError(`Response exceeded maximum allowed size of ${maxBytes} bytes (read ${bytesRead} bytes).`, bytesRead, maxBytes);
+                                }
+                                chunks.push(value);
+                            }
+                        }
+                    }
+                    finally {
+                        reader.releaseLock();
+                    }
+                }
+                const totalBuffer = Buffer.concat(chunks);
+                const bodyText = totalBuffer.toString('utf-8');
+                return {
+                    url: urlStr,
+                    finalUrl: currentUrl,
+                    status: res.status,
+                    statusText: res.statusText,
+                    headers,
+                    body: bodyText,
+                    contentType,
+                    bytesRead,
+                };
+            }
+            catch (err) {
+                if (err instanceof DocOrbitError)
+                    throw err;
+                if (controller.signal.aborted) {
+                    const reason = controller.signal.reason;
+                    if (reason instanceof DocOrbitError)
+                        throw reason;
+                    throw new FetchTimeoutError(`Request timed out after ${timeoutMs}ms`, timeoutMs);
+                }
+                throw new DocOrbitError(`Fetch failure for "${currentUrl}": ${err instanceof Error ? err.message : String(err)}`);
+            }
+            finally {
+                clearTimeout(timeoutId);
+            }
+        }
+        throw new DocOrbitError(`Exceeded maximum allowed redirects (${maxRedirects}).`);
+    }
+}

package/dist/packages/crawler/src/index.js

@@ -0,0 +1,2 @@
+export * from "./config.js";
+export * from "./fetcher.js";

package/dist/packages/discovery/src/index.d.ts

@@ -0,0 +1,11 @@
+import { DiscoveryCoordinator } from './provider.ts';
+export * from './provider.ts';
+export * from './ranker.ts';
+export * from './providers/llms-txt.ts';
+export * from './providers/openapi.ts';
+export * from './providers/markdown.ts';
+export * from './providers/sitemap.ts';
+export * from './providers/github.ts';
+export * from './providers/skill.ts';
+export * from './providers/generic.ts';
+export declare function createDefaultDiscoveryCoordinator(): DiscoveryCoordinator;

package/dist/packages/discovery/src/index.js

@@ -0,0 +1,29 @@
+import { DiscoveryCoordinator } from "./provider.js";
+import { LlmsTxtProvider } from "./providers/llms-txt.js";
+import { OpenApiProvider } from "./providers/openapi.js";
+import { MarkdownProvider } from "./providers/markdown.js";
+import { SitemapProvider } from "./providers/sitemap.js";
+import { GithubProvider } from "./providers/github.js";
+import { SkillProvider } from "./providers/skill.js";
+import { GenericWebProvider } from "./providers/generic.js";
+export * from "./provider.js";
+export * from "./ranker.js";
+export * from "./providers/llms-txt.js";
+export * from "./providers/openapi.js";
+export * from "./providers/markdown.js";
+export * from "./providers/sitemap.js";
+export * from "./providers/github.js";
+export * from "./providers/skill.js";
+export * from "./providers/generic.js";
+export function createDefaultDiscoveryCoordinator() {
+    const coordinator = new DiscoveryCoordinator();
+    coordinator
+        .register(new LlmsTxtProvider())
+        .register(new OpenApiProvider())
+        .register(new MarkdownProvider())
+        .register(new SitemapProvider())
+        .register(new SkillProvider())
+        .register(new GithubProvider())
+        .register(new GenericWebProvider());
+    return coordinator;
+}

package/dist/packages/discovery/src/provider.d.ts

@@ -0,0 +1,12 @@
+import type { DiscoveredSource } from '../../shared/src/index.ts';
+import { SecureFetcher } from '../../crawler/src/index.ts';
+export interface DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}
+export declare class DiscoveryCoordinator {
+    private providers;
+    register(provider: DiscoveryProvider): this;
+    getProviders(): DiscoveryProvider[];
+    discoverAll(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/provider.js

@@ -0,0 +1,30 @@
+export class DiscoveryCoordinator {
+    providers = [];
+    register(provider) {
+        this.providers.push(provider);
+        return this;
+    }
+    getProviders() {
+        return [...this.providers];
+    }
+    async discoverAll(targetUrl, fetcher) {
+        const results = await Promise.allSettled(this.providers.map(p => p.discover(targetUrl, fetcher)));
+        const allSources = [];
+        for (const res of results) {
+            if (res.status === 'fulfilled') {
+                allSources.push(...res.value);
+            }
+        }
+        // Deduplicate sources by normalized URL
+        const seen = new Set();
+        const deduped = [];
+        for (const s of allSources) {
+            const normalizedUrl = s.url.replace(/\/$/, '').toLowerCase();
+            if (!seen.has(normalizedUrl)) {
+                seen.add(normalizedUrl);
+                deduped.push(s);
+            }
+        }
+        return deduped;
+    }
+}

package/dist/packages/discovery/src/providers/generic.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class GenericWebProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/providers/generic.js

@@ -0,0 +1,94 @@
+export class GenericWebProvider {
+    name = 'generic_web';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        try {
+            const res = await fetcher.fetch(targetUrl, {
+                timeoutMs: 5000,
+            });
+            if (res.status >= 200 && res.status < 300) {
+                let framework = 'unknown';
+                const bodyLower = res.body.toLowerCase();
+                // Check generator meta tags or distinctive DOM anchors first
+                const metaGenerator = res.body.match(/<meta\b[^>]*\bname=["']generator["'][^>]*\bcontent=["']([^"']+)["']/i);
+                const generatorVal = metaGenerator ? metaGenerator[1].toLowerCase() : '';
+                if (generatorVal.includes('docusaurus') || res.body.includes('__docusaurus') || res.body.includes('docusaurus-plugin')) {
+                    framework = 'docusaurus';
+                }
+                else if (generatorVal.includes('mintlify') || res.body.includes('_mintlify') || res.body.includes('mintlify.css') || res.body.includes('mintlify.com')) {
+                    framework = 'mintlify';
+                }
+                else if (generatorVal.includes('nextra') || res.body.includes('__nextra')) {
+                    framework = 'nextra';
+                }
+                else if (generatorVal.includes('gitbook') || res.body.includes('gitbook-root') || res.body.includes('data-gitbook')) {
+                    framework = 'gitbook';
+                }
+                else if (generatorVal.includes('mkdocs') || res.body.includes('mkdocs')) {
+                    framework = 'mkdocs';
+                }
+                else if (generatorVal.includes('sphinx') || res.body.includes('sphinxsidebar')) {
+                    framework = 'sphinx';
+                }
+                // Section 18: Detect candidate doc sections without blindly crawling
+                const candidateSections = [];
+                const sectionRegex = /href=["']([^"'#?]+(?:\/docs|\/api|\/reference|\/guides|\/tutorials|\/changelog)[^"'#?]*)["']/gi;
+                let sMatch;
+                while ((sMatch = sectionRegex.exec(res.body)) !== null) {
+                    try {
+                        const secUrl = new URL(sMatch[1], targetUrl).href;
+                        if (new URL(secUrl).origin === new URL(targetUrl).origin && !candidateSections.includes(secUrl)) {
+                            candidateSections.push(secUrl);
+                            if (candidateSections.length >= 10)
+                                break;
+                        }
+                    }
+                    catch {
+                        // Ignore
+                    }
+                }
+                const linkHeader = res.headers['link'];
+                if (linkHeader) {
+                    const llmsMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?(?:llms-txt|alternate)["']?/i);
+                    if (llmsMatch) {
+                        try {
+                            const fullLlmsUrl = new URL(llmsMatch[1], targetUrl).href;
+                            results.push({
+                                url: fullLlmsUrl,
+                                type: 'llms_txt',
+                                discoveredBy: `${this.name}:link-header`,
+                                status: 'valid',
+                                confidence: 0.99,
+                                authority: 'official',
+                                machineReadable: true,
+                            });
+                        }
+                        catch {
+                            // Ignore
+                        }
+                    }
+                }
+                results.push({
+                    url: res.finalUrl || targetUrl,
+                    type: 'web',
+                    discoveredBy: this.name,
+                    status: 'valid',
+                    contentType: res.contentType,
+                    confidence: 0.85,
+                    authority: 'official',
+                    machineReadable: false,
+                    metadata: {
+                        framework,
+                        status: res.status,
+                        bytes: res.bytesRead,
+                        candidateSections,
+                    },
+                });
+            }
+        }
+        catch {
+            // Continue
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/providers/github.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class GithubProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}