docorbit 0.1.0 → 0.1.2

package/dist/packages/discovery/src/providers/github.js

@@ -0,0 +1,52 @@
+export class GithubProvider {
+    name = 'github';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        const isDirectGithub = /^https?:\/\/(?:www\.)?github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+/i.test(targetUrl);
+        if (isDirectGithub) {
+            results.push({
+                url: targetUrl,
+                type: 'github',
+                discoveredBy: this.name,
+                status: 'valid',
+                confidence: 0.99,
+                authority: 'official',
+                machineReadable: false,
+            });
+            return results;
+        }
+        try {
+            const res = await fetcher.fetch(targetUrl, {
+                timeoutMs: 4000,
+            });
+            if (res.status >= 200 && res.status < 300 && res.body) {
+                const githubMatches = res.body.matchAll(/href=["'](https?:\/\/(?:www\.)?github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+(?:\/)?)(?:["']|[/?#])/gi);
+                const seenRepos = new Set();
+                for (const m of githubMatches) {
+                    const repoUrl = m[1].replace(/\/$/, '');
+                    if (repoUrl.includes('/topics') || repoUrl.includes('/features') || repoUrl.includes('/pricing'))
+                        continue;
+                    if (!seenRepos.has(repoUrl)) {
+                        seenRepos.add(repoUrl);
+                        results.push({
+                            url: repoUrl,
+                            type: 'github',
+                            discoveredBy: this.name,
+                            status: 'valid',
+                            confidence: 0.85,
+                            authority: 'official',
+                            machineReadable: false,
+                            metadata: {
+                                foundOn: targetUrl,
+                            },
+                        });
+                    }
+                }
+            }
+        }
+        catch {
+            // Continue
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/providers/llms-txt.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class LlmsTxtProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/providers/llms-txt.js

@@ -0,0 +1,60 @@
+import { isValidLlmsTxt } from "../../../normalizer/src/index.js";
+export class LlmsTxtProvider {
+    name = 'llms_txt';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        let baseUrl;
+        try {
+            baseUrl = new URL(targetUrl);
+        }
+        catch {
+            return results;
+        }
+        const cleanPath = baseUrl.pathname.replace(/\/$/, '');
+        // Probe both subpath-relative and origin-root paths
+        const candidatePaths = [
+            { path: '/llms.txt', type: 'llms_txt' },
+            { path: '/llms-full.txt', type: 'llms_full_txt' },
+            { path: '/.well-known/llms.txt', type: 'llms_txt' },
+        ];
+        if (cleanPath && cleanPath !== '') {
+            candidatePaths.unshift({ path: `${cleanPath}/llms.txt`, type: 'llms_txt' }, { path: `${cleanPath}/llms-full.txt`, type: 'llms_full_txt' });
+        }
+        const seenUrls = new Set();
+        for (const item of candidatePaths) {
+            const probeUrl = new URL(item.path, baseUrl.origin).href;
+            if (seenUrls.has(probeUrl))
+                continue;
+            seenUrls.add(probeUrl);
+            try {
+                const res = await fetcher.fetch(probeUrl, {
+                    timeoutMs: 4000,
+                });
+                if (res.status >= 200 && res.status < 300) {
+                    const isValid = isValidLlmsTxt(res.body);
+                    const isNotHtml = !res.contentType.includes('text/html') && !res.body.includes('<html');
+                    if (isValid || (item.type === 'llms_full_txt' && isNotHtml)) {
+                        results.push({
+                            url: probeUrl,
+                            type: item.type,
+                            discoveredBy: this.name,
+                            status: 'valid',
+                            contentType: res.contentType,
+                            confidence: 0.98,
+                            authority: 'official',
+                            machineReadable: true,
+                            metadata: {
+                                bytes: res.bytesRead,
+                                path: item.path,
+                            },
+                        });
+                    }
+                }
+            }
+            catch {
+                // Continue
+            }
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/providers/markdown.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class MarkdownProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/providers/markdown.js

@@ -0,0 +1,38 @@
+export class MarkdownProvider {
+    name = 'markdown';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        const mdProbeUrl = targetUrl.endsWith('.md')
+            ? targetUrl
+            : targetUrl.replace(/\/$/, '') + '.md';
+        try {
+            const res = await fetcher.fetch(mdProbeUrl, {
+                timeoutMs: 4000,
+            });
+            if (res.status >= 200 && res.status < 300) {
+                const isMarkdown = res.contentType.includes('text/markdown') ||
+                    res.contentType.includes('text/plain') ||
+                    res.body.startsWith('#');
+                if (isMarkdown && !res.body.includes('<html')) {
+                    results.push({
+                        url: mdProbeUrl,
+                        type: 'markdown',
+                        discoveredBy: this.name,
+                        status: 'valid',
+                        contentType: res.contentType,
+                        confidence: 0.95,
+                        authority: 'official',
+                        machineReadable: true,
+                        metadata: {
+                            bytes: res.bytesRead,
+                        },
+                    });
+                }
+            }
+        }
+        catch {
+            // Continue
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/providers/openapi.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class OpenApiProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/providers/openapi.js

@@ -0,0 +1,77 @@
+import { detectOpenApiSpec } from "../../../normalizer/src/index.js";
+export class OpenApiProvider {
+    name = 'openapi';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        let baseUrl;
+        try {
+            baseUrl = new URL(targetUrl);
+        }
+        catch {
+            return results;
+        }
+        const cleanPath = baseUrl.pathname.replace(/\/$/, '');
+        const candidatePaths = [
+            '/openapi.json',
+            '/openapi.yaml',
+            '/swagger.json',
+            '/v3/api-docs',
+            '/api-docs',
+        ];
+        if (cleanPath && cleanPath !== '') {
+            candidatePaths.unshift(`${cleanPath}/openapi.json`, `${cleanPath}/openapi.yaml`, `${cleanPath}/swagger.json`);
+        }
+        // Also probe targetUrl HTML for <link rel="service-doc">
+        try {
+            const pageRes = await fetcher.fetch(targetUrl, { timeoutMs: 4000 });
+            if (pageRes.status >= 200 && pageRes.status < 300 && pageRes.body) {
+                const linkMatch = pageRes.body.match(/<link\b[^>]*\brel=["'](?:service-doc|openapi|swagger)["'][^>]*\bhref=["']([^"']+)["']/i);
+                if (linkMatch) {
+                    const docHref = new URL(linkMatch[1], targetUrl).href;
+                    candidatePaths.unshift(new URL(docHref).pathname);
+                }
+            }
+        }
+        catch {
+            // Continue
+        }
+        const seenUrls = new Set();
+        for (const path of candidatePaths) {
+            const probeUrl = new URL(path, baseUrl.origin).href;
+            if (seenUrls.has(probeUrl))
+                continue;
+            seenUrls.add(probeUrl);
+            try {
+                const res = await fetcher.fetch(probeUrl, {
+                    timeoutMs: 4000,
+                });
+                if (res.status >= 200 && res.status < 300) {
+                    const spec = detectOpenApiSpec(res.body, probeUrl);
+                    if (spec) {
+                        results.push({
+                            url: probeUrl,
+                            type: 'openapi',
+                            discoveredBy: this.name,
+                            status: 'valid',
+                            contentType: res.contentType,
+                            confidence: 0.99,
+                            authority: 'official',
+                            machineReadable: true,
+                            metadata: {
+                                specVersion: spec.specVersion,
+                                title: spec.title,
+                                pathCount: spec.pathCount,
+                                servers: spec.servers,
+                            },
+                        });
+                        break;
+                    }
+                }
+            }
+            catch {
+                // Continue
+            }
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/providers/sitemap.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class SitemapProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/providers/sitemap.js

@@ -0,0 +1,53 @@
+export class SitemapProvider {
+    name = 'sitemap';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        let baseUrl;
+        try {
+            baseUrl = new URL(targetUrl);
+        }
+        catch {
+            return results;
+        }
+        const cleanPath = baseUrl.pathname.replace(/\/$/, '');
+        const candidatePaths = ['/sitemap.xml', '/sitemap_index.xml'];
+        if (cleanPath && cleanPath !== '') {
+            candidatePaths.unshift(`${cleanPath}/sitemap.xml`);
+        }
+        const seenUrls = new Set();
+        for (const path of candidatePaths) {
+            const probeUrl = new URL(path, baseUrl.origin).href;
+            if (seenUrls.has(probeUrl))
+                continue;
+            seenUrls.add(probeUrl);
+            try {
+                const res = await fetcher.fetch(probeUrl, {
+                    timeoutMs: 4000,
+                });
+                if (res.status >= 200 && res.status < 300) {
+                    const isXml = res.contentType.includes('xml') || res.body.includes('<urlset') || res.body.includes('<sitemapindex');
+                    if (isXml) {
+                        results.push({
+                            url: probeUrl,
+                            type: 'sitemap',
+                            discoveredBy: this.name,
+                            status: 'valid',
+                            contentType: res.contentType,
+                            confidence: 0.9,
+                            authority: 'official',
+                            machineReadable: true,
+                            metadata: {
+                                bytes: res.bytesRead,
+                            },
+                        });
+                        break;
+                    }
+                }
+            }
+            catch {
+                // Continue
+            }
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/providers/skill.d.ts

@@ -0,0 +1,7 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+export declare class SkillProvider implements DiscoveryProvider {
+    name: string;
+    discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}

package/dist/packages/discovery/src/providers/skill.js

@@ -0,0 +1,46 @@
+export class SkillProvider {
+    name = 'skill';
+    async discover(targetUrl, fetcher) {
+        const results = [];
+        let baseUrl;
+        try {
+            baseUrl = new URL(targetUrl);
+        }
+        catch {
+            return results;
+        }
+        const candidatePaths = ['/skill.md', '/.well-known/skills/skill.md'];
+        for (const path of candidatePaths) {
+            const probeUrl = new URL(path, baseUrl.origin).href;
+            try {
+                const res = await fetcher.fetch(probeUrl, {
+                    timeoutMs: 4000,
+                });
+                if (res.status >= 200 && res.status < 300) {
+                    const hasYamlHeader = /^---\s*\n([\s\S]*?)\n---/m.test(res.body);
+                    const hasName = /\bname:\s*.+/i.test(res.body);
+                    if (hasYamlHeader && hasName) {
+                        results.push({
+                            url: probeUrl,
+                            type: 'skill',
+                            discoveredBy: this.name,
+                            status: 'valid',
+                            contentType: res.contentType,
+                            confidence: 0.95,
+                            authority: 'official',
+                            machineReadable: true,
+                            metadata: {
+                                bytes: res.bytesRead,
+                            },
+                        });
+                        break;
+                    }
+                }
+            }
+            catch {
+                // Continue
+            }
+        }
+        return results;
+    }
+}

package/dist/packages/discovery/src/ranker.d.ts

@@ -0,0 +1,6 @@
+import type { DiscoveredSource, SourcePurpose, SourceRankResult } from '../../shared/src/index.ts';
+/**
+ * Ranks discovered sources based on the intended purpose:
+ * navigation, conceptual, api, examples, or implementation.
+ */
+export declare function rankSources(sources: DiscoveredSource[], purpose?: SourcePurpose): SourceRankResult;

package/dist/packages/discovery/src/ranker.js

@@ -0,0 +1,107 @@
+const PURPOSE_TYPE_WEIGHTS = {
+    navigation: {
+        llms_txt: 1.0,
+        sitemap: 0.9,
+        web: 0.7,
+        llms_full_txt: 0.6,
+        markdown: 0.5,
+        skill: 0.45,
+        openapi: 0.4,
+        github: 0.3,
+    },
+    conceptual: {
+        markdown: 1.0,
+        llms_full_txt: 0.95,
+        llms_txt: 0.85,
+        web: 0.75,
+        skill: 0.6,
+        openapi: 0.5,
+        github: 0.45,
+        sitemap: 0.3,
+    },
+    api: {
+        openapi: 1.0,
+        markdown: 0.7,
+        llms_full_txt: 0.65,
+        web: 0.5,
+        llms_txt: 0.4,
+        github: 0.35,
+        skill: 0.3,
+        sitemap: 0.2,
+    },
+    examples: {
+        github: 1.0,
+        markdown: 0.85,
+        llms_full_txt: 0.8,
+        web: 0.6,
+        skill: 0.5,
+        llms_txt: 0.45,
+        openapi: 0.4,
+        sitemap: 0.2,
+    },
+    implementation: {
+        skill: 1.0,
+        markdown: 0.9,
+        llms_txt: 0.85,
+        llms_full_txt: 0.8,
+        web: 0.65,
+        openapi: 0.6,
+        github: 0.55,
+        sitemap: 0.2,
+    },
+};
+const AUTHORITY_WEIGHTS = {
+    official: 1.0,
+    community: 0.8,
+    third_party: 0.6,
+};
+function scoreSource(source, purpose) {
+    if (source.status === 'unreachable')
+        return 0;
+    if (source.status === 'invalid')
+        return 0.05;
+    const typeWeights = PURPOSE_TYPE_WEIGHTS[purpose] || PURPOSE_TYPE_WEIGHTS.conceptual;
+    const baseWeight = typeWeights[source.type] || 0.4;
+    const authorityWeight = AUTHORITY_WEIGHTS[source.authority] || 0.6;
+    const machineBonus = source.machineReadable ? 0.05 : 0.0;
+    const score = (baseWeight * authorityWeight + machineBonus) * source.confidence;
+    return Math.min(1.0, Math.max(0.0, score));
+}
+/**
+ * Ranks discovered sources based on the intended purpose:
+ * navigation, conceptual, api, examples, or implementation.
+ */
+export function rankSources(sources, purpose = 'conceptual') {
+    if (!sources || sources.length === 0) {
+        return {
+            purpose,
+            recommended: null,
+            ranked: [],
+            rationale: `No sources available to rank for purpose "${purpose}".`,
+        };
+    }
+    const scored = sources.map(s => ({
+        source: s,
+        score: scoreSource(s, purpose),
+    }));
+    scored.sort((a, b) => {
+        if (b.score !== a.score)
+            return b.score - a.score;
+        return a.source.url.localeCompare(b.source.url);
+    });
+    const ranked = scored.map(item => item.source);
+    const recommended = ranked.find(s => s.status === 'valid') || null;
+    let rationale = `Ranked ${ranked.length} source(s) for purpose "${purpose}".`;
+    if (recommended) {
+        rationale += ` Recommended "${recommended.url}" (${recommended.type}, authority: ${recommended.authority}, machine-readable: ${recommended.machineReadable}).`;
+    }
+    else {
+        rationale += ' No valid candidate sources found.';
+    }
+    return {
+        purpose,
+        recommended,
+        ranked,
+        rationale,
+    };
+}

package/dist/packages/evaluation/src/dataset.d.ts

@@ -0,0 +1,2 @@
+import type { BenchmarkTaskDef } from './types.ts';
+export declare const BENCHMARK_DATASET: BenchmarkTaskDef[];