docorbit 0.1.0

package/packages/discovery/src/providers/generic.ts

@@ -0,0 +1,98 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class GenericWebProvider implements DiscoveryProvider {
+  name = 'generic_web';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+
+    try {
+      const res = await fetcher.fetch(targetUrl, {
+        timeoutMs: 5000,
+      });
+
+      if (res.status >= 200 && res.status < 300) {
+        let framework = 'unknown';
+        const bodyLower = res.body.toLowerCase();
+
+        // Check generator meta tags or distinctive DOM anchors first
+        const metaGenerator = res.body.match(/<meta\b[^>]*\bname=["']generator["'][^>]*\bcontent=["']([^"']+)["']/i);
+        const generatorVal = metaGenerator ? metaGenerator[1].toLowerCase() : '';
+
+        if (generatorVal.includes('docusaurus') || res.body.includes('__docusaurus') || res.body.includes('docusaurus-plugin')) {
+          framework = 'docusaurus';
+        } else if (generatorVal.includes('mintlify') || res.body.includes('_mintlify') || res.body.includes('mintlify.css') || res.body.includes('mintlify.com')) {
+          framework = 'mintlify';
+        } else if (generatorVal.includes('nextra') || res.body.includes('__nextra')) {
+          framework = 'nextra';
+        } else if (generatorVal.includes('gitbook') || res.body.includes('gitbook-root') || res.body.includes('data-gitbook')) {
+          framework = 'gitbook';
+        } else if (generatorVal.includes('mkdocs') || res.body.includes('mkdocs')) {
+          framework = 'mkdocs';
+        } else if (generatorVal.includes('sphinx') || res.body.includes('sphinxsidebar')) {
+          framework = 'sphinx';
+        }
+
+        // Section 18: Detect candidate doc sections without blindly crawling
+        const candidateSections: string[] = [];
+        const sectionRegex = /href=["']([^"'#?]+(?:\/docs|\/api|\/reference|\/guides|\/tutorials|\/changelog)[^"'#?]*)["']/gi;
+        let sMatch: RegExpExecArray | null;
+        while ((sMatch = sectionRegex.exec(res.body)) !== null) {
+          try {
+            const secUrl = new URL(sMatch[1], targetUrl).href;
+            if (new URL(secUrl).origin === new URL(targetUrl).origin && !candidateSections.includes(secUrl)) {
+              candidateSections.push(secUrl);
+              if (candidateSections.length >= 10) break;
+            }
+          } catch {
+            // Ignore
+          }
+        }
+
+        const linkHeader = res.headers['link'];
+        if (linkHeader) {
+          const llmsMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?(?:llms-txt|alternate)["']?/i);
+          if (llmsMatch) {
+            try {
+              const fullLlmsUrl = new URL(llmsMatch[1], targetUrl).href;
+              results.push({
+                url: fullLlmsUrl,
+                type: 'llms_txt',
+                discoveredBy: `${this.name}:link-header`,
+                status: 'valid',
+                confidence: 0.99,
+                authority: 'official',
+                machineReadable: true,
+              });
+            } catch {
+              // Ignore
+            }
+          }
+        }
+
+        results.push({
+          url: res.finalUrl || targetUrl,
+          type: 'web',
+          discoveredBy: this.name,
+          status: 'valid',
+          contentType: res.contentType,
+          confidence: 0.85,
+          authority: 'official',
+          machineReadable: false,
+          metadata: {
+            framework,
+            status: res.status,
+            bytes: res.bytesRead,
+            candidateSections,
+          },
+        });
+      }
+    } catch {
+      // Continue
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/providers/github.ts

@@ -0,0 +1,61 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class GithubProvider implements DiscoveryProvider {
+  name = 'github';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+
+    const isDirectGithub = /^https?:\/\/(?:www\.)?github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+/i.test(targetUrl);
+    if (isDirectGithub) {
+      results.push({
+        url: targetUrl,
+        type: 'github',
+        discoveredBy: this.name,
+        status: 'valid',
+        confidence: 0.99,
+        authority: 'official',
+        machineReadable: false,
+      });
+      return results;
+    }
+
+    try {
+      const res = await fetcher.fetch(targetUrl, {
+        timeoutMs: 4000,
+      });
+
+      if (res.status >= 200 && res.status < 300 && res.body) {
+        const githubMatches = res.body.matchAll(/href=["'](https?:\/\/(?:www\.)?github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+(?:\/)?)(?:["']|[/?#])/gi);
+        const seenRepos = new Set<string>();
+
+        for (const m of githubMatches) {
+          const repoUrl = m[1].replace(/\/$/, '');
+          if (repoUrl.includes('/topics') || repoUrl.includes('/features') || repoUrl.includes('/pricing')) continue;
+
+          if (!seenRepos.has(repoUrl)) {
+            seenRepos.add(repoUrl);
+            results.push({
+              url: repoUrl,
+              type: 'github',
+              discoveredBy: this.name,
+              status: 'valid',
+              confidence: 0.85,
+              authority: 'official',
+              machineReadable: false,
+              metadata: {
+                foundOn: targetUrl,
+              },
+            });
+          }
+        }
+      }
+    } catch {
+      // Continue
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/providers/llms-txt.ts

@@ -0,0 +1,73 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import { isValidLlmsTxt } from '../../../normalizer/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class LlmsTxtProvider implements DiscoveryProvider {
+  name = 'llms_txt';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+    let baseUrl: URL;
+    try {
+      baseUrl = new URL(targetUrl);
+    } catch {
+      return results;
+    }
+
+    const cleanPath = baseUrl.pathname.replace(/\/$/, '');
+
+    // Probe both subpath-relative and origin-root paths
+    const candidatePaths: Array<{ path: string; type: 'llms_txt' | 'llms_full_txt' }> = [
+      { path: '/llms.txt', type: 'llms_txt' },
+      { path: '/llms-full.txt', type: 'llms_full_txt' },
+      { path: '/.well-known/llms.txt', type: 'llms_txt' },
+    ];
+
+    if (cleanPath && cleanPath !== '') {
+      candidatePaths.unshift(
+        { path: `${cleanPath}/llms.txt`, type: 'llms_txt' },
+        { path: `${cleanPath}/llms-full.txt`, type: 'llms_full_txt' }
+      );
+    }
+
+    const seenUrls = new Set<string>();
+
+    for (const item of candidatePaths) {
+      const probeUrl = new URL(item.path, baseUrl.origin).href;
+      if (seenUrls.has(probeUrl)) continue;
+      seenUrls.add(probeUrl);
+
+      try {
+        const res = await fetcher.fetch(probeUrl, {
+          timeoutMs: 4000,
+        });
+
+        if (res.status >= 200 && res.status < 300) {
+          const isValid = isValidLlmsTxt(res.body);
+          const isNotHtml = !res.contentType.includes('text/html') && !res.body.includes('<html');
+          if (isValid || (item.type === 'llms_full_txt' && isNotHtml)) {
+            results.push({
+              url: probeUrl,
+              type: item.type,
+              discoveredBy: this.name,
+              status: 'valid',
+              contentType: res.contentType,
+              confidence: 0.98,
+              authority: 'official',
+              machineReadable: true,
+              metadata: {
+                bytes: res.bytesRead,
+                path: item.path,
+              },
+            });
+          }
+        }
+      } catch {
+        // Continue
+      }
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/providers/markdown.ts

@@ -0,0 +1,48 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class MarkdownProvider implements DiscoveryProvider {
+  name = 'markdown';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+
+    const mdProbeUrl = targetUrl.endsWith('.md')
+      ? targetUrl
+      : targetUrl.replace(/\/$/, '') + '.md';
+
+    try {
+      const res = await fetcher.fetch(mdProbeUrl, {
+        timeoutMs: 4000,
+      });
+
+      if (res.status >= 200 && res.status < 300) {
+        const isMarkdown =
+          res.contentType.includes('text/markdown') ||
+          res.contentType.includes('text/plain') ||
+          res.body.startsWith('#');
+
+        if (isMarkdown && !res.body.includes('<html')) {
+          results.push({
+            url: mdProbeUrl,
+            type: 'markdown',
+            discoveredBy: this.name,
+            status: 'valid',
+            contentType: res.contentType,
+            confidence: 0.95,
+            authority: 'official',
+            machineReadable: true,
+            metadata: {
+              bytes: res.bytesRead,
+            },
+          });
+        }
+      }
+    } catch {
+      // Continue
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/providers/openapi.ts

@@ -0,0 +1,91 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import { detectOpenApiSpec } from '../../../normalizer/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class OpenApiProvider implements DiscoveryProvider {
+  name = 'openapi';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+    let baseUrl: URL;
+    try {
+      baseUrl = new URL(targetUrl);
+    } catch {
+      return results;
+    }
+
+    const cleanPath = baseUrl.pathname.replace(/\/$/, '');
+
+    const candidatePaths = [
+      '/openapi.json',
+      '/openapi.yaml',
+      '/swagger.json',
+      '/v3/api-docs',
+      '/api-docs',
+    ];
+
+    if (cleanPath && cleanPath !== '') {
+      candidatePaths.unshift(
+        `${cleanPath}/openapi.json`,
+        `${cleanPath}/openapi.yaml`,
+        `${cleanPath}/swagger.json`
+      );
+    }
+
+    // Also probe targetUrl HTML for <link rel="service-doc">
+    try {
+      const pageRes = await fetcher.fetch(targetUrl, { timeoutMs: 4000 });
+      if (pageRes.status >= 200 && pageRes.status < 300 && pageRes.body) {
+        const linkMatch = pageRes.body.match(/<link\b[^>]*\brel=["'](?:service-doc|openapi|swagger)["'][^>]*\bhref=["']([^"']+)["']/i);
+        if (linkMatch) {
+          const docHref = new URL(linkMatch[1], targetUrl).href;
+          candidatePaths.unshift(new URL(docHref).pathname);
+        }
+      }
+    } catch {
+      // Continue
+    }
+
+    const seenUrls = new Set<string>();
+
+    for (const path of candidatePaths) {
+      const probeUrl = new URL(path, baseUrl.origin).href;
+      if (seenUrls.has(probeUrl)) continue;
+      seenUrls.add(probeUrl);
+
+      try {
+        const res = await fetcher.fetch(probeUrl, {
+          timeoutMs: 4000,
+        });
+
+        if (res.status >= 200 && res.status < 300) {
+          const spec = detectOpenApiSpec(res.body, probeUrl);
+          if (spec) {
+            results.push({
+              url: probeUrl,
+              type: 'openapi',
+              discoveredBy: this.name,
+              status: 'valid',
+              contentType: res.contentType,
+              confidence: 0.99,
+              authority: 'official',
+              machineReadable: true,
+              metadata: {
+                specVersion: spec.specVersion,
+                title: spec.title,
+                pathCount: spec.pathCount,
+                servers: spec.servers,
+              },
+            });
+            break;
+          }
+        }
+      } catch {
+        // Continue
+      }
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/providers/sitemap.ts

@@ -0,0 +1,62 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class SitemapProvider implements DiscoveryProvider {
+  name = 'sitemap';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+    let baseUrl: URL;
+    try {
+      baseUrl = new URL(targetUrl);
+    } catch {
+      return results;
+    }
+
+    const cleanPath = baseUrl.pathname.replace(/\/$/, '');
+
+    const candidatePaths = ['/sitemap.xml', '/sitemap_index.xml'];
+    if (cleanPath && cleanPath !== '') {
+      candidatePaths.unshift(`${cleanPath}/sitemap.xml`);
+    }
+
+    const seenUrls = new Set<string>();
+
+    for (const path of candidatePaths) {
+      const probeUrl = new URL(path, baseUrl.origin).href;
+      if (seenUrls.has(probeUrl)) continue;
+      seenUrls.add(probeUrl);
+
+      try {
+        const res = await fetcher.fetch(probeUrl, {
+          timeoutMs: 4000,
+        });
+
+        if (res.status >= 200 && res.status < 300) {
+          const isXml = res.contentType.includes('xml') || res.body.includes('<urlset') || res.body.includes('<sitemapindex');
+          if (isXml) {
+            results.push({
+              url: probeUrl,
+              type: 'sitemap',
+              discoveredBy: this.name,
+              status: 'valid',
+              contentType: res.contentType,
+              confidence: 0.9,
+              authority: 'official',
+              machineReadable: true,
+              metadata: {
+                bytes: res.bytesRead,
+              },
+            });
+            break;
+          }
+        }
+      } catch {
+        // Continue
+      }
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/providers/skill.ts

@@ -0,0 +1,54 @@
+import type { DiscoveredSource } from '../../../shared/src/index.ts';
+import { SecureFetcher } from '../../../crawler/src/index.ts';
+import type { DiscoveryProvider } from '../provider.ts';
+
+export class SkillProvider implements DiscoveryProvider {
+  name = 'skill';
+
+  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results: DiscoveredSource[] = [];
+    let baseUrl: URL;
+    try {
+      baseUrl = new URL(targetUrl);
+    } catch {
+      return results;
+    }
+
+    const candidatePaths = ['/skill.md', '/.well-known/skills/skill.md'];
+
+    for (const path of candidatePaths) {
+      const probeUrl = new URL(path, baseUrl.origin).href;
+      try {
+        const res = await fetcher.fetch(probeUrl, {
+          timeoutMs: 4000,
+        });
+
+        if (res.status >= 200 && res.status < 300) {
+          const hasYamlHeader = /^---\s*\n([\s\S]*?)\n---/m.test(res.body);
+          const hasName = /\bname:\s*.+/i.test(res.body);
+
+          if (hasYamlHeader && hasName) {
+            results.push({
+              url: probeUrl,
+              type: 'skill',
+              discoveredBy: this.name,
+              status: 'valid',
+              contentType: res.contentType,
+              confidence: 0.95,
+              authority: 'official',
+              machineReadable: true,
+              metadata: {
+                bytes: res.bytesRead,
+              },
+            });
+            break;
+          }
+        }
+      } catch {
+        // Continue
+      }
+    }
+
+    return results;
+  }
+}

package/packages/discovery/src/ranker.ts

@@ -0,0 +1,123 @@
+import type {
+  DiscoveredSource,
+  SourcePurpose,
+  SourceRankResult,
+  SourceType,
+} from '../../shared/src/index.ts';
+
+const PURPOSE_TYPE_WEIGHTS: Record<SourcePurpose, Record<SourceType, number>> = {
+  navigation: {
+    llms_txt: 1.0,
+    sitemap: 0.9,
+    web: 0.7,
+    llms_full_txt: 0.6,
+    markdown: 0.5,
+    skill: 0.45,
+    openapi: 0.4,
+    github: 0.3,
+  },
+  conceptual: {
+    markdown: 1.0,
+    llms_full_txt: 0.95,
+    llms_txt: 0.85,
+    web: 0.75,
+    skill: 0.6,
+    openapi: 0.5,
+    github: 0.45,
+    sitemap: 0.3,
+  },
+  api: {
+    openapi: 1.0,
+    markdown: 0.7,
+    llms_full_txt: 0.65,
+    web: 0.5,
+    llms_txt: 0.4,
+    github: 0.35,
+    skill: 0.3,
+    sitemap: 0.2,
+  },
+  examples: {
+    github: 1.0,
+    markdown: 0.85,
+    llms_full_txt: 0.8,
+    web: 0.6,
+    skill: 0.5,
+    llms_txt: 0.45,
+    openapi: 0.4,
+    sitemap: 0.2,
+  },
+  implementation: {
+    skill: 1.0,
+    markdown: 0.9,
+    llms_txt: 0.85,
+    llms_full_txt: 0.8,
+    web: 0.65,
+    openapi: 0.6,
+    github: 0.55,
+    sitemap: 0.2,
+  },
+};
+
+const AUTHORITY_WEIGHTS = {
+  official: 1.0,
+  community: 0.8,
+  third_party: 0.6,
+};
+
+function scoreSource(source: DiscoveredSource, purpose: SourcePurpose): number {
+  if (source.status === 'unreachable') return 0;
+  if (source.status === 'invalid') return 0.05;
+
+  const typeWeights = PURPOSE_TYPE_WEIGHTS[purpose] || PURPOSE_TYPE_WEIGHTS.conceptual;
+  const baseWeight = typeWeights[source.type] || 0.4;
+  const authorityWeight = AUTHORITY_WEIGHTS[source.authority] || 0.6;
+  const machineBonus = source.machineReadable ? 0.05 : 0.0;
+
+  const score = (baseWeight * authorityWeight + machineBonus) * source.confidence;
+  return Math.min(1.0, Math.max(0.0, score));
+}
+
+/**
+ * Ranks discovered sources based on the intended purpose:
+ * navigation, conceptual, api, examples, or implementation.
+ */
+export function rankSources(
+  sources: DiscoveredSource[],
+  purpose: SourcePurpose = 'conceptual'
+): SourceRankResult {
+  if (!sources || sources.length === 0) {
+    return {
+      purpose,
+      recommended: null,
+      ranked: [],
+      rationale: `No sources available to rank for purpose "${purpose}".`,
+    };
+  }
+
+  const scored = sources.map(s => ({
+    source: s,
+    score: scoreSource(s, purpose),
+  }));
+
+  scored.sort((a, b) => {
+    if (b.score !== a.score) return b.score - a.score;
+    return a.source.url.localeCompare(b.source.url);
+  });
+
+  const ranked = scored.map(item => item.source);
+  const recommended = ranked.find(s => s.status === 'valid') || null;
+
+  let rationale = `Ranked ${ranked.length} source(s) for purpose "${purpose}".`;
+  if (recommended) {
+    rationale += ` Recommended "${recommended.url}" (${recommended.type}, authority: ${recommended.authority}, machine-readable: ${recommended.machineReadable}).`;
+  } else {
+    rationale += ' No valid candidate sources found.';
+  }
+
+  return {
+    purpose,
+    recommended,
+    ranked,
+    rationale,
+  };
+}