docorbit 0.1.0 → 0.1.2

package/packages/discovery/src/providers/generic.ts

@@ -1,98 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class GenericWebProvider implements DiscoveryProvider {
-  name = 'generic_web';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-
-    try {
-      const res = await fetcher.fetch(targetUrl, {
-        timeoutMs: 5000,
-      });
-
-      if (res.status >= 200 && res.status < 300) {
-        let framework = 'unknown';
-        const bodyLower = res.body.toLowerCase();
-
-        // Check generator meta tags or distinctive DOM anchors first
-        const metaGenerator = res.body.match(/<meta\b[^>]*\bname=["']generator["'][^>]*\bcontent=["']([^"']+)["']/i);
-        const generatorVal = metaGenerator ? metaGenerator[1].toLowerCase() : '';
-
-        if (generatorVal.includes('docusaurus') || res.body.includes('__docusaurus') || res.body.includes('docusaurus-plugin')) {
-          framework = 'docusaurus';
-        } else if (generatorVal.includes('mintlify') || res.body.includes('_mintlify') || res.body.includes('mintlify.css') || res.body.includes('mintlify.com')) {
-          framework = 'mintlify';
-        } else if (generatorVal.includes('nextra') || res.body.includes('__nextra')) {
-          framework = 'nextra';
-        } else if (generatorVal.includes('gitbook') || res.body.includes('gitbook-root') || res.body.includes('data-gitbook')) {
-          framework = 'gitbook';
-        } else if (generatorVal.includes('mkdocs') || res.body.includes('mkdocs')) {
-          framework = 'mkdocs';
-        } else if (generatorVal.includes('sphinx') || res.body.includes('sphinxsidebar')) {
-          framework = 'sphinx';
-        }
-
-        // Section 18: Detect candidate doc sections without blindly crawling
-        const candidateSections: string[] = [];
-        const sectionRegex = /href=["']([^"'#?]+(?:\/docs|\/api|\/reference|\/guides|\/tutorials|\/changelog)[^"'#?]*)["']/gi;
-        let sMatch: RegExpExecArray | null;
-        while ((sMatch = sectionRegex.exec(res.body)) !== null) {
-          try {
-            const secUrl = new URL(sMatch[1], targetUrl).href;
-            if (new URL(secUrl).origin === new URL(targetUrl).origin && !candidateSections.includes(secUrl)) {
-              candidateSections.push(secUrl);
-              if (candidateSections.length >= 10) break;
-            }
-          } catch {
-            // Ignore
-          }
-        }
-
-        const linkHeader = res.headers['link'];
-        if (linkHeader) {
-          const llmsMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?(?:llms-txt|alternate)["']?/i);
-          if (llmsMatch) {
-            try {
-              const fullLlmsUrl = new URL(llmsMatch[1], targetUrl).href;
-              results.push({
-                url: fullLlmsUrl,
-                type: 'llms_txt',
-                discoveredBy: `${this.name}:link-header`,
-                status: 'valid',
-                confidence: 0.99,
-                authority: 'official',
-                machineReadable: true,
-              });
-            } catch {
-              // Ignore
-            }
-          }
-        }
-
-        results.push({
-          url: res.finalUrl || targetUrl,
-          type: 'web',
-          discoveredBy: this.name,
-          status: 'valid',
-          contentType: res.contentType,
-          confidence: 0.85,
-          authority: 'official',
-          machineReadable: false,
-          metadata: {
-            framework,
-            status: res.status,
-            bytes: res.bytesRead,
-            candidateSections,
-          },
-        });
-      }
-    } catch {
-      // Continue
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/providers/github.ts

@@ -1,61 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class GithubProvider implements DiscoveryProvider {
-  name = 'github';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-
-    const isDirectGithub = /^https?:\/\/(?:www\.)?github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+/i.test(targetUrl);
-    if (isDirectGithub) {
-      results.push({
-        url: targetUrl,
-        type: 'github',
-        discoveredBy: this.name,
-        status: 'valid',
-        confidence: 0.99,
-        authority: 'official',
-        machineReadable: false,
-      });
-      return results;
-    }
-
-    try {
-      const res = await fetcher.fetch(targetUrl, {
-        timeoutMs: 4000,
-      });
-
-      if (res.status >= 200 && res.status < 300 && res.body) {
-        const githubMatches = res.body.matchAll(/href=["'](https?:\/\/(?:www\.)?github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+(?:\/)?)(?:["']|[/?#])/gi);
-        const seenRepos = new Set<string>();
-
-        for (const m of githubMatches) {
-          const repoUrl = m[1].replace(/\/$/, '');
-          if (repoUrl.includes('/topics') || repoUrl.includes('/features') || repoUrl.includes('/pricing')) continue;
-
-          if (!seenRepos.has(repoUrl)) {
-            seenRepos.add(repoUrl);
-            results.push({
-              url: repoUrl,
-              type: 'github',
-              discoveredBy: this.name,
-              status: 'valid',
-              confidence: 0.85,
-              authority: 'official',
-              machineReadable: false,
-              metadata: {
-                foundOn: targetUrl,
-              },
-            });
-          }
-        }
-      }
-    } catch {
-      // Continue
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/providers/llms-txt.ts

@@ -1,73 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import { isValidLlmsTxt } from '../../../normalizer/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class LlmsTxtProvider implements DiscoveryProvider {
-  name = 'llms_txt';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-    let baseUrl: URL;
-    try {
-      baseUrl = new URL(targetUrl);
-    } catch {
-      return results;
-    }
-
-    const cleanPath = baseUrl.pathname.replace(/\/$/, '');
-
-    // Probe both subpath-relative and origin-root paths
-    const candidatePaths: Array<{ path: string; type: 'llms_txt' | 'llms_full_txt' }> = [
-      { path: '/llms.txt', type: 'llms_txt' },
-      { path: '/llms-full.txt', type: 'llms_full_txt' },
-      { path: '/.well-known/llms.txt', type: 'llms_txt' },
-    ];
-
-    if (cleanPath && cleanPath !== '') {
-      candidatePaths.unshift(
-        { path: `${cleanPath}/llms.txt`, type: 'llms_txt' },
-        { path: `${cleanPath}/llms-full.txt`, type: 'llms_full_txt' }
-      );
-    }
-
-    const seenUrls = new Set<string>();
-
-    for (const item of candidatePaths) {
-      const probeUrl = new URL(item.path, baseUrl.origin).href;
-      if (seenUrls.has(probeUrl)) continue;
-      seenUrls.add(probeUrl);
-
-      try {
-        const res = await fetcher.fetch(probeUrl, {
-          timeoutMs: 4000,
-        });
-
-        if (res.status >= 200 && res.status < 300) {
-          const isValid = isValidLlmsTxt(res.body);
-          const isNotHtml = !res.contentType.includes('text/html') && !res.body.includes('<html');
-          if (isValid || (item.type === 'llms_full_txt' && isNotHtml)) {
-            results.push({
-              url: probeUrl,
-              type: item.type,
-              discoveredBy: this.name,
-              status: 'valid',
-              contentType: res.contentType,
-              confidence: 0.98,
-              authority: 'official',
-              machineReadable: true,
-              metadata: {
-                bytes: res.bytesRead,
-                path: item.path,
-              },
-            });
-          }
-        }
-      } catch {
-        // Continue
-      }
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/providers/markdown.ts

@@ -1,48 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class MarkdownProvider implements DiscoveryProvider {
-  name = 'markdown';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-
-    const mdProbeUrl = targetUrl.endsWith('.md')
-      ? targetUrl
-      : targetUrl.replace(/\/$/, '') + '.md';
-
-    try {
-      const res = await fetcher.fetch(mdProbeUrl, {
-        timeoutMs: 4000,
-      });
-
-      if (res.status >= 200 && res.status < 300) {
-        const isMarkdown =
-          res.contentType.includes('text/markdown') ||
-          res.contentType.includes('text/plain') ||
-          res.body.startsWith('#');
-
-        if (isMarkdown && !res.body.includes('<html')) {
-          results.push({
-            url: mdProbeUrl,
-            type: 'markdown',
-            discoveredBy: this.name,
-            status: 'valid',
-            contentType: res.contentType,
-            confidence: 0.95,
-            authority: 'official',
-            machineReadable: true,
-            metadata: {
-              bytes: res.bytesRead,
-            },
-          });
-        }
-      }
-    } catch {
-      // Continue
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/providers/openapi.ts

@@ -1,91 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import { detectOpenApiSpec } from '../../../normalizer/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class OpenApiProvider implements DiscoveryProvider {
-  name = 'openapi';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-    let baseUrl: URL;
-    try {
-      baseUrl = new URL(targetUrl);
-    } catch {
-      return results;
-    }
-
-    const cleanPath = baseUrl.pathname.replace(/\/$/, '');
-
-    const candidatePaths = [
-      '/openapi.json',
-      '/openapi.yaml',
-      '/swagger.json',
-      '/v3/api-docs',
-      '/api-docs',
-    ];
-
-    if (cleanPath && cleanPath !== '') {
-      candidatePaths.unshift(
-        `${cleanPath}/openapi.json`,
-        `${cleanPath}/openapi.yaml`,
-        `${cleanPath}/swagger.json`
-      );
-    }
-
-    // Also probe targetUrl HTML for <link rel="service-doc">
-    try {
-      const pageRes = await fetcher.fetch(targetUrl, { timeoutMs: 4000 });
-      if (pageRes.status >= 200 && pageRes.status < 300 && pageRes.body) {
-        const linkMatch = pageRes.body.match(/<link\b[^>]*\brel=["'](?:service-doc|openapi|swagger)["'][^>]*\bhref=["']([^"']+)["']/i);
-        if (linkMatch) {
-          const docHref = new URL(linkMatch[1], targetUrl).href;
-          candidatePaths.unshift(new URL(docHref).pathname);
-        }
-      }
-    } catch {
-      // Continue
-    }
-
-    const seenUrls = new Set<string>();
-
-    for (const path of candidatePaths) {
-      const probeUrl = new URL(path, baseUrl.origin).href;
-      if (seenUrls.has(probeUrl)) continue;
-      seenUrls.add(probeUrl);
-
-      try {
-        const res = await fetcher.fetch(probeUrl, {
-          timeoutMs: 4000,
-        });
-
-        if (res.status >= 200 && res.status < 300) {
-          const spec = detectOpenApiSpec(res.body, probeUrl);
-          if (spec) {
-            results.push({
-              url: probeUrl,
-              type: 'openapi',
-              discoveredBy: this.name,
-              status: 'valid',
-              contentType: res.contentType,
-              confidence: 0.99,
-              authority: 'official',
-              machineReadable: true,
-              metadata: {
-                specVersion: spec.specVersion,
-                title: spec.title,
-                pathCount: spec.pathCount,
-                servers: spec.servers,
-              },
-            });
-            break;
-          }
-        }
-      } catch {
-        // Continue
-      }
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/providers/sitemap.ts

@@ -1,62 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class SitemapProvider implements DiscoveryProvider {
-  name = 'sitemap';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-    let baseUrl: URL;
-    try {
-      baseUrl = new URL(targetUrl);
-    } catch {
-      return results;
-    }
-
-    const cleanPath = baseUrl.pathname.replace(/\/$/, '');
-
-    const candidatePaths = ['/sitemap.xml', '/sitemap_index.xml'];
-    if (cleanPath && cleanPath !== '') {
-      candidatePaths.unshift(`${cleanPath}/sitemap.xml`);
-    }
-
-    const seenUrls = new Set<string>();
-
-    for (const path of candidatePaths) {
-      const probeUrl = new URL(path, baseUrl.origin).href;
-      if (seenUrls.has(probeUrl)) continue;
-      seenUrls.add(probeUrl);
-
-      try {
-        const res = await fetcher.fetch(probeUrl, {
-          timeoutMs: 4000,
-        });
-
-        if (res.status >= 200 && res.status < 300) {
-          const isXml = res.contentType.includes('xml') || res.body.includes('<urlset') || res.body.includes('<sitemapindex');
-          if (isXml) {
-            results.push({
-              url: probeUrl,
-              type: 'sitemap',
-              discoveredBy: this.name,
-              status: 'valid',
-              contentType: res.contentType,
-              confidence: 0.9,
-              authority: 'official',
-              machineReadable: true,
-              metadata: {
-                bytes: res.bytesRead,
-              },
-            });
-            break;
-          }
-        }
-      } catch {
-        // Continue
-      }
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/providers/skill.ts

@@ -1,54 +0,0 @@
-import type { DiscoveredSource } from '../../../shared/src/index.ts';
-import { SecureFetcher } from '../../../crawler/src/index.ts';
-import type { DiscoveryProvider } from '../provider.ts';
-
-export class SkillProvider implements DiscoveryProvider {
-  name = 'skill';
-
-  async discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results: DiscoveredSource[] = [];
-    let baseUrl: URL;
-    try {
-      baseUrl = new URL(targetUrl);
-    } catch {
-      return results;
-    }
-
-    const candidatePaths = ['/skill.md', '/.well-known/skills/skill.md'];
-
-    for (const path of candidatePaths) {
-      const probeUrl = new URL(path, baseUrl.origin).href;
-      try {
-        const res = await fetcher.fetch(probeUrl, {
-          timeoutMs: 4000,
-        });
-
-        if (res.status >= 200 && res.status < 300) {
-          const hasYamlHeader = /^---\s*\n([\s\S]*?)\n---/m.test(res.body);
-          const hasName = /\bname:\s*.+/i.test(res.body);
-
-          if (hasYamlHeader && hasName) {
-            results.push({
-              url: probeUrl,
-              type: 'skill',
-              discoveredBy: this.name,
-              status: 'valid',
-              contentType: res.contentType,
-              confidence: 0.95,
-              authority: 'official',
-              machineReadable: true,
-              metadata: {
-                bytes: res.bytesRead,
-              },
-            });
-            break;
-          }
-        }
-      } catch {
-        // Continue
-      }
-    }
-
-    return results;
-  }
-}

package/packages/discovery/src/ranker.ts

@@ -1,123 +0,0 @@
-import type {
-  DiscoveredSource,
-  SourcePurpose,
-  SourceRankResult,
-  SourceType,
-} from '../../shared/src/index.ts';
-
-const PURPOSE_TYPE_WEIGHTS: Record<SourcePurpose, Record<SourceType, number>> = {
-  navigation: {
-    llms_txt: 1.0,
-    sitemap: 0.9,
-    web: 0.7,
-    llms_full_txt: 0.6,
-    markdown: 0.5,
-    skill: 0.45,
-    openapi: 0.4,
-    github: 0.3,
-  },
-  conceptual: {
-    markdown: 1.0,
-    llms_full_txt: 0.95,
-    llms_txt: 0.85,
-    web: 0.75,
-    skill: 0.6,
-    openapi: 0.5,
-    github: 0.45,
-    sitemap: 0.3,
-  },
-  api: {
-    openapi: 1.0,
-    markdown: 0.7,
-    llms_full_txt: 0.65,
-    web: 0.5,
-    llms_txt: 0.4,
-    github: 0.35,
-    skill: 0.3,
-    sitemap: 0.2,
-  },
-  examples: {
-    github: 1.0,
-    markdown: 0.85,
-    llms_full_txt: 0.8,
-    web: 0.6,
-    skill: 0.5,
-    llms_txt: 0.45,
-    openapi: 0.4,
-    sitemap: 0.2,
-  },
-  implementation: {
-    skill: 1.0,
-    markdown: 0.9,
-    llms_txt: 0.85,
-    llms_full_txt: 0.8,
-    web: 0.65,
-    openapi: 0.6,
-    github: 0.55,
-    sitemap: 0.2,
-  },
-};
-
-const AUTHORITY_WEIGHTS = {
-  official: 1.0,
-  community: 0.8,
-  third_party: 0.6,
-};
-
-function scoreSource(source: DiscoveredSource, purpose: SourcePurpose): number {
-  if (source.status === 'unreachable') return 0;
-  if (source.status === 'invalid') return 0.05;
-
-  const typeWeights = PURPOSE_TYPE_WEIGHTS[purpose] || PURPOSE_TYPE_WEIGHTS.conceptual;
-  const baseWeight = typeWeights[source.type] || 0.4;
-  const authorityWeight = AUTHORITY_WEIGHTS[source.authority] || 0.6;
-  const machineBonus = source.machineReadable ? 0.05 : 0.0;
-
-  const score = (baseWeight * authorityWeight + machineBonus) * source.confidence;
-  return Math.min(1.0, Math.max(0.0, score));
-}
-
-/**
- * Ranks discovered sources based on the intended purpose:
- * navigation, conceptual, api, examples, or implementation.
- */
-export function rankSources(
-  sources: DiscoveredSource[],
-  purpose: SourcePurpose = 'conceptual'
-): SourceRankResult {
-  if (!sources || sources.length === 0) {
-    return {
-      purpose,
-      recommended: null,
-      ranked: [],
-      rationale: `No sources available to rank for purpose "${purpose}".`,
-    };
-  }
-
-  const scored = sources.map(s => ({
-    source: s,
-    score: scoreSource(s, purpose),
-  }));
-
-  scored.sort((a, b) => {
-    if (b.score !== a.score) return b.score - a.score;
-    return a.source.url.localeCompare(b.source.url);
-  });
-
-  const ranked = scored.map(item => item.source);
-  const recommended = ranked.find(s => s.status === 'valid') || null;
-
-  let rationale = `Ranked ${ranked.length} source(s) for purpose "${purpose}".`;
-  if (recommended) {
-    rationale += ` Recommended "${recommended.url}" (${recommended.type}, authority: ${recommended.authority}, machine-readable: ${recommended.machineReadable}).`;
-  } else {
-    rationale += ' No valid candidate sources found.';
-  }
-
-  return {
-    purpose,
-    recommended,
-    ranked,
-    rationale,
-  };
-}