docorbit 0.1.0 → 0.1.2

package/packages/core/src/pipeline.ts

@@ -1,331 +0,0 @@
-import type {
-  DiscoveredSource,
-  NormalizedPage,
-  CrawlerConfig,
-  CrawlPolicy,
-  Target,
-  SourcePurpose,
-} from '../../shared/src/index.ts';
-import { validateTargetUrl } from '../../security/src/index.ts';
-import { SecureFetcher, DEFAULT_CRAWLER_CONFIG } from '../../crawler/src/index.ts';
-import { createDefaultDiscoveryCoordinator, rankSources } from '../../discovery/src/index.ts';
-import {
-  buildNormalizedPage,
-  parseLlmsTxt,
-  isValidLlmsTxt,
-  slicePageIntoChunks,
-  parseOpenApiEndpoints,
-  detectOpenApiSpec,
-  extractIndexedExamples,
-  extractPitfalls,
-} from '../../normalizer/src/index.ts';
-import { DocOrbitRepository } from '../../storage/src/index.ts';
-
-export interface IngestionOptions {
-  crawlerConfig?: Partial<CrawlerConfig>;
-  crawlPolicy?: Partial<CrawlPolicy>;
-  allowLocalhostForTesting?: boolean;
-}
-
-export interface IngestionResult {
-  targetUrl: string;
-  target?: Target;
-  primarySourceId: string;
-  sourcesDiscovered: DiscoveredSource[];
-  selectedSources: Array<{
-    purpose: SourcePurpose;
-    sourceId: string;
-  }>;
-  pages: NormalizedPage[];
-  pagesDiscovered: number;
-  pagesFetched: number;
-  pagesStored: number;
-  snapshotId: string;
-  warnings: string[];
-  errors: Array<{
-    url: string;
-    error: string;
-  }>;
-  durationMs: number;
-  stats: {
-    totalPages: number;
-    totalBytes: number;
-    totalEstimatedTokens: number;
-    totalCodeExamples: number;
-    machineReadableSources: number;
-    totalChunks: number;
-  };
-}
-
-
-export class IngestionPipeline {
-  private repository: DocOrbitRepository;
-  private fetcher: SecureFetcher;
-  private config: CrawlerConfig;
-  private policy: CrawlPolicy;
-  private allowLocalhostForTesting: boolean;
-
-  constructor(repository: DocOrbitRepository, options: IngestionOptions = {}) {
-    this.repository = repository;
-    this.config = {
-      ...DEFAULT_CRAWLER_CONFIG,
-      ...options.crawlerConfig,
-    };
-    this.allowLocalhostForTesting = options.allowLocalhostForTesting ?? false;
-    this.policy = {
-      purpose: 'conceptual',
-      maxPages: this.config.maxPages,
-      maxDepth: this.config.maxDepth || 2,
-      followLlmsReferences: true,
-      followExternalDomains: false,
-      ...options.crawlPolicy,
-    };
-    this.fetcher = new SecureFetcher({
-      timeoutMs: this.config.timeoutMs,
-      maxBytes: this.config.maxBytesPerResponse,
-      maxRedirects: this.config.maxRedirects,
-      userAgent: this.config.userAgent,
-      allowLocalhostForTesting: this.allowLocalhostForTesting,
-    });
-  }
-
-  async ingest(targetUrl: string): Promise<IngestionResult> {
-    const startTime = Date.now();
-
-    // 1. SSRF & Protocol validation before making any discovery probes
-    await validateTargetUrl(targetUrl, {
-      allowLocalhostForTesting: this.allowLocalhostForTesting,
-    });
-
-    const coordinator = createDefaultDiscoveryCoordinator();
-    const discovered = await coordinator.discoverAll(targetUrl, this.fetcher);
-
-    const sourceIds: Record<string, string> = {};
-    for (const s of discovered) {
-      const id = this.repository.saveSource(s);
-      sourceIds[s.url] = id;
-    }
-
-    const conceptualRank = rankSources(discovered, 'conceptual');
-    const apiRank = rankSources(discovered, 'api');
-
-    const primarySource = conceptualRank.recommended || discovered[0];
-    const primarySourceId = primarySource
-      ? (sourceIds[primarySource.url] || this.repository.saveSource(primarySource))
-      : this.repository.saveSource({
-          url: targetUrl,
-          type: 'web',
-          discoveredBy: 'direct',
-          status: 'valid',
-          confidence: 1.0,
-          authority: 'official',
-          machineReadable: false,
-        });
-
-    const purposes: SourcePurpose[] = ['navigation', 'conceptual', 'api', 'examples', 'implementation'];
-    const selectedSources = purposes.map(p => {
-      const ranked = rankSources(discovered, p);
-      return {
-        purpose: p,
-        sourceId: ranked.recommended ? (sourceIds[ranked.recommended.url] || primarySourceId) : primarySourceId,
-      };
-    });
-
-    const queue: Array<{ url: string; depth: number }> = [];
-    const visited = new Set<string>();
-    const ingestedPages: NormalizedPage[] = [];
-    const warnings: string[] = [];
-    const errors: Array<{ url: string; error: string }> = [];
-    let pagesDiscovered = 0;
-
-    const enqueue = (url: string, depth: number) => {
-      try {
-        const parsed = new URL(url);
-        const normalized = `${parsed.origin}${parsed.pathname}`;
-        pagesDiscovered++;
-        if (depth > this.policy.maxDepth) return;
-        if (!visited.has(normalized) && queue.length < this.policy.maxPages) {
-          queue.push({ url: normalized, depth });
-        }
-      } catch {
-        // Ignore
-      }
-    };
-
-    // 1. Prioritize targetUrl as the primary entry point (depth: 0)
-    enqueue(targetUrl, 0);
-
-    // 2. If an OpenAPI spec is available, enqueue it (depth: 0)
-    if (apiRank.recommended && apiRank.recommended.type === 'openapi' && apiRank.recommended.url !== targetUrl) {
-      enqueue(apiRank.recommended.url, 0);
-    }
-
-    // 3. If llms.txt or llms-full.txt is available, enqueue it (depth: 0)
-    const llmsFull = discovered.find(s => s.type === 'llms_full_txt' && s.status === 'valid');
-    const llms = discovered.find(s => s.type === 'llms_txt' && s.status === 'valid');
-
-    if (llmsFull && llmsFull.url !== targetUrl) {
-      enqueue(llmsFull.url, 0);
-    } else if (llms && llms.url !== targetUrl) {
-      enqueue(llms.url, 0);
-    }
-
-    if (primarySource && primarySource.url !== targetUrl) {
-      enqueue(primarySource.url, 0);
-    }
-
-    const targetOrigin = new URL(targetUrl).origin;
-
-    while (queue.length > 0 && ingestedPages.length < this.policy.maxPages) {
-      const item = queue.shift()!;
-      const currentUrl = item.url;
-      const currentDepth = item.depth;
-
-      if (visited.has(currentUrl)) continue;
-      visited.add(currentUrl);
-
-      try {
-        const res = await this.fetcher.fetch(currentUrl);
-        if (res.status < 200 || res.status >= 300) {
-          errors.push({ url: currentUrl, error: `HTTP status ${res.status}` });
-          continue;
-        }
-
-        const page = buildNormalizedPage({
-          sourceId: primarySourceId,
-          url: res.finalUrl || currentUrl,
-          rawContent: res.body,
-          contentType: res.contentType,
-          sourceUrl: primarySource?.url || targetUrl,
-          targetUrl,
-          discoveredBy: primarySource?.discoveredBy || 'direct',
-          fetchedAt: new Date().toISOString(),
-        });
-
-        this.repository.savePage(page);
-        ingestedPages.push(page);
-
-        // Subpage link discovery
-        if (isValidLlmsTxt(res.body)) {
-          if (this.policy.followLlmsReferences && currentDepth < this.policy.maxDepth) {
-            const llmsDoc = parseLlmsTxt(res.body, currentUrl);
-            for (const sec of llmsDoc.sections) {
-              for (const lnk of sec.links) {
-                if (ingestedPages.length + queue.length < this.policy.maxPages) {
-                  try {
-                    const parsedLnk = new URL(lnk.url);
-                    if (this.policy.followExternalDomains || parsedLnk.origin === targetOrigin) {
-                      enqueue(lnk.url, currentDepth + 1);
-                    }
-                  } catch {
-                    // Ignore
-                  }
-                }
-              }
-            }
-          }
-        } else {
-          if (currentDepth < this.policy.maxDepth) {
-            for (const lnk of page.links) {
-              if (ingestedPages.length + queue.length < this.policy.maxPages) {
-                try {
-                  const linkUrl = new URL(lnk.url);
-                  if (this.policy.followExternalDomains || linkUrl.origin === targetOrigin) {
-                    enqueue(lnk.url, currentDepth + 1);
-                  }
-                } catch {
-                  // Ignore
-                }
-              }
-            }
-          }
-        }
-      } catch (err: unknown) {
-        errors.push({ url: currentUrl, error: err instanceof Error ? err.message : String(err) });
-      }
-    }
-
-    const snapshotId = this.repository.createSnapshot(primarySourceId, {
-      targetUrl,
-      pageCount: ingestedPages.length,
-      ingestedAt: new Date().toISOString(),
-    });
-
-    // Milestone 2: Semantic slicing of pages into standalone DocumentChunks
-    let totalChunks = 0;
-    for (const p of ingestedPages) {
-      const slicingResult = slicePageIntoChunks(p, snapshotId);
-      this.repository.saveChunks(
-        slicingResult.chunks,
-        slicingResult.relationships,
-        slicingResult.codeSnippets,
-        slicingResult.symbols
-      );
-      totalChunks += slicingResult.chunks.length;
-
-      // Milestone 4: API Endpoints
-      if (p.openApiSummary || detectOpenApiSpec(p.content, p.url)) {
-        const endpoints = parseOpenApiEndpoints(p.content, p.id, snapshotId, p.url, p.docVersion);
-        if (endpoints.length > 0) {
-          this.repository.saveApiEndpoints(endpoints);
-        }
-      }
-
-      // Milestone 4: Indexed Examples
-      const examples = extractIndexedExamples(p, slicingResult.chunks, primarySource?.authority || 'official', p.docVersion, snapshotId);
-      if (examples.length > 0) {
-        this.repository.saveIndexedExamples(examples);
-      }
-
-      // Milestone 4: Pitfalls
-      const pitfalls = extractPitfalls(p, slicingResult.chunks, p.docVersion, snapshotId);
-      if (pitfalls.length > 0) {
-        this.repository.savePitfalls(pitfalls);
-      }
-    }
-
-    const durationMs = Date.now() - startTime;
-
-    let totalBytes = 0;
-    let totalEstimatedTokens = 0;
-    let totalCodeExamples = 0;
-
-    for (const p of ingestedPages) {
-      totalBytes += p.rawBytes;
-      totalEstimatedTokens += p.estimatedTokens;
-      totalCodeExamples += p.codeExamples.length;
-      if (p.securityAnnotations.length > 0) {
-        warnings.push(`Page "${p.url}" generated ${p.securityAnnotations.length} security alert(s).`);
-      }
-    }
-
-    return {
-      targetUrl,
-      target: {
-        type: 'url',
-        value: targetUrl,
-        normalizedUrl: targetUrl,
-      },
-      primarySourceId,
-      sourcesDiscovered: discovered,
-      selectedSources,
-      pages: ingestedPages,
-      pagesDiscovered,
-      pagesFetched: visited.size,
-      pagesStored: ingestedPages.length,
-      snapshotId,
-      warnings,
-      errors,
-      durationMs,
-      stats: {
-        totalPages: ingestedPages.length,
-        totalBytes,
-        totalEstimatedTokens,
-        totalCodeExamples,
-        machineReadableSources: discovered.filter(s => s.status === 'valid' && s.machineReadable).length,
-        totalChunks,
-      },
-    };
-  }
-}
-

package/packages/crawler/src/config.ts

@@ -1,12 +0,0 @@
-import type { CrawlerConfig } from '../../shared/src/index.ts';
-
-export const DEFAULT_CRAWLER_CONFIG: CrawlerConfig = {
-  maxPages: 50,
-  maxDepth: 3,
-  maxBytesPerResponse: 10 * 1024 * 1024, // 10MB
-  timeoutMs: 10000,                      // 10 seconds
-  maxRedirects: 5,
-  concurrency: 5,
-  allowedProtocols: ['http:', 'https:'],
-  userAgent: 'DocOrbit/1.0 (+https://github.com/docorbit/docorbit)',
-};

package/packages/crawler/src/fetcher.ts

@@ -1,185 +0,0 @@
-import {
-  FetchTimeoutError,
-  PayloadTooLargeError,
-  DocOrbitError,
-} from '../../shared/src/index.ts';
-import { validateTargetUrl } from '../../security/src/index.ts';
-import type { SsrfValidationOptions } from '../../security/src/index.ts';
-import { DEFAULT_CRAWLER_CONFIG } from './config.ts';
-
-export interface FetchResult {
-  url: string;
-  finalUrl: string;
-  status: number;
-  statusText: string;
-  headers: Record<string, string>;
-  body: string;
-  contentType: string;
-  bytesRead: number;
-}
-
-export interface FetchOptions extends SsrfValidationOptions {
-  timeoutMs?: number;
-  maxBytes?: number;
-  maxRedirects?: number;
-  userAgent?: string;
-  headers?: Record<string, string>;
-  method?: string;
-  fetchFn?: typeof fetch;
-  allowCrossDomainRedirects?: boolean;
-}
-
-export class SecureFetcher {
-  private timeoutMs: number;
-  private maxBytes: number;
-  private maxRedirects: number;
-  private userAgent: string;
-  private allowLocalhostForTesting: boolean;
-  private fetchFn: typeof fetch;
-
-  constructor(defaults: Partial<FetchOptions> = {}) {
-    this.timeoutMs = defaults.timeoutMs ?? DEFAULT_CRAWLER_CONFIG.timeoutMs;
-    this.maxBytes = defaults.maxBytes ?? DEFAULT_CRAWLER_CONFIG.maxBytesPerResponse;
-    this.maxRedirects = defaults.maxRedirects ?? DEFAULT_CRAWLER_CONFIG.maxRedirects;
-    this.userAgent = defaults.userAgent ?? DEFAULT_CRAWLER_CONFIG.userAgent;
-    this.allowLocalhostForTesting = defaults.allowLocalhostForTesting ?? false;
-    this.fetchFn = defaults.fetchFn ?? globalThis.fetch;
-  }
-
-  async fetch(urlStr: string, options: FetchOptions = {}): Promise<FetchResult> {
-    const timeoutMs = options.timeoutMs ?? this.timeoutMs;
-    const maxBytes = options.maxBytes ?? this.maxBytes;
-    const maxRedirects = options.maxRedirects ?? this.maxRedirects;
-    const userAgent = options.userAgent ?? this.userAgent;
-    const method = options.method ?? 'GET';
-    const allowLocalhost = options.allowLocalhostForTesting ?? this.allowLocalhostForTesting;
-    const activeFetch = options.fetchFn ?? this.fetchFn;
-
-    let currentUrl = urlStr;
-    let redirectCount = 0;
-    const seenRedirects = new Set<string>();
-    seenRedirects.add(currentUrl);
-
-    while (redirectCount <= maxRedirects) {
-      // 1. SSRF & Protocol validation before making the hop
-      await validateTargetUrl(currentUrl, {
-        allowLocalhostForTesting: allowLocalhost,
-      });
-
-      const controller = new AbortController();
-      const timeoutId = setTimeout(() => {
-        controller.abort(new FetchTimeoutError(`Request timed out after ${timeoutMs}ms`, timeoutMs));
-      }, timeoutMs);
-
-      try {
-        const reqHeaders: Record<string, string> = {
-          'User-Agent': userAgent,
-          Accept: 'text/html,application/xhtml+xml,text/plain,text/markdown,application/json,*/*',
-          ...options.headers,
-        };
-
-        const res = await activeFetch(currentUrl, {
-          method,
-          headers: reqHeaders,
-          redirect: 'manual',
-          signal: controller.signal,
-        });
-
-        // 2. Handle HTTP redirects (301, 302, 303, 307, 308)
-        if ([301, 302, 303, 307, 308].includes(res.status)) {
-          const location = res.headers.get('location');
-          if (!location) {
-            throw new DocOrbitError(`Redirect response (${res.status}) missing Location header.`);
-          }
-
-          redirectCount++;
-          if (redirectCount > maxRedirects) {
-            throw new DocOrbitError(`Exceeded maximum allowed redirects (${maxRedirects}).`);
-          }
-
-          const nextUrl = new URL(location, currentUrl).href;
-
-          // Check for redirect loop
-          if (seenRedirects.has(nextUrl)) {
-            throw new DocOrbitError(`Redirect loop detected: "${nextUrl}" already visited in redirect chain.`);
-          }
-          seenRedirects.add(nextUrl);
-
-          // Enforce domain/origin policy
-          if (options.allowCrossDomainRedirects === false) {
-            const origOrigin = new URL(urlStr).origin;
-            const nextOrigin = new URL(nextUrl).origin;
-            if (origOrigin !== nextOrigin) {
-              throw new DocOrbitError(`Cross-origin redirect from "${origOrigin}" to "${nextOrigin}" prohibited by policy.`);
-            }
-          }
-
-          currentUrl = nextUrl;
-          continue;
-        }
-
-        // 3. Extract response headers
-        const headers: Record<string, string> = {};
-        res.headers.forEach((val, key) => {
-          headers[key.toLowerCase()] = val;
-        });
-
-        const contentType = headers['content-type'] || 'application/octet-stream';
-
-        // 4. Stream response body with byte counting
-        let bytesRead = 0;
-        const chunks: Uint8Array[] = [];
-
-        if (res.body) {
-          const reader = res.body.getReader();
-          try {
-            while (true) {
-              const { done, value } = await reader.read();
-              if (done) break;
-              if (value) {
-                bytesRead += value.length;
-                if (bytesRead > maxBytes) {
-                  controller.abort();
-                  throw new PayloadTooLargeError(
-                    `Response exceeded maximum allowed size of ${maxBytes} bytes (read ${bytesRead} bytes).`,
-                    bytesRead,
-                    maxBytes
-                  );
-                }
-                chunks.push(value);
-              }
-            }
-          } finally {
-            reader.releaseLock();
-          }
-        }
-
-        const totalBuffer = Buffer.concat(chunks);
-        const bodyText = totalBuffer.toString('utf-8');
-
-        return {
-          url: urlStr,
-          finalUrl: currentUrl,
-          status: res.status,
-          statusText: res.statusText,
-          headers,
-          body: bodyText,
-          contentType,
-          bytesRead,
-        };
-      } catch (err: unknown) {
-        if (err instanceof DocOrbitError) throw err;
-        if (controller.signal.aborted) {
-          const reason = controller.signal.reason;
-          if (reason instanceof DocOrbitError) throw reason;
-          throw new FetchTimeoutError(`Request timed out after ${timeoutMs}ms`, timeoutMs);
-        }
-        throw new DocOrbitError(`Fetch failure for "${currentUrl}": ${err instanceof Error ? err.message : String(err)}`);
-      } finally {
-        clearTimeout(timeoutId);
-      }
-    }
-
-    throw new DocOrbitError(`Exceeded maximum allowed redirects (${maxRedirects}).`);
-  }
-}

package/packages/discovery/src/index.ts

@@ -1,31 +0,0 @@
-import { DiscoveryCoordinator } from './provider.ts';
-import { LlmsTxtProvider } from './providers/llms-txt.ts';
-import { OpenApiProvider } from './providers/openapi.ts';
-import { MarkdownProvider } from './providers/markdown.ts';
-import { SitemapProvider } from './providers/sitemap.ts';
-import { GithubProvider } from './providers/github.ts';
-import { SkillProvider } from './providers/skill.ts';
-import { GenericWebProvider } from './providers/generic.ts';
-
-export * from './provider.ts';
-export * from './ranker.ts';
-export * from './providers/llms-txt.ts';
-export * from './providers/openapi.ts';
-export * from './providers/markdown.ts';
-export * from './providers/sitemap.ts';
-export * from './providers/github.ts';
-export * from './providers/skill.ts';
-export * from './providers/generic.ts';
-
-export function createDefaultDiscoveryCoordinator(): DiscoveryCoordinator {
-  const coordinator = new DiscoveryCoordinator();
-  coordinator
-    .register(new LlmsTxtProvider())
-    .register(new OpenApiProvider())
-    .register(new MarkdownProvider())
-    .register(new SitemapProvider())
-    .register(new SkillProvider())
-    .register(new GithubProvider())
-    .register(new GenericWebProvider());
-  return coordinator;
-}

package/packages/discovery/src/provider.ts

@@ -1,47 +0,0 @@
-import type { DiscoveredSource } from '../../shared/src/index.ts';
-import { SecureFetcher } from '../../crawler/src/index.ts';
-
-export interface DiscoveryProvider {
-  name: string;
-  discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
-}
-
-export class DiscoveryCoordinator {
-  private providers: DiscoveryProvider[] = [];
-
-  register(provider: DiscoveryProvider): this {
-    this.providers.push(provider);
-    return this;
-  }
-
-  getProviders(): DiscoveryProvider[] {
-    return [...this.providers];
-  }
-
-  async discoverAll(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
-    const results = await Promise.allSettled(
-      this.providers.map(p => p.discover(targetUrl, fetcher))
-    );
-
-    const allSources: DiscoveredSource[] = [];
-    for (const res of results) {
-      if (res.status === 'fulfilled') {
-        allSources.push(...res.value);
-      }
-    }
-
-    // Deduplicate sources by normalized URL
-    const seen = new Set<string>();
-    const deduped: DiscoveredSource[] = [];
-
-    for (const s of allSources) {
-      const normalizedUrl = s.url.replace(/\/$/, '').toLowerCase();
-      if (!seen.has(normalizedUrl)) {
-        seen.add(normalizedUrl);
-        deduped.push(s);
-      }
-    }
-
-    return deduped;
-  }
-}