docorbit 0.1.0

package/packages/core/src/pipeline.ts

@@ -0,0 +1,331 @@
+import type {
+  DiscoveredSource,
+  NormalizedPage,
+  CrawlerConfig,
+  CrawlPolicy,
+  Target,
+  SourcePurpose,
+} from '../../shared/src/index.ts';
+import { validateTargetUrl } from '../../security/src/index.ts';
+import { SecureFetcher, DEFAULT_CRAWLER_CONFIG } from '../../crawler/src/index.ts';
+import { createDefaultDiscoveryCoordinator, rankSources } from '../../discovery/src/index.ts';
+import {
+  buildNormalizedPage,
+  parseLlmsTxt,
+  isValidLlmsTxt,
+  slicePageIntoChunks,
+  parseOpenApiEndpoints,
+  detectOpenApiSpec,
+  extractIndexedExamples,
+  extractPitfalls,
+} from '../../normalizer/src/index.ts';
+import { DocOrbitRepository } from '../../storage/src/index.ts';
+
+export interface IngestionOptions {
+  crawlerConfig?: Partial<CrawlerConfig>;
+  crawlPolicy?: Partial<CrawlPolicy>;
+  allowLocalhostForTesting?: boolean;
+}
+
+export interface IngestionResult {
+  targetUrl: string;
+  target?: Target;
+  primarySourceId: string;
+  sourcesDiscovered: DiscoveredSource[];
+  selectedSources: Array<{
+    purpose: SourcePurpose;
+    sourceId: string;
+  }>;
+  pages: NormalizedPage[];
+  pagesDiscovered: number;
+  pagesFetched: number;
+  pagesStored: number;
+  snapshotId: string;
+  warnings: string[];
+  errors: Array<{
+    url: string;
+    error: string;
+  }>;
+  durationMs: number;
+  stats: {
+    totalPages: number;
+    totalBytes: number;
+    totalEstimatedTokens: number;
+    totalCodeExamples: number;
+    machineReadableSources: number;
+    totalChunks: number;
+  };
+}
+
+
+export class IngestionPipeline {
+  private repository: DocOrbitRepository;
+  private fetcher: SecureFetcher;
+  private config: CrawlerConfig;
+  private policy: CrawlPolicy;
+  private allowLocalhostForTesting: boolean;
+
+  constructor(repository: DocOrbitRepository, options: IngestionOptions = {}) {
+    this.repository = repository;
+    this.config = {
+      ...DEFAULT_CRAWLER_CONFIG,
+      ...options.crawlerConfig,
+    };
+    this.allowLocalhostForTesting = options.allowLocalhostForTesting ?? false;
+    this.policy = {
+      purpose: 'conceptual',
+      maxPages: this.config.maxPages,
+      maxDepth: this.config.maxDepth || 2,
+      followLlmsReferences: true,
+      followExternalDomains: false,
+      ...options.crawlPolicy,
+    };
+    this.fetcher = new SecureFetcher({
+      timeoutMs: this.config.timeoutMs,
+      maxBytes: this.config.maxBytesPerResponse,
+      maxRedirects: this.config.maxRedirects,
+      userAgent: this.config.userAgent,
+      allowLocalhostForTesting: this.allowLocalhostForTesting,
+    });
+  }
+
+  async ingest(targetUrl: string): Promise<IngestionResult> {
+    const startTime = Date.now();
+
+    // 1. SSRF & Protocol validation before making any discovery probes
+    await validateTargetUrl(targetUrl, {
+      allowLocalhostForTesting: this.allowLocalhostForTesting,
+    });
+
+    const coordinator = createDefaultDiscoveryCoordinator();
+    const discovered = await coordinator.discoverAll(targetUrl, this.fetcher);
+
+    const sourceIds: Record<string, string> = {};
+    for (const s of discovered) {
+      const id = this.repository.saveSource(s);
+      sourceIds[s.url] = id;
+    }
+
+    const conceptualRank = rankSources(discovered, 'conceptual');
+    const apiRank = rankSources(discovered, 'api');
+
+    const primarySource = conceptualRank.recommended || discovered[0];
+    const primarySourceId = primarySource
+      ? (sourceIds[primarySource.url] || this.repository.saveSource(primarySource))
+      : this.repository.saveSource({
+          url: targetUrl,
+          type: 'web',
+          discoveredBy: 'direct',
+          status: 'valid',
+          confidence: 1.0,
+          authority: 'official',
+          machineReadable: false,
+        });
+
+    const purposes: SourcePurpose[] = ['navigation', 'conceptual', 'api', 'examples', 'implementation'];
+    const selectedSources = purposes.map(p => {
+      const ranked = rankSources(discovered, p);
+      return {
+        purpose: p,
+        sourceId: ranked.recommended ? (sourceIds[ranked.recommended.url] || primarySourceId) : primarySourceId,
+      };
+    });
+
+    const queue: Array<{ url: string; depth: number }> = [];
+    const visited = new Set<string>();
+    const ingestedPages: NormalizedPage[] = [];
+    const warnings: string[] = [];
+    const errors: Array<{ url: string; error: string }> = [];
+    let pagesDiscovered = 0;
+
+    const enqueue = (url: string, depth: number) => {
+      try {
+        const parsed = new URL(url);
+        const normalized = `${parsed.origin}${parsed.pathname}`;
+        pagesDiscovered++;
+        if (depth > this.policy.maxDepth) return;
+        if (!visited.has(normalized) && queue.length < this.policy.maxPages) {
+          queue.push({ url: normalized, depth });
+        }
+      } catch {
+        // Ignore
+      }
+    };
+
+    // 1. Prioritize targetUrl as the primary entry point (depth: 0)
+    enqueue(targetUrl, 0);
+
+    // 2. If an OpenAPI spec is available, enqueue it (depth: 0)
+    if (apiRank.recommended && apiRank.recommended.type === 'openapi' && apiRank.recommended.url !== targetUrl) {
+      enqueue(apiRank.recommended.url, 0);
+    }
+
+    // 3. If llms.txt or llms-full.txt is available, enqueue it (depth: 0)
+    const llmsFull = discovered.find(s => s.type === 'llms_full_txt' && s.status === 'valid');
+    const llms = discovered.find(s => s.type === 'llms_txt' && s.status === 'valid');
+
+    if (llmsFull && llmsFull.url !== targetUrl) {
+      enqueue(llmsFull.url, 0);
+    } else if (llms && llms.url !== targetUrl) {
+      enqueue(llms.url, 0);
+    }
+
+    if (primarySource && primarySource.url !== targetUrl) {
+      enqueue(primarySource.url, 0);
+    }
+
+    const targetOrigin = new URL(targetUrl).origin;
+
+    while (queue.length > 0 && ingestedPages.length < this.policy.maxPages) {
+      const item = queue.shift()!;
+      const currentUrl = item.url;
+      const currentDepth = item.depth;
+
+      if (visited.has(currentUrl)) continue;
+      visited.add(currentUrl);
+
+      try {
+        const res = await this.fetcher.fetch(currentUrl);
+        if (res.status < 200 || res.status >= 300) {
+          errors.push({ url: currentUrl, error: `HTTP status ${res.status}` });
+          continue;
+        }
+
+        const page = buildNormalizedPage({
+          sourceId: primarySourceId,
+          url: res.finalUrl || currentUrl,
+          rawContent: res.body,
+          contentType: res.contentType,
+          sourceUrl: primarySource?.url || targetUrl,
+          targetUrl,
+          discoveredBy: primarySource?.discoveredBy || 'direct',
+          fetchedAt: new Date().toISOString(),
+        });
+
+        this.repository.savePage(page);
+        ingestedPages.push(page);
+
+        // Subpage link discovery
+        if (isValidLlmsTxt(res.body)) {
+          if (this.policy.followLlmsReferences && currentDepth < this.policy.maxDepth) {
+            const llmsDoc = parseLlmsTxt(res.body, currentUrl);
+            for (const sec of llmsDoc.sections) {
+              for (const lnk of sec.links) {
+                if (ingestedPages.length + queue.length < this.policy.maxPages) {
+                  try {
+                    const parsedLnk = new URL(lnk.url);
+                    if (this.policy.followExternalDomains || parsedLnk.origin === targetOrigin) {
+                      enqueue(lnk.url, currentDepth + 1);
+                    }
+                  } catch {
+                    // Ignore
+                  }
+                }
+              }
+            }
+          }
+        } else {
+          if (currentDepth < this.policy.maxDepth) {
+            for (const lnk of page.links) {
+              if (ingestedPages.length + queue.length < this.policy.maxPages) {
+                try {
+                  const linkUrl = new URL(lnk.url);
+                  if (this.policy.followExternalDomains || linkUrl.origin === targetOrigin) {
+                    enqueue(lnk.url, currentDepth + 1);
+                  }
+                } catch {
+                  // Ignore
+                }
+              }
+            }
+          }
+        }
+      } catch (err: unknown) {
+        errors.push({ url: currentUrl, error: err instanceof Error ? err.message : String(err) });
+      }
+    }
+
+    const snapshotId = this.repository.createSnapshot(primarySourceId, {
+      targetUrl,
+      pageCount: ingestedPages.length,
+      ingestedAt: new Date().toISOString(),
+    });
+
+    // Milestone 2: Semantic slicing of pages into standalone DocumentChunks
+    let totalChunks = 0;
+    for (const p of ingestedPages) {
+      const slicingResult = slicePageIntoChunks(p, snapshotId);
+      this.repository.saveChunks(
+        slicingResult.chunks,
+        slicingResult.relationships,
+        slicingResult.codeSnippets,
+        slicingResult.symbols
+      );
+      totalChunks += slicingResult.chunks.length;
+
+      // Milestone 4: API Endpoints
+      if (p.openApiSummary || detectOpenApiSpec(p.content, p.url)) {
+        const endpoints = parseOpenApiEndpoints(p.content, p.id, snapshotId, p.url, p.docVersion);
+        if (endpoints.length > 0) {
+          this.repository.saveApiEndpoints(endpoints);
+        }
+      }
+
+      // Milestone 4: Indexed Examples
+      const examples = extractIndexedExamples(p, slicingResult.chunks, primarySource?.authority || 'official', p.docVersion, snapshotId);
+      if (examples.length > 0) {
+        this.repository.saveIndexedExamples(examples);
+      }
+
+      // Milestone 4: Pitfalls
+      const pitfalls = extractPitfalls(p, slicingResult.chunks, p.docVersion, snapshotId);
+      if (pitfalls.length > 0) {
+        this.repository.savePitfalls(pitfalls);
+      }
+    }
+
+    const durationMs = Date.now() - startTime;
+
+    let totalBytes = 0;
+    let totalEstimatedTokens = 0;
+    let totalCodeExamples = 0;
+
+    for (const p of ingestedPages) {
+      totalBytes += p.rawBytes;
+      totalEstimatedTokens += p.estimatedTokens;
+      totalCodeExamples += p.codeExamples.length;
+      if (p.securityAnnotations.length > 0) {
+        warnings.push(`Page "${p.url}" generated ${p.securityAnnotations.length} security alert(s).`);
+      }
+    }
+
+    return {
+      targetUrl,
+      target: {
+        type: 'url',
+        value: targetUrl,
+        normalizedUrl: targetUrl,
+      },
+      primarySourceId,
+      sourcesDiscovered: discovered,
+      selectedSources,
+      pages: ingestedPages,
+      pagesDiscovered,
+      pagesFetched: visited.size,
+      pagesStored: ingestedPages.length,
+      snapshotId,
+      warnings,
+      errors,
+      durationMs,
+      stats: {
+        totalPages: ingestedPages.length,
+        totalBytes,
+        totalEstimatedTokens,
+        totalCodeExamples,
+        machineReadableSources: discovered.filter(s => s.status === 'valid' && s.machineReadable).length,
+        totalChunks,
+      },
+    };
+  }
+}
+

package/packages/crawler/src/config.ts

@@ -0,0 +1,12 @@
+import type { CrawlerConfig } from '../../shared/src/index.ts';
+
+export const DEFAULT_CRAWLER_CONFIG: CrawlerConfig = {
+  maxPages: 50,
+  maxDepth: 3,
+  maxBytesPerResponse: 10 * 1024 * 1024, // 10MB
+  timeoutMs: 10000,                      // 10 seconds
+  maxRedirects: 5,
+  concurrency: 5,
+  allowedProtocols: ['http:', 'https:'],
+  userAgent: 'DocOrbit/1.0 (+https://github.com/docorbit/docorbit)',
+};

package/packages/crawler/src/fetcher.ts

@@ -0,0 +1,185 @@
+import {
+  FetchTimeoutError,
+  PayloadTooLargeError,
+  DocOrbitError,
+} from '../../shared/src/index.ts';
+import { validateTargetUrl } from '../../security/src/index.ts';
+import type { SsrfValidationOptions } from '../../security/src/index.ts';
+import { DEFAULT_CRAWLER_CONFIG } from './config.ts';
+
+export interface FetchResult {
+  url: string;
+  finalUrl: string;
+  status: number;
+  statusText: string;
+  headers: Record<string, string>;
+  body: string;
+  contentType: string;
+  bytesRead: number;
+}
+
+export interface FetchOptions extends SsrfValidationOptions {
+  timeoutMs?: number;
+  maxBytes?: number;
+  maxRedirects?: number;
+  userAgent?: string;
+  headers?: Record<string, string>;
+  method?: string;
+  fetchFn?: typeof fetch;
+  allowCrossDomainRedirects?: boolean;
+}
+
+export class SecureFetcher {
+  private timeoutMs: number;
+  private maxBytes: number;
+  private maxRedirects: number;
+  private userAgent: string;
+  private allowLocalhostForTesting: boolean;
+  private fetchFn: typeof fetch;
+
+  constructor(defaults: Partial<FetchOptions> = {}) {
+    this.timeoutMs = defaults.timeoutMs ?? DEFAULT_CRAWLER_CONFIG.timeoutMs;
+    this.maxBytes = defaults.maxBytes ?? DEFAULT_CRAWLER_CONFIG.maxBytesPerResponse;
+    this.maxRedirects = defaults.maxRedirects ?? DEFAULT_CRAWLER_CONFIG.maxRedirects;
+    this.userAgent = defaults.userAgent ?? DEFAULT_CRAWLER_CONFIG.userAgent;
+    this.allowLocalhostForTesting = defaults.allowLocalhostForTesting ?? false;
+    this.fetchFn = defaults.fetchFn ?? globalThis.fetch;
+  }
+
+  async fetch(urlStr: string, options: FetchOptions = {}): Promise<FetchResult> {
+    const timeoutMs = options.timeoutMs ?? this.timeoutMs;
+    const maxBytes = options.maxBytes ?? this.maxBytes;
+    const maxRedirects = options.maxRedirects ?? this.maxRedirects;
+    const userAgent = options.userAgent ?? this.userAgent;
+    const method = options.method ?? 'GET';
+    const allowLocalhost = options.allowLocalhostForTesting ?? this.allowLocalhostForTesting;
+    const activeFetch = options.fetchFn ?? this.fetchFn;
+
+    let currentUrl = urlStr;
+    let redirectCount = 0;
+    const seenRedirects = new Set<string>();
+    seenRedirects.add(currentUrl);
+
+    while (redirectCount <= maxRedirects) {
+      // 1. SSRF & Protocol validation before making the hop
+      await validateTargetUrl(currentUrl, {
+        allowLocalhostForTesting: allowLocalhost,
+      });
+
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => {
+        controller.abort(new FetchTimeoutError(`Request timed out after ${timeoutMs}ms`, timeoutMs));
+      }, timeoutMs);
+
+      try {
+        const reqHeaders: Record<string, string> = {
+          'User-Agent': userAgent,
+          Accept: 'text/html,application/xhtml+xml,text/plain,text/markdown,application/json,*/*',
+          ...options.headers,
+        };
+
+        const res = await activeFetch(currentUrl, {
+          method,
+          headers: reqHeaders,
+          redirect: 'manual',
+          signal: controller.signal,
+        });
+
+        // 2. Handle HTTP redirects (301, 302, 303, 307, 308)
+        if ([301, 302, 303, 307, 308].includes(res.status)) {
+          const location = res.headers.get('location');
+          if (!location) {
+            throw new DocOrbitError(`Redirect response (${res.status}) missing Location header.`);
+          }
+
+          redirectCount++;
+          if (redirectCount > maxRedirects) {
+            throw new DocOrbitError(`Exceeded maximum allowed redirects (${maxRedirects}).`);
+          }
+
+          const nextUrl = new URL(location, currentUrl).href;
+
+          // Check for redirect loop
+          if (seenRedirects.has(nextUrl)) {
+            throw new DocOrbitError(`Redirect loop detected: "${nextUrl}" already visited in redirect chain.`);
+          }
+          seenRedirects.add(nextUrl);
+
+          // Enforce domain/origin policy
+          if (options.allowCrossDomainRedirects === false) {
+            const origOrigin = new URL(urlStr).origin;
+            const nextOrigin = new URL(nextUrl).origin;
+            if (origOrigin !== nextOrigin) {
+              throw new DocOrbitError(`Cross-origin redirect from "${origOrigin}" to "${nextOrigin}" prohibited by policy.`);
+            }
+          }
+
+          currentUrl = nextUrl;
+          continue;
+        }
+
+        // 3. Extract response headers
+        const headers: Record<string, string> = {};
+        res.headers.forEach((val, key) => {
+          headers[key.toLowerCase()] = val;
+        });
+
+        const contentType = headers['content-type'] || 'application/octet-stream';
+
+        // 4. Stream response body with byte counting
+        let bytesRead = 0;
+        const chunks: Uint8Array[] = [];
+
+        if (res.body) {
+          const reader = res.body.getReader();
+          try {
+            while (true) {
+              const { done, value } = await reader.read();
+              if (done) break;
+              if (value) {
+                bytesRead += value.length;
+                if (bytesRead > maxBytes) {
+                  controller.abort();
+                  throw new PayloadTooLargeError(
+                    `Response exceeded maximum allowed size of ${maxBytes} bytes (read ${bytesRead} bytes).`,
+                    bytesRead,
+                    maxBytes
+                  );
+                }
+                chunks.push(value);
+              }
+            }
+          } finally {
+            reader.releaseLock();
+          }
+        }
+
+        const totalBuffer = Buffer.concat(chunks);
+        const bodyText = totalBuffer.toString('utf-8');
+
+        return {
+          url: urlStr,
+          finalUrl: currentUrl,
+          status: res.status,
+          statusText: res.statusText,
+          headers,
+          body: bodyText,
+          contentType,
+          bytesRead,
+        };
+      } catch (err: unknown) {
+        if (err instanceof DocOrbitError) throw err;
+        if (controller.signal.aborted) {
+          const reason = controller.signal.reason;
+          if (reason instanceof DocOrbitError) throw reason;
+          throw new FetchTimeoutError(`Request timed out after ${timeoutMs}ms`, timeoutMs);
+        }
+        throw new DocOrbitError(`Fetch failure for "${currentUrl}": ${err instanceof Error ? err.message : String(err)}`);
+      } finally {
+        clearTimeout(timeoutId);
+      }
+    }
+
+    throw new DocOrbitError(`Exceeded maximum allowed redirects (${maxRedirects}).`);
+  }
+}

package/packages/crawler/src/index.ts

@@ -0,0 +1,2 @@
+export * from './config.ts';
+export * from './fetcher.ts';

package/packages/discovery/src/index.ts

@@ -0,0 +1,31 @@
+import { DiscoveryCoordinator } from './provider.ts';
+import { LlmsTxtProvider } from './providers/llms-txt.ts';
+import { OpenApiProvider } from './providers/openapi.ts';
+import { MarkdownProvider } from './providers/markdown.ts';
+import { SitemapProvider } from './providers/sitemap.ts';
+import { GithubProvider } from './providers/github.ts';
+import { SkillProvider } from './providers/skill.ts';
+import { GenericWebProvider } from './providers/generic.ts';
+
+export * from './provider.ts';
+export * from './ranker.ts';
+export * from './providers/llms-txt.ts';
+export * from './providers/openapi.ts';
+export * from './providers/markdown.ts';
+export * from './providers/sitemap.ts';
+export * from './providers/github.ts';
+export * from './providers/skill.ts';
+export * from './providers/generic.ts';
+
+export function createDefaultDiscoveryCoordinator(): DiscoveryCoordinator {
+  const coordinator = new DiscoveryCoordinator();
+  coordinator
+    .register(new LlmsTxtProvider())
+    .register(new OpenApiProvider())
+    .register(new MarkdownProvider())
+    .register(new SitemapProvider())
+    .register(new SkillProvider())
+    .register(new GithubProvider())
+    .register(new GenericWebProvider());
+  return coordinator;
+}

package/packages/discovery/src/provider.ts

@@ -0,0 +1,47 @@
+import type { DiscoveredSource } from '../../shared/src/index.ts';
+import { SecureFetcher } from '../../crawler/src/index.ts';
+
+export interface DiscoveryProvider {
+  name: string;
+  discover(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]>;
+}
+
+export class DiscoveryCoordinator {
+  private providers: DiscoveryProvider[] = [];
+
+  register(provider: DiscoveryProvider): this {
+    this.providers.push(provider);
+    return this;
+  }
+
+  getProviders(): DiscoveryProvider[] {
+    return [...this.providers];
+  }
+
+  async discoverAll(targetUrl: string, fetcher: SecureFetcher): Promise<DiscoveredSource[]> {
+    const results = await Promise.allSettled(
+      this.providers.map(p => p.discover(targetUrl, fetcher))
+    );
+
+    const allSources: DiscoveredSource[] = [];
+    for (const res of results) {
+      if (res.status === 'fulfilled') {
+        allSources.push(...res.value);
+      }
+    }
+
+    // Deduplicate sources by normalized URL
+    const seen = new Set<string>();
+    const deduped: DiscoveredSource[] = [];
+
+    for (const s of allSources) {
+      const normalizedUrl = s.url.replace(/\/$/, '').toLowerCase();
+      if (!seen.has(normalizedUrl)) {
+        seen.add(normalizedUrl);
+        deduped.push(s);
+      }
+    }
+
+    return deduped;
+  }
+}