docorbit 0.1.0 → 0.1.1

package/packages/retrieval/src/scorer.ts

@@ -1,139 +0,0 @@
-import type {
-  DocumentChunk,
-  SymbolReference,
-  ChunkCode,
-  ScoringWeights,
-  QueryIntent,
-  SourceAuthority,
-  SearchResult,
-} from '../../shared/src/index.ts';
-
-export interface ScoreCandidateInput {
-  chunk: DocumentChunk;
-  ftsRank: number;
-  symbols: SymbolReference[];
-  codeSnippets: ChunkCode[];
-  query: string;
-  intent: QueryIntent;
-  authority?: SourceAuthority;
-  targetDocVersion?: string;
-  weights: ScoringWeights;
-}
-
-export function scoreChunkCandidate(input: ScoreCandidateInput): SearchResult {
-  const { chunk, ftsRank, symbols, codeSnippets, query, intent, authority = 'official', targetDocVersion, weights } = input;
-
-  const matchReasons: string[] = [];
-  const queryLower = query.toLowerCase().trim();
-  const queryTokens = queryLower
-    .replace(/[^\w\s-]/g, ' ')
-    .split(/\s+/)
-    .filter(t => t.length > 2);
-
-  // 1. Base BM25 score from FTS5 (FTS5 rank is negative where smaller = more relevant)
-  const normalizedBm25 = Math.max(0.1, -ftsRank);
-  let totalScore = normalizedBm25 * weights.bm25Weight;
-  matchReasons.push(`BM25 base (${normalizedBm25.toFixed(2)})`);
-
-  // 2. Exact phrase match in content
-  if (chunk.content.toLowerCase().includes(queryLower)) {
-    totalScore += weights.exactPhraseBonus;
-    matchReasons.push(`Exact phrase match (+${weights.exactPhraseBonus})`);
-  }
-
-  // 3. Title match
-  const titleLower = (chunk.title || '').toLowerCase();
-  const titleMatches = queryTokens.filter(t => titleLower.includes(t));
-  if (titleMatches.length > 0) {
-    const boost = weights.titleBonus * (titleMatches.length / queryTokens.length);
-    totalScore += boost;
-    matchReasons.push(`Title match [${titleMatches.join(', ')}] (+${boost.toFixed(1)})`);
-  }
-
-  // 4. Section path match
-  const pathString = chunk.sectionPath.join(' ').toLowerCase();
-  const pathMatches = queryTokens.filter(t => pathString.includes(t));
-  if (pathMatches.length > 0) {
-    const boost = weights.sectionPathBonus * (pathMatches.length / queryTokens.length);
-    totalScore += boost;
-    matchReasons.push(`Section breadcrumbs match [${pathMatches.join(', ')}] (+${boost.toFixed(1)})`);
-  }
-
-  // 5. Symbol match (shallow deterministic regex symbols)
-  const matchedSymbols = symbols.filter(s => {
-    const symLower = s.name.toLowerCase();
-    return queryLower.includes(symLower) || symLower.includes(queryLower);
-  });
-  if (matchedSymbols.length > 0) {
-    totalScore += weights.symbolMatchBonus;
-    matchReasons.push(`Symbol match [${matchedSymbols.map(s => s.name).join(', ')}] (+${weights.symbolMatchBonus})`);
-  }
-
-  // 6. Query intent alignment
-  let intentBoost = 0;
-  if (intent === 'api' && chunk.chunkType === 'api') {
-    intentBoost = weights.intentTypeBonus;
-  } else if (intent === 'examples' && (chunk.chunkType === 'example' || chunk.chunkType === 'code')) {
-    intentBoost = weights.intentTypeBonus;
-  } else if (intent === 'troubleshooting' && (chunk.chunkType === 'warning' || /error|exception|fail|status|code/i.test(chunk.content))) {
-    intentBoost = weights.intentTypeBonus;
-  } else if (intent === 'implementation' && (chunk.chunkType === 'mixed' || chunk.chunkType === 'example')) {
-    intentBoost = weights.intentTypeBonus;
-  } else if (intent === 'conceptual' && chunk.chunkType === 'prose') {
-    intentBoost = weights.intentTypeBonus;
-  } else if (intent === 'configuration' && (symbols.some(s => s.kind === 'config') || chunk.chunkType === 'prose')) {
-    intentBoost = weights.intentTypeBonus;
-  }
-
-  if (intentBoost > 0) {
-    totalScore += intentBoost;
-    matchReasons.push(`Intent alignment '${intent}' for chunkType '${chunk.chunkType}' (+${intentBoost})`);
-  }
-
-  // 7. Documentation version boosting & nuance
-  if (targetDocVersion && chunk.docVersion) {
-    const chunkVerClean = chunk.docVersion.replace(/^[v=]/, '');
-    const targetVerClean = targetDocVersion.replace(/^[v=]/, '');
-
-    const chunkMajor = chunkVerClean.split('.')[0];
-    const targetMajor = targetVerClean.split('.')[0];
-
-    if (chunkVerClean === targetVerClean || chunk.docVersion === targetDocVersion) {
-      // Exact version match
-      totalScore += weights.versionBonus;
-      matchReasons.push(`Target doc version exact match [${chunk.docVersion}] (+${weights.versionBonus})`);
-    } else if (chunkMajor === targetMajor) {
-      // Same major version (e.g. target 14.2.3, chunk v14 or 14.1)
-      const majorBoost = weights.versionBonus * 0.85;
-      totalScore += majorBoost;
-      matchReasons.push(`Target doc major match [${chunk.docVersion}] (+${majorBoost.toFixed(1)})`);
-    } else if (
-      chunk.docVersion === 'latest' ||
-      chunk.docVersion === 'stable' ||
-      chunk.docVersion === 'current'
-    ) {
-      // Unversioned / latest documentation: neutral supplemental context
-      matchReasons.push(`Supplemental unversioned/latest documentation [${chunk.docVersion}]`);
-    } else {
-      // Evidence of major version incompatibility (e.g. target v14 vs chunk v16)
-      const penalty = weights.versionBonus * 0.5;
-      totalScore -= penalty;
-      matchReasons.push(`Version major discrepancy [chunk ${chunk.docVersion} vs target ${targetDocVersion}] (-${penalty.toFixed(1)})`);
-    }
-  }
-
-  // 8. Authority weighting
-  const authMultiplier = weights.authorityWeights[authority] ?? 1.0;
-  totalScore *= authMultiplier;
-  if (authMultiplier !== 1.0) {
-    matchReasons.push(`Authority weighting: x${authMultiplier}`);
-  }
-
-  return {
-    chunk,
-    score: parseFloat(totalScore.toFixed(3)),
-    matchReasons,
-    symbols,
-    codeSnippets,
-  };
-}

package/packages/retrieval/src/weights.ts

@@ -1,31 +0,0 @@
-import type { ScoringWeights } from '../../shared/src/index.ts';
-
-export const DEFAULT_SCORING_WEIGHTS_V1: ScoringWeights = {
-  version: '1.0.0',
-  bm25Weight: 1.0,
-  exactPhraseBonus: 5.0,
-  titleBonus: 8.0,
-  sectionPathBonus: 4.0,
-  symbolMatchBonus: 10.0,
-  intentTypeBonus: 6.0,
-  versionBonus: 12.0,
-  authorityWeights: {
-    official: 1.0,
-    community: 0.8,
-    third_party: 0.6,
-  },
-};
-
-export function resolveScoringWeights(overrides?: Partial<ScoringWeights>): ScoringWeights {
-  if (!overrides) {
-    return { ...DEFAULT_SCORING_WEIGHTS_V1 };
-  }
-  return {
-    ...DEFAULT_SCORING_WEIGHTS_V1,
-    ...overrides,
-    authorityWeights: {
-      ...DEFAULT_SCORING_WEIGHTS_V1.authorityWeights,
-      ...(overrides.authorityWeights || {}),
-    },
-  };
-}

package/packages/security/src/annotations.ts

@@ -1,112 +0,0 @@
-import type { SecurityAnnotation } from '../../shared/src/index.ts';
-
-const MULTILINE_INJECTION_PATTERNS = [
-  { pattern: /<system\b[^>]*>([\s\S]*?)<\/system>/gi, severity: 'high' as const, label: '<system> block' },
-  { pattern: /\[SYSTEM(?:\s+INSTRUCTION)?\]([\s\S]*?)\[\/SYSTEM(?:\s+INSTRUCTION)?\]/gi, severity: 'high' as const, label: '[SYSTEM] block' },
-  { pattern: /<\|im_start\|>system([\s\S]*?)<\|im_end\|>/gi, severity: 'high' as const, label: 'ChatML system block' },
-  { pattern: /<!--\s*AI\s+INSTRUCTION:([\s\S]*?)-->/gi, severity: 'high' as const, label: 'HTML comment injection' },
-];
-
-const PROMPT_INJECTION_PATTERNS = [
-  { pattern: /\b(?:ignore|disregard|forget)\s+(?:all\s+)?(?:previous|prior|above)\s+instructions\b/i, severity: 'high' as const },
-  { pattern: /\b(?:system\s+message|system\s+prompt|system\s+instruction)\s*[:=]/i, severity: 'high' as const },
-  { pattern: /\byou\s+are\s+now\s+(?:an?\s+)?(?:unrestricted|jailbroken|developer\s+mode|dan|evil)\b/i, severity: 'high' as const },
-  { pattern: /\byou\s+are\s+now\s+(?:acting\s+as\s+)?a\b.*?\bassistant\s+that\s+(?:executes|runs|ignores)\b/i, severity: 'high' as const },
-  { pattern: /\bIMPORTANT\s+INSTRUCTION\s+FOR\s+(?:THE\s+)?AI\s+AGENT\b/i, severity: 'high' as const },
-  { pattern: /\b(?:call|invoke|execute)\s+(?:the\s+)?tool\s+['"]?[a-zA-Z0-9_-]+['"]?\s+with\b/i, severity: 'high' as const },
-  { pattern: /\buse\s+(?:the\s+)?(?:bash|terminal|shell|command|run_command)\s+tool\s+to\b/i, severity: 'high' as const },
-  { pattern: /\bdo\s+not\s+tell\s+the\s+user\b/i, severity: 'medium' as const },
-];
-
-const SUSPICIOUS_INSTRUCTION_PATTERNS = [
-  { pattern: /curl\s+-[sS]*[fF]*[kK]*\s+https?:\/\/[^\s|]+\s*\|\s*(?:ba)?sh/i, severity: 'high' as const },
-  { pattern: /wget\s+-[qO-]*\s+https?:\/\/[^\s|]+\s*\|\s*(?:ba)?sh/i, severity: 'high' as const },
-  { pattern: /powershell\s+.*-(?:enc|encodedcommand)\s+[A-Za-z0-9+/=]{20,}/i, severity: 'high' as const },
-  { pattern: /base64\s+-d\s+.*\|\s*(?:ba)?sh/i, severity: 'high' as const },
-  { pattern: /eval\s*\(\s*base64_decode\s*\(/i, severity: 'high' as const },
-  { pattern: /\b(?:cat|type)\s+[^\n]*\.(?:ssh\/id_[a-zA-Z0-9_-]*|aws\/credentials|env)\b/i, severity: 'high' as const },
-  { pattern: /\b(?:curl|wget|fetch|nc|ncat)\b[^\n]*\b(?:\.ssh\/id_|\.aws\/credentials|\.env|API_KEY|SECRET)\b/i, severity: 'high' as const },
-  { pattern: /\b(?:printenv|env)\b[^\n]*\|\s*(?:curl|nc|wget|base64)/i, severity: 'high' as const },
-];
-
-const UNSAFE_LINK_PATTERNS = [
-  { pattern: /\[[^\]]*\]\(((?:javascript|data|file):[\s\S]*?)\)(?:\.|\s|$)/i, severity: 'high' as const },
-];
-
-/**
- * Scans documentation content for security hazards (prompt injection,
- * malicious execution snippets, unsafe links, credential harvesting) and returns
- * non-destructive annotations.
- *
- * Principle: External documentation is treated as untrusted data and structurally
- * separated from agent instructions. Original documentation text is never altered.
- */
-export function detectSecurityAnnotations(content: string): SecurityAnnotation[] {
-  const annotations: SecurityAnnotation[] = [];
-  if (!content) return annotations;
-
-  // 1. Multiline patterns across whole content
-  for (const rule of MULTILINE_INJECTION_PATTERNS) {
-    let match: RegExpExecArray | null;
-    const regex = new RegExp(rule.pattern.source, rule.pattern.flags);
-    while ((match = regex.exec(content)) !== null) {
-      const matchStart = match.index;
-      const lineNumber = content.slice(0, matchStart).split('\n').length;
-      annotations.push({
-        type: 'prompt_injection_suspected',
-        location: `Line ${lineNumber} (${rule.label})`,
-        evidence: match[0].slice(0, 150).trim(),
-        severity: rule.severity,
-      });
-    }
-  }
-
-  // 2. Line-by-line checks
-  const lines = content.split('\n');
-
-  for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
-    const line = lines[lineIdx];
-    const loc = `Line ${lineIdx + 1}`;
-
-    // Prompt injection checks
-    for (const rule of PROMPT_INJECTION_PATTERNS) {
-      const match = line.match(rule.pattern);
-      if (match) {
-        annotations.push({
-          type: 'prompt_injection_suspected',
-          location: loc,
-          evidence: match[0].trim(),
-          severity: rule.severity,
-        });
-      }
-    }
-
-    // Suspicious shell execution / credential theft checks
-    for (const rule of SUSPICIOUS_INSTRUCTION_PATTERNS) {
-      const match = line.match(rule.pattern);
-      if (match) {
-        annotations.push({
-          type: 'suspicious_instruction',
-          location: loc,
-          evidence: match[0].trim(),
-          severity: rule.severity,
-        });
-      }
-    }
-
-    // Unsafe link protocols check
-    for (const rule of UNSAFE_LINK_PATTERNS) {
-      const match = line.match(rule.pattern);
-      if (match) {
-        annotations.push({
-          type: 'unsafe_link',
-          location: loc,
-          evidence: match[1] || match[0],
-          severity: rule.severity,
-        });
-      }
-    }
-  }
-
-  return annotations;
-}

package/packages/security/src/ssrf.ts

@@ -1,153 +0,0 @@
-import { lookup } from 'node:dns/promises';
-import { isIP } from 'node:net';
-import { SsrfError } from '../../shared/src/index.ts';
-
-/**
- * Checks whether an IPv4 or IPv6 address is in a private, link-local, loopback,
- * or cloud provider metadata range.
- */
-export function isPrivateOrBlockedIp(ip: string): boolean {
-  // IPv4 check
-  if (isIP(ip) === 4) {
-    const parts = ip.split('.').map(Number);
-    if (parts.length !== 4 || parts.some(isNaN)) return true;
-
-    const [b0, b1, b2, b3] = parts;
-
-    // 0.0.0.0/8 (Current network)
-    if (b0 === 0) return true;
-
-    // 127.0.0.0/8 (Loopback)
-    if (b0 === 127) return true;
-
-    // 10.0.0.0/8 (RFC 1918 Private)
-    if (b0 === 10) return true;
-
-    // 172.16.0.0/12 (RFC 1918 Private: 172.16.0.0 - 172.31.255.255)
-    if (b0 === 172 && b1 >= 16 && b1 <= 31) return true;
-
-    // 192.168.0.0/16 (RFC 1918 Private)
-    if (b0 === 192 && b1 === 168) return true;
-
-    // 169.254.0.0/16 (Link-local & AWS/GCP/Azure instance metadata 169.254.169.254)
-    if (b0 === 169 && b1 === 254) return true;
-
-    // 100.64.0.0/10 (Carrier-grade NAT)
-    if (b0 === 100 && b1 >= 64 && b1 <= 127) return true;
-
-    // 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24 (TEST-NET-1, 2, 3)
-    if (b0 === 192 && b1 === 0 && b2 === 2) return true;
-    if (b0 === 198 && b1 === 51 && b2 === 100) return true;
-    if (b0 === 203 && b1 === 0 && b2 === 113) return true;
-
-    // 224.0.0.0/4 (Multicast) & 240.0.0.0/4 (Reserved)
-    if (b0 >= 224) return true;
-
-    return false;
-  }
-
-  // IPv6 check
-  if (isIP(ip) === 6) {
-    const normalized = ip.toLowerCase();
-
-    // ::1 (Loopback)
-    if (normalized === '::1' || normalized === '0:0:0:0:0:0:0:1') return true;
-
-    // :: (Unspecified)
-    if (normalized === '::' || normalized === '0:0:0:0:0:0:0:0') return true;
-
-    // fc00::/7 (Unique local addresses / private)
-    if (normalized.startsWith('fc') || normalized.startsWith('fd')) return true;
-
-    // fe80::/10 (Link-local unicast)
-    if (normalized.startsWith('fe8') || normalized.startsWith('fe9') || normalized.startsWith('fea') || normalized.startsWith('feb')) {
-      return true;
-    }
-
-    // IPv4-mapped IPv6 addresses (::ffff:x.x.x.x)
-    if (normalized.startsWith('::ffff:')) {
-      const ipv4Part = normalized.slice(7);
-      return isPrivateOrBlockedIp(ipv4Part);
-    }
-
-    return false;
-  }
-
-  return true; // Not a recognized IP, block by default
-}
-
-export interface SsrfValidationOptions {
-  allowLocalhostForTesting?: boolean;
-}
-
-/**
- * Validates a target URL against SSRF threats:
- * - Ensures valid HTTP/HTTPS protocol
- * - Validates hostname does not resolve to private or loopback IP
- */
-export async function validateTargetUrl(
-  urlStr: string,
-  options: SsrfValidationOptions = {}
-): Promise<{ url: URL; resolvedIp: string }> {
-  let url: URL;
-  try {
-    url = new URL(urlStr);
-  } catch {
-    throw new SsrfError(`Malformed target URL: "${urlStr}"`);
-  }
-
-  // Protocol check
-  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
-    throw new SsrfError(`Disallowed protocol "${url.protocol}". Only http: and https: are allowed.`);
-  }
-
-  const hostname = url.hostname.toLowerCase();
-  const cleanHost = (hostname.startsWith('[') && hostname.endsWith(']'))
-    ? hostname.slice(1, -1)
-    : hostname;
-
-  // Handle explicit localhost / loopback strings
-  if (cleanHost === 'localhost' || cleanHost === '127.0.0.1' || cleanHost === '::1') {
-    if (options.allowLocalhostForTesting) {
-      return { url, resolvedIp: cleanHost === 'localhost' ? '127.0.0.1' : cleanHost };
-    }
-    throw new SsrfError(`Access to localhost is prohibited for security.`, '127.0.0.1', cleanHost);
-  }
-
-  // If the hostname itself is a literal IP address (IPv4 or IPv6)
-  if (isIP(cleanHost)) {
-    if (isPrivateOrBlockedIp(cleanHost)) {
-      if (options.allowLocalhostForTesting && (cleanHost === '127.0.0.1' || cleanHost === '::1')) {
-        return { url, resolvedIp: cleanHost };
-      }
-      throw new SsrfError(`Access to private/blocked IP address "${cleanHost}" is prohibited.`, cleanHost, cleanHost);
-    }
-    return { url, resolvedIp: cleanHost };
-  }
-
-  // Resolve hostname via DNS
-  try {
-    const res = await lookup(cleanHost, { all: true });
-    if (!res || res.length === 0) {
-      throw new SsrfError(`Hostname "${cleanHost}" could not be resolved via DNS.`);
-    }
-
-    for (const record of res) {
-      if (isPrivateOrBlockedIp(record.address)) {
-        if (options.allowLocalhostForTesting && (record.address === '127.0.0.1' || record.address === '::1')) {
-          continue;
-        }
-        throw new SsrfError(
-          `Hostname "${cleanHost}" resolved to blocked/private IP address "${record.address}".`,
-          record.address,
-          cleanHost
-        );
-      }
-    }
-
-    return { url, resolvedIp: res[0].address };
-  } catch (err: unknown) {
-    if (err instanceof SsrfError) throw err;
-    throw new SsrfError(`DNS resolution failure for host "${cleanHost}": ${err instanceof Error ? err.message : String(err)}`);
-  }
-}

package/packages/shared/src/errors.ts

@@ -1,53 +0,0 @@
-export class DocOrbitError extends Error {
-  code: string;
-
-  constructor(message: string, code: string = 'DOCORBIT_ERROR') {
-    super(message);
-    this.name = 'DocOrbitError';
-    this.code = code;
-  }
-}
-
-export class SsrfError extends DocOrbitError {
-  ip?: string;
-  host?: string;
-
-  constructor(message: string, ip?: string, host?: string) {
-    super(message, 'SSRF_BLOCKED');
-    this.name = 'SsrfError';
-    this.ip = ip;
-    this.host = host;
-  }
-}
-
-export class FetchTimeoutError extends DocOrbitError {
-  timeoutMs?: number;
-
-  constructor(message: string, timeoutMs?: number) {
-    super(message, 'FETCH_TIMEOUT');
-    this.name = 'FetchTimeoutError';
-    this.timeoutMs = timeoutMs;
-  }
-}
-
-export class PayloadTooLargeError extends DocOrbitError {
-  byteCount?: number;
-  maxBytes?: number;
-
-  constructor(message: string, byteCount?: number, maxBytes?: number) {
-    super(message, 'PAYLOAD_TOO_LARGE');
-    this.name = 'PayloadTooLargeError';
-    this.byteCount = byteCount;
-    this.maxBytes = maxBytes;
-  }
-}
-
-export class ValidationError extends DocOrbitError {
-  validationDetails?: unknown;
-
-  constructor(message: string, validationDetails?: unknown) {
-    super(message, 'VALIDATION_FAILED');
-    this.name = 'ValidationError';
-    this.validationDetails = validationDetails;
-  }
-}

package/packages/shared/src/hashing.ts

@@ -1,23 +0,0 @@
-import { createHash } from 'node:crypto';
-
-export function computeSha256(content: string | Buffer): string {
-  return createHash('sha256').update(content).digest('hex');
-}
-
-export function computeContentHash(content: string): string {
-  // Normalize line endings and line-by-line trailing whitespace for canonical stability
-  const normalized = content
-    .replace(/\r\n/g, '\n')
-    .replace(/\r/g, '\n')
-    .split('\n')
-    .map(line => line.trimEnd())
-    .join('\n')
-    .trim();
-  return computeSha256(normalized);
-}
-
-export function estimateTokenCount(text: string): number {
-  // Heuristic: roughly ~4 characters per token for English and Markdown/code
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
-}