@dependabit/detector 0.1.16 → 0.1.18

package/src/parsers/code-comments.ts

@@ -1,178 +0,0 @@
-/**
- * Code Comment Parser
- * Extracts URLs and references from code comments
- */
-
-export interface CommentReference {
-  url: string;
-  context: string;
-  file: string;
-  line: number;
-  commentType: 'single-line' | 'multi-line' | 'jsdoc';
-}
-
-/**
- * Parse code files and extract references from comments
- */
-export function parseCodeComments(content: string, filePath: string): CommentReference[] {
-  const references: CommentReference[] = [];
-  const extension = getFileExtension(filePath);
-  const commentStyle = getCommentStyle(extension);
-
-  if (!commentStyle) {
-    return references; // Unsupported file type
-  }
-
-  const lines = content.split('\n');
-  let inMultiLineComment = false;
-
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i];
-    if (!line) continue;
-
-    const lineNumber = i + 1;
-
-    // Check for multi-line comment start/end
-    if (commentStyle.multiLine) {
-      if (line.includes(commentStyle.multiLine.start)) {
-        inMultiLineComment = true;
-      }
-      if (inMultiLineComment) {
-        const urls = extractUrls(line);
-        for (const url of urls) {
-          references.push({
-            url,
-            context: line.trim(),
-            file: filePath,
-            line: lineNumber,
-            commentType: line.includes('/**') ? 'jsdoc' : 'multi-line'
-          });
-        }
-      }
-      if (line.includes(commentStyle.multiLine.end)) {
-        inMultiLineComment = false;
-      }
-      continue;
-    }
-
-    // Check for single-line comments
-    if (commentStyle.singleLine) {
-      const commentStart = line.indexOf(commentStyle.singleLine);
-      if (commentStart !== -1) {
-        const comment = line.substring(commentStart);
-        const urls = extractUrls(comment);
-        for (const url of urls) {
-          references.push({
-            url,
-            context: comment.trim(),
-            file: filePath,
-            line: lineNumber,
-            commentType: 'single-line'
-          });
-        }
-      }
-    }
-  }
-
-  return references;
-}
-
-function getFileExtension(filePath: string): string {
-  const match = filePath.match(/\.([^.]+)$/);
-  const ext = match?.[1];
-  return ext ? ext.toLowerCase() : '';
-}
-
-interface CommentStyle {
-  singleLine?: string;
-  multiLine?: { start: string; end: string };
-}
-
-function getCommentStyle(extension: string): CommentStyle | null {
-  const styles: Record<string, CommentStyle> = {
-    // JavaScript/TypeScript
-    js: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-    ts: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-    jsx: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-    tsx: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // Python
-    py: { singleLine: '#' },
-
-    // Ruby
-    rb: { singleLine: '#', multiLine: { start: '=begin', end: '=end' } },
-
-    // Go
-    go: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // Rust
-    rs: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // C/C++
-    c: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-    cpp: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-    h: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // Java/Kotlin
-    java: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-    kt: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // C#
-    cs: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // PHP
-    php: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
-
-    // Shell
-    sh: { singleLine: '#' },
-    bash: { singleLine: '#' },
-
-    // YAML
-    yml: { singleLine: '#' },
-    yaml: { singleLine: '#' }
-  };
-
-  return styles[extension] || null;
-}
-
-function extractUrls(text: string): string[] {
-  const urls: string[] = [];
-  const regex = /https?:\/\/[^\s<>()[\]'"]+/g;
-  let match;
-
-  while ((match = regex.exec(text)) !== null) {
-    urls.push(match[0]);
-  }
-
-  return urls;
-}
-
-/**
- * Extract specification and RFC references from comments
- */
-export function extractSpecReferences(content: string): Array<{ spec: string; context: string }> {
-  const references: Array<{ spec: string; context: string }> = [];
-  const lines = content.split('\n');
-
-  for (const line of lines) {
-    // Match RFC references
-    const rfcMatch = /RFC\s*(\d+)/i.exec(line);
-    if (rfcMatch) {
-      references.push({
-        spec: `RFC ${rfcMatch[1]}`,
-        context: line.trim()
-      });
-    }
-
-    // Match standard references (ISO, IEEE, etc.)
-    const standardMatch = /(ISO|IEEE|ECMA|W3C)[\s-]*(\d+(?:[-.]\d+)*)/i.exec(line);
-    if (standardMatch) {
-      references.push({
-        spec: `${standardMatch[1]} ${standardMatch[2]}`,
-        context: line.trim()
-      });
-    }
-  }
-
-  return references;
-}

package/src/parsers/package-files.ts

@@ -1,156 +0,0 @@
-/**
- * Package File Parser
- * Extracts metadata and references from package manager files
- * EXCLUDES actual dependencies (handled by dependabot)
- */
-
-export interface PackageMetadata {
-  repository?: string;
-  homepage?: string;
-  documentation?: string;
-  urls: string[]; // URLs found in descriptions, etc.
-}
-
-/**
- * Parse package.json and extract metadata URLs (NOT dependencies)
- */
-export function parsePackageJson(content: string): PackageMetadata {
-  try {
-    const pkg = JSON.parse(content);
-    const urls: string[] = [];
-
-    // Extract repository URL
-    let repository: string | undefined = undefined;
-    if (typeof pkg.repository === 'string') {
-      repository = pkg.repository;
-    } else if (pkg.repository && pkg.repository.url) {
-      repository = pkg.repository.url;
-    }
-
-    // Extract homepage
-    const homepage: string | undefined = pkg.homepage;
-
-    // Extract documentation (not standard but sometimes present)
-    const documentation: string | undefined = pkg.documentation || pkg.docs;
-
-    // Extract URLs from description
-    if (pkg.description) {
-      const descUrls = extractUrls(pkg.description);
-      urls.push(...descUrls);
-    }
-
-    // Note: We DO NOT extract dependencies/devDependencies
-    // Those are handled by dependabot
-
-    return {
-      ...(repository !== undefined && { repository }),
-      ...(homepage !== undefined && { homepage }),
-      ...(documentation !== undefined && { documentation }),
-      urls
-    };
-  } catch  {
-    return { urls: [] };
-  }
-}
-
-/**
- * Parse requirements.txt and extract URLs from comments
- * EXCLUDES actual packages (handled by dependabot)
- */
-export function parseRequirementsTxt(content: string): PackageMetadata {
-  const urls: string[] = [];
-  const lines = content.split('\n');
-
-  for (const line of lines) {
-    // Only extract URLs from comments
-    if (line.trim().startsWith('#')) {
-      const commentUrls = extractUrls(line);
-      urls.push(...commentUrls);
-    }
-    // Skip actual package lines - dependabot handles those
-  }
-
-  return { urls };
-}
-
-/**
- * Parse Cargo.toml and extract metadata URLs
- * EXCLUDES actual dependencies (handled by dependabot)
- */
-export function parseCargoToml(content: string): PackageMetadata {
-  const urls: string[] = [];
-  let repository: string | undefined = undefined;
-  let homepage: string | undefined = undefined;
-  let documentation: string | undefined = undefined;
-
-  const lines = content.split('\n');
-  let inPackageSection = false;
-
-  for (const line of lines) {
-    if (line.trim() === '[package]') {
-      inPackageSection = true;
-      continue;
-    }
-
-    if (line.trim().startsWith('[') && line.trim() !== '[package]') {
-      inPackageSection = false;
-      continue;
-    }
-
-    if (inPackageSection) {
-      const repoMatch = /repository\s*=\s*"([^"]+)"/.exec(line);
-      if (repoMatch && repoMatch[1]) repository = repoMatch[1];
-
-      const homepageMatch = /homepage\s*=\s*"([^"]+)"/.exec(line);
-      if (homepageMatch && homepageMatch[1]) homepage = homepageMatch[1];
-
-      const docMatch = /documentation\s*=\s*"([^"]+)"/.exec(line);
-      if (docMatch && docMatch[1]) documentation = docMatch[1];
-    }
-
-    // Extract URLs from comments
-    if (line.trim().startsWith('#')) {
-      const commentUrls = extractUrls(line);
-      urls.push(...commentUrls);
-    }
-  }
-
-  return {
-    ...(repository !== undefined && { repository }),
-    ...(homepage !== undefined && { homepage }),
-    ...(documentation !== undefined && { documentation }),
-    urls
-  };
-}
-
-/**
- * Parse go.mod and extract URLs from comments
- * EXCLUDES actual dependencies (handled by dependabot)
- */
-export function parseGoMod(content: string): PackageMetadata {
-  const urls: string[] = [];
-  const lines = content.split('\n');
-
-  for (const line of lines) {
-    // Only extract URLs from comments
-    if (line.trim().startsWith('//')) {
-      const commentUrls = extractUrls(line);
-      urls.push(...commentUrls);
-    }
-    // Skip actual require lines - dependabot handles those
-  }
-
-  return { urls };
-}
-
-function extractUrls(text: string): string[] {
-  const urls: string[] = [];
-  const regex = /https?:\/\/[^\s<>()[\]'"]+/g;
-  let match;
-
-  while ((match = regex.exec(text)) !== null) {
-    urls.push(match[0]);
-  }
-
-  return urls;
-}

package/src/parsers/readme.ts

@@ -1,191 +0,0 @@
-/**
- * README Parser
- * Extracts URLs and references from README and markdown files
- */
-
-export interface ExtractedReference {
-  url: string;
-  context: string; // Surrounding text
-  line?: number;
-  type: 'markdown-link' | 'bare-url' | 'reference-link';
-}
-
-// Patterns to skip (package managers, CI badges, shields.io, placeholders)
-const SKIP_PATTERNS = [
-  /npmjs\.com\/package/,
-  /pypi\.org\/project/,
-  /crates\.io\/crates/,
-  /rubygems\.org\/gems/,
-  /packagist\.org\/packages/,
-  /shields\.io/,
-  /badge(s)?\..*\.svg/,
-  /travis-ci\.(org|com)/,
-  /circleci\.com/,
-  /github\.com\/.*\/actions/, // GitHub Actions badges
-  /example\.com/, // placeholder domain
-  /example\.org/, // placeholder domain
-  /localhost/, // local dev URLs
-  /127\.0\.0\.1/, // loopback address
-  /\[.*\]/, // template placeholders (e.g., issues/[NUMBER])
-  /github\.com\/user\/repo/ // common placeholder in docs
-];
-
-/**
- * Parse README content and extract external references
- */
-export function parseReadme(content: string, filePath = 'README.md'): ExtractedReference[] {
-  const references: ExtractedReference[] = [];
-  const lines = content.split('\n');
-
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i];
-    if (!line) continue; // Skip undefined or empty lines
-
-    const lineNumber = i + 1;
-
-    // Extract markdown links [text](url)
-    const markdownLinks = extractMarkdownLinks(line);
-    for (const { url, text } of markdownLinks) {
-      if (!shouldSkipUrl(url)) {
-        references.push({
-          url,
-          context: text || line.trim(),
-          line: lineNumber,
-          type: 'markdown-link'
-        });
-      }
-    }
-
-    // Extract reference-style links [text]: url
-    const referenceLinks = extractReferenceLinks(line);
-    for (const { url, text } of referenceLinks) {
-      if (!shouldSkipUrl(url)) {
-        references.push({
-          url,
-          context: text || line.trim(),
-          line: lineNumber,
-          type: 'reference-link'
-        });
-      }
-    }
-
-    // Extract bare URLs
-    const bareUrls = extractBareUrls(line);
-    for (const url of bareUrls) {
-      if (!shouldSkipUrl(url)) {
-        references.push({
-          url,
-          context: line.trim(),
-          line: lineNumber,
-          type: 'bare-url'
-        });
-      }
-    }
-  }
-
-  // Deduplicate by URL
-  return deduplicateReferences(references);
-}
-
-function extractMarkdownLinks(line: string): Array<{ url: string; text: string }> {
-  const links: Array<{ url: string; text: string }> = [];
-  const regex = /\[([^\]]+)\]\(([^)]+)\)/g;
-  let match;
-
-  while ((match = regex.exec(line)) !== null) {
-    const text = match[1];
-    const url = match[2];
-    if (text !== undefined && url !== undefined) {
-      links.push({ text, url });
-    }
-  }
-
-  return links;
-}
-
-function extractReferenceLinks(line: string): Array<{ url: string; text: string }> {
-  const links: Array<{ url: string; text: string }> = [];
-  const regex = /^\[([^\]]+)\]:\s+(.+)$/;
-  const match = regex.exec(line);
-
-  if (match) {
-    const text = match[1];
-    const url = match[2];
-    if (text !== undefined && url !== undefined) {
-      links.push({ text, url });
-    }
-  }
-
-  return links;
-}
-
-function extractBareUrls(line: string): string[] {
-  const urls: string[] = [];
-  const regex = /https?:\/\/[^\s<>()[\]]+/g;
-  let match;
-
-  while ((match = regex.exec(line)) !== null) {
-    urls.push(match[0]);
-  }
-
-  return urls;
-}
-
-function shouldSkipUrl(url: string): boolean {
-  return SKIP_PATTERNS.some((pattern) => pattern.test(url));
-}
-
-function deduplicateReferences(references: ExtractedReference[]): ExtractedReference[] {
-  const seen = new Set<string>();
-  return references.filter((ref) => {
-    if (seen.has(ref.url)) {
-      return false;
-    }
-    seen.add(ref.url);
-    return true;
-  });
-}
-
-/**
- * Extract GitHub repository mentions (owner/repo format)
- */
-export function extractGitHubReferences(
-  content: string
-): Array<{ owner: string; repo: string; context: string }> {
-  const references: Array<{ owner: string; repo: string; context: string }> = [];
-  const lines = content.split('\n');
-
-  for (const line of lines) {
-    // Match owner/repo pattern not in URLs, with basic length and context constraints
-    const regex =
-      /(?<!https?:\/\/github\.com\/)(?:^|[\s(])([a-zA-Z0-9_-]{2,}\/[a-zA-Z0-9_.-]{2,})(?=$|[\s),.;])/g;
-    let match;
-
-    while ((match = regex.exec(line)) !== null) {
-      const ownerRepo = match[1];
-      if (ownerRepo) {
-        const parts = ownerRepo.split('/');
-        const owner = parts[0];
-        const repo = parts[1];
-
-        if (
-          owner &&
-          repo &&
-          owner.length >= 2 &&
-          repo.length >= 2 &&
-          owner !== 'owner' &&
-          repo !== 'repo' &&
-          !(/^\d+$/.test(owner) && /^\d+$/.test(repo))
-        ) {
-          references.push({
-            owner,
-            repo,
-            context: line.trim()
-          });
-        }
-      }
-    }
-  }
-
-  return references;
-}

package/test/detector.test.ts

@@ -1,169 +0,0 @@
-import { describe, it, expect, beforeEach, vi } from 'vitest';
-import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { Detector } from '../src/detector.js';
-
-const createDetector = (): Detector =>
-  new Detector({
-    repoPath: '.',
-    llmProvider: {
-      analyze: vi.fn(async () => ({
-        dependencies: [],
-        usage: { totalTokens: 0, latencyMs: 0 },
-        rawResponse: '{}'
-      }))
-    }
-  });
-
-describe('Detector', () => {
-  beforeEach(() => {
-    vi.clearAllMocks();
-  });
-
-  describe('detectDependencies', () => {
-    it('should orchestrate all parsers', async () => {
-      // Expected: Call README parser, code comment parser, package file parser
-      expect(true).toBe(true);
-    });
-
-    it('should aggregate results from all parsers', async () => {
-      // Expected: Combine results from multiple parsers
-      expect(true).toBe(true);
-    });
-
-    it('should send aggregated content to LLM for analysis', async () => {
-      // Expected: Pass extracted content to LLM provider
-      expect(true).toBe(true);
-    });
-
-    it('should deduplicate dependencies by URL', async () => {
-      // Expected: Same URL from multiple sources = one dependency
-      expect(true).toBe(true);
-    });
-
-    it('should calculate confidence scores', async () => {
-      // Expected: LLM confidence * detection method weight
-      expect(true).toBe(true);
-    });
-
-    it('should generate UUIDs for each dependency', async () => {
-      expect(true).toBe(true);
-    });
-
-    it('should include detection metadata', async () => {
-      // Expected: detectionMethod, detectedAt, referencedIn
-      expect(true).toBe(true);
-    });
-
-    it('should handle empty repository', async () => {
-      // Expected: Return empty dependencies array
-      expect(true).toBe(true);
-    });
-
-    it('should handle LLM failures gracefully', async () => {
-      // Expected: Continue with parser results, log error
-      expect(true).toBe(true);
-    });
-
-    it('should keep GitHub repo links when LLM parser misses them', async () => {
-      const repoPath = await mkdtemp(join(tmpdir(), 'dependabit-detector-'));
-
-      try {
-        await mkdir(join(repoPath, 'docs'), { recursive: true });
-        await writeFile(
-          join(repoPath, 'docs', 'tooling.md'),
-          '- [spec-kit](https://github.com/github/spec-kit)\n'
-        );
-
-        const detector = new Detector({
-          repoPath,
-          llmProvider: {
-            analyze: vi.fn(async () => ({
-              dependencies: [],
-              usage: { totalTokens: 0, latencyMs: 0 },
-              rawResponse: '{}'
-            })),
-            getSupportedModels: vi.fn(() => ['test-model']),
-            getRateLimit: vi.fn(async () => ({
-              remaining: 100,
-              limit: 100,
-              resetAt: new Date(Date.now() + 60_000)
-            })),
-            validateConfig: vi.fn(() => true)
-          },
-          repoOwner: 'pradeepmouli',
-          repoName: 'dependabit'
-        });
-
-        const result = await detector.detectDependencies();
-        const dep = result.dependencies.find((d) => d.url === 'https://github.com/github/spec-kit');
-
-        expect(dep).toBeDefined();
-        expect(dep?.type).toBe('reference-implementation');
-        expect(dep?.accessMethod).toBe('github-api');
-      } finally {
-        await rm(repoPath, { recursive: true, force: true });
-      }
-    });
-  });
-
-  describe('classifyDependency', () => {
-    it('should classify GitHub URLs as repository', () => {
-      const url = 'https://github.com/owner/repo';
-      expect(url).toContain('github.com');
-    });
-
-    it('should classify GitHub repository links as reference-implementation', () => {
-      const detector = createDetector();
-      const type = (detector as any).determineDependencyType(
-        'https://github.com/github/spec-kit',
-        'spec-kit'
-      );
-
-      expect(type).toBe('reference-implementation');
-    });
-
-    it('should classify arXiv URLs as research-paper', () => {
-      const url = 'https://arxiv.org/abs/1706.03762';
-      expect(url).toContain('arxiv.org');
-    });
-
-    it('should classify OpenAPI specs as schema', () => {
-      const url = 'https://api.example.com/openapi.yaml';
-      expect(url).toContain('openapi');
-    });
-
-    it('should classify documentation sites as documentation', () => {
-      const url = 'https://docs.example.com/guide';
-      expect(url).toContain('docs.');
-    });
-
-    it('should use LLM for ambiguous URLs', () => {
-      const url = 'https://example.com/some-resource';
-      expect(typeof url).toBe('string');
-    });
-  });
-
-  describe('determineAccessMethod', () => {
-    it('should use github-api for GitHub URLs', () => {
-      const url = 'https://github.com/owner/repo';
-      expect(url).toContain('github.com');
-    });
-
-    it('should use arxiv for arXiv URLs', () => {
-      const url = 'https://arxiv.org/abs/1234.5678';
-      expect(url).toContain('arxiv.org');
-    });
-
-    it('should use openapi for OpenAPI spec URLs', () => {
-      const url = 'https://api.example.com/openapi.json';
-      expect(url).toContain('openapi');
-    });
-
-    it('should use http as fallback', () => {
-      const url = 'https://example.com/docs';
-      expect(url).toContain('http');
-    });
-  });
-});