@dependabit/detector 0.1.1

package/src/parsers/code-comments.ts

@@ -0,0 +1,178 @@
+/**
+ * Code Comment Parser
+ * Extracts URLs and references from code comments
+ */
+
+export interface CommentReference {
+  url: string;
+  context: string;
+  file: string;
+  line: number;
+  commentType: 'single-line' | 'multi-line' | 'jsdoc';
+}
+
+/**
+ * Parse code files and extract references from comments
+ */
+export function parseCodeComments(content: string, filePath: string): CommentReference[] {
+  const references: CommentReference[] = [];
+  const extension = getFileExtension(filePath);
+  const commentStyle = getCommentStyle(extension);
+
+  if (!commentStyle) {
+    return references; // Unsupported file type
+  }
+
+  const lines = content.split('\n');
+  let inMultiLineComment = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue;
+
+    const lineNumber = i + 1;
+
+    // Check for multi-line comment start/end
+    if (commentStyle.multiLine) {
+      if (line.includes(commentStyle.multiLine.start)) {
+        inMultiLineComment = true;
+      }
+      if (inMultiLineComment) {
+        const urls = extractUrls(line);
+        for (const url of urls) {
+          references.push({
+            url,
+            context: line.trim(),
+            file: filePath,
+            line: lineNumber,
+            commentType: line.includes('/**') ? 'jsdoc' : 'multi-line'
+          });
+        }
+      }
+      if (line.includes(commentStyle.multiLine.end)) {
+        inMultiLineComment = false;
+      }
+      continue;
+    }
+
+    // Check for single-line comments
+    if (commentStyle.singleLine) {
+      const commentStart = line.indexOf(commentStyle.singleLine);
+      if (commentStart !== -1) {
+        const comment = line.substring(commentStart);
+        const urls = extractUrls(comment);
+        for (const url of urls) {
+          references.push({
+            url,
+            context: comment.trim(),
+            file: filePath,
+            line: lineNumber,
+            commentType: 'single-line'
+          });
+        }
+      }
+    }
+  }
+
+  return references;
+}
+
+function getFileExtension(filePath: string): string {
+  const match = filePath.match(/\.([^.]+)$/);
+  const ext = match?.[1];
+  return ext ? ext.toLowerCase() : '';
+}
+
+interface CommentStyle {
+  singleLine?: string;
+  multiLine?: { start: string; end: string };
+}
+
+function getCommentStyle(extension: string): CommentStyle | null {
+  const styles: Record<string, CommentStyle> = {
+    // JavaScript/TypeScript
+    js: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+    ts: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+    jsx: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+    tsx: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // Python
+    py: { singleLine: '#' },
+
+    // Ruby
+    rb: { singleLine: '#', multiLine: { start: '=begin', end: '=end' } },
+
+    // Go
+    go: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // Rust
+    rs: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // C/C++
+    c: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+    cpp: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+    h: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // Java/Kotlin
+    java: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+    kt: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // C#
+    cs: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // PHP
+    php: { singleLine: '//', multiLine: { start: '/*', end: '*/' } },
+
+    // Shell
+    sh: { singleLine: '#' },
+    bash: { singleLine: '#' },
+
+    // YAML
+    yml: { singleLine: '#' },
+    yaml: { singleLine: '#' }
+  };
+
+  return styles[extension] || null;
+}
+
+function extractUrls(text: string): string[] {
+  const urls: string[] = [];
+  const regex = /https?:\/\/[^\s<>()[\]'"]+/g;
+  let match;
+
+  while ((match = regex.exec(text)) !== null) {
+    urls.push(match[0]);
+  }
+
+  return urls;
+}
+
+/**
+ * Extract specification and RFC references from comments
+ */
+export function extractSpecReferences(content: string): Array<{ spec: string; context: string }> {
+  const references: Array<{ spec: string; context: string }> = [];
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    // Match RFC references
+    const rfcMatch = /RFC\s*(\d+)/i.exec(line);
+    if (rfcMatch) {
+      references.push({
+        spec: `RFC ${rfcMatch[1]}`,
+        context: line.trim()
+      });
+    }
+
+    // Match standard references (ISO, IEEE, etc.)
+    const standardMatch = /(ISO|IEEE|ECMA|W3C)[\s-]*(\d+(?:[-.]\d+)*)/i.exec(line);
+    if (standardMatch) {
+      references.push({
+        spec: `${standardMatch[1]} ${standardMatch[2]}`,
+        context: line.trim()
+      });
+    }
+  }
+
+  return references;
+}

package/src/parsers/package-files.ts

@@ -0,0 +1,156 @@
+/**
+ * Package File Parser
+ * Extracts metadata and references from package manager files
+ * EXCLUDES actual dependencies (handled by dependabot)
+ */
+
+export interface PackageMetadata {
+  repository?: string;
+  homepage?: string;
+  documentation?: string;
+  urls: string[]; // URLs found in descriptions, etc.
+}
+
+/**
+ * Parse package.json and extract metadata URLs (NOT dependencies)
+ */
+export function parsePackageJson(content: string): PackageMetadata {
+  try {
+    const pkg = JSON.parse(content);
+    const urls: string[] = [];
+
+    // Extract repository URL
+    let repository: string | undefined = undefined;
+    if (typeof pkg.repository === 'string') {
+      repository = pkg.repository;
+    } else if (pkg.repository && pkg.repository.url) {
+      repository = pkg.repository.url;
+    }
+
+    // Extract homepage
+    const homepage: string | undefined = pkg.homepage;
+
+    // Extract documentation (not standard but sometimes present)
+    const documentation: string | undefined = pkg.documentation || pkg.docs;
+
+    // Extract URLs from description
+    if (pkg.description) {
+      const descUrls = extractUrls(pkg.description);
+      urls.push(...descUrls);
+    }
+
+    // Note: We DO NOT extract dependencies/devDependencies
+    // Those are handled by dependabot
+
+    return {
+      ...(repository !== undefined && { repository }),
+      ...(homepage !== undefined && { homepage }),
+      ...(documentation !== undefined && { documentation }),
+      urls
+    };
+  } catch  {
+    return { urls: [] };
+  }
+}
+
+/**
+ * Parse requirements.txt and extract URLs from comments
+ * EXCLUDES actual packages (handled by dependabot)
+ */
+export function parseRequirementsTxt(content: string): PackageMetadata {
+  const urls: string[] = [];
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    // Only extract URLs from comments
+    if (line.trim().startsWith('#')) {
+      const commentUrls = extractUrls(line);
+      urls.push(...commentUrls);
+    }
+    // Skip actual package lines - dependabot handles those
+  }
+
+  return { urls };
+}
+
+/**
+ * Parse Cargo.toml and extract metadata URLs
+ * EXCLUDES actual dependencies (handled by dependabot)
+ */
+export function parseCargoToml(content: string): PackageMetadata {
+  const urls: string[] = [];
+  let repository: string | undefined = undefined;
+  let homepage: string | undefined = undefined;
+  let documentation: string | undefined = undefined;
+
+  const lines = content.split('\n');
+  let inPackageSection = false;
+
+  for (const line of lines) {
+    if (line.trim() === '[package]') {
+      inPackageSection = true;
+      continue;
+    }
+
+    if (line.trim().startsWith('[') && line.trim() !== '[package]') {
+      inPackageSection = false;
+      continue;
+    }
+
+    if (inPackageSection) {
+      const repoMatch = /repository\s*=\s*"([^"]+)"/.exec(line);
+      if (repoMatch && repoMatch[1]) repository = repoMatch[1];
+
+      const homepageMatch = /homepage\s*=\s*"([^"]+)"/.exec(line);
+      if (homepageMatch && homepageMatch[1]) homepage = homepageMatch[1];
+
+      const docMatch = /documentation\s*=\s*"([^"]+)"/.exec(line);
+      if (docMatch && docMatch[1]) documentation = docMatch[1];
+    }
+
+    // Extract URLs from comments
+    if (line.trim().startsWith('#')) {
+      const commentUrls = extractUrls(line);
+      urls.push(...commentUrls);
+    }
+  }
+
+  return {
+    ...(repository !== undefined && { repository }),
+    ...(homepage !== undefined && { homepage }),
+    ...(documentation !== undefined && { documentation }),
+    urls
+  };
+}
+
+/**
+ * Parse go.mod and extract URLs from comments
+ * EXCLUDES actual dependencies (handled by dependabot)
+ */
+export function parseGoMod(content: string): PackageMetadata {
+  const urls: string[] = [];
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    // Only extract URLs from comments
+    if (line.trim().startsWith('//')) {
+      const commentUrls = extractUrls(line);
+      urls.push(...commentUrls);
+    }
+    // Skip actual require lines - dependabot handles those
+  }
+
+  return { urls };
+}
+
+function extractUrls(text: string): string[] {
+  const urls: string[] = [];
+  const regex = /https?:\/\/[^\s<>()[\]'"]+/g;
+  let match;
+
+  while ((match = regex.exec(text)) !== null) {
+    urls.push(match[0]);
+  }
+
+  return urls;
+}

package/src/parsers/readme.ts

@@ -0,0 +1,185 @@
+/**
+ * README Parser
+ * Extracts URLs and references from README and markdown files
+ */
+
+export interface ExtractedReference {
+  url: string;
+  context: string; // Surrounding text
+  line?: number;
+  type: 'markdown-link' | 'bare-url' | 'reference-link';
+}
+
+// Patterns to skip (package managers, CI badges, shields.io)
+const SKIP_PATTERNS = [
+  /npmjs\.com\/package/,
+  /pypi\.org\/project/,
+  /crates\.io\/crates/,
+  /rubygems\.org\/gems/,
+  /packagist\.org\/packages/,
+  /shields\.io/,
+  /badge(s)?\..*\.svg/,
+  /travis-ci\.(org|com)/,
+  /circleci\.com/,
+  /github\.com\/.*\/actions/ // GitHub Actions badges
+];
+
+/**
+ * Parse README content and extract external references
+ */
+export function parseReadme(content: string, filePath = 'README.md'): ExtractedReference[] {
+  const references: ExtractedReference[] = [];
+  const lines = content.split('\n');
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line) continue; // Skip undefined or empty lines
+
+    const lineNumber = i + 1;
+
+    // Extract markdown links [text](url)
+    const markdownLinks = extractMarkdownLinks(line);
+    for (const { url, text } of markdownLinks) {
+      if (!shouldSkipUrl(url)) {
+        references.push({
+          url,
+          context: text || line.trim(),
+          line: lineNumber,
+          type: 'markdown-link'
+        });
+      }
+    }
+
+    // Extract reference-style links [text]: url
+    const referenceLinks = extractReferenceLinks(line);
+    for (const { url, text } of referenceLinks) {
+      if (!shouldSkipUrl(url)) {
+        references.push({
+          url,
+          context: text || line.trim(),
+          line: lineNumber,
+          type: 'reference-link'
+        });
+      }
+    }
+
+    // Extract bare URLs
+    const bareUrls = extractBareUrls(line);
+    for (const url of bareUrls) {
+      if (!shouldSkipUrl(url)) {
+        references.push({
+          url,
+          context: line.trim(),
+          line: lineNumber,
+          type: 'bare-url'
+        });
+      }
+    }
+  }
+
+  // Deduplicate by URL
+  return deduplicateReferences(references);
+}
+
+function extractMarkdownLinks(line: string): Array<{ url: string; text: string }> {
+  const links: Array<{ url: string; text: string }> = [];
+  const regex = /\[([^\]]+)\]\(([^)]+)\)/g;
+  let match;
+
+  while ((match = regex.exec(line)) !== null) {
+    const text = match[1];
+    const url = match[2];
+    if (text !== undefined && url !== undefined) {
+      links.push({ text, url });
+    }
+  }
+
+  return links;
+}
+
+function extractReferenceLinks(line: string): Array<{ url: string; text: string }> {
+  const links: Array<{ url: string; text: string }> = [];
+  const regex = /^\[([^\]]+)\]:\s+(.+)$/;
+  const match = regex.exec(line);
+
+  if (match) {
+    const text = match[1];
+    const url = match[2];
+    if (text !== undefined && url !== undefined) {
+      links.push({ text, url });
+    }
+  }
+
+  return links;
+}
+
+function extractBareUrls(line: string): string[] {
+  const urls: string[] = [];
+  const regex = /https?:\/\/[^\s<>()[\]]+/g;
+  let match;
+
+  while ((match = regex.exec(line)) !== null) {
+    urls.push(match[0]);
+  }
+
+  return urls;
+}
+
+function shouldSkipUrl(url: string): boolean {
+  return SKIP_PATTERNS.some((pattern) => pattern.test(url));
+}
+
+function deduplicateReferences(references: ExtractedReference[]): ExtractedReference[] {
+  const seen = new Set<string>();
+  return references.filter((ref) => {
+    if (seen.has(ref.url)) {
+      return false;
+    }
+    seen.add(ref.url);
+    return true;
+  });
+}
+
+/**
+ * Extract GitHub repository mentions (owner/repo format)
+ */
+export function extractGitHubReferences(
+  content: string
+): Array<{ owner: string; repo: string; context: string }> {
+  const references: Array<{ owner: string; repo: string; context: string }> = [];
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    // Match owner/repo pattern not in URLs, with basic length and context constraints
+    const regex =
+      /(?<!https?:\/\/github\.com\/)(?:^|[\s(])([a-zA-Z0-9_-]{2,}\/[a-zA-Z0-9_.-]{2,})(?=$|[\s),.;])/g;
+    let match;
+
+    while ((match = regex.exec(line)) !== null) {
+      const ownerRepo = match[1];
+      if (ownerRepo) {
+        const parts = ownerRepo.split('/');
+        const owner = parts[0];
+        const repo = parts[1];
+
+        if (
+          owner &&
+          repo &&
+          owner.length >= 2 &&
+          repo.length >= 2 &&
+          owner !== 'owner' &&
+          repo !== 'repo' &&
+          !(/^\d+$/.test(owner) && /^\d+$/.test(repo))
+        ) {
+          references.push({
+            owner,
+            repo,
+            context: line.trim()
+          });
+        }
+      }
+    }
+  }
+
+  return references;
+}

package/test/detector.test.ts

@@ -0,0 +1,102 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+describe('Detector', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('detectDependencies', () => {
+    it('should orchestrate all parsers', async () => {
+      // Expected: Call README parser, code comment parser, package file parser
+      expect(true).toBe(true);
+    });
+
+    it('should aggregate results from all parsers', async () => {
+      // Expected: Combine results from multiple parsers
+      expect(true).toBe(true);
+    });
+
+    it('should send aggregated content to LLM for analysis', async () => {
+      // Expected: Pass extracted content to LLM provider
+      expect(true).toBe(true);
+    });
+
+    it('should deduplicate dependencies by URL', async () => {
+      // Expected: Same URL from multiple sources = one dependency
+      expect(true).toBe(true);
+    });
+
+    it('should calculate confidence scores', async () => {
+      // Expected: LLM confidence * detection method weight
+      expect(true).toBe(true);
+    });
+
+    it('should generate UUIDs for each dependency', async () => {
+      expect(true).toBe(true);
+    });
+
+    it('should include detection metadata', async () => {
+      // Expected: detectionMethod, detectedAt, referencedIn
+      expect(true).toBe(true);
+    });
+
+    it('should handle empty repository', async () => {
+      // Expected: Return empty dependencies array
+      expect(true).toBe(true);
+    });
+
+    it('should handle LLM failures gracefully', async () => {
+      // Expected: Continue with parser results, log error
+      expect(true).toBe(true);
+    });
+  });
+
+  describe('classifyDependency', () => {
+    it('should classify GitHub URLs as repository', () => {
+      const url = 'https://github.com/owner/repo';
+      expect(url).toContain('github.com');
+    });
+
+    it('should classify arXiv URLs as research-paper', () => {
+      const url = 'https://arxiv.org/abs/1706.03762';
+      expect(url).toContain('arxiv.org');
+    });
+
+    it('should classify OpenAPI specs as schema', () => {
+      const url = 'https://api.example.com/openapi.yaml';
+      expect(url).toContain('openapi');
+    });
+
+    it('should classify documentation sites as documentation', () => {
+      const url = 'https://docs.example.com/guide';
+      expect(url).toContain('docs.');
+    });
+
+    it('should use LLM for ambiguous URLs', () => {
+      const url = 'https://example.com/some-resource';
+      expect(typeof url).toBe('string');
+    });
+  });
+
+  describe('determineAccessMethod', () => {
+    it('should use github-api for GitHub URLs', () => {
+      const url = 'https://github.com/owner/repo';
+      expect(url).toContain('github.com');
+    });
+
+    it('should use arxiv for arXiv URLs', () => {
+      const url = 'https://arxiv.org/abs/1234.5678';
+      expect(url).toContain('arxiv.org');
+    });
+
+    it('should use openapi for OpenAPI spec URLs', () => {
+      const url = 'https://api.example.com/openapi.json';
+      expect(url).toContain('openapi');
+    });
+
+    it('should use http as fallback', () => {
+      const url = 'https://example.com/docs';
+      expect(url).toContain('http');
+    });
+  });
+});