@dependabit/detector 0.1.2 → 0.1.15

package/package.json

@@ -1,6 +1,11 @@
 {
   "name": "@dependabit/detector",
-  "version": "0.1.2",
+  "version": "0.1.15",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/pradeepmouli/dependabit",
+    "directory": "packages/detector"
+  },
   "description": "LLM-based dependency detection for external resources",
   "type": "module",
   "main": "dist/index.js",
@@ -15,8 +20,8 @@
     "@actions/core": "^3.0.0",
     "ignore": "^7.0.5",
     "zod": "^4.3.6",
-    "@dependabit/manifest": "0.1.2",
-    "@dependabit/github-client": "0.1.2"
+    "@dependabit/github-client": "0.1.13",
+    "@dependabit/manifest": "0.1.13"
   },
   "devDependencies": {
     "@types/node": "^25.2.3",
@@ -33,7 +38,7 @@
   "license": "MIT",
   "scripts": {
     "build": "tsgo -p tsconfig.json",
-    "clean": "rm -rf dist",
+    "clean": "rm -rf dist tsconfig.tsbuildinfo",
     "dev": "tsx watch src/index.ts",
     "type-check": "tsgo --noEmit -p tsconfig.json",
     "test": "vitest run",

package/src/detector.ts

@@ -6,7 +6,7 @@
 import { readdir, readFile } from 'node:fs/promises';
 import { execSync } from 'node:child_process';
 import { homedir } from 'node:os';
-import { dirname, join, relative, resolve, normalize, sep } from 'node:path';
+import { basename, dirname, join, relative, resolve, normalize, sep } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import ignore, { type Ignore } from 'ignore';
 import type { LLMProvider } from './llm/client.js';
@@ -31,6 +31,8 @@ export interface DetectorOptions {
   llmProvider: LLMProvider;
   ignorePatterns?: string[];
   useGitExcludes?: boolean;
+  repoOwner?: string;
+  repoName?: string;
 }
 
 export interface DetectionResult {
@@ -46,22 +48,17 @@ export interface DetectionResult {
 
 const DEFAULT_IGNORE_PATTERNS = [
   'node_modules',
-  '.git',
-  '.github',
-  '.claude',
-  '.codex',
   'dist',
   'build',
   'target',
   'vendor',
-  '.venv',
   'venv',
   '__pycache__',
-  'coverage',
-  '.next',
-  '.nuxt'
+  'coverage'
 ];
 
+const ALLOWED_DOT_DIRECTORIES = new Set<string>();
+
 /**
  * Main detector class
  */
@@ -69,13 +66,38 @@ export class Detector {
   private options: Required<DetectorOptions>;
   private ignoreMatcher: Ignore | null = null;
   private ignoreMatcherLoaded = false;
+  private skipUrlPatterns: RegExp[];
 
   constructor(options: DetectorOptions) {
     this.options = {
       ...options,
       ignorePatterns: options.ignorePatterns || DEFAULT_IGNORE_PATTERNS,
-      useGitExcludes: options.useGitExcludes ?? true
+      useGitExcludes: options.useGitExcludes ?? true,
+      repoOwner: options.repoOwner || '',
+      repoName: options.repoName || ''
     };
+
+    this.skipUrlPatterns = [
+      /example\.com/,
+      /example\.org/,
+      /localhost/,
+      /127\.0\.0\.1/,
+      /\[.*\]/, // template placeholders like [NUMBER]
+      /github\.com\/user\/repo/, // common placeholder in docs
+      /your-?username/i, // template placeholder: your-username or yourusername
+      /you\/your-project/i, // template: you/your-project
+      /github\.com\/YOUR-?USERNAME/i, // GitHub template: YOUR-USERNAME
+      /github\.com\/your-?username/i, // GitHub template: your-username
+      // Self-reference: skip if URL matches current repository
+      ...(this.options.repoOwner && this.options.repoName
+        ? [
+            new RegExp(
+              `github\\.com\\/${this.options.repoOwner}\\/${this.options.repoName}(?:\\.git)?(?:[/?#]|$)`,
+              'i'
+            )
+          ]
+        : [])
+    ];
   }
 
   /**
@@ -128,7 +150,33 @@ export class Detector {
       filesScanned++;
     }
 
-    // 1b. Parse package files for metadata (NOT dependencies)
+    // 1b. Parse non-README documentation files
+    const documentationFiles = await this.findFiles(this.options.repoPath, /\.(md|txt|rst|adoc)$/i);
+    for (const file of documentationFiles) {
+      if (/^README/i.test(basename(file))) {
+        continue;
+      }
+
+      const content = await readFile(file, 'utf-8');
+      const references = parseReadme(content, relative(this.options.repoPath, file));
+
+      for (const ref of references) {
+        this.addReference(
+          allReferences,
+          ref.url,
+          {
+            file: relative(this.options.repoPath, file),
+            ...(ref.line !== undefined && { line: ref.line }),
+            text: ref.context
+          },
+          'llm-analysis'
+        );
+      }
+
+      filesScanned++;
+    }
+
+    // 1c. Parse package files for metadata (NOT dependencies)
     const packageFiles = await this.findPackageFiles(this.options.repoPath);
     for (const file of packageFiles) {
       const content = await readFile(file, 'utf-8');
@@ -154,7 +202,7 @@ export class Detector {
       filesScanned++;
     }
 
-    // 1c. Parse code comments from source files
+    // 1d. Parse code comments from source files
     const sourceFiles = await this.findSourceFiles(this.options.repoPath);
     for (const file of sourceFiles.slice(0, 50)) {
       // Limit to 50 files for performance
@@ -232,6 +280,8 @@ Return as JSON with "dependencies" array.`;
     const now = new Date().toISOString();
 
     for (const [url, data] of allReferences) {
+      // Skip entries that will end up with low confidence
+      // (entries that can't be typed programmatically and LLM assigns low confidence)
       // Prepare context for potential LLM use
       const context = data.contexts
         .map((c) => `${c.file}${c.line ? `:${c.line}` : ''}: ${c.text}`)
@@ -269,6 +319,11 @@ Return as JSON with "dependencies" array.`;
         typeConfidence = 0.3;
       }
 
+      // Skip low-confidence entries
+      if (typeConfidence < 0.5) {
+        continue;
+      }
+
       // Step 5: Try programmatic access method determination
       let accessMethod: AccessMethod | null = this.determineAccessMethod(url);
 
@@ -364,6 +419,27 @@ Return as JSON: {"accessMethod": "...", "confidence": 0.0-1.0}`;
     context: { file: string; line?: number; text: string },
     detectionMethod: DetectionMethod
   ): void {
+    // Skip URLs that don't start with http:// or https://
+    if (!/^https?:\/\//.test(url)) {
+      return;
+    }
+
+    // Skip URLs matching skip patterns (placeholders, localhost, etc.)
+    if (this.skipUrlPatterns.some((pattern) => pattern.test(url))) {
+      return;
+    }
+
+    // Skip self-references (URLs pointing to the repo itself)
+    if (this.options.repoOwner && this.options.repoName) {
+      const selfPattern = new RegExp(
+        `github\\.com[/:]${this.escapeRegExp(this.options.repoOwner)}/${this.escapeRegExp(this.options.repoName)}(?:/|$|#|\\?)`,
+        'i'
+      );
+      if (selfPattern.test(url)) {
+        return;
+      }
+    }
+
     if (!map.has(url)) {
       map.set(url, {
         url,
@@ -374,6 +450,10 @@ Return as JSON: {"accessMethod": "...", "confidence": 0.0-1.0}`;
     map.get(url)!.contexts.push(context);
   }
 
+  private escapeRegExp(str: string): string {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+
   /**
    * Programmatically determine access method based on URL patterns
    * Returns null if cannot be determined programmatically
@@ -412,6 +492,10 @@ Return as JSON: {"accessMethod": "...", "confidence": 0.0-1.0}`;
     const lowerUrl = url.toLowerCase();
     const lowerContext = context.toLowerCase();
 
+    const isGitHubRepositoryUrl = /^https?:\/\/github\.com\/[^/]+\/[^/#?]+(?:$|[/?#])/.test(
+      lowerUrl
+    );
+
     // Research papers
     if (
       lowerUrl.includes('arxiv.org') ||
@@ -447,10 +531,11 @@ Return as JSON: {"accessMethod": "...", "confidence": 0.0-1.0}`;
 
     // Reference implementations (GitHub repos)
     if (
-      lowerUrl.includes('github.com') &&
-      (lowerContext.includes('example') ||
-        lowerContext.includes('implementation') ||
-        lowerContext.includes('reference'))
+      isGitHubRepositoryUrl ||
+      (lowerUrl.includes('github.com') &&
+        (lowerContext.includes('example') ||
+          lowerContext.includes('implementation') ||
+          lowerContext.includes('reference')))
     ) {
       return 'reference-implementation';
     }
@@ -526,7 +611,7 @@ Return as JSON: {"accessMethod": "...", "confidence": 0.0-1.0}`;
           files.push(fullPath);
         }
       }
-    } catch  {
+    } catch {
       // Ignore errors (permission denied, etc.)
     }
 
@@ -560,11 +645,14 @@ Return as JSON: {"accessMethod": "...", "confidence": 0.0-1.0}`;
 
   private isDotDirectoryPath(filePath: string): boolean {
     const segments = normalize(filePath).split(sep);
-    return segments.some((segment) => segment.startsWith('.') && segment.length > 1);
+    return segments.some(
+      (segment) =>
+        segment.startsWith('.') && segment.length > 1 && !ALLOWED_DOT_DIRECTORIES.has(segment)
+    );
   }
 
   private isDotDirectoryName(name: string): boolean {
-    return name.startsWith('.') && name.length > 1;
+    return name.startsWith('.') && name.length > 1 && !ALLOWED_DOT_DIRECTORIES.has(name);
   }
 
   private async isGitIgnored(filePath: string, isDirectory: boolean): Promise<boolean> {

package/src/llm/copilot.ts

@@ -3,7 +3,7 @@
  * Integrates with GitHub Copilot via CLI commands
  */
 
-import { exec } from 'node:child_process';
+import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
 import type {
   LLMProvider,
@@ -15,7 +15,7 @@ import type {
 } from './client.js';
 import { SYSTEM_PROMPT } from './prompts.js';
 
-const execAsync = promisify(exec);
+const execFileAsync = promisify(execFile);
 
 export class GitHubCopilotProvider implements LLMProvider {
   private config: Required<LLMProviderConfig>;
@@ -44,14 +44,17 @@ export class GitHubCopilotProvider implements LLMProvider {
       // Combine system prompt and user prompt for CLI
       const fullPrompt = `${SYSTEM_PROMPT}\n\n${prompt}`;
 
-      // Escape the prompt for shell safety (basic escaping)
-      const escapedPrompt = fullPrompt.replace(/"/g, '\\"').replace(/\$/g, '\\$');
-
-      // Use gh copilot suggest command to get AI response
-      // The --yes flag auto-accepts the suggestion, --shell-out returns raw output
-      const command = `echo "${escapedPrompt}" | gh copilot suggest --yes 2>&1`;
-
-      const { stdout, stderr } = await execAsync(command, {
+      // Use execFile to avoid shell escaping issues and command injection
+      // Pass prompt directly as an argument without manual escaping
+      const args = [
+        'copilot',
+        '-p',
+        fullPrompt,
+        '--silent',
+        '--allow-all-tools'
+      ];
+
+      const { stdout, stderr } = await execFileAsync('gh', args, {
         maxBuffer: 10 * 1024 * 1024, // 10MB buffer for large responses
         timeout: 60000 // 60 second timeout
       });

package/src/llm/prompts.ts

@@ -17,11 +17,20 @@ INCLUDE these types of dependencies:
 
 EXCLUDE these (handled by dependabot):
 - NPM packages in package.json
-- Python packages in requirements.txt  
+- Python packages in requirements.txt
 - Rust crates in Cargo.toml
 - Docker images in Dockerfile
 - Any declared package manager dependencies
 
+ALSO EXCLUDE (false positives):
+- URLs that reference the repository itself (self-references)
+- Relative file paths (e.g., CONTRIBUTING.md, docs/guide.md, ./src/utils)
+- Placeholder URLs used in documentation examples (example.com, example.org, localhost)
+- Internal documentation links within the same repository
+- URLs with template variables or placeholders (e.g., issues/[NUMBER], user/repo)
+
+Only return dependencies with confidence >= 0.7.
+
 For each dependency found, provide:
 1. url: The complete URL
 2. name: A descriptive name

package/src/parsers/readme.ts

@@ -10,7 +10,7 @@ export interface ExtractedReference {
   type: 'markdown-link' | 'bare-url' | 'reference-link';
 }
 
-// Patterns to skip (package managers, CI badges, shields.io)
+// Patterns to skip (package managers, CI badges, shields.io, placeholders)
 const SKIP_PATTERNS = [
   /npmjs\.com\/package/,
   /pypi\.org\/project/,
@@ -21,7 +21,13 @@ const SKIP_PATTERNS = [
   /badge(s)?\..*\.svg/,
   /travis-ci\.(org|com)/,
   /circleci\.com/,
-  /github\.com\/.*\/actions/ // GitHub Actions badges
+  /github\.com\/.*\/actions/, // GitHub Actions badges
+  /example\.com/, // placeholder domain
+  /example\.org/, // placeholder domain
+  /localhost/, // local dev URLs
+  /127\.0\.0\.1/, // loopback address
+  /\[.*\]/, // template placeholders (e.g., issues/[NUMBER])
+  /github\.com\/user\/repo/ // common placeholder in docs
 ];
 
 /**

package/test/detector.test.ts

@@ -1,4 +1,20 @@
 import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Detector } from '../src/detector.js';
+
+const createDetector = (): Detector =>
+  new Detector({
+    repoPath: '.',
+    llmProvider: {
+      analyze: vi.fn(async () => ({
+        dependencies: [],
+        usage: { totalTokens: 0, latencyMs: 0 },
+        rawResponse: '{}'
+      }))
+    }
+  });
 
 describe('Detector', () => {
   beforeEach(() => {
@@ -49,6 +65,47 @@ describe('Detector', () => {
       // Expected: Continue with parser results, log error
       expect(true).toBe(true);
     });
+
+    it('should keep GitHub repo links when LLM parser misses them', async () => {
+      const repoPath = await mkdtemp(join(tmpdir(), 'dependabit-detector-'));
+
+      try {
+        await mkdir(join(repoPath, 'docs'), { recursive: true });
+        await writeFile(
+          join(repoPath, 'docs', 'tooling.md'),
+          '- [spec-kit](https://github.com/github/spec-kit)\n'
+        );
+
+        const detector = new Detector({
+          repoPath,
+          llmProvider: {
+            analyze: vi.fn(async () => ({
+              dependencies: [],
+              usage: { totalTokens: 0, latencyMs: 0 },
+              rawResponse: '{}'
+            })),
+            getSupportedModels: vi.fn(() => ['test-model']),
+            getRateLimit: vi.fn(async () => ({
+              remaining: 100,
+              limit: 100,
+              resetAt: new Date(Date.now() + 60_000)
+            })),
+            validateConfig: vi.fn(() => true)
+          },
+          repoOwner: 'pradeepmouli',
+          repoName: 'dependabit'
+        });
+
+        const result = await detector.detectDependencies();
+        const dep = result.dependencies.find((d) => d.url === 'https://github.com/github/spec-kit');
+
+        expect(dep).toBeDefined();
+        expect(dep?.type).toBe('reference-implementation');
+        expect(dep?.accessMethod).toBe('github-api');
+      } finally {
+        await rm(repoPath, { recursive: true, force: true });
+      }
+    });
   });
 
   describe('classifyDependency', () => {
@@ -57,6 +114,16 @@ describe('Detector', () => {
       expect(url).toContain('github.com');
     });
 
+    it('should classify GitHub repository links as reference-implementation', () => {
+      const detector = createDetector();
+      const type = (detector as any).determineDependencyType(
+        'https://github.com/github/spec-kit',
+        'spec-kit'
+      );
+
+      expect(type).toBe('reference-implementation');
+    });
+
     it('should classify arXiv URLs as research-paper', () => {
       const url = 'https://arxiv.org/abs/1706.03762';
       expect(url).toContain('arxiv.org');