@goshenkata/dryscan-core 1.0.0

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@goshenkata/dryscan-core",
+  "version": "1.0.0",
+  "description": "Core library for DryScan - semantic code duplication analyzer",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --sourcemap --clean --outDir dist",
+    "clean": "rm -rf dist",
+    "test": "tsx ../node_modules/mocha/bin/mocha \"test/**/*.test.mjs\"",
+    "coverage": "c8 tsx ../node_modules/mocha/bin/mocha \"test/**/*.test.mjs\""
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "keywords": [
+    "code-analysis",
+    "duplication-detection",
+    "semantic-analysis"
+  ],
+  "author": "Goshenkata",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/debug": "^4.1.12",
+    "@types/node": "^25.0.3",
+    "chai": "^6.2.2",
+    "mocha": "^11.7.5",
+    "sinon": "^21.0.1",
+    "tsup": "^8.5.1",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  },
+  "dependencies": {
+    "@langchain/core": "^1.1.8",
+    "@langchain/google-genai": "^2.1.3",
+    "@langchain/ollama": "^1.1.0",
+    "better-sqlite3": "^12.5.0",
+    "debug": "^4.4.3",
+    "glob-gitignore": "^1.0.15",
+    "ignore": "^7.0.5",
+    "jsonschema": "^1.5.0",
+    "langchain": "^1.2.3",
+    "minimatch": "^10.1.1",
+    "reflect-metadata": "^0.2.2",
+    "short-uuid": "^6.0.3",
+    "tree-sitter": "^0.25.0",
+    "tree-sitter-java": "^0.23.5",
+    "tree-sitter-python": "^0.25.0",
+    "typeorm": "^0.3.28",
+    "upath": "^2.0.1"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/DryScan.ts

@@ -0,0 +1,169 @@
+import upath from "upath";
+import fs from "fs/promises";
+import { DuplicateAnalysisResult, DuplicateReport } from "./types";
+import { DRYSCAN_DIR, INDEX_DB } from "./const";
+import { defaultExtractors, IndexUnitExtractor } from "./IndexUnitExtractor";
+import { DryScanDatabase } from "./db/DryScanDatabase";
+import { RepositoryInitializer, InitOptions as InitServiceOptions } from "./services/RepositoryInitializer";
+import { UpdateService } from "./services/UpdateService";
+import { DuplicateService } from "./services/DuplicateService";
+import { ExclusionService } from "./services/ExclusionService";
+import { DryScanServiceDeps } from "./services/types";
+import { configStore } from "./config/configStore";
+import { DryConfig } from "./types";
+import { PairingService } from "./services/PairingService";
+
+export type InitOptions = InitServiceOptions;
+
+
+export class DryScan {
+  repoPath: string;
+  private readonly extractor: IndexUnitExtractor;
+  private db: DryScanDatabase;
+  private readonly services: {
+    initializer: RepositoryInitializer;
+    updater: UpdateService;
+    duplicate: DuplicateService;
+    exclusion: ExclusionService;
+  };
+  private readonly serviceDeps: DryScanServiceDeps;
+
+  constructor(
+    repoPath: string,
+    extractor?: IndexUnitExtractor,
+    db?: DryScanDatabase
+  ) {
+    this.repoPath = repoPath;
+    this.extractor = extractor ?? new IndexUnitExtractor(repoPath, defaultExtractors(repoPath));
+    this.db = db ?? new DryScanDatabase();
+
+    this.serviceDeps = {
+      repoPath: this.repoPath,
+      db: this.db,
+      extractor: this.extractor,
+      pairing: new PairingService(this.extractor),
+    };
+
+    const exclusion = new ExclusionService(this.serviceDeps);
+    this.services = {
+      initializer: new RepositoryInitializer(this.serviceDeps, exclusion),
+      updater: new UpdateService(this.serviceDeps, exclusion),
+      duplicate: new DuplicateService(this.serviceDeps),
+      exclusion,
+    };
+  }
+
+  /**
+   * Initializes the DryScan repository with a 3-phase analysis:
+   * Phase 1: Extract and save all functions
+   * Phase 2: Resolve and save internal dependencies
+   * Phase 3: Compute and save semantic embeddings
+   */
+  async init(options?: InitOptions): Promise<void> {
+    console.log(`[DryScan] Initializing repository at ${this.repoPath}`);
+    console.log("[DryScan] Preparing database and cache...");
+    await configStore.init(this.repoPath);
+    await this.ensureDatabase();
+    if (await this.isInitialized()) {
+      console.log("[DryScan] Repository already initialized; skipping full init.");
+      return;
+    }
+    console.log("[DryScan] Starting initial scan (may take a moment)...");
+    await this.services.initializer.init(options);
+    console.log("[DryScan] Initial scan complete.");
+  }
+
+  /**
+   * Updates the index by detecting changed, new, and deleted files.
+   * Only reprocesses units in changed files for efficiency.
+   * Delegates to DryScanUpdater module for implementation.
+   * 
+   * Update process:
+   * 1. List all current source files in repository
+   * 2. For each file, check if it's new, changed, or unchanged (via mtime + checksum)
+   * 3. Remove old units from changed/deleted files
+   * 4. Extract and save units from new/changed files
+   * 5. Recompute internal dependencies for affected units
+   * 6. Recompute embeddings for affected units
+   * 7. Update file tracking metadata
+   */
+  async updateIndex(): Promise<void> {
+    console.log(`[DryScan] Updating index at ${this.repoPath}...`);
+    console.log("[DryScan] Checking for file changes...");
+    const start = Date.now();
+    await this.ensureDatabase();
+    await this.services.updater.updateIndex();
+    const duration = Date.now() - start;
+    console.log(`[DryScan] Index update complete. Took ${duration}ms.`);
+  }
+
+
+  /**
+   * Runs duplicate detection and returns a normalized report payload ready for persistence or display.
+   */
+  async buildDuplicateReport(): Promise<DuplicateReport> {
+    const config = await this.loadConfig();
+    const analysis = await this.findDuplicates(config);
+    return {
+      version: 1,
+      generatedAt: new Date().toISOString(),
+      threshold: config.threshold,
+      score: analysis.score,
+      duplicates: analysis.duplicates,
+    };
+  }
+
+  /**
+   * Finds duplicate code blocks using cosine similarity on embeddings.
+   * Automatically updates the index before searching to ensure results are current.
+   * Compares all function pairs and returns groups with similarity above the configured threshold.
+   *
+   * @returns Analysis result with duplicate groups and duplication score
+   */
+  private async findDuplicates(config: DryConfig): Promise<DuplicateAnalysisResult> {
+    console.log(`[DryScan] Finding duplicates (threshold: ${config.threshold})...`);
+    await this.ensureDatabase();
+
+    console.log("[DryScan] Updating index...");
+    const updateStart = Date.now();
+    await this.updateIndex();
+    const updateDuration = Date.now() - updateStart;
+    console.log(`[DryScan] Index update  took ${updateDuration}ms.`);
+
+    console.log("[DryScan] Detecting duplicates...");
+    const dupStart = Date.now();
+    const result = await this.services.duplicate.findDuplicates(config);
+    const dupDuration = Date.now() - dupStart;
+    console.log(`[DryScan] Duplicate detection took ${dupDuration}ms.`);
+
+    return result;
+  }
+
+  /**
+   * Cleans excludedPairs entries that no longer match any indexed units.
+   * Runs an update first to ensure the index reflects current code.
+   */
+  async cleanExclusions(): Promise<{ removed: number; kept: number }> {
+    await this.updateIndex();
+    return this.services.exclusion.cleanExclusions();
+  }
+
+  private async ensureDatabase(): Promise<void> {
+    if (this.db.isInitialized()) return;
+    const dbPath = upath.join(this.repoPath, DRYSCAN_DIR, INDEX_DB);
+    await fs.mkdir(upath.dirname(dbPath), { recursive: true });
+    await this.db.init(dbPath);
+  }
+
+  private async loadConfig(): Promise<DryConfig> {
+    return configStore.get(this.repoPath);
+  }
+
+  private async isInitialized(): Promise<boolean> {
+    if (!this.db.isInitialized()) return false;
+    const unitCount = await this.db.countUnits();
+    const initialized = unitCount > 0;
+    console.log(`[DryScan] Initialization check: ${unitCount} indexed units`);
+    return initialized;
+  }
+}

package/src/DryScanUpdater.ts

@@ -0,0 +1,236 @@
+import path from "path";
+import fs from "fs/promises";
+import debug from "debug";
+import { IndexUnit } from "./types";
+import { IndexUnitExtractor } from "./IndexUnitExtractor";
+import { DryScanDatabase } from "./db/DryScanDatabase";
+import { FileEntity } from "./db/entities/FileEntity";
+import { EmbeddingService } from "./services/EmbeddingService";
+
+const log = debug("DryScan:Updater");
+
+/**
+ * DryScan Updater Module
+ * 
+ * This module contains all incremental update logic for DryScan.
+ * Separated from DryScan.ts to keep that file focused on core operations.
+ * 
+ * Represents the result of change detection.
+ * Categorizes files into added, changed, deleted, and unchanged.
+ */
+export interface FileChangeSet {
+  added: string[];
+  changed: string[];
+  deleted: string[];
+  unchanged: string[];
+}
+
+/**
+ * Detects which files have been added, changed, or deleted since last scan.
+ * Uses mtime as fast check, then checksum for verification.
+ * 
+ * @param repoPath - Root path of the repository
+ * @param extractor - Index unit extractor instance for file operations
+ * @param db - Database instance for retrieving tracked files
+ * @returns Change set with categorized file paths
+ */
+export async function detectFileChanges(
+  repoPath: string,
+  extractor: IndexUnitExtractor,
+  db: DryScanDatabase
+): Promise<FileChangeSet> {
+  // Get current files in repository
+  const currentFiles = await extractor.listSourceFiles(repoPath);
+  const currentFileSet = new Set(currentFiles);
+
+  // Get tracked files from database
+  const trackedFiles = await db.getAllFiles();
+  const trackedFileMap = new Map(trackedFiles.map(f => [f.filePath, f]));
+
+  const added: string[] = [];
+  const changed: string[] = [];
+  const unchanged: string[] = [];
+
+  // Check each current file
+  for (const filePath of currentFiles) {
+    const tracked = trackedFileMap.get(filePath);
+    
+    if (!tracked) {
+      // New file
+      added.push(filePath);
+      continue;
+    }
+
+    // Check if file changed using mtime first (fast check)
+    const fullPath = path.join(repoPath, filePath);
+    const stat = await fs.stat(fullPath);
+    
+    if (stat.mtimeMs !== tracked.mtime) {
+      // Mtime changed, verify with checksum
+      const currentChecksum = await extractor.computeChecksum(fullPath);
+      if (currentChecksum !== tracked.checksum) {
+        changed.push(filePath);
+      } else {
+        // Mtime changed but content same
+        unchanged.push(filePath);
+      }
+    } else {
+      unchanged.push(filePath);
+    }
+  }
+
+  // Find deleted files
+  const deleted = trackedFiles
+    .map(f => f.filePath)
+    .filter(fp => !currentFileSet.has(fp));
+
+  return { added, changed, deleted, unchanged };
+}
+
+/**
+ * Extracts index units from a list of files.
+ * Used during incremental updates.
+ * 
+ * @param filePaths - Array of relative file paths to extract from
+ * @param extractor - Index unit extractor instance
+ * @returns Array of extracted units
+ */
+export async function extractUnitsFromFiles(
+  filePaths: string[],
+  extractor: IndexUnitExtractor
+): Promise<IndexUnit[]> {
+  const allUnits: IndexUnit[] = [];
+  
+  for (const relPath of filePaths) {
+    const functions = await extractor.scan(relPath);
+    allUnits.push(...functions);
+  }
+  
+  return allUnits;
+}
+
+/**
+ * Updates file tracking metadata after processing changes.
+ * Removes deleted files, updates changed files, adds new files.
+ * 
+ * @param changeSet - Set of file changes to apply
+ * @param repoPath - Root path of the repository
+ * @param extractor - Index unit extractor for checksum computation
+ * @param db - Database instance for file tracking
+ */
+export async function updateFileTracking(
+  changeSet: FileChangeSet,
+  repoPath: string,
+  extractor: IndexUnitExtractor,
+  db: DryScanDatabase
+): Promise<void> {
+  // Remove deleted files
+  if (changeSet.deleted.length > 0) {
+    if (typeof (db as any).removeFilesByFilePaths === "function") {
+      await (db as any).removeFilesByFilePaths(changeSet.deleted);
+    } else if (typeof (db as any).removeFiles === "function") {
+      await (db as any).removeFiles(changeSet.deleted);
+    }
+  }
+
+  // Create file entities for new and changed files
+  const filesToTrack = [...changeSet.added, ...changeSet.changed];
+  if (filesToTrack.length > 0) {
+    const fileEntities: FileEntity[] = [];
+    
+    for (const relPath of filesToTrack) {
+      const fullPath = path.join(repoPath, relPath);
+      const stat = await fs.stat(fullPath);
+      const checksum = await extractor.computeChecksum(fullPath);
+      
+      const fileEntity = new FileEntity();
+      fileEntity.filePath = relPath;
+      fileEntity.checksum = checksum;
+      fileEntity.mtime = stat.mtimeMs;
+      
+      fileEntities.push(fileEntity);
+    }
+    
+    await db.saveFiles(fileEntities);
+  }
+}
+
+/**
+ * Performs incremental update of the DryScan index.
+ * Detects file changes and reprocesses only affected files.
+ * 
+ * @param repoPath - Root path of the repository
+ * @param extractor - Index unit extractor instance
+ * @param db - Database instance (must be initialized)
+ */
+export async function performIncrementalUpdate(
+  repoPath: string,
+  extractor: IndexUnitExtractor,
+  db: DryScanDatabase,
+): Promise<FileChangeSet> {
+  log("Starting incremental update");
+  const embeddingService = new EmbeddingService(repoPath);
+  
+  // Step 1: Detect changes
+  const changeSet = await detectFileChanges(repoPath, extractor, db);
+  
+  if (changeSet.changed.length === 0 && 
+      changeSet.added.length === 0 && 
+      changeSet.deleted.length === 0) {
+    log("No changes detected. Index is up to date.");
+    return changeSet;
+  }
+
+  log(`Changes detected: ${changeSet.added.length} added, ${changeSet.changed.length} changed, ${changeSet.deleted.length} deleted`);
+
+  // Step 2: Remove old data for changed/deleted files
+  const filesToRemove = [...changeSet.changed, ...changeSet.deleted];
+  if (filesToRemove.length > 0) {
+      await db.removeUnitsByFilePaths(filesToRemove);
+      log(`Removed units from ${filesToRemove.length} files`);
+  }
+
+  // Step 3: Extract functions from new/changed files
+  const filesToProcess = [...changeSet.added, ...changeSet.changed];
+  if (filesToProcess.length > 0) {
+    const newUnits = await extractUnitsFromFiles(filesToProcess, extractor);
+      await db.saveUnits(newUnits);
+      log(`Extracted and saved ${newUnits.length} units from ${filesToProcess.length} files`);
+
+    // Step 4: Recompute embeddings for affected units only
+    const total = newUnits.length;
+    if (total > 0) {
+      log(`Recomputing embeddings for ${total} units`);
+      const progressInterval = Math.max(1, Math.ceil(total / 10));
+      const updatedWithEmbeddings = [] as IndexUnit[];
+
+      for (let i = 0; i < total; i++) {
+        const unit = newUnits[i];
+        try {
+          const enriched = await embeddingService.addEmbedding(unit);
+          updatedWithEmbeddings.push(enriched);
+        } catch (err: any) {
+          console.error(
+            `[DryScan] embedding failed for ${unit.filePath} (${unit.name}): ${err?.message || err}`
+          );
+          throw err;
+        }
+
+        const completed = i + 1;
+        if (completed === total || completed % progressInterval === 0) {
+          const pct = Math.floor((completed / total) * 100);
+          console.log(`[DryScan] Incremental embeddings ${completed}/${total} (${pct}%)`);
+        }
+      }
+
+      await db.updateUnits(updatedWithEmbeddings);
+      log(`Recomputed embeddings for ${updatedWithEmbeddings.length} units`);
+    }
+  }
+
+  // Step 5: Update file tracking
+  await updateFileTracking(changeSet, repoPath, extractor, db);
+  log("Incremental update complete");
+
+  return changeSet;
+}

package/src/Gitignore.ts

@@ -0,0 +1,71 @@
+import path from "path";
+import fs from "fs/promises";
+import upath from "upath";
+import { glob } from "glob-gitignore";
+import ignore, { Ignore } from "ignore";
+import { DryConfig } from "./types";
+
+/**
+ * Gitignore helper that builds ignore matchers by combining default rules,
+ * repo .gitignore files, and config-driven exclusions.
+ */
+export class Gitignore {
+  private readonly defaultIgnores = [".git/**", ".dry/**"];
+
+  constructor(private readonly root: string) {}
+
+  async buildMatcher(config: DryConfig): Promise<Ignore> {
+    const rules = await this.resolveRules(config);
+    return ignore({ allowRelativePaths: true }).add(rules);
+  }
+
+  private async resolveRules(config: DryConfig): Promise<string[]> {
+    const gitignoreRules = await this.loadGitignoreRules();
+    const configRules = config.excludedPaths || [];
+    return [...this.defaultIgnores, ...gitignoreRules, ...configRules];
+  }
+
+  private async loadGitignoreRules(): Promise<string[]> {
+    const gitignoreFiles = await glob("**/.gitignore", {
+      cwd: this.root,
+      dot: true,
+      nodir: true,
+      ignore: this.defaultIgnores,
+    });
+
+    const rules: string[] = [];
+
+    for (const file of gitignoreFiles) {
+      const absPath = path.join(this.root, file);
+      const dir = upath.normalizeTrim(upath.dirname(file));
+      const content = await fs.readFile(absPath, "utf8").catch(() => "");
+      const lines = content.split(/\r?\n/);
+
+      for (const raw of lines) {
+        const trimmed = raw.trim();
+        if (!trimmed || trimmed.startsWith("#")) continue;
+
+        const negated = trimmed.startsWith("!");
+        const body = negated ? trimmed.slice(1) : trimmed;
+
+        const scoped = this.scopeRule(body, dir);
+        if (!scoped) continue;
+
+        rules.push(negated ? `!${scoped}` : scoped);
+      }
+    }
+
+    return rules;
+  }
+
+  private scopeRule(rule: string, gitignoreDir: string): string | null {
+    const cleaned = rule.replace(/^\//, "");
+    if (!cleaned) return null;
+
+    if (!gitignoreDir || gitignoreDir === ".") {
+      return cleaned;
+    }
+
+    return upath.normalizeTrim(upath.join(gitignoreDir, cleaned));
+  }
+}