@goshenkata/dryscan-core 1.0.0

package/src/services/DuplicationCache.ts

@@ -0,0 +1,104 @@
+import { DuplicateGroup } from "../types";
+
+/**
+ * In-memory cache for duplicate comparison scores.
+ * Stores a global map of comparison keys and a per-file index for fast invalidation.
+ */
+export class DuplicationCache {
+  private static instance: DuplicationCache | null = null;
+
+  private readonly comparisons = new Map<string, number>();
+  private readonly fileIndex = new Map<string, Set<string>>();
+  private initialized = false;
+
+  static getInstance(): DuplicationCache {
+    if (!DuplicationCache.instance) {
+      DuplicationCache.instance = new DuplicationCache();
+    }
+    return DuplicationCache.instance;
+  }
+
+  /**
+   * Updates the cache with fresh duplicate groups. Not awaited by callers to avoid blocking.
+   */
+  async update(groups: DuplicateGroup[]): Promise<void> {
+    if (!groups) return;
+
+    for (const group of groups) {
+      const key = this.makeKey(group.left.id, group.right.id);
+      this.comparisons.set(key, group.similarity);
+      this.addKeyForFile(group.left.filePath, key);
+      this.addKeyForFile(group.right.filePath, key);
+    }
+
+    this.initialized = this.initialized || groups.length > 0;
+  }
+
+  /**
+   * Retrieves a cached similarity if present and valid for both file paths.
+   * Returns null when the cache has not been initialized or when the pair is missing.
+   */
+  get(leftId: string, rightId: string, leftFilePath: string, rightFilePath: string): number | null {
+    if (!this.initialized) return null;
+
+    const key = this.makeKey(leftId, rightId);
+    if (!this.fileHasKey(leftFilePath, key) || !this.fileHasKey(rightFilePath, key)) {
+      return null;
+    }
+
+    const value = this.comparisons.get(key);
+    return typeof value === "number" ? value : null;
+  }
+
+  /**
+   * Invalidates all cached comparisons involving the provided file paths.
+   */
+  async invalidate(paths: string[]): Promise<void> {
+    if (!this.initialized || !paths || paths.length === 0) return;
+
+    const unique = new Set(paths);
+    for (const filePath of unique) {
+      const keys = this.fileIndex.get(filePath);
+      if (!keys) continue;
+
+      for (const key of keys) {
+        this.comparisons.delete(key);
+        for (const [otherPath, otherKeys] of this.fileIndex.entries()) {
+          if (otherKeys.delete(key) && otherKeys.size === 0) {
+            this.fileIndex.delete(otherPath);
+          }
+        }
+      }
+
+      this.fileIndex.delete(filePath);
+    }
+
+    if (this.comparisons.size === 0) {
+      this.initialized = false;
+    }
+  }
+
+  /**
+   * Clears all cached data. Intended for test setup.
+   */
+  clear(): void {
+    this.comparisons.clear();
+    this.fileIndex.clear();
+    this.initialized = false;
+  }
+
+  private addKeyForFile(filePath: string, key: string): void {
+    const current = this.fileIndex.get(filePath) ?? new Set<string>();
+    current.add(key);
+    this.fileIndex.set(filePath, current);
+  }
+
+  private fileHasKey(filePath: string, key: string): boolean {
+    const keys = this.fileIndex.get(filePath);
+    return keys ? keys.has(key) : false;
+  }
+
+  private makeKey(leftId: string, rightId: string): string {
+    return [leftId, rightId].sort().join("::");
+  }
+}

package/src/services/EmbeddingService.ts

@@ -0,0 +1,58 @@
+import debug from "debug";
+import { OllamaEmbeddings } from "@langchain/ollama";
+import { GoogleGenerativeAIEmbeddings } from "@langchain/google-genai";
+import { TaskType } from "@google/generative-ai";
+import { IndexUnit } from "../types";
+import { configStore } from "../config/configStore";
+
+const log = debug("DryScan:EmbeddingService");
+
+export class EmbeddingService {
+    constructor(private readonly repoPath: string) { }
+
+    async addEmbedding(fn: IndexUnit): Promise<IndexUnit> {
+        const config = await configStore.get(this.repoPath);
+        const maxContext = config?.contextLength ?? 2048;
+        if (fn.code.length > maxContext) {
+            log(
+                "Skipping embedding for %s (code length %d exceeds context %d)",
+                fn.id,
+                fn.code.length,
+                maxContext
+            );
+            return { ...fn, embedding: null };
+        }
+
+        const model = config.embeddingModel ?? undefined
+        const source = config.embeddingSource
+        if (!source) {
+            const message = `Embedding source is not configured for repository at ${this.repoPath}`;
+            log(message);
+            throw new Error(message);
+        }
+
+        const embeddings = this.buildProvider(source, model);
+        const embedding = await embeddings.embedQuery(fn.code);
+        return { ...fn, embedding };
+    }
+
+    private buildProvider(source: string, model: string) {
+        if (source === "google") {
+            return new GoogleGenerativeAIEmbeddings({
+                model: model ?? "gemini-embedding-001",
+                taskType: TaskType.SEMANTIC_SIMILARITY,
+            });
+        }
+
+        if (/^https?:\/\//i.test(source)) {
+            return new OllamaEmbeddings({
+                model: model ?? "embeddinggemma",
+                baseUrl: source,
+            });
+        }
+
+        const message = `Unsupported embedding source: ${source || "(empty)"}`;
+        log(message);
+        throw new Error(message);
+    }
+}

package/src/services/ExclusionService.ts

@@ -0,0 +1,102 @@
+import { DryConfig } from "../types";
+import { configStore } from "../config/configStore";
+import { DryScanServiceDeps } from "./types";
+import { IndexUnitType } from "../types";
+import { minimatch } from "minimatch";
+import { ParsedPairKey } from "./PairingService";
+
+export class ExclusionService {
+  private config?: DryConfig;
+
+  constructor(private readonly deps: DryScanServiceDeps) {}
+
+  async cleanupExcludedFiles(): Promise<void> {
+    const config = await this.loadConfig();
+    if (!config.excludedPaths || config.excludedPaths.length === 0) return;
+
+    const units = await this.deps.db.getAllUnits();
+    const files = await this.deps.db.getAllFiles();
+
+    const unitPathsToRemove = new Set<string>();
+    for (const unit of units) {
+      if (this.pathExcluded(unit.filePath)) {
+        unitPathsToRemove.add(unit.filePath);
+      }
+    }
+
+    const filePathsToRemove = new Set<string>();
+    for (const file of files) {
+      if (this.pathExcluded(file.filePath)) {
+        filePathsToRemove.add(file.filePath);
+      }
+    }
+
+    const paths = [...new Set([...unitPathsToRemove, ...filePathsToRemove])];
+    if (paths.length > 0) {
+      await this.deps.db.removeUnitsByFilePaths(paths);
+      await this.deps.db.removeFilesByFilePaths(paths);
+    }
+  }
+
+  async cleanExclusions(): Promise<{ removed: number; kept: number }> {
+    const config = await this.loadConfig();
+    const units = await this.deps.db.getAllUnits();
+
+    const actualPairsByType = {
+      [IndexUnitType.CLASS]: this.buildPairKeys(units, IndexUnitType.CLASS),
+      [IndexUnitType.FUNCTION]: this.buildPairKeys(units, IndexUnitType.FUNCTION),
+      [IndexUnitType.BLOCK]: this.buildPairKeys(units, IndexUnitType.BLOCK),
+    };
+
+    const kept: string[] = [];
+    const removed: string[] = [];
+
+    for (const entry of config.excludedPairs || []) {
+      const parsed = this.deps.pairing.parsePairKey(entry);
+      if (!parsed) {
+        removed.push(entry);
+        continue;
+      }
+
+      const candidates = actualPairsByType[parsed.type];
+      const matched = candidates.some((actual) => this.deps.pairing.pairKeyMatches(actual, parsed));
+      if (matched) {
+        kept.push(entry);
+      } else {
+        removed.push(entry);
+      }
+    }
+
+    const nextConfig: DryConfig = { ...config, excludedPairs: kept };
+    await configStore.save(this.deps.repoPath, nextConfig);
+    this.config = nextConfig;
+
+    return { removed: removed.length, kept: kept.length };
+  }
+
+  private pathExcluded(filePath: string): boolean {
+    const config = this.config;
+    if (!config || !config.excludedPaths || config.excludedPaths.length === 0) return false;
+    return config.excludedPaths.some((pattern) => minimatch(filePath, pattern, { dot: true }));
+  }
+
+  private buildPairKeys(units: any[], type: IndexUnitType): ParsedPairKey[] {
+    const typed = units.filter((u) => u.unitType === type);
+    const pairs: ParsedPairKey[] = [];
+    for (let i = 0; i < typed.length; i++) {
+      for (let j = i + 1; j < typed.length; j++) {
+        const key = this.deps.pairing.pairKeyForUnits(typed[i], typed[j]);
+        const parsed = key ? this.deps.pairing.parsePairKey(key) : null;
+        if (parsed) {
+          pairs.push(parsed);
+        }
+      }
+    }
+    return pairs;
+  }
+
+  private async loadConfig(): Promise<DryConfig> {
+    this.config = await configStore.get(this.deps.repoPath);
+    return this.config;
+  }
+}

package/src/services/PairingService.ts

@@ -0,0 +1,145 @@
+import crypto from "node:crypto";
+import debug from "debug";
+import { minimatch } from "minimatch";
+import { LanguageExtractor } from "../extractors/LanguageExtractor";
+import { IndexUnitExtractor } from "../IndexUnitExtractor";
+import { IndexUnit, IndexUnitType } from "../types";
+import { BLOCK_HASH_ALGO } from "../const";
+
+const log = debug("DryScan:pairs");
+
+type UnitLike = Pick<IndexUnit, "unitType" | "filePath" | "name" | "code">;
+
+export interface ParsedPairKey {
+  type: IndexUnitType;
+  left: string;
+  right: string;
+  key: string;
+}
+
+/**
+ * Service for building and parsing pair keys with extractor-aware labeling.
+ */
+export class PairingService {
+  constructor(private readonly indexUnitExtractor: IndexUnitExtractor) {}
+
+  /**
+   * Creates a stable, order-independent key for two units of the same type.
+   * Returns null when units differ in type so callers can skip invalid pairs.
+   */
+  pairKeyForUnits(left: UnitLike, right: UnitLike): string | null {
+    if (left.unitType !== right.unitType) {
+      log("Skipping pair with mismatched types: %s vs %s", left.unitType, right.unitType);
+      return null;
+    }
+    const type = left.unitType;
+    const leftLabel = this.unitLabel(left);
+    const rightLabel = this.unitLabel(right);
+    const [a, b] = [leftLabel, rightLabel].sort();
+    return `${type}|${a}|${b}`;
+  }
+
+  /**
+   * Parses a raw pair key into its components, returning null for malformed values.
+   * Sorting is applied so callers can compare pairs without worrying about order.
+   */
+  parsePairKey(value: string): ParsedPairKey | null {
+    const parts = value.split("|");
+    if (parts.length !== 3) {
+      log("Invalid pair key format: %s", value);
+      return null;
+    }
+    const [typeRaw, leftRaw, rightRaw] = parts;
+    const type = this.stringToUnitType(typeRaw);
+    if (!type) {
+      log("Unknown unit type in pair key: %s", typeRaw);
+      return null;
+    }
+    const [left, right] = [leftRaw, rightRaw].sort();
+    return { type, left, right, key: `${type}|${left}|${right}` };
+  }
+
+  /**
+   * Checks whether an actual pair key satisfies a pattern, with glob matching for class paths.
+   */
+  pairKeyMatches(actual: ParsedPairKey, pattern: ParsedPairKey): boolean {
+    if (actual.type !== pattern.type) return false;
+    if (actual.type === IndexUnitType.CLASS) {
+      // Allow glob matching for class file paths.
+      const forward =
+        minimatch(actual.left, pattern.left, { dot: true }) &&
+        minimatch(actual.right, pattern.right, { dot: true });
+      const swapped =
+        minimatch(actual.left, pattern.right, { dot: true }) &&
+        minimatch(actual.right, pattern.left, { dot: true });
+      return forward || swapped;
+    }
+
+    // Functions and blocks use exact matching on canonical strings.
+    return (
+      (actual.left === pattern.left && actual.right === pattern.right) ||
+      (actual.left === pattern.right && actual.right === pattern.left)
+    );
+  }
+
+  /**
+   * Derives a reversible, extractor-aware label for a unit.
+   * Extractors may override; fallback uses a fixed format per unit type.
+   */
+  unitLabel(unit: UnitLike): string {
+    const extractor = this.findExtractor(unit.filePath);
+    const customLabel = extractor?.unitLabel?.(unit as IndexUnit);
+    if (customLabel) return customLabel;
+
+    switch (unit.unitType) {
+      case IndexUnitType.CLASS:
+        return unit.filePath;
+      case IndexUnitType.FUNCTION:
+        return this.canonicalFunctionSignature(unit);
+      case IndexUnitType.BLOCK:
+        return this.normalizedBlockHash(unit);
+      default:
+        return unit.name;
+    }
+  }
+
+  private findExtractor(filePath: string): LanguageExtractor | undefined {
+    return this.indexUnitExtractor.extractors.find((ex) => ex.supports(filePath));
+  }
+
+  private canonicalFunctionSignature(unit: UnitLike): string {
+    const arity = this.extractArity(unit.code);
+    return `${unit.name}(arity:${arity})`;
+  }
+
+  /**
+   * Normalizes block code (strips comments/whitespace) and hashes it for pair matching.
+   */
+  private normalizedBlockHash(unit: UnitLike): string {
+    const normalized = this.normalizeCode(unit.code);
+    return crypto.createHash(BLOCK_HASH_ALGO).update(normalized).digest("hex");
+  }
+
+  private stringToUnitType(value: string): IndexUnitType | null {
+    if (value === IndexUnitType.CLASS) return IndexUnitType.CLASS;
+    if (value === IndexUnitType.FUNCTION) return IndexUnitType.FUNCTION;
+    if (value === IndexUnitType.BLOCK) return IndexUnitType.BLOCK;
+    return null;
+  }
+
+  private extractArity(code: string): number {
+    const match = code.match(/^[^{]*?\(([^)]*)\)/s);
+    if (!match) return 0;
+    const params = match[1]
+      .split(",")
+      .map((p) => p.trim())
+      .filter(Boolean);
+    return params.length;
+  }
+
+  private normalizeCode(code: string): string {
+    const withoutBlockComments = code.replace(/\/\*[\s\S]*?\*\//g, "");
+    const withoutLineComments = withoutBlockComments.replace(/\/\/[^\n\r]*/g, "");
+    return withoutLineComments.replace(/\s+/g, "");
+  }
+}

package/src/services/RepositoryInitializer.ts

@@ -0,0 +1,93 @@
+import path from "path";
+import fs from "fs/promises";
+import { DryScanServiceDeps } from "./types";
+import { ExclusionService } from "./ExclusionService";
+import { IndexUnit } from "../types";
+import { EmbeddingService } from "./EmbeddingService";
+import { FileEntity } from "../db/entities/FileEntity";
+import { IndexUnitExtractor } from "../IndexUnitExtractor";
+
+export interface InitOptions {
+  skipEmbeddings?: boolean;
+}
+
+export class RepositoryInitializer {
+  constructor(
+    private readonly deps: DryScanServiceDeps,
+    private readonly exclusionService: ExclusionService
+  ) {}
+
+  async init(options?: InitOptions): Promise<void> {
+    const extractor = this.deps.extractor;
+
+    console.log("[DryScan] Phase 1/3: Extracting code units...");
+    await this.initUnits(extractor);
+    console.log("[DryScan] Phase 2/3: Computing embeddings (may be slow)...");
+    await this.computeEmbeddings(options?.skipEmbeddings === true);
+    console.log("[DryScan] Phase 3/3: Tracking files...");
+    await this.trackFiles(extractor);
+    await this.exclusionService.cleanupExcludedFiles();
+    console.log("[DryScan] Initialization phases complete.");
+  }
+
+  private async initUnits(extractor: IndexUnitExtractor): Promise<void> {
+    const units = await extractor.scan(this.deps.repoPath);
+    console.log(`[DryScan] Extracted ${units.length} index units.`);
+    await this.deps.db.saveUnits(units);
+  }
+
+  private async computeEmbeddings(skipEmbeddings: boolean): Promise<void> {
+    if (skipEmbeddings) {
+      console.log("[DryScan] Skipping embedding computation by request.");
+      return;
+    }
+    const allUnits: IndexUnit[] = await this.deps.db.getAllUnits();
+    const total = allUnits.length;
+    console.log(`[DryScan] Computing embeddings for ${total} units...`);
+
+    const updated: IndexUnit[] = [];
+    const progressInterval = Math.max(1, Math.ceil(total / 10));
+    const embeddingService = new EmbeddingService(this.deps.repoPath);
+
+    for (let i = 0; i < total; i++) {
+      const unit = allUnits[i];
+      try {
+        const enriched = await embeddingService.addEmbedding(unit);
+        updated.push(enriched);
+      } catch (err: any) {
+        console.error(
+          `[DryScan] Embedding failed for ${unit.filePath} (${unit.name}): ${err?.message || err}`
+        );
+        throw err;
+      }
+
+      const completed = i + 1;
+      if (completed === total || completed % progressInterval === 0) {
+        const pct = Math.floor((completed / total) * 100);
+        console.log(`[DryScan] Embeddings ${completed}/${total} (${pct}%)`);
+      }
+    }
+
+    await this.deps.db.updateUnits(updated);
+  }
+
+  private async trackFiles(extractor: IndexUnitExtractor): Promise<void> {
+    const allFunctions = await extractor.listSourceFiles(this.deps.repoPath);
+    const fileEntities: FileEntity[] = [];
+
+    for (const relPath of allFunctions) {
+      const fullPath = path.join(this.deps.repoPath, relPath);
+      const stat = await fs.stat(fullPath);
+      const checksum = await extractor.computeChecksum(fullPath);
+
+      const fileEntity = new FileEntity();
+      fileEntity.filePath = relPath;
+      fileEntity.checksum = checksum;
+      fileEntity.mtime = stat.mtimeMs;
+      fileEntities.push(fileEntity);
+    }
+
+    await this.deps.db.saveFiles(fileEntities);
+    console.log(`[DryScan] Tracked ${fileEntities.length} files.`);
+  }
+}

package/src/services/UpdateService.ts

@@ -0,0 +1,28 @@
+import debug from "debug";
+import { DryScanServiceDeps } from "./types";
+import { ExclusionService } from "./ExclusionService";
+import { performIncrementalUpdate } from "../DryScanUpdater";
+import { DuplicationCache } from "./DuplicationCache";
+
+const log = debug("DryScan:UpdateService");
+
+export class UpdateService {
+  constructor(
+    private readonly deps: DryScanServiceDeps,
+    private readonly exclusionService: ExclusionService
+  ) {}
+
+  async updateIndex(): Promise<void> {
+    const extractor = this.deps.extractor;
+    const cache = DuplicationCache.getInstance();
+
+    try {
+      const changeSet = await performIncrementalUpdate(this.deps.repoPath, extractor, this.deps.db);
+      await this.exclusionService.cleanupExcludedFiles();
+      await cache.invalidate([...changeSet.changed, ...changeSet.deleted]);
+    } catch (err) {
+      log("Error during index update:", err);
+      throw err;
+    }
+  }
+}

package/src/services/types.ts

@@ -0,0 +1,10 @@
+import { DryScanDatabase } from "../db/DryScanDatabase";
+import { IndexUnitExtractor } from "../IndexUnitExtractor";
+import { PairingService } from "./PairingService";
+
+export interface DryScanServiceDeps {
+  repoPath: string;
+  db: DryScanDatabase;
+  extractor: IndexUnitExtractor;
+  pairing: PairingService;
+}

package/src/types/glob-gitignore.d.ts

@@ -0,0 +1,7 @@
+declare module "glob-gitignore" {
+  import type { GlobOptions } from "glob";
+  export function glob(patterns: string | string[], options?: GlobOptions & { ignore?: string | string[] }): Promise<string[]>;
+  export function sync(patterns: string | string[], options?: GlobOptions & { ignore?: string | string[] }): string[];
+  export function hasMagic(patterns: string | string[], options?: GlobOptions): boolean;
+  export default glob;
+}

package/src/types/short-uuid.d.ts

@@ -0,0 +1,7 @@
+declare module "short-uuid" {
+  export function generate(): string;
+  const shortUuid: {
+    generate: () => string;
+  };
+  export default shortUuid;
+}

package/src/types/tree-sitter-langs.d.ts

@@ -0,0 +1,4 @@
+declare module "tree-sitter-java" {
+  const Java: any;
+  export = Java;
+}

package/src/types.ts

@@ -0,0 +1,76 @@
+export enum IndexUnitType {
+  CLASS = "class",
+  FUNCTION = "function",
+  BLOCK = "block",
+}
+
+export interface DuplicateGroup {
+  id: string;
+  similarity: number;
+  left: DuplicateSide;
+  right: DuplicateSide;
+  shortId: string;
+  exclusionString: string;
+}
+
+export interface DuplicationScore {
+  score: number;
+  grade: 'Excellent' | 'Good' | 'Fair' | 'Poor' | 'Critical';
+  totalLines: number;
+  duplicateLines: number;
+  duplicateGroups: number;
+}
+
+export interface DuplicateAnalysisResult {
+  duplicates: DuplicateGroup[];
+  score: DuplicationScore;
+}
+
+export interface DuplicateReport {
+  version: number;
+  generatedAt: string;
+  threshold: number;
+  score: DuplicationScore;
+  duplicates: DuplicateGroup[];
+}
+
+export interface DuplicateSide {
+  id: string;
+  name: string;
+  filePath: string;
+  startLine: number;
+  endLine: number;
+  code: string;
+  unitType: IndexUnitType;
+}
+
+export interface DryConfig {
+  excludedPaths: string[];
+  excludedPairs: string[];
+  minLines: number;
+  minBlockLines: number;
+  threshold: number;
+  embeddingModel: string;
+  embeddingSource?: string;
+  contextLength: number;
+}
+
+export interface IndexUnit {
+  id: string;
+  name: string;
+  filePath: string;
+  startLine: number;
+  endLine: number;
+  code: string;
+  unitType: IndexUnitType;
+  parentId?: string | null;
+  parent?: IndexUnit | null;
+  children?: IndexUnit[];
+  embedding?: number[] | null;
+}
+
+export interface EmbeddingResult {
+  processed: number;
+  updated: number;
+  errors: string[];
+}