@goshenkata/dryscan-core 1.0.0

package/src/extractors/java.ts

@@ -0,0 +1,335 @@
+import crypto from "node:crypto";
+import Parser from "tree-sitter";
+import Java from "tree-sitter-java";
+import { LanguageExtractor } from "./LanguageExtractor";
+import { IndexUnit, IndexUnitType } from "../types";
+import { indexConfig } from "../config/indexConfig";
+import { DryConfig } from "../types";
+import { configStore } from "../config/configStore";
+import { BLOCK_HASH_ALGO } from "../const";
+
+export class JavaExtractor implements LanguageExtractor {
+  readonly id = "java";
+  readonly exts = [".java"];
+
+  private parser: Parser;
+  private readonly repoPath: string;
+  private config?: DryConfig;
+
+  constructor(repoPath: string) {
+    this.repoPath = repoPath;
+    this.parser = new Parser();
+    this.parser.setLanguage(Java);
+  }
+
+  supports(filePath: string): boolean {
+    const lower = filePath.toLowerCase();
+    return this.exts.some((ext) => lower.endsWith(ext));
+  }
+
+  async extractFromText(fileRelPath: string, source: string): Promise<IndexUnit[]> {
+    if (!source.trim()) return [];
+
+    this.config = await configStore.get(this.repoPath);
+
+    const tree = this.parser.parse(source);
+    const units: IndexUnit[] = [];
+
+    const visit = (node: Parser.SyntaxNode, currentClass?: IndexUnit) => {
+      if (this.isClassNode(node)) {
+        const className = this.getClassName(node, source) || "<anonymous>";
+        if (this.isDtoClass(node, source, className)) {
+          return;
+        }
+        const startLine = node.startPosition.row;
+        const endLine = node.endPosition.row;
+        const classLength = endLine - startLine;
+        const skipClass = this.shouldSkip(IndexUnitType.CLASS, className, classLength);
+        const classId = this.buildId(IndexUnitType.CLASS, className, startLine, endLine);
+        const code = this.stripComments(this.stripClassBody(node, source));
+        const classUnit: IndexUnit = {
+          id: classId,
+          name: className,
+          filePath: fileRelPath,
+          startLine,
+          endLine,
+          code,
+          unitType: IndexUnitType.CLASS,
+          children: [],
+        };
+        if (!skipClass) {
+          units.push(classUnit);
+        }
+
+        for (let i = 0; i < node.namedChildCount; i++) {
+          const child = node.namedChild(i);
+          if (child) visit(child, skipClass ? undefined : classUnit);
+        }
+        return;
+      }
+
+      if (this.isFunctionNode(node)) {
+        const fnUnit = this.buildFunctionUnit(node, source, fileRelPath, currentClass);
+        const fnLength = fnUnit.endLine - fnUnit.startLine;
+        const bodyNode = this.getFunctionBody(node);
+        const skipFunction = this.shouldSkip(IndexUnitType.FUNCTION, fnUnit.name, fnLength);
+
+        if (skipFunction) {
+          return;
+        }
+
+        units.push(fnUnit);
+
+        if (bodyNode) {
+          const blocks = this.extractBlocks(bodyNode, source, fileRelPath, fnUnit);
+          units.push(...blocks);
+        }
+      }
+
+      for (let i = 0; i < node.namedChildCount; i++) {
+        const child = node.namedChild(i);
+        if (child) visit(child, currentClass);
+      }
+    };
+
+    visit(tree.rootNode);
+
+    return units;
+  }
+
+  unitLabel(unit: IndexUnit): string | null {
+    if (unit.unitType === IndexUnitType.CLASS) return unit.filePath;
+    if (unit.unitType === IndexUnitType.FUNCTION) return this.canonicalFunctionSignature(unit);
+    if (unit.unitType === IndexUnitType.BLOCK) return this.normalizedBlockHash(unit);
+    return unit.name;
+  }
+
+  private isClassNode(node: Parser.SyntaxNode): boolean {
+    return node.type === "class_declaration";
+  }
+
+  private getClassName(node: Parser.SyntaxNode, source: string): string | null {
+    const nameNode = node.childForFieldName?.("name");
+    return nameNode ? source.slice(nameNode.startIndex, nameNode.endIndex) : null;
+  }
+
+  private isFunctionNode(node: Parser.SyntaxNode): boolean {
+    return node.type === "method_declaration" || node.type === "constructor_declaration";
+  }
+
+  private getFunctionName(node: Parser.SyntaxNode, source: string, parentClass?: IndexUnit): string | null {
+    const nameNode = node.childForFieldName?.("name");
+    const nameText = nameNode ? source.slice(nameNode.startIndex, nameNode.endIndex) : "<anonymous>";
+    return parentClass ? `${parentClass.name}.${nameText}` : nameText;
+  }
+
+  private getFunctionBody(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
+    return node.childForFieldName?.("body") ?? null;
+  }
+
+  private isBlockNode(node: Parser.SyntaxNode): boolean {
+    return node.type === "block";
+  }
+
+  private getMethodBodiesForClass(node: Parser.SyntaxNode): Parser.SyntaxNode[] {
+    const bodies: Parser.SyntaxNode[] = [];
+    const classBody = node.children.find(child => child.type === "class_body");
+    if (!classBody) return bodies;
+    
+    for (let i = 0; i < classBody.namedChildCount; i++) {
+      const child = classBody.namedChild(i);
+      if (!child) continue;
+      if (child.type === "method_declaration" || child.type === "constructor_declaration") {
+        const body = child.childForFieldName?.("body");
+        if (body) bodies.push(body);
+      }
+    }
+    return bodies;
+  }
+
+  private canonicalFunctionSignature(unit: IndexUnit): string {
+    const arity = this.extractArity(unit.code);
+    return `${unit.name}(arity:${arity})`;
+  }
+
+  private normalizedBlockHash(unit: IndexUnit): string {
+    const normalized = this.normalizeCode(unit.code);
+    return crypto.createHash(BLOCK_HASH_ALGO).update(normalized).digest("hex");
+  }
+
+  private shouldSkip(unitType: IndexUnitType, name: string, lineCount: number): boolean {
+    if (!this.config) {
+      throw new Error("Config not loaded before skip evaluation");
+    }
+    const config = this.config;
+    const minLines = unitType === IndexUnitType.BLOCK
+      ? Math.max(indexConfig.blockMinLines, config.minBlockLines ?? 0)
+      : config.minLines;
+    const belowMin = minLines > 0 && lineCount < minLines;
+    const trivial = unitType === IndexUnitType.FUNCTION && this.isTrivialFunction(name);
+    return belowMin || trivial;
+  }
+
+  private isTrivialFunction(fullName: string): boolean {
+    const simpleName = fullName.split(".").pop() || fullName;
+    const isGetter = /^(get|is)[A-Z]/.test(simpleName);
+    const isSetter = /^set[A-Z]/.test(simpleName);
+    return isGetter || isSetter;
+  }
+
+  private isDtoClass(node: Parser.SyntaxNode, source: string, className: string): boolean {
+    const classBody = node.children.find((child) => child.type === "class_body");
+    if (!classBody) return false;
+
+    let hasField = false;
+
+    for (let i = 0; i < classBody.namedChildCount; i++) {
+      const child = classBody.namedChild(i);
+      if (!child) continue;
+
+      if (child.type === "field_declaration") {
+        hasField = true;
+        continue;
+      }
+
+      if (child.type.includes("annotation")) {
+        continue;
+      }
+
+      if (child.type === "method_declaration" || child.type === "constructor_declaration") {
+        const simpleName = this.getSimpleFunctionName(child, source);
+        const fullName = `${className}.${simpleName}`;
+        if (!this.isTrivialFunction(fullName)) {
+          return false;
+        }
+        continue;
+      }
+
+      return false;
+    }
+
+    return hasField;
+  }
+
+  private getSimpleFunctionName(node: Parser.SyntaxNode, source: string): string {
+    const nameNode = node.childForFieldName?.("name");
+    return nameNode ? source.slice(nameNode.startIndex, nameNode.endIndex) : "<anonymous>";
+  }
+
+  private buildFunctionUnit(
+    node: Parser.SyntaxNode,
+    source: string,
+    file: string,
+    parentClass?: IndexUnit
+  ): IndexUnit {
+    const name = this.getFunctionName(node, source, parentClass) || "<anonymous>";
+    const startLine = node.startPosition.row;
+    const endLine = node.endPosition.row;
+    const id = this.buildId(IndexUnitType.FUNCTION, name, startLine, endLine);
+    const unit: IndexUnit = {
+      id,
+      name,
+      filePath: file,
+      startLine,
+      endLine,
+      code: this.stripComments(source.slice(node.startIndex, node.endIndex)),
+      unitType: IndexUnitType.FUNCTION,
+      parentId: parentClass?.id,
+      parent: parentClass,
+    };
+    if (parentClass) {
+      parentClass.children = parentClass.children || [];
+      parentClass.children.push(unit);
+    }
+    return unit;
+  }
+
+  private extractBlocks(
+    bodyNode: Parser.SyntaxNode,
+    source: string,
+    file: string,
+    parentFunction: IndexUnit
+  ): IndexUnit[] {
+    const blocks: IndexUnit[] = [];
+
+    const visit = (n: Parser.SyntaxNode) => {
+      if (this.isBlockNode(n)) {
+        const startLine = n.startPosition.row;
+        const endLine = n.endPosition.row;
+        const lineCount = endLine - startLine;
+        if (this.shouldSkip(IndexUnitType.BLOCK, parentFunction.name, lineCount)) {
+          return;
+        }
+        if (lineCount >= indexConfig.blockMinLines) {
+          const id = this.buildId(IndexUnitType.BLOCK, parentFunction.name, startLine, endLine);
+          const blockUnit: IndexUnit = {
+            id,
+            name: parentFunction.name,
+            filePath: file,
+            startLine,
+            endLine,
+            code: this.stripComments(source.slice(n.startIndex, n.endIndex)),
+            unitType: IndexUnitType.BLOCK,
+            parentId: parentFunction.id,
+            parent: parentFunction,
+          };
+          parentFunction.children = parentFunction.children || [];
+          parentFunction.children.push(blockUnit);
+          blocks.push(blockUnit);
+        }
+      }
+
+      for (let i = 0; i < n.namedChildCount; i++) {
+        const child = n.namedChild(i);
+        if (child) visit(child);
+      }
+    };
+
+    visit(bodyNode);
+    return blocks;
+  }
+
+  private stripClassBody(node: Parser.SyntaxNode, source: string): string {
+    const classStart = node.startIndex;
+    let code = source.slice(classStart, node.endIndex);
+
+    const methodBodies: Array<{ start: number; end: number }> = [];
+    const candidates = this.getMethodBodiesForClass(node);
+
+    for (const body of candidates) {
+      methodBodies.push({ start: body.startIndex - classStart, end: body.endIndex - classStart });
+    }
+
+    methodBodies.sort((a, b) => b.start - a.start);
+    for (const body of methodBodies) {
+      code = code.slice(0, body.start) + " { }" + code.slice(body.end);
+    }
+
+    return code;
+  }
+
+  private buildId(type: IndexUnitType, name: string, startLine: number, endLine: number): string {
+    return `${type}:${name}:${startLine}-${endLine}`;
+  }
+
+  private extractArity(code: string): number {
+    const match = code.match(/^[^{]*?\(([^)]*)\)/s);
+    if (!match) return 0;
+    const params = match[1]
+      .split(",")
+      .map((p) => p.trim())
+      .filter(Boolean);
+    return params.length;
+  }
+
+  private normalizeCode(code: string): string {
+    const withoutBlockComments = code.replace(/\/\*[\s\S]*?\*\//g, "");
+    const withoutLineComments = withoutBlockComments.replace(/\/\/[^\n\r]*/g, "");
+    return withoutLineComments.replace(/\s+/g, "");
+  }
+
+  private stripComments(code: string): string {
+    const withoutBlockComments = code.replace(/\/\*[\s\S]*?\*\//g, (match) => match.replace(/[^\n\r]/g, ""));
+    return withoutBlockComments.replace(/\/\/[^\n\r]*/g, "");
+  }
+}

package/src/index.ts

@@ -0,0 +1,9 @@
+// Public surface: keep minimal API for consumers
+export { DryScan } from './DryScan';
+export { configStore } from './config/configStore';
+export {
+	DuplicateGroup,
+	DuplicationScore,
+	DuplicateReport,
+	DryConfig,
+} from './types';

package/src/services/DuplicateService.ts

@@ -0,0 +1,274 @@
+import debug from "debug";
+import shortUuid from "short-uuid";
+import { cosineSimilarity } from "@langchain/core/utils/math";
+import { DryScanServiceDeps } from "./types";
+import { DuplicateAnalysisResult, DuplicateGroup, DuplicationScore, IndexUnit, IndexUnitType } from "../types";
+import { indexConfig } from "../config/indexConfig";
+import { DryConfig } from "../types";
+import { DuplicationCache } from "./DuplicationCache";
+
+const log = debug("DryScan:DuplicateService");
+
+export class DuplicateService {
+  private config?: DryConfig;
+  private readonly cache = DuplicationCache.getInstance();
+
+  constructor(private readonly deps: DryScanServiceDeps) {}
+
+  async findDuplicates(config: DryConfig): Promise<DuplicateAnalysisResult> {
+    this.config = config;
+    const allUnits = await this.deps.db.getAllUnits();
+    if (allUnits.length < 2) {
+      const score = this.computeDuplicationScore([], allUnits);
+      return { duplicates: [], score };
+    }
+
+    const thresholds = this.resolveThresholds(config.threshold);
+    const duplicates = this.computeDuplicates(allUnits, thresholds);
+    const filteredDuplicates = duplicates.filter((group) => !this.isGroupExcluded(group));
+    log("Found %d duplicate groups", filteredDuplicates.length);
+
+    // Update cache asynchronously; no need to block the main flow.
+    this.cache.update(filteredDuplicates).catch((err) => log("Cache update failed: %O", err));
+
+    const score = this.computeDuplicationScore(filteredDuplicates, allUnits);
+    return { duplicates: filteredDuplicates, score };
+  }
+
+  private resolveThresholds(functionThreshold?: number): { function: number; block: number; class: number } {
+    const defaults = indexConfig.thresholds;
+    const clamp = (value: number) => Math.min(1, Math.max(0, value));
+
+    const base = functionThreshold ?? defaults.function;
+    const blockOffset = defaults.block - defaults.function;
+    const classOffset = defaults.class - defaults.function;
+
+    const functionThresholdValue = clamp(base);
+    return {
+      function: functionThresholdValue,
+      block: clamp(functionThresholdValue + blockOffset),
+      class: clamp(functionThresholdValue + classOffset),
+    };
+  }
+
+  private computeDuplicates(
+    units: IndexUnit[],
+    thresholds: { function: number; block: number; class: number }
+  ): DuplicateGroup[] {
+    const duplicates: DuplicateGroup[] = [];
+    const byType = new Map<IndexUnitType, IndexUnit[]>();
+
+    for (const unit of units) {
+      const list = byType.get(unit.unitType) ?? [];
+      list.push(unit);
+      byType.set(unit.unitType, list);
+    }
+
+    for (const [type, typedUnits] of byType.entries()) {
+      const threshold = this.getThreshold(type, thresholds);
+
+      for (let i = 0; i < typedUnits.length; i++) {
+        for (let j = i + 1; j < typedUnits.length; j++) {
+          const left = typedUnits[i];
+          const right = typedUnits[j];
+
+          if (this.shouldSkipComparison(left, right)) continue;
+
+          const cached = this.cache.get(left.id, right.id, left.filePath, right.filePath);
+          let similarity: number | null = null;
+
+          if (cached !== null) {
+            similarity = cached;
+          } else {
+            if (!left.embedding || !right.embedding) continue;
+            similarity = this.computeWeightedSimilarity(left, right);
+          }
+
+          if (similarity === null) continue;
+
+          if (similarity >= threshold) {
+            const exclusionString = this.deps.pairing.pairKeyForUnits(left, right);
+            if (!exclusionString) continue;
+
+            duplicates.push({
+              id: `${left.id}::${right.id}`,
+              similarity,
+              shortId: shortUuid.generate(),
+              exclusionString,
+              left: {
+                id: left.id,
+                name: left.name,
+                filePath: left.filePath,
+                startLine: left.startLine,
+                endLine: left.endLine,
+                code: left.code,
+                unitType: left.unitType,
+              },
+              right: {
+                id: right.id,
+                name: right.name,
+                filePath: right.filePath,
+                startLine: right.startLine,
+                endLine: right.endLine,
+                code: right.code,
+                unitType: right.unitType,
+              },
+            });
+          }
+        }
+      }
+    }
+
+    return duplicates.sort((a, b) => b.similarity - a.similarity);
+  }
+
+  private isGroupExcluded(group: DuplicateGroup): boolean {
+    const config = this.config;
+    if (!config || !config.excludedPairs || config.excludedPairs.length === 0) return false;
+    const key = this.deps.pairing.pairKeyForUnits(group.left, group.right);
+    if (!key) return false;
+    const actual = this.deps.pairing.parsePairKey(key);
+    if (!actual) return false;
+    return config.excludedPairs.some((entry) => {
+      const parsed = this.deps.pairing.parsePairKey(entry);
+      return parsed ? this.deps.pairing.pairKeyMatches(actual, parsed) : false;
+    });
+  }
+
+  private getThreshold(type: IndexUnitType, thresholds: { function: number; block: number; class: number }): number {
+    if (type === IndexUnitType.CLASS) return thresholds.class;
+    if (type === IndexUnitType.BLOCK) return thresholds.block;
+    return thresholds.function;
+  }
+
+  private computeWeightedSimilarity(left: IndexUnit, right: IndexUnit): number {
+    const selfSimilarity = this.similarityWithFallback(left, right);
+
+    if (left.unitType === IndexUnitType.CLASS) {
+      return selfSimilarity * indexConfig.weights.class.self;
+    }
+
+    if (left.unitType === IndexUnitType.FUNCTION) {
+      const parentClassSimilarity = this.parentSimilarity(left, right, IndexUnitType.CLASS);
+      const weights = indexConfig.weights.function;
+      return (weights.self * selfSimilarity) + (weights.parentClass * parentClassSimilarity);
+    }
+
+    const weights = indexConfig.weights.block;
+    const parentFuncSim = this.parentSimilarity(left, right, IndexUnitType.FUNCTION);
+    const parentClassSim = this.parentSimilarity(left, right, IndexUnitType.CLASS);
+    return (
+      weights.self * selfSimilarity +
+      weights.parentFunction * parentFuncSim +
+      weights.parentClass * parentClassSim
+    );
+  }
+
+  private parentSimilarity(left: IndexUnit, right: IndexUnit, targetType: IndexUnitType): number {
+    const leftParent = this.findParentOfType(left, targetType);
+    const rightParent = this.findParentOfType(right, targetType);
+    if (!leftParent || !rightParent) return 0;
+    return this.similarityWithFallback(leftParent, rightParent);
+  }
+
+  private similarityWithFallback(left: IndexUnit, right: IndexUnit): number {
+    const leftHasEmbedding = this.hasVector(left);
+    const rightHasEmbedding = this.hasVector(right);
+
+    if (leftHasEmbedding && rightHasEmbedding) {
+      return cosineSimilarity([left.embedding as number[]], [right.embedding as number[]])[0][0];
+    }
+
+    return this.childSimilarity(left, right);
+  }
+
+  private childSimilarity(left: IndexUnit, right: IndexUnit): number {
+    const leftChildren = left.children ?? [];
+    const rightChildren = right.children ?? [];
+    if (leftChildren.length === 0 || rightChildren.length === 0) return 0;
+
+    let best = 0;
+    for (const lChild of leftChildren) {
+      for (const rChild of rightChildren) {
+        if (lChild.unitType !== rChild.unitType) continue;
+        const sim = this.similarityWithFallback(lChild, rChild);
+        if (sim > best) best = sim;
+      }
+    }
+    return best;
+  }
+
+  private hasVector(unit: IndexUnit): boolean {
+    return Array.isArray(unit.embedding) && unit.embedding.length > 0;
+  }
+
+  private shouldSkipComparison(left: IndexUnit, right: IndexUnit): boolean {
+    if (left.unitType !== IndexUnitType.BLOCK || right.unitType !== IndexUnitType.BLOCK) {
+      return false;
+    }
+
+    if (left.filePath !== right.filePath) {
+      return false;
+    }
+
+    const leftContainsRight = left.startLine <= right.startLine && left.endLine >= right.endLine;
+    const rightContainsLeft = right.startLine <= left.startLine && right.endLine >= left.endLine;
+    return leftContainsRight || rightContainsLeft;
+  }
+
+  private findParentOfType(unit: IndexUnit, targetType: IndexUnitType): IndexUnit | null {
+    let current: IndexUnit | undefined | null = unit.parent;
+    while (current) {
+      if (current.unitType === targetType) return current;
+      current = current.parent;
+    }
+    return null;
+  }
+
+  private computeDuplicationScore(duplicates: DuplicateGroup[], allUnits: IndexUnit[]): DuplicationScore {
+    const totalLines = this.calculateTotalLines(allUnits);
+
+    if (totalLines === 0 || duplicates.length === 0) {
+      return {
+        score: 0,
+        grade: "Excellent",
+        totalLines,
+        duplicateLines: 0,
+        duplicateGroups: 0,
+      };
+    }
+
+    const weightedDuplicateLines = duplicates.reduce((sum, group) => {
+      const leftLines = group.left.endLine - group.left.startLine + 1;
+      const rightLines = group.right.endLine - group.right.startLine + 1;
+      const avgLines = (leftLines + rightLines) / 2;
+      return sum + group.similarity * avgLines;
+    }, 0);
+
+    const score = (weightedDuplicateLines / totalLines) * 100;
+    const grade = this.getScoreGrade(score);
+
+    return {
+      score,
+      grade,
+      totalLines,
+      duplicateLines: Math.round(weightedDuplicateLines),
+      duplicateGroups: duplicates.length,
+    };
+  }
+
+  private calculateTotalLines(units: IndexUnit[]): number {
+    return units.reduce((sum, unit) => {
+      const lines = unit.endLine - unit.startLine + 1;
+      return sum + lines;
+    }, 0);
+  }
+
+  private getScoreGrade(score: number): DuplicationScore["grade"] {
+    if (score < 5) return "Excellent";
+    if (score < 15) return "Good";
+    if (score < 30) return "Fair";
+    if (score < 50) return "Poor";
+    return "Critical";
+  }
+}