@goshenkata/dryscan-core 1.0.0

package/src/IndexUnitExtractor.ts

@@ -0,0 +1,208 @@
+import path from "path";
+import type { Stats } from "fs";
+import fs from "fs/promises";
+import upath from "upath";
+import crypto from "node:crypto";
+import debug from "debug";
+import { glob } from "glob-gitignore";
+import { IndexUnit } from "./types";
+import { LanguageExtractor } from "./extractors/LanguageExtractor";
+import { JavaExtractor } from "./extractors/java";
+import { FILE_CHECKSUM_ALGO } from "./const";
+import { configStore } from "./config/configStore";
+import { DryConfig } from "./types";
+import { Gitignore } from "./Gitignore"
+import { Ignore } from "ignore";
+
+const log = debug("DryScan:Extractor");
+
+export type { LanguageExtractor } from "./extractors/LanguageExtractor";
+/**
+ * Returns the default set of language extractors supported by DryScan.
+ * Extend/override by passing custom extractors into the IndexUnitExtractor constructor.
+ */
+export function defaultExtractors(repoPath: string): LanguageExtractor[] {
+  return [new JavaExtractor(repoPath)];
+}
+
+/**
+ * Extracts and indexes code units (classes, functions, blocks) for a repository.
+ * Owns shared file-system helpers and delegates language-specific parsing to LanguageExtractors.
+ */
+export class IndexUnitExtractor {
+  private readonly root: string;
+  readonly extractors: LanguageExtractor[];
+  private readonly gitignore: Gitignore;
+
+  constructor(
+    rootPath: string,
+    extractors?: LanguageExtractor[]
+  ) {
+    this.root = rootPath;
+    this.extractors = extractors ?? defaultExtractors(rootPath);
+    this.gitignore = new Gitignore(this.root);
+    log("Initialized extractor for %s", this.root);
+  }
+
+  /**
+   * Lists all supported source files from a path. Honors exclusion globs from config.
+   */
+  async listSourceFiles(dirPath: string): Promise<string[]> {
+    const target = await this.resolveTarget(dirPath);
+    const config = await this.loadConfig();
+    const ignoreMatcher = await this.gitignore.buildMatcher(config);
+
+    if (target.stat.isFile()) {
+      return this.filterSingleFile(target.baseRel, ignoreMatcher);
+    }
+
+    const matches = await this.globSourceFiles(target.baseRel);
+    return this.filterSupportedFiles(matches, ignoreMatcher);
+  }
+
+  /**
+   * Computes MD5 checksum of file content to track changes.
+   */
+  async computeChecksum(filePath: string): Promise<string> {
+    const fullPath = path.isAbsolute(filePath)
+      ? filePath
+      : path.join(this.root, filePath);
+
+    const content = await fs.readFile(fullPath, "utf8");
+    return crypto.createHash(FILE_CHECKSUM_ALGO).update(content).digest("hex");
+  }
+
+  /**
+   * Scans a file or directory and extracts indexable units using the matching LanguageExtractor.
+   * The returned units have repo-relative file paths and no embedding attached.
+   */
+  async scan(targetPath: string): Promise<IndexUnit[]> {
+    const fullPath = path.isAbsolute(targetPath)
+      ? targetPath
+      : path.join(this.root, targetPath);
+
+    const stat = await fs.stat(fullPath).catch(() => null);
+    if (!stat) {
+      throw new Error(`Path not found: ${fullPath}`);
+    }
+
+    if (stat.isDirectory()) {
+      log("Scanning directory %s", fullPath);
+      return this.scanDirectory(fullPath);
+    }
+
+    return this.scanFile(fullPath);
+  }
+
+
+  /**
+   * Scans a directory recursively, extracting units from supported files while honoring exclusions.
+   */
+  private async scanDirectory(dir: string): Promise<IndexUnit[]> {
+    const out: IndexUnit[] = [];
+    const relDir = this.relPath(dir);
+    const files = await this.listSourceFiles(relDir);
+    for (const relFile of files) {
+      const absFile = path.join(this.root, relFile);
+      const extracted = await this.tryScanSupportedFile(absFile);
+      out.push(...extracted);
+    }
+    return out;
+  }
+
+  /**
+   * Scans a single file and extracts supported units.
+   */
+  private async scanFile(filePath: string): Promise<IndexUnit[]> {
+    return this.tryScanSupportedFile(filePath, true);
+  }
+
+  /**
+   * Extracts units from a supported file.
+   * Optionally throws when the file type is unsupported (used when scanning an explicit file).
+   */
+  private async tryScanSupportedFile(filePath: string, throwOnUnsupported = false): Promise<IndexUnit[]> {
+    const extractor = this.extractors.find(ex => ex.supports(filePath));
+    if (!extractor) {
+      if (throwOnUnsupported) {
+        throw new Error(`Unsupported file type: ${filePath}`);
+      }
+      return [];
+    }
+    const rel = this.relPath(filePath);
+    if (await this.shouldExclude(rel)) {
+      log("Skipping excluded file %s", rel);
+      return [];
+    }
+    const source = await fs.readFile(filePath, "utf8");
+    const units = await extractor.extractFromText(rel, source);
+    log("Extracted %d units from %s", units.length, rel);
+    return units.map(unit => ({
+      ...unit,
+      filePath: rel,
+      embedding: undefined,
+    }));
+  }
+
+  /**
+   * Converts an absolute path to a repo-relative, normalized (POSIX-style) path.
+   * This keeps paths stable across platforms and consistent in the index/DB.
+   */
+  private relPath(absPath: string): string {
+    return this.normalizeRelPath(upath.relative(this.root, absPath));
+  }
+
+  /**
+   * Returns true if a repo-relative path matches any configured exclusion glob.
+   */
+  private async shouldExclude(relPath: string): Promise<boolean> {
+    const config = await this.loadConfig();
+    const ignoreMatcher = await this.gitignore.buildMatcher(config);
+    return ignoreMatcher.ignores(this.normalizeRelPath(relPath));
+  }
+
+  private async loadConfig(): Promise<DryConfig> {
+    return await configStore.get(this.root);
+  }
+
+  /**
+   * Normalizes repo-relative paths and strips leading "./" to keep matcher inputs consistent.
+   */
+  private normalizeRelPath(relPath: string): string {
+    const normalized = upath.normalizeTrim(relPath);
+    return normalized.startsWith("./") ? normalized.slice(2) : normalized;
+  }
+
+  private async resolveTarget(dirPath: string): Promise<{ fullPath: string; baseRel: string; stat: Stats; }> {
+    const fullPath = path.isAbsolute(dirPath) ? dirPath : path.join(this.root, dirPath);
+    const stat = await fs.stat(fullPath).catch(() => null);
+    if (!stat) {
+      throw new Error(`Path not found: ${fullPath}`);
+    }
+    const baseRel = this.relPath(fullPath);
+    log("Listing source files under %s", fullPath);
+    return { fullPath, baseRel, stat };
+  }
+
+  private async filterSingleFile(baseRel: string, ignoreMatcher: Ignore): Promise<string[]> {
+    const relFile = this.normalizeRelPath(baseRel);
+    if (ignoreMatcher.ignores(relFile)) return [];
+    return this.extractors.some((ex) => ex.supports(relFile)) ? [relFile] : [];
+  }
+
+  private async globSourceFiles(baseRel: string): Promise<string[]> {
+    const pattern = baseRel ? `${baseRel.replace(/\\/g, "/")}/**/*` : "**/*";
+    const matches = await glob(pattern, {
+      cwd: this.root,
+      dot: false,
+      nodir: true,
+    });
+    return matches.map((p: string) => this.normalizeRelPath(p));
+  }
+
+  private filterSupportedFiles(relPaths: string[], ignoreMatcher: Ignore): string[] {
+    return relPaths
+      .filter((relPath: string) => !ignoreMatcher.ignores(relPath))
+      .filter((relPath: string) => this.extractors.some((ex) => ex.supports(relPath)));
+  }
+}

package/src/config/configStore.ts

@@ -0,0 +1,55 @@
+import upath from "upath";
+import { DryConfig } from "../types";
+import { ensureDefaultConfig, resolveDryConfig, saveDryConfig } from "./dryconfig";
+
+class ConfigStore {
+  private readonly cache = new Map<string, DryConfig>();
+  private readonly loading = new Map<string, Promise<DryConfig>>();
+
+  async init(repoPath: string): Promise<DryConfig> {
+    const key = this.normalize(repoPath);
+    return this.load(key, repoPath);
+  }
+
+  async get(repoPath: string): Promise<DryConfig> {
+    const key = this.normalize(repoPath);
+    const cached = this.cache.get(key);
+    if (cached) return cached;
+    return this.load(key, repoPath);
+  }
+
+  async refresh(repoPath: string): Promise<DryConfig> {
+    const key = this.normalize(repoPath);
+    this.cache.delete(key);
+    return this.load(key, repoPath);
+  }
+
+  async save(repoPath: string, config: DryConfig): Promise<void> {
+    const key = this.normalize(repoPath);
+    await saveDryConfig(repoPath, config);
+    this.cache.set(key, config);
+  }
+
+  private async load(key: string, repoPath: string): Promise<DryConfig> {
+    const existing = this.loading.get(key);
+    if (existing) return existing;
+
+    const promise = ensureDefaultConfig(repoPath).then(() => resolveDryConfig(repoPath)).then((config) => {
+      this.cache.set(key, config);
+      this.loading.delete(key);
+      return config;
+    }).catch((err) => {
+      this.loading.delete(key);
+      throw err;
+    });
+
+    this.loading.set(key, promise);
+    return promise;
+  }
+
+  private normalize(repoPath: string): string {
+    return upath.normalizeTrim(upath.resolve(repoPath));
+  }
+}
+
+export const configStore = new ConfigStore();

package/src/config/dryconfig.ts

@@ -0,0 +1,117 @@
+import fs from "fs/promises";
+import upath from "upath";
+import { Validator, Schema } from "jsonschema";
+import { DryConfig } from "../types";
+
+// Baseline config used when no file is present; exported so tests and constructors can seed defaults.
+export const DEFAULT_CONFIG: DryConfig = {
+  excludedPaths: [
+    "**/test/**",
+  ],
+  excludedPairs: [],
+  minLines: 3,
+  minBlockLines: 5,
+  threshold: 0.88,
+  embeddingModel: "embeddinggemma",
+  embeddingSource: "http://localhost:11434",
+  contextLength: 2048,
+};
+
+const validator = new Validator();
+
+const partialConfigSchema: Schema = {
+  type: "object",
+  properties: {
+    excludedPaths: { type: "array", items: { type: "string" } },
+    excludedPairs: { type: "array", items: { type: "string" } },
+    minLines: { type: "number" },
+    minBlockLines: { type: "number" },
+    threshold: { type: "number" },
+    embeddingModel: { type: "string" },
+    embeddingSource: { type: "string" },
+    contextLength: { type: "number" },
+  },
+};
+
+const fullConfigSchema: Schema = {
+  ...partialConfigSchema,
+  required: [
+    "excludedPaths",
+    "excludedPairs",
+    "minLines",
+    "minBlockLines",
+    "threshold",
+    "embeddingModel",
+    "contextLength",
+  ],
+};
+
+function validateConfig(raw: unknown, schema: Schema, source: string): any {
+  const result = validator.validate(raw, schema);
+  if (!result.valid) {
+    const details = result.errors.map((e) => e.stack).join("; ");
+    throw new Error(`${source} config is invalid: ${details}`);
+  }
+  return raw;
+}
+
+async function readConfigFile(repoPath: string): Promise<Partial<DryConfig>> {
+  const configPath = upath.join(repoPath, "dryconfig.json");
+  try {
+    const content = await fs.readFile(configPath, "utf8");
+    let parsed: Partial<DryConfig> = {};
+    try {
+      parsed = JSON.parse(content) as Partial<DryConfig>;
+    } catch (parseErr) {
+      throw new Error(`Invalid JSON in ${configPath}: ${(parseErr as Error).message}`);
+    }
+    return parsed;
+  } catch (err: any) {
+    if (err?.code === "ENOENT") {
+      return {};
+    }
+    throw err;
+  }
+}
+
+/**
+ * Resolves the effective config for a repo using defaults merged with any file config.
+ */
+export async function resolveDryConfig(repoPath: string): Promise<DryConfig> {
+  const fileConfigRaw = await readConfigFile(repoPath);
+  validateConfig(fileConfigRaw, partialConfigSchema, "Config file");
+
+  const merged = { ...DEFAULT_CONFIG, ...fileConfigRaw };
+  validateConfig(merged, fullConfigSchema, "Merged");
+  return merged as DryConfig;
+}
+
+// Backwards-compatible helper used by existing callers (file + defaults).
+export async function loadDryConfig(repoPath: string): Promise<DryConfig> {
+  return resolveDryConfig(repoPath);
+}
+
+export async function saveDryConfig(repoPath: string, config: DryConfig): Promise<void> {
+  const configPath = upath.join(repoPath, "dryconfig.json");
+  validateConfig(config, fullConfigSchema, "Config to save");
+  await fs.writeFile(configPath, JSON.stringify(config, null, 2), "utf8");
+}
+
+export async function ensureDefaultConfig(repoPath: string): Promise<void> {
+  const configPath = upath.join(repoPath, "dryconfig.json");
+  const repoExists = await fs.stat(repoPath).then((s) => s.isDirectory()).catch((err: any) => {
+    if (err?.code === "ENOENT") return false;
+    throw err;
+  });
+
+  if (!repoExists) return;
+
+  const exists = await fs.stat(configPath).then(() => true).catch((err: any) => {
+    if (err?.code === "ENOENT") return false;
+    throw err;
+  });
+
+  if (!exists) {
+    await saveDryConfig(repoPath, DEFAULT_CONFIG);
+  }
+}

package/src/config/indexConfig.ts

@@ -0,0 +1,13 @@
+export const indexConfig = {
+  blockMinLines: 5,
+  thresholds: {
+    class: 0.88,
+    function: 0.88,
+    block: 0.88,
+  },
+  weights: {
+    class: { self: 1 },
+    function: { self: 0.8, parentClass: 0.2 },
+    block: { self: 0.7, parentFunction: 0.2, parentClass: 0.1 },
+  },
+};

package/src/const.ts

@@ -0,0 +1,5 @@
+export const DRYSCAN_DIR = ".dry";
+export const INDEX_DB = "index.db";
+export const REPORTS_DIR = "reports";
+export const FILE_CHECKSUM_ALGO = "md5";
+export const BLOCK_HASH_ALGO = "sha1";

package/src/db/DryScanDatabase.ts

@@ -0,0 +1,128 @@
+import "reflect-metadata";
+import fs from "fs/promises";
+import upath from "upath";
+import { DataSource, Repository, In } from "typeorm";
+import { FileEntity } from "./entities/FileEntity";
+import { IndexUnit } from "../types";
+import { IndexUnitEntity } from "./entities/IndexUnitEntity";
+
+export class DryScanDatabase {
+  private dataSource?: DataSource;
+  private unitRepository?: Repository<IndexUnitEntity>;
+  private fileRepository?: Repository<FileEntity>;
+
+  isInitialized(): boolean {
+    return !!this.dataSource?.isInitialized;
+  }
+
+  async init(dbPath: string): Promise<void> {
+    await fs.mkdir(upath.dirname(dbPath), { recursive: true });
+
+    this.dataSource = new DataSource({
+      type: "better-sqlite3",
+      database: dbPath,
+      entities: [IndexUnitEntity, FileEntity],
+      synchronize: true,
+      logging: false,
+    });
+
+    await this.dataSource.initialize();
+    this.unitRepository = this.dataSource.getRepository(IndexUnitEntity);
+    this.fileRepository = this.dataSource.getRepository(FileEntity);
+  }
+
+  async saveUnit(unit: IndexUnit): Promise<void> {
+    await this.saveUnits(unit);
+  }
+
+  async saveUnits(units: IndexUnit | IndexUnit[]): Promise<void> {
+    if (!this.unitRepository) throw new Error("Database not initialized");
+    const payload = Array.isArray(units) ? units : [units];
+    await this.unitRepository.save(payload);
+  }
+
+  async getUnit(id: string): Promise<IndexUnit | null> {
+    if (!this.unitRepository) throw new Error("Database not initialized");
+    return this.unitRepository.findOne({ 
+      where: { id },
+      relations: ["children", "parent"]
+    });
+  }
+
+  async getAllUnits(): Promise<IndexUnit[]> {
+    if (!this.unitRepository) throw new Error("Database not initialized");
+    return this.unitRepository.find({ relations: ["children", "parent"] });
+  }
+
+  async updateUnit(unit: IndexUnit): Promise<void> {
+    await this.saveUnits(unit);
+  }
+
+  async updateUnits(units: IndexUnit | IndexUnit[]): Promise<void> {
+    await this.saveUnits(units);
+  }
+
+  /**
+   * Returns total count of indexed units.
+   */
+  async countUnits(): Promise<number> {
+    if (!this.unitRepository) throw new Error("Database not initialized");
+    return this.unitRepository.count();
+  }
+
+  /**
+   * Removes index units by their file paths.
+   * Used during incremental updates when files change.
+   */
+  async removeUnitsByFilePaths(filePaths: string[]): Promise<void> {
+    if (!this.unitRepository) throw new Error("Database not initialized");
+    await this.unitRepository.delete({ filePath: In(filePaths) });
+  }
+
+  /**
+   * Saves file metadata (path, checksum, mtime) to track changes.
+   */
+  async saveFile(file: FileEntity): Promise<void> {
+    if (!this.fileRepository) throw new Error("Database not initialized");
+    await this.fileRepository.save(file);
+  }
+
+  /**
+   * Saves multiple file metadata entries.
+   */
+  async saveFiles(files: FileEntity[]): Promise<void> {
+    if (!this.fileRepository) throw new Error("Database not initialized");
+    await this.fileRepository.save(files);
+  }
+
+  /**
+   * Gets file metadata by file path.
+   */
+  async getFile(filePath: string): Promise<FileEntity | null> {
+    if (!this.fileRepository) throw new Error("Database not initialized");
+    return this.fileRepository.findOne({ where: { filePath } });
+  }
+
+  /**
+   * Gets all tracked files.
+   */
+  async getAllFiles(): Promise<FileEntity[]> {
+    if (!this.fileRepository) throw new Error("Database not initialized");
+    return this.fileRepository.find();
+  }
+
+  /**
+   * Removes file metadata entries by file paths.
+   * Used when files are deleted from repository.
+   */
+  async removeFilesByFilePaths(filePaths: string[]): Promise<void> {
+    if (!this.fileRepository) throw new Error("Database not initialized");
+    await this.fileRepository.delete({ filePath: In(filePaths) });
+  }
+
+  async close(): Promise<void> {
+    if (this.dataSource?.isInitialized) {
+      await this.dataSource.destroy();
+    }
+  }
+}

package/src/db/entities/FileEntity.ts

@@ -0,0 +1,29 @@
+import { Entity, PrimaryColumn, Column } from "typeorm";
+
+/**
+ * Represents a tracked source file in the repository.
+ * Used to detect changes via checksum and mtime for incremental updates.
+ */
+@Entity("files")
+export class FileEntity {
+  /**
+   * Relative path to the file from repository root.
+   * Used as primary key for uniqueness.
+   */
+  @PrimaryColumn("text")
+  filePath!: string;
+
+  /**
+   * MD5 checksum of file content.
+   * Used to detect content changes.
+   */
+  @Column("text")
+  checksum!: string;
+
+  /**
+   * Last modification time in milliseconds since epoch.
+   * Used as fast sanity check before computing checksum.
+   */
+  @Column("integer")
+  mtime!: number;
+}

package/src/db/entities/IndexUnitEntity.ts

@@ -0,0 +1,50 @@
+import {
+  Column,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  OneToMany,
+  PrimaryColumn,
+  RelationId,
+} from "typeorm";
+import { IndexUnit, IndexUnitType } from "../../types";
+
+@Entity("index_units")
+export class IndexUnitEntity implements IndexUnit {
+  @PrimaryColumn("text")
+  id!: string;
+
+  @Column("text")
+  name!: string;
+
+  @Column("text")
+  filePath!: string;
+
+  @Column("integer")
+  startLine!: number;
+
+  @Column("integer")
+  endLine!: number;
+
+  @Column("text")
+  code!: string;
+
+  @Column("text")
+  unitType!: IndexUnitType;
+
+  @ManyToOne(() => IndexUnitEntity, (unit) => unit.children, {
+    nullable: true,
+    onDelete: "CASCADE",
+  })
+  @JoinColumn({ name: "parent_id" })
+  parent?: IndexUnitEntity | null;
+
+  @RelationId((unit: IndexUnitEntity) => unit.parent)
+  parentId?: string | null;
+
+  @OneToMany(() => IndexUnitEntity, (unit) => unit.parent, { nullable: true })
+  children?: IndexUnitEntity[];
+
+  @Column("simple-array", { nullable: true })
+  embedding?: number[] | null;
+}

package/src/extractors/LanguageExtractor.ts

@@ -0,0 +1,9 @@
+import { IndexUnit } from "../types";
+
+export interface LanguageExtractor {
+  readonly id: string;
+  readonly exts: string[];
+  supports(filePath: string): boolean;
+  extractFromText(filePath: string, source: string): Promise<IndexUnit[]>;
+  unitLabel(unit: IndexUnit): string | null;
+}