@lzehrung/codegraph 2.1.0

package/dist/duplicates/unitCache.js

@@ -0,0 +1,352 @@
+import fs from "node:fs";
+import path from "node:path";
+import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from "node:zlib";
+import { getNativeDuplicateTokens, getNativeTreeSitterSupportedLanguageIds, isNativeDuplicateTokenizationAvailable, isNativeTreeSitterDisabledByEnv, } from "../native/treeSitterNative.js";
+import { SqliteDatabase } from "../sqlite-driver.js";
+import { logWithLevel } from "../logging.js";
+import { cacheDatabasePath } from "../indexer/build-cache/module-cache.js";
+import { createSqliteTableIfMissing, ensureSqliteVersionedTableSchema, recreateSqliteTable, sqliteTableColumns, } from "../util/sqliteSchema.js";
+import { lruMapGet } from "../util/lruMap.js";
+import { fileIdentityKey, normalizePath } from "../util/paths.js";
+// v3: drop dead `text`/`normalizedTokens` fields (never read after construction) and
+// brotli-compress the payload; both cut on-disk cache size by roughly 8x.
+export const DUPLICATE_UNIT_CACHE_VERSION = 3;
+export const DUPLICATE_UNIT_CACHE_SCHEMA_VERSION = 1;
+export const DUPLICATE_UNIT_CACHE_TABLE = "duplicate_unit_cache";
+export const DUPLICATE_UNIT_CACHE_SCHEMA_VERSION_KEY = "duplicate_unit_cache.schema_version";
+export const DUPLICATE_TOKENIZER_REVISION = 2;
+export const DUPLICATE_UNIT_CACHE_MAX_ROWS = 5_000;
+export const DUPLICATE_UNIT_CACHE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1_000;
+export const DUPLICATE_UNIT_MEMORY_CACHE_MAX_ENTRIES = 2_048;
+export const DUPLICATE_UNIT_MEMORY_CACHE_MAX_BYTES = 64 * 1024 * 1024;
+export const DUPLICATE_UNIT_CACHE_COLUMNS = [
+    { name: "file", definition: "TEXT NOT NULL" },
+    { name: "variant", definition: "TEXT NOT NULL" },
+    { name: "sig", definition: "TEXT NOT NULL" },
+    { name: "version", definition: "INTEGER NOT NULL" },
+    { name: "payload", definition: "BLOB NOT NULL" },
+    { name: "updated_at", definition: "INTEGER NOT NULL" },
+];
+export const duplicateUnitMemoryCache = new Map();
+export const duplicateUnitDiskDatabases = new Map();
+let duplicateUnitMemoryCacheBytes = 0;
+function estimateDuplicateUnitCacheEntryBytes(entry) {
+    let bytes = entry.sig.length * 2 + 64;
+    for (const unit of entry.units) {
+        bytes += unit.id.length * 2 + unit.file.length * 2 + unit.absoluteFile.length * 2 + 128;
+        bytes += unit.tokenSet.size * 24 + unit.signatures.size * 24;
+        bytes += unit.tokenCount * 4;
+    }
+    return bytes;
+}
+export function clearDuplicateUnitMemoryCache() {
+    duplicateUnitMemoryCache.clear();
+    duplicateUnitMemoryCacheBytes = 0;
+}
+export function clearDuplicateUnitMemoryCacheForRoot(projectRoot) {
+    const rootPrefix = normalizePath(projectRoot).replace(/\/$/, "");
+    const rootKey = fileIdentityKey(rootPrefix);
+    for (const key of [...duplicateUnitMemoryCache.keys()]) {
+        const file = key.split("\u0000")[0] ?? "";
+        const normalizedFile = normalizePath(file);
+        if (fileIdentityKey(normalizedFile) === rootKey ||
+            normalizedFile === rootPrefix ||
+            normalizedFile.startsWith(`${rootPrefix}/`)) {
+            const entry = duplicateUnitMemoryCache.get(key);
+            if (entry)
+                duplicateUnitMemoryCacheBytes = Math.max(0, duplicateUnitMemoryCacheBytes - estimateDuplicateUnitCacheEntryBytes(entry));
+            duplicateUnitMemoryCache.delete(key);
+        }
+    }
+}
+function readDuplicateUnitMemoryCache(key) {
+    return lruMapGet(duplicateUnitMemoryCache, key);
+}
+function writeDuplicateUnitMemoryCache(key, entry) {
+    const existing = duplicateUnitMemoryCache.get(key);
+    if (existing) {
+        duplicateUnitMemoryCacheBytes = Math.max(0, duplicateUnitMemoryCacheBytes - estimateDuplicateUnitCacheEntryBytes(existing));
+        duplicateUnitMemoryCache.delete(key);
+    }
+    const entryBytes = estimateDuplicateUnitCacheEntryBytes(entry);
+    while (duplicateUnitMemoryCache.size > 0 &&
+        (duplicateUnitMemoryCache.size >= DUPLICATE_UNIT_MEMORY_CACHE_MAX_ENTRIES ||
+            duplicateUnitMemoryCacheBytes + entryBytes > DUPLICATE_UNIT_MEMORY_CACHE_MAX_BYTES)) {
+        const oldestKey = duplicateUnitMemoryCache.keys().next().value;
+        if (oldestKey === undefined)
+            break;
+        const oldest = duplicateUnitMemoryCache.get(oldestKey);
+        duplicateUnitMemoryCache.delete(oldestKey);
+        if (oldest) {
+            duplicateUnitMemoryCacheBytes = Math.max(0, duplicateUnitMemoryCacheBytes - estimateDuplicateUnitCacheEntryBytes(oldest));
+        }
+    }
+    duplicateUnitMemoryCache.set(key, entry);
+    duplicateUnitMemoryCacheBytes += entryBytes;
+}
+export function duplicateUnitCacheVariant(index, minTokens, maxTokens, shingleSize, windowSize) {
+    const nativeMode = normalizedDuplicateUnitCacheNativeMode(index.nativeMode);
+    return JSON.stringify({
+        version: DUPLICATE_UNIT_CACHE_VERSION,
+        tokenizerRevision: DUPLICATE_TOKENIZER_REVISION,
+        nativeMode,
+        nativeDuplicateTokens: isNativeDuplicateTokenizationAvailable(index.nativeMode),
+        nativeSyntaxLanguages: getNativeTreeSitterSupportedLanguageIds(index.nativeMode),
+        nativeEnvDisabled: nativeMode === undefined ? isNativeTreeSitterDisabledByEnv() : undefined,
+        minTokens,
+        maxTokens,
+        shingleSize,
+        windowSize,
+    });
+}
+export function normalizedDuplicateUnitCacheNativeMode(nativeMode) {
+    if (nativeMode === undefined || nativeMode === "auto")
+        return undefined;
+    return nativeMode;
+}
+export function duplicateUnitCacheSignature(index, file) {
+    const entry = index.manifestEntries?.get(file);
+    return entry?.gitSig ?? entry?.sig;
+}
+export function duplicateUnitCacheKey(file, variant) {
+    return `${file}\u0000${variant}`;
+}
+export function duplicateUnitCacheDatabasePath(projectRoot, opts) {
+    return cacheDatabasePath(projectRoot, opts, "duplicate-unit-cache.sqlite");
+}
+function duplicateUnitCacheDatabasePathForRoot(cacheRootDir) {
+    return path.join(cacheRootDir, "duplicate-unit-cache.sqlite").replace(/\\/g, "/");
+}
+export function createDuplicateUnitCacheTable(db) {
+    createSqliteTableIfMissing(db, DUPLICATE_UNIT_CACHE_TABLE, DUPLICATE_UNIT_CACHE_COLUMNS, [
+        "PRIMARY KEY (file, variant)",
+    ]);
+}
+export function recreateDuplicateUnitCacheTable(db) {
+    recreateSqliteTable(db, DUPLICATE_UNIT_CACHE_TABLE, createDuplicateUnitCacheTable);
+}
+export function migrateDuplicateUnitCacheTable(db) {
+    const columns = sqliteTableColumns(db, DUPLICATE_UNIT_CACHE_TABLE);
+    if (!columns.size) {
+        createDuplicateUnitCacheTable(db);
+        return;
+    }
+    if (!columns.has("file") || !columns.has("variant")) {
+        recreateDuplicateUnitCacheTable(db);
+        return;
+    }
+    if (!columns.has("sig"))
+        db.exec("ALTER TABLE duplicate_unit_cache ADD COLUMN sig TEXT NOT NULL DEFAULT '';");
+    if (!columns.has("version"))
+        db.exec("ALTER TABLE duplicate_unit_cache ADD COLUMN version INTEGER NOT NULL DEFAULT 0;");
+    if (!columns.has("payload"))
+        db.exec("ALTER TABLE duplicate_unit_cache ADD COLUMN payload BLOB NOT NULL DEFAULT (x'');");
+    if (!columns.has("updated_at")) {
+        db.exec("ALTER TABLE duplicate_unit_cache ADD COLUMN updated_at INTEGER NOT NULL DEFAULT 0;");
+    }
+}
+export function ensureDuplicateUnitCacheSchema(db) {
+    ensureSqliteVersionedTableSchema({
+        db,
+        tableName: DUPLICATE_UNIT_CACHE_TABLE,
+        schemaVersionKey: DUPLICATE_UNIT_CACHE_SCHEMA_VERSION_KEY,
+        schemaVersion: DUPLICATE_UNIT_CACHE_SCHEMA_VERSION,
+        createTable: createDuplicateUnitCacheTable,
+        migrateTable: migrateDuplicateUnitCacheTable,
+    });
+}
+export function createDuplicateUnitDiskStatements(db) {
+    return {
+        load: db.prepare("SELECT sig, version, payload FROM duplicate_unit_cache WHERE file = ? AND variant = ?"),
+        write: db.prepare(`INSERT INTO duplicate_unit_cache (file, variant, sig, version, payload, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?)
+       ON CONFLICT(file, variant) DO UPDATE SET
+         sig = excluded.sig,
+         version = excluded.version,
+         payload = excluded.payload,
+         updated_at = excluded.updated_at`),
+        pruneExpired: db.prepare("DELETE FROM duplicate_unit_cache WHERE updated_at < ?"),
+        pruneOverflow: db.prepare("DELETE FROM duplicate_unit_cache WHERE rowid IN (SELECT rowid FROM duplicate_unit_cache ORDER BY updated_at DESC LIMIT -1 OFFSET ?)"),
+    };
+}
+export function maintainDuplicateUnitDiskCache(index) {
+    if (index.cacheMode !== "disk" || !index.cacheRootDir)
+        return;
+    const dbPath = duplicateUnitCacheDatabasePathForRoot(index.cacheRootDir);
+    const entry = duplicateUnitDiskDatabases.get(dbPath);
+    if (!entry?.db || !entry.statements)
+        return;
+    const { db, statements } = entry;
+    try {
+        db.transaction(() => {
+            statements.pruneExpired.run(Date.now() - DUPLICATE_UNIT_CACHE_MAX_AGE_MS);
+            statements.pruneOverflow.run(DUPLICATE_UNIT_CACHE_MAX_ROWS);
+        })();
+    }
+    catch (error) {
+        logWithLevel(undefined, "warn", `Warning: Failed to prune duplicate cache ${dbPath}:`, error);
+    }
+}
+export function duplicateUnitDiskCache(index) {
+    if (index.cacheMode !== "disk" || !index.cacheRootDir)
+        return null;
+    const dbPath = duplicateUnitCacheDatabasePathForRoot(index.cacheRootDir);
+    let entry = duplicateUnitDiskDatabases.get(dbPath);
+    if (!entry) {
+        entry = { leases: 0, closeRequested: false };
+        duplicateUnitDiskDatabases.set(dbPath, entry);
+    }
+    if (!entry.db) {
+        fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+        const db = new SqliteDatabase(dbPath);
+        db.pragma("journal_mode = WAL");
+        db.pragma("synchronous = NORMAL");
+        ensureDuplicateUnitCacheSchema(db);
+        entry.db = db;
+        entry.statements = createDuplicateUnitDiskStatements(db);
+    }
+    return entry;
+}
+export function closeDuplicateUnitDiskDatabaseEntry(dbPath, entry) {
+    if (entry.leases > 0) {
+        entry.closeRequested = true;
+        return;
+    }
+    if (!entry.db) {
+        duplicateUnitDiskDatabases.delete(dbPath);
+        return;
+    }
+    try {
+        entry.db.pragma("wal_checkpoint(TRUNCATE)");
+    }
+    catch {
+        // checkpoint best-effort
+    }
+    try {
+        entry.db.close();
+        duplicateUnitDiskDatabases.delete(dbPath);
+    }
+    catch {
+        // Keep handle for later retry if close fails.
+    }
+}
+export function closeDuplicateUnitCacheDatabase(projectRoot, opts) {
+    clearDuplicateUnitMemoryCacheForRoot(projectRoot);
+    const dbPath = duplicateUnitCacheDatabasePath(projectRoot, opts);
+    const entry = duplicateUnitDiskDatabases.get(dbPath);
+    if (!entry) {
+        return;
+    }
+    closeDuplicateUnitDiskDatabaseEntry(dbPath, entry);
+}
+export function leaseDuplicateUnitCacheForIndex(index) {
+    if (index.cacheMode !== "disk" || !index.cacheRootDir)
+        return () => { };
+    const dbPath = duplicateUnitCacheDatabasePathForRoot(index.cacheRootDir);
+    let entry = duplicateUnitDiskDatabases.get(dbPath);
+    if (!entry) {
+        entry = { leases: 0, closeRequested: false };
+        duplicateUnitDiskDatabases.set(dbPath, entry);
+    }
+    entry.leases++;
+    return () => {
+        entry.leases = Math.max(0, entry.leases - 1);
+        if (entry.closeRequested) {
+            closeDuplicateUnitDiskDatabaseEntry(dbPath, entry);
+        }
+    };
+}
+export function closeDuplicateUnitCacheForIndex(index) {
+    if (index.cacheMode !== "disk" || !index.cacheRootDir)
+        return;
+    const dbPath = duplicateUnitCacheDatabasePathForRoot(index.cacheRootDir);
+    const entry = duplicateUnitDiskDatabases.get(dbPath);
+    if (!entry)
+        return;
+    closeDuplicateUnitDiskDatabaseEntry(dbPath, entry);
+}
+export function tryLoadDuplicateUnitsFromCache(index, file, variant) {
+    const sig = duplicateUnitCacheSignature(index, file);
+    if (!sig)
+        return null;
+    const key = duplicateUnitCacheKey(file, variant);
+    if (index.cacheMode === "memory") {
+        const entry = readDuplicateUnitMemoryCache(key);
+        if (entry && entry.sig === sig)
+            return entry.units;
+        return null;
+    }
+    if (index.cacheMode === "disk") {
+        try {
+            const entry = duplicateUnitDiskCache(index);
+            const row = entry?.statements?.load.get(file, variant);
+            if (!row || row.sig !== sig || row.version !== DUPLICATE_UNIT_CACHE_VERSION)
+                return null;
+            const parsed = JSON.parse(brotliDecompressSync(row.payload).toString("utf8"));
+            return deserializeDuplicateUnits(parsed);
+        }
+        catch {
+            return null;
+        }
+    }
+    return null;
+}
+export function writeDuplicateUnitsToCache(index, file, variant, units) {
+    const sig = duplicateUnitCacheSignature(index, file);
+    if (!sig)
+        return;
+    const key = duplicateUnitCacheKey(file, variant);
+    if (index.cacheMode === "memory") {
+        writeDuplicateUnitMemoryCache(key, { sig, units });
+        return;
+    }
+    if (index.cacheMode === "disk") {
+        try {
+            const entry = duplicateUnitDiskCache(index);
+            const payload = brotliCompressSync(JSON.stringify(serializeDuplicateUnits(units)), {
+                params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 },
+            });
+            entry?.statements?.write.run(file, variant, sig, DUPLICATE_UNIT_CACHE_VERSION, payload, Date.now());
+        }
+        catch {
+            // best-effort cache
+        }
+    }
+}
+export function deserializeDuplicateUnits(value) {
+    if (!Array.isArray(value) || !value.every(isDuplicateSerializedUnit))
+        return null;
+    return value.map((unit) => ({
+        ...unit,
+        tokenSet: new Set(unit.tokenSet),
+        signatures: new Set(unit.signatures),
+    }));
+}
+export function serializeDuplicateUnits(units) {
+    return units.map((unit) => ({
+        ...unit,
+        tokenSet: [...unit.tokenSet],
+        signatures: [...unit.signatures],
+    }));
+}
+export function isDuplicateSerializedUnit(value) {
+    if (!value || typeof value !== "object")
+        return false;
+    const unit = value;
+    return (typeof unit.id === "string" &&
+        typeof unit.file === "string" &&
+        typeof unit.absoluteFile === "string" &&
+        typeof unit.rawHash === "string" &&
+        typeof unit.normalizedHash === "string" &&
+        Array.isArray(unit.tokenSet) &&
+        Array.isArray(unit.signatures) &&
+        typeof unit.startLine === "number" &&
+        typeof unit.endLine === "number" &&
+        typeof unit.languageId === "string" &&
+        (unit.kind === "symbol" || unit.kind === "chunk") &&
+        typeof unit.tokenCount === "number" &&
+        typeof unit.handle === "string" &&
+        typeof unit.fileHandle === "string" &&
+        typeof unit.chunkHandle === "string");
+}
+//# sourceMappingURL=unitCache.js.map

package/dist/duplicates/unitCache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unitCache.js","sourceRoot":"","sources":["../../src/duplicates/unitCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,WAAW,CAAC;AAEjG,OAAO,EACL,wBAAwB,EACxB,uCAAuC,EACvC,sCAAsC,EACtC,+BAA+B,GAChC,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,cAAc,EAAwB,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,wCAAwC,CAAC;AAC3E,OAAO,EACL,0BAA0B,EAC1B,gCAAgC,EAChC,mBAAmB,EACnB,kBAAkB,GAEnB,MAAM,yBAAyB,CAAC;AASjC,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAElE,qFAAqF;AACrF,0EAA0E;AAC1E,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAC9C,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AACrD,MAAM,CAAC,MAAM,0BAA0B,GAAG,sBAAsB,CAAC;AACjE,MAAM,CAAC,MAAM,uCAAuC,GAAG,qCAAqC,CAAC;AAC7F,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAC9C,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,CAAC;AACnD,MAAM,CAAC,MAAM,+BAA+B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC;AACzE,MAAM,CAAC,MAAM,uCAAuC,GAAG,KAAK,CAAC;AAC7D,MAAM,CAAC,MAAM,qCAAqC,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACtE,MAAM,CAAC,MAAM,4BAA4B,GAAiC;IACxE,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE;IAC7C,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE;IAChD,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE;IAC5C,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE;IACnD,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE;IAChD,EAAE,IAAI,EAAE,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE;CACvD,CAAC;AAEF,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAmC,CAAC;AACnF,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,GAAG,EAA0C,CAAC;AAE5F,IAAI,6BAA6B,GAAG,CAAC,CAAC;AAEtC,SAAS,oCAAoC,CAAC,KAA8B;IAC1E,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,KAAK,IAAI,IAAI,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC;QACxF,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,EAAE,CAAC;QAC7D,KAAK,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,6BAA6B;IAC3C,wBAAwB,CAAC,KAAK,EAAE,CAAC;IACjC,6BAA6B,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,oCAAoC,CAAC,WAAmB;IACtE,MAAM,UAAU,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5C,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAC3C,IACE,eAAe,CAAC,cAAc,CAAC,KAAK,OAAO;YAC3C,cAAc,KAAK,UAAU;YAC7B,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU,GAAG,CAAC,EAC3C,CAAC;YACD,MAAM,KAAK,GAAG,wBAAwB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,KAAK;gBACP,6BAA6B,GAAG,IAAI,CAAC,GAAG,CACtC,CAAC,EACD,6BAA6B,GAAG,oCAAoC,CAAC,KAAK,CAAC,CAC5E,CAAC;YACJ,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,4BAA4B,CAAC,GAAW;IAC/C,OAAO,SAAS,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,6BAA6B,CAAC,GAAW,EAAE,KAA8B;IAChF,MAAM,QAAQ,GAAG,wBAAwB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,QAAQ,EAAE,CAAC;QACb,6BAA6B,GAAG,IAAI,CAAC,GAAG,CACtC,CAAC,EACD,6BAA6B,GAAG,oCAAoC,CAAC,QAAQ,CAAC,CAC/E,CAAC;QACF,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,UAAU,GAAG,oCAAoC,CAAC,KAAK,CAAC,CAAC;IAC/D,OACE,wBAAwB,CAAC,IAAI,GAAG,CAAC;QACjC,CAAC,wBAAwB,CAAC,IAAI,IAAI,uCAAuC;YACvE,6BAA6B,GAAG,UAAU,GAAG,qCAAqC,CAAC,EACrF,CAAC;QACD,MAAM,SAAS,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAA2B,CAAC;QACrF,IAAI,SAAS,KAAK,SAAS;YAAE,MAAM;QACnC,MAAM,MAAM,GAAG,wBAAwB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvD,wBAAwB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,MAAM,EAAE,CAAC;YACX,6BAA6B,GAAG,IAAI,CAAC,GAAG,CACtC,CAAC,EACD,6BAA6B,GAAG,oCAAoC,CAAC,MAAM,CAAC,CAC7E,CAAC;QACJ,CAAC;IACH,CAAC;IACD,wBAAwB,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzC,6BAA6B,IAAI,UAAU,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,KAAmB,EACnB,SAAiB,EACjB,SAAiB,EACjB,WAAmB,EACnB,UAAkB;IAElB,MAAM,UAAU,GAAG,sCAAsC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5E,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO,EAAE,4BAA4B;QACrC,iBAAiB,EAAE,4BAA4B;QAC/C,UAAU;QACV,qBAAqB,EAAE,sCAAsC,CAAC,KAAK,CAAC,UAAU,CAAC;QAC/E,qBAAqB,EAAE,uCAAuC,CAAC,KAAK,CAAC,UAAU,CAAC;QAChF,iBAAiB,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC,SAAS;QAC3F,SAAS;QACT,SAAS;QACT,WAAW;QACX,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,UAAkD;IAElD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC;IACxE,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,KAAmB,EAAE,IAAY;IAC3E,MAAM,KAAK,GAAG,KAAK,CAAC,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,KAAK,EAAE,MAAM,IAAI,KAAK,EAAE,GAAG,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAe;IACjE,OAAO,GAAG,IAAI,SAAS,OAAO,EAAE,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,WAAmB,EAAE,IAAmB;IACrF,OAAO,iBAAiB,CAAC,WAAW,EAAE,IAAI,EAAE,6BAA6B,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,qCAAqC,CAAC,YAAoB;IACjE,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,6BAA6B,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,EAAkB;IAC9D,0BAA0B,CAAC,EAAE,EAAE,0BAA0B,EAAE,4BAA4B,EAAE;QACvF,6BAA6B;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,EAAkB;IAChE,mBAAmB,CAAC,EAAE,EAAE,0BAA0B,EAAE,6BAA6B,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,EAAkB;IAC/D,MAAM,OAAO,GAAG,kBAAkB,CAAC,EAAE,EAAE,0BAA0B,CAAC,CAAC;IACnE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,6BAA6B,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACpD,+BAA+B,CAAC,EAAE,CAAC,CAAC;QACpC,OAAO;IACT,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,EAAE,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;IAC9G,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QACzB,EAAE,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;IAC7F,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QACzB,EAAE,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;IAC9F,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IAChG,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,EAAkB;IAC/D,gCAAgC,CAAC;QAC/B,EAAE;QACF,SAAS,EAAE,0BAA0B;QACrC,gBAAgB,EAAE,uCAAuC;QACzD,aAAa,EAAE,mCAAmC;QAClD,WAAW,EAAE,6BAA6B;QAC1C,YAAY,EAAE,8BAA8B;KAC7C,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,EAAkB;IAClE,OAAO;QACL,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,uFAAuF,CAAC;QACzG,KAAK,EAAE,EAAE,CAAC,OAAO,CACf;;;;;;0CAMoC,CACrC;QACD,YAAY,EAAE,EAAE,CAAC,OAAO,CAAC,uDAAuD,CAAC;QACjF,aAAa,EAAE,EAAE,CAAC,OAAO,CACvB,qIAAqI,CACtI;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,KAAmB;IAChE,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO;IAC9D,MAAM,MAAM,GAAG,qCAAqC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU;QAAE,OAAO;IAC5C,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;IACjC,IAAI,CAAC;QACH,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,+BAA+B,CAAC,CAAC;YAC1E,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC9D,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,4CAA4C,MAAM,GAAG,EAAE,KAAK,CAAC,CAAC;IAChG,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAmB;IACxD,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IACnE,MAAM,MAAM,GAAG,qCAAqC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACzE,IAAI,KAAK,GAAG,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAC7C,0BAA0B,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACd,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,MAAM,EAAE,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACtC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAChC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAClC,8BAA8B,CAAC,EAAE,CAAC,CAAC;QACnC,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC;QACd,KAAK,CAAC,UAAU,GAAG,iCAAiC,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,mCAAmC,CAAC,MAAc,EAAE,KAAqC;IACvG,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC;QAC5B,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QACd,0BAA0B,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1C,OAAO;IACT,CAAC;IACD,IAAI,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,yBAAyB;IAC3B,CAAC;IACD,IAAI,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACjB,0BAA0B,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,8CAA8C;IAChD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,WAAmB,EAAE,IAAmB;IACtF,oCAAoC,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,8BAA8B,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;IACT,CAAC;IACD,mCAAmC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,KAAmB;IACjE,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,qCAAqC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACzE,IAAI,KAAK,GAAG,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QAC7C,0BAA0B,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAChD,CAAC;IACD,KAAK,CAAC,MAAM,EAAE,CAAC;IACf,OAAO,GAAG,EAAE;QACV,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7C,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC;YACzB,mCAAmC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,KAAmB;IACjE,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,YAAY;QAAE,OAAO;IAC9D,MAAM,MAAM,GAAG,qCAAqC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACrD,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,mCAAmC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,KAAmB,EACnB,IAAY,EACZ,OAAe;IAEf,MAAM,GAAG,GAAG,2BAA2B,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,GAAG,GAAG,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,IAAI,KAAK,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,KAAK,IAAI,KAAK,CAAC,GAAG,KAAK,GAAG;YAAE,OAAO,KAAK,CAAC,KAAK,CAAC;QACnD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,GAAG,GAAG,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CACc,CAAC;YACpE,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,KAAK,4BAA4B;gBAAE,OAAO,IAAI,CAAC;YACzF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAY,CAAC;YACzF,OAAO,yBAAyB,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAmB,EACnB,IAAY,EACZ,OAAe,EACf,KAA8B;IAE9B,MAAM,GAAG,GAAG,2BAA2B,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,GAAG,GAAG,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,IAAI,KAAK,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACjC,6BAA6B,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,KAAK,MAAM,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE;gBACjF,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,oBAAoB,CAAC,EAAE,CAAC,EAAE;aACpD,CAAC,CAAC;YACH,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,4BAA4B,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACtG,CAAC;QAAC,MAAM,CAAC;YACP,oBAAoB;QACtB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAc;IACtD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,yBAAyB,CAAC;QAAE,OAAO,IAAI,CAAC;IAClF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,GAAG,IAAI;QACP,QAAQ,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;QAChC,UAAU,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC;KACrC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAA8B;IACpE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,GAAG,IAAI;QACP,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,KAAc;IACtD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,MAAM,IAAI,GAAG,KAAyC,CAAC;IACvD,OAAO,CACL,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ;QAC3B,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;QAC7B,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ;QACrC,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAChC,OAAO,IAAI,CAAC,cAAc,KAAK,QAAQ;QACvC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC5B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ;QAClC,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAChC,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;QACnC,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC;QACjD,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;QACnC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ;QAC/B,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;QACnC,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CACrC,CAAC;AACJ,CAAC"}

package/dist/duplicates/units.d.ts

@@ -0,0 +1,60 @@
+import { type Chunk } from "../chunking/chunkFile.js";
+import type { ParsedFileContext } from "../indexer/parse-context.js";
+import { SymbolKind, type ProjectIndex, type SymbolDef } from "../indexer/types.js";
+import type { SyntaxNodeLike } from "../languages/types.js";
+import type { CollectedDuplicateUnits, DuplicateAstContext, DuplicateAstContextCache, DuplicateInternalUnit, DuplicateUnitCollectionOptions, DuplicateUnitDraft, DuplicateUnitRef, LanguageForFileResult } from "./types.js";
+export declare const DEFAULT_MAX_FINGERPRINTS = 128;
+export declare function lineSpan(unit: Pick<DuplicateUnitRef, "startLine" | "endLine">): number;
+export declare const textLanguageByExtension: Record<string, string>;
+export declare const chunkLanguageAliases: Record<string, string>;
+export declare const symbolUnitKinds: Set<SymbolKind>;
+export declare function hashText(value: string): string;
+export declare function shortHashText(value: string): string;
+/** Builds hashed token windows used as local structural fingerprints. */
+export declare function makeShingles(tokens: readonly string[], size: number): string[];
+/** Keeps stable representative fingerprints from nearby shingle windows. */
+export declare function winnowShingles(shingles: readonly string[], windowSize: number, maxFingerprints: number): Set<string>;
+export declare function languageForFile(filePath: string): LanguageForFileResult | undefined;
+export declare function formatDuplicateFileHandle(file: string): string;
+export declare function formatDuplicateChunkHandle(file: string, line: number): string;
+export declare function formatDuplicateSqlHandle(file: string, name: string, line: number): string;
+export declare function sqlHandleForDuplicateSymbol(symbol: SymbolDef, file: string): string | undefined;
+export declare function formatDuplicateSymbolHandle(file: string, name: string, line: number, column: number): string;
+export declare function normalizeDetectionFile(filePath: string, projectRoot: string | undefined): string;
+export declare function internalUnitId(unit: DuplicateUnitDraft, absoluteFile: string): string;
+export declare function normalizedDuplicateTokens(text: string, nativeMode: ProjectIndex["nativeMode"] | undefined): string[];
+export declare function duplicateTextLines(text: string): string[];
+export declare function maskedDuplicateStatements(text: string): string[];
+export declare function looksLikeImportStatement(statement: string): boolean;
+export declare function looksLikeBarrelStatement(statement: string): boolean;
+export declare function duplicateTextHeuristics(text: string): {
+    looksLikeImportList: boolean;
+    looksLikeBarrel: boolean;
+};
+/** Adds hashes, normalized tokens, and fingerprints to a reportable unit. */
+export declare function buildInternalUnit(unit: DuplicateUnitDraft, absoluteFile: string, text: string, shingleSize: number, windowSize: number, nativeMode: ProjectIndex["nativeMode"] | undefined, handles?: {
+    symbolHandle?: string;
+    sqlHandle?: string;
+}, astShapeHash?: string): DuplicateInternalUnit;
+export declare function makeSymbolUnit(symbol: SymbolDef, chunk: Chunk, projectRoot: string | undefined, nativeMode: ProjectIndex["nativeMode"] | undefined, shingleSize: number, windowSize: number, astContext: DuplicateAstContext | undefined): DuplicateInternalUnit | undefined;
+export declare function makeDuplicateChunks(filePath: string, languageId: string, textOnly: boolean, source: string, minTokens: number, maxTokens: number): Chunk[];
+export declare function makeSymbolSourceChunks(filePath: string, languageId: string, textOnly: boolean, source: string, maxTokens: number): Chunk[];
+export declare function findChunkForSymbol(symbol: SymbolDef, chunks: readonly Chunk[]): Chunk | undefined;
+export declare function getDuplicateAstContext(index: ProjectIndex, file: string, source: string, cache: DuplicateAstContextCache): Promise<DuplicateAstContext | undefined>;
+export declare function astContextFromParsed(parsed: ParsedFileContext): DuplicateAstContext;
+export declare function astShapeHashForRange(context: DuplicateAstContext | undefined, startLine: number, endLine: number): string | undefined;
+export declare function lineRangeToByteRange(context: DuplicateAstContext, startLine: number, endLine: number): {
+    start: number;
+    end: number;
+} | undefined;
+export declare function findSmallestCoveringNode(node: SyntaxNodeLike, startIndex: number, endIndex: number): SyntaxNodeLike | null;
+export declare function normalizedAstShape(root: SyntaxNodeLike): string;
+export declare function isAstShapeIgnoredNode(node: SyntaxNodeLike): boolean;
+export declare function normalizeAstNodeType(type: string): string;
+/** Falls back to semantic chunks so body-level clones are still visible. */
+export declare function makeChunkUnits(filePath: string, chunks: readonly Chunk[], projectRoot: string | undefined, nativeMode: ProjectIndex["nativeMode"] | undefined, shingleSize: number, windowSize: number, astContext: DuplicateAstContext | undefined): DuplicateInternalUnit[];
+export declare function shouldKeepUnit(unit: DuplicateInternalUnit, includeSmall: boolean, minTokens: number): boolean;
+/** Reads files and creates comparable symbol and chunk units. */
+export declare function collectDuplicateUnits(index: ProjectIndex, options: DuplicateUnitCollectionOptions): Promise<CollectedDuplicateUnits>;
+export declare function buildDuplicateUnitsForFile(index: ProjectIndex, file: string, projectRoot: string | undefined, minTokens: number, maxTokens: number, shingleSize: number, windowSize: number, astContextCache: DuplicateAstContextCache): Promise<DuplicateInternalUnit[]>;
+//# sourceMappingURL=units.d.ts.map

package/dist/duplicates/units.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"units.d.ts","sourceRoot":"","sources":["../../src/duplicates/units.ts"],"names":[],"mappings":"AAIA,OAAO,EAAa,KAAK,KAAK,EAAE,MAAM,0BAA0B,CAAC;AAQjE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAQpF,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,uBAAuB,CAAC;AAM5E,OAAO,KAAK,EACV,uBAAuB,EACvB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,8BAA8B,EAC9B,kBAAkB,EAClB,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAC5C,wBAAgB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,gBAAgB,EAAE,WAAW,GAAG,SAAS,CAAC,GAAG,MAAM,CAEtF;AAED,eAAO,MAAM,uBAAuB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ1D,CAAC;AAEF,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAGvD,CAAC;AAEF,eAAO,MAAM,eAAe,iBAQ1B,CAAC;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE9C;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,yEAAyE;AACzE,wBAAgB,YAAY,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAO9E;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAepH;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,qBAAqB,GAAG,SAAS,CAUnF;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE9D;AAED,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAE7E;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEzF;AAED,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAM/F;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAE5G;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAGhG;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAErF;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,SAAS,GAAG,MAAM,EAAE,CAKpH;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAKzD;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAMhE;AAED,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAQnE;AAED,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAKnE;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,mBAAmB,EAAE,OAAO,CAAC;IAAC,eAAe,EAAE,OAAO,CAAA;CAAE,CAQhH;AAED,6EAA6E;AAC7E,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,kBAAkB,EACxB,YAAY,EAAE,MAAM,EACpB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,SAAS,EAClD,OAAO,GAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,EAC3D,YAAY,CAAC,EAAE,MAAM,GACpB,qBAAqB,CA0BvB;AAED,wBAAgB,cAAc,CAC5B,MAAM,EAAE,SAAS,EACjB,KAAK,EAAE,KAAK,EACZ,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,SAAS,EAClD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,mBAAmB,GAAG,SAAS,GAC1C,qBAAqB,GAAG,SAAS,CA+BnC;AAED,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,OAAO,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,KAAK,EAAE,CAMT;AAED,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,OAAO,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,KAAK,EAAE,CAGT;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,KAAK,EAAE,GAAG,KAAK,GAAG,SAAS,CAWjG;AAED,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,mBAAmB,GAAG,SAAS,CAAC,CA+B1C;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,iBAAiB,GAAG,mBAAmB,CAMnF;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,mBAAmB,GAAG,SAAS,EACxC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,MAAM,GAAG,SAAS,CASpB;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,mBAAmB,EAC5B,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAgB5C;AAED,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,cAAc,EACpB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,cAAc,GAAG,IAAI,CAQvB;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,cAAc,GAAG,MAAM,CAe/D;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAGnE;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASzD;AAED,4EAA4E;AAC5E,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,SAAS,KAAK,EAAE,EACxB,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,GAAG,SAAS,EAClD,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,mBAAmB,GAAG,SAAS,GAC1C,qBAAqB,EAAE,CAsBzB;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,qBAAqB,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAG7G;AAED,iEAAiE;AACjE,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,8BAA8B,GACtC,OAAO,CAAC,uBAAuB,CAAC,CAuDlC;AAED,wBAAsB,0BAA0B,CAC9C,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,wBAAwB,GACxC,OAAO,CAAC,qBAAqB,EAAE,CAAC,CA0BlC"}