omp-plugin-duplicate-detector 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detector-worker.js +147 -129
- package/package.json +1 -1
- package/src/detector-worker.ts +22 -3
- package/src/disk-cache.ts +19 -14
- package/src/duplicate-ledger.ts +215 -4
- package/src/index.ts +127 -13
- package/src/source-aware-index.ts +6 -1
- package/src/tui-notification.ts +8 -2
- package/src/worker-protocol.ts +1 -0
package/dist/detector-worker.js
CHANGED
|
@@ -977,11 +977,11 @@ import * as path5 from "path";
|
|
|
977
977
|
|
|
978
978
|
// src/disk-cache.ts
|
|
979
979
|
import { Database } from "bun:sqlite";
|
|
980
|
-
import * as
|
|
981
|
-
import * as
|
|
982
|
-
import * as
|
|
980
|
+
import * as crypto2 from "crypto";
|
|
981
|
+
import * as fsSync2 from "fs";
|
|
982
|
+
import * as fs2 from "fs/promises";
|
|
983
983
|
import * as os from "os";
|
|
984
|
-
import * as
|
|
984
|
+
import * as path4 from "path";
|
|
985
985
|
import * as zlib from "zlib";
|
|
986
986
|
|
|
987
987
|
// node_modules/eventemitter3/index.mjs
|
|
@@ -12292,6 +12292,95 @@ async function getTrackedGitFiles(rootDir, options = {}) {
|
|
|
12292
12292
|
}
|
|
12293
12293
|
}
|
|
12294
12294
|
|
|
12295
|
+
// src/repo-context.ts
|
|
12296
|
+
import * as crypto from "crypto";
|
|
12297
|
+
import * as fsSync from "fs";
|
|
12298
|
+
import * as fs from "fs/promises";
|
|
12299
|
+
import * as path3 from "path";
|
|
12300
|
+
function canonicalizePath(targetPath) {
|
|
12301
|
+
const resolved = path3.resolve(targetPath);
|
|
12302
|
+
let current = resolved;
|
|
12303
|
+
let suffix = "";
|
|
12304
|
+
while (current && current !== path3.dirname(current)) {
|
|
12305
|
+
try {
|
|
12306
|
+
if (fsSync.existsSync(current)) {
|
|
12307
|
+
const real = fsSync.realpathSync(current);
|
|
12308
|
+
return suffix ? path3.join(real, suffix) : real;
|
|
12309
|
+
}
|
|
12310
|
+
} catch {}
|
|
12311
|
+
suffix = suffix ? path3.join(path3.basename(current), suffix) : path3.basename(current);
|
|
12312
|
+
current = path3.dirname(current);
|
|
12313
|
+
}
|
|
12314
|
+
return resolved;
|
|
12315
|
+
}
|
|
12316
|
+
function isOmpWorktreePath(targetPath) {
|
|
12317
|
+
const normalized = targetPath.replace(/\\/g, "/");
|
|
12318
|
+
return normalized.includes("/.omp/wt/") || normalized.includes("/.omp/worktrees/");
|
|
12319
|
+
}
|
|
12320
|
+
async function resolveRepositoryContext(cwd, signal) {
|
|
12321
|
+
const canonicalCwd = canonicalizePath(cwd);
|
|
12322
|
+
try {
|
|
12323
|
+
const { stdout } = await execGit([
|
|
12324
|
+
"rev-parse",
|
|
12325
|
+
"--path-format=absolute",
|
|
12326
|
+
"--show-toplevel",
|
|
12327
|
+
"--git-dir",
|
|
12328
|
+
"--git-common-dir"
|
|
12329
|
+
], canonicalCwd, { signal });
|
|
12330
|
+
const lines = stdout.split(`
|
|
12331
|
+
`).map((l) => l.trim()).filter(Boolean);
|
|
12332
|
+
const resolveEntry = (val) => {
|
|
12333
|
+
if (!val)
|
|
12334
|
+
return "";
|
|
12335
|
+
return path3.isAbsolute(val) ? val : path3.resolve(canonicalCwd, val);
|
|
12336
|
+
};
|
|
12337
|
+
const workspaceRoot = canonicalizePath(resolveEntry(lines[0]));
|
|
12338
|
+
const gitDir = canonicalizePath(resolveEntry(lines[1] || path3.join(workspaceRoot, ".git")));
|
|
12339
|
+
const commonGitDir = canonicalizePath(resolveEntry(lines[2] || lines[1] || path3.join(workspaceRoot, ".git")));
|
|
12340
|
+
let repositoryObjectDir = path3.join(commonGitDir, "objects");
|
|
12341
|
+
const isOmpIsolation = isOmpWorktreePath(workspaceRoot) || isOmpWorktreePath(canonicalCwd);
|
|
12342
|
+
if (isOmpIsolation && gitDir === commonGitDir) {
|
|
12343
|
+
const alternatesFile = path3.join(gitDir, "objects", "info", "alternates");
|
|
12344
|
+
try {
|
|
12345
|
+
if (fsSync.existsSync(alternatesFile)) {
|
|
12346
|
+
const content = await fs.readFile(alternatesFile, "utf-8");
|
|
12347
|
+
const altLines = content.split(`
|
|
12348
|
+
`).map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
12349
|
+
for (const line of altLines) {
|
|
12350
|
+
const resolvedAlt = path3.isAbsolute(line) ? line : path3.resolve(path3.join(gitDir, "objects"), line);
|
|
12351
|
+
const canonicalAlt = canonicalizePath(resolvedAlt);
|
|
12352
|
+
if (fsSync.existsSync(canonicalAlt)) {
|
|
12353
|
+
repositoryObjectDir = canonicalAlt;
|
|
12354
|
+
break;
|
|
12355
|
+
}
|
|
12356
|
+
}
|
|
12357
|
+
}
|
|
12358
|
+
} catch {}
|
|
12359
|
+
}
|
|
12360
|
+
repositoryObjectDir = canonicalizePath(repositoryObjectDir);
|
|
12361
|
+
const repositoryKey = crypto.createHash("sha256").update(`git-object-dir\x00${repositoryObjectDir}`).digest("hex").slice(0, 16);
|
|
12362
|
+
return {
|
|
12363
|
+
workspaceRoot,
|
|
12364
|
+
isGit: true,
|
|
12365
|
+
gitDir,
|
|
12366
|
+
commonGitDir,
|
|
12367
|
+
repositoryObjectDir,
|
|
12368
|
+
repositoryKey,
|
|
12369
|
+
isOmpIsolation
|
|
12370
|
+
};
|
|
12371
|
+
} catch {
|
|
12372
|
+
const repositoryObjectDir = path3.join(canonicalCwd, ".non-git");
|
|
12373
|
+
const repositoryKey = crypto.createHash("sha256").update(`directory\x00${canonicalCwd}`).digest("hex").slice(0, 16);
|
|
12374
|
+
return {
|
|
12375
|
+
workspaceRoot: canonicalCwd,
|
|
12376
|
+
isGit: false,
|
|
12377
|
+
repositoryObjectDir,
|
|
12378
|
+
repositoryKey,
|
|
12379
|
+
isOmpIsolation: false
|
|
12380
|
+
};
|
|
12381
|
+
}
|
|
12382
|
+
}
|
|
12383
|
+
|
|
12295
12384
|
// src/source-aware-index.ts
|
|
12296
12385
|
class CompactSourceFrame {
|
|
12297
12386
|
id;
|
|
@@ -12673,6 +12762,7 @@ class SourceAwareCloneIndex {
|
|
|
12673
12762
|
const detectedClones = [];
|
|
12674
12763
|
const { insertFrames = false } = options;
|
|
12675
12764
|
let activeClones = new Map;
|
|
12765
|
+
const canonicalSourceId = canonicalizePath(sourceId);
|
|
12676
12766
|
for (const frame of frames) {
|
|
12677
12767
|
const frameStartLine = "startLine" in frame ? frame.startLine : frame.start.loc?.start.line ?? frame.start.line ?? 1;
|
|
12678
12768
|
const frameStartCol = "startCol" in frame ? frame.startCol : frame.start.loc?.start.column ?? frame.start.column ?? 1;
|
|
@@ -12720,7 +12810,8 @@ class SourceAwareCloneIndex {
|
|
|
12720
12810
|
const targetEndCol = targetFrame.endCol;
|
|
12721
12811
|
const targetEndPos = targetFrame.endPos;
|
|
12722
12812
|
const targetEndRange = targetFrame.endRange;
|
|
12723
|
-
|
|
12813
|
+
const isSameSource = targetFrame.sourceId === sourceId || canonicalizePath(targetFrame.sourceId) === canonicalSourceId;
|
|
12814
|
+
if (isSameSource && targetStartLine === frameStartLine && targetStartCol === frameStartCol) {
|
|
12724
12815
|
continue;
|
|
12725
12816
|
}
|
|
12726
12817
|
const offsetKey = `${targetFrame.sourceId}:${targetStartRange - frameStartRange}`;
|
|
@@ -12838,20 +12929,20 @@ class SourceAwareCloneIndex {
|
|
|
12838
12929
|
|
|
12839
12930
|
// src/disk-cache.ts
|
|
12840
12931
|
var DEFAULT_MAX_CACHE_BYTES = 250 * 1024 * 1024;
|
|
12841
|
-
var TOKENIZER_CACHE_VERSION = "
|
|
12932
|
+
var TOKENIZER_CACHE_VERSION = "5.0";
|
|
12842
12933
|
function getDefaultCacheDir() {
|
|
12843
12934
|
if (process.platform === "win32") {
|
|
12844
12935
|
const localAppData = process.env.LOCALAPPDATA;
|
|
12845
12936
|
if (localAppData) {
|
|
12846
|
-
return
|
|
12937
|
+
return path4.join(localAppData, "omp", "duplicate-detector");
|
|
12847
12938
|
}
|
|
12848
|
-
return
|
|
12939
|
+
return path4.join(os.homedir(), "AppData", "Local", "omp", "duplicate-detector");
|
|
12849
12940
|
}
|
|
12850
12941
|
const xdgCacheHome = process.env.XDG_CACHE_HOME;
|
|
12851
12942
|
if (xdgCacheHome) {
|
|
12852
|
-
return
|
|
12943
|
+
return path4.join(xdgCacheHome, "omp", "duplicate-detector");
|
|
12853
12944
|
}
|
|
12854
|
-
return
|
|
12945
|
+
return path4.join(os.homedir(), ".cache", "omp", "duplicate-detector");
|
|
12855
12946
|
}
|
|
12856
12947
|
function computeConfigFingerprint(config) {
|
|
12857
12948
|
if (!config)
|
|
@@ -12871,15 +12962,15 @@ function computeConfigFingerprint(config) {
|
|
|
12871
12962
|
crossFormats: config.crossFormats ?? false,
|
|
12872
12963
|
formatsExts: sortedFormats
|
|
12873
12964
|
};
|
|
12874
|
-
return
|
|
12965
|
+
return crypto2.createHash("sha256").update(JSON.stringify(canonical)).digest("hex").slice(0, 16);
|
|
12875
12966
|
}
|
|
12876
12967
|
function computeWorkspaceCachePath(baseDir, repositoryKeyOrRootDir, configFingerprint) {
|
|
12877
12968
|
const isKey = /^[0-9a-f]{16}$/i.test(repositoryKeyOrRootDir);
|
|
12878
|
-
const repoKey = isKey ? repositoryKeyOrRootDir :
|
|
12879
|
-
return
|
|
12969
|
+
const repoKey = isKey ? repositoryKeyOrRootDir : crypto2.createHash("sha256").update(path4.resolve(repositoryKeyOrRootDir)).digest("hex").slice(0, 16);
|
|
12970
|
+
return path4.join(baseDir, `${repoKey}_${configFingerprint}_v${CACHE_FORMAT_VERSION}.sqlite`);
|
|
12880
12971
|
}
|
|
12881
12972
|
var CACHE_FORMAT_MAGIC = "DUP3";
|
|
12882
|
-
var CACHE_FORMAT_VERSION =
|
|
12973
|
+
var CACHE_FORMAT_VERSION = 5;
|
|
12883
12974
|
function packBinaryShard(shard) {
|
|
12884
12975
|
return packBinaryShardV3(shard, shard.tokens ?? []);
|
|
12885
12976
|
}
|
|
@@ -12913,20 +13004,22 @@ function packBinaryShardV3(shard, tokens) {
|
|
|
12913
13004
|
const dLenBuf = Buffer.allocUnsafe(colBytes);
|
|
12914
13005
|
let prevLine = 0;
|
|
12915
13006
|
let prevCol = 0;
|
|
12916
|
-
let
|
|
13007
|
+
let prevRangeStart = 0;
|
|
12917
13008
|
for (let i = 0;i < tokenCount; i++) {
|
|
12918
13009
|
const tok = tokens[i];
|
|
12919
13010
|
const dLine = tok.line - prevLine;
|
|
12920
13011
|
const dCol = tok.column - prevCol;
|
|
12921
|
-
const
|
|
12922
|
-
const
|
|
13012
|
+
const startRange = Array.isArray(tok.range) && tok.range.length >= 2 ? tok.range[0] : tok.position ?? i;
|
|
13013
|
+
const endRange = Array.isArray(tok.range) && tok.range.length >= 2 ? tok.range[1] : startRange;
|
|
13014
|
+
const len = Math.max(0, endRange - startRange);
|
|
13015
|
+
const dRangeStart = startRange - prevRangeStart;
|
|
12923
13016
|
dLinesBuf.writeInt32LE(dLine, i * 4);
|
|
12924
13017
|
dColsBuf.writeInt32LE(dCol, i * 4);
|
|
12925
|
-
dPosBuf.writeInt32LE(
|
|
13018
|
+
dPosBuf.writeInt32LE(dRangeStart, i * 4);
|
|
12926
13019
|
dLenBuf.writeUInt32LE(len, i * 4);
|
|
12927
13020
|
prevLine = tok.line;
|
|
12928
13021
|
prevCol = tok.column;
|
|
12929
|
-
|
|
13022
|
+
prevRangeStart = startRange;
|
|
12930
13023
|
}
|
|
12931
13024
|
const meta = {
|
|
12932
13025
|
sourceId: shard.sourceId,
|
|
@@ -12975,7 +13068,7 @@ function unpackBinaryShard(compressed) {
|
|
|
12975
13068
|
function unpackBinaryShardV3(buf) {
|
|
12976
13069
|
try {
|
|
12977
13070
|
const version = buf.readUInt16LE(4);
|
|
12978
|
-
if (version
|
|
13071
|
+
if (version !== CACHE_FORMAT_VERSION)
|
|
12979
13072
|
return null;
|
|
12980
13073
|
const metaLen = buf.readUInt16LE(6);
|
|
12981
13074
|
const tokenCount = buf.readUInt32LE(8);
|
|
@@ -13027,11 +13120,11 @@ function unpackBinaryShardV3(buf) {
|
|
|
13027
13120
|
const tokens = new Array(tokenCount);
|
|
13028
13121
|
let curLine = 0;
|
|
13029
13122
|
let curCol = 0;
|
|
13030
|
-
let
|
|
13123
|
+
let curRangeStart = 0;
|
|
13031
13124
|
for (let i = 0;i < tokenCount; i++) {
|
|
13032
13125
|
curLine += dLines[i];
|
|
13033
13126
|
curCol += dCols[i];
|
|
13034
|
-
|
|
13127
|
+
curRangeStart += dPos[i];
|
|
13035
13128
|
const len = dLens[i];
|
|
13036
13129
|
const dictIdx = indices[i];
|
|
13037
13130
|
const hash2 = dictionary[dictIdx] ?? "";
|
|
@@ -13039,8 +13132,8 @@ function unpackBinaryShardV3(buf) {
|
|
|
13039
13132
|
hash: hash2,
|
|
13040
13133
|
line: curLine,
|
|
13041
13134
|
column: curCol,
|
|
13042
|
-
position:
|
|
13043
|
-
range: [
|
|
13135
|
+
position: i,
|
|
13136
|
+
range: [curRangeStart, curRangeStart + len]
|
|
13044
13137
|
};
|
|
13045
13138
|
}
|
|
13046
13139
|
const minTokens = meta.minTokens ?? 40;
|
|
@@ -13081,9 +13174,9 @@ class DiskCacheManager {
|
|
|
13081
13174
|
#deleteAllStmt = null;
|
|
13082
13175
|
#closed = false;
|
|
13083
13176
|
constructor(options) {
|
|
13084
|
-
this.rootDir =
|
|
13085
|
-
this.repositoryKey = options.repositoryKey ??
|
|
13086
|
-
this.baseCacheDir = options.cacheDir ?
|
|
13177
|
+
this.rootDir = path4.resolve(options.rootDir);
|
|
13178
|
+
this.repositoryKey = options.repositoryKey ?? crypto2.createHash("sha256").update(this.rootDir).digest("hex").slice(0, 16);
|
|
13179
|
+
this.baseCacheDir = options.cacheDir ? path4.resolve(options.cacheDir) : getDefaultCacheDir();
|
|
13087
13180
|
this.configFingerprint = computeConfigFingerprint(options.config);
|
|
13088
13181
|
this.dbPath = computeWorkspaceCachePath(this.baseCacheDir, this.repositoryKey, this.configFingerprint);
|
|
13089
13182
|
this.workspaceCacheDir = this.baseCacheDir;
|
|
@@ -13096,9 +13189,9 @@ class DiskCacheManager {
|
|
|
13096
13189
|
return this.#db;
|
|
13097
13190
|
let db = null;
|
|
13098
13191
|
try {
|
|
13099
|
-
const dir =
|
|
13100
|
-
if (!
|
|
13101
|
-
|
|
13192
|
+
const dir = path4.dirname(this.dbPath);
|
|
13193
|
+
if (!fsSync2.existsSync(dir)) {
|
|
13194
|
+
fsSync2.mkdirSync(dir, { recursive: true });
|
|
13102
13195
|
}
|
|
13103
13196
|
db = new Database(this.dbPath, { create: true });
|
|
13104
13197
|
db.exec("PRAGMA busy_timeout = 2000;");
|
|
@@ -13177,7 +13270,7 @@ class DiskCacheManager {
|
|
|
13177
13270
|
const db = this.#getDb();
|
|
13178
13271
|
if (!db || !this.#saveStmt)
|
|
13179
13272
|
return;
|
|
13180
|
-
const targetRelPath = relPath ?? (
|
|
13273
|
+
const targetRelPath = relPath ?? (path4.isAbsolute(shard.sourceId) ? path4.relative(this.rootDir, shard.sourceId) : shard.sourceId);
|
|
13181
13274
|
const normalizedRelPath = targetRelPath.replace(/\\/g, "/");
|
|
13182
13275
|
const payload = packBinaryShard(shard);
|
|
13183
13276
|
this.#saveStmt.run(normalizedRelPath, shard.contentHash, payload, Date.now());
|
|
@@ -13213,7 +13306,7 @@ class DiskCacheManager {
|
|
|
13213
13306
|
const now = Date.now();
|
|
13214
13307
|
const prepared = [];
|
|
13215
13308
|
for (const item of items) {
|
|
13216
|
-
const targetRelPath = item.relPath ?? (
|
|
13309
|
+
const targetRelPath = item.relPath ?? (path4.isAbsolute(item.shard.sourceId) ? path4.relative(root, item.shard.sourceId) : item.shard.sourceId);
|
|
13217
13310
|
const normalizedRelPath = targetRelPath.replace(/\\/g, "/");
|
|
13218
13311
|
const payload = packBinaryShard(item.shard);
|
|
13219
13312
|
prepared.push({
|
|
@@ -13295,108 +13388,19 @@ class DiskCacheManager {
|
|
|
13295
13388
|
async function cleanupLegacyCacheFiles(customCacheDir) {
|
|
13296
13389
|
const baseDir = customCacheDir ?? getDefaultCacheDir();
|
|
13297
13390
|
try {
|
|
13298
|
-
if (!
|
|
13391
|
+
if (!fsSync2.existsSync(baseDir))
|
|
13299
13392
|
return;
|
|
13300
|
-
const entries = await
|
|
13393
|
+
const entries = await fs2.readdir(baseDir);
|
|
13301
13394
|
for (const entry of entries) {
|
|
13302
13395
|
if (entry.endsWith(".sqlite") && !entry.includes(`_v${CACHE_FORMAT_VERSION}.sqlite`)) {
|
|
13303
|
-
await
|
|
13304
|
-
await
|
|
13305
|
-
await
|
|
13396
|
+
await fs2.unlink(path4.join(baseDir, entry)).catch(() => {});
|
|
13397
|
+
await fs2.unlink(path4.join(baseDir, `${entry}-wal`)).catch(() => {});
|
|
13398
|
+
await fs2.unlink(path4.join(baseDir, `${entry}-shm`)).catch(() => {});
|
|
13306
13399
|
}
|
|
13307
13400
|
}
|
|
13308
13401
|
} catch {}
|
|
13309
13402
|
}
|
|
13310
13403
|
|
|
13311
|
-
// src/repo-context.ts
|
|
13312
|
-
import * as crypto2 from "crypto";
|
|
13313
|
-
import * as fsSync2 from "fs";
|
|
13314
|
-
import * as fs2 from "fs/promises";
|
|
13315
|
-
import * as path4 from "path";
|
|
13316
|
-
function canonicalizePath(targetPath) {
|
|
13317
|
-
const resolved = path4.resolve(targetPath);
|
|
13318
|
-
let current = resolved;
|
|
13319
|
-
let suffix = "";
|
|
13320
|
-
while (current && current !== path4.dirname(current)) {
|
|
13321
|
-
try {
|
|
13322
|
-
if (fsSync2.existsSync(current)) {
|
|
13323
|
-
const real = fsSync2.realpathSync(current);
|
|
13324
|
-
return suffix ? path4.join(real, suffix) : real;
|
|
13325
|
-
}
|
|
13326
|
-
} catch {}
|
|
13327
|
-
suffix = suffix ? path4.join(path4.basename(current), suffix) : path4.basename(current);
|
|
13328
|
-
current = path4.dirname(current);
|
|
13329
|
-
}
|
|
13330
|
-
return resolved;
|
|
13331
|
-
}
|
|
13332
|
-
function isOmpWorktreePath(targetPath) {
|
|
13333
|
-
const normalized = targetPath.replace(/\\/g, "/");
|
|
13334
|
-
return normalized.includes("/.omp/wt/") || normalized.includes("/.omp/worktrees/");
|
|
13335
|
-
}
|
|
13336
|
-
async function resolveRepositoryContext(cwd, signal) {
|
|
13337
|
-
const canonicalCwd = canonicalizePath(cwd);
|
|
13338
|
-
try {
|
|
13339
|
-
const { stdout } = await execGit([
|
|
13340
|
-
"rev-parse",
|
|
13341
|
-
"--path-format=absolute",
|
|
13342
|
-
"--show-toplevel",
|
|
13343
|
-
"--git-dir",
|
|
13344
|
-
"--git-common-dir"
|
|
13345
|
-
], canonicalCwd, { signal });
|
|
13346
|
-
const lines = stdout.split(`
|
|
13347
|
-
`).map((l) => l.trim()).filter(Boolean);
|
|
13348
|
-
const resolveEntry = (val) => {
|
|
13349
|
-
if (!val)
|
|
13350
|
-
return "";
|
|
13351
|
-
return path4.isAbsolute(val) ? val : path4.resolve(canonicalCwd, val);
|
|
13352
|
-
};
|
|
13353
|
-
const workspaceRoot = canonicalizePath(resolveEntry(lines[0]));
|
|
13354
|
-
const gitDir = canonicalizePath(resolveEntry(lines[1] || path4.join(workspaceRoot, ".git")));
|
|
13355
|
-
const commonGitDir = canonicalizePath(resolveEntry(lines[2] || lines[1] || path4.join(workspaceRoot, ".git")));
|
|
13356
|
-
let repositoryObjectDir = path4.join(commonGitDir, "objects");
|
|
13357
|
-
const isOmpIsolation = isOmpWorktreePath(workspaceRoot) || isOmpWorktreePath(canonicalCwd);
|
|
13358
|
-
if (isOmpIsolation && gitDir === commonGitDir) {
|
|
13359
|
-
const alternatesFile = path4.join(gitDir, "objects", "info", "alternates");
|
|
13360
|
-
try {
|
|
13361
|
-
if (fsSync2.existsSync(alternatesFile)) {
|
|
13362
|
-
const content = await fs2.readFile(alternatesFile, "utf-8");
|
|
13363
|
-
const altLines = content.split(`
|
|
13364
|
-
`).map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
13365
|
-
for (const line of altLines) {
|
|
13366
|
-
const resolvedAlt = path4.isAbsolute(line) ? line : path4.resolve(path4.join(gitDir, "objects"), line);
|
|
13367
|
-
const canonicalAlt = canonicalizePath(resolvedAlt);
|
|
13368
|
-
if (fsSync2.existsSync(canonicalAlt)) {
|
|
13369
|
-
repositoryObjectDir = canonicalAlt;
|
|
13370
|
-
break;
|
|
13371
|
-
}
|
|
13372
|
-
}
|
|
13373
|
-
}
|
|
13374
|
-
} catch {}
|
|
13375
|
-
}
|
|
13376
|
-
repositoryObjectDir = canonicalizePath(repositoryObjectDir);
|
|
13377
|
-
const repositoryKey = crypto2.createHash("sha256").update(`git-object-dir\x00${repositoryObjectDir}`).digest("hex").slice(0, 16);
|
|
13378
|
-
return {
|
|
13379
|
-
workspaceRoot,
|
|
13380
|
-
isGit: true,
|
|
13381
|
-
gitDir,
|
|
13382
|
-
commonGitDir,
|
|
13383
|
-
repositoryObjectDir,
|
|
13384
|
-
repositoryKey,
|
|
13385
|
-
isOmpIsolation
|
|
13386
|
-
};
|
|
13387
|
-
} catch {
|
|
13388
|
-
const repositoryObjectDir = path4.join(canonicalCwd, ".non-git");
|
|
13389
|
-
const repositoryKey = crypto2.createHash("sha256").update(`directory\x00${canonicalCwd}`).digest("hex").slice(0, 16);
|
|
13390
|
-
return {
|
|
13391
|
-
workspaceRoot: canonicalCwd,
|
|
13392
|
-
isGit: false,
|
|
13393
|
-
repositoryObjectDir,
|
|
13394
|
-
repositoryKey,
|
|
13395
|
-
isOmpIsolation: false
|
|
13396
|
-
};
|
|
13397
|
-
}
|
|
13398
|
-
}
|
|
13399
|
-
|
|
13400
13404
|
// src/worker-protocol.ts
|
|
13401
13405
|
var REQUEST_TYPES = {
|
|
13402
13406
|
openWorkspace: true,
|
|
@@ -13763,7 +13767,8 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13763
13767
|
totalSourceBytes,
|
|
13764
13768
|
cloneCount: currentIndex.clones.length,
|
|
13765
13769
|
durationMs: Date.now() - startTime,
|
|
13766
|
-
status: baselineStatus
|
|
13770
|
+
status: baselineStatus,
|
|
13771
|
+
baselineClones: currentIndex.getClones()
|
|
13767
13772
|
}));
|
|
13768
13773
|
self.postMessage(createStatusEvent("ready", "Baseline indexing complete"));
|
|
13769
13774
|
return { indexedCount, status: baselineStatus };
|
|
@@ -13906,6 +13911,15 @@ async function handleWorkerRequest(msg) {
|
|
|
13906
13911
|
}
|
|
13907
13912
|
activeAbortController = new AbortController;
|
|
13908
13913
|
const recResult = await runIncrementalGitReconciliation(currentRootDir, options, activeAbortController.signal);
|
|
13914
|
+
self.postMessage(createCompleteEvent({
|
|
13915
|
+
indexedCount: recResult.indexedCount,
|
|
13916
|
+
cachedCount: 0,
|
|
13917
|
+
totalSourceBytes: 0,
|
|
13918
|
+
cloneCount: currentIndex.clones.length,
|
|
13919
|
+
durationMs: 0,
|
|
13920
|
+
status: recResult.status,
|
|
13921
|
+
baselineClones: currentIndex.getClones()
|
|
13922
|
+
}));
|
|
13909
13923
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13910
13924
|
started: true,
|
|
13911
13925
|
rootDir: currentRootDir,
|
|
@@ -13965,12 +13979,16 @@ async function handleWorkerRequest(msg) {
|
|
|
13965
13979
|
}
|
|
13966
13980
|
case "checkAndUpdate": {
|
|
13967
13981
|
const { filePath, content, format, revision = 1 } = msg.payload;
|
|
13968
|
-
const rawClones = currentIndex.updateSource(filePath, content, format);
|
|
13969
13982
|
const canonicalFilePath = canonicalizePath(filePath);
|
|
13970
13983
|
const resolvedPath = path5.resolve(filePath);
|
|
13984
|
+
currentIndex.removeSource(canonicalFilePath);
|
|
13985
|
+
currentIndex.removeSource(resolvedPath);
|
|
13986
|
+
currentIndex.removeSource(filePath);
|
|
13987
|
+
const rawClones = currentIndex.addSource(filePath, content, format);
|
|
13988
|
+
cacheSourceShard(canonicalFilePath, content);
|
|
13971
13989
|
cacheSourceShard(filePath, content);
|
|
13972
13990
|
const clones = filterAndEvictStaleClones(rawClones, filePath);
|
|
13973
|
-
const fileClones = clones.filter((c) => c.duplicationA.sourceId === filePath || c.duplicationB.sourceId === filePath || path5.resolve(c.duplicationA.sourceId) === resolvedPath || path5.resolve(c.duplicationB.sourceId) === resolvedPath || canonicalizePath(c.duplicationA.sourceId) === canonicalFilePath || canonicalizePath(c.duplicationB.sourceId) === canonicalFilePath);
|
|
13991
|
+
const fileClones = clones.filter((c) => c.duplicationA.sourceId === filePath || c.duplicationB.sourceId === filePath || c.duplicationA.sourceId === canonicalFilePath || c.duplicationB.sourceId === canonicalFilePath || path5.resolve(c.duplicationA.sourceId) === resolvedPath || path5.resolve(c.duplicationB.sourceId) === resolvedPath || canonicalizePath(c.duplicationA.sourceId) === canonicalFilePath || canonicalizePath(c.duplicationB.sourceId) === canonicalFilePath);
|
|
13974
13992
|
const watchEntry = {
|
|
13975
13993
|
revision,
|
|
13976
13994
|
lastKnownCloneCount: fileClones.length
|
package/package.json
CHANGED
package/src/detector-worker.ts
CHANGED
|
@@ -466,6 +466,7 @@ async function runBaselineIndexing(
|
|
|
466
466
|
cloneCount: currentIndex.clones.length,
|
|
467
467
|
durationMs: Date.now() - startTime,
|
|
468
468
|
status: baselineStatus,
|
|
469
|
+
baselineClones: currentIndex.getClones(),
|
|
469
470
|
}),
|
|
470
471
|
);
|
|
471
472
|
self.postMessage(
|
|
@@ -649,6 +650,17 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
649
650
|
options,
|
|
650
651
|
activeAbortController.signal,
|
|
651
652
|
);
|
|
653
|
+
self.postMessage(
|
|
654
|
+
createCompleteEvent({
|
|
655
|
+
indexedCount: recResult.indexedCount,
|
|
656
|
+
cachedCount: 0,
|
|
657
|
+
totalSourceBytes: 0,
|
|
658
|
+
cloneCount: currentIndex.clones.length,
|
|
659
|
+
durationMs: 0,
|
|
660
|
+
status: recResult.status,
|
|
661
|
+
baselineClones: currentIndex.getClones(),
|
|
662
|
+
}),
|
|
663
|
+
);
|
|
652
664
|
self.postMessage(
|
|
653
665
|
createSuccessResponse(msg.id, {
|
|
654
666
|
started: true,
|
|
@@ -731,24 +743,31 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
731
743
|
|
|
732
744
|
case "checkAndUpdate": {
|
|
733
745
|
const { filePath, content, format, revision = 1 } = msg.payload;
|
|
734
|
-
const rawClones = currentIndex.updateSource(filePath, content, format);
|
|
735
746
|
const canonicalFilePath = canonicalizePath(filePath);
|
|
736
747
|
const resolvedPath = path.resolve(filePath);
|
|
737
748
|
|
|
749
|
+
// Remove all path variations to ensure no duplicate self-match across symlinks
|
|
750
|
+
currentIndex.removeSource(canonicalFilePath);
|
|
751
|
+
currentIndex.removeSource(resolvedPath);
|
|
752
|
+
currentIndex.removeSource(filePath);
|
|
753
|
+
|
|
754
|
+
const rawClones = currentIndex.addSource(filePath, content, format);
|
|
755
|
+
|
|
756
|
+
cacheSourceShard(canonicalFilePath, content);
|
|
738
757
|
cacheSourceShard(filePath, content);
|
|
739
758
|
|
|
740
759
|
const clones = filterAndEvictStaleClones(rawClones, filePath);
|
|
741
|
-
|
|
742
760
|
const fileClones = clones.filter(
|
|
743
761
|
(c) =>
|
|
744
762
|
c.duplicationA.sourceId === filePath ||
|
|
745
763
|
c.duplicationB.sourceId === filePath ||
|
|
764
|
+
c.duplicationA.sourceId === canonicalFilePath ||
|
|
765
|
+
c.duplicationB.sourceId === canonicalFilePath ||
|
|
746
766
|
path.resolve(c.duplicationA.sourceId) === resolvedPath ||
|
|
747
767
|
path.resolve(c.duplicationB.sourceId) === resolvedPath ||
|
|
748
768
|
canonicalizePath(c.duplicationA.sourceId) === canonicalFilePath ||
|
|
749
769
|
canonicalizePath(c.duplicationB.sourceId) === canonicalFilePath,
|
|
750
770
|
);
|
|
751
|
-
|
|
752
771
|
const watchEntry = {
|
|
753
772
|
revision,
|
|
754
773
|
lastKnownCloneCount: fileClones.length,
|
package/src/disk-cache.ts
CHANGED
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
import type { WorkspaceOptions } from "./worker-protocol";
|
|
22
22
|
|
|
23
23
|
const DEFAULT_MAX_CACHE_BYTES = 250 * 1024 * 1024; // 250 MB
|
|
24
|
-
const TOKENIZER_CACHE_VERSION = "
|
|
24
|
+
const TOKENIZER_CACHE_VERSION = "5.0";
|
|
25
25
|
|
|
26
26
|
export interface DiskCacheOptions {
|
|
27
27
|
/** Root directory of the workspace */
|
|
@@ -136,7 +136,7 @@ export function computeShardKey(
|
|
|
136
136
|
export const CACHE_FORMAT_MAGIC = "DUP3";
|
|
137
137
|
|
|
138
138
|
/** Current binary format & SQLite schema version */
|
|
139
|
-
export const CACHE_FORMAT_VERSION =
|
|
139
|
+
export const CACHE_FORMAT_VERSION = 5;
|
|
140
140
|
|
|
141
141
|
/**
|
|
142
142
|
* Encodes a SerializedSourceShard into a high-density, zlib-compressed binary buffer (DUP3 format).
|
|
@@ -187,26 +187,31 @@ function packBinaryShardV3(
|
|
|
187
187
|
|
|
188
188
|
let prevLine = 0;
|
|
189
189
|
let prevCol = 0;
|
|
190
|
-
let
|
|
190
|
+
let prevRangeStart = 0;
|
|
191
191
|
|
|
192
192
|
for (let i = 0; i < tokenCount; i++) {
|
|
193
193
|
const tok = tokens[i]!;
|
|
194
194
|
const dLine = tok.line - prevLine;
|
|
195
195
|
const dCol = tok.column - prevCol;
|
|
196
|
-
const
|
|
197
|
-
const len =
|
|
196
|
+
const startRange =
|
|
198
197
|
Array.isArray(tok.range) && tok.range.length >= 2
|
|
199
|
-
? tok.range[
|
|
200
|
-
:
|
|
198
|
+
? tok.range[0]
|
|
199
|
+
: (tok.position ?? i);
|
|
200
|
+
const endRange =
|
|
201
|
+
Array.isArray(tok.range) && tok.range.length >= 2
|
|
202
|
+
? tok.range[1]
|
|
203
|
+
: startRange;
|
|
204
|
+
const len = Math.max(0, endRange - startRange);
|
|
205
|
+
const dRangeStart = startRange - prevRangeStart;
|
|
201
206
|
|
|
202
207
|
dLinesBuf.writeInt32LE(dLine, i * 4);
|
|
203
208
|
dColsBuf.writeInt32LE(dCol, i * 4);
|
|
204
|
-
dPosBuf.writeInt32LE(
|
|
209
|
+
dPosBuf.writeInt32LE(dRangeStart, i * 4);
|
|
205
210
|
dLenBuf.writeUInt32LE(len, i * 4);
|
|
206
211
|
|
|
207
212
|
prevLine = tok.line;
|
|
208
213
|
prevCol = tok.column;
|
|
209
|
-
|
|
214
|
+
prevRangeStart = startRange;
|
|
210
215
|
}
|
|
211
216
|
|
|
212
217
|
const meta = {
|
|
@@ -267,7 +272,7 @@ export function unpackBinaryShard(
|
|
|
267
272
|
function unpackBinaryShardV3(buf: Buffer): SerializedSourceShard | null {
|
|
268
273
|
try {
|
|
269
274
|
const version = buf.readUInt16LE(4);
|
|
270
|
-
if (version
|
|
275
|
+
if (version !== CACHE_FORMAT_VERSION) return null;
|
|
271
276
|
|
|
272
277
|
const metaLen = buf.readUInt16LE(6);
|
|
273
278
|
const tokenCount = buf.readUInt32LE(8);
|
|
@@ -329,12 +334,12 @@ function unpackBinaryShardV3(buf: Buffer): SerializedSourceShard | null {
|
|
|
329
334
|
const tokens: SerializedToken[] = new Array(tokenCount);
|
|
330
335
|
let curLine = 0;
|
|
331
336
|
let curCol = 0;
|
|
332
|
-
let
|
|
337
|
+
let curRangeStart = 0;
|
|
333
338
|
|
|
334
339
|
for (let i = 0; i < tokenCount; i++) {
|
|
335
340
|
curLine += dLines[i]!;
|
|
336
341
|
curCol += dCols[i]!;
|
|
337
|
-
|
|
342
|
+
curRangeStart += dPos[i]!;
|
|
338
343
|
const len = dLens[i]!;
|
|
339
344
|
const dictIdx = indices[i]!;
|
|
340
345
|
const hash = dictionary[dictIdx] ?? "";
|
|
@@ -343,8 +348,8 @@ function unpackBinaryShardV3(buf: Buffer): SerializedSourceShard | null {
|
|
|
343
348
|
hash,
|
|
344
349
|
line: curLine,
|
|
345
350
|
column: curCol,
|
|
346
|
-
position:
|
|
347
|
-
range: [
|
|
351
|
+
position: i,
|
|
352
|
+
range: [curRangeStart, curRangeStart + len],
|
|
348
353
|
};
|
|
349
354
|
}
|
|
350
355
|
|
package/src/duplicate-ledger.ts
CHANGED
|
@@ -1,6 +1,144 @@
|
|
|
1
1
|
import type { IClone } from "@jscpd/core";
|
|
2
2
|
import { toDisplayPath } from "./tui-notification";
|
|
3
3
|
|
|
4
|
+
export interface LineRange {
|
|
5
|
+
start: number;
|
|
6
|
+
end: number;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Computes 1-indexed changed/added line ranges in newText relative to oldText.
|
|
11
|
+
* If oldText is null or undefined (e.g. newly created file), returns the full range of newText.
|
|
12
|
+
*/
|
|
13
|
+
export function computeChangedLineRanges(
|
|
14
|
+
oldText: string | null | undefined,
|
|
15
|
+
newText: string,
|
|
16
|
+
): LineRange[] {
|
|
17
|
+
if (oldText == null) {
|
|
18
|
+
const lines = newText.split(/\r?\n/);
|
|
19
|
+
return [{ start: 1, end: Math.max(1, lines.length) }];
|
|
20
|
+
}
|
|
21
|
+
if (oldText === newText || newText.length === 0) {
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const oldLines = oldText.split(/\r?\n/);
|
|
26
|
+
const newLines = newText.split(/\r?\n/);
|
|
27
|
+
|
|
28
|
+
if (oldLines.length === 0) {
|
|
29
|
+
return [{ start: 1, end: Math.max(1, newLines.length) }];
|
|
30
|
+
}
|
|
31
|
+
if (newLines.length === 0) {
|
|
32
|
+
return [];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Trim common prefix
|
|
36
|
+
let prefix = 0;
|
|
37
|
+
while (
|
|
38
|
+
prefix < oldLines.length &&
|
|
39
|
+
prefix < newLines.length &&
|
|
40
|
+
oldLines[prefix] === newLines[prefix]
|
|
41
|
+
) {
|
|
42
|
+
prefix++;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Trim common suffix
|
|
46
|
+
let oldSuffix = oldLines.length - 1;
|
|
47
|
+
let newSuffix = newLines.length - 1;
|
|
48
|
+
while (
|
|
49
|
+
oldSuffix >= prefix &&
|
|
50
|
+
newSuffix >= prefix &&
|
|
51
|
+
oldLines[oldSuffix] === newLines[newSuffix]
|
|
52
|
+
) {
|
|
53
|
+
oldSuffix--;
|
|
54
|
+
newSuffix--;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const oldTrimmed = oldLines.slice(prefix, oldSuffix + 1);
|
|
58
|
+
const newTrimmed = newLines.slice(prefix, newSuffix + 1);
|
|
59
|
+
|
|
60
|
+
// Pure deletions: lines were removed, but no new lines were introduced
|
|
61
|
+
if (newTrimmed.length === 0) {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Compute LCS on the trimmed core if reasonably bounded (up to 500k cells)
|
|
66
|
+
if (
|
|
67
|
+
oldTrimmed.length > 0 &&
|
|
68
|
+
newTrimmed.length > 0 &&
|
|
69
|
+
oldTrimmed.length * newTrimmed.length <= 500000
|
|
70
|
+
) {
|
|
71
|
+
const m = oldTrimmed.length;
|
|
72
|
+
const n = newTrimmed.length;
|
|
73
|
+
const dp = Array.from({ length: m + 1 }, () => new Int32Array(n + 1));
|
|
74
|
+
for (let i = 1; i <= m; i++) {
|
|
75
|
+
for (let j = 1; j <= n; j++) {
|
|
76
|
+
if (oldTrimmed[i - 1] === newTrimmed[j - 1]) {
|
|
77
|
+
dp[i]![j] = dp[i - 1]![j - 1]! + 1;
|
|
78
|
+
} else {
|
|
79
|
+
dp[i]![j] = Math.max(dp[i - 1]![j]!, dp[i]![j - 1]!);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const matchedInNew = new Uint8Array(n);
|
|
85
|
+
let i = m;
|
|
86
|
+
let j = n;
|
|
87
|
+
while (i > 0 && j > 0) {
|
|
88
|
+
if (oldTrimmed[i - 1] === newTrimmed[j - 1]) {
|
|
89
|
+
matchedInNew[j - 1] = 1;
|
|
90
|
+
i--;
|
|
91
|
+
j--;
|
|
92
|
+
} else if (dp[i - 1]![j]! >= dp[i]![j - 1]!) {
|
|
93
|
+
i--;
|
|
94
|
+
} else {
|
|
95
|
+
j--;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const ranges: LineRange[] = [];
|
|
100
|
+
let rangeStart = -1;
|
|
101
|
+
for (let idx = 0; idx < n; idx++) {
|
|
102
|
+
if (!matchedInNew[idx]) {
|
|
103
|
+
if (rangeStart === -1) rangeStart = prefix + idx + 1;
|
|
104
|
+
} else {
|
|
105
|
+
if (rangeStart !== -1) {
|
|
106
|
+
ranges.push({ start: rangeStart, end: prefix + idx });
|
|
107
|
+
rangeStart = -1;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (rangeStart !== -1) {
|
|
112
|
+
ranges.push({ start: rangeStart, end: prefix + n });
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return ranges;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (prefix <= newSuffix) {
|
|
119
|
+
return [{ start: prefix + 1, end: newSuffix + 1 }];
|
|
120
|
+
}
|
|
121
|
+
return [];
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Checks whether a clone's target mutation span intersects any of the changed line ranges.
|
|
126
|
+
*/
|
|
127
|
+
export function cloneIntersectsRanges(
|
|
128
|
+
clone: IClone,
|
|
129
|
+
ranges: LineRange[],
|
|
130
|
+
side: "duplicationA" | "duplicationB" = "duplicationA",
|
|
131
|
+
): boolean {
|
|
132
|
+
if (ranges.length === 0) return false;
|
|
133
|
+
const span =
|
|
134
|
+
side === "duplicationB" ? clone.duplicationB : clone.duplicationA;
|
|
135
|
+
const start = span.start.line;
|
|
136
|
+
const end = span.end.line;
|
|
137
|
+
return ranges.some(
|
|
138
|
+
(range) => Math.max(start, range.start) <= Math.min(end, range.end),
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
4
142
|
/**
|
|
5
143
|
* Unique identifier for a duplicate match.
|
|
6
144
|
*/
|
|
@@ -31,28 +169,99 @@ function extractLineRange(
|
|
|
31
169
|
*/
|
|
32
170
|
export class DuplicateLedger {
|
|
33
171
|
readonly #seen = new Map<string, Set<string>>();
|
|
172
|
+
readonly #baseline = new Map<string, Set<string>>();
|
|
173
|
+
|
|
174
|
+
#addBaseline(key: string, id: string): void {
|
|
175
|
+
if (!key) return;
|
|
176
|
+
let set = this.#baseline.get(key);
|
|
177
|
+
if (!set) {
|
|
178
|
+
set = new Set<string>();
|
|
179
|
+
this.#baseline.set(key, set);
|
|
180
|
+
}
|
|
181
|
+
set.add(id);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Seed pre-existing baseline clones from workspace indexing so that legacy repository
|
|
186
|
+
* duplications are treated as pre-existing and not flagged as newly introduced duplicates.
|
|
187
|
+
*/
|
|
188
|
+
seedBaseline(clones: IClone[], basePath?: string): void {
|
|
189
|
+
for (const clone of clones) {
|
|
190
|
+
const a = clone.duplicationA;
|
|
191
|
+
const b = clone.duplicationB;
|
|
192
|
+
const lenA = a.end.line - a.start.line + 1;
|
|
193
|
+
const lenB = b.end.line - b.start.line + 1;
|
|
194
|
+
|
|
195
|
+
const relA = toDisplayPath(a.sourceId, basePath);
|
|
196
|
+
const relB = toDisplayPath(b.sourceId, basePath);
|
|
197
|
+
|
|
198
|
+
const idA_raw = `${b.sourceId}:${b.start.line}-${b.end.line}::${a.sourceId}::${lenA}`;
|
|
199
|
+
const idA_rel = `${relB}:${b.start.line}-${b.end.line}::${relA}::${lenA}`;
|
|
200
|
+
|
|
201
|
+
const idB_raw = `${a.sourceId}:${a.start.line}-${a.end.line}::${b.sourceId}::${lenB}`;
|
|
202
|
+
const idB_rel = `${relA}:${a.start.line}-${a.end.line}::${relB}::${lenB}`;
|
|
203
|
+
|
|
204
|
+
// Register bidirectional identities under both files' raw and display keys
|
|
205
|
+
const allIds = [idA_raw, idA_rel, idB_raw, idB_rel];
|
|
206
|
+
const allKeys = [relA, a.sourceId, relB, b.sourceId];
|
|
207
|
+
|
|
208
|
+
for (const key of allKeys) {
|
|
209
|
+
for (const id of allIds) {
|
|
210
|
+
this.#addBaseline(key, id);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
34
215
|
|
|
35
216
|
/**
|
|
36
217
|
* Filter a list of clones, returning only those not yet seen for the target file.
|
|
37
218
|
* Updates the ledger with newly seen identities.
|
|
38
219
|
*/
|
|
39
|
-
filterFreshClones(
|
|
220
|
+
filterFreshClones(
|
|
221
|
+
filePath: string,
|
|
222
|
+
clones: IClone[],
|
|
223
|
+
basePath?: string,
|
|
224
|
+
): IClone[] {
|
|
225
|
+
const displayPath = toDisplayPath(filePath, basePath);
|
|
40
226
|
const previous = this.#seen.get(filePath);
|
|
227
|
+
const prevDisplay = this.#seen.get(displayPath);
|
|
228
|
+
const baseline = this.#baseline.get(filePath);
|
|
229
|
+
const baseDisplay = this.#baseline.get(displayPath);
|
|
41
230
|
const fresh: IClone[] = [];
|
|
42
231
|
const currentIdentities = new Set<string>();
|
|
43
232
|
|
|
44
233
|
for (const clone of clones) {
|
|
45
234
|
const id = cloneIdentity(clone);
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
235
|
+
const a = clone.duplicationA;
|
|
236
|
+
const b = clone.duplicationB;
|
|
237
|
+
const len = a.end.line - a.start.line + 1;
|
|
238
|
+
const relId = `${toDisplayPath(b.sourceId, basePath)}:${b.start.line}-${b.end.line}::${toDisplayPath(a.sourceId, basePath)}::${len}`;
|
|
239
|
+
|
|
240
|
+
if (!currentIdentities.has(id)) {
|
|
241
|
+
currentIdentities.add(id);
|
|
242
|
+
currentIdentities.add(relId);
|
|
243
|
+
const isBaseline =
|
|
244
|
+
baseline?.has(id) ||
|
|
245
|
+
baseline?.has(relId) ||
|
|
246
|
+
baseDisplay?.has(id) ||
|
|
247
|
+
baseDisplay?.has(relId);
|
|
248
|
+
const isPrevious =
|
|
249
|
+
previous?.has(id) ||
|
|
250
|
+
previous?.has(relId) ||
|
|
251
|
+
prevDisplay?.has(id) ||
|
|
252
|
+
prevDisplay?.has(relId);
|
|
253
|
+
if (!isBaseline && !isPrevious) {
|
|
254
|
+
fresh.push(clone);
|
|
255
|
+
}
|
|
49
256
|
}
|
|
50
257
|
}
|
|
51
258
|
|
|
52
259
|
if (currentIdentities.size === 0) {
|
|
53
260
|
this.#seen.delete(filePath);
|
|
261
|
+
this.#seen.delete(displayPath);
|
|
54
262
|
} else {
|
|
55
263
|
this.#seen.set(filePath, currentIdentities);
|
|
264
|
+
this.#seen.set(displayPath, currentIdentities);
|
|
56
265
|
}
|
|
57
266
|
|
|
58
267
|
return fresh;
|
|
@@ -137,8 +346,10 @@ export class DuplicateLedger {
|
|
|
137
346
|
clear(filePath?: string): void {
|
|
138
347
|
if (filePath) {
|
|
139
348
|
this.#seen.delete(filePath);
|
|
349
|
+
this.#baseline.delete(filePath);
|
|
140
350
|
} else {
|
|
141
351
|
this.#seen.clear();
|
|
352
|
+
this.#baseline.clear();
|
|
142
353
|
}
|
|
143
354
|
}
|
|
144
355
|
}
|
package/src/index.ts
CHANGED
|
@@ -9,7 +9,12 @@ import {
|
|
|
9
9
|
type JscpdProjectConfig,
|
|
10
10
|
} from "./config-loader";
|
|
11
11
|
import { DuplicateDetectorCoordinator } from "./coordinator";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
cloneIntersectsRanges,
|
|
14
|
+
computeChangedLineRanges,
|
|
15
|
+
DuplicateLedger,
|
|
16
|
+
type LineRange,
|
|
17
|
+
} from "./duplicate-ledger";
|
|
13
18
|
import {
|
|
14
19
|
type BaselineStatus,
|
|
15
20
|
createIgnoreFilter,
|
|
@@ -231,6 +236,21 @@ function extractSettingsObject(
|
|
|
231
236
|
return undefined;
|
|
232
237
|
}
|
|
233
238
|
|
|
239
|
+
function extractPathFromToolInput(input: unknown): string | undefined {
|
|
240
|
+
if (!input || typeof input !== "object") return undefined;
|
|
241
|
+
const obj = input as Record<string, unknown>;
|
|
242
|
+
if (typeof obj.path === "string" && obj.path) {
|
|
243
|
+
return obj.path;
|
|
244
|
+
}
|
|
245
|
+
if (typeof obj.input === "string" && obj.input) {
|
|
246
|
+
const match = obj.input.match(/^\[([^#\]\r\n]+)(?:#[a-zA-Z0-9_-]+)?\]/m);
|
|
247
|
+
if (match?.[1]) {
|
|
248
|
+
return match[1].trim();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
|
253
|
+
|
|
234
254
|
export function formatBaselineMessage(
|
|
235
255
|
status: BaselineStatus,
|
|
236
256
|
count: number,
|
|
@@ -337,6 +357,8 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
337
357
|
let lastCtx: ExtensionContext | undefined;
|
|
338
358
|
const ledger = new DuplicateLedger();
|
|
339
359
|
const fileRevisions = new Map<string, number>();
|
|
360
|
+
const fileChangedRanges = new Map<string, LineRange[]>();
|
|
361
|
+
const preMutationContent = new Map<string, string | null>();
|
|
340
362
|
const coordinator = new DuplicateDetectorCoordinator();
|
|
341
363
|
let lastKnownHead: string | null = null;
|
|
342
364
|
let workerFailureNotified = false;
|
|
@@ -474,6 +496,9 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
474
496
|
notifyWorkerFailure(payload.error ?? "Baseline indexing failed");
|
|
475
497
|
return;
|
|
476
498
|
}
|
|
499
|
+
if (payload.baselineClones && payload.baselineClones.length > 0) {
|
|
500
|
+
ledger.seedBaseline(payload.baselineClones, currentCwd);
|
|
501
|
+
}
|
|
477
502
|
workerFailureNotified = false;
|
|
478
503
|
notifyBaselineStatus(
|
|
479
504
|
pi,
|
|
@@ -527,8 +552,25 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
527
552
|
: undefined;
|
|
528
553
|
|
|
529
554
|
if (!targetRel) return;
|
|
555
|
+
const targetRanges = fileChangedRanges.get(targetRel);
|
|
556
|
+
if (targetRanges) {
|
|
557
|
+
const isIntraFile = relA === relB;
|
|
558
|
+
const intersects = isIntraFile
|
|
559
|
+
? cloneIntersectsRanges(clone, targetRanges, "duplicationA") ||
|
|
560
|
+
cloneIntersectsRanges(clone, targetRanges, "duplicationB")
|
|
561
|
+
: cloneIntersectsRanges(
|
|
562
|
+
clone,
|
|
563
|
+
targetRanges,
|
|
564
|
+
targetRel === relB ? "duplicationB" : "duplicationA",
|
|
565
|
+
);
|
|
566
|
+
if (!intersects) return;
|
|
567
|
+
}
|
|
530
568
|
|
|
531
|
-
const freshClones = ledger.filterFreshClones(
|
|
569
|
+
const freshClones = ledger.filterFreshClones(
|
|
570
|
+
targetRel,
|
|
571
|
+
[clone],
|
|
572
|
+
currentCwd,
|
|
573
|
+
);
|
|
532
574
|
if (freshClones.length > 0) {
|
|
533
575
|
const fullPath = path.isAbsolute(targetRel)
|
|
534
576
|
? targetRel
|
|
@@ -573,6 +615,8 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
573
615
|
pi.on("session_switch", async (event, ctx) => {
|
|
574
616
|
ledger.clear();
|
|
575
617
|
fileRevisions.clear();
|
|
618
|
+
fileChangedRanges.clear();
|
|
619
|
+
preMutationContent.clear();
|
|
576
620
|
workerFailureNotified = false;
|
|
577
621
|
lastKnownHead = null;
|
|
578
622
|
if (ctx?.cwd) {
|
|
@@ -615,6 +659,8 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
615
659
|
pi.on("session_branch", async () => {
|
|
616
660
|
ledger.clear();
|
|
617
661
|
fileRevisions.clear();
|
|
662
|
+
fileChangedRanges.clear();
|
|
663
|
+
preMutationContent.clear();
|
|
618
664
|
});
|
|
619
665
|
|
|
620
666
|
pi.on("session_shutdown", async () => {
|
|
@@ -687,9 +733,44 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
687
733
|
});
|
|
688
734
|
|
|
689
735
|
// Intercept write and edit tool executions to detect clones in newly added/modified code
|
|
736
|
+
// Pre-capture file content before write or edit mutation so diff calculation on tool_result
|
|
737
|
+
// knows exactly which line ranges were newly introduced vs pre-existing.
|
|
738
|
+
pi.on("tool_call", async (event, ctx) => {
|
|
739
|
+
if (event.toolName !== "write" && event.toolName !== "edit") return;
|
|
740
|
+
const rawPath = extractPathFromToolInput(event.input);
|
|
741
|
+
if (!rawPath || rawPath.includes("://")) return;
|
|
742
|
+
const fullPath = path.isAbsolute(rawPath)
|
|
743
|
+
? path.resolve(rawPath)
|
|
744
|
+
: path.resolve(ctx.cwd, rawPath);
|
|
745
|
+
try {
|
|
746
|
+
const file = Bun.file(fullPath);
|
|
747
|
+
if (await file.exists()) {
|
|
748
|
+
preMutationContent.set(fullPath, await file.text());
|
|
749
|
+
} else {
|
|
750
|
+
preMutationContent.set(fullPath, null);
|
|
751
|
+
}
|
|
752
|
+
} catch {
|
|
753
|
+
preMutationContent.set(fullPath, null);
|
|
754
|
+
}
|
|
755
|
+
});
|
|
756
|
+
|
|
690
757
|
pi.on(
|
|
691
758
|
"tool_result",
|
|
692
759
|
async (event, ctx): Promise<ToolResultEventResult | void> => {
|
|
760
|
+
let capturedOldContent: string | null | undefined;
|
|
761
|
+
let resolvedFullPath: string | undefined;
|
|
762
|
+
|
|
763
|
+
if (event.toolName === "write" || event.toolName === "edit") {
|
|
764
|
+
const rawPath = extractPathFromToolInput(event.input);
|
|
765
|
+
if (rawPath && !rawPath.includes("://") && ctx?.cwd) {
|
|
766
|
+
resolvedFullPath = path.isAbsolute(rawPath)
|
|
767
|
+
? path.resolve(rawPath)
|
|
768
|
+
: path.resolve(ctx.cwd, rawPath);
|
|
769
|
+
capturedOldContent = preMutationContent.get(resolvedFullPath);
|
|
770
|
+
preMutationContent.delete(resolvedFullPath);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
693
774
|
if (event.isError) return;
|
|
694
775
|
if (!isEnabledForProject) return;
|
|
695
776
|
if (!config.checkOnMutation) return;
|
|
@@ -699,15 +780,9 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
699
780
|
return;
|
|
700
781
|
}
|
|
701
782
|
if (event.toolName !== "write" && event.toolName !== "edit") return;
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
// Skip internal protocol URLs (e.g. xd://, local://)
|
|
706
|
-
if (rawPath.includes("://")) return;
|
|
707
|
-
|
|
708
|
-
const fullPath = path.isAbsolute(rawPath)
|
|
709
|
-
? path.resolve(rawPath)
|
|
710
|
-
: path.resolve(ctx.cwd, rawPath);
|
|
783
|
+
if (!resolvedFullPath) return;
|
|
784
|
+
|
|
785
|
+
const fullPath = resolvedFullPath;
|
|
711
786
|
const relPath = path.relative(ctx.cwd, fullPath);
|
|
712
787
|
const normalizedRelPath = relPath.replace(/\\/g, "/");
|
|
713
788
|
|
|
@@ -720,7 +795,6 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
720
795
|
) {
|
|
721
796
|
return;
|
|
722
797
|
}
|
|
723
|
-
|
|
724
798
|
try {
|
|
725
799
|
// Skip ignored files (matching ignore patterns, noise files, or test files)
|
|
726
800
|
const ignoreFilter = createIgnoreFilter(config.ignorePatterns, {
|
|
@@ -737,6 +811,27 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
737
811
|
// Skip generated files
|
|
738
812
|
if (isGeneratedContent(content)) return;
|
|
739
813
|
|
|
814
|
+
let oldContent = capturedOldContent;
|
|
815
|
+
if (oldContent === undefined && ctx?.cwd) {
|
|
816
|
+
try {
|
|
817
|
+
const isGit = await isInsideGitWorkTree(ctx.cwd);
|
|
818
|
+
if (isGit) {
|
|
819
|
+
const { stdout } = await execGit(
|
|
820
|
+
["show", `HEAD:${normalizedRelPath}`],
|
|
821
|
+
ctx.cwd,
|
|
822
|
+
);
|
|
823
|
+
oldContent = stdout;
|
|
824
|
+
} else {
|
|
825
|
+
oldContent = null;
|
|
826
|
+
}
|
|
827
|
+
} catch {
|
|
828
|
+
oldContent = null;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
const changedRanges = computeChangedLineRanges(oldContent, content);
|
|
833
|
+
fileChangedRanges.set(normalizedRelPath, changedRanges);
|
|
834
|
+
|
|
740
835
|
const revision = (fileRevisions.get(normalizedRelPath) ?? 0) + 1;
|
|
741
836
|
fileRevisions.set(normalizedRelPath, revision);
|
|
742
837
|
|
|
@@ -745,7 +840,24 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
745
840
|
content,
|
|
746
841
|
revision,
|
|
747
842
|
);
|
|
748
|
-
const
|
|
843
|
+
const mutatingClones = clones.filter((c) => {
|
|
844
|
+
if (cloneIntersectsRanges(c, changedRanges, "duplicationA")) {
|
|
845
|
+
return true;
|
|
846
|
+
}
|
|
847
|
+
const isIntraFile =
|
|
848
|
+
c.duplicationA.sourceId === c.duplicationB.sourceId ||
|
|
849
|
+
path.resolve(c.duplicationA.sourceId) ===
|
|
850
|
+
path.resolve(c.duplicationB.sourceId);
|
|
851
|
+
return (
|
|
852
|
+
isIntraFile &&
|
|
853
|
+
cloneIntersectsRanges(c, changedRanges, "duplicationB")
|
|
854
|
+
);
|
|
855
|
+
});
|
|
856
|
+
const freshClones = ledger.filterFreshClones(
|
|
857
|
+
normalizedRelPath,
|
|
858
|
+
mutatingClones,
|
|
859
|
+
currentCwd,
|
|
860
|
+
);
|
|
749
861
|
|
|
750
862
|
if (freshClones.length > 0) {
|
|
751
863
|
const fullReminder = ledger.formatReminder(
|
|
@@ -872,6 +984,8 @@ export default function duplicateDetectorExtension(pi: ExtensionAPI): void {
|
|
|
872
984
|
isEnabledForProject = false;
|
|
873
985
|
ledger.clear();
|
|
874
986
|
fileRevisions.clear();
|
|
987
|
+
fileChangedRanges.clear();
|
|
988
|
+
preMutationContent.clear();
|
|
875
989
|
ctx.ui.notify("Duplicate detector disabled for this project.", "info");
|
|
876
990
|
notifyBaselineStatus(pi, ctx as ExtensionContext, "disabled", 0);
|
|
877
991
|
} else if (action === "status" || action === "") {
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
} from "@jscpd/core";
|
|
15
15
|
import { Tokenizer } from "@jscpd/tokenizer";
|
|
16
16
|
import { getSupportedCodeFormat } from "./jscpd-engine";
|
|
17
|
+
import { canonicalizePath } from "./repo-context";
|
|
17
18
|
export class CompactSourceFrame {
|
|
18
19
|
constructor(
|
|
19
20
|
public readonly id: string,
|
|
@@ -689,6 +690,7 @@ export class SourceAwareCloneIndex {
|
|
|
689
690
|
const { insertFrames = false } = options;
|
|
690
691
|
let activeClones = new Map<string, ActiveCloneCandidate>();
|
|
691
692
|
|
|
693
|
+
const canonicalSourceId = canonicalizePath(sourceId);
|
|
692
694
|
for (const frame of frames) {
|
|
693
695
|
const frameStartLine =
|
|
694
696
|
"startLine" in frame
|
|
@@ -793,8 +795,11 @@ export class SourceAwareCloneIndex {
|
|
|
793
795
|
const targetEndRange = targetFrame.endRange;
|
|
794
796
|
|
|
795
797
|
// Disallow exact self-match at identical line/column position
|
|
798
|
+
const isSameSource =
|
|
799
|
+
targetFrame.sourceId === sourceId ||
|
|
800
|
+
canonicalizePath(targetFrame.sourceId) === canonicalSourceId;
|
|
796
801
|
if (
|
|
797
|
-
|
|
802
|
+
isSameSource &&
|
|
798
803
|
targetStartLine === frameStartLine &&
|
|
799
804
|
targetStartCol === frameStartCol
|
|
800
805
|
) {
|
package/src/tui-notification.ts
CHANGED
|
@@ -132,8 +132,14 @@ export function toDisplayPath(sourceId: string, basePath?: string): string {
|
|
|
132
132
|
(typeof process !== "undefined" && process.cwd ? process.cwd() : "")
|
|
133
133
|
)?.replace(/\\/g, "/");
|
|
134
134
|
|
|
135
|
-
if (root
|
|
136
|
-
|
|
135
|
+
if (root) {
|
|
136
|
+
if (clean.startsWith(root)) {
|
|
137
|
+
clean = clean.slice(root.length);
|
|
138
|
+
} else if (clean.startsWith(`/private${root}`)) {
|
|
139
|
+
clean = clean.slice(`/private${root}`.length);
|
|
140
|
+
} else if (root.startsWith("/private") && clean.startsWith(root.slice(8))) {
|
|
141
|
+
clean = clean.slice(root.slice(8).length);
|
|
142
|
+
}
|
|
137
143
|
if (clean.startsWith("/")) clean = clean.slice(1);
|
|
138
144
|
}
|
|
139
145
|
return clean || sourceId;
|