omp-plugin-duplicate-detector 0.1.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/detector-worker.js +507 -276
- package/package.json +1 -1
- package/src/coordinator.ts +7 -3
- package/src/detector-worker.ts +156 -28
- package/src/disk-cache.ts +383 -314
- package/src/index.ts +134 -7
- package/src/project-state.ts +10 -4
- package/src/repo-context.ts +167 -0
- package/src/source-aware-index.ts +14 -1
package/dist/detector-worker.js
CHANGED
|
@@ -970,9 +970,10 @@ var require_ignore = __commonJS(function(exports, module) {
|
|
|
970
970
|
});
|
|
971
971
|
|
|
972
972
|
// src/detector-worker.ts
|
|
973
|
-
import * as
|
|
974
|
-
import * as
|
|
975
|
-
import * as
|
|
973
|
+
import * as crypto3 from "crypto";
|
|
974
|
+
import * as fsSync3 from "fs";
|
|
975
|
+
import * as fs3 from "fs/promises";
|
|
976
|
+
import * as path5 from "path";
|
|
976
977
|
|
|
977
978
|
// src/disk-cache.ts
|
|
978
979
|
import { Database } from "bun:sqlite";
|
|
@@ -12565,7 +12566,9 @@ class SourceAwareCloneIndex {
|
|
|
12565
12566
|
}
|
|
12566
12567
|
let normalizedFrames;
|
|
12567
12568
|
const shardTokens = shard.tokens;
|
|
12568
|
-
if (shard.frames
|
|
12569
|
+
if (shardTokens && shardTokens.length > 0 && (shard.minTokens !== this.#minTokens || !shard.frames || shard.frames.length === 0)) {
|
|
12570
|
+
normalizedFrames = reconstructFramesFromTokens(shardTokens, sourceId, this.#minTokens, this.#hashFunction);
|
|
12571
|
+
} else if (shard.frames && shard.frames.length > 0) {
|
|
12569
12572
|
normalizedFrames = shard.frames.map((f) => {
|
|
12570
12573
|
if (f instanceof CompactSourceFrame && f.sourceId === sourceId) {
|
|
12571
12574
|
return f;
|
|
@@ -12835,6 +12838,7 @@ class SourceAwareCloneIndex {
|
|
|
12835
12838
|
|
|
12836
12839
|
// src/disk-cache.ts
|
|
12837
12840
|
var DEFAULT_MAX_CACHE_BYTES = 250 * 1024 * 1024;
|
|
12841
|
+
var TOKENIZER_CACHE_VERSION = "4.0";
|
|
12838
12842
|
function getDefaultCacheDir() {
|
|
12839
12843
|
if (process.platform === "win32") {
|
|
12840
12844
|
const localAppData = process.env.LOCALAPPDATA;
|
|
@@ -12851,7 +12855,7 @@ function getDefaultCacheDir() {
|
|
|
12851
12855
|
}
|
|
12852
12856
|
function computeConfigFingerprint(config) {
|
|
12853
12857
|
if (!config)
|
|
12854
|
-
return
|
|
12858
|
+
return `default_${TOKENIZER_CACHE_VERSION}`;
|
|
12855
12859
|
let sortedFormats;
|
|
12856
12860
|
if (config.formatsExts) {
|
|
12857
12861
|
sortedFormats = {};
|
|
@@ -12860,6 +12864,7 @@ function computeConfigFingerprint(config) {
|
|
|
12860
12864
|
}
|
|
12861
12865
|
}
|
|
12862
12866
|
const canonical = {
|
|
12867
|
+
version: TOKENIZER_CACHE_VERSION,
|
|
12863
12868
|
minTokens: config.minTokens ?? 40,
|
|
12864
12869
|
minLines: config.minLines ?? 5,
|
|
12865
12870
|
maxLines: config.maxLines ?? 500,
|
|
@@ -12868,201 +12873,199 @@ function computeConfigFingerprint(config) {
|
|
|
12868
12873
|
};
|
|
12869
12874
|
return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex").slice(0, 16);
|
|
12870
12875
|
}
|
|
12871
|
-
function computeWorkspaceCachePath(baseDir,
|
|
12872
|
-
const
|
|
12873
|
-
const
|
|
12874
|
-
return path3.join(baseDir, `${
|
|
12876
|
+
function computeWorkspaceCachePath(baseDir, repositoryKeyOrRootDir, configFingerprint) {
|
|
12877
|
+
const isKey = /^[0-9a-f]{16}$/i.test(repositoryKeyOrRootDir);
|
|
12878
|
+
const repoKey = isKey ? repositoryKeyOrRootDir : crypto.createHash("sha256").update(path3.resolve(repositoryKeyOrRootDir)).digest("hex").slice(0, 16);
|
|
12879
|
+
return path3.join(baseDir, `${repoKey}_${configFingerprint}_v${CACHE_FORMAT_VERSION}.sqlite`);
|
|
12875
12880
|
}
|
|
12876
12881
|
var CACHE_FORMAT_MAGIC = "DUP3";
|
|
12877
|
-
var CACHE_FORMAT_VERSION =
|
|
12882
|
+
var CACHE_FORMAT_VERSION = 4;
|
|
12878
12883
|
function packBinaryShard(shard) {
|
|
12879
12884
|
return packBinaryShardV3(shard, shard.tokens ?? []);
|
|
12880
12885
|
}
|
|
12881
12886
|
function packBinaryShardV3(shard, tokens) {
|
|
12882
|
-
const srcIdBuf = Buffer.from(shard.sourceId, "utf8");
|
|
12883
|
-
const formatBuf = Buffer.from(shard.format, "utf8");
|
|
12884
|
-
const hashBuf = Buffer.from(shard.contentHash, "utf8");
|
|
12885
12887
|
const tokenCount = tokens.length;
|
|
12886
|
-
const
|
|
12887
|
-
const
|
|
12888
|
-
const tokenIndices = new Uint16Array(tokenCount);
|
|
12888
|
+
const dictionary = [];
|
|
12889
|
+
const dictMap = new Map;
|
|
12889
12890
|
for (let i = 0;i < tokenCount; i++) {
|
|
12890
12891
|
const h = tokens[i].hash;
|
|
12891
|
-
|
|
12892
|
-
|
|
12893
|
-
|
|
12894
|
-
|
|
12895
|
-
|
|
12896
|
-
|
|
12897
|
-
|
|
12898
|
-
|
|
12899
|
-
|
|
12900
|
-
|
|
12901
|
-
|
|
12902
|
-
const
|
|
12903
|
-
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
12908
|
-
|
|
12909
|
-
|
|
12910
|
-
|
|
12911
|
-
|
|
12912
|
-
|
|
12913
|
-
|
|
12914
|
-
|
|
12915
|
-
|
|
12916
|
-
buf.writeUInt32LE(shard.size, pos);
|
|
12917
|
-
pos += 4;
|
|
12918
|
-
buf.writeUInt32LE(shard.lines, pos);
|
|
12919
|
-
pos += 4;
|
|
12920
|
-
buf.writeUInt32LE(shard.tokenCount, pos);
|
|
12921
|
-
pos += 4;
|
|
12922
|
-
buf.writeDoubleLE(shard.updatedAt ?? Date.now(), pos);
|
|
12923
|
-
pos += 8;
|
|
12924
|
-
buf.writeUInt16LE(minTokens, pos);
|
|
12925
|
-
pos += 2;
|
|
12926
|
-
buf.writeUInt16LE(srcIdBuf.length, pos);
|
|
12927
|
-
pos += 2;
|
|
12928
|
-
srcIdBuf.copy(buf, pos);
|
|
12929
|
-
pos += srcIdBuf.length;
|
|
12930
|
-
buf.writeUInt16LE(dictCount, pos);
|
|
12931
|
-
pos += 2;
|
|
12932
|
-
buf.writeUInt32LE(tokenCount, pos);
|
|
12933
|
-
pos += 4;
|
|
12934
|
-
for (const h of dict.keys()) {
|
|
12935
|
-
const hexHash = h.length === 20 ? h : h.padEnd(20, "0");
|
|
12936
|
-
buf.write(hexHash, pos, 10, "hex");
|
|
12937
|
-
pos += 10;
|
|
12938
|
-
}
|
|
12939
|
-
const dictIdxOffset = pos;
|
|
12940
|
-
const deltaLineOffset = dictIdxOffset + tokenCount * 2;
|
|
12941
|
-
const colOffset = deltaLineOffset + tokenCount * 2;
|
|
12942
|
-
const deltaRangeOffset = colOffset + tokenCount * 2;
|
|
12943
|
-
const lenOffset = deltaRangeOffset + tokenCount * 4;
|
|
12944
|
-
let prevLine = 1;
|
|
12945
|
-
let prevRangeStart = 0;
|
|
12892
|
+
if (!dictMap.has(h)) {
|
|
12893
|
+
dictMap.set(h, dictionary.length);
|
|
12894
|
+
dictionary.push(h);
|
|
12895
|
+
}
|
|
12896
|
+
}
|
|
12897
|
+
const dictCount = dictionary.length;
|
|
12898
|
+
const dictEntries = new Array(dictCount);
|
|
12899
|
+
for (let i = 0;i < dictCount; i++) {
|
|
12900
|
+
const hex = dictionary[i];
|
|
12901
|
+
dictEntries[i] = Buffer.from(hex, "hex");
|
|
12902
|
+
}
|
|
12903
|
+
const dictBuf = Buffer.concat(dictEntries);
|
|
12904
|
+
const indicesBuf = Buffer.allocUnsafe(tokenCount * 2);
|
|
12905
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
12906
|
+
const idx = dictMap.get(tokens[i].hash);
|
|
12907
|
+
indicesBuf.writeUInt16LE(idx, i * 2);
|
|
12908
|
+
}
|
|
12909
|
+
const colBytes = tokenCount * 4;
|
|
12910
|
+
const dLinesBuf = Buffer.allocUnsafe(colBytes);
|
|
12911
|
+
const dColsBuf = Buffer.allocUnsafe(colBytes);
|
|
12912
|
+
const dPosBuf = Buffer.allocUnsafe(colBytes);
|
|
12913
|
+
const dLenBuf = Buffer.allocUnsafe(colBytes);
|
|
12914
|
+
let prevLine = 0;
|
|
12915
|
+
let prevCol = 0;
|
|
12916
|
+
let prevPos = 0;
|
|
12946
12917
|
for (let i = 0;i < tokenCount; i++) {
|
|
12947
|
-
const
|
|
12948
|
-
const
|
|
12949
|
-
const
|
|
12950
|
-
const
|
|
12951
|
-
const
|
|
12952
|
-
|
|
12953
|
-
|
|
12954
|
-
|
|
12955
|
-
|
|
12956
|
-
|
|
12957
|
-
|
|
12958
|
-
|
|
12959
|
-
|
|
12960
|
-
|
|
12961
|
-
|
|
12962
|
-
|
|
12918
|
+
const tok = tokens[i];
|
|
12919
|
+
const dLine = tok.line - prevLine;
|
|
12920
|
+
const dCol = tok.column - prevCol;
|
|
12921
|
+
const dPos = tok.position - prevPos;
|
|
12922
|
+
const len = Array.isArray(tok.range) && tok.range.length >= 2 ? tok.range[1] - tok.range[0] : 0;
|
|
12923
|
+
dLinesBuf.writeInt32LE(dLine, i * 4);
|
|
12924
|
+
dColsBuf.writeInt32LE(dCol, i * 4);
|
|
12925
|
+
dPosBuf.writeInt32LE(dPos, i * 4);
|
|
12926
|
+
dLenBuf.writeUInt32LE(len, i * 4);
|
|
12927
|
+
prevLine = tok.line;
|
|
12928
|
+
prevCol = tok.column;
|
|
12929
|
+
prevPos = tok.position;
|
|
12930
|
+
}
|
|
12931
|
+
const meta = {
|
|
12932
|
+
sourceId: shard.sourceId,
|
|
12933
|
+
contentHash: shard.contentHash,
|
|
12934
|
+
format: shard.format,
|
|
12935
|
+
size: shard.size,
|
|
12936
|
+
lines: shard.lines,
|
|
12937
|
+
tokenCount: shard.tokenCount,
|
|
12938
|
+
minTokens: shard.minTokens,
|
|
12939
|
+
updatedAt: shard.updatedAt ?? Date.now()
|
|
12940
|
+
};
|
|
12941
|
+
const metaJson = Buffer.from(JSON.stringify(meta), "utf-8");
|
|
12942
|
+
const HEADER_SIZE = 16;
|
|
12943
|
+
const header = Buffer.allocUnsafe(HEADER_SIZE);
|
|
12944
|
+
header.write(CACHE_FORMAT_MAGIC, 0, 4, "ascii");
|
|
12945
|
+
header.writeUInt16LE(CACHE_FORMAT_VERSION, 4);
|
|
12946
|
+
header.writeUInt16LE(metaJson.length, 6);
|
|
12947
|
+
header.writeUInt32LE(tokenCount, 8);
|
|
12948
|
+
header.writeUInt32LE(dictCount, 12);
|
|
12949
|
+
const rawUncompressed = Buffer.concat([
|
|
12950
|
+
header,
|
|
12951
|
+
metaJson,
|
|
12952
|
+
dictBuf,
|
|
12953
|
+
indicesBuf,
|
|
12954
|
+
dLinesBuf,
|
|
12955
|
+
dColsBuf,
|
|
12956
|
+
dPosBuf,
|
|
12957
|
+
dLenBuf
|
|
12958
|
+
]);
|
|
12959
|
+
return zlib.deflateSync(rawUncompressed, { level: 6 });
|
|
12963
12960
|
}
|
|
12964
12961
|
function unpackBinaryShard(compressed) {
|
|
12965
12962
|
try {
|
|
12966
|
-
const
|
|
12967
|
-
if (
|
|
12968
|
-
return null;
|
|
12969
|
-
const magic = buf.toString("ascii", 0, 4);
|
|
12970
|
-
if (magic !== CACHE_FORMAT_MAGIC) {
|
|
12963
|
+
const raw = zlib.inflateSync(compressed);
|
|
12964
|
+
if (raw.length < 16)
|
|
12971
12965
|
return null;
|
|
12966
|
+
const magic = raw.toString("ascii", 0, 4);
|
|
12967
|
+
if (magic === CACHE_FORMAT_MAGIC) {
|
|
12968
|
+
return unpackBinaryShardV3(raw);
|
|
12972
12969
|
}
|
|
12973
|
-
return
|
|
12970
|
+
return null;
|
|
12974
12971
|
} catch {
|
|
12975
12972
|
return null;
|
|
12976
12973
|
}
|
|
12977
12974
|
}
|
|
12978
12975
|
function unpackBinaryShardV3(buf) {
|
|
12979
|
-
|
|
12980
|
-
|
|
12981
|
-
|
|
12982
|
-
|
|
12983
|
-
|
|
12984
|
-
|
|
12985
|
-
|
|
12986
|
-
|
|
12987
|
-
|
|
12988
|
-
|
|
12989
|
-
|
|
12990
|
-
|
|
12991
|
-
|
|
12992
|
-
|
|
12993
|
-
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
|
|
12999
|
-
|
|
13000
|
-
|
|
13001
|
-
|
|
13002
|
-
|
|
13003
|
-
|
|
13004
|
-
|
|
13005
|
-
|
|
13006
|
-
|
|
13007
|
-
|
|
13008
|
-
|
|
13009
|
-
|
|
13010
|
-
|
|
13011
|
-
|
|
13012
|
-
|
|
13013
|
-
|
|
13014
|
-
|
|
13015
|
-
|
|
13016
|
-
|
|
13017
|
-
|
|
13018
|
-
|
|
13019
|
-
|
|
13020
|
-
|
|
13021
|
-
|
|
13022
|
-
|
|
13023
|
-
|
|
13024
|
-
|
|
13025
|
-
|
|
13026
|
-
|
|
13027
|
-
|
|
13028
|
-
|
|
13029
|
-
|
|
13030
|
-
const
|
|
13031
|
-
|
|
13032
|
-
|
|
13033
|
-
|
|
13034
|
-
|
|
13035
|
-
|
|
13036
|
-
|
|
13037
|
-
|
|
13038
|
-
|
|
12976
|
+
try {
|
|
12977
|
+
const version = buf.readUInt16LE(4);
|
|
12978
|
+
if (version < 3 || version > CACHE_FORMAT_VERSION)
|
|
12979
|
+
return null;
|
|
12980
|
+
const metaLen = buf.readUInt16LE(6);
|
|
12981
|
+
const tokenCount = buf.readUInt32LE(8);
|
|
12982
|
+
const dictCount = buf.readUInt32LE(12);
|
|
12983
|
+
let offset = 16;
|
|
12984
|
+
const metaJsonBuf = buf.subarray(offset, offset + metaLen);
|
|
12985
|
+
offset += metaLen;
|
|
12986
|
+
const meta = JSON.parse(metaJsonBuf.toString("utf-8"));
|
|
12987
|
+
const TOKEN_HASH_RAW_BYTES = 10;
|
|
12988
|
+
const dictByteLen = dictCount * TOKEN_HASH_RAW_BYTES;
|
|
12989
|
+
if (buf.length < offset + dictByteLen)
|
|
12990
|
+
return null;
|
|
12991
|
+
const dictionary = new Array(dictCount);
|
|
12992
|
+
for (let i = 0;i < dictCount; i++) {
|
|
12993
|
+
dictionary[i] = buf.subarray(offset + i * 10, offset + (i + 1) * 10).toString("hex");
|
|
12994
|
+
}
|
|
12995
|
+
offset += dictByteLen;
|
|
12996
|
+
const indicesByteLen = tokenCount * 2;
|
|
12997
|
+
if (buf.length < offset + indicesByteLen)
|
|
12998
|
+
return null;
|
|
12999
|
+
const indices = new Uint16Array(tokenCount);
|
|
13000
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13001
|
+
indices[i] = buf.readUInt16LE(offset + i * 2);
|
|
13002
|
+
}
|
|
13003
|
+
offset += indicesByteLen;
|
|
13004
|
+
const colBytes = tokenCount * 4;
|
|
13005
|
+
if (buf.length < offset + colBytes * 4)
|
|
13006
|
+
return null;
|
|
13007
|
+
const dLines = new Int32Array(tokenCount);
|
|
13008
|
+
const dCols = new Int32Array(tokenCount);
|
|
13009
|
+
const dPos = new Int32Array(tokenCount);
|
|
13010
|
+
const dLens = new Uint32Array(tokenCount);
|
|
13011
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13012
|
+
dLines[i] = buf.readInt32LE(offset + i * 4);
|
|
13013
|
+
}
|
|
13014
|
+
offset += colBytes;
|
|
13015
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13016
|
+
dCols[i] = buf.readInt32LE(offset + i * 4);
|
|
13017
|
+
}
|
|
13018
|
+
offset += colBytes;
|
|
13019
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13020
|
+
dPos[i] = buf.readInt32LE(offset + i * 4);
|
|
13021
|
+
}
|
|
13022
|
+
offset += colBytes;
|
|
13023
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13024
|
+
dLens[i] = buf.readUInt32LE(offset + i * 4);
|
|
13025
|
+
}
|
|
13026
|
+
offset += colBytes;
|
|
13027
|
+
const tokens = new Array(tokenCount);
|
|
13028
|
+
let curLine = 0;
|
|
13029
|
+
let curCol = 0;
|
|
13030
|
+
let curPos = 0;
|
|
13031
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13032
|
+
curLine += dLines[i];
|
|
13033
|
+
curCol += dCols[i];
|
|
13034
|
+
curPos += dPos[i];
|
|
13035
|
+
const len = dLens[i];
|
|
13036
|
+
const dictIdx = indices[i];
|
|
13037
|
+
const hash2 = dictionary[dictIdx] ?? "";
|
|
13038
|
+
tokens[i] = {
|
|
13039
|
+
hash: hash2,
|
|
13040
|
+
line: curLine,
|
|
13041
|
+
column: curCol,
|
|
13042
|
+
position: curPos,
|
|
13043
|
+
range: [curPos, curPos + len]
|
|
13044
|
+
};
|
|
13045
|
+
}
|
|
13046
|
+
const minTokens = meta.minTokens ?? 40;
|
|
13047
|
+
const frames = reconstructFramesFromTokens(tokens, meta.sourceId, minTokens);
|
|
13048
|
+
return {
|
|
13049
|
+
version,
|
|
13050
|
+
sourceId: meta.sourceId,
|
|
13051
|
+
contentHash: meta.contentHash,
|
|
13052
|
+
format: meta.format,
|
|
13053
|
+
size: meta.size,
|
|
13054
|
+
lines: meta.lines,
|
|
13055
|
+
tokenCount,
|
|
13056
|
+
minTokens,
|
|
13057
|
+
updatedAt: meta.updatedAt,
|
|
13058
|
+
tokens,
|
|
13059
|
+
frames
|
|
13039
13060
|
};
|
|
13040
|
-
|
|
13041
|
-
|
|
13061
|
+
} catch {
|
|
13062
|
+
return null;
|
|
13042
13063
|
}
|
|
13043
|
-
let memoizedFrames = null;
|
|
13044
|
-
return {
|
|
13045
|
-
version: 1,
|
|
13046
|
-
sourceId,
|
|
13047
|
-
contentHash,
|
|
13048
|
-
format,
|
|
13049
|
-
size,
|
|
13050
|
-
lines,
|
|
13051
|
-
tokenCount,
|
|
13052
|
-
minTokens,
|
|
13053
|
-
updatedAt,
|
|
13054
|
-
tokens,
|
|
13055
|
-
get frames() {
|
|
13056
|
-
if (!memoizedFrames) {
|
|
13057
|
-
memoizedFrames = reconstructFramesFromTokens(tokens, sourceId, minTokens || 40);
|
|
13058
|
-
}
|
|
13059
|
-
return memoizedFrames;
|
|
13060
|
-
}
|
|
13061
|
-
};
|
|
13062
13064
|
}
|
|
13063
13065
|
|
|
13064
13066
|
class DiskCacheManager {
|
|
13065
13067
|
rootDir;
|
|
13068
|
+
repositoryKey;
|
|
13066
13069
|
baseCacheDir;
|
|
13067
13070
|
dbPath;
|
|
13068
13071
|
workspaceCacheDir;
|
|
@@ -13071,17 +13074,18 @@ class DiskCacheManager {
|
|
|
13071
13074
|
#db = null;
|
|
13072
13075
|
#getStmt = null;
|
|
13073
13076
|
#saveStmt = null;
|
|
13074
|
-
#updateMtimeStmt = null;
|
|
13075
13077
|
#deleteStmt = null;
|
|
13078
|
+
#deleteByRelPathStmt = null;
|
|
13076
13079
|
#totalSizeStmt = null;
|
|
13077
13080
|
#oldestShardsStmt = null;
|
|
13078
13081
|
#deleteAllStmt = null;
|
|
13079
13082
|
#closed = false;
|
|
13080
13083
|
constructor(options) {
|
|
13081
13084
|
this.rootDir = path3.resolve(options.rootDir);
|
|
13085
|
+
this.repositoryKey = options.repositoryKey ?? crypto.createHash("sha256").update(this.rootDir).digest("hex").slice(0, 16);
|
|
13082
13086
|
this.baseCacheDir = options.cacheDir ? path3.resolve(options.cacheDir) : getDefaultCacheDir();
|
|
13083
13087
|
this.configFingerprint = computeConfigFingerprint(options.config);
|
|
13084
|
-
this.dbPath = computeWorkspaceCachePath(this.baseCacheDir, this.
|
|
13088
|
+
this.dbPath = computeWorkspaceCachePath(this.baseCacheDir, this.repositoryKey, this.configFingerprint);
|
|
13085
13089
|
this.workspaceCacheDir = this.baseCacheDir;
|
|
13086
13090
|
this.maxBytes = options.maxBytes ?? DEFAULT_MAX_CACHE_BYTES;
|
|
13087
13091
|
}
|
|
@@ -13090,48 +13094,61 @@ class DiskCacheManager {
|
|
|
13090
13094
|
return null;
|
|
13091
13095
|
if (this.#db)
|
|
13092
13096
|
return this.#db;
|
|
13097
|
+
let db = null;
|
|
13093
13098
|
try {
|
|
13094
13099
|
const dir = path3.dirname(this.dbPath);
|
|
13095
13100
|
if (!fsSync.existsSync(dir)) {
|
|
13096
13101
|
fsSync.mkdirSync(dir, { recursive: true });
|
|
13097
13102
|
}
|
|
13098
|
-
|
|
13099
|
-
db.exec("PRAGMA
|
|
13103
|
+
db = new Database(this.dbPath, { create: true });
|
|
13104
|
+
db.exec("PRAGMA busy_timeout = 2000;");
|
|
13105
|
+
const journalRow = db.query("PRAGMA journal_mode = WAL;").get();
|
|
13106
|
+
if (journalRow?.journal_mode?.toLowerCase() !== "wal") {
|
|
13107
|
+
try {
|
|
13108
|
+
db.close();
|
|
13109
|
+
} catch {}
|
|
13110
|
+
return null;
|
|
13111
|
+
}
|
|
13100
13112
|
db.exec("PRAGMA synchronous = NORMAL;");
|
|
13101
13113
|
db.exec("PRAGMA temp_store = MEMORY;");
|
|
13102
13114
|
const versionRow = db.query("PRAGMA user_version;").get();
|
|
13103
13115
|
const schemaVersion = versionRow?.user_version ?? 0;
|
|
13104
13116
|
if (schemaVersion !== CACHE_FORMAT_VERSION) {
|
|
13105
|
-
|
|
13117
|
+
try {
|
|
13118
|
+
db.exec("DELETE FROM shards;");
|
|
13119
|
+
} catch {}
|
|
13106
13120
|
db.exec(`PRAGMA user_version = ${CACHE_FORMAT_VERSION};`);
|
|
13107
13121
|
}
|
|
13108
13122
|
db.exec(`
|
|
13109
13123
|
CREATE TABLE IF NOT EXISTS shards (
|
|
13110
|
-
rel_path TEXT NOT NULL
|
|
13124
|
+
rel_path TEXT NOT NULL,
|
|
13111
13125
|
content_hash TEXT NOT NULL,
|
|
13112
13126
|
payload BLOB NOT NULL,
|
|
13113
|
-
mtime REAL NOT NULL
|
|
13127
|
+
mtime REAL NOT NULL,
|
|
13128
|
+
PRIMARY KEY (rel_path, content_hash)
|
|
13114
13129
|
);
|
|
13115
|
-
CREATE INDEX IF NOT EXISTS idx_shards_content_hash ON shards(content_hash);
|
|
13116
13130
|
CREATE INDEX IF NOT EXISTS idx_shards_mtime ON shards(mtime);
|
|
13117
13131
|
`);
|
|
13118
|
-
this.#getStmt = db.prepare("SELECT payload
|
|
13132
|
+
this.#getStmt = db.prepare("SELECT payload FROM shards WHERE rel_path = ?1 AND content_hash = ?2");
|
|
13119
13133
|
this.#saveStmt = db.prepare(`
|
|
13120
13134
|
INSERT INTO shards (rel_path, content_hash, payload, mtime)
|
|
13121
13135
|
VALUES (?1, ?2, ?3, ?4)
|
|
13122
|
-
ON CONFLICT(rel_path) DO UPDATE SET
|
|
13123
|
-
content_hash = excluded.content_hash,
|
|
13124
|
-
payload = excluded.payload,
|
|
13136
|
+
ON CONFLICT(rel_path, content_hash) DO UPDATE SET
|
|
13125
13137
|
mtime = excluded.mtime
|
|
13126
13138
|
`);
|
|
13127
|
-
this.#
|
|
13128
|
-
this.#
|
|
13139
|
+
this.#deleteStmt = db.prepare("DELETE FROM shards WHERE rel_path = ?1 AND content_hash = ?2");
|
|
13140
|
+
this.#deleteByRelPathStmt = db.prepare("DELETE FROM shards WHERE rel_path = ?1");
|
|
13129
13141
|
this.#totalSizeStmt = db.prepare("SELECT COALESCE(SUM(LENGTH(payload)), 0) as total FROM shards");
|
|
13130
|
-
this.#oldestShardsStmt = db.prepare("SELECT
|
|
13142
|
+
this.#oldestShardsStmt = db.prepare("SELECT rowid, LENGTH(payload) as size FROM shards ORDER BY mtime ASC LIMIT ?1");
|
|
13131
13143
|
this.#deleteAllStmt = db.prepare("DELETE FROM shards");
|
|
13132
13144
|
this.#db = db;
|
|
13133
13145
|
return db;
|
|
13134
13146
|
} catch {
|
|
13147
|
+
if (db) {
|
|
13148
|
+
try {
|
|
13149
|
+
db.close();
|
|
13150
|
+
} catch {}
|
|
13151
|
+
}
|
|
13135
13152
|
return null;
|
|
13136
13153
|
}
|
|
13137
13154
|
}
|
|
@@ -13141,21 +13158,15 @@ class DiskCacheManager {
|
|
|
13141
13158
|
if (!db || !this.#getStmt)
|
|
13142
13159
|
return null;
|
|
13143
13160
|
const normalizedRelPath = relPath.replace(/\\/g, "/");
|
|
13144
|
-
const row = this.#getStmt.get(normalizedRelPath);
|
|
13145
|
-
if (!row
|
|
13161
|
+
const row = this.#getStmt.get(normalizedRelPath, contentHash);
|
|
13162
|
+
if (!row) {
|
|
13146
13163
|
return null;
|
|
13147
13164
|
}
|
|
13148
13165
|
const payloadBuf = Buffer.isBuffer(row.payload) ? row.payload : Buffer.from(row.payload.buffer, row.payload.byteOffset, row.payload.byteLength);
|
|
13149
13166
|
const shard = unpackBinaryShard(payloadBuf);
|
|
13150
|
-
if (shard && shard.contentHash === contentHash &&
|
|
13151
|
-
try {
|
|
13152
|
-
this.#updateMtimeStmt?.run(Date.now(), normalizedRelPath);
|
|
13153
|
-
} catch {}
|
|
13167
|
+
if (shard && shard.contentHash === contentHash && Array.isArray(shard.frames)) {
|
|
13154
13168
|
return shard;
|
|
13155
13169
|
}
|
|
13156
|
-
try {
|
|
13157
|
-
this.#deleteStmt?.run(normalizedRelPath);
|
|
13158
|
-
} catch {}
|
|
13159
13170
|
return null;
|
|
13160
13171
|
} catch {
|
|
13161
13172
|
return null;
|
|
@@ -13172,6 +13183,53 @@ class DiskCacheManager {
|
|
|
13172
13183
|
this.#saveStmt.run(normalizedRelPath, shard.contentHash, payload, Date.now());
|
|
13173
13184
|
} catch {}
|
|
13174
13185
|
}
|
|
13186
|
+
async deleteShard(relPath, contentHash) {
|
|
13187
|
+
try {
|
|
13188
|
+
const db = this.#getDb();
|
|
13189
|
+
if (!db || !this.#deleteStmt)
|
|
13190
|
+
return;
|
|
13191
|
+
const normalizedRelPath = relPath.replace(/\\/g, "/");
|
|
13192
|
+
this.#deleteStmt.run(normalizedRelPath, contentHash);
|
|
13193
|
+
} catch {}
|
|
13194
|
+
}
|
|
13195
|
+
async deleteByRelPath(relPath) {
|
|
13196
|
+
try {
|
|
13197
|
+
const db = this.#getDb();
|
|
13198
|
+
if (!db || !this.#deleteByRelPathStmt)
|
|
13199
|
+
return;
|
|
13200
|
+
const normalizedRelPath = relPath.replace(/\\/g, "/");
|
|
13201
|
+
this.#deleteByRelPathStmt.run(normalizedRelPath);
|
|
13202
|
+
} catch {}
|
|
13203
|
+
}
|
|
13204
|
+
async saveShards(items) {
|
|
13205
|
+
if (items.length === 0)
|
|
13206
|
+
return;
|
|
13207
|
+
try {
|
|
13208
|
+
const db = this.#getDb();
|
|
13209
|
+
if (!db || !this.#saveStmt)
|
|
13210
|
+
return;
|
|
13211
|
+
const stmt = this.#saveStmt;
|
|
13212
|
+
const root = this.rootDir;
|
|
13213
|
+
const now = Date.now();
|
|
13214
|
+
const prepared = [];
|
|
13215
|
+
for (const item of items) {
|
|
13216
|
+
const targetRelPath = item.relPath ?? (path3.isAbsolute(item.shard.sourceId) ? path3.relative(root, item.shard.sourceId) : item.shard.sourceId);
|
|
13217
|
+
const normalizedRelPath = targetRelPath.replace(/\\/g, "/");
|
|
13218
|
+
const payload = packBinaryShard(item.shard);
|
|
13219
|
+
prepared.push({
|
|
13220
|
+
relPath: normalizedRelPath,
|
|
13221
|
+
hash: item.shard.contentHash,
|
|
13222
|
+
payload
|
|
13223
|
+
});
|
|
13224
|
+
}
|
|
13225
|
+
const tx = db.transaction((entries) => {
|
|
13226
|
+
for (const entry of entries) {
|
|
13227
|
+
stmt.run(entry.relPath, entry.hash, entry.payload, now);
|
|
13228
|
+
}
|
|
13229
|
+
});
|
|
13230
|
+
tx(prepared);
|
|
13231
|
+
} catch {}
|
|
13232
|
+
}
|
|
13175
13233
|
async prune(maxBytes) {
|
|
13176
13234
|
const budget = maxBytes !== undefined ? maxBytes : this.maxBytes;
|
|
13177
13235
|
try {
|
|
@@ -13180,53 +13238,41 @@ class DiskCacheManager {
|
|
|
13180
13238
|
return;
|
|
13181
13239
|
if (budget <= 0) {
|
|
13182
13240
|
this.#deleteAllStmt?.run();
|
|
13183
|
-
try {
|
|
13184
|
-
db.exec("VACUUM;");
|
|
13185
|
-
} catch {}
|
|
13186
13241
|
return;
|
|
13187
13242
|
}
|
|
13188
|
-
|
|
13189
|
-
|
|
13190
|
-
|
|
13191
|
-
return;
|
|
13192
|
-
}
|
|
13193
|
-
const oldestShards = this.#oldestShardsStmt?.all() ?? [];
|
|
13194
|
-
let deletedAny = false;
|
|
13195
|
-
for (const entry of oldestShards) {
|
|
13243
|
+
for (let iter = 0;iter < 10; iter++) {
|
|
13244
|
+
const totalRow = this.#totalSizeStmt?.get();
|
|
13245
|
+
const totalSize = totalRow?.total ?? 0;
|
|
13196
13246
|
if (totalSize <= budget) {
|
|
13197
13247
|
break;
|
|
13198
13248
|
}
|
|
13199
|
-
|
|
13200
|
-
|
|
13201
|
-
|
|
13202
|
-
|
|
13203
|
-
|
|
13204
|
-
|
|
13205
|
-
|
|
13206
|
-
|
|
13207
|
-
|
|
13208
|
-
|
|
13249
|
+
const excess = totalSize - budget;
|
|
13250
|
+
const rows = this.#oldestShardsStmt?.all(100) ?? [];
|
|
13251
|
+
if (rows.length === 0)
|
|
13252
|
+
break;
|
|
13253
|
+
const rowidsToDelete = [];
|
|
13254
|
+
let freed = 0;
|
|
13255
|
+
for (const r of rows) {
|
|
13256
|
+
rowidsToDelete.push(r.rowid);
|
|
13257
|
+
freed += r.size;
|
|
13258
|
+
if (freed >= excess)
|
|
13259
|
+
break;
|
|
13260
|
+
}
|
|
13261
|
+
if (rowidsToDelete.length > 0) {
|
|
13262
|
+
const deleteBatchStmt = db.prepare(`DELETE FROM shards WHERE rowid IN (${rowidsToDelete.join(",")})`);
|
|
13263
|
+
deleteBatchStmt.run();
|
|
13264
|
+
} else {
|
|
13265
|
+
break;
|
|
13266
|
+
}
|
|
13209
13267
|
}
|
|
13210
13268
|
} catch {}
|
|
13211
13269
|
}
|
|
13212
13270
|
async clear() {
|
|
13213
13271
|
try {
|
|
13214
|
-
|
|
13215
|
-
|
|
13216
|
-
|
|
13217
|
-
|
|
13218
|
-
this.#db = null;
|
|
13219
|
-
this.#getStmt = null;
|
|
13220
|
-
this.#saveStmt = null;
|
|
13221
|
-
this.#updateMtimeStmt = null;
|
|
13222
|
-
this.#deleteStmt = null;
|
|
13223
|
-
this.#totalSizeStmt = null;
|
|
13224
|
-
this.#oldestShardsStmt = null;
|
|
13225
|
-
this.#deleteAllStmt = null;
|
|
13226
|
-
}
|
|
13227
|
-
await fs.unlink(this.dbPath).catch(() => {});
|
|
13228
|
-
await fs.unlink(`${this.dbPath}-wal`).catch(() => {});
|
|
13229
|
-
await fs.unlink(`${this.dbPath}-shm`).catch(() => {});
|
|
13272
|
+
const db = this.#getDb();
|
|
13273
|
+
if (db && this.#deleteAllStmt) {
|
|
13274
|
+
this.#deleteAllStmt.run();
|
|
13275
|
+
}
|
|
13230
13276
|
} catch {}
|
|
13231
13277
|
}
|
|
13232
13278
|
close() {
|
|
@@ -13238,14 +13284,118 @@ class DiskCacheManager {
|
|
|
13238
13284
|
this.#db = null;
|
|
13239
13285
|
this.#getStmt = null;
|
|
13240
13286
|
this.#saveStmt = null;
|
|
13241
|
-
this.#updateMtimeStmt = null;
|
|
13242
13287
|
this.#deleteStmt = null;
|
|
13288
|
+
this.#deleteByRelPathStmt = null;
|
|
13243
13289
|
this.#totalSizeStmt = null;
|
|
13244
13290
|
this.#oldestShardsStmt = null;
|
|
13245
13291
|
this.#deleteAllStmt = null;
|
|
13246
13292
|
}
|
|
13247
13293
|
}
|
|
13248
13294
|
}
|
|
13295
|
+
async function cleanupLegacyCacheFiles(customCacheDir) {
|
|
13296
|
+
const baseDir = customCacheDir ?? getDefaultCacheDir();
|
|
13297
|
+
try {
|
|
13298
|
+
if (!fsSync.existsSync(baseDir))
|
|
13299
|
+
return;
|
|
13300
|
+
const entries = await fs.readdir(baseDir);
|
|
13301
|
+
for (const entry of entries) {
|
|
13302
|
+
if (entry.endsWith(".sqlite") && !entry.includes(`_v${CACHE_FORMAT_VERSION}.sqlite`)) {
|
|
13303
|
+
await fs.unlink(path3.join(baseDir, entry)).catch(() => {});
|
|
13304
|
+
await fs.unlink(path3.join(baseDir, `${entry}-wal`)).catch(() => {});
|
|
13305
|
+
await fs.unlink(path3.join(baseDir, `${entry}-shm`)).catch(() => {});
|
|
13306
|
+
}
|
|
13307
|
+
}
|
|
13308
|
+
} catch {}
|
|
13309
|
+
}
|
|
13310
|
+
|
|
13311
|
+
// src/repo-context.ts
|
|
13312
|
+
import * as crypto2 from "crypto";
|
|
13313
|
+
import * as fsSync2 from "fs";
|
|
13314
|
+
import * as fs2 from "fs/promises";
|
|
13315
|
+
import * as path4 from "path";
|
|
13316
|
+
function canonicalizePath(targetPath) {
|
|
13317
|
+
const resolved = path4.resolve(targetPath);
|
|
13318
|
+
let current = resolved;
|
|
13319
|
+
let suffix = "";
|
|
13320
|
+
while (current && current !== path4.dirname(current)) {
|
|
13321
|
+
try {
|
|
13322
|
+
if (fsSync2.existsSync(current)) {
|
|
13323
|
+
const real = fsSync2.realpathSync(current);
|
|
13324
|
+
return suffix ? path4.join(real, suffix) : real;
|
|
13325
|
+
}
|
|
13326
|
+
} catch {}
|
|
13327
|
+
suffix = suffix ? path4.join(path4.basename(current), suffix) : path4.basename(current);
|
|
13328
|
+
current = path4.dirname(current);
|
|
13329
|
+
}
|
|
13330
|
+
return resolved;
|
|
13331
|
+
}
|
|
13332
|
+
function isOmpWorktreePath(targetPath) {
|
|
13333
|
+
const normalized = targetPath.replace(/\\/g, "/");
|
|
13334
|
+
return normalized.includes("/.omp/wt/") || normalized.includes("/.omp/worktrees/");
|
|
13335
|
+
}
|
|
13336
|
+
async function resolveRepositoryContext(cwd, signal) {
|
|
13337
|
+
const canonicalCwd = canonicalizePath(cwd);
|
|
13338
|
+
try {
|
|
13339
|
+
const { stdout } = await execGit([
|
|
13340
|
+
"rev-parse",
|
|
13341
|
+
"--path-format=absolute",
|
|
13342
|
+
"--show-toplevel",
|
|
13343
|
+
"--git-dir",
|
|
13344
|
+
"--git-common-dir"
|
|
13345
|
+
], canonicalCwd, { signal });
|
|
13346
|
+
const lines = stdout.split(`
|
|
13347
|
+
`).map((l) => l.trim()).filter(Boolean);
|
|
13348
|
+
const resolveEntry = (val) => {
|
|
13349
|
+
if (!val)
|
|
13350
|
+
return "";
|
|
13351
|
+
return path4.isAbsolute(val) ? val : path4.resolve(canonicalCwd, val);
|
|
13352
|
+
};
|
|
13353
|
+
const workspaceRoot = canonicalizePath(resolveEntry(lines[0]));
|
|
13354
|
+
const gitDir = canonicalizePath(resolveEntry(lines[1] || path4.join(workspaceRoot, ".git")));
|
|
13355
|
+
const commonGitDir = canonicalizePath(resolveEntry(lines[2] || lines[1] || path4.join(workspaceRoot, ".git")));
|
|
13356
|
+
let repositoryObjectDir = path4.join(commonGitDir, "objects");
|
|
13357
|
+
const isOmpIsolation = isOmpWorktreePath(workspaceRoot) || isOmpWorktreePath(canonicalCwd);
|
|
13358
|
+
if (isOmpIsolation && gitDir === commonGitDir) {
|
|
13359
|
+
const alternatesFile = path4.join(gitDir, "objects", "info", "alternates");
|
|
13360
|
+
try {
|
|
13361
|
+
if (fsSync2.existsSync(alternatesFile)) {
|
|
13362
|
+
const content = await fs2.readFile(alternatesFile, "utf-8");
|
|
13363
|
+
const altLines = content.split(`
|
|
13364
|
+
`).map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
13365
|
+
for (const line of altLines) {
|
|
13366
|
+
const resolvedAlt = path4.isAbsolute(line) ? line : path4.resolve(path4.join(gitDir, "objects"), line);
|
|
13367
|
+
const canonicalAlt = canonicalizePath(resolvedAlt);
|
|
13368
|
+
if (fsSync2.existsSync(canonicalAlt)) {
|
|
13369
|
+
repositoryObjectDir = canonicalAlt;
|
|
13370
|
+
break;
|
|
13371
|
+
}
|
|
13372
|
+
}
|
|
13373
|
+
}
|
|
13374
|
+
} catch {}
|
|
13375
|
+
}
|
|
13376
|
+
repositoryObjectDir = canonicalizePath(repositoryObjectDir);
|
|
13377
|
+
const repositoryKey = crypto2.createHash("sha256").update(`git-object-dir\x00${repositoryObjectDir}`).digest("hex").slice(0, 16);
|
|
13378
|
+
return {
|
|
13379
|
+
workspaceRoot,
|
|
13380
|
+
isGit: true,
|
|
13381
|
+
gitDir,
|
|
13382
|
+
commonGitDir,
|
|
13383
|
+
repositoryObjectDir,
|
|
13384
|
+
repositoryKey,
|
|
13385
|
+
isOmpIsolation
|
|
13386
|
+
};
|
|
13387
|
+
} catch {
|
|
13388
|
+
const repositoryObjectDir = path4.join(canonicalCwd, ".non-git");
|
|
13389
|
+
const repositoryKey = crypto2.createHash("sha256").update(`directory\x00${canonicalCwd}`).digest("hex").slice(0, 16);
|
|
13390
|
+
return {
|
|
13391
|
+
workspaceRoot: canonicalCwd,
|
|
13392
|
+
isGit: false,
|
|
13393
|
+
repositoryObjectDir,
|
|
13394
|
+
repositoryKey,
|
|
13395
|
+
isOmpIsolation: false
|
|
13396
|
+
};
|
|
13397
|
+
}
|
|
13398
|
+
}
|
|
13249
13399
|
|
|
13250
13400
|
// src/worker-protocol.ts
|
|
13251
13401
|
var REQUEST_TYPES = {
|
|
@@ -13351,22 +13501,74 @@ function areOptionsEqual(a, b) {
|
|
|
13351
13501
|
const bFormats = b.formatsExts ? JSON.stringify(b.formatsExts) : "";
|
|
13352
13502
|
return aFormats === bFormats;
|
|
13353
13503
|
}
|
|
13504
|
+
function filterAndEvictStaleClones(clones, activeFilePath) {
|
|
13505
|
+
if (!currentRootDir || !fsSync3.existsSync(currentRootDir)) {
|
|
13506
|
+
return clones;
|
|
13507
|
+
}
|
|
13508
|
+
const validClones = [];
|
|
13509
|
+
const evictedSources = new Set;
|
|
13510
|
+
const activeRes = activeFilePath ? path5.resolve(activeFilePath) : null;
|
|
13511
|
+
const activeCan = activeFilePath ? canonicalizePath(activeFilePath) : null;
|
|
13512
|
+
const isActive = (src) => {
|
|
13513
|
+
if (!activeFilePath)
|
|
13514
|
+
return false;
|
|
13515
|
+
if (src === activeFilePath)
|
|
13516
|
+
return true;
|
|
13517
|
+
if (activeRes && path5.resolve(src) === activeRes)
|
|
13518
|
+
return true;
|
|
13519
|
+
if (activeCan && canonicalizePath(src) === activeCan)
|
|
13520
|
+
return true;
|
|
13521
|
+
return false;
|
|
13522
|
+
};
|
|
13523
|
+
const isStaleSource = (src) => {
|
|
13524
|
+
if (isActive(src))
|
|
13525
|
+
return false;
|
|
13526
|
+
if (evictedSources.has(src))
|
|
13527
|
+
return true;
|
|
13528
|
+
const can = canonicalizePath(path5.isAbsolute(src) ? src : path5.resolve(currentRootDir, src));
|
|
13529
|
+
const rel = path5.relative(currentRootDir, can);
|
|
13530
|
+
const isInside = !rel.startsWith("..") && !path5.isAbsolute(rel);
|
|
13531
|
+
if (isInside && !fsSync3.existsSync(can)) {
|
|
13532
|
+
evictedSources.add(src);
|
|
13533
|
+
currentIndex.removeSource(src);
|
|
13534
|
+
watchedRevisions.delete(src);
|
|
13535
|
+
watchedRevisions.delete(can);
|
|
13536
|
+
watchedRevisions.delete(path5.resolve(src));
|
|
13537
|
+
if (currentDiskCache) {
|
|
13538
|
+
currentDiskCache.deleteByRelPath(rel).catch(() => {});
|
|
13539
|
+
}
|
|
13540
|
+
return true;
|
|
13541
|
+
}
|
|
13542
|
+
return false;
|
|
13543
|
+
};
|
|
13544
|
+
for (const clone2 of clones) {
|
|
13545
|
+
const srcA = clone2.duplicationA.sourceId;
|
|
13546
|
+
const srcB = clone2.duplicationB.sourceId;
|
|
13547
|
+
if (isStaleSource(srcA) || isStaleSource(srcB)) {
|
|
13548
|
+
continue;
|
|
13549
|
+
}
|
|
13550
|
+
validClones.push(clone2);
|
|
13551
|
+
}
|
|
13552
|
+
return validClones;
|
|
13553
|
+
}
|
|
13354
13554
|
function notifyLateFindings(clones) {
|
|
13355
13555
|
for (const clone2 of clones) {
|
|
13356
13556
|
const srcA = clone2.duplicationA.sourceId;
|
|
13357
13557
|
const srcB = clone2.duplicationB.sourceId;
|
|
13358
|
-
const resA =
|
|
13359
|
-
const resB =
|
|
13360
|
-
const
|
|
13361
|
-
const
|
|
13558
|
+
const resA = path5.resolve(srcA);
|
|
13559
|
+
const resB = path5.resolve(srcB);
|
|
13560
|
+
const canA = canonicalizePath(srcA);
|
|
13561
|
+
const canB = canonicalizePath(srcB);
|
|
13562
|
+
const isWatchedA = watchedRevisions.has(srcA) || watchedRevisions.has(resA) || watchedRevisions.has(canA);
|
|
13563
|
+
const isWatchedB = watchedRevisions.has(srcB) || watchedRevisions.has(resB) || watchedRevisions.has(canB);
|
|
13362
13564
|
if (isWatchedA || isWatchedB) {
|
|
13363
13565
|
if (isWatchedA) {
|
|
13364
|
-
const entry = watchedRevisions.get(srcA) ?? watchedRevisions.get(resA);
|
|
13566
|
+
const entry = watchedRevisions.get(srcA) ?? watchedRevisions.get(resA) ?? watchedRevisions.get(canA);
|
|
13365
13567
|
if (entry)
|
|
13366
13568
|
entry.lastKnownCloneCount++;
|
|
13367
13569
|
}
|
|
13368
13570
|
if (isWatchedB) {
|
|
13369
|
-
const entry = watchedRevisions.get(srcB) ?? watchedRevisions.get(resB);
|
|
13571
|
+
const entry = watchedRevisions.get(srcB) ?? watchedRevisions.get(resB) ?? watchedRevisions.get(canB);
|
|
13370
13572
|
if (entry)
|
|
13371
13573
|
entry.lastKnownCloneCount++;
|
|
13372
13574
|
}
|
|
@@ -13377,16 +13579,16 @@ function notifyLateFindings(clones) {
|
|
|
13377
13579
|
function cacheSourceShard(filePath, content) {
|
|
13378
13580
|
if (!currentDiskCache)
|
|
13379
13581
|
return;
|
|
13380
|
-
const contentHash =
|
|
13381
|
-
const relPath =
|
|
13582
|
+
const contentHash = crypto3.createHash("sha256").update(content).digest("hex");
|
|
13583
|
+
const relPath = path5.relative(currentRootDir, filePath).replace(/\\/g, "/");
|
|
13382
13584
|
const shard = currentIndex.exportSourceShard(filePath, contentHash);
|
|
13383
13585
|
if (shard) {
|
|
13384
13586
|
currentDiskCache.saveShard(shard, relPath).catch(() => {});
|
|
13385
13587
|
}
|
|
13386
13588
|
}
|
|
13387
13589
|
function yieldTask() {
|
|
13388
|
-
const { promise, resolve:
|
|
13389
|
-
setTimeout(
|
|
13590
|
+
const { promise, resolve: resolve5 } = Promise.withResolvers();
|
|
13591
|
+
setTimeout(resolve5, 0);
|
|
13390
13592
|
return promise;
|
|
13391
13593
|
}
|
|
13392
13594
|
async function runBaselineIndexing(rootDir, options, signal) {
|
|
@@ -13453,11 +13655,11 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13453
13655
|
return null;
|
|
13454
13656
|
}
|
|
13455
13657
|
try {
|
|
13456
|
-
const stat2 = await
|
|
13658
|
+
const stat2 = await fs3.stat(filePath);
|
|
13457
13659
|
if (stat2.size > MAX_FILE_SIZE_BYTES || stat2.size <= 0) {
|
|
13458
13660
|
return null;
|
|
13459
13661
|
}
|
|
13460
|
-
const resolved =
|
|
13662
|
+
const resolved = path5.resolve(filePath);
|
|
13461
13663
|
if (currentIndex.hasSource(filePath) || currentIndex.hasSource(resolved)) {
|
|
13462
13664
|
return {
|
|
13463
13665
|
filePath,
|
|
@@ -13470,12 +13672,12 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13470
13672
|
alreadyIndexed: true
|
|
13471
13673
|
};
|
|
13472
13674
|
}
|
|
13473
|
-
const content = await
|
|
13675
|
+
const content = await fs3.readFile(filePath, "utf8");
|
|
13474
13676
|
if (signal.aborted || isGeneratedContent(content)) {
|
|
13475
13677
|
return null;
|
|
13476
13678
|
}
|
|
13477
|
-
const contentHash =
|
|
13478
|
-
const relPath =
|
|
13679
|
+
const contentHash = crypto3.createHash("sha256").update(content).digest("hex");
|
|
13680
|
+
const relPath = path5.relative(rootDir, filePath).replace(/\\/g, "/");
|
|
13479
13681
|
const cachedShard = currentDiskCache ? await currentDiskCache.getShard(relPath, contentHash) : null;
|
|
13480
13682
|
let isNewlyTokenized = false;
|
|
13481
13683
|
let shard = cachedShard;
|
|
@@ -13499,6 +13701,7 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13499
13701
|
}));
|
|
13500
13702
|
if (signal.aborted)
|
|
13501
13703
|
return { indexedCount, status: "cancelled" };
|
|
13704
|
+
const shardsToSave = [];
|
|
13502
13705
|
for (const item of fileItems) {
|
|
13503
13706
|
if (!item)
|
|
13504
13707
|
continue;
|
|
@@ -13513,7 +13716,10 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13513
13716
|
newClones = currentIndex.hydrateSourceShard(item.cachedShard);
|
|
13514
13717
|
if (item.isNewlyTokenized) {
|
|
13515
13718
|
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
13516
|
-
|
|
13719
|
+
shardsToSave.push({
|
|
13720
|
+
shard: item.cachedShard,
|
|
13721
|
+
relPath: item.relPath
|
|
13722
|
+
});
|
|
13517
13723
|
}
|
|
13518
13724
|
} else {
|
|
13519
13725
|
cachedCount++;
|
|
@@ -13523,7 +13729,7 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13523
13729
|
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
13524
13730
|
const shard = currentIndex.exportSourceShard(item.filePath, item.contentHash);
|
|
13525
13731
|
if (shard) {
|
|
13526
|
-
|
|
13732
|
+
shardsToSave.push({ shard, relPath: item.relPath });
|
|
13527
13733
|
}
|
|
13528
13734
|
}
|
|
13529
13735
|
}
|
|
@@ -13533,6 +13739,9 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13533
13739
|
notifyLateFindings(newClones);
|
|
13534
13740
|
}
|
|
13535
13741
|
}
|
|
13742
|
+
if (shardsToSave.length > 0 && currentDiskCache) {
|
|
13743
|
+
await currentDiskCache.saveShards(shardsToSave);
|
|
13744
|
+
}
|
|
13536
13745
|
const processedCount = Math.min(i + batch.length, totalFiles);
|
|
13537
13746
|
const percentage = totalFiles > 0 ? Math.round(processedCount / totalFiles * 100) : 100;
|
|
13538
13747
|
self.postMessage(createProgressEvent({
|
|
@@ -13619,12 +13828,12 @@ async function runIncrementalGitReconciliation(rootDir, options, signal) {
|
|
|
13619
13828
|
const oldRelPath = entries[i]?.trim();
|
|
13620
13829
|
i++;
|
|
13621
13830
|
if (oldRelPath) {
|
|
13622
|
-
const oldFullPath =
|
|
13831
|
+
const oldFullPath = path5.resolve(rootDir, oldRelPath);
|
|
13623
13832
|
currentIndex.removeSource(oldFullPath);
|
|
13624
13833
|
currentIndex.removeSource(oldRelPath);
|
|
13625
13834
|
}
|
|
13626
13835
|
}
|
|
13627
|
-
const fullPath =
|
|
13836
|
+
const fullPath = path5.resolve(rootDir, relPath);
|
|
13628
13837
|
if (ignoreFilter(relPath)) {
|
|
13629
13838
|
currentIndex.removeSource(fullPath);
|
|
13630
13839
|
currentIndex.removeSource(relPath);
|
|
@@ -13639,12 +13848,12 @@ async function runIncrementalGitReconciliation(rootDir, options, signal) {
|
|
|
13639
13848
|
currentIndex.removeSource(fullPath);
|
|
13640
13849
|
continue;
|
|
13641
13850
|
}
|
|
13642
|
-
const stat2 = await
|
|
13851
|
+
const stat2 = await fs3.stat(fullPath);
|
|
13643
13852
|
if (stat2.size > MAX_FILE_SIZE_BYTES || stat2.size <= 0) {
|
|
13644
13853
|
currentIndex.removeSource(fullPath);
|
|
13645
13854
|
continue;
|
|
13646
13855
|
}
|
|
13647
|
-
const content = await
|
|
13856
|
+
const content = await fs3.readFile(fullPath, "utf8");
|
|
13648
13857
|
if (signal.aborted)
|
|
13649
13858
|
return {
|
|
13650
13859
|
indexedCount: currentIndex.stats().sourceCount,
|
|
@@ -13689,16 +13898,17 @@ async function handleWorkerRequest(msg) {
|
|
|
13689
13898
|
switch (msg.type) {
|
|
13690
13899
|
case "openWorkspace": {
|
|
13691
13900
|
const { rootDir, options } = msg.payload;
|
|
13692
|
-
|
|
13901
|
+
const canonicalRootDir = canonicalizePath(rootDir);
|
|
13902
|
+
if ((currentRootDir === canonicalRootDir || currentRootDir === rootDir) && areOptionsEqual(currentOptions, options) && (isBaselineComplete || isBaselineIndexing)) {
|
|
13693
13903
|
if (isBaselineComplete) {
|
|
13694
13904
|
if (activeAbortController) {
|
|
13695
13905
|
activeAbortController.abort();
|
|
13696
13906
|
}
|
|
13697
13907
|
activeAbortController = new AbortController;
|
|
13698
|
-
const recResult = await runIncrementalGitReconciliation(
|
|
13908
|
+
const recResult = await runIncrementalGitReconciliation(currentRootDir, options, activeAbortController.signal);
|
|
13699
13909
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13700
13910
|
started: true,
|
|
13701
|
-
rootDir,
|
|
13911
|
+
rootDir: currentRootDir,
|
|
13702
13912
|
reused: true,
|
|
13703
13913
|
indexedCount: recResult.indexedCount,
|
|
13704
13914
|
status: recResult.status
|
|
@@ -13706,7 +13916,7 @@ async function handleWorkerRequest(msg) {
|
|
|
13706
13916
|
} else {
|
|
13707
13917
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13708
13918
|
started: true,
|
|
13709
|
-
rootDir,
|
|
13919
|
+
rootDir: currentRootDir,
|
|
13710
13920
|
reused: true,
|
|
13711
13921
|
indexedCount: currentIndex.stats().sourceCount,
|
|
13712
13922
|
status: "complete"
|
|
@@ -13722,11 +13932,14 @@ async function handleWorkerRequest(msg) {
|
|
|
13722
13932
|
currentDiskCache.close();
|
|
13723
13933
|
currentDiskCache = null;
|
|
13724
13934
|
}
|
|
13725
|
-
|
|
13935
|
+
const repoContext = await resolveRepositoryContext(rootDir, activeAbortController.signal);
|
|
13936
|
+
const effectiveRoot = repoContext.workspaceRoot;
|
|
13937
|
+
currentRootDir = effectiveRoot;
|
|
13726
13938
|
currentOptions = options;
|
|
13727
13939
|
currentIndex = new SourceAwareCloneIndex(options);
|
|
13728
13940
|
currentDiskCache = new DiskCacheManager({
|
|
13729
|
-
rootDir,
|
|
13941
|
+
rootDir: effectiveRoot,
|
|
13942
|
+
repositoryKey: repoContext.repositoryKey,
|
|
13730
13943
|
cacheDir: options?.cacheDir,
|
|
13731
13944
|
config: options,
|
|
13732
13945
|
maxBytes: options?.maxCacheBytes
|
|
@@ -13734,48 +13947,58 @@ async function handleWorkerRequest(msg) {
|
|
|
13734
13947
|
watchedRevisions.clear();
|
|
13735
13948
|
isBaselineIndexing = true;
|
|
13736
13949
|
isBaselineComplete = false;
|
|
13950
|
+
cleanupLegacyCacheFiles(options?.cacheDir).catch(() => {});
|
|
13737
13951
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13738
13952
|
started: true,
|
|
13739
|
-
rootDir,
|
|
13953
|
+
rootDir: effectiveRoot,
|
|
13740
13954
|
reused: false
|
|
13741
13955
|
}));
|
|
13742
|
-
runBaselineIndexing(
|
|
13956
|
+
runBaselineIndexing(effectiveRoot, options, activeAbortController.signal).catch(() => {});
|
|
13743
13957
|
break;
|
|
13744
13958
|
}
|
|
13745
13959
|
case "checkSnippet": {
|
|
13746
13960
|
const { filePath, content, format } = msg.payload;
|
|
13747
|
-
const
|
|
13961
|
+
const rawClones = currentIndex.checkSnippet(filePath, content, format);
|
|
13962
|
+
const clones = filterAndEvictStaleClones(rawClones, filePath);
|
|
13748
13963
|
self.postMessage(createSuccessResponse(msg.id, clones));
|
|
13749
13964
|
break;
|
|
13750
13965
|
}
|
|
13751
13966
|
case "checkAndUpdate": {
|
|
13752
13967
|
const { filePath, content, format, revision = 1 } = msg.payload;
|
|
13753
|
-
const
|
|
13754
|
-
const
|
|
13968
|
+
const rawClones = currentIndex.updateSource(filePath, content, format);
|
|
13969
|
+
const canonicalFilePath = canonicalizePath(filePath);
|
|
13970
|
+
const resolvedPath = path5.resolve(filePath);
|
|
13755
13971
|
cacheSourceShard(filePath, content);
|
|
13756
|
-
const
|
|
13972
|
+
const clones = filterAndEvictStaleClones(rawClones, filePath);
|
|
13973
|
+
const fileClones = clones.filter((c) => c.duplicationA.sourceId === filePath || c.duplicationB.sourceId === filePath || path5.resolve(c.duplicationA.sourceId) === resolvedPath || path5.resolve(c.duplicationB.sourceId) === resolvedPath || canonicalizePath(c.duplicationA.sourceId) === canonicalFilePath || canonicalizePath(c.duplicationB.sourceId) === canonicalFilePath);
|
|
13757
13974
|
const watchEntry = {
|
|
13758
13975
|
revision,
|
|
13759
13976
|
lastKnownCloneCount: fileClones.length
|
|
13760
13977
|
};
|
|
13761
13978
|
watchedRevisions.set(filePath, watchEntry);
|
|
13762
13979
|
watchedRevisions.set(resolvedPath, watchEntry);
|
|
13980
|
+
watchedRevisions.set(canonicalFilePath, watchEntry);
|
|
13763
13981
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13764
|
-
clones,
|
|
13982
|
+
clones: fileClones,
|
|
13765
13983
|
isComplete: isBaselineComplete
|
|
13766
13984
|
}));
|
|
13767
13985
|
break;
|
|
13768
13986
|
}
|
|
13769
13987
|
case "updateFile": {
|
|
13770
13988
|
const { filePath, content, format } = msg.payload;
|
|
13771
|
-
const
|
|
13989
|
+
const rawClones = currentIndex.updateSource(filePath, content, format);
|
|
13772
13990
|
cacheSourceShard(filePath, content);
|
|
13991
|
+
const clones = filterAndEvictStaleClones(rawClones, filePath);
|
|
13773
13992
|
self.postMessage(createSuccessResponse(msg.id, { clones }));
|
|
13774
13993
|
break;
|
|
13775
13994
|
}
|
|
13776
13995
|
case "removeFile": {
|
|
13777
13996
|
const { filePath } = msg.payload;
|
|
13778
13997
|
currentIndex.removeSource(filePath);
|
|
13998
|
+
if (currentDiskCache) {
|
|
13999
|
+
const relPath = path5.isAbsolute(filePath) && currentRootDir ? path5.relative(currentRootDir, filePath) : filePath;
|
|
14000
|
+
currentDiskCache.deleteByRelPath(relPath).catch(() => {});
|
|
14001
|
+
}
|
|
13779
14002
|
self.postMessage(createSuccessResponse(msg.id, { removed: true }));
|
|
13780
14003
|
break;
|
|
13781
14004
|
}
|
|
@@ -13791,11 +14014,15 @@ async function handleWorkerRequest(msg) {
|
|
|
13791
14014
|
} else {
|
|
13792
14015
|
if (!getSupportedCodeFormat(fileEntry.filePath, currentOptions?.formatsExts)) {
|
|
13793
14016
|
currentIndex.removeSource(fileEntry.filePath);
|
|
14017
|
+
if (currentDiskCache) {
|
|
14018
|
+
const relPath = path5.isAbsolute(fileEntry.filePath) && currentRootDir ? path5.relative(currentRootDir, fileEntry.filePath) : fileEntry.filePath;
|
|
14019
|
+
currentDiskCache.deleteByRelPath(relPath).catch(() => {});
|
|
14020
|
+
}
|
|
13794
14021
|
continue;
|
|
13795
14022
|
}
|
|
13796
|
-
const stat2 = await
|
|
14023
|
+
const stat2 = await fs3.stat(fileEntry.filePath);
|
|
13797
14024
|
if (stat2.size <= MAX_FILE_SIZE_BYTES && stat2.size > 0) {
|
|
13798
|
-
const content = await
|
|
14025
|
+
const content = await fs3.readFile(fileEntry.filePath, "utf8");
|
|
13799
14026
|
if (!isGeneratedContent(content)) {
|
|
13800
14027
|
currentIndex.updateSource(fileEntry.filePath, content);
|
|
13801
14028
|
cacheSourceShard(fileEntry.filePath, content);
|
|
@@ -13805,6 +14032,10 @@ async function handleWorkerRequest(msg) {
|
|
|
13805
14032
|
}
|
|
13806
14033
|
} catch {
|
|
13807
14034
|
currentIndex.removeSource(fileEntry.filePath);
|
|
14035
|
+
if (currentDiskCache) {
|
|
14036
|
+
const relPath = path5.isAbsolute(fileEntry.filePath) && currentRootDir ? path5.relative(currentRootDir, fileEntry.filePath) : fileEntry.filePath;
|
|
14037
|
+
currentDiskCache.deleteByRelPath(relPath).catch(() => {});
|
|
14038
|
+
}
|
|
13808
14039
|
}
|
|
13809
14040
|
}
|
|
13810
14041
|
self.postMessage(createSuccessResponse(msg.id, { reconciledCount }));
|
|
@@ -13839,9 +14070,9 @@ async function handleWorkerRequest(msg) {
|
|
|
13839
14070
|
try {
|
|
13840
14071
|
if (!getSupportedCodeFormat(file, optionsToUse?.formatsExts))
|
|
13841
14072
|
continue;
|
|
13842
|
-
const stat2 = await
|
|
14073
|
+
const stat2 = await fs3.stat(file);
|
|
13843
14074
|
if (stat2.size <= MAX_FILE_SIZE_BYTES && stat2.size > 0) {
|
|
13844
|
-
const content = await
|
|
14075
|
+
const content = await fs3.readFile(file, "utf8");
|
|
13845
14076
|
if (!isGeneratedContent(content)) {
|
|
13846
14077
|
indexToUse.addSource(file, content);
|
|
13847
14078
|
}
|
|
@@ -13853,7 +14084,7 @@ async function handleWorkerRequest(msg) {
|
|
|
13853
14084
|
clones = currentIndex.getClones();
|
|
13854
14085
|
}
|
|
13855
14086
|
} else {
|
|
13856
|
-
clones = currentIndex.getClones();
|
|
14087
|
+
clones = filterAndEvictStaleClones(currentIndex.getClones());
|
|
13857
14088
|
}
|
|
13858
14089
|
self.postMessage(createSuccessResponse(msg.id, clones));
|
|
13859
14090
|
break;
|