omp-plugin-duplicate-detector 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -17
- package/dist/detector-worker.js +418 -272
- package/package.json +1 -1
- package/src/coordinator.ts +16 -4
- package/src/detector-worker.ts +63 -24
- package/src/disk-cache.ts +365 -314
- package/src/index.ts +134 -7
- package/src/project-state.ts +10 -4
- package/src/repo-context.ts +158 -0
- package/src/source-aware-index.ts +14 -1
package/dist/detector-worker.js
CHANGED
|
@@ -970,9 +970,9 @@ var require_ignore = __commonJS(function(exports, module) {
|
|
|
970
970
|
});
|
|
971
971
|
|
|
972
972
|
// src/detector-worker.ts
|
|
973
|
-
import * as
|
|
974
|
-
import * as
|
|
975
|
-
import * as
|
|
973
|
+
import * as crypto3 from "crypto";
|
|
974
|
+
import * as fs3 from "fs/promises";
|
|
975
|
+
import * as path5 from "path";
|
|
976
976
|
|
|
977
977
|
// src/disk-cache.ts
|
|
978
978
|
import { Database } from "bun:sqlite";
|
|
@@ -12565,7 +12565,9 @@ class SourceAwareCloneIndex {
|
|
|
12565
12565
|
}
|
|
12566
12566
|
let normalizedFrames;
|
|
12567
12567
|
const shardTokens = shard.tokens;
|
|
12568
|
-
if (shard.frames
|
|
12568
|
+
if (shardTokens && shardTokens.length > 0 && (shard.minTokens !== this.#minTokens || !shard.frames || shard.frames.length === 0)) {
|
|
12569
|
+
normalizedFrames = reconstructFramesFromTokens(shardTokens, sourceId, this.#minTokens, this.#hashFunction);
|
|
12570
|
+
} else if (shard.frames && shard.frames.length > 0) {
|
|
12569
12571
|
normalizedFrames = shard.frames.map((f) => {
|
|
12570
12572
|
if (f instanceof CompactSourceFrame && f.sourceId === sourceId) {
|
|
12571
12573
|
return f;
|
|
@@ -12835,6 +12837,7 @@ class SourceAwareCloneIndex {
|
|
|
12835
12837
|
|
|
12836
12838
|
// src/disk-cache.ts
|
|
12837
12839
|
var DEFAULT_MAX_CACHE_BYTES = 250 * 1024 * 1024;
|
|
12840
|
+
var TOKENIZER_CACHE_VERSION = "4.0";
|
|
12838
12841
|
function getDefaultCacheDir() {
|
|
12839
12842
|
if (process.platform === "win32") {
|
|
12840
12843
|
const localAppData = process.env.LOCALAPPDATA;
|
|
@@ -12851,7 +12854,7 @@ function getDefaultCacheDir() {
|
|
|
12851
12854
|
}
|
|
12852
12855
|
function computeConfigFingerprint(config) {
|
|
12853
12856
|
if (!config)
|
|
12854
|
-
return
|
|
12857
|
+
return `default_${TOKENIZER_CACHE_VERSION}`;
|
|
12855
12858
|
let sortedFormats;
|
|
12856
12859
|
if (config.formatsExts) {
|
|
12857
12860
|
sortedFormats = {};
|
|
@@ -12860,6 +12863,7 @@ function computeConfigFingerprint(config) {
|
|
|
12860
12863
|
}
|
|
12861
12864
|
}
|
|
12862
12865
|
const canonical = {
|
|
12866
|
+
version: TOKENIZER_CACHE_VERSION,
|
|
12863
12867
|
minTokens: config.minTokens ?? 40,
|
|
12864
12868
|
minLines: config.minLines ?? 5,
|
|
12865
12869
|
maxLines: config.maxLines ?? 500,
|
|
@@ -12868,201 +12872,199 @@ function computeConfigFingerprint(config) {
|
|
|
12868
12872
|
};
|
|
12869
12873
|
return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex").slice(0, 16);
|
|
12870
12874
|
}
|
|
12871
|
-
function computeWorkspaceCachePath(baseDir,
|
|
12872
|
-
const
|
|
12873
|
-
const
|
|
12874
|
-
return path3.join(baseDir, `${
|
|
12875
|
+
function computeWorkspaceCachePath(baseDir, repositoryKeyOrRootDir, configFingerprint) {
|
|
12876
|
+
const isKey = /^[0-9a-f]{16}$/i.test(repositoryKeyOrRootDir);
|
|
12877
|
+
const repoKey = isKey ? repositoryKeyOrRootDir : crypto.createHash("sha256").update(path3.resolve(repositoryKeyOrRootDir)).digest("hex").slice(0, 16);
|
|
12878
|
+
return path3.join(baseDir, `${repoKey}_${configFingerprint}_v${CACHE_FORMAT_VERSION}.sqlite`);
|
|
12875
12879
|
}
|
|
12876
12880
|
var CACHE_FORMAT_MAGIC = "DUP3";
|
|
12877
|
-
var CACHE_FORMAT_VERSION =
|
|
12881
|
+
var CACHE_FORMAT_VERSION = 4;
|
|
12878
12882
|
function packBinaryShard(shard) {
|
|
12879
12883
|
return packBinaryShardV3(shard, shard.tokens ?? []);
|
|
12880
12884
|
}
|
|
12881
12885
|
function packBinaryShardV3(shard, tokens) {
|
|
12882
|
-
const srcIdBuf = Buffer.from(shard.sourceId, "utf8");
|
|
12883
|
-
const formatBuf = Buffer.from(shard.format, "utf8");
|
|
12884
|
-
const hashBuf = Buffer.from(shard.contentHash, "utf8");
|
|
12885
12886
|
const tokenCount = tokens.length;
|
|
12886
|
-
const
|
|
12887
|
-
const
|
|
12888
|
-
const tokenIndices = new Uint16Array(tokenCount);
|
|
12887
|
+
const dictionary = [];
|
|
12888
|
+
const dictMap = new Map;
|
|
12889
12889
|
for (let i = 0;i < tokenCount; i++) {
|
|
12890
12890
|
const h = tokens[i].hash;
|
|
12891
|
-
|
|
12892
|
-
|
|
12893
|
-
|
|
12894
|
-
|
|
12895
|
-
|
|
12896
|
-
|
|
12897
|
-
|
|
12898
|
-
|
|
12899
|
-
|
|
12900
|
-
|
|
12901
|
-
|
|
12902
|
-
const
|
|
12903
|
-
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
|
|
12908
|
-
|
|
12909
|
-
|
|
12910
|
-
|
|
12911
|
-
|
|
12912
|
-
|
|
12913
|
-
|
|
12914
|
-
|
|
12915
|
-
|
|
12916
|
-
buf.writeUInt32LE(shard.size, pos);
|
|
12917
|
-
pos += 4;
|
|
12918
|
-
buf.writeUInt32LE(shard.lines, pos);
|
|
12919
|
-
pos += 4;
|
|
12920
|
-
buf.writeUInt32LE(shard.tokenCount, pos);
|
|
12921
|
-
pos += 4;
|
|
12922
|
-
buf.writeDoubleLE(shard.updatedAt ?? Date.now(), pos);
|
|
12923
|
-
pos += 8;
|
|
12924
|
-
buf.writeUInt16LE(minTokens, pos);
|
|
12925
|
-
pos += 2;
|
|
12926
|
-
buf.writeUInt16LE(srcIdBuf.length, pos);
|
|
12927
|
-
pos += 2;
|
|
12928
|
-
srcIdBuf.copy(buf, pos);
|
|
12929
|
-
pos += srcIdBuf.length;
|
|
12930
|
-
buf.writeUInt16LE(dictCount, pos);
|
|
12931
|
-
pos += 2;
|
|
12932
|
-
buf.writeUInt32LE(tokenCount, pos);
|
|
12933
|
-
pos += 4;
|
|
12934
|
-
for (const h of dict.keys()) {
|
|
12935
|
-
const hexHash = h.length === 20 ? h : h.padEnd(20, "0");
|
|
12936
|
-
buf.write(hexHash, pos, 10, "hex");
|
|
12937
|
-
pos += 10;
|
|
12938
|
-
}
|
|
12939
|
-
const dictIdxOffset = pos;
|
|
12940
|
-
const deltaLineOffset = dictIdxOffset + tokenCount * 2;
|
|
12941
|
-
const colOffset = deltaLineOffset + tokenCount * 2;
|
|
12942
|
-
const deltaRangeOffset = colOffset + tokenCount * 2;
|
|
12943
|
-
const lenOffset = deltaRangeOffset + tokenCount * 4;
|
|
12944
|
-
let prevLine = 1;
|
|
12945
|
-
let prevRangeStart = 0;
|
|
12891
|
+
if (!dictMap.has(h)) {
|
|
12892
|
+
dictMap.set(h, dictionary.length);
|
|
12893
|
+
dictionary.push(h);
|
|
12894
|
+
}
|
|
12895
|
+
}
|
|
12896
|
+
const dictCount = dictionary.length;
|
|
12897
|
+
const dictEntries = new Array(dictCount);
|
|
12898
|
+
for (let i = 0;i < dictCount; i++) {
|
|
12899
|
+
const hex = dictionary[i];
|
|
12900
|
+
dictEntries[i] = Buffer.from(hex, "hex");
|
|
12901
|
+
}
|
|
12902
|
+
const dictBuf = Buffer.concat(dictEntries);
|
|
12903
|
+
const indicesBuf = Buffer.allocUnsafe(tokenCount * 2);
|
|
12904
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
12905
|
+
const idx = dictMap.get(tokens[i].hash);
|
|
12906
|
+
indicesBuf.writeUInt16LE(idx, i * 2);
|
|
12907
|
+
}
|
|
12908
|
+
const colBytes = tokenCount * 4;
|
|
12909
|
+
const dLinesBuf = Buffer.allocUnsafe(colBytes);
|
|
12910
|
+
const dColsBuf = Buffer.allocUnsafe(colBytes);
|
|
12911
|
+
const dPosBuf = Buffer.allocUnsafe(colBytes);
|
|
12912
|
+
const dLenBuf = Buffer.allocUnsafe(colBytes);
|
|
12913
|
+
let prevLine = 0;
|
|
12914
|
+
let prevCol = 0;
|
|
12915
|
+
let prevPos = 0;
|
|
12946
12916
|
for (let i = 0;i < tokenCount; i++) {
|
|
12947
|
-
const
|
|
12948
|
-
const
|
|
12949
|
-
const
|
|
12950
|
-
const
|
|
12951
|
-
const
|
|
12952
|
-
|
|
12953
|
-
|
|
12954
|
-
|
|
12955
|
-
|
|
12956
|
-
|
|
12957
|
-
|
|
12958
|
-
|
|
12959
|
-
|
|
12960
|
-
|
|
12961
|
-
|
|
12962
|
-
|
|
12917
|
+
const tok = tokens[i];
|
|
12918
|
+
const dLine = tok.line - prevLine;
|
|
12919
|
+
const dCol = tok.column - prevCol;
|
|
12920
|
+
const dPos = tok.position - prevPos;
|
|
12921
|
+
const len = Array.isArray(tok.range) && tok.range.length >= 2 ? tok.range[1] - tok.range[0] : 0;
|
|
12922
|
+
dLinesBuf.writeInt32LE(dLine, i * 4);
|
|
12923
|
+
dColsBuf.writeInt32LE(dCol, i * 4);
|
|
12924
|
+
dPosBuf.writeInt32LE(dPos, i * 4);
|
|
12925
|
+
dLenBuf.writeUInt32LE(len, i * 4);
|
|
12926
|
+
prevLine = tok.line;
|
|
12927
|
+
prevCol = tok.column;
|
|
12928
|
+
prevPos = tok.position;
|
|
12929
|
+
}
|
|
12930
|
+
const meta = {
|
|
12931
|
+
sourceId: shard.sourceId,
|
|
12932
|
+
contentHash: shard.contentHash,
|
|
12933
|
+
format: shard.format,
|
|
12934
|
+
size: shard.size,
|
|
12935
|
+
lines: shard.lines,
|
|
12936
|
+
tokenCount: shard.tokenCount,
|
|
12937
|
+
minTokens: shard.minTokens,
|
|
12938
|
+
updatedAt: shard.updatedAt ?? Date.now()
|
|
12939
|
+
};
|
|
12940
|
+
const metaJson = Buffer.from(JSON.stringify(meta), "utf-8");
|
|
12941
|
+
const HEADER_SIZE = 16;
|
|
12942
|
+
const header = Buffer.allocUnsafe(HEADER_SIZE);
|
|
12943
|
+
header.write(CACHE_FORMAT_MAGIC, 0, 4, "ascii");
|
|
12944
|
+
header.writeUInt16LE(CACHE_FORMAT_VERSION, 4);
|
|
12945
|
+
header.writeUInt16LE(metaJson.length, 6);
|
|
12946
|
+
header.writeUInt32LE(tokenCount, 8);
|
|
12947
|
+
header.writeUInt32LE(dictCount, 12);
|
|
12948
|
+
const rawUncompressed = Buffer.concat([
|
|
12949
|
+
header,
|
|
12950
|
+
metaJson,
|
|
12951
|
+
dictBuf,
|
|
12952
|
+
indicesBuf,
|
|
12953
|
+
dLinesBuf,
|
|
12954
|
+
dColsBuf,
|
|
12955
|
+
dPosBuf,
|
|
12956
|
+
dLenBuf
|
|
12957
|
+
]);
|
|
12958
|
+
return zlib.deflateSync(rawUncompressed, { level: 6 });
|
|
12963
12959
|
}
|
|
12964
12960
|
function unpackBinaryShard(compressed) {
|
|
12965
12961
|
try {
|
|
12966
|
-
const
|
|
12967
|
-
if (
|
|
12968
|
-
return null;
|
|
12969
|
-
const magic = buf.toString("ascii", 0, 4);
|
|
12970
|
-
if (magic !== CACHE_FORMAT_MAGIC) {
|
|
12962
|
+
const raw = zlib.inflateSync(compressed);
|
|
12963
|
+
if (raw.length < 16)
|
|
12971
12964
|
return null;
|
|
12965
|
+
const magic = raw.toString("ascii", 0, 4);
|
|
12966
|
+
if (magic === CACHE_FORMAT_MAGIC) {
|
|
12967
|
+
return unpackBinaryShardV3(raw);
|
|
12972
12968
|
}
|
|
12973
|
-
return
|
|
12969
|
+
return null;
|
|
12974
12970
|
} catch {
|
|
12975
12971
|
return null;
|
|
12976
12972
|
}
|
|
12977
12973
|
}
|
|
12978
12974
|
function unpackBinaryShardV3(buf) {
|
|
12979
|
-
|
|
12980
|
-
|
|
12981
|
-
|
|
12982
|
-
|
|
12983
|
-
|
|
12984
|
-
|
|
12985
|
-
|
|
12986
|
-
|
|
12987
|
-
|
|
12988
|
-
|
|
12989
|
-
|
|
12990
|
-
|
|
12991
|
-
|
|
12992
|
-
|
|
12993
|
-
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
|
|
12999
|
-
|
|
13000
|
-
|
|
13001
|
-
|
|
13002
|
-
|
|
13003
|
-
|
|
13004
|
-
|
|
13005
|
-
|
|
13006
|
-
|
|
13007
|
-
|
|
13008
|
-
|
|
13009
|
-
|
|
13010
|
-
|
|
13011
|
-
|
|
13012
|
-
|
|
13013
|
-
|
|
13014
|
-
|
|
13015
|
-
|
|
13016
|
-
|
|
13017
|
-
|
|
13018
|
-
|
|
13019
|
-
|
|
13020
|
-
|
|
13021
|
-
|
|
13022
|
-
|
|
13023
|
-
|
|
13024
|
-
|
|
13025
|
-
|
|
13026
|
-
|
|
13027
|
-
|
|
13028
|
-
|
|
13029
|
-
|
|
13030
|
-
const
|
|
13031
|
-
|
|
13032
|
-
|
|
13033
|
-
|
|
13034
|
-
|
|
13035
|
-
|
|
13036
|
-
|
|
13037
|
-
|
|
13038
|
-
|
|
12975
|
+
try {
|
|
12976
|
+
const version = buf.readUInt16LE(4);
|
|
12977
|
+
if (version < 3 || version > CACHE_FORMAT_VERSION)
|
|
12978
|
+
return null;
|
|
12979
|
+
const metaLen = buf.readUInt16LE(6);
|
|
12980
|
+
const tokenCount = buf.readUInt32LE(8);
|
|
12981
|
+
const dictCount = buf.readUInt32LE(12);
|
|
12982
|
+
let offset = 16;
|
|
12983
|
+
const metaJsonBuf = buf.subarray(offset, offset + metaLen);
|
|
12984
|
+
offset += metaLen;
|
|
12985
|
+
const meta = JSON.parse(metaJsonBuf.toString("utf-8"));
|
|
12986
|
+
const TOKEN_HASH_RAW_BYTES = 10;
|
|
12987
|
+
const dictByteLen = dictCount * TOKEN_HASH_RAW_BYTES;
|
|
12988
|
+
if (buf.length < offset + dictByteLen)
|
|
12989
|
+
return null;
|
|
12990
|
+
const dictionary = new Array(dictCount);
|
|
12991
|
+
for (let i = 0;i < dictCount; i++) {
|
|
12992
|
+
dictionary[i] = buf.subarray(offset + i * 10, offset + (i + 1) * 10).toString("hex");
|
|
12993
|
+
}
|
|
12994
|
+
offset += dictByteLen;
|
|
12995
|
+
const indicesByteLen = tokenCount * 2;
|
|
12996
|
+
if (buf.length < offset + indicesByteLen)
|
|
12997
|
+
return null;
|
|
12998
|
+
const indices = new Uint16Array(tokenCount);
|
|
12999
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13000
|
+
indices[i] = buf.readUInt16LE(offset + i * 2);
|
|
13001
|
+
}
|
|
13002
|
+
offset += indicesByteLen;
|
|
13003
|
+
const colBytes = tokenCount * 4;
|
|
13004
|
+
if (buf.length < offset + colBytes * 4)
|
|
13005
|
+
return null;
|
|
13006
|
+
const dLines = new Int32Array(tokenCount);
|
|
13007
|
+
const dCols = new Int32Array(tokenCount);
|
|
13008
|
+
const dPos = new Int32Array(tokenCount);
|
|
13009
|
+
const dLens = new Uint32Array(tokenCount);
|
|
13010
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13011
|
+
dLines[i] = buf.readInt32LE(offset + i * 4);
|
|
13012
|
+
}
|
|
13013
|
+
offset += colBytes;
|
|
13014
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13015
|
+
dCols[i] = buf.readInt32LE(offset + i * 4);
|
|
13016
|
+
}
|
|
13017
|
+
offset += colBytes;
|
|
13018
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13019
|
+
dPos[i] = buf.readInt32LE(offset + i * 4);
|
|
13020
|
+
}
|
|
13021
|
+
offset += colBytes;
|
|
13022
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13023
|
+
dLens[i] = buf.readUInt32LE(offset + i * 4);
|
|
13024
|
+
}
|
|
13025
|
+
offset += colBytes;
|
|
13026
|
+
const tokens = new Array(tokenCount);
|
|
13027
|
+
let curLine = 0;
|
|
13028
|
+
let curCol = 0;
|
|
13029
|
+
let curPos = 0;
|
|
13030
|
+
for (let i = 0;i < tokenCount; i++) {
|
|
13031
|
+
curLine += dLines[i];
|
|
13032
|
+
curCol += dCols[i];
|
|
13033
|
+
curPos += dPos[i];
|
|
13034
|
+
const len = dLens[i];
|
|
13035
|
+
const dictIdx = indices[i];
|
|
13036
|
+
const hash2 = dictionary[dictIdx] ?? "";
|
|
13037
|
+
tokens[i] = {
|
|
13038
|
+
hash: hash2,
|
|
13039
|
+
line: curLine,
|
|
13040
|
+
column: curCol,
|
|
13041
|
+
position: curPos,
|
|
13042
|
+
range: [curPos, curPos + len]
|
|
13043
|
+
};
|
|
13044
|
+
}
|
|
13045
|
+
const minTokens = meta.minTokens ?? 40;
|
|
13046
|
+
const frames = reconstructFramesFromTokens(tokens, meta.sourceId, minTokens);
|
|
13047
|
+
return {
|
|
13048
|
+
version,
|
|
13049
|
+
sourceId: meta.sourceId,
|
|
13050
|
+
contentHash: meta.contentHash,
|
|
13051
|
+
format: meta.format,
|
|
13052
|
+
size: meta.size,
|
|
13053
|
+
lines: meta.lines,
|
|
13054
|
+
tokenCount,
|
|
13055
|
+
minTokens,
|
|
13056
|
+
updatedAt: meta.updatedAt,
|
|
13057
|
+
tokens,
|
|
13058
|
+
frames
|
|
13039
13059
|
};
|
|
13040
|
-
|
|
13041
|
-
|
|
13060
|
+
} catch {
|
|
13061
|
+
return null;
|
|
13042
13062
|
}
|
|
13043
|
-
let memoizedFrames = null;
|
|
13044
|
-
return {
|
|
13045
|
-
version: 1,
|
|
13046
|
-
sourceId,
|
|
13047
|
-
contentHash,
|
|
13048
|
-
format,
|
|
13049
|
-
size,
|
|
13050
|
-
lines,
|
|
13051
|
-
tokenCount,
|
|
13052
|
-
minTokens,
|
|
13053
|
-
updatedAt,
|
|
13054
|
-
tokens,
|
|
13055
|
-
get frames() {
|
|
13056
|
-
if (!memoizedFrames) {
|
|
13057
|
-
memoizedFrames = reconstructFramesFromTokens(tokens, sourceId, minTokens || 40);
|
|
13058
|
-
}
|
|
13059
|
-
return memoizedFrames;
|
|
13060
|
-
}
|
|
13061
|
-
};
|
|
13062
13063
|
}
|
|
13063
13064
|
|
|
13064
13065
|
class DiskCacheManager {
|
|
13065
13066
|
rootDir;
|
|
13067
|
+
repositoryKey;
|
|
13066
13068
|
baseCacheDir;
|
|
13067
13069
|
dbPath;
|
|
13068
13070
|
workspaceCacheDir;
|
|
@@ -13071,7 +13073,6 @@ class DiskCacheManager {
|
|
|
13071
13073
|
#db = null;
|
|
13072
13074
|
#getStmt = null;
|
|
13073
13075
|
#saveStmt = null;
|
|
13074
|
-
#updateMtimeStmt = null;
|
|
13075
13076
|
#deleteStmt = null;
|
|
13076
13077
|
#totalSizeStmt = null;
|
|
13077
13078
|
#oldestShardsStmt = null;
|
|
@@ -13079,9 +13080,10 @@ class DiskCacheManager {
|
|
|
13079
13080
|
#closed = false;
|
|
13080
13081
|
constructor(options) {
|
|
13081
13082
|
this.rootDir = path3.resolve(options.rootDir);
|
|
13083
|
+
this.repositoryKey = options.repositoryKey ?? crypto.createHash("sha256").update(this.rootDir).digest("hex").slice(0, 16);
|
|
13082
13084
|
this.baseCacheDir = options.cacheDir ? path3.resolve(options.cacheDir) : getDefaultCacheDir();
|
|
13083
13085
|
this.configFingerprint = computeConfigFingerprint(options.config);
|
|
13084
|
-
this.dbPath = computeWorkspaceCachePath(this.baseCacheDir, this.
|
|
13086
|
+
this.dbPath = computeWorkspaceCachePath(this.baseCacheDir, this.repositoryKey, this.configFingerprint);
|
|
13085
13087
|
this.workspaceCacheDir = this.baseCacheDir;
|
|
13086
13088
|
this.maxBytes = options.maxBytes ?? DEFAULT_MAX_CACHE_BYTES;
|
|
13087
13089
|
}
|
|
@@ -13090,48 +13092,60 @@ class DiskCacheManager {
|
|
|
13090
13092
|
return null;
|
|
13091
13093
|
if (this.#db)
|
|
13092
13094
|
return this.#db;
|
|
13095
|
+
let db = null;
|
|
13093
13096
|
try {
|
|
13094
13097
|
const dir = path3.dirname(this.dbPath);
|
|
13095
13098
|
if (!fsSync.existsSync(dir)) {
|
|
13096
13099
|
fsSync.mkdirSync(dir, { recursive: true });
|
|
13097
13100
|
}
|
|
13098
|
-
|
|
13099
|
-
db.exec("PRAGMA
|
|
13101
|
+
db = new Database(this.dbPath, { create: true });
|
|
13102
|
+
db.exec("PRAGMA busy_timeout = 2000;");
|
|
13103
|
+
const journalRow = db.query("PRAGMA journal_mode = WAL;").get();
|
|
13104
|
+
if (journalRow?.journal_mode?.toLowerCase() !== "wal") {
|
|
13105
|
+
try {
|
|
13106
|
+
db.close();
|
|
13107
|
+
} catch {}
|
|
13108
|
+
return null;
|
|
13109
|
+
}
|
|
13100
13110
|
db.exec("PRAGMA synchronous = NORMAL;");
|
|
13101
13111
|
db.exec("PRAGMA temp_store = MEMORY;");
|
|
13102
13112
|
const versionRow = db.query("PRAGMA user_version;").get();
|
|
13103
13113
|
const schemaVersion = versionRow?.user_version ?? 0;
|
|
13104
13114
|
if (schemaVersion !== CACHE_FORMAT_VERSION) {
|
|
13105
|
-
|
|
13115
|
+
try {
|
|
13116
|
+
db.exec("DELETE FROM shards;");
|
|
13117
|
+
} catch {}
|
|
13106
13118
|
db.exec(`PRAGMA user_version = ${CACHE_FORMAT_VERSION};`);
|
|
13107
13119
|
}
|
|
13108
13120
|
db.exec(`
|
|
13109
13121
|
CREATE TABLE IF NOT EXISTS shards (
|
|
13110
|
-
rel_path TEXT NOT NULL
|
|
13122
|
+
rel_path TEXT NOT NULL,
|
|
13111
13123
|
content_hash TEXT NOT NULL,
|
|
13112
13124
|
payload BLOB NOT NULL,
|
|
13113
|
-
mtime REAL NOT NULL
|
|
13125
|
+
mtime REAL NOT NULL,
|
|
13126
|
+
PRIMARY KEY (rel_path, content_hash)
|
|
13114
13127
|
);
|
|
13115
|
-
CREATE INDEX IF NOT EXISTS idx_shards_content_hash ON shards(content_hash);
|
|
13116
13128
|
CREATE INDEX IF NOT EXISTS idx_shards_mtime ON shards(mtime);
|
|
13117
13129
|
`);
|
|
13118
|
-
this.#getStmt = db.prepare("SELECT payload
|
|
13130
|
+
this.#getStmt = db.prepare("SELECT payload FROM shards WHERE rel_path = ?1 AND content_hash = ?2");
|
|
13119
13131
|
this.#saveStmt = db.prepare(`
|
|
13120
13132
|
INSERT INTO shards (rel_path, content_hash, payload, mtime)
|
|
13121
13133
|
VALUES (?1, ?2, ?3, ?4)
|
|
13122
|
-
ON CONFLICT(rel_path) DO UPDATE SET
|
|
13123
|
-
content_hash = excluded.content_hash,
|
|
13124
|
-
payload = excluded.payload,
|
|
13134
|
+
ON CONFLICT(rel_path, content_hash) DO UPDATE SET
|
|
13125
13135
|
mtime = excluded.mtime
|
|
13126
13136
|
`);
|
|
13127
|
-
this.#
|
|
13128
|
-
this.#deleteStmt = db.prepare("DELETE FROM shards WHERE rel_path = ?1");
|
|
13137
|
+
this.#deleteStmt = db.prepare("DELETE FROM shards WHERE rel_path = ?1 AND content_hash = ?2");
|
|
13129
13138
|
this.#totalSizeStmt = db.prepare("SELECT COALESCE(SUM(LENGTH(payload)), 0) as total FROM shards");
|
|
13130
|
-
this.#oldestShardsStmt = db.prepare("SELECT
|
|
13139
|
+
this.#oldestShardsStmt = db.prepare("SELECT rowid, LENGTH(payload) as size FROM shards ORDER BY mtime ASC LIMIT ?1");
|
|
13131
13140
|
this.#deleteAllStmt = db.prepare("DELETE FROM shards");
|
|
13132
13141
|
this.#db = db;
|
|
13133
13142
|
return db;
|
|
13134
13143
|
} catch {
|
|
13144
|
+
if (db) {
|
|
13145
|
+
try {
|
|
13146
|
+
db.close();
|
|
13147
|
+
} catch {}
|
|
13148
|
+
}
|
|
13135
13149
|
return null;
|
|
13136
13150
|
}
|
|
13137
13151
|
}
|
|
@@ -13141,21 +13155,15 @@ class DiskCacheManager {
|
|
|
13141
13155
|
if (!db || !this.#getStmt)
|
|
13142
13156
|
return null;
|
|
13143
13157
|
const normalizedRelPath = relPath.replace(/\\/g, "/");
|
|
13144
|
-
const row = this.#getStmt.get(normalizedRelPath);
|
|
13145
|
-
if (!row
|
|
13158
|
+
const row = this.#getStmt.get(normalizedRelPath, contentHash);
|
|
13159
|
+
if (!row) {
|
|
13146
13160
|
return null;
|
|
13147
13161
|
}
|
|
13148
13162
|
const payloadBuf = Buffer.isBuffer(row.payload) ? row.payload : Buffer.from(row.payload.buffer, row.payload.byteOffset, row.payload.byteLength);
|
|
13149
13163
|
const shard = unpackBinaryShard(payloadBuf);
|
|
13150
|
-
if (shard && shard.contentHash === contentHash &&
|
|
13151
|
-
try {
|
|
13152
|
-
this.#updateMtimeStmt?.run(Date.now(), normalizedRelPath);
|
|
13153
|
-
} catch {}
|
|
13164
|
+
if (shard && shard.contentHash === contentHash && Array.isArray(shard.frames)) {
|
|
13154
13165
|
return shard;
|
|
13155
13166
|
}
|
|
13156
|
-
try {
|
|
13157
|
-
this.#deleteStmt?.run(normalizedRelPath);
|
|
13158
|
-
} catch {}
|
|
13159
13167
|
return null;
|
|
13160
13168
|
} catch {
|
|
13161
13169
|
return null;
|
|
@@ -13172,6 +13180,44 @@ class DiskCacheManager {
|
|
|
13172
13180
|
this.#saveStmt.run(normalizedRelPath, shard.contentHash, payload, Date.now());
|
|
13173
13181
|
} catch {}
|
|
13174
13182
|
}
|
|
13183
|
+
async deleteShard(relPath, contentHash) {
|
|
13184
|
+
try {
|
|
13185
|
+
const db = this.#getDb();
|
|
13186
|
+
if (!db || !this.#deleteStmt)
|
|
13187
|
+
return;
|
|
13188
|
+
const normalizedRelPath = relPath.replace(/\\/g, "/");
|
|
13189
|
+
this.#deleteStmt.run(normalizedRelPath, contentHash);
|
|
13190
|
+
} catch {}
|
|
13191
|
+
}
|
|
13192
|
+
async saveShards(items) {
|
|
13193
|
+
if (items.length === 0)
|
|
13194
|
+
return;
|
|
13195
|
+
try {
|
|
13196
|
+
const db = this.#getDb();
|
|
13197
|
+
if (!db || !this.#saveStmt)
|
|
13198
|
+
return;
|
|
13199
|
+
const stmt = this.#saveStmt;
|
|
13200
|
+
const root = this.rootDir;
|
|
13201
|
+
const now = Date.now();
|
|
13202
|
+
const prepared = [];
|
|
13203
|
+
for (const item of items) {
|
|
13204
|
+
const targetRelPath = item.relPath ?? (path3.isAbsolute(item.shard.sourceId) ? path3.relative(root, item.shard.sourceId) : item.shard.sourceId);
|
|
13205
|
+
const normalizedRelPath = targetRelPath.replace(/\\/g, "/");
|
|
13206
|
+
const payload = packBinaryShard(item.shard);
|
|
13207
|
+
prepared.push({
|
|
13208
|
+
relPath: normalizedRelPath,
|
|
13209
|
+
hash: item.shard.contentHash,
|
|
13210
|
+
payload
|
|
13211
|
+
});
|
|
13212
|
+
}
|
|
13213
|
+
const tx = db.transaction((entries) => {
|
|
13214
|
+
for (const entry of entries) {
|
|
13215
|
+
stmt.run(entry.relPath, entry.hash, entry.payload, now);
|
|
13216
|
+
}
|
|
13217
|
+
});
|
|
13218
|
+
tx(prepared);
|
|
13219
|
+
} catch {}
|
|
13220
|
+
}
|
|
13175
13221
|
async prune(maxBytes) {
|
|
13176
13222
|
const budget = maxBytes !== undefined ? maxBytes : this.maxBytes;
|
|
13177
13223
|
try {
|
|
@@ -13180,53 +13226,41 @@ class DiskCacheManager {
|
|
|
13180
13226
|
return;
|
|
13181
13227
|
if (budget <= 0) {
|
|
13182
13228
|
this.#deleteAllStmt?.run();
|
|
13183
|
-
try {
|
|
13184
|
-
db.exec("VACUUM;");
|
|
13185
|
-
} catch {}
|
|
13186
|
-
return;
|
|
13187
|
-
}
|
|
13188
|
-
const totalRow = this.#totalSizeStmt?.get();
|
|
13189
|
-
let totalSize = totalRow?.total ?? 0;
|
|
13190
|
-
if (totalSize <= budget) {
|
|
13191
13229
|
return;
|
|
13192
13230
|
}
|
|
13193
|
-
|
|
13194
|
-
|
|
13195
|
-
|
|
13231
|
+
for (let iter = 0;iter < 10; iter++) {
|
|
13232
|
+
const totalRow = this.#totalSizeStmt?.get();
|
|
13233
|
+
const totalSize = totalRow?.total ?? 0;
|
|
13196
13234
|
if (totalSize <= budget) {
|
|
13197
13235
|
break;
|
|
13198
13236
|
}
|
|
13199
|
-
|
|
13200
|
-
|
|
13201
|
-
|
|
13202
|
-
|
|
13203
|
-
|
|
13204
|
-
|
|
13205
|
-
|
|
13206
|
-
|
|
13207
|
-
|
|
13208
|
-
|
|
13237
|
+
const excess = totalSize - budget;
|
|
13238
|
+
const rows = this.#oldestShardsStmt?.all(100) ?? [];
|
|
13239
|
+
if (rows.length === 0)
|
|
13240
|
+
break;
|
|
13241
|
+
const rowidsToDelete = [];
|
|
13242
|
+
let freed = 0;
|
|
13243
|
+
for (const r of rows) {
|
|
13244
|
+
rowidsToDelete.push(r.rowid);
|
|
13245
|
+
freed += r.size;
|
|
13246
|
+
if (freed >= excess)
|
|
13247
|
+
break;
|
|
13248
|
+
}
|
|
13249
|
+
if (rowidsToDelete.length > 0) {
|
|
13250
|
+
const deleteBatchStmt = db.prepare(`DELETE FROM shards WHERE rowid IN (${rowidsToDelete.join(",")})`);
|
|
13251
|
+
deleteBatchStmt.run();
|
|
13252
|
+
} else {
|
|
13253
|
+
break;
|
|
13254
|
+
}
|
|
13209
13255
|
}
|
|
13210
13256
|
} catch {}
|
|
13211
13257
|
}
|
|
13212
13258
|
async clear() {
|
|
13213
13259
|
try {
|
|
13214
|
-
|
|
13215
|
-
|
|
13216
|
-
|
|
13217
|
-
|
|
13218
|
-
this.#db = null;
|
|
13219
|
-
this.#getStmt = null;
|
|
13220
|
-
this.#saveStmt = null;
|
|
13221
|
-
this.#updateMtimeStmt = null;
|
|
13222
|
-
this.#deleteStmt = null;
|
|
13223
|
-
this.#totalSizeStmt = null;
|
|
13224
|
-
this.#oldestShardsStmt = null;
|
|
13225
|
-
this.#deleteAllStmt = null;
|
|
13226
|
-
}
|
|
13227
|
-
await fs.unlink(this.dbPath).catch(() => {});
|
|
13228
|
-
await fs.unlink(`${this.dbPath}-wal`).catch(() => {});
|
|
13229
|
-
await fs.unlink(`${this.dbPath}-shm`).catch(() => {});
|
|
13260
|
+
const db = this.#getDb();
|
|
13261
|
+
if (db && this.#deleteAllStmt) {
|
|
13262
|
+
this.#deleteAllStmt.run();
|
|
13263
|
+
}
|
|
13230
13264
|
} catch {}
|
|
13231
13265
|
}
|
|
13232
13266
|
close() {
|
|
@@ -13238,7 +13272,6 @@ class DiskCacheManager {
|
|
|
13238
13272
|
this.#db = null;
|
|
13239
13273
|
this.#getStmt = null;
|
|
13240
13274
|
this.#saveStmt = null;
|
|
13241
|
-
this.#updateMtimeStmt = null;
|
|
13242
13275
|
this.#deleteStmt = null;
|
|
13243
13276
|
this.#totalSizeStmt = null;
|
|
13244
13277
|
this.#oldestShardsStmt = null;
|
|
@@ -13246,6 +13279,103 @@ class DiskCacheManager {
|
|
|
13246
13279
|
}
|
|
13247
13280
|
}
|
|
13248
13281
|
}
|
|
13282
|
+
async function cleanupLegacyCacheFiles(customCacheDir) {
|
|
13283
|
+
const baseDir = customCacheDir ?? getDefaultCacheDir();
|
|
13284
|
+
try {
|
|
13285
|
+
if (!fsSync.existsSync(baseDir))
|
|
13286
|
+
return;
|
|
13287
|
+
const entries = await fs.readdir(baseDir);
|
|
13288
|
+
for (const entry of entries) {
|
|
13289
|
+
if (entry.endsWith(".sqlite") && !entry.includes(`_v${CACHE_FORMAT_VERSION}.sqlite`)) {
|
|
13290
|
+
await fs.unlink(path3.join(baseDir, entry)).catch(() => {});
|
|
13291
|
+
await fs.unlink(path3.join(baseDir, `${entry}-wal`)).catch(() => {});
|
|
13292
|
+
await fs.unlink(path3.join(baseDir, `${entry}-shm`)).catch(() => {});
|
|
13293
|
+
}
|
|
13294
|
+
}
|
|
13295
|
+
} catch {}
|
|
13296
|
+
}
|
|
13297
|
+
|
|
13298
|
+
// src/repo-context.ts
|
|
13299
|
+
import * as crypto2 from "crypto";
|
|
13300
|
+
import * as fsSync2 from "fs";
|
|
13301
|
+
import * as fs2 from "fs/promises";
|
|
13302
|
+
import * as path4 from "path";
|
|
13303
|
+
function canonicalizePath(targetPath) {
|
|
13304
|
+
const resolved = path4.resolve(targetPath);
|
|
13305
|
+
try {
|
|
13306
|
+
if (fsSync2.existsSync(resolved)) {
|
|
13307
|
+
return fsSync2.realpathSync(resolved);
|
|
13308
|
+
}
|
|
13309
|
+
} catch {}
|
|
13310
|
+
return resolved;
|
|
13311
|
+
}
|
|
13312
|
+
function isOmpWorktreePath(targetPath) {
|
|
13313
|
+
const normalized = targetPath.replace(/\\/g, "/");
|
|
13314
|
+
return normalized.includes("/.omp/wt/") || normalized.includes("/.omp/worktrees/");
|
|
13315
|
+
}
|
|
13316
|
+
async function resolveRepositoryContext(cwd, signal) {
|
|
13317
|
+
const canonicalCwd = canonicalizePath(cwd);
|
|
13318
|
+
try {
|
|
13319
|
+
const { stdout } = await execGit([
|
|
13320
|
+
"rev-parse",
|
|
13321
|
+
"--path-format=absolute",
|
|
13322
|
+
"--show-toplevel",
|
|
13323
|
+
"--git-dir",
|
|
13324
|
+
"--git-common-dir"
|
|
13325
|
+
], canonicalCwd, { signal });
|
|
13326
|
+
const lines = stdout.split(`
|
|
13327
|
+
`).map((l) => l.trim()).filter(Boolean);
|
|
13328
|
+
const resolveEntry = (val) => {
|
|
13329
|
+
if (!val)
|
|
13330
|
+
return "";
|
|
13331
|
+
return path4.isAbsolute(val) ? val : path4.resolve(canonicalCwd, val);
|
|
13332
|
+
};
|
|
13333
|
+
const workspaceRoot = canonicalizePath(resolveEntry(lines[0]));
|
|
13334
|
+
const gitDir = canonicalizePath(resolveEntry(lines[1] || path4.join(workspaceRoot, ".git")));
|
|
13335
|
+
const commonGitDir = canonicalizePath(resolveEntry(lines[2] || lines[1] || path4.join(workspaceRoot, ".git")));
|
|
13336
|
+
let repositoryObjectDir = path4.join(commonGitDir, "objects");
|
|
13337
|
+
const isOmpIsolation = isOmpWorktreePath(workspaceRoot) || isOmpWorktreePath(canonicalCwd);
|
|
13338
|
+
if (isOmpIsolation && gitDir === commonGitDir) {
|
|
13339
|
+
const alternatesFile = path4.join(gitDir, "objects", "info", "alternates");
|
|
13340
|
+
try {
|
|
13341
|
+
if (fsSync2.existsSync(alternatesFile)) {
|
|
13342
|
+
const content = await fs2.readFile(alternatesFile, "utf-8");
|
|
13343
|
+
const altLines = content.split(`
|
|
13344
|
+
`).map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
|
|
13345
|
+
for (const line of altLines) {
|
|
13346
|
+
const resolvedAlt = path4.isAbsolute(line) ? line : path4.resolve(path4.join(gitDir, "objects"), line);
|
|
13347
|
+
const canonicalAlt = canonicalizePath(resolvedAlt);
|
|
13348
|
+
if (fsSync2.existsSync(canonicalAlt)) {
|
|
13349
|
+
repositoryObjectDir = canonicalAlt;
|
|
13350
|
+
break;
|
|
13351
|
+
}
|
|
13352
|
+
}
|
|
13353
|
+
}
|
|
13354
|
+
} catch {}
|
|
13355
|
+
}
|
|
13356
|
+
repositoryObjectDir = canonicalizePath(repositoryObjectDir);
|
|
13357
|
+
const repositoryKey = crypto2.createHash("sha256").update(`git-object-dir\x00${repositoryObjectDir}`).digest("hex").slice(0, 16);
|
|
13358
|
+
return {
|
|
13359
|
+
workspaceRoot,
|
|
13360
|
+
isGit: true,
|
|
13361
|
+
gitDir,
|
|
13362
|
+
commonGitDir,
|
|
13363
|
+
repositoryObjectDir,
|
|
13364
|
+
repositoryKey,
|
|
13365
|
+
isOmpIsolation
|
|
13366
|
+
};
|
|
13367
|
+
} catch {
|
|
13368
|
+
const repositoryObjectDir = path4.join(canonicalCwd, ".non-git");
|
|
13369
|
+
const repositoryKey = crypto2.createHash("sha256").update(`directory\x00${canonicalCwd}`).digest("hex").slice(0, 16);
|
|
13370
|
+
return {
|
|
13371
|
+
workspaceRoot: canonicalCwd,
|
|
13372
|
+
isGit: false,
|
|
13373
|
+
repositoryObjectDir,
|
|
13374
|
+
repositoryKey,
|
|
13375
|
+
isOmpIsolation: false
|
|
13376
|
+
};
|
|
13377
|
+
}
|
|
13378
|
+
}
|
|
13249
13379
|
|
|
13250
13380
|
// src/worker-protocol.ts
|
|
13251
13381
|
var REQUEST_TYPES = {
|
|
@@ -13355,18 +13485,20 @@ function notifyLateFindings(clones) {
|
|
|
13355
13485
|
for (const clone2 of clones) {
|
|
13356
13486
|
const srcA = clone2.duplicationA.sourceId;
|
|
13357
13487
|
const srcB = clone2.duplicationB.sourceId;
|
|
13358
|
-
const resA =
|
|
13359
|
-
const resB =
|
|
13360
|
-
const
|
|
13361
|
-
const
|
|
13488
|
+
const resA = path5.resolve(srcA);
|
|
13489
|
+
const resB = path5.resolve(srcB);
|
|
13490
|
+
const canA = canonicalizePath(srcA);
|
|
13491
|
+
const canB = canonicalizePath(srcB);
|
|
13492
|
+
const isWatchedA = watchedRevisions.has(srcA) || watchedRevisions.has(resA) || watchedRevisions.has(canA);
|
|
13493
|
+
const isWatchedB = watchedRevisions.has(srcB) || watchedRevisions.has(resB) || watchedRevisions.has(canB);
|
|
13362
13494
|
if (isWatchedA || isWatchedB) {
|
|
13363
13495
|
if (isWatchedA) {
|
|
13364
|
-
const entry = watchedRevisions.get(srcA) ?? watchedRevisions.get(resA);
|
|
13496
|
+
const entry = watchedRevisions.get(srcA) ?? watchedRevisions.get(resA) ?? watchedRevisions.get(canA);
|
|
13365
13497
|
if (entry)
|
|
13366
13498
|
entry.lastKnownCloneCount++;
|
|
13367
13499
|
}
|
|
13368
13500
|
if (isWatchedB) {
|
|
13369
|
-
const entry = watchedRevisions.get(srcB) ?? watchedRevisions.get(resB);
|
|
13501
|
+
const entry = watchedRevisions.get(srcB) ?? watchedRevisions.get(resB) ?? watchedRevisions.get(canB);
|
|
13370
13502
|
if (entry)
|
|
13371
13503
|
entry.lastKnownCloneCount++;
|
|
13372
13504
|
}
|
|
@@ -13377,16 +13509,16 @@ function notifyLateFindings(clones) {
|
|
|
13377
13509
|
function cacheSourceShard(filePath, content) {
|
|
13378
13510
|
if (!currentDiskCache)
|
|
13379
13511
|
return;
|
|
13380
|
-
const contentHash =
|
|
13381
|
-
const relPath =
|
|
13512
|
+
const contentHash = crypto3.createHash("sha256").update(content).digest("hex");
|
|
13513
|
+
const relPath = path5.relative(currentRootDir, filePath).replace(/\\/g, "/");
|
|
13382
13514
|
const shard = currentIndex.exportSourceShard(filePath, contentHash);
|
|
13383
13515
|
if (shard) {
|
|
13384
13516
|
currentDiskCache.saveShard(shard, relPath).catch(() => {});
|
|
13385
13517
|
}
|
|
13386
13518
|
}
|
|
13387
13519
|
function yieldTask() {
|
|
13388
|
-
const { promise, resolve:
|
|
13389
|
-
setTimeout(
|
|
13520
|
+
const { promise, resolve: resolve5 } = Promise.withResolvers();
|
|
13521
|
+
setTimeout(resolve5, 0);
|
|
13390
13522
|
return promise;
|
|
13391
13523
|
}
|
|
13392
13524
|
async function runBaselineIndexing(rootDir, options, signal) {
|
|
@@ -13453,11 +13585,11 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13453
13585
|
return null;
|
|
13454
13586
|
}
|
|
13455
13587
|
try {
|
|
13456
|
-
const stat2 = await
|
|
13588
|
+
const stat2 = await fs3.stat(filePath);
|
|
13457
13589
|
if (stat2.size > MAX_FILE_SIZE_BYTES || stat2.size <= 0) {
|
|
13458
13590
|
return null;
|
|
13459
13591
|
}
|
|
13460
|
-
const resolved =
|
|
13592
|
+
const resolved = path5.resolve(filePath);
|
|
13461
13593
|
if (currentIndex.hasSource(filePath) || currentIndex.hasSource(resolved)) {
|
|
13462
13594
|
return {
|
|
13463
13595
|
filePath,
|
|
@@ -13470,12 +13602,12 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13470
13602
|
alreadyIndexed: true
|
|
13471
13603
|
};
|
|
13472
13604
|
}
|
|
13473
|
-
const content = await
|
|
13605
|
+
const content = await fs3.readFile(filePath, "utf8");
|
|
13474
13606
|
if (signal.aborted || isGeneratedContent(content)) {
|
|
13475
13607
|
return null;
|
|
13476
13608
|
}
|
|
13477
|
-
const contentHash =
|
|
13478
|
-
const relPath =
|
|
13609
|
+
const contentHash = crypto3.createHash("sha256").update(content).digest("hex");
|
|
13610
|
+
const relPath = path5.relative(rootDir, filePath).replace(/\\/g, "/");
|
|
13479
13611
|
const cachedShard = currentDiskCache ? await currentDiskCache.getShard(relPath, contentHash) : null;
|
|
13480
13612
|
let isNewlyTokenized = false;
|
|
13481
13613
|
let shard = cachedShard;
|
|
@@ -13499,6 +13631,7 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13499
13631
|
}));
|
|
13500
13632
|
if (signal.aborted)
|
|
13501
13633
|
return { indexedCount, status: "cancelled" };
|
|
13634
|
+
const shardsToSave = [];
|
|
13502
13635
|
for (const item of fileItems) {
|
|
13503
13636
|
if (!item)
|
|
13504
13637
|
continue;
|
|
@@ -13513,7 +13646,10 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13513
13646
|
newClones = currentIndex.hydrateSourceShard(item.cachedShard);
|
|
13514
13647
|
if (item.isNewlyTokenized) {
|
|
13515
13648
|
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
13516
|
-
|
|
13649
|
+
shardsToSave.push({
|
|
13650
|
+
shard: item.cachedShard,
|
|
13651
|
+
relPath: item.relPath
|
|
13652
|
+
});
|
|
13517
13653
|
}
|
|
13518
13654
|
} else {
|
|
13519
13655
|
cachedCount++;
|
|
@@ -13523,7 +13659,7 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13523
13659
|
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
13524
13660
|
const shard = currentIndex.exportSourceShard(item.filePath, item.contentHash);
|
|
13525
13661
|
if (shard) {
|
|
13526
|
-
|
|
13662
|
+
shardsToSave.push({ shard, relPath: item.relPath });
|
|
13527
13663
|
}
|
|
13528
13664
|
}
|
|
13529
13665
|
}
|
|
@@ -13533,6 +13669,9 @@ async function runBaselineIndexing(rootDir, options, signal) {
|
|
|
13533
13669
|
notifyLateFindings(newClones);
|
|
13534
13670
|
}
|
|
13535
13671
|
}
|
|
13672
|
+
if (shardsToSave.length > 0 && currentDiskCache) {
|
|
13673
|
+
await currentDiskCache.saveShards(shardsToSave);
|
|
13674
|
+
}
|
|
13536
13675
|
const processedCount = Math.min(i + batch.length, totalFiles);
|
|
13537
13676
|
const percentage = totalFiles > 0 ? Math.round(processedCount / totalFiles * 100) : 100;
|
|
13538
13677
|
self.postMessage(createProgressEvent({
|
|
@@ -13619,12 +13758,12 @@ async function runIncrementalGitReconciliation(rootDir, options, signal) {
|
|
|
13619
13758
|
const oldRelPath = entries[i]?.trim();
|
|
13620
13759
|
i++;
|
|
13621
13760
|
if (oldRelPath) {
|
|
13622
|
-
const oldFullPath =
|
|
13761
|
+
const oldFullPath = path5.resolve(rootDir, oldRelPath);
|
|
13623
13762
|
currentIndex.removeSource(oldFullPath);
|
|
13624
13763
|
currentIndex.removeSource(oldRelPath);
|
|
13625
13764
|
}
|
|
13626
13765
|
}
|
|
13627
|
-
const fullPath =
|
|
13766
|
+
const fullPath = path5.resolve(rootDir, relPath);
|
|
13628
13767
|
if (ignoreFilter(relPath)) {
|
|
13629
13768
|
currentIndex.removeSource(fullPath);
|
|
13630
13769
|
currentIndex.removeSource(relPath);
|
|
@@ -13639,12 +13778,12 @@ async function runIncrementalGitReconciliation(rootDir, options, signal) {
|
|
|
13639
13778
|
currentIndex.removeSource(fullPath);
|
|
13640
13779
|
continue;
|
|
13641
13780
|
}
|
|
13642
|
-
const stat2 = await
|
|
13781
|
+
const stat2 = await fs3.stat(fullPath);
|
|
13643
13782
|
if (stat2.size > MAX_FILE_SIZE_BYTES || stat2.size <= 0) {
|
|
13644
13783
|
currentIndex.removeSource(fullPath);
|
|
13645
13784
|
continue;
|
|
13646
13785
|
}
|
|
13647
|
-
const content = await
|
|
13786
|
+
const content = await fs3.readFile(fullPath, "utf8");
|
|
13648
13787
|
if (signal.aborted)
|
|
13649
13788
|
return {
|
|
13650
13789
|
indexedCount: currentIndex.stats().sourceCount,
|
|
@@ -13689,16 +13828,17 @@ async function handleWorkerRequest(msg) {
|
|
|
13689
13828
|
switch (msg.type) {
|
|
13690
13829
|
case "openWorkspace": {
|
|
13691
13830
|
const { rootDir, options } = msg.payload;
|
|
13692
|
-
|
|
13831
|
+
const canonicalRootDir = canonicalizePath(rootDir);
|
|
13832
|
+
if ((currentRootDir === canonicalRootDir || currentRootDir === rootDir) && areOptionsEqual(currentOptions, options) && (isBaselineComplete || isBaselineIndexing)) {
|
|
13693
13833
|
if (isBaselineComplete) {
|
|
13694
13834
|
if (activeAbortController) {
|
|
13695
13835
|
activeAbortController.abort();
|
|
13696
13836
|
}
|
|
13697
13837
|
activeAbortController = new AbortController;
|
|
13698
|
-
const recResult = await runIncrementalGitReconciliation(
|
|
13838
|
+
const recResult = await runIncrementalGitReconciliation(currentRootDir, options, activeAbortController.signal);
|
|
13699
13839
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13700
13840
|
started: true,
|
|
13701
|
-
rootDir,
|
|
13841
|
+
rootDir: currentRootDir,
|
|
13702
13842
|
reused: true,
|
|
13703
13843
|
indexedCount: recResult.indexedCount,
|
|
13704
13844
|
status: recResult.status
|
|
@@ -13706,7 +13846,7 @@ async function handleWorkerRequest(msg) {
|
|
|
13706
13846
|
} else {
|
|
13707
13847
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13708
13848
|
started: true,
|
|
13709
|
-
rootDir,
|
|
13849
|
+
rootDir: currentRootDir,
|
|
13710
13850
|
reused: true,
|
|
13711
13851
|
indexedCount: currentIndex.stats().sourceCount,
|
|
13712
13852
|
status: "complete"
|
|
@@ -13722,11 +13862,14 @@ async function handleWorkerRequest(msg) {
|
|
|
13722
13862
|
currentDiskCache.close();
|
|
13723
13863
|
currentDiskCache = null;
|
|
13724
13864
|
}
|
|
13725
|
-
|
|
13865
|
+
const repoContext = await resolveRepositoryContext(rootDir, activeAbortController.signal);
|
|
13866
|
+
const effectiveRoot = repoContext.workspaceRoot;
|
|
13867
|
+
currentRootDir = effectiveRoot;
|
|
13726
13868
|
currentOptions = options;
|
|
13727
13869
|
currentIndex = new SourceAwareCloneIndex(options);
|
|
13728
13870
|
currentDiskCache = new DiskCacheManager({
|
|
13729
|
-
rootDir,
|
|
13871
|
+
rootDir: effectiveRoot,
|
|
13872
|
+
repositoryKey: repoContext.repositoryKey,
|
|
13730
13873
|
cacheDir: options?.cacheDir,
|
|
13731
13874
|
config: options,
|
|
13732
13875
|
maxBytes: options?.maxCacheBytes
|
|
@@ -13734,12 +13877,13 @@ async function handleWorkerRequest(msg) {
|
|
|
13734
13877
|
watchedRevisions.clear();
|
|
13735
13878
|
isBaselineIndexing = true;
|
|
13736
13879
|
isBaselineComplete = false;
|
|
13880
|
+
cleanupLegacyCacheFiles(options?.cacheDir).catch(() => {});
|
|
13737
13881
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13738
13882
|
started: true,
|
|
13739
|
-
rootDir,
|
|
13883
|
+
rootDir: effectiveRoot,
|
|
13740
13884
|
reused: false
|
|
13741
13885
|
}));
|
|
13742
|
-
runBaselineIndexing(
|
|
13886
|
+
runBaselineIndexing(effectiveRoot, options, activeAbortController.signal).catch(() => {});
|
|
13743
13887
|
break;
|
|
13744
13888
|
}
|
|
13745
13889
|
case "checkSnippet": {
|
|
@@ -13751,17 +13895,19 @@ async function handleWorkerRequest(msg) {
|
|
|
13751
13895
|
case "checkAndUpdate": {
|
|
13752
13896
|
const { filePath, content, format, revision = 1 } = msg.payload;
|
|
13753
13897
|
const clones = currentIndex.updateSource(filePath, content, format);
|
|
13754
|
-
const
|
|
13898
|
+
const canonicalFilePath = canonicalizePath(filePath);
|
|
13899
|
+
const resolvedPath = path5.resolve(filePath);
|
|
13755
13900
|
cacheSourceShard(filePath, content);
|
|
13756
|
-
const fileClones = clones.filter((c) => c.duplicationA.sourceId === filePath || c.duplicationB.sourceId === filePath ||
|
|
13901
|
+
const fileClones = clones.filter((c) => c.duplicationA.sourceId === filePath || c.duplicationB.sourceId === filePath || path5.resolve(c.duplicationA.sourceId) === resolvedPath || path5.resolve(c.duplicationB.sourceId) === resolvedPath || canonicalizePath(c.duplicationA.sourceId) === canonicalFilePath || canonicalizePath(c.duplicationB.sourceId) === canonicalFilePath);
|
|
13757
13902
|
const watchEntry = {
|
|
13758
13903
|
revision,
|
|
13759
13904
|
lastKnownCloneCount: fileClones.length
|
|
13760
13905
|
};
|
|
13761
13906
|
watchedRevisions.set(filePath, watchEntry);
|
|
13762
13907
|
watchedRevisions.set(resolvedPath, watchEntry);
|
|
13908
|
+
watchedRevisions.set(canonicalFilePath, watchEntry);
|
|
13763
13909
|
self.postMessage(createSuccessResponse(msg.id, {
|
|
13764
|
-
clones,
|
|
13910
|
+
clones: fileClones,
|
|
13765
13911
|
isComplete: isBaselineComplete
|
|
13766
13912
|
}));
|
|
13767
13913
|
break;
|
|
@@ -13793,9 +13939,9 @@ async function handleWorkerRequest(msg) {
|
|
|
13793
13939
|
currentIndex.removeSource(fileEntry.filePath);
|
|
13794
13940
|
continue;
|
|
13795
13941
|
}
|
|
13796
|
-
const stat2 = await
|
|
13942
|
+
const stat2 = await fs3.stat(fileEntry.filePath);
|
|
13797
13943
|
if (stat2.size <= MAX_FILE_SIZE_BYTES && stat2.size > 0) {
|
|
13798
|
-
const content = await
|
|
13944
|
+
const content = await fs3.readFile(fileEntry.filePath, "utf8");
|
|
13799
13945
|
if (!isGeneratedContent(content)) {
|
|
13800
13946
|
currentIndex.updateSource(fileEntry.filePath, content);
|
|
13801
13947
|
cacheSourceShard(fileEntry.filePath, content);
|
|
@@ -13839,9 +13985,9 @@ async function handleWorkerRequest(msg) {
|
|
|
13839
13985
|
try {
|
|
13840
13986
|
if (!getSupportedCodeFormat(file, optionsToUse?.formatsExts))
|
|
13841
13987
|
continue;
|
|
13842
|
-
const stat2 = await
|
|
13988
|
+
const stat2 = await fs3.stat(file);
|
|
13843
13989
|
if (stat2.size <= MAX_FILE_SIZE_BYTES && stat2.size > 0) {
|
|
13844
|
-
const content = await
|
|
13990
|
+
const content = await fs3.readFile(file, "utf8");
|
|
13845
13991
|
if (!isGeneratedContent(content)) {
|
|
13846
13992
|
indexToUse.addSource(file, content);
|
|
13847
13993
|
}
|