omp-plugin-duplicate-detector 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -0
- package/dist/detector-worker.js +13897 -0
- package/package.json +105 -0
- package/src/config-loader.ts +336 -0
- package/src/coordinator.ts +536 -0
- package/src/detector-worker.ts +828 -0
- package/src/disk-cache.ts +703 -0
- package/src/duplicate-ledger.ts +144 -0
- package/src/index.ts +807 -0
- package/src/jscpd-engine.ts +797 -0
- package/src/project-state.ts +129 -0
- package/src/source-aware-index.ts +919 -0
- package/src/test-detector.ts +337 -0
- package/src/tui-notification.ts +464 -0
- package/src/worker-protocol.ts +330 -0
- package/types/global.d.ts +19 -0
- package/types/jscpd.d.ts +139 -0
|
@@ -0,0 +1,828 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker thread entry point for duplicate detection.
|
|
3
|
+
* Runs in Bun.Worker, managing background baseline indexing and answering RPC requests
|
|
4
|
+
* from the main thread via SourceAwareCloneIndex.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import * as crypto from "node:crypto";
|
|
8
|
+
import * as fs from "node:fs/promises";
|
|
9
|
+
import * as path from "node:path";
|
|
10
|
+
import type { IClone } from "@jscpd/core";
|
|
11
|
+
import { DiskCacheManager } from "./disk-cache";
|
|
12
|
+
import {
|
|
13
|
+
type BaselineStatus,
|
|
14
|
+
createIgnoreFilter,
|
|
15
|
+
execGit,
|
|
16
|
+
getSupportedCodeFormat,
|
|
17
|
+
getTrackedGitFiles,
|
|
18
|
+
isGeneratedContent,
|
|
19
|
+
isInsideGitWorkTree,
|
|
20
|
+
MAX_FILE_SIZE_BYTES,
|
|
21
|
+
MAX_GIT_PATHS,
|
|
22
|
+
MAX_INDEXED_FILES,
|
|
23
|
+
MAX_TOTAL_SOURCE_BYTES,
|
|
24
|
+
} from "./jscpd-engine";
|
|
25
|
+
import {
|
|
26
|
+
type SerializedSourceShard,
|
|
27
|
+
SourceAwareCloneIndex,
|
|
28
|
+
} from "./source-aware-index";
|
|
29
|
+
import {
|
|
30
|
+
createCompleteEvent,
|
|
31
|
+
createErrorResponse,
|
|
32
|
+
createLateFindingEvent,
|
|
33
|
+
createProgressEvent,
|
|
34
|
+
createStatusEvent,
|
|
35
|
+
createSuccessResponse,
|
|
36
|
+
isWorkerRequest,
|
|
37
|
+
type WorkerRequestMessage,
|
|
38
|
+
type WorkspaceOptions,
|
|
39
|
+
} from "./worker-protocol";
|
|
40
|
+
|
|
41
|
+
// Type declaration for Worker global scope
|
|
42
|
+
declare const self: {
|
|
43
|
+
onmessage: ((event: MessageEvent<unknown>) => void) | null;
|
|
44
|
+
postMessage: (message: unknown) => void;
|
|
45
|
+
close?: () => void;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// ============================================================================
|
|
49
|
+
// Worker State
|
|
50
|
+
// ============================================================================
|
|
51
|
+
|
|
52
|
+
let currentIndex: SourceAwareCloneIndex = new SourceAwareCloneIndex();
|
|
53
|
+
let currentDiskCache: DiskCacheManager | null = null;
|
|
54
|
+
let currentRootDir = "";
|
|
55
|
+
let currentOptions: WorkspaceOptions | undefined;
|
|
56
|
+
let activeAbortController: AbortController | null = null;
|
|
57
|
+
let isBaselineIndexing = false;
|
|
58
|
+
let isBaselineComplete = false;
|
|
59
|
+
const watchedRevisions = new Map<
|
|
60
|
+
string,
|
|
61
|
+
{ revision: number; lastKnownCloneCount: number }
|
|
62
|
+
>();
|
|
63
|
+
|
|
64
|
+
const BATCH_SIZE = 32;
|
|
65
|
+
|
|
66
|
+
function areOptionsEqual(a?: WorkspaceOptions, b?: WorkspaceOptions): boolean {
|
|
67
|
+
if (a === b) return true;
|
|
68
|
+
if (!a && !b) return true;
|
|
69
|
+
if (!a || !b) return false;
|
|
70
|
+
if (a.minTokens !== b.minTokens) return false;
|
|
71
|
+
if (a.minLines !== b.minLines) return false;
|
|
72
|
+
if (a.maxLines !== b.maxLines) return false;
|
|
73
|
+
if (a.maxIndexedFiles !== b.maxIndexedFiles) return false;
|
|
74
|
+
if (a.ignoreTests !== b.ignoreTests) return false;
|
|
75
|
+
const aCustom = (a.customTestPatterns ?? []).slice().sort().join(",");
|
|
76
|
+
const bCustom = (b.customTestPatterns ?? []).slice().sort().join(",");
|
|
77
|
+
if (aCustom !== bCustom) return false;
|
|
78
|
+
const aExclude = (a.excludeTestPatterns ?? []).slice().sort().join(",");
|
|
79
|
+
const bExclude = (b.excludeTestPatterns ?? []).slice().sort().join(",");
|
|
80
|
+
if (aExclude !== bExclude) return false;
|
|
81
|
+
const aIgnores = (a.ignorePatterns ?? []).slice().sort().join(",");
|
|
82
|
+
const bIgnores = (b.ignorePatterns ?? []).slice().sort().join(",");
|
|
83
|
+
if (aIgnores !== bIgnores) return false;
|
|
84
|
+
if (a.cacheDir !== b.cacheDir) return false;
|
|
85
|
+
if (a.maxCacheBytes !== b.maxCacheBytes) return false;
|
|
86
|
+
const aFormats = a.formatsExts ? JSON.stringify(a.formatsExts) : "";
|
|
87
|
+
const bFormats = b.formatsExts ? JSON.stringify(b.formatsExts) : "";
|
|
88
|
+
return aFormats === bFormats;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function notifyLateFindings(clones: IClone[]): void {
|
|
92
|
+
for (const clone of clones) {
|
|
93
|
+
const srcA = clone.duplicationA.sourceId;
|
|
94
|
+
const srcB = clone.duplicationB.sourceId;
|
|
95
|
+
const resA = path.resolve(srcA);
|
|
96
|
+
const resB = path.resolve(srcB);
|
|
97
|
+
|
|
98
|
+
const isWatchedA = watchedRevisions.has(srcA) || watchedRevisions.has(resA);
|
|
99
|
+
const isWatchedB = watchedRevisions.has(srcB) || watchedRevisions.has(resB);
|
|
100
|
+
|
|
101
|
+
if (isWatchedA || isWatchedB) {
|
|
102
|
+
if (isWatchedA) {
|
|
103
|
+
const entry = watchedRevisions.get(srcA) ?? watchedRevisions.get(resA);
|
|
104
|
+
if (entry) entry.lastKnownCloneCount++;
|
|
105
|
+
}
|
|
106
|
+
if (isWatchedB) {
|
|
107
|
+
const entry = watchedRevisions.get(srcB) ?? watchedRevisions.get(resB);
|
|
108
|
+
if (entry) entry.lastKnownCloneCount++;
|
|
109
|
+
}
|
|
110
|
+
self.postMessage(createLateFindingEvent(clone));
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function cacheSourceShard(filePath: string, content: string): void {
|
|
116
|
+
if (!currentDiskCache) return;
|
|
117
|
+
const contentHash = crypto.createHash("sha256").update(content).digest("hex");
|
|
118
|
+
const relPath = path.relative(currentRootDir, filePath).replace(/\\/g, "/");
|
|
119
|
+
const shard = currentIndex.exportSourceShard(filePath, contentHash);
|
|
120
|
+
if (shard) {
|
|
121
|
+
currentDiskCache.saveShard(shard, relPath).catch(() => {});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ============================================================================
|
|
126
|
+
// Baseline Indexing Worker Task
|
|
127
|
+
// ============================================================================
|
|
128
|
+
|
|
129
|
+
function yieldTask(): Promise<void> {
|
|
130
|
+
const { promise, resolve } = Promise.withResolvers<void>();
|
|
131
|
+
setTimeout(resolve, 0);
|
|
132
|
+
return promise;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
interface BatchItem {
|
|
136
|
+
filePath: string;
|
|
137
|
+
content: string | null;
|
|
138
|
+
contentHash: string | null;
|
|
139
|
+
relPath: string | null;
|
|
140
|
+
size: number;
|
|
141
|
+
cachedShard: SerializedSourceShard | null;
|
|
142
|
+
isNewlyTokenized: boolean;
|
|
143
|
+
alreadyIndexed: boolean;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async function runBaselineIndexing(
|
|
147
|
+
rootDir: string,
|
|
148
|
+
options: WorkspaceOptions | undefined,
|
|
149
|
+
signal: AbortSignal,
|
|
150
|
+
): Promise<{ indexedCount: number; status: BaselineStatus }> {
|
|
151
|
+
const startTime = Date.now();
|
|
152
|
+
let indexedCount = 0;
|
|
153
|
+
let cachedCount = 0;
|
|
154
|
+
let totalSourceBytes = 0;
|
|
155
|
+
|
|
156
|
+
try {
|
|
157
|
+
const isGit = await isInsideGitWorkTree(rootDir, signal);
|
|
158
|
+
if (signal.aborted) return { indexedCount: 0, status: "cancelled" };
|
|
159
|
+
|
|
160
|
+
if (!isGit) {
|
|
161
|
+
isBaselineIndexing = false;
|
|
162
|
+
isBaselineComplete = true;
|
|
163
|
+
self.postMessage(
|
|
164
|
+
createStatusEvent("idle", "Workspace is not inside a Git repository"),
|
|
165
|
+
);
|
|
166
|
+
self.postMessage(
|
|
167
|
+
createCompleteEvent({
|
|
168
|
+
indexedCount: 0,
|
|
169
|
+
cachedCount: 0,
|
|
170
|
+
totalSourceBytes: 0,
|
|
171
|
+
cloneCount: 0,
|
|
172
|
+
durationMs: Date.now() - startTime,
|
|
173
|
+
status: "skipped_not_git",
|
|
174
|
+
}),
|
|
175
|
+
);
|
|
176
|
+
return { indexedCount: 0, status: "skipped_not_git" };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
self.postMessage(
|
|
180
|
+
createStatusEvent("indexing", "Enumerating tracked Git files..."),
|
|
181
|
+
);
|
|
182
|
+
|
|
183
|
+
const maxIndexedFiles = options?.maxIndexedFiles ?? MAX_INDEXED_FILES;
|
|
184
|
+
const trackedFiles = await getTrackedGitFiles(rootDir, {
|
|
185
|
+
userIgnorePatterns: options?.ignorePatterns,
|
|
186
|
+
ignoreTests: options?.ignoreTests,
|
|
187
|
+
customTestPatterns: options?.customTestPatterns,
|
|
188
|
+
excludeTestPatterns: options?.excludeTestPatterns,
|
|
189
|
+
signal,
|
|
190
|
+
maxPaths: Math.max(MAX_GIT_PATHS, maxIndexedFiles),
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
if (signal.aborted) return { indexedCount: 0, status: "cancelled" };
|
|
194
|
+
|
|
195
|
+
const totalFiles = trackedFiles.length;
|
|
196
|
+
self.postMessage(
|
|
197
|
+
createProgressEvent({
|
|
198
|
+
phase: "indexing",
|
|
199
|
+
indexedCount: 0,
|
|
200
|
+
totalFiles,
|
|
201
|
+
percentage: 0,
|
|
202
|
+
}),
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
let baselineStatus:
|
|
206
|
+
| "complete"
|
|
207
|
+
| "capped_file_count"
|
|
208
|
+
| "capped_source_bytes" = "complete";
|
|
209
|
+
|
|
210
|
+
for (let i = 0; i < trackedFiles.length; i += BATCH_SIZE) {
|
|
211
|
+
if (signal.aborted) return { indexedCount, status: "cancelled" };
|
|
212
|
+
|
|
213
|
+
if (indexedCount >= maxIndexedFiles) {
|
|
214
|
+
baselineStatus = "capped_file_count";
|
|
215
|
+
self.postMessage(
|
|
216
|
+
createStatusEvent("ready", "Indexed file limit reached"),
|
|
217
|
+
);
|
|
218
|
+
break;
|
|
219
|
+
}
|
|
220
|
+
if (totalSourceBytes >= MAX_TOTAL_SOURCE_BYTES) {
|
|
221
|
+
baselineStatus = "capped_source_bytes";
|
|
222
|
+
self.postMessage(
|
|
223
|
+
createStatusEvent("ready", "Source byte limit reached"),
|
|
224
|
+
);
|
|
225
|
+
break;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const batch = trackedFiles.slice(i, i + BATCH_SIZE);
|
|
229
|
+
|
|
230
|
+
// Process file batch I/O and cache lookups concurrently
|
|
231
|
+
const fileItems = await Promise.all(
|
|
232
|
+
batch.map(async (filePath): Promise<BatchItem | null> => {
|
|
233
|
+
if (signal.aborted) return null;
|
|
234
|
+
|
|
235
|
+
// Early skip if file format is unsupported by code duplicate detector
|
|
236
|
+
if (!getSupportedCodeFormat(filePath, options?.formatsExts)) {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
try {
|
|
241
|
+
const stat = await fs.stat(filePath);
|
|
242
|
+
if (stat.size > MAX_FILE_SIZE_BYTES || stat.size <= 0) {
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const resolved = path.resolve(filePath);
|
|
247
|
+
if (
|
|
248
|
+
currentIndex.hasSource(filePath) ||
|
|
249
|
+
currentIndex.hasSource(resolved)
|
|
250
|
+
) {
|
|
251
|
+
return {
|
|
252
|
+
filePath,
|
|
253
|
+
content: null,
|
|
254
|
+
contentHash: null,
|
|
255
|
+
relPath: null,
|
|
256
|
+
size: stat.size,
|
|
257
|
+
cachedShard: null,
|
|
258
|
+
isNewlyTokenized: false,
|
|
259
|
+
alreadyIndexed: true,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const content = await fs.readFile(filePath, "utf8");
|
|
264
|
+
if (signal.aborted || isGeneratedContent(content)) {
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const contentHash = crypto
|
|
269
|
+
.createHash("sha256")
|
|
270
|
+
.update(content)
|
|
271
|
+
.digest("hex");
|
|
272
|
+
const relPath = path
|
|
273
|
+
.relative(rootDir, filePath)
|
|
274
|
+
.replace(/\\/g, "/");
|
|
275
|
+
|
|
276
|
+
const cachedShard = currentDiskCache
|
|
277
|
+
? await currentDiskCache.getShard(relPath, contentHash)
|
|
278
|
+
: null;
|
|
279
|
+
|
|
280
|
+
let isNewlyTokenized = false;
|
|
281
|
+
let shard = cachedShard;
|
|
282
|
+
if (!shard) {
|
|
283
|
+
shard = currentIndex.tokenizeSource(
|
|
284
|
+
filePath,
|
|
285
|
+
content,
|
|
286
|
+
contentHash,
|
|
287
|
+
);
|
|
288
|
+
isNewlyTokenized = true;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
return {
|
|
292
|
+
filePath,
|
|
293
|
+
content: shard ? null : content,
|
|
294
|
+
contentHash,
|
|
295
|
+
relPath,
|
|
296
|
+
size: stat.size,
|
|
297
|
+
cachedShard: shard,
|
|
298
|
+
isNewlyTokenized,
|
|
299
|
+
alreadyIndexed: false,
|
|
300
|
+
};
|
|
301
|
+
} catch {
|
|
302
|
+
return null;
|
|
303
|
+
}
|
|
304
|
+
}),
|
|
305
|
+
);
|
|
306
|
+
|
|
307
|
+
if (signal.aborted) return { indexedCount, status: "cancelled" };
|
|
308
|
+
|
|
309
|
+
// Ingest items into index and notify late findings
|
|
310
|
+
for (const item of fileItems) {
|
|
311
|
+
if (!item) continue;
|
|
312
|
+
|
|
313
|
+
if (item.alreadyIndexed) {
|
|
314
|
+
indexedCount++;
|
|
315
|
+
totalSourceBytes += item.size;
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
let newClones: IClone[] = [];
|
|
320
|
+
|
|
321
|
+
if (item.cachedShard) {
|
|
322
|
+
item.cachedShard.sourceId = item.filePath;
|
|
323
|
+
newClones = currentIndex.hydrateSourceShard(item.cachedShard);
|
|
324
|
+
if (item.isNewlyTokenized) {
|
|
325
|
+
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
326
|
+
currentDiskCache
|
|
327
|
+
.saveShard(item.cachedShard, item.relPath)
|
|
328
|
+
.catch(() => {});
|
|
329
|
+
}
|
|
330
|
+
} else {
|
|
331
|
+
cachedCount++;
|
|
332
|
+
}
|
|
333
|
+
} else if (item.content) {
|
|
334
|
+
newClones = currentIndex.addSource(item.filePath, item.content);
|
|
335
|
+
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
336
|
+
const shard = currentIndex.exportSourceShard(
|
|
337
|
+
item.filePath,
|
|
338
|
+
item.contentHash,
|
|
339
|
+
);
|
|
340
|
+
if (shard) {
|
|
341
|
+
currentDiskCache.saveShard(shard, item.relPath).catch(() => {});
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
indexedCount++;
|
|
346
|
+
totalSourceBytes += item.size;
|
|
347
|
+
|
|
348
|
+
if (newClones.length > 0) {
|
|
349
|
+
notifyLateFindings(newClones);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const processedCount = Math.min(i + batch.length, totalFiles);
|
|
354
|
+
const percentage =
|
|
355
|
+
totalFiles > 0 ? Math.round((processedCount / totalFiles) * 100) : 100;
|
|
356
|
+
self.postMessage(
|
|
357
|
+
createProgressEvent({
|
|
358
|
+
phase: "indexing",
|
|
359
|
+
indexedCount,
|
|
360
|
+
totalFiles,
|
|
361
|
+
currentFile: batch[batch.length - 1],
|
|
362
|
+
percentage,
|
|
363
|
+
}),
|
|
364
|
+
);
|
|
365
|
+
|
|
366
|
+
// Yield macrotask once per batch to process pending RPC requests
|
|
367
|
+
await yieldTask();
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
if (!signal.aborted) {
|
|
371
|
+
isBaselineIndexing = false;
|
|
372
|
+
isBaselineComplete = true;
|
|
373
|
+
currentDiskCache?.prune().catch(() => {});
|
|
374
|
+
self.postMessage(
|
|
375
|
+
createCompleteEvent({
|
|
376
|
+
indexedCount,
|
|
377
|
+
cachedCount,
|
|
378
|
+
totalSourceBytes,
|
|
379
|
+
cloneCount: currentIndex.clones.length,
|
|
380
|
+
durationMs: Date.now() - startTime,
|
|
381
|
+
status: baselineStatus,
|
|
382
|
+
}),
|
|
383
|
+
);
|
|
384
|
+
self.postMessage(
|
|
385
|
+
createStatusEvent("ready", "Baseline indexing complete"),
|
|
386
|
+
);
|
|
387
|
+
return { indexedCount, status: baselineStatus };
|
|
388
|
+
}
|
|
389
|
+
return { indexedCount, status: "cancelled" };
|
|
390
|
+
} catch (err) {
|
|
391
|
+
if (signal.aborted) return { indexedCount, status: "cancelled" };
|
|
392
|
+
isBaselineIndexing = false;
|
|
393
|
+
isBaselineComplete = false;
|
|
394
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
395
|
+
self.postMessage(createStatusEvent("error", error));
|
|
396
|
+
self.postMessage(
|
|
397
|
+
createCompleteEvent({
|
|
398
|
+
indexedCount,
|
|
399
|
+
cachedCount,
|
|
400
|
+
totalSourceBytes,
|
|
401
|
+
cloneCount: currentIndex.clones.length,
|
|
402
|
+
durationMs: Date.now() - startTime,
|
|
403
|
+
status: "failed",
|
|
404
|
+
error,
|
|
405
|
+
}),
|
|
406
|
+
);
|
|
407
|
+
return { indexedCount, status: "failed" };
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
async function runIncrementalGitReconciliation(
|
|
412
|
+
rootDir: string,
|
|
413
|
+
options: WorkspaceOptions | undefined,
|
|
414
|
+
signal: AbortSignal,
|
|
415
|
+
): Promise<{ indexedCount: number; status: BaselineStatus }> {
|
|
416
|
+
try {
|
|
417
|
+
const isGit = await isInsideGitWorkTree(rootDir, signal);
|
|
418
|
+
if (!isGit || signal.aborted) {
|
|
419
|
+
return {
|
|
420
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
421
|
+
status: !isGit ? "skipped_not_git" : "complete",
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
const ignoreFilter = createIgnoreFilter(options?.ignorePatterns, {
|
|
425
|
+
ignoreTests: options?.ignoreTests,
|
|
426
|
+
customTestPatterns: options?.customTestPatterns,
|
|
427
|
+
excludeTestPatterns: options?.excludeTestPatterns,
|
|
428
|
+
});
|
|
429
|
+
const { stdout } = await execGit(
|
|
430
|
+
["status", "--porcelain", "-z", "--", "."],
|
|
431
|
+
rootDir,
|
|
432
|
+
{ signal },
|
|
433
|
+
);
|
|
434
|
+
|
|
435
|
+
if (signal.aborted)
|
|
436
|
+
return {
|
|
437
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
438
|
+
status: "cancelled",
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
const entries = stdout.split("\0");
|
|
442
|
+
let i = 0;
|
|
443
|
+
while (i < entries.length) {
|
|
444
|
+
if (signal.aborted)
|
|
445
|
+
return {
|
|
446
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
447
|
+
status: "cancelled",
|
|
448
|
+
};
|
|
449
|
+
const entry = entries[i];
|
|
450
|
+
i++;
|
|
451
|
+
if (!entry || entry.length < 4) continue;
|
|
452
|
+
|
|
453
|
+
const statusCode = entry.slice(0, 2);
|
|
454
|
+
const relPath = entry.slice(3).trim();
|
|
455
|
+
if (!relPath) continue;
|
|
456
|
+
|
|
457
|
+
if (statusCode.includes("R") && i < entries.length) {
|
|
458
|
+
const oldRelPath = entries[i]?.trim();
|
|
459
|
+
i++;
|
|
460
|
+
if (oldRelPath) {
|
|
461
|
+
const oldFullPath = path.resolve(rootDir, oldRelPath);
|
|
462
|
+
currentIndex.removeSource(oldFullPath);
|
|
463
|
+
currentIndex.removeSource(oldRelPath);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
const fullPath = path.resolve(rootDir, relPath);
|
|
468
|
+
|
|
469
|
+
if (ignoreFilter(relPath)) {
|
|
470
|
+
currentIndex.removeSource(fullPath);
|
|
471
|
+
currentIndex.removeSource(relPath);
|
|
472
|
+
continue;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
if (statusCode.includes("D")) {
|
|
476
|
+
currentIndex.removeSource(fullPath);
|
|
477
|
+
currentIndex.removeSource(relPath);
|
|
478
|
+
} else {
|
|
479
|
+
try {
|
|
480
|
+
if (!getSupportedCodeFormat(fullPath, options?.formatsExts)) {
|
|
481
|
+
currentIndex.removeSource(fullPath);
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
const stat = await fs.stat(fullPath);
|
|
485
|
+
if (stat.size > MAX_FILE_SIZE_BYTES || stat.size <= 0) {
|
|
486
|
+
currentIndex.removeSource(fullPath);
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
const content = await fs.readFile(fullPath, "utf8");
|
|
490
|
+
if (signal.aborted)
|
|
491
|
+
return {
|
|
492
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
493
|
+
status: "cancelled",
|
|
494
|
+
};
|
|
495
|
+
|
|
496
|
+
if (isGeneratedContent(content)) {
|
|
497
|
+
currentIndex.removeSource(fullPath);
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
const newClones = currentIndex.updateSource(fullPath, content);
|
|
502
|
+
cacheSourceShard(fullPath, content);
|
|
503
|
+
if (newClones.length > 0) {
|
|
504
|
+
notifyLateFindings(newClones);
|
|
505
|
+
}
|
|
506
|
+
} catch {
|
|
507
|
+
currentIndex.removeSource(fullPath);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
await yieldTask();
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
if (!signal.aborted) {
|
|
515
|
+
self.postMessage(
|
|
516
|
+
createCompleteEvent({
|
|
517
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
518
|
+
totalSourceBytes: 0,
|
|
519
|
+
cloneCount: currentIndex.clones.length,
|
|
520
|
+
durationMs: 0,
|
|
521
|
+
status: "complete",
|
|
522
|
+
}),
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
return {
|
|
526
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
527
|
+
status: "complete",
|
|
528
|
+
};
|
|
529
|
+
} catch {
|
|
530
|
+
// Non-fatal error during incremental reconciliation
|
|
531
|
+
return {
|
|
532
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
533
|
+
status: "complete",
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// ============================================================================
|
|
539
|
+
// Message Dispatcher
|
|
540
|
+
// ============================================================================
|
|
541
|
+
|
|
542
|
+
async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
543
|
+
switch (msg.type) {
|
|
544
|
+
case "openWorkspace": {
|
|
545
|
+
const { rootDir, options } = msg.payload;
|
|
546
|
+
|
|
547
|
+
// If rootDir === currentRootDir and options match and baseline is already complete or indexing,
|
|
548
|
+
// avoid total reset; trigger an incremental git status check instead.
|
|
549
|
+
if (
|
|
550
|
+
currentRootDir === rootDir &&
|
|
551
|
+
areOptionsEqual(currentOptions, options) &&
|
|
552
|
+
(isBaselineComplete || isBaselineIndexing)
|
|
553
|
+
) {
|
|
554
|
+
if (isBaselineComplete) {
|
|
555
|
+
if (activeAbortController) {
|
|
556
|
+
activeAbortController.abort();
|
|
557
|
+
}
|
|
558
|
+
activeAbortController = new AbortController();
|
|
559
|
+
const recResult = await runIncrementalGitReconciliation(
|
|
560
|
+
rootDir,
|
|
561
|
+
options,
|
|
562
|
+
activeAbortController.signal,
|
|
563
|
+
);
|
|
564
|
+
self.postMessage(
|
|
565
|
+
createSuccessResponse(msg.id, {
|
|
566
|
+
started: true,
|
|
567
|
+
rootDir,
|
|
568
|
+
reused: true,
|
|
569
|
+
indexedCount: recResult.indexedCount,
|
|
570
|
+
status: recResult.status,
|
|
571
|
+
}),
|
|
572
|
+
);
|
|
573
|
+
} else {
|
|
574
|
+
self.postMessage(
|
|
575
|
+
createSuccessResponse(msg.id, {
|
|
576
|
+
started: true,
|
|
577
|
+
rootDir,
|
|
578
|
+
reused: true,
|
|
579
|
+
indexedCount: currentIndex.stats().sourceCount,
|
|
580
|
+
status: "complete" as BaselineStatus,
|
|
581
|
+
}),
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
break;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Abort any ongoing indexing and close previous workspace cache
|
|
588
|
+
if (activeAbortController) {
|
|
589
|
+
activeAbortController.abort();
|
|
590
|
+
}
|
|
591
|
+
activeAbortController = new AbortController();
|
|
592
|
+
if (currentDiskCache) {
|
|
593
|
+
currentDiskCache.close();
|
|
594
|
+
currentDiskCache = null;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
currentRootDir = rootDir;
|
|
598
|
+
currentOptions = options;
|
|
599
|
+
currentIndex = new SourceAwareCloneIndex(options);
|
|
600
|
+
currentDiskCache = new DiskCacheManager({
|
|
601
|
+
rootDir,
|
|
602
|
+
cacheDir: options?.cacheDir,
|
|
603
|
+
config: options,
|
|
604
|
+
maxBytes: options?.maxCacheBytes,
|
|
605
|
+
});
|
|
606
|
+
watchedRevisions.clear();
|
|
607
|
+
isBaselineIndexing = true;
|
|
608
|
+
isBaselineComplete = false;
|
|
609
|
+
|
|
610
|
+
self.postMessage(
|
|
611
|
+
createSuccessResponse(msg.id, {
|
|
612
|
+
started: true,
|
|
613
|
+
rootDir,
|
|
614
|
+
reused: false,
|
|
615
|
+
}),
|
|
616
|
+
);
|
|
617
|
+
|
|
618
|
+
// Run background indexing task (posts complete event when finished)
|
|
619
|
+
runBaselineIndexing(rootDir, options, activeAbortController.signal).catch(
|
|
620
|
+
() => {},
|
|
621
|
+
);
|
|
622
|
+
break;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
case "checkSnippet": {
|
|
626
|
+
const { filePath, content, format } = msg.payload;
|
|
627
|
+
const clones = currentIndex.checkSnippet(filePath, content, format);
|
|
628
|
+
self.postMessage(createSuccessResponse(msg.id, clones));
|
|
629
|
+
break;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
case "checkAndUpdate": {
|
|
633
|
+
const { filePath, content, format, revision = 1 } = msg.payload;
|
|
634
|
+
const clones = currentIndex.updateSource(filePath, content, format);
|
|
635
|
+
const resolvedPath = path.resolve(filePath);
|
|
636
|
+
|
|
637
|
+
cacheSourceShard(filePath, content);
|
|
638
|
+
|
|
639
|
+
const fileClones = clones.filter(
|
|
640
|
+
(c) =>
|
|
641
|
+
c.duplicationA.sourceId === filePath ||
|
|
642
|
+
c.duplicationB.sourceId === filePath ||
|
|
643
|
+
path.resolve(c.duplicationA.sourceId) === resolvedPath ||
|
|
644
|
+
path.resolve(c.duplicationB.sourceId) === resolvedPath,
|
|
645
|
+
);
|
|
646
|
+
|
|
647
|
+
const watchEntry = {
|
|
648
|
+
revision,
|
|
649
|
+
lastKnownCloneCount: fileClones.length,
|
|
650
|
+
};
|
|
651
|
+
watchedRevisions.set(filePath, watchEntry);
|
|
652
|
+
watchedRevisions.set(resolvedPath, watchEntry);
|
|
653
|
+
|
|
654
|
+
self.postMessage(
|
|
655
|
+
createSuccessResponse(msg.id, {
|
|
656
|
+
clones,
|
|
657
|
+
isComplete: isBaselineComplete,
|
|
658
|
+
}),
|
|
659
|
+
);
|
|
660
|
+
break;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
case "updateFile": {
|
|
664
|
+
const { filePath, content, format } = msg.payload;
|
|
665
|
+
const clones = currentIndex.updateSource(filePath, content, format);
|
|
666
|
+
cacheSourceShard(filePath, content);
|
|
667
|
+
self.postMessage(createSuccessResponse(msg.id, { clones }));
|
|
668
|
+
break;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
case "removeFile": {
|
|
672
|
+
const { filePath } = msg.payload;
|
|
673
|
+
currentIndex.removeSource(filePath);
|
|
674
|
+
self.postMessage(createSuccessResponse(msg.id, { removed: true }));
|
|
675
|
+
break;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
case "reconcile": {
|
|
679
|
+
const { files } = msg.payload;
|
|
680
|
+
let reconciledCount = 0;
|
|
681
|
+
|
|
682
|
+
for (const fileEntry of files) {
|
|
683
|
+
try {
|
|
684
|
+
if (fileEntry.content !== undefined) {
|
|
685
|
+
currentIndex.updateSource(fileEntry.filePath, fileEntry.content);
|
|
686
|
+
cacheSourceShard(fileEntry.filePath, fileEntry.content);
|
|
687
|
+
reconciledCount++;
|
|
688
|
+
} else {
|
|
689
|
+
if (
|
|
690
|
+
!getSupportedCodeFormat(
|
|
691
|
+
fileEntry.filePath,
|
|
692
|
+
currentOptions?.formatsExts,
|
|
693
|
+
)
|
|
694
|
+
) {
|
|
695
|
+
currentIndex.removeSource(fileEntry.filePath);
|
|
696
|
+
continue;
|
|
697
|
+
}
|
|
698
|
+
const stat = await fs.stat(fileEntry.filePath);
|
|
699
|
+
if (stat.size <= MAX_FILE_SIZE_BYTES && stat.size > 0) {
|
|
700
|
+
const content = await fs.readFile(fileEntry.filePath, "utf8");
|
|
701
|
+
if (!isGeneratedContent(content)) {
|
|
702
|
+
currentIndex.updateSource(fileEntry.filePath, content);
|
|
703
|
+
cacheSourceShard(fileEntry.filePath, content);
|
|
704
|
+
reconciledCount++;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
} catch {
|
|
709
|
+
// If file cannot be read or no longer exists, remove from index
|
|
710
|
+
currentIndex.removeSource(fileEntry.filePath);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
self.postMessage(createSuccessResponse(msg.id, { reconciledCount }));
|
|
715
|
+
break;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
case "scan": {
|
|
719
|
+
const targetPath = msg.payload?.targetPath;
|
|
720
|
+
const scanOptions = msg.payload?.options;
|
|
721
|
+
let clones: IClone[] = [];
|
|
722
|
+
|
|
723
|
+
if (scanOptions || (targetPath && targetPath !== currentRootDir)) {
|
|
724
|
+
try {
|
|
725
|
+
const pathToScan = targetPath || currentRootDir;
|
|
726
|
+
const isGit = await isInsideGitWorkTree(pathToScan);
|
|
727
|
+
const filesToScan: string[] = [];
|
|
728
|
+
const optionsToUse = scanOptions || currentOptions;
|
|
729
|
+
const indexToUse = new SourceAwareCloneIndex({
|
|
730
|
+
minTokens: optionsToUse?.minTokens,
|
|
731
|
+
minLines: optionsToUse?.minLines,
|
|
732
|
+
maxLines: optionsToUse?.maxLines,
|
|
733
|
+
formatsExts: optionsToUse?.formatsExts,
|
|
734
|
+
});
|
|
735
|
+
|
|
736
|
+
if (isGit) {
|
|
737
|
+
const gitFiles = await getTrackedGitFiles(pathToScan, {
|
|
738
|
+
userIgnorePatterns: optionsToUse?.ignorePatterns,
|
|
739
|
+
ignoreTests: optionsToUse?.ignoreTests,
|
|
740
|
+
customTestPatterns: optionsToUse?.customTestPatterns,
|
|
741
|
+
excludeTestPatterns: optionsToUse?.excludeTestPatterns,
|
|
742
|
+
});
|
|
743
|
+
filesToScan.push(...gitFiles);
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
for (const file of filesToScan) {
|
|
747
|
+
try {
|
|
748
|
+
if (!getSupportedCodeFormat(file, optionsToUse?.formatsExts))
|
|
749
|
+
continue;
|
|
750
|
+
const stat = await fs.stat(file);
|
|
751
|
+
if (stat.size <= MAX_FILE_SIZE_BYTES && stat.size > 0) {
|
|
752
|
+
const content = await fs.readFile(file, "utf8");
|
|
753
|
+
if (!isGeneratedContent(content)) {
|
|
754
|
+
indexToUse.addSource(file, content);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
} catch {
|
|
758
|
+
// Ignore individual file read errors
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
clones = indexToUse.getClones();
|
|
763
|
+
} catch {
|
|
764
|
+
clones = currentIndex.getClones();
|
|
765
|
+
}
|
|
766
|
+
} else {
|
|
767
|
+
clones = currentIndex.getClones();
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
self.postMessage(createSuccessResponse(msg.id, clones));
|
|
771
|
+
break;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
case "close": {
|
|
775
|
+
if (activeAbortController) {
|
|
776
|
+
activeAbortController.abort();
|
|
777
|
+
activeAbortController = null;
|
|
778
|
+
}
|
|
779
|
+
currentIndex.reset();
|
|
780
|
+
if (currentDiskCache) {
|
|
781
|
+
currentDiskCache.close();
|
|
782
|
+
currentDiskCache = null;
|
|
783
|
+
}
|
|
784
|
+
watchedRevisions.clear();
|
|
785
|
+
isBaselineIndexing = false;
|
|
786
|
+
isBaselineComplete = false;
|
|
787
|
+
|
|
788
|
+
self.postMessage(createSuccessResponse(msg.id, { closed: true }));
|
|
789
|
+
self.postMessage(createStatusEvent("closed"));
|
|
790
|
+
break;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
default: {
|
|
794
|
+
const unknownMsg = msg as Record<string, unknown>;
|
|
795
|
+
const reqId =
|
|
796
|
+
typeof unknownMsg.id === "string" ? unknownMsg.id : "unknown";
|
|
797
|
+
const reqType =
|
|
798
|
+
typeof unknownMsg.type === "string" ? unknownMsg.type : "unknown";
|
|
799
|
+
self.postMessage(
|
|
800
|
+
createErrorResponse(reqId, `Unsupported request type: ${reqType}`),
|
|
801
|
+
);
|
|
802
|
+
break;
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
// ============================================================================
|
|
808
|
+
// Worker Listener
|
|
809
|
+
// ============================================================================
|
|
810
|
+
|
|
811
|
+
self.onmessage = async (event: MessageEvent<unknown>) => {
|
|
812
|
+
const rawData = event.data;
|
|
813
|
+
|
|
814
|
+
if (!isWorkerRequest(rawData)) {
|
|
815
|
+
return;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
try {
|
|
819
|
+
await handleWorkerRequest(rawData);
|
|
820
|
+
} catch (err) {
|
|
821
|
+
self.postMessage(
|
|
822
|
+
createErrorResponse(
|
|
823
|
+
rawData.id,
|
|
824
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
825
|
+
),
|
|
826
|
+
);
|
|
827
|
+
}
|
|
828
|
+
};
|