omp-plugin-duplicate-detector 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,828 @@
1
+ /**
2
+ * Worker thread entry point for duplicate detection.
3
+ * Runs in Bun.Worker, managing background baseline indexing and answering RPC requests
4
+ * from the main thread via SourceAwareCloneIndex.
5
+ */
6
+
7
+ import * as crypto from "node:crypto";
8
+ import * as fs from "node:fs/promises";
9
+ import * as path from "node:path";
10
+ import type { IClone } from "@jscpd/core";
11
+ import { DiskCacheManager } from "./disk-cache";
12
+ import {
13
+ type BaselineStatus,
14
+ createIgnoreFilter,
15
+ execGit,
16
+ getSupportedCodeFormat,
17
+ getTrackedGitFiles,
18
+ isGeneratedContent,
19
+ isInsideGitWorkTree,
20
+ MAX_FILE_SIZE_BYTES,
21
+ MAX_GIT_PATHS,
22
+ MAX_INDEXED_FILES,
23
+ MAX_TOTAL_SOURCE_BYTES,
24
+ } from "./jscpd-engine";
25
+ import {
26
+ type SerializedSourceShard,
27
+ SourceAwareCloneIndex,
28
+ } from "./source-aware-index";
29
+ import {
30
+ createCompleteEvent,
31
+ createErrorResponse,
32
+ createLateFindingEvent,
33
+ createProgressEvent,
34
+ createStatusEvent,
35
+ createSuccessResponse,
36
+ isWorkerRequest,
37
+ type WorkerRequestMessage,
38
+ type WorkspaceOptions,
39
+ } from "./worker-protocol";
40
+
41
+ // Type declaration for Worker global scope
42
+ declare const self: {
43
+ onmessage: ((event: MessageEvent<unknown>) => void) | null;
44
+ postMessage: (message: unknown) => void;
45
+ close?: () => void;
46
+ };
47
+
48
+ // ============================================================================
49
+ // Worker State
50
+ // ============================================================================
51
+
52
+ let currentIndex: SourceAwareCloneIndex = new SourceAwareCloneIndex();
53
+ let currentDiskCache: DiskCacheManager | null = null;
54
+ let currentRootDir = "";
55
+ let currentOptions: WorkspaceOptions | undefined;
56
+ let activeAbortController: AbortController | null = null;
57
+ let isBaselineIndexing = false;
58
+ let isBaselineComplete = false;
59
+ const watchedRevisions = new Map<
60
+ string,
61
+ { revision: number; lastKnownCloneCount: number }
62
+ >();
63
+
64
+ const BATCH_SIZE = 32;
65
+
66
+ function areOptionsEqual(a?: WorkspaceOptions, b?: WorkspaceOptions): boolean {
67
+ if (a === b) return true;
68
+ if (!a && !b) return true;
69
+ if (!a || !b) return false;
70
+ if (a.minTokens !== b.minTokens) return false;
71
+ if (a.minLines !== b.minLines) return false;
72
+ if (a.maxLines !== b.maxLines) return false;
73
+ if (a.maxIndexedFiles !== b.maxIndexedFiles) return false;
74
+ if (a.ignoreTests !== b.ignoreTests) return false;
75
+ const aCustom = (a.customTestPatterns ?? []).slice().sort().join(",");
76
+ const bCustom = (b.customTestPatterns ?? []).slice().sort().join(",");
77
+ if (aCustom !== bCustom) return false;
78
+ const aExclude = (a.excludeTestPatterns ?? []).slice().sort().join(",");
79
+ const bExclude = (b.excludeTestPatterns ?? []).slice().sort().join(",");
80
+ if (aExclude !== bExclude) return false;
81
+ const aIgnores = (a.ignorePatterns ?? []).slice().sort().join(",");
82
+ const bIgnores = (b.ignorePatterns ?? []).slice().sort().join(",");
83
+ if (aIgnores !== bIgnores) return false;
84
+ if (a.cacheDir !== b.cacheDir) return false;
85
+ if (a.maxCacheBytes !== b.maxCacheBytes) return false;
86
+ const aFormats = a.formatsExts ? JSON.stringify(a.formatsExts) : "";
87
+ const bFormats = b.formatsExts ? JSON.stringify(b.formatsExts) : "";
88
+ return aFormats === bFormats;
89
+ }
90
+
91
+ function notifyLateFindings(clones: IClone[]): void {
92
+ for (const clone of clones) {
93
+ const srcA = clone.duplicationA.sourceId;
94
+ const srcB = clone.duplicationB.sourceId;
95
+ const resA = path.resolve(srcA);
96
+ const resB = path.resolve(srcB);
97
+
98
+ const isWatchedA = watchedRevisions.has(srcA) || watchedRevisions.has(resA);
99
+ const isWatchedB = watchedRevisions.has(srcB) || watchedRevisions.has(resB);
100
+
101
+ if (isWatchedA || isWatchedB) {
102
+ if (isWatchedA) {
103
+ const entry = watchedRevisions.get(srcA) ?? watchedRevisions.get(resA);
104
+ if (entry) entry.lastKnownCloneCount++;
105
+ }
106
+ if (isWatchedB) {
107
+ const entry = watchedRevisions.get(srcB) ?? watchedRevisions.get(resB);
108
+ if (entry) entry.lastKnownCloneCount++;
109
+ }
110
+ self.postMessage(createLateFindingEvent(clone));
111
+ }
112
+ }
113
+ }
114
+
115
+ function cacheSourceShard(filePath: string, content: string): void {
116
+ if (!currentDiskCache) return;
117
+ const contentHash = crypto.createHash("sha256").update(content).digest("hex");
118
+ const relPath = path.relative(currentRootDir, filePath).replace(/\\/g, "/");
119
+ const shard = currentIndex.exportSourceShard(filePath, contentHash);
120
+ if (shard) {
121
+ currentDiskCache.saveShard(shard, relPath).catch(() => {});
122
+ }
123
+ }
124
+
125
+ // ============================================================================
126
+ // Baseline Indexing Worker Task
127
+ // ============================================================================
128
+
129
+ function yieldTask(): Promise<void> {
130
+ const { promise, resolve } = Promise.withResolvers<void>();
131
+ setTimeout(resolve, 0);
132
+ return promise;
133
+ }
134
+
135
+ interface BatchItem {
136
+ filePath: string;
137
+ content: string | null;
138
+ contentHash: string | null;
139
+ relPath: string | null;
140
+ size: number;
141
+ cachedShard: SerializedSourceShard | null;
142
+ isNewlyTokenized: boolean;
143
+ alreadyIndexed: boolean;
144
+ }
145
+
146
+ async function runBaselineIndexing(
147
+ rootDir: string,
148
+ options: WorkspaceOptions | undefined,
149
+ signal: AbortSignal,
150
+ ): Promise<{ indexedCount: number; status: BaselineStatus }> {
151
+ const startTime = Date.now();
152
+ let indexedCount = 0;
153
+ let cachedCount = 0;
154
+ let totalSourceBytes = 0;
155
+
156
+ try {
157
+ const isGit = await isInsideGitWorkTree(rootDir, signal);
158
+ if (signal.aborted) return { indexedCount: 0, status: "cancelled" };
159
+
160
+ if (!isGit) {
161
+ isBaselineIndexing = false;
162
+ isBaselineComplete = true;
163
+ self.postMessage(
164
+ createStatusEvent("idle", "Workspace is not inside a Git repository"),
165
+ );
166
+ self.postMessage(
167
+ createCompleteEvent({
168
+ indexedCount: 0,
169
+ cachedCount: 0,
170
+ totalSourceBytes: 0,
171
+ cloneCount: 0,
172
+ durationMs: Date.now() - startTime,
173
+ status: "skipped_not_git",
174
+ }),
175
+ );
176
+ return { indexedCount: 0, status: "skipped_not_git" };
177
+ }
178
+
179
+ self.postMessage(
180
+ createStatusEvent("indexing", "Enumerating tracked Git files..."),
181
+ );
182
+
183
+ const maxIndexedFiles = options?.maxIndexedFiles ?? MAX_INDEXED_FILES;
184
+ const trackedFiles = await getTrackedGitFiles(rootDir, {
185
+ userIgnorePatterns: options?.ignorePatterns,
186
+ ignoreTests: options?.ignoreTests,
187
+ customTestPatterns: options?.customTestPatterns,
188
+ excludeTestPatterns: options?.excludeTestPatterns,
189
+ signal,
190
+ maxPaths: Math.max(MAX_GIT_PATHS, maxIndexedFiles),
191
+ });
192
+
193
+ if (signal.aborted) return { indexedCount: 0, status: "cancelled" };
194
+
195
+ const totalFiles = trackedFiles.length;
196
+ self.postMessage(
197
+ createProgressEvent({
198
+ phase: "indexing",
199
+ indexedCount: 0,
200
+ totalFiles,
201
+ percentage: 0,
202
+ }),
203
+ );
204
+
205
+ let baselineStatus:
206
+ | "complete"
207
+ | "capped_file_count"
208
+ | "capped_source_bytes" = "complete";
209
+
210
+ for (let i = 0; i < trackedFiles.length; i += BATCH_SIZE) {
211
+ if (signal.aborted) return { indexedCount, status: "cancelled" };
212
+
213
+ if (indexedCount >= maxIndexedFiles) {
214
+ baselineStatus = "capped_file_count";
215
+ self.postMessage(
216
+ createStatusEvent("ready", "Indexed file limit reached"),
217
+ );
218
+ break;
219
+ }
220
+ if (totalSourceBytes >= MAX_TOTAL_SOURCE_BYTES) {
221
+ baselineStatus = "capped_source_bytes";
222
+ self.postMessage(
223
+ createStatusEvent("ready", "Source byte limit reached"),
224
+ );
225
+ break;
226
+ }
227
+
228
+ const batch = trackedFiles.slice(i, i + BATCH_SIZE);
229
+
230
+ // Process file batch I/O and cache lookups concurrently
231
+ const fileItems = await Promise.all(
232
+ batch.map(async (filePath): Promise<BatchItem | null> => {
233
+ if (signal.aborted) return null;
234
+
235
+ // Early skip if file format is unsupported by code duplicate detector
236
+ if (!getSupportedCodeFormat(filePath, options?.formatsExts)) {
237
+ return null;
238
+ }
239
+
240
+ try {
241
+ const stat = await fs.stat(filePath);
242
+ if (stat.size > MAX_FILE_SIZE_BYTES || stat.size <= 0) {
243
+ return null;
244
+ }
245
+
246
+ const resolved = path.resolve(filePath);
247
+ if (
248
+ currentIndex.hasSource(filePath) ||
249
+ currentIndex.hasSource(resolved)
250
+ ) {
251
+ return {
252
+ filePath,
253
+ content: null,
254
+ contentHash: null,
255
+ relPath: null,
256
+ size: stat.size,
257
+ cachedShard: null,
258
+ isNewlyTokenized: false,
259
+ alreadyIndexed: true,
260
+ };
261
+ }
262
+
263
+ const content = await fs.readFile(filePath, "utf8");
264
+ if (signal.aborted || isGeneratedContent(content)) {
265
+ return null;
266
+ }
267
+
268
+ const contentHash = crypto
269
+ .createHash("sha256")
270
+ .update(content)
271
+ .digest("hex");
272
+ const relPath = path
273
+ .relative(rootDir, filePath)
274
+ .replace(/\\/g, "/");
275
+
276
+ const cachedShard = currentDiskCache
277
+ ? await currentDiskCache.getShard(relPath, contentHash)
278
+ : null;
279
+
280
+ let isNewlyTokenized = false;
281
+ let shard = cachedShard;
282
+ if (!shard) {
283
+ shard = currentIndex.tokenizeSource(
284
+ filePath,
285
+ content,
286
+ contentHash,
287
+ );
288
+ isNewlyTokenized = true;
289
+ }
290
+
291
+ return {
292
+ filePath,
293
+ content: shard ? null : content,
294
+ contentHash,
295
+ relPath,
296
+ size: stat.size,
297
+ cachedShard: shard,
298
+ isNewlyTokenized,
299
+ alreadyIndexed: false,
300
+ };
301
+ } catch {
302
+ return null;
303
+ }
304
+ }),
305
+ );
306
+
307
+ if (signal.aborted) return { indexedCount, status: "cancelled" };
308
+
309
+ // Ingest items into index and notify late findings
310
+ for (const item of fileItems) {
311
+ if (!item) continue;
312
+
313
+ if (item.alreadyIndexed) {
314
+ indexedCount++;
315
+ totalSourceBytes += item.size;
316
+ continue;
317
+ }
318
+
319
+ let newClones: IClone[] = [];
320
+
321
+ if (item.cachedShard) {
322
+ item.cachedShard.sourceId = item.filePath;
323
+ newClones = currentIndex.hydrateSourceShard(item.cachedShard);
324
+ if (item.isNewlyTokenized) {
325
+ if (currentDiskCache && item.contentHash && item.relPath) {
326
+ currentDiskCache
327
+ .saveShard(item.cachedShard, item.relPath)
328
+ .catch(() => {});
329
+ }
330
+ } else {
331
+ cachedCount++;
332
+ }
333
+ } else if (item.content) {
334
+ newClones = currentIndex.addSource(item.filePath, item.content);
335
+ if (currentDiskCache && item.contentHash && item.relPath) {
336
+ const shard = currentIndex.exportSourceShard(
337
+ item.filePath,
338
+ item.contentHash,
339
+ );
340
+ if (shard) {
341
+ currentDiskCache.saveShard(shard, item.relPath).catch(() => {});
342
+ }
343
+ }
344
+ }
345
+ indexedCount++;
346
+ totalSourceBytes += item.size;
347
+
348
+ if (newClones.length > 0) {
349
+ notifyLateFindings(newClones);
350
+ }
351
+ }
352
+
353
+ const processedCount = Math.min(i + batch.length, totalFiles);
354
+ const percentage =
355
+ totalFiles > 0 ? Math.round((processedCount / totalFiles) * 100) : 100;
356
+ self.postMessage(
357
+ createProgressEvent({
358
+ phase: "indexing",
359
+ indexedCount,
360
+ totalFiles,
361
+ currentFile: batch[batch.length - 1],
362
+ percentage,
363
+ }),
364
+ );
365
+
366
+ // Yield macrotask once per batch to process pending RPC requests
367
+ await yieldTask();
368
+ }
369
+
370
+ if (!signal.aborted) {
371
+ isBaselineIndexing = false;
372
+ isBaselineComplete = true;
373
+ currentDiskCache?.prune().catch(() => {});
374
+ self.postMessage(
375
+ createCompleteEvent({
376
+ indexedCount,
377
+ cachedCount,
378
+ totalSourceBytes,
379
+ cloneCount: currentIndex.clones.length,
380
+ durationMs: Date.now() - startTime,
381
+ status: baselineStatus,
382
+ }),
383
+ );
384
+ self.postMessage(
385
+ createStatusEvent("ready", "Baseline indexing complete"),
386
+ );
387
+ return { indexedCount, status: baselineStatus };
388
+ }
389
+ return { indexedCount, status: "cancelled" };
390
+ } catch (err) {
391
+ if (signal.aborted) return { indexedCount, status: "cancelled" };
392
+ isBaselineIndexing = false;
393
+ isBaselineComplete = false;
394
+ const error = err instanceof Error ? err.message : String(err);
395
+ self.postMessage(createStatusEvent("error", error));
396
+ self.postMessage(
397
+ createCompleteEvent({
398
+ indexedCount,
399
+ cachedCount,
400
+ totalSourceBytes,
401
+ cloneCount: currentIndex.clones.length,
402
+ durationMs: Date.now() - startTime,
403
+ status: "failed",
404
+ error,
405
+ }),
406
+ );
407
+ return { indexedCount, status: "failed" };
408
+ }
409
+ }
410
+
411
+ async function runIncrementalGitReconciliation(
412
+ rootDir: string,
413
+ options: WorkspaceOptions | undefined,
414
+ signal: AbortSignal,
415
+ ): Promise<{ indexedCount: number; status: BaselineStatus }> {
416
+ try {
417
+ const isGit = await isInsideGitWorkTree(rootDir, signal);
418
+ if (!isGit || signal.aborted) {
419
+ return {
420
+ indexedCount: currentIndex.stats().sourceCount,
421
+ status: !isGit ? "skipped_not_git" : "complete",
422
+ };
423
+ }
424
+ const ignoreFilter = createIgnoreFilter(options?.ignorePatterns, {
425
+ ignoreTests: options?.ignoreTests,
426
+ customTestPatterns: options?.customTestPatterns,
427
+ excludeTestPatterns: options?.excludeTestPatterns,
428
+ });
429
+ const { stdout } = await execGit(
430
+ ["status", "--porcelain", "-z", "--", "."],
431
+ rootDir,
432
+ { signal },
433
+ );
434
+
435
+ if (signal.aborted)
436
+ return {
437
+ indexedCount: currentIndex.stats().sourceCount,
438
+ status: "cancelled",
439
+ };
440
+
441
+ const entries = stdout.split("\0");
442
+ let i = 0;
443
+ while (i < entries.length) {
444
+ if (signal.aborted)
445
+ return {
446
+ indexedCount: currentIndex.stats().sourceCount,
447
+ status: "cancelled",
448
+ };
449
+ const entry = entries[i];
450
+ i++;
451
+ if (!entry || entry.length < 4) continue;
452
+
453
+ const statusCode = entry.slice(0, 2);
454
+ const relPath = entry.slice(3).trim();
455
+ if (!relPath) continue;
456
+
457
+ if (statusCode.includes("R") && i < entries.length) {
458
+ const oldRelPath = entries[i]?.trim();
459
+ i++;
460
+ if (oldRelPath) {
461
+ const oldFullPath = path.resolve(rootDir, oldRelPath);
462
+ currentIndex.removeSource(oldFullPath);
463
+ currentIndex.removeSource(oldRelPath);
464
+ }
465
+ }
466
+
467
+ const fullPath = path.resolve(rootDir, relPath);
468
+
469
+ if (ignoreFilter(relPath)) {
470
+ currentIndex.removeSource(fullPath);
471
+ currentIndex.removeSource(relPath);
472
+ continue;
473
+ }
474
+
475
+ if (statusCode.includes("D")) {
476
+ currentIndex.removeSource(fullPath);
477
+ currentIndex.removeSource(relPath);
478
+ } else {
479
+ try {
480
+ if (!getSupportedCodeFormat(fullPath, options?.formatsExts)) {
481
+ currentIndex.removeSource(fullPath);
482
+ continue;
483
+ }
484
+ const stat = await fs.stat(fullPath);
485
+ if (stat.size > MAX_FILE_SIZE_BYTES || stat.size <= 0) {
486
+ currentIndex.removeSource(fullPath);
487
+ continue;
488
+ }
489
+ const content = await fs.readFile(fullPath, "utf8");
490
+ if (signal.aborted)
491
+ return {
492
+ indexedCount: currentIndex.stats().sourceCount,
493
+ status: "cancelled",
494
+ };
495
+
496
+ if (isGeneratedContent(content)) {
497
+ currentIndex.removeSource(fullPath);
498
+ continue;
499
+ }
500
+
501
+ const newClones = currentIndex.updateSource(fullPath, content);
502
+ cacheSourceShard(fullPath, content);
503
+ if (newClones.length > 0) {
504
+ notifyLateFindings(newClones);
505
+ }
506
+ } catch {
507
+ currentIndex.removeSource(fullPath);
508
+ }
509
+ }
510
+
511
+ await yieldTask();
512
+ }
513
+
514
+ if (!signal.aborted) {
515
+ self.postMessage(
516
+ createCompleteEvent({
517
+ indexedCount: currentIndex.stats().sourceCount,
518
+ totalSourceBytes: 0,
519
+ cloneCount: currentIndex.clones.length,
520
+ durationMs: 0,
521
+ status: "complete",
522
+ }),
523
+ );
524
+ }
525
+ return {
526
+ indexedCount: currentIndex.stats().sourceCount,
527
+ status: "complete",
528
+ };
529
+ } catch {
530
+ // Non-fatal error during incremental reconciliation
531
+ return {
532
+ indexedCount: currentIndex.stats().sourceCount,
533
+ status: "complete",
534
+ };
535
+ }
536
+ }
537
+
538
+ // ============================================================================
539
+ // Message Dispatcher
540
+ // ============================================================================
541
+
542
+ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
543
+ switch (msg.type) {
544
+ case "openWorkspace": {
545
+ const { rootDir, options } = msg.payload;
546
+
547
+ // If rootDir === currentRootDir and options match and baseline is already complete or indexing,
548
+ // avoid total reset; trigger an incremental git status check instead.
549
+ if (
550
+ currentRootDir === rootDir &&
551
+ areOptionsEqual(currentOptions, options) &&
552
+ (isBaselineComplete || isBaselineIndexing)
553
+ ) {
554
+ if (isBaselineComplete) {
555
+ if (activeAbortController) {
556
+ activeAbortController.abort();
557
+ }
558
+ activeAbortController = new AbortController();
559
+ const recResult = await runIncrementalGitReconciliation(
560
+ rootDir,
561
+ options,
562
+ activeAbortController.signal,
563
+ );
564
+ self.postMessage(
565
+ createSuccessResponse(msg.id, {
566
+ started: true,
567
+ rootDir,
568
+ reused: true,
569
+ indexedCount: recResult.indexedCount,
570
+ status: recResult.status,
571
+ }),
572
+ );
573
+ } else {
574
+ self.postMessage(
575
+ createSuccessResponse(msg.id, {
576
+ started: true,
577
+ rootDir,
578
+ reused: true,
579
+ indexedCount: currentIndex.stats().sourceCount,
580
+ status: "complete" as BaselineStatus,
581
+ }),
582
+ );
583
+ }
584
+ break;
585
+ }
586
+
587
+ // Abort any ongoing indexing and close previous workspace cache
588
+ if (activeAbortController) {
589
+ activeAbortController.abort();
590
+ }
591
+ activeAbortController = new AbortController();
592
+ if (currentDiskCache) {
593
+ currentDiskCache.close();
594
+ currentDiskCache = null;
595
+ }
596
+
597
+ currentRootDir = rootDir;
598
+ currentOptions = options;
599
+ currentIndex = new SourceAwareCloneIndex(options);
600
+ currentDiskCache = new DiskCacheManager({
601
+ rootDir,
602
+ cacheDir: options?.cacheDir,
603
+ config: options,
604
+ maxBytes: options?.maxCacheBytes,
605
+ });
606
+ watchedRevisions.clear();
607
+ isBaselineIndexing = true;
608
+ isBaselineComplete = false;
609
+
610
+ self.postMessage(
611
+ createSuccessResponse(msg.id, {
612
+ started: true,
613
+ rootDir,
614
+ reused: false,
615
+ }),
616
+ );
617
+
618
+ // Run background indexing task (posts complete event when finished)
619
+ runBaselineIndexing(rootDir, options, activeAbortController.signal).catch(
620
+ () => {},
621
+ );
622
+ break;
623
+ }
624
+
625
+ case "checkSnippet": {
626
+ const { filePath, content, format } = msg.payload;
627
+ const clones = currentIndex.checkSnippet(filePath, content, format);
628
+ self.postMessage(createSuccessResponse(msg.id, clones));
629
+ break;
630
+ }
631
+
632
+ case "checkAndUpdate": {
633
+ const { filePath, content, format, revision = 1 } = msg.payload;
634
+ const clones = currentIndex.updateSource(filePath, content, format);
635
+ const resolvedPath = path.resolve(filePath);
636
+
637
+ cacheSourceShard(filePath, content);
638
+
639
+ const fileClones = clones.filter(
640
+ (c) =>
641
+ c.duplicationA.sourceId === filePath ||
642
+ c.duplicationB.sourceId === filePath ||
643
+ path.resolve(c.duplicationA.sourceId) === resolvedPath ||
644
+ path.resolve(c.duplicationB.sourceId) === resolvedPath,
645
+ );
646
+
647
+ const watchEntry = {
648
+ revision,
649
+ lastKnownCloneCount: fileClones.length,
650
+ };
651
+ watchedRevisions.set(filePath, watchEntry);
652
+ watchedRevisions.set(resolvedPath, watchEntry);
653
+
654
+ self.postMessage(
655
+ createSuccessResponse(msg.id, {
656
+ clones,
657
+ isComplete: isBaselineComplete,
658
+ }),
659
+ );
660
+ break;
661
+ }
662
+
663
+ case "updateFile": {
664
+ const { filePath, content, format } = msg.payload;
665
+ const clones = currentIndex.updateSource(filePath, content, format);
666
+ cacheSourceShard(filePath, content);
667
+ self.postMessage(createSuccessResponse(msg.id, { clones }));
668
+ break;
669
+ }
670
+
671
+ case "removeFile": {
672
+ const { filePath } = msg.payload;
673
+ currentIndex.removeSource(filePath);
674
+ self.postMessage(createSuccessResponse(msg.id, { removed: true }));
675
+ break;
676
+ }
677
+
678
+ case "reconcile": {
679
+ const { files } = msg.payload;
680
+ let reconciledCount = 0;
681
+
682
+ for (const fileEntry of files) {
683
+ try {
684
+ if (fileEntry.content !== undefined) {
685
+ currentIndex.updateSource(fileEntry.filePath, fileEntry.content);
686
+ cacheSourceShard(fileEntry.filePath, fileEntry.content);
687
+ reconciledCount++;
688
+ } else {
689
+ if (
690
+ !getSupportedCodeFormat(
691
+ fileEntry.filePath,
692
+ currentOptions?.formatsExts,
693
+ )
694
+ ) {
695
+ currentIndex.removeSource(fileEntry.filePath);
696
+ continue;
697
+ }
698
+ const stat = await fs.stat(fileEntry.filePath);
699
+ if (stat.size <= MAX_FILE_SIZE_BYTES && stat.size > 0) {
700
+ const content = await fs.readFile(fileEntry.filePath, "utf8");
701
+ if (!isGeneratedContent(content)) {
702
+ currentIndex.updateSource(fileEntry.filePath, content);
703
+ cacheSourceShard(fileEntry.filePath, content);
704
+ reconciledCount++;
705
+ }
706
+ }
707
+ }
708
+ } catch {
709
+ // If file cannot be read or no longer exists, remove from index
710
+ currentIndex.removeSource(fileEntry.filePath);
711
+ }
712
+ }
713
+
714
+ self.postMessage(createSuccessResponse(msg.id, { reconciledCount }));
715
+ break;
716
+ }
717
+
718
+ case "scan": {
719
+ const targetPath = msg.payload?.targetPath;
720
+ const scanOptions = msg.payload?.options;
721
+ let clones: IClone[] = [];
722
+
723
+ if (scanOptions || (targetPath && targetPath !== currentRootDir)) {
724
+ try {
725
+ const pathToScan = targetPath || currentRootDir;
726
+ const isGit = await isInsideGitWorkTree(pathToScan);
727
+ const filesToScan: string[] = [];
728
+ const optionsToUse = scanOptions || currentOptions;
729
+ const indexToUse = new SourceAwareCloneIndex({
730
+ minTokens: optionsToUse?.minTokens,
731
+ minLines: optionsToUse?.minLines,
732
+ maxLines: optionsToUse?.maxLines,
733
+ formatsExts: optionsToUse?.formatsExts,
734
+ });
735
+
736
+ if (isGit) {
737
+ const gitFiles = await getTrackedGitFiles(pathToScan, {
738
+ userIgnorePatterns: optionsToUse?.ignorePatterns,
739
+ ignoreTests: optionsToUse?.ignoreTests,
740
+ customTestPatterns: optionsToUse?.customTestPatterns,
741
+ excludeTestPatterns: optionsToUse?.excludeTestPatterns,
742
+ });
743
+ filesToScan.push(...gitFiles);
744
+ }
745
+
746
+ for (const file of filesToScan) {
747
+ try {
748
+ if (!getSupportedCodeFormat(file, optionsToUse?.formatsExts))
749
+ continue;
750
+ const stat = await fs.stat(file);
751
+ if (stat.size <= MAX_FILE_SIZE_BYTES && stat.size > 0) {
752
+ const content = await fs.readFile(file, "utf8");
753
+ if (!isGeneratedContent(content)) {
754
+ indexToUse.addSource(file, content);
755
+ }
756
+ }
757
+ } catch {
758
+ // Ignore individual file read errors
759
+ }
760
+ }
761
+
762
+ clones = indexToUse.getClones();
763
+ } catch {
764
+ clones = currentIndex.getClones();
765
+ }
766
+ } else {
767
+ clones = currentIndex.getClones();
768
+ }
769
+
770
+ self.postMessage(createSuccessResponse(msg.id, clones));
771
+ break;
772
+ }
773
+
774
+ case "close": {
775
+ if (activeAbortController) {
776
+ activeAbortController.abort();
777
+ activeAbortController = null;
778
+ }
779
+ currentIndex.reset();
780
+ if (currentDiskCache) {
781
+ currentDiskCache.close();
782
+ currentDiskCache = null;
783
+ }
784
+ watchedRevisions.clear();
785
+ isBaselineIndexing = false;
786
+ isBaselineComplete = false;
787
+
788
+ self.postMessage(createSuccessResponse(msg.id, { closed: true }));
789
+ self.postMessage(createStatusEvent("closed"));
790
+ break;
791
+ }
792
+
793
+ default: {
794
+ const unknownMsg = msg as Record<string, unknown>;
795
+ const reqId =
796
+ typeof unknownMsg.id === "string" ? unknownMsg.id : "unknown";
797
+ const reqType =
798
+ typeof unknownMsg.type === "string" ? unknownMsg.type : "unknown";
799
+ self.postMessage(
800
+ createErrorResponse(reqId, `Unsupported request type: ${reqType}`),
801
+ );
802
+ break;
803
+ }
804
+ }
805
+ }
806
+
807
+ // ============================================================================
808
+ // Worker Listener
809
+ // ============================================================================
810
+
811
+ self.onmessage = async (event: MessageEvent<unknown>) => {
812
+ const rawData = event.data;
813
+
814
+ if (!isWorkerRequest(rawData)) {
815
+ return;
816
+ }
817
+
818
+ try {
819
+ await handleWorkerRequest(rawData);
820
+ } catch (err) {
821
+ self.postMessage(
822
+ createErrorResponse(
823
+ rawData.id,
824
+ err instanceof Error ? err : new Error(String(err)),
825
+ ),
826
+ );
827
+ }
828
+ };