omp-plugin-duplicate-detector 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -17
- package/dist/detector-worker.js +418 -272
- package/package.json +1 -1
- package/src/coordinator.ts +16 -4
- package/src/detector-worker.ts +63 -24
- package/src/disk-cache.ts +365 -314
- package/src/index.ts +134 -7
- package/src/project-state.ts +10 -4
- package/src/repo-context.ts +158 -0
- package/src/source-aware-index.ts +14 -1
package/package.json
CHANGED
package/src/coordinator.ts
CHANGED
|
@@ -158,6 +158,9 @@ export class DuplicateDetectorCoordinator extends EventEmitter<CoordinatorEvents
|
|
|
158
158
|
|
|
159
159
|
try {
|
|
160
160
|
const worker = new Worker(this.#workerUrl);
|
|
161
|
+
if (typeof worker.unref === "function") {
|
|
162
|
+
worker.unref();
|
|
163
|
+
}
|
|
161
164
|
|
|
162
165
|
worker.onmessage = (event: MessageEvent<unknown>) => {
|
|
163
166
|
this.#handleWorkerMessage(event.data);
|
|
@@ -264,6 +267,9 @@ export class DuplicateDetectorCoordinator extends EventEmitter<CoordinatorEvents
|
|
|
264
267
|
).catch(() => {});
|
|
265
268
|
}
|
|
266
269
|
}, delay);
|
|
270
|
+
if (typeof this.#restartTimer.unref === "function") {
|
|
271
|
+
this.#restartTimer.unref();
|
|
272
|
+
}
|
|
267
273
|
}
|
|
268
274
|
}
|
|
269
275
|
|
|
@@ -300,7 +306,9 @@ export class DuplicateDetectorCoordinator extends EventEmitter<CoordinatorEvents
|
|
|
300
306
|
),
|
|
301
307
|
);
|
|
302
308
|
}, this.#requestTimeoutMs);
|
|
303
|
-
|
|
309
|
+
if (typeof timeoutTimer.unref === "function") {
|
|
310
|
+
timeoutTimer.unref();
|
|
311
|
+
}
|
|
304
312
|
this.#pendingRequests.set(id, {
|
|
305
313
|
resolve: resolve as (value: unknown) => void,
|
|
306
314
|
reject,
|
|
@@ -465,15 +473,19 @@ export class DuplicateDetectorCoordinator extends EventEmitter<CoordinatorEvents
|
|
|
465
473
|
/**
|
|
466
474
|
* Reconcile multiple modified or removed files with the index.
|
|
467
475
|
*/
|
|
468
|
-
async reconcile(
|
|
476
|
+
async reconcile(
|
|
477
|
+
files: ReconcileFileEntry[],
|
|
478
|
+
): Promise<{ reconciledCount: number }> {
|
|
469
479
|
try {
|
|
470
480
|
const payload: ReconcilePayload = { files };
|
|
471
|
-
await this.#sendRequest<{ reconciledCount: number }>(
|
|
481
|
+
return await this.#sendRequest<{ reconciledCount: number }>(
|
|
472
482
|
"reconcile",
|
|
473
483
|
payload,
|
|
474
484
|
);
|
|
475
485
|
} catch (err) {
|
|
476
|
-
|
|
486
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
487
|
+
this.emit("error", error);
|
|
488
|
+
throw error;
|
|
477
489
|
}
|
|
478
490
|
}
|
|
479
491
|
|
package/src/detector-worker.ts
CHANGED
|
@@ -8,7 +8,7 @@ import * as crypto from "node:crypto";
|
|
|
8
8
|
import * as fs from "node:fs/promises";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import type { IClone } from "@jscpd/core";
|
|
11
|
-
import { DiskCacheManager } from "./disk-cache";
|
|
11
|
+
import { cleanupLegacyCacheFiles, DiskCacheManager } from "./disk-cache";
|
|
12
12
|
import {
|
|
13
13
|
type BaselineStatus,
|
|
14
14
|
createIgnoreFilter,
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
MAX_INDEXED_FILES,
|
|
23
23
|
MAX_TOTAL_SOURCE_BYTES,
|
|
24
24
|
} from "./jscpd-engine";
|
|
25
|
+
import { canonicalizePath, resolveRepositoryContext } from "./repo-context";
|
|
25
26
|
import {
|
|
26
27
|
type SerializedSourceShard,
|
|
27
28
|
SourceAwareCloneIndex,
|
|
@@ -87,24 +88,37 @@ function areOptionsEqual(a?: WorkspaceOptions, b?: WorkspaceOptions): boolean {
|
|
|
87
88
|
const bFormats = b.formatsExts ? JSON.stringify(b.formatsExts) : "";
|
|
88
89
|
return aFormats === bFormats;
|
|
89
90
|
}
|
|
90
|
-
|
|
91
91
|
function notifyLateFindings(clones: IClone[]): void {
|
|
92
92
|
for (const clone of clones) {
|
|
93
93
|
const srcA = clone.duplicationA.sourceId;
|
|
94
94
|
const srcB = clone.duplicationB.sourceId;
|
|
95
95
|
const resA = path.resolve(srcA);
|
|
96
96
|
const resB = path.resolve(srcB);
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
97
|
+
const canA = canonicalizePath(srcA);
|
|
98
|
+
const canB = canonicalizePath(srcB);
|
|
99
|
+
|
|
100
|
+
const isWatchedA =
|
|
101
|
+
watchedRevisions.has(srcA) ||
|
|
102
|
+
watchedRevisions.has(resA) ||
|
|
103
|
+
watchedRevisions.has(canA);
|
|
104
|
+
const isWatchedB =
|
|
105
|
+
watchedRevisions.has(srcB) ||
|
|
106
|
+
watchedRevisions.has(resB) ||
|
|
107
|
+
watchedRevisions.has(canB);
|
|
100
108
|
|
|
101
109
|
if (isWatchedA || isWatchedB) {
|
|
102
110
|
if (isWatchedA) {
|
|
103
|
-
const entry =
|
|
111
|
+
const entry =
|
|
112
|
+
watchedRevisions.get(srcA) ??
|
|
113
|
+
watchedRevisions.get(resA) ??
|
|
114
|
+
watchedRevisions.get(canA);
|
|
104
115
|
if (entry) entry.lastKnownCloneCount++;
|
|
105
116
|
}
|
|
106
117
|
if (isWatchedB) {
|
|
107
|
-
const entry =
|
|
118
|
+
const entry =
|
|
119
|
+
watchedRevisions.get(srcB) ??
|
|
120
|
+
watchedRevisions.get(resB) ??
|
|
121
|
+
watchedRevisions.get(canB);
|
|
108
122
|
if (entry) entry.lastKnownCloneCount++;
|
|
109
123
|
}
|
|
110
124
|
self.postMessage(createLateFindingEvent(clone));
|
|
@@ -307,6 +321,10 @@ async function runBaselineIndexing(
|
|
|
307
321
|
if (signal.aborted) return { indexedCount, status: "cancelled" };
|
|
308
322
|
|
|
309
323
|
// Ingest items into index and notify late findings
|
|
324
|
+
const shardsToSave: Array<{
|
|
325
|
+
shard: SerializedSourceShard;
|
|
326
|
+
relPath: string;
|
|
327
|
+
}> = [];
|
|
310
328
|
for (const item of fileItems) {
|
|
311
329
|
if (!item) continue;
|
|
312
330
|
|
|
@@ -323,9 +341,10 @@ async function runBaselineIndexing(
|
|
|
323
341
|
newClones = currentIndex.hydrateSourceShard(item.cachedShard);
|
|
324
342
|
if (item.isNewlyTokenized) {
|
|
325
343
|
if (currentDiskCache && item.contentHash && item.relPath) {
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
.
|
|
344
|
+
shardsToSave.push({
|
|
345
|
+
shard: item.cachedShard,
|
|
346
|
+
relPath: item.relPath,
|
|
347
|
+
});
|
|
329
348
|
}
|
|
330
349
|
} else {
|
|
331
350
|
cachedCount++;
|
|
@@ -338,7 +357,7 @@ async function runBaselineIndexing(
|
|
|
338
357
|
item.contentHash,
|
|
339
358
|
);
|
|
340
359
|
if (shard) {
|
|
341
|
-
|
|
360
|
+
shardsToSave.push({ shard, relPath: item.relPath });
|
|
342
361
|
}
|
|
343
362
|
}
|
|
344
363
|
}
|
|
@@ -350,6 +369,10 @@ async function runBaselineIndexing(
|
|
|
350
369
|
}
|
|
351
370
|
}
|
|
352
371
|
|
|
372
|
+
if (shardsToSave.length > 0 && currentDiskCache) {
|
|
373
|
+
await currentDiskCache.saveShards(shardsToSave);
|
|
374
|
+
}
|
|
375
|
+
|
|
353
376
|
const processedCount = Math.min(i + batch.length, totalFiles);
|
|
354
377
|
const percentage =
|
|
355
378
|
totalFiles > 0 ? Math.round((processedCount / totalFiles) * 100) : 100;
|
|
@@ -543,11 +566,12 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
543
566
|
switch (msg.type) {
|
|
544
567
|
case "openWorkspace": {
|
|
545
568
|
const { rootDir, options } = msg.payload;
|
|
569
|
+
const canonicalRootDir = canonicalizePath(rootDir);
|
|
546
570
|
|
|
547
|
-
// If
|
|
571
|
+
// If canonicalRootDir === currentRootDir and options match and baseline is already complete or indexing,
|
|
548
572
|
// avoid total reset; trigger an incremental git status check instead.
|
|
549
573
|
if (
|
|
550
|
-
currentRootDir === rootDir &&
|
|
574
|
+
(currentRootDir === canonicalRootDir || currentRootDir === rootDir) &&
|
|
551
575
|
areOptionsEqual(currentOptions, options) &&
|
|
552
576
|
(isBaselineComplete || isBaselineIndexing)
|
|
553
577
|
) {
|
|
@@ -557,14 +581,14 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
557
581
|
}
|
|
558
582
|
activeAbortController = new AbortController();
|
|
559
583
|
const recResult = await runIncrementalGitReconciliation(
|
|
560
|
-
|
|
584
|
+
currentRootDir,
|
|
561
585
|
options,
|
|
562
586
|
activeAbortController.signal,
|
|
563
587
|
);
|
|
564
588
|
self.postMessage(
|
|
565
589
|
createSuccessResponse(msg.id, {
|
|
566
590
|
started: true,
|
|
567
|
-
rootDir,
|
|
591
|
+
rootDir: currentRootDir,
|
|
568
592
|
reused: true,
|
|
569
593
|
indexedCount: recResult.indexedCount,
|
|
570
594
|
status: recResult.status,
|
|
@@ -574,7 +598,7 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
574
598
|
self.postMessage(
|
|
575
599
|
createSuccessResponse(msg.id, {
|
|
576
600
|
started: true,
|
|
577
|
-
rootDir,
|
|
601
|
+
rootDir: currentRootDir,
|
|
578
602
|
reused: true,
|
|
579
603
|
indexedCount: currentIndex.stats().sourceCount,
|
|
580
604
|
status: "complete" as BaselineStatus,
|
|
@@ -594,11 +618,17 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
594
618
|
currentDiskCache = null;
|
|
595
619
|
}
|
|
596
620
|
|
|
597
|
-
|
|
621
|
+
const repoContext = await resolveRepositoryContext(
|
|
622
|
+
rootDir,
|
|
623
|
+
activeAbortController.signal,
|
|
624
|
+
);
|
|
625
|
+
const effectiveRoot = repoContext.workspaceRoot;
|
|
626
|
+
currentRootDir = effectiveRoot;
|
|
598
627
|
currentOptions = options;
|
|
599
628
|
currentIndex = new SourceAwareCloneIndex(options);
|
|
600
629
|
currentDiskCache = new DiskCacheManager({
|
|
601
|
-
rootDir,
|
|
630
|
+
rootDir: effectiveRoot,
|
|
631
|
+
repositoryKey: repoContext.repositoryKey,
|
|
602
632
|
cacheDir: options?.cacheDir,
|
|
603
633
|
config: options,
|
|
604
634
|
maxBytes: options?.maxCacheBytes,
|
|
@@ -607,18 +637,23 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
607
637
|
isBaselineIndexing = true;
|
|
608
638
|
isBaselineComplete = false;
|
|
609
639
|
|
|
640
|
+
// Clean up legacy pre-v4 caches in background
|
|
641
|
+
cleanupLegacyCacheFiles(options?.cacheDir).catch(() => {});
|
|
642
|
+
|
|
610
643
|
self.postMessage(
|
|
611
644
|
createSuccessResponse(msg.id, {
|
|
612
645
|
started: true,
|
|
613
|
-
rootDir,
|
|
646
|
+
rootDir: effectiveRoot,
|
|
614
647
|
reused: false,
|
|
615
648
|
}),
|
|
616
649
|
);
|
|
617
650
|
|
|
618
651
|
// Run background indexing task (posts complete event when finished)
|
|
619
|
-
runBaselineIndexing(
|
|
620
|
-
|
|
621
|
-
|
|
652
|
+
runBaselineIndexing(
|
|
653
|
+
effectiveRoot,
|
|
654
|
+
options,
|
|
655
|
+
activeAbortController.signal,
|
|
656
|
+
).catch(() => {});
|
|
622
657
|
break;
|
|
623
658
|
}
|
|
624
659
|
|
|
@@ -632,6 +667,7 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
632
667
|
case "checkAndUpdate": {
|
|
633
668
|
const { filePath, content, format, revision = 1 } = msg.payload;
|
|
634
669
|
const clones = currentIndex.updateSource(filePath, content, format);
|
|
670
|
+
const canonicalFilePath = canonicalizePath(filePath);
|
|
635
671
|
const resolvedPath = path.resolve(filePath);
|
|
636
672
|
|
|
637
673
|
cacheSourceShard(filePath, content);
|
|
@@ -641,7 +677,9 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
641
677
|
c.duplicationA.sourceId === filePath ||
|
|
642
678
|
c.duplicationB.sourceId === filePath ||
|
|
643
679
|
path.resolve(c.duplicationA.sourceId) === resolvedPath ||
|
|
644
|
-
path.resolve(c.duplicationB.sourceId) === resolvedPath
|
|
680
|
+
path.resolve(c.duplicationB.sourceId) === resolvedPath ||
|
|
681
|
+
canonicalizePath(c.duplicationA.sourceId) === canonicalFilePath ||
|
|
682
|
+
canonicalizePath(c.duplicationB.sourceId) === canonicalFilePath,
|
|
645
683
|
);
|
|
646
684
|
|
|
647
685
|
const watchEntry = {
|
|
@@ -650,10 +688,11 @@ async function handleWorkerRequest(msg: WorkerRequestMessage): Promise<void> {
|
|
|
650
688
|
};
|
|
651
689
|
watchedRevisions.set(filePath, watchEntry);
|
|
652
690
|
watchedRevisions.set(resolvedPath, watchEntry);
|
|
691
|
+
watchedRevisions.set(canonicalFilePath, watchEntry);
|
|
653
692
|
|
|
654
693
|
self.postMessage(
|
|
655
694
|
createSuccessResponse(msg.id, {
|
|
656
|
-
clones,
|
|
695
|
+
clones: fileClones,
|
|
657
696
|
isComplete: isBaselineComplete,
|
|
658
697
|
}),
|
|
659
698
|
);
|