clone-alert 0.3.0

package/dist/core.js

@@ -0,0 +1,442 @@
+"use strict";
+/**
+ * Language-agnostic CPD core. Operates on a flat stream of tokens supplied by
+ * the tokenizers (see tokenizers.ts) as `RawToken[]`.
+ *
+ * Token storage is struct-of-arrays over typed arrays (`Int32Array`): instead of
+ * ~N `TokenEntry` objects we keep parallel numeric columns. Full `TokenEntry`
+ * objects are materialized lazily, only for the marks that land in a match.
+ *
+ * @packageDocumentation
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CpdCore = exports.Match = exports.Mark = exports.TokenEntry = exports.TS_LIT = exports.TS_ID = exports.S = void 0;
+/**
+ * Normalization sentinel prefix, taken from the Unicode private-use area so it
+ * is guaranteed never to collide with real source token images. Framework token
+ * namespaces (Angular/Vue/Svelte) live in their own extension modules
+ * (src/angular.ts, etc.) and are built on top of this shared sentinel.
+ */
+exports.S = '\uE000';
+/** Normalized identifier (TS). */
+exports.TS_ID = `${exports.S}ID`;
+/** Normalized literal (TS). */
+exports.TS_LIT = `${exports.S}LIT`;
+/** A fully materialized token with its image, interned id, and source location. */
+class TokenEntry {
+    image;
+    identifier;
+    index;
+    file;
+    beginLine;
+    beginColumn;
+    endLine;
+    endColumn;
+    constructor(image, identifier, index, file, beginLine, beginColumn, endLine = beginLine, endColumn = beginColumn) {
+        this.image = image;
+        this.identifier = identifier;
+        this.index = index;
+        this.file = file;
+        this.beginLine = beginLine;
+        this.beginColumn = beginColumn;
+        this.endLine = endLine;
+        this.endColumn = endColumn;
+    }
+}
+exports.TokenEntry = TokenEntry;
+/** A single occurrence of a duplicated span, anchored at its starting token. */
+class Mark {
+    token;
+    constructor(token) {
+        this.token = token;
+    }
+}
+exports.Mark = Mark;
+/** A set of marks that share an identical duplicated token span. */
+class Match {
+    tokenCount;
+    /** Dedupe by token index (PMD uses a TreeSet keyed by index, not by reference). */
+    markMap = new Map();
+    /**
+     * Cache of sorted marks. The `marks` getter is hit millions of times in the
+     * hot reportMatch path; without the cache every call did Array.from + sort.
+     * Invalidated only in addMark, i.e. when the mark set actually changes.
+     */
+    marksSorted = null;
+    constructor(tokenCount, first, second) {
+        this.tokenCount = tokenCount;
+        this.markMap.set(first.token.index, first);
+        this.markMap.set(second.token.index, second);
+    }
+    addMark(entry) {
+        if (!this.markMap.has(entry.index)) {
+            this.markMap.set(entry.index, new Mark(entry));
+            this.marksSorted = null;
+        }
+    }
+    get markCount() {
+        return this.markMap.size;
+    }
+    get marks() {
+        if (this.marksSorted === null) {
+            this.marksSorted = Array.from(this.markMap.values()).sort((a, b) => a.token.index - b.token.index);
+        }
+        return this.marksSorted;
+    }
+}
+exports.Match = Match;
+/** The duplicate-detection engine: ingests token streams and reports matches. */
+class CpdCore {
+    minTileSize;
+    // Token columns (struct-of-arrays). Grown geometrically in ensureCapacity().
+    ids = new Int32Array(0); // interned image; 0 == EOF/barrier
+    fileIds = new Int32Array(0);
+    beginLines = new Int32Array(0);
+    beginColumns = new Int32Array(0);
+    endLines = new Int32Array(0);
+    endColumns = new Int32Array(0);
+    size = 0;
+    capacity = 0;
+    // Interning tables: id -> string. idImages[0] == '' (EOF).
+    imageToId = new Map();
+    idImages = [''];
+    fileToId = new Map();
+    fileNames = [];
+    constructor(minTileSize = 50) {
+        this.minTileSize = minTileSize;
+    }
+    intern(image) {
+        let id = this.imageToId.get(image);
+        if (id === undefined) {
+            id = this.idImages.length;
+            this.imageToId.set(image, id);
+            this.idImages.push(image);
+        }
+        return id;
+    }
+    fileId(file) {
+        let id = this.fileToId.get(file);
+        if (id === undefined) {
+            id = this.fileNames.length;
+            this.fileToId.set(file, id);
+            this.fileNames.push(file);
+        }
+        return id;
+    }
+    ensureCapacity(extra) {
+        const need = this.size + extra;
+        if (need <= this.capacity)
+            return;
+        let cap = this.capacity === 0 ? 1024 : this.capacity;
+        while (cap < need)
+            cap *= 2;
+        this.ids = growInt32(this.ids, cap);
+        this.fileIds = growInt32(this.fileIds, cap);
+        this.beginLines = growInt32(this.beginLines, cap);
+        this.beginColumns = growInt32(this.beginColumns, cap);
+        this.endLines = growInt32(this.endLines, cap);
+        this.endColumns = growInt32(this.endColumns, cap);
+        this.capacity = cap;
+    }
+    pushToken(id, fileId, bl, bc, el, ec) {
+        const i = this.size++;
+        this.ids[i] = id;
+        this.fileIds[i] = fileId;
+        this.beginLines[i] = bl;
+        this.beginColumns[i] = bc;
+        this.endLines[i] = el;
+        this.endColumns[i] = ec;
+    }
+    /** Add one file's token stream. An EOF barrier is always appended at the end. */
+    addFile(file, raw) {
+        const fileId = this.fileId(file);
+        this.ensureCapacity(raw.length + 1);
+        for (const r of raw) {
+            if (r.barrier) {
+                this.pushToken(0, fileId, r.line, r.column, r.line, r.column);
+                continue;
+            }
+            this.pushToken(this.intern(r.image), fileId, r.line, r.column, r.endLine ?? r.line, r.endColumn ?? r.column);
+        }
+        this.pushToken(0, fileId, 0, 0, 0, 0); // EOF
+    }
+    get tokenCount() {
+        return this.size;
+    }
+    /** Raw access to the id column for the collector's hot loops (module-internal). */
+    get idColumn() {
+        return this.ids;
+    }
+    /** Materialize a TokenEntry by absolute index. Returns undefined when out of range. */
+    entryAt(index) {
+        if (index < 0 || index >= this.size)
+            return undefined;
+        const id = this.ids[index];
+        return new TokenEntry(this.idImages[id], id, index, this.fileNames[this.fileIds[index]], this.beginLines[index], this.beginColumns[index], this.endLines[index], this.endColumns[index]);
+    }
+    analyze() {
+        if (this.size < this.minTileSize)
+            return [];
+        const { markIndices, markHashes, markCount } = this.hash();
+        if (markCount === 0)
+            return [];
+        // Group by equal hash. This used to be a comparator sort over a boxed
+        // number[] — the most expensive part of the core (O(n log n) with a
+        // megamorphic closure over 3.4M items). Now it is a stable LSD radix sort
+        // by the 32-bit hash on a Uint32Array: O(n) linear passes, no closures, no
+        // boxing. markIndices is strictly decreasing (hash() walks right-to-left),
+        // so the initial permutation by ascending index is a reversal; radix
+        // stability preserves ascending index within an equal hash (required by
+        // MatchCollector.collect).
+        const order = radixSortByHash(markHashes, markCount);
+        const collector = new MatchCollector(this, this.minTileSize);
+        let start = 0;
+        while (start < markCount) {
+            const h = markHashes[order[start]];
+            let end = start + 1;
+            while (end < markCount && markHashes[order[end]] === h)
+                end++;
+            if (end - start > 1) {
+                // The run is already sorted by ascending index (the sort tie-break).
+                const group = new Int32Array(end - start);
+                for (let k = start; k < end; k++)
+                    group[k - start] = markIndices[order[k]];
+                collector.collect(group);
+            }
+            start = end;
+        }
+        const matches = collector.getMatches();
+        // Deterministic report order. Does not affect detection. For a line-by-line
+        // diff against PMD, sort both dumps by (file, line) instead.
+        matches.sort((a, b) => {
+            const byLen = b.tokenCount - a.tokenCount;
+            if (byLen !== 0)
+                return byLen;
+            const byMarks = b.markCount - a.markCount;
+            if (byMarks !== 0)
+                return byMarks;
+            return a.marks[0].token.index - b.marks[0].token.index;
+        });
+        return matches;
+    }
+    // Karp-Rabin sliding window, right-to-left. All arithmetic is 32-bit (| 0 /
+    // Math.imul); float64 would produce hashes different from the Java original.
+    //
+    // Returns parallel columns (token index, its hash) ordered by descending index
+    // — the same mark set the Java original distributed across buckets.
+    hash() {
+        const ids = this.ids;
+        const n = this.size;
+        const MOD = 37;
+        let lastMod = 1;
+        for (let i = 0; i < this.minTileSize; i++) {
+            lastMod = Math.imul(lastMod, MOD);
+        }
+        let lastHash = 0;
+        const markIndices = new Int32Array(n);
+        const markHashes = new Int32Array(n);
+        let m = 0;
+        for (let i = n - 1; i >= 0; i--) {
+            if (ids[i] !== 0) {
+                const aheadIndex = i + this.minTileSize;
+                const last = aheadIndex < n ? ids[aheadIndex] : 0;
+                lastHash = (Math.imul(MOD, lastHash) + ids[i] - Math.imul(lastMod, last)) | 0;
+                markIndices[m] = i;
+                markHashes[m] = lastHash;
+                m++;
+            }
+            else {
+                // EOF/barrier: reset the hash and skip the minTileSize-1 positions
+                // before it (their windows would cross the boundary). The warm-up
+                // advances the OUTER i.
+                lastHash = 0;
+                const end = Math.max(0, i - this.minTileSize + 1);
+                for (; i > end; i--) {
+                    const id = ids[i - 1];
+                    lastHash = (Math.imul(MOD, lastHash) + id) | 0;
+                    if (id === 0)
+                        break;
+                }
+            }
+        }
+        return { markIndices, markHashes, markCount: m };
+    }
+}
+exports.CpdCore = CpdCore;
+function growInt32(src, capacity) {
+    const dst = new Int32Array(capacity);
+    dst.set(src);
+    return dst;
+}
+// Stable LSD radix sort of the permutation [0..count) by key markHashes[pos].
+// Order is by ascending signed hash; on equal hashes stability preserves the
+// order of the starting permutation. We start from positions in descending order
+// (count-1..0): because markHashes/markIndices run by descending token index,
+// this yields ascending index within every equal-hash group. 4 byte passes
+// instead of an O(n log n) comparator over a boxed number[].
+function radixSortByHash(markHashes, count) {
+    // Signed int32 -> monotonic uint32 (flip the top bit) so the byte-wise radix
+    // produces a correct signed order.
+    const keys = new Uint32Array(count);
+    for (let i = 0; i < count; i++)
+        keys[i] = (markHashes[i] ^ 0x80000000) >>> 0;
+    let src = new Uint32Array(count);
+    for (let i = 0; i < count; i++)
+        src[i] = count - 1 - i;
+    let dst = new Uint32Array(count);
+    const counts = new Int32Array(257);
+    for (let shift = 0; shift < 32; shift += 8) {
+        counts.fill(0);
+        for (let i = 0; i < count; i++)
+            counts[((keys[src[i]] >>> shift) & 0xff) + 1]++;
+        for (let b = 0; b < 256; b++)
+            counts[b + 1] += counts[b];
+        for (let i = 0; i < count; i++) {
+            const p = src[i];
+            dst[counts[(keys[p] >>> shift) & 0xff]++] = p;
+        }
+        const tmp = src;
+        src = dst;
+        dst = tmp;
+    }
+    return src;
+}
+// Port of MatchCollector.java with no change to the algorithm (it is correct).
+// Marks are represented by the absolute token index (number); positions and ids
+// are read from the SoA columns.
+class MatchCollector {
+    ma;
+    minTileSize;
+    matchTree = new Map();
+    tokenMatchSets = new Map();
+    ids;
+    tokenCount;
+    constructor(ma, minTileSize) {
+        this.ma = ma;
+        this.minTileSize = minTileSize;
+        this.ids = ma.idColumn;
+        this.tokenCount = ma.tokenCount;
+    }
+    collect(marks) {
+        let skipped = 0;
+        for (let i = 0; i < marks.length - 1; i += skipped + 1) {
+            skipped = 0;
+            const mark1 = marks[i];
+            for (let j = i + 1; j < marks.length; j++) {
+                const mark2 = marks[j];
+                const diff = mark1 - mark2;
+                if (-diff < this.minTileSize) {
+                    skipped++;
+                    continue;
+                }
+                if (this.hasPreviousDupe(mark1, mark2)) {
+                    continue;
+                }
+                const dupes = this.countDuplicateTokens(mark1, mark2);
+                if (dupes < this.minTileSize) {
+                    continue;
+                }
+                if (diff + dupes >= 1) {
+                    continue; // self-overlap
+                }
+                this.reportMatch(mark1, mark2, dupes);
+            }
+        }
+    }
+    reportMatch(mark1, mark2, dupes) {
+        if (this.tokenMatchSets.get(mark1)?.has(mark2)) {
+            return;
+        }
+        let lowestKey = mark1;
+        const set1 = this.tokenMatchSets.get(mark1);
+        if (set1) {
+            for (const key of set1) {
+                if (key < lowestKey)
+                    lowestKey = key;
+            }
+        }
+        let matches = this.matchTree.get(lowestKey);
+        if (!matches) {
+            matches = [];
+            this.matchTree.set(lowestKey, matches);
+        }
+        for (let i = 0; i < matches.length; i++) {
+            const m = matches[i];
+            for (const otherMark of m.marks) {
+                const otherEnd = otherMark.token.index;
+                if (otherEnd === mark1)
+                    continue;
+                if (otherEnd < mark2 && otherEnd + m.tokenCount >= mark2 + dupes) {
+                    return; // nested inside an existing match
+                }
+                else if (mark2 < otherEnd && mark2 + dupes >= otherEnd + m.tokenCount) {
+                    matches.splice(i, 1); // replace it
+                    i--;
+                    break;
+                }
+                else if (dupes === m.tokenCount) {
+                    for (const other of m.marks) {
+                        this.registerTokenMatch(other.token.index, mark2);
+                    }
+                    m.addMark(this.entry(mark2));
+                    return;
+                }
+            }
+        }
+        matches.push(new Match(dupes, new Mark(this.entry(mark1)), new Mark(this.entry(mark2))));
+        this.registerTokenMatch(mark1, mark2);
+    }
+    // Materialize a TokenEntry for a mark. The index is guaranteed in range (marks
+    // come from the token stream), so undefined is impossible here.
+    entry(index) {
+        const entry = this.ma.entryAt(index);
+        if (!entry)
+            throw new Error(`token index out of range: ${index}`);
+        return entry;
+    }
+    registerTokenMatch(mark1, mark2) {
+        let s1 = this.tokenMatchSets.get(mark1);
+        if (!s1) {
+            s1 = new Set();
+            this.tokenMatchSets.set(mark1, s1);
+        }
+        let s2 = this.tokenMatchSets.get(mark2);
+        if (!s2) {
+            s2 = new Set();
+            this.tokenMatchSets.set(mark2, s2);
+        }
+        s1.add(mark2);
+        s2.add(mark1);
+    }
+    getMatches() {
+        const result = [];
+        for (const matches of this.matchTree.values()) {
+            result.push(...matches);
+        }
+        return result;
+    }
+    hasPreviousDupe(mark1, mark2) {
+        if (mark1 === 0)
+            return false;
+        return !this.matchEnded(mark1 - 1, mark2 - 1);
+    }
+    countDuplicateTokens(mark1, mark2) {
+        let index = 0;
+        for (;;) {
+            if (this.matchEnded(mark1 + index, mark2 + index))
+                break;
+            index++;
+        }
+        return index;
+    }
+    // True once the windows diverge: one of the indices is out of range, the ids
+    // differ, or it is EOF (id === 0). Equivalent to matchEnded(token1, token2) on
+    // TokenEntry.
+    matchEnded(a, b) {
+        if (a < 0 || b < 0 || a >= this.tokenCount || b >= this.tokenCount)
+            return true;
+        const id1 = this.ids[a];
+        const id2 = this.ids[b];
+        return id1 !== id2 || id1 === 0 || id2 === 0;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+import { type Mark, type Match } from './core';
+import { type TokenizeOptions } from './tokenizers';
+export { Mark, Match, TokenEntry } from './core';
+export interface CpdOptions extends TokenizeOptions {
+    minTileSize?: number;
+    /** Extract inline templates from `@Component` for `.ts` files (default: false). */
+    angularInlineTemplates?: boolean;
+}
+export interface MatchLocation {
+    path: string;
+    startLine: number;
+    startColumn: number;
+    endLine: number;
+    endColumn: number;
+}
+/** High-level copy-paste detector: add sources, then {@link run} to get matches. */
+export declare class Cpd {
+    private core;
+    private opts;
+    constructor(opts?: CpdOptions);
+    addPath(filePath: string): void;
+    addSource(filePath: string, source: string): void;
+    run(): Match[];
+    locationForMark(mark: Mark, tokenCount: number): MatchLocation;
+    /** Plain text report for eyeballing / diff tests. */
+    report(matches?: Match[]): string;
+}

package/dist/index.js

@@ -0,0 +1,153 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Cpd = exports.TokenEntry = exports.Match = exports.Mark = void 0;
+/**
+ * Public entry point. The {@link Cpd} facade dispatches each source by extension
+ * to the right tokenizer, feeds the tokens to {@link CpdCore}, and materializes
+ * match locations for reporting.
+ *
+ * @packageDocumentation
+ */
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const angular_1 = require("./angular");
+const core_1 = require("./core");
+const svelte_1 = require("./svelte");
+const tokenizers_1 = require("./tokenizers");
+const vue_1 = require("./vue");
+var core_2 = require("./core");
+Object.defineProperty(exports, "Mark", { enumerable: true, get: function () { return core_2.Mark; } });
+Object.defineProperty(exports, "Match", { enumerable: true, get: function () { return core_2.Match; } });
+Object.defineProperty(exports, "TokenEntry", { enumerable: true, get: function () { return core_2.TokenEntry; } });
+const TS_EXT = new Set(['.ts', '.mts', '.cts']);
+const JSX_EXT = new Set(['.tsx', '.jsx', '.js', '.mjs', '.cjs']);
+const HTML_EXT = new Set(['.html', '.htm']);
+/** High-level copy-paste detector: add sources, then {@link run} to get matches. */
+class Cpd {
+    core;
+    opts;
+    constructor(opts = {}) {
+        this.opts = {
+            minTileSize: opts.minTileSize ?? 50,
+            ignoreIdentifiers: opts.ignoreIdentifiers ?? false,
+            ignoreLiterals: opts.ignoreLiterals ?? false,
+            pmdTypescriptCompatibility: opts.pmdTypescriptCompatibility ?? true,
+            svelteTemplates: opts.svelteTemplates ?? true,
+            vueTemplates: opts.vueTemplates ?? true,
+            angularInlineTemplates: opts.angularInlineTemplates ?? false,
+        };
+        this.core = new core_1.CpdCore(this.opts.minTileSize);
+    }
+    addPath(filePath) {
+        this.addSource(filePath, fs.readFileSync(filePath, 'utf-8'));
+    }
+    addSource(filePath, source) {
+        const ext = path.extname(filePath).toLowerCase();
+        const tok = {
+            ignoreIdentifiers: this.opts.ignoreIdentifiers,
+            ignoreLiterals: this.opts.ignoreLiterals,
+            pmdTypescriptCompatibility: this.opts.pmdTypescriptCompatibility,
+            svelteTemplates: this.opts.svelteTemplates,
+            vueTemplates: this.opts.vueTemplates,
+        };
+        if (TS_EXT.has(ext)) {
+            const script = (0, tokenizers_1.tokenizeTypeScript)(filePath, source, tok, (0, tokenizers_1.scriptKindFor)(ext));
+            const all = [...script];
+            if (this.opts.angularInlineTemplates && source.includes('@Component')) {
+                for (const tpl of (0, angular_1.extractAngularInlineTemplates)(filePath, source)) {
+                    const tplTokens = (0, angular_1.tokenizeAngularHtml)(filePath, tpl.code, { line: tpl.line, col: tpl.col }, tok);
+                    if (tplTokens.length) {
+                        all.push({ image: '', line: tpl.line, column: tpl.col, barrier: true });
+                        all.push(...tplTokens);
+                    }
+                }
+            }
+            this.core.addFile(filePath, all);
+            return;
+        }
+        if (JSX_EXT.has(ext)) {
+            this.core.addFile(filePath, (0, tokenizers_1.tokenizeTypeScript)(filePath, source, tok, (0, tokenizers_1.scriptKindFor)(ext)));
+            return;
+        }
+        if (ext === '.vue') {
+            this.core.addFile(filePath, (0, vue_1.tokenizeVue)(filePath, source, tok));
+            return;
+        }
+        if (ext === '.svelte') {
+            this.core.addFile(filePath, (0, svelte_1.tokenizeSvelte)(filePath, source, tok));
+            return;
+        }
+        if (HTML_EXT.has(ext)) {
+            // External Angular template (plain HTML parses too).
+            this.core.addFile(filePath, (0, angular_1.tokenizeAngularHtml)(filePath, source, { line: 1, col: 1 }, tok));
+            return;
+        }
+        // Unknown extension: ignore.
+    }
+    run() {
+        return this.core.analyze();
+    }
+    locationForMark(mark, tokenCount) {
+        const start = mark.token;
+        const end = this.core.entryAt(start.index + tokenCount - 1) ?? start;
+        return {
+            path: start.file,
+            startLine: start.beginLine,
+            startColumn: start.beginColumn,
+            endLine: end.endLine,
+            endColumn: end.endColumn,
+        };
+    }
+    /** Plain text report for eyeballing / diff tests. */
+    report(matches = this.run()) {
+        const lines = [];
+        for (const m of matches) {
+            const marks = m.marks;
+            lines.push(`Found a ${m.tokenCount} token (${m.markCount} occurrences) duplication:`);
+            for (const mk of marks) {
+                const t = mk.token;
+                lines.push(`  ${t.file}:${t.beginLine}:${t.beginColumn}`);
+            }
+            lines.push('');
+        }
+        return lines.join('\n');
+    }
+}
+exports.Cpd = Cpd;
+// Example:
+//   const cpd = new Cpd({ minTileSize: 50 });
+//   for (const f of files) cpd.addPath(f);
+//   console.log(cpd.report());

package/dist/svelte.d.ts

@@ -0,0 +1,4 @@
+import { type RawToken } from './core';
+import { type TokenizeOptions } from './tokenizers';
+/** Tokenize a `.svelte` component (`<script>` blocks + markup). */
+export declare function tokenizeSvelte(filePath: string, source: string, options?: TokenizeOptions): RawToken[];