clone-alert 0.3.0

package/dist/cli.js

@@ -0,0 +1,407 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.collectFiles = collectFiles;
+exports.main = main;
+exports.parseArgs = parseArgs;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const index_1 = require("./index");
+const DEFAULT_EXTENSIONS = [
+    '.ts',
+    '.tsx',
+    '.js',
+    '.jsx',
+    '.mts',
+    '.cts',
+    '.mjs',
+    '.cjs',
+    '.vue',
+    '.svelte',
+    '.html',
+    '.htm',
+];
+const HELP = `Usage: clone-alert [options] [<path>...]
+
+PMD CPD-like copy-paste detector for TS/JS and common frontend templates.
+
+Options:
+  --files <path[,path...]>        Files or directories to scan. Can be repeated.
+  --minimum-tokens <n>            Minimum duplicated token span. Default: 50.
+  --minimum-tile-size <n>         Alias for --minimum-tokens.
+  --format <text|xml|json>        Report format. Default: text.
+  --extensions <ext[,ext...]>     Extensions to include. Default: ts,tsx,js,jsx,vue,svelte,html.
+  --exclude <glob[,glob...]>      Exclude files or directories. Can be repeated.
+  --ignore-identifiers            Normalize identifiers.
+  --no-ignore-identifiers         Compare exact identifiers. Default.
+  --ignore-literals               Normalize literals.
+  --no-ignore-literals            Compare exact literals. Default.
+  --pmd-typescript-compatibility  Match PMD typescript granularity for .ts/.tsx:
+                                  split template literals into per-atom tokens
+                                  (backtick, \${, }, one per text char) and
+                                  collapse regexp. .js/.jsx stay native. Default.
+  --no-pmd-typescript-compatibility
+                                  Tokenize .ts/.tsx with the native TypeScript
+                                  scanner (a template literal stays one token).
+  --svelte-templates              Tokenize .svelte markup (ast.fragment), not
+                                  just <script>. Default.
+  --no-svelte-templates           Tokenize only <script> in .svelte files.
+                                  Use to run markup and code at different
+                                  --minimum-tokens thresholds.
+  --vue-templates                 Tokenize .vue markup (descriptor.template.ast),
+                                  not just <script>. Default.
+  --no-vue-templates              Skip .vue markup; scan the <script> block
+                                  alone (handy for a code-only threshold pass).
+  --angular-inline-templates      Also scan Angular @Component inline templates.
+  --skip-angular-inline-templates Do not scan inline Angular templates. Default.
+  --fail-on-violation             Exit with code 4 when duplications are found.
+  -h, --help                      Show this help.
+  -V, --version                   Show version.
+
+Examples:
+  clone-alert --minimum-tokens 50 --files src
+  clone-alert --minimum-tokens 30 --format xml src test
+`;
+function main(argv) {
+    let options;
+    try {
+        options = parseArgs(argv);
+    }
+    catch (error) {
+        console.error(`clone-alert: ${error.message}`);
+        console.error("Try 'clone-alert --help' for more information.");
+        return 2;
+    }
+    if (options.paths.length === 0) {
+        console.error('clone-alert: missing files or directories to scan');
+        console.error("Try 'clone-alert --help' for more information.");
+        return 2;
+    }
+    let files;
+    try {
+        files = collectFiles(options.paths, options.extensions, options.excludePatterns);
+    }
+    catch (error) {
+        console.error(`clone-alert: ${error.message}`);
+        return 2;
+    }
+    if (files.length === 0) {
+        console.error('clone-alert: no supported files found');
+        return 2;
+    }
+    const cpd = new index_1.Cpd(options);
+    for (const file of files) {
+        cpd.addPath(file);
+    }
+    const matches = cpd.run();
+    process.stdout.write(formatReport(options.format, cpd, matches));
+    return options.failOnViolation && matches.length > 0 ? 4 : 0;
+}
+function parseArgs(argv) {
+    const paths = [];
+    const extensions = new Set(DEFAULT_EXTENSIONS);
+    const excludePatterns = [];
+    let minTileSize = 50;
+    let ignoreIdentifiers = false;
+    let ignoreLiterals = false;
+    let pmdTypescriptCompatibility = true;
+    let svelteTemplates = true;
+    let vueTemplates = true;
+    let angularInlineTemplates = false;
+    let format = 'text';
+    let failOnViolation = false;
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg === '-h' || arg === '--help') {
+            console.log(HELP);
+            process.exit(0);
+        }
+        if (arg === '-V' || arg === '--version') {
+            console.log(readVersion());
+            process.exit(0);
+        }
+        if (arg === '--files') {
+            paths.push(...splitList(requireValue(argv, ++i, arg)));
+            continue;
+        }
+        if (arg.startsWith('--files=')) {
+            paths.push(...splitList(arg.slice('--files='.length)));
+            continue;
+        }
+        if (arg === '--minimum-tokens' || arg === '--minimum-tile-size') {
+            minTileSize = parsePositiveInteger(requireValue(argv, ++i, arg), arg);
+            continue;
+        }
+        if (arg.startsWith('--minimum-tokens=')) {
+            minTileSize = parsePositiveInteger(arg.slice('--minimum-tokens='.length), '--minimum-tokens');
+            continue;
+        }
+        if (arg.startsWith('--minimum-tile-size=')) {
+            minTileSize = parsePositiveInteger(arg.slice('--minimum-tile-size='.length), '--minimum-tile-size');
+            continue;
+        }
+        if (arg === '--format') {
+            format = parseFormat(requireValue(argv, ++i, arg));
+            continue;
+        }
+        if (arg.startsWith('--format=')) {
+            format = parseFormat(arg.slice('--format='.length));
+            continue;
+        }
+        if (arg === '--extensions') {
+            replaceExtensions(extensions, requireValue(argv, ++i, arg));
+            continue;
+        }
+        if (arg.startsWith('--extensions=')) {
+            replaceExtensions(extensions, arg.slice('--extensions='.length));
+            continue;
+        }
+        if (arg === '--exclude') {
+            excludePatterns.push(...splitList(requireValue(argv, ++i, arg)));
+            continue;
+        }
+        if (arg.startsWith('--exclude=')) {
+            excludePatterns.push(...splitList(arg.slice('--exclude='.length)));
+            continue;
+        }
+        if (arg === '--ignore-identifiers') {
+            ignoreIdentifiers = true;
+            continue;
+        }
+        if (arg === '--no-ignore-identifiers') {
+            ignoreIdentifiers = false;
+            continue;
+        }
+        if (arg === '--ignore-literals') {
+            ignoreLiterals = true;
+            continue;
+        }
+        if (arg === '--no-ignore-literals') {
+            ignoreLiterals = false;
+            continue;
+        }
+        if (arg === '--pmd-typescript-compatibility') {
+            pmdTypescriptCompatibility = true;
+            continue;
+        }
+        if (arg === '--no-pmd-typescript-compatibility') {
+            pmdTypescriptCompatibility = false;
+            continue;
+        }
+        if (arg === '--svelte-templates') {
+            svelteTemplates = true;
+            continue;
+        }
+        if (arg === '--no-svelte-templates') {
+            svelteTemplates = false;
+            continue;
+        }
+        if (arg === '--vue-templates') {
+            vueTemplates = true;
+            continue;
+        }
+        if (arg === '--no-vue-templates') {
+            vueTemplates = false;
+            continue;
+        }
+        if (arg === '--angular-inline-templates') {
+            angularInlineTemplates = true;
+            continue;
+        }
+        if (arg === '--skip-angular-inline-templates') {
+            angularInlineTemplates = false;
+            continue;
+        }
+        if (arg === '--fail-on-violation') {
+            failOnViolation = true;
+            continue;
+        }
+        if (arg.startsWith('-')) {
+            throw new Error(`unknown option: ${arg}`);
+        }
+        paths.push(arg);
+    }
+    return {
+        paths,
+        extensions,
+        excludePatterns,
+        minTileSize,
+        ignoreIdentifiers,
+        ignoreLiterals,
+        pmdTypescriptCompatibility,
+        svelteTemplates,
+        vueTemplates,
+        angularInlineTemplates,
+        format,
+        failOnViolation,
+    };
+}
+function requireValue(argv, index, option) {
+    const value = argv[index];
+    if (!value || value.startsWith('-')) {
+        throw new Error(`${option} requires a value`);
+    }
+    return value;
+}
+function splitList(value) {
+    return value
+        .split(',')
+        .map((item) => item.trim())
+        .filter(Boolean);
+}
+function parsePositiveInteger(value, option) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < 1) {
+        throw new Error(`${option} must be a positive integer`);
+    }
+    return parsed;
+}
+function parseFormat(value) {
+    if (value === 'text' || value === 'xml' || value === 'json') {
+        return value;
+    }
+    throw new Error('--format must be one of: text, xml, json');
+}
+function replaceExtensions(target, value) {
+    target.clear();
+    for (const ext of splitList(value)) {
+        target.add(ext.startsWith('.') ? ext.toLowerCase() : `.${ext.toLowerCase()}`);
+    }
+}
+function collectFiles(paths, extensions, excludePatterns = []) {
+    const files = [];
+    const seen = new Set();
+    const excludeMatchers = excludePatterns.map((pattern) => globToRegExp(toPosix(pattern)));
+    const visit = (entry) => {
+        const full = path.resolve(entry);
+        if (!fs.existsSync(full)) {
+            throw new Error(`path does not exist: ${entry}`);
+        }
+        const stat = fs.statSync(full);
+        if (stat.isDirectory()) {
+            if (isExcluded(`${full}${path.sep}`, excludeMatchers))
+                return;
+            for (const child of fs.readdirSync(full).sort()) {
+                if (child === 'node_modules' || child === '.git' || child === 'dist')
+                    continue;
+                visit(path.join(full, child));
+            }
+            return;
+        }
+        if (!stat.isFile())
+            return;
+        if (isExcluded(full, excludeMatchers))
+            return;
+        if (!extensions.has(path.extname(full).toLowerCase()))
+            return;
+        if (seen.has(full))
+            return;
+        seen.add(full);
+        files.push(full);
+    };
+    for (const entry of paths)
+        visit(entry);
+    return files;
+}
+function formatReport(format, cpd, matches) {
+    if (format === 'json') {
+        return `${JSON.stringify({ duplicates: matches.map((match) => matchToJson(match, cpd)) }, null, 2)}\n`;
+    }
+    if (format === 'xml') {
+        return formatXml(matches, cpd);
+    }
+    return cpd.report(matches);
+}
+function matchToJson(match, cpd) {
+    const files = match.marks.map((mark) => cpd.locationForMark(mark, match.tokenCount));
+    return {
+        lines: Math.max(0, ...files.map((file) => file.endLine - file.startLine + 1)),
+        tokens: match.tokenCount,
+        files,
+    };
+}
+function formatXml(matches, cpd) {
+    const lines = ['<?xml version="1.0" encoding="UTF-8"?>', '<pmd-cpd>'];
+    for (const match of matches) {
+        const duplicate = matchToJson(match, cpd);
+        lines.push(`  <duplication lines="${duplicate.lines}" tokens="${match.tokenCount}" occurrences="${match.markCount}">`);
+        for (const mark of match.marks) {
+            const location = cpd.locationForMark(mark, match.tokenCount);
+            lines.push(`    <file path="${escapeXml(location.path)}" line="${location.startLine}" endline="${location.endLine}" column="${location.startColumn}" endcolumn="${location.endColumn}" />`);
+        }
+        lines.push('  </duplication>');
+    }
+    lines.push('</pmd-cpd>');
+    return `${lines.join('\n')}\n`;
+}
+function isExcluded(filePath, matchers) {
+    const normalized = toPosix(filePath);
+    return matchers.some((matcher) => matcher.test(normalized));
+}
+function toPosix(value) {
+    return value.split(path.sep).join('/');
+}
+function globToRegExp(pattern) {
+    let source = '';
+    for (let index = 0; index < pattern.length; index++) {
+        const char = pattern[index];
+        if (char === '*') {
+            if (pattern[index + 1] === '*') {
+                source += '.*';
+                index++;
+            }
+            else {
+                source += '[^/]*';
+            }
+            continue;
+        }
+        source += escapeRegExp(char);
+    }
+    return new RegExp(`^${source}$`);
+}
+function escapeRegExp(char) {
+    return /[\\^$+?.()|[\]{}]/.test(char) ? `\\${char}` : char;
+}
+function escapeXml(value) {
+    return value.replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+function readVersion() {
+    const pkg = JSON.parse(fs.readFileSync(path.resolve(__dirname, '..', 'package.json'), 'utf-8'));
+    return pkg.version ?? '0.0.0';
+}
+if (require.main === module) {
+    process.exitCode = main(process.argv.slice(2));
+}

package/dist/core.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Language-agnostic CPD core. Operates on a flat stream of tokens supplied by
+ * the tokenizers (see tokenizers.ts) as `RawToken[]`.
+ *
+ * Token storage is struct-of-arrays over typed arrays (`Int32Array`): instead of
+ * ~N `TokenEntry` objects we keep parallel numeric columns. Full `TokenEntry`
+ * objects are materialized lazily, only for the marks that land in a match.
+ *
+ * @packageDocumentation
+ */
+/**
+ * Normalization sentinel prefix, taken from the Unicode private-use area so it
+ * is guaranteed never to collide with real source token images. Framework token
+ * namespaces (Angular/Vue/Svelte) live in their own extension modules
+ * (src/angular.ts, etc.) and are built on top of this shared sentinel.
+ */
+export declare const S = "\uE000";
+/** Normalized identifier (TS). */
+export declare const TS_ID = "\uE000ID";
+/** Normalized literal (TS). */
+export declare const TS_LIT = "\uE000LIT";
+/** A raw token as emitted by a tokenizer, before it is interned into the core. */
+export interface RawToken {
+    image: string;
+    /** 1-based. */
+    line: number;
+    /** 1-based. */
+    column: number;
+    /** 1-based, PMD-style token end position. */
+    endLine?: number;
+    /** 1-based, PMD-style exclusive end column. */
+    endColumn?: number;
+    /** Forced break; inserts an EOF token (id 0) so matches cannot span it. */
+    barrier?: boolean;
+}
+/** A fully materialized token with its image, interned id, and source location. */
+export declare class TokenEntry {
+    image: string;
+    identifier: number;
+    index: number;
+    file: string;
+    beginLine: number;
+    beginColumn: number;
+    endLine: number;
+    endColumn: number;
+    constructor(image: string, identifier: number, index: number, file: string, beginLine: number, beginColumn: number, endLine?: number, endColumn?: number);
+}
+/** A single occurrence of a duplicated span, anchored at its starting token. */
+export declare class Mark {
+    token: TokenEntry;
+    constructor(token: TokenEntry);
+}
+/** A set of marks that share an identical duplicated token span. */
+export declare class Match {
+    tokenCount: number;
+    /** Dedupe by token index (PMD uses a TreeSet keyed by index, not by reference). */
+    private markMap;
+    /**
+     * Cache of sorted marks. The `marks` getter is hit millions of times in the
+     * hot reportMatch path; without the cache every call did Array.from + sort.
+     * Invalidated only in addMark, i.e. when the mark set actually changes.
+     */
+    private marksSorted;
+    constructor(tokenCount: number, first: Mark, second: Mark);
+    addMark(entry: TokenEntry): void;
+    get markCount(): number;
+    get marks(): Mark[];
+}
+/** The duplicate-detection engine: ingests token streams and reports matches. */
+export declare class CpdCore {
+    private minTileSize;
+    private ids;
+    private fileIds;
+    private beginLines;
+    private beginColumns;
+    private endLines;
+    private endColumns;
+    private size;
+    private capacity;
+    private imageToId;
+    private idImages;
+    private fileToId;
+    private fileNames;
+    constructor(minTileSize?: number);
+    private intern;
+    private fileId;
+    private ensureCapacity;
+    private pushToken;
+    /** Add one file's token stream. An EOF barrier is always appended at the end. */
+    addFile(file: string, raw: RawToken[]): void;
+    get tokenCount(): number;
+    /** Raw access to the id column for the collector's hot loops (module-internal). */
+    get idColumn(): Int32Array;
+    /** Materialize a TokenEntry by absolute index. Returns undefined when out of range. */
+    entryAt(index: number): TokenEntry | undefined;
+    analyze(): Match[];
+    private hash;
+}