clone-alert 0.3.0 → 0.4.0

package/README.md

@@ -6,6 +6,7 @@
 [![license](https://img.shields.io/npm/l/clone-alert.svg)](./LICENSE)
 [![node](https://img.shields.io/node/v/clone-alert.svg)](https://nodejs.org)
 [![types](https://img.shields.io/npm/types/clone-alert.svg)](https://www.npmjs.com/package/clone-alert)
+[![clone-alert](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/BaryshevRS/clone-alert/main/clone-alert-badge.json)](#duplication-badge)
 
 **clone-alert** finds duplicated and copy‑pasted code across your codebase by comparing **token streams** — the same proven approach as [PMD CPD](https://pmd.github.io/) (Copy‑Paste Detector), but built natively for the JavaScript/TypeScript ecosystem and your frontend templates. Catch code clones, enforce **DRY**, reduce technical debt, and fail your build when duplication creeps in.
 
@@ -18,11 +19,12 @@ npx clone-alert --minimum-tokens 50 --files src
 ## Why clone-alert?
 
 - 🎯 **PMD CPD‑compatible** — a faithful port of PMD's match algorithm and JavaScript/TypeScript tokenizers, validated against PMD's own golden fixtures.
-- ⚡ **Fast on large monorepos** — a struct‑of‑arrays token core with a Karp–Rabin rolling hash and radix‑sorted buckets. In our [benchmarks](#benchmarks) it runs **10–30× faster** than PMD CPD while using **1.4–2.5× less memory**, on real codebases from Next.js to nx.
+- ⚡ **Fast on large monorepos** — a struct‑of‑arrays token core with a Karp–Rabin rolling hash and radix‑sorted buckets. In our [benchmarks](#benchmarks) it runs **10–27× faster** than PMD CPD while using **1.3–2.6× less memory**, on real codebases from Next.js to nx.
 - 🧩 **Frontend templates, natively** — tokenizes **Vue** `<template>`, **Svelte** markup, and **Angular** templates, not just `<script>` blocks. Detects template‑to‑script duplication too.
 - 🧪 **Zero‑config CLI** — sensible defaults, recursive directory scan, `node_modules`/`.git`/`dist` skipped automatically.
 - 📦 **Tiny footprint** — a single runtime dependency (`typescript`). Framework parsers are **optional peer dependencies**, loaded only when needed.
-- 🛠 **CI‑ready** — `text`, `json`, and PMD‑style `xml` reports, plus `--fail-on-violation` (exit code `4`).
+- 🛠 **CI‑ready** — `text`, `json`, PMD‑style `xml` / `csv`, and **SARIF** (GitHub Code Scanning) reports; fails the build on duplication by default (exit code `4`), like PMD CPD.
+- 📉 **Baseline for adoption** — accept the clones an existing project already has and fail CI only on **new** ones. Fingerprints are content‑based, so the baseline survives code moving around.
 - 🔇 **Inline suppression** — ignore known duplication with `CPD-OFF` / `CPD-ON` comment markers.
 
 ## Supported languages & frameworks
@@ -59,14 +61,19 @@ Requires **Node.js 18+**.
 ## Quick start
 
 ```sh
-# Scan a folder and print a human‑readable report
+# Scan a folder and print a human‑readable report.
+# Like PMD CPD, this exits 4 when duplication is found — so it fails CI out of the box.
 clone-alert --minimum-tokens 50 --files src
 
-# Fail CI when duplication is found (exit code 4)
-clone-alert --minimum-tokens 50 --files src --fail-on-violation
+# Just want the report, never a failing exit code? Opt out:
+clone-alert --minimum-tokens 50 --files src --no-fail-on-violation
 
-# Machine‑readable output for dashboards
-clone-alert --format json --files src,packages > duplication.json
+# Machine‑readable output for dashboards (don't fail the job that builds the artifact)
+clone-alert --format json --files src,packages --no-fail-on-violation > duplication.json
+
+# Adopt an existing project: accept today's clones, fail only on new ones
+clone-alert --files src --baseline .clone-alert-baseline.json --update-baseline
+clone-alert --files src --baseline .clone-alert-baseline.json --fail-on-violation
 ```
 
 ## Usage
@@ -80,11 +87,16 @@ clone-alert [options] [<path>...]
 | Option | Description |
 | --- | --- |
 | `--files <path[,path...]>` | Files or directories to scan. Can be repeated. |
+| `--file-list <path>` | Read newline-separated paths to scan from a file. |
 | `--minimum-tokens <n>` | Minimum duplicated token span. Default: `50`. |
 | `--minimum-tile-size <n>` | Alias for `--minimum-tokens`. |
-| `--format <text\|xml\|json>` | Report format. Default: `text`. |
+| `--format <fmt>` | `text` (default), `xml`, `json`, `sarif`, `csv`, `csv_with_linecount_per_file`, `markdown`, `ai`. `sarif` targets GitHub Code Scanning; the two `csv` formats mirror PMD's CSV renderers. `xml`/`json`/`markdown` embed the duplicated code (PMD's `<codefragment>`, a jscpd-style `fragment` field, and a fenced code block respectively). `ai` is a compact, token-frugal listing for LLM pipelines. `shields` prints a [shields.io endpoint](#duplication-badge) JSON for a duplication badge. `text` and `ai` end with a `N clones · X% duplicated lines` summary. |
 | `--extensions <ext[,ext...]>` | Extensions to include during recursive scans. |
-| `--exclude <glob[,glob...]>` | Exclude files or directories (glob). Can be repeated. |
+| `--exclude <glob[,glob...]>` | Exclude files or directories (glob). Can be repeated. Prunes the walk, not a post-filter — excluded directories are never read. |
+| `--non-recursive` | Scan only the top level of each directory. |
+| `--gitignore` / `--no-gitignore` | Skip files ignored by `.gitignore` (nested files and the repo-root file honored). On by default. |
+| `--skip-duplicate-files` | Skip files with the same name and byte length (PMD parity). |
+| `--skip-lexical-errors` | Skip files that fail to tokenize instead of aborting the whole run. |
 | `--ignore-identifiers` / `--no-ignore-identifiers` | Normalize or compare identifier names. Strict by default, like PMD. |
 | `--ignore-literals` / `--no-ignore-literals` | Normalize or compare literals. Strict by default, like PMD. |
 | `--pmd-typescript-compatibility` / `--no-…` | Match PMD `typescript` granularity for `.ts/.tsx` (split template literals into atoms, collapse regexp). On by default. |
@@ -92,7 +104,9 @@ clone-alert [options] [<path>...]
 | `--vue-templates` / `--no-vue-templates` | Tokenize `.vue` markup, not just `<script>`. On by default. |
 | `--angular-inline-templates` | Also scan Angular `@Component` inline templates. |
 | `--skip-angular-inline-templates` | Do not scan inline Angular templates (explicit default). |
-| `--fail-on-violation` | Exit with code `4` when duplications are found. |
+| `--fail-on-violation` / `--no-fail-on-violation` | Exit with code `4` when duplications are found. **On by default**, like PMD CPD; pass `--no-fail-on-violation` to always exit `0`. |
+| `--baseline <path>` | Ignore duplications recorded in this baseline file; report and fail only on **new** ones. Matched by content fingerprint, so accepted clones stay suppressed even after the code moves. |
+| `--update-baseline` | Write/regenerate the baseline file at `--baseline` with all current duplications, then exit `0`. Run once to adopt existing debt. |
 | `-h, --help` | Show help. |
 | `-V, --version` | Show version. |
 
@@ -159,6 +173,94 @@ const generatedTableB = { /* ... */ };
 // CPD-ON
 ```
 
+## Baseline (adopting an existing project)
+
+A fresh project can have thousands of pre‑existing clones — enough to light up CI red on day one. A **baseline** lets you accept that debt and gate only on what's added afterwards.
+
+Generate it once, commit it, then check against it in CI:
+
+```sh
+# 1. Record today's duplications (writes the file, exits 0)
+clone-alert --files src --baseline .clone-alert-baseline.json --update-baseline
+
+# 2. In CI: fail only on clones not in the baseline
+clone-alert --files src --baseline .clone-alert-baseline.json --fail-on-violation
+```
+
+The baseline is a small, sorted JSON file you commit and review in pull requests:
+
+```json
+{
+  "version": 1,
+  "clones": [
+    {
+      "fingerprint": "00a034a93cd6e7e3",
+      "tokens": 414,
+      "files": ["src/server/webkit/webview/wvPage.ts", "src/server/webkit/wkPage.ts"]
+    }
+  ]
+}
+```
+
+Each clone is matched by a **content fingerprint** hashed over its tokens only — no line numbers, no file paths. So a baselined clone stays suppressed when the code is moved, reformatted, or shifted by edits above it, and the file produces a stable, churn‑free diff. Introduce a genuinely new duplication and CI fails on that one alone. Re‑run `--update-baseline` to re‑adopt after an intentional change.
+
+> The baseline filters the already‑computed match set, so it adds no measurable cost to a scan — there's no separate cache to persist between CI runs.
+
+## GitHub Code Scanning (SARIF)
+
+`--format sarif` emits a SARIF 2.1.0 log that GitHub ingests as code‑scanning alerts, shown inline in pull requests and in the repository's Security tab. Each duplication's stable content fingerprint is written to `partialFingerprints`, so GitHub tracks an alert across commits and **does not re‑raise it when the clone simply moves**. Artifact URIs are relative to the working directory, so they map onto the checked‑out tree.
+
+```yaml
+# .github/workflows/clone-alert.yml
+name: clone-alert
+on: [push, pull_request]
+jobs:
+  duplication:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write   # required to upload SARIF
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with: { node-version: 20 }
+      # --no-fail-on-violation so the step exits 0 and the SARIF still uploads;
+      # GitHub surfaces the duplications as code-scanning alerts instead.
+      - run: npx clone-alert src --format sarif --no-fail-on-violation > clone-alert.sarif
+      - uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: clone-alert.sarif
+```
+
+Combine it with a committed `--baseline` to surface only the duplications added after adoption.
+
+## Duplication badge
+
+Show off how clean your codebase is with a [shields.io](https://shields.io/badges/endpoint-badge) badge. `--format shields` prints a shields **endpoint JSON** to stdout — host it (a committed file, a gist, anywhere reachable) and point shields at it:
+
+```sh
+clone-alert src --minimum-tokens 70 --format shields --no-fail-on-violation > clone-alert-badge.json
+```
+
+```md
+[![clone-alert](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/OWNER/REPO/main/clone-alert-badge.json)](https://github.com/BaryshevRS/clone-alert)
+```
+
+shields fetches the JSON and renders the badge, so it refreshes whenever you regenerate the file. The color comes from a fixed scale, tuned to reward near‑zero duplication:
+
+| Result | Color | |
+| --- | --- | --- |
+| **0 clones** | 🟢 bright green | the flex — zero copy‑paste |
+| **≤ 3%** | 🟢 green | clean |
+| **≤ 10%** | 🟡 yellow | has some debt |
+| **> 10%** | 🔴 red | needs attention |
+
+The percentage is `duplicated lines / total scanned lines`, so it tracks your chosen `--minimum-tokens` (and which files you scan — exclude `**/*.test.*` and fixtures to badge production code only). Regenerate it in CI to keep it fresh:
+
+```yaml
+      - run: npx clone-alert src --minimum-tokens 70 --format shields --no-fail-on-violation > clone-alert-badge.json
+      # then commit the file (or push it to a gist) so shields serves the latest value
+```
+
 ## Programmatic API
 
 clone-alert ships with TypeScript types and a small Node API:
@@ -184,17 +286,17 @@ Framework template tokens live in a separate namespace from script tokens, so ma
 
 ## Benchmarks
 
-clone-alert is a drop-in for PMD CPD that runs **10–30× faster** on **1.4–2.5× less memory** — on the same files, finding the same clones.
+clone-alert is a drop-in for PMD CPD that runs **10–27× faster** on **1.3–2.6× less memory** — on the same files, finding the same clones.
 
 Measured with [`npm run compare:pmd`](#development) on five real-world TypeScript codebases. Only pure `.ts` is compared (the exact file set PMD's `typescript` lexer can parse), so all tools see byte-identical input. macOS, Node 20, `--minimum-tokens 50`, JVM start-up counted for PMD as in real CLI use:
 
 | Repository | clone-alert | PMD CPD | Speed‑up | Peak RAM (clone vs PMD) | Agreement with PMD¹ |
 | --- | --- | --- | --- | --- | --- |
-| `nestjs/nest` | **1.8 s** | 52.7 s | **30×** | 203 MB vs 500 MB (2.5× less) | 100% |
-| `angular/components` | **1.5 s** | 42.5 s | **28×** | 338 MB vs 577 MB (1.7× less) | 95%² |
-| `microsoft/playwright` | **9.3 s** | 153 s | **16×** | 838 MB vs 1.4 GB (1.7× less) | 99.98% |
-| `vercel/next.js` | **5.8 s** | 73.4 s | **13×** | 1.3 GB vs 1.9 GB (1.4× less) | 99.2% |
-| `nrwl/nx` | **8.6 s** | 84.1 s | **10×** | 2.1 GB vs 3.3 GB (1.6× less) | 99.9% |
+| `nestjs/nest` | **0.7 s** | 15.4 s | **23×** | 203 MB vs 526 MB (2.6× less) | 100% |
+| `angular/components` | **1.6 s** | 41.9 s | **27×** | 338 MB vs 632 MB (1.9× less) | 95%² |
+| `microsoft/playwright` | **3.6 s** | 58.7 s | **16×** | 836 MB vs 1.6 GB (1.9× less) | 99.98% |
+| `vercel/next.js` | **6.0 s** | 73.6 s | **12×** | 1.3 GB vs 1.7 GB (1.3× less) | 99.2% |
+| `nrwl/nx` | **8.1 s** | 83.2 s | **10×** | 2.1 GB vs 3.2 GB (1.5× less) | 99.9% |
 
 <sub>¹ Jaccard overlap of the file pairs both tools flag as duplicated. ² `angular/components` ships ~20 near‑identical table demos sharing the same 398‑token block. clone-alert and PMD cut that clone's boundary **identically** (398, 391, 390, 210… tokens, token‑for‑token); they only disagree on *which* of the interchangeable demo files get grouped into the same `<duplication>` — a symmetric clustering tie‑break, not missed or mis‑sized duplication. On this small sample (~2 000 file pairs) that grouping noise is the whole 5%.</sub>
 
@@ -220,8 +322,15 @@ Because the tokens are identical and the match engine is a faithful port of PMD'
 | Svelte markup | ✅ (Svelte 5+) | ➖ | ➖ |
 | Angular templates | ✅ | ➖ | flat HTML only |
 | PMD CPD algorithm parity | ✅ | — | ➖ |
+| CI baseline (fail only on new) | ✅ committed fingerprint file | ➖ | ⚠️ via on‑disk cache¹ |
+| SARIF / GitHub Code Scanning | ✅ | ➖ | ✅ |
+| Report formats | text, xml, json, sarif, csv, markdown, ai, shields | text, xml, csv, vs | many |
+| PMD CLI flags (`--file-list`, `--non-recursive`, `--skip-duplicate-files`, `--skip-lexical-errors`) | ✅ | ✅ | ➖ |
+| `.gitignore` aware | ✅ (on by default, prunes walk) | ➖ | ✅ |
 | Install size | tiny (1 dep) | JVM required | npm package |
 
+¹ jscpd derives "new vs known" from a persistent store (LevelDB) that you must keep between runs; clone-alert commits a small, reviewable JSON baseline and stays stateless.
+
 ## Development
 
 ```sh

package/dist/baseline.d.ts

@@ -0,0 +1,20 @@
+import type { Cpd, Match } from './index';
+/** One accepted clone, as persisted in the baseline file. */
+export interface CloneRecord {
+    fingerprint: string;
+    tokens: number;
+    files: string[];
+}
+/**
+ * Stable content fingerprint of a match: see {@link hashImages}. Both occurrences
+ * of a match share identical span content, so the choice of mark does not matter.
+ */
+export declare function fingerprint(cpd: Cpd, match: Match): string;
+/** Fingerprints of every clone recorded in the baseline file. */
+export declare function readBaseline(baselinePath: string): Set<string>;
+/**
+ * Serialize the accepted clones to the baseline file. Entries are sorted by
+ * fingerprint and carry no line/column, so the file stays a stable, reviewable,
+ * churn-free diff as long as the duplicated content itself does not change.
+ */
+export declare function writeBaseline(baselinePath: string, clones: CloneRecord[]): void;

package/dist/baseline.js

@@ -0,0 +1,105 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fingerprint = fingerprint;
+exports.readBaseline = readBaseline;
+exports.writeBaseline = writeBaseline;
+/**
+ * Baseline support for the CLI: the content-fingerprint hash plus reading and
+ * writing the baseline JSON file. This is purely an adoption/CI concern, kept out
+ * of the core engine (a faithful PMD port) and the tokenizers.
+ *
+ * @packageDocumentation
+ */
+const fs = __importStar(require("node:fs"));
+/**
+ * Stable content fingerprint of a match: see {@link hashImages}. Both occurrences
+ * of a match share identical span content, so the choice of mark does not matter.
+ */
+function fingerprint(cpd, match) {
+    return hashImages(cpd.spanImages(match));
+}
+/**
+ * 64-bit hash (16 hex chars) over the token *images* only — no file path, no
+ * line/column — so it is unchanged when the duplicated code moves within or
+ * between files. Deterministic across runs and machines (a pure function of the
+ * image strings). A boundary byte between tokens keeps `["ab","c"]` distinct from
+ * `["a","bc"]`.
+ */
+function hashImages(images) {
+    let a = 0x811c9dc5 | 0; // FNV-1a basis
+    let b = 0x85ebca6b | 0; // second lane, distinct multiplier
+    for (const image of images) {
+        for (let c = 0; c < image.length; c++) {
+            const ch = image.charCodeAt(c);
+            a = Math.imul(a ^ ch, 0x01000193);
+            b = Math.imul(b ^ ch, 0xc2b2ae35);
+        }
+        a = Math.imul(a ^ 0xff, 0x01000193);
+        b = Math.imul(b ^ 0xff, 0xc2b2ae35);
+    }
+    return toHex8(a) + toHex8(b);
+}
+function toHex8(h) {
+    return (h >>> 0).toString(16).padStart(8, '0');
+}
+/** Fingerprints of every clone recorded in the baseline file. */
+function readBaseline(baselinePath) {
+    if (!fs.existsSync(baselinePath)) {
+        throw new Error(`baseline file not found: ${baselinePath} (run with --update-baseline to create it)`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(fs.readFileSync(baselinePath, 'utf-8'));
+    }
+    catch {
+        throw new Error(`baseline file is not valid JSON: ${baselinePath}`);
+    }
+    const fingerprints = new Set();
+    for (const clone of parsed.clones ?? []) {
+        if (clone.fingerprint)
+            fingerprints.add(clone.fingerprint);
+    }
+    return fingerprints;
+}
+/**
+ * Serialize the accepted clones to the baseline file. Entries are sorted by
+ * fingerprint and carry no line/column, so the file stays a stable, reviewable,
+ * churn-free diff as long as the duplicated content itself does not change.
+ */
+function writeBaseline(baselinePath, clones) {
+    const sorted = [...clones].sort((x, y) => x.fingerprint < y.fingerprint ? -1 : x.fingerprint > y.fingerprint ? 1 : 0);
+    fs.writeFileSync(baselinePath, `${JSON.stringify({ version: 1, clones: sorted }, null, 2)}\n`);
+}

package/dist/cli.d.ts

@@ -1,14 +1,20 @@
 #!/usr/bin/env node
+import { collectFiles } from './files';
 import { type CpdOptions } from './index';
-type ReportFormat = 'text' | 'xml' | 'json';
+type ReportFormat = 'text' | 'xml' | 'json' | 'sarif' | 'csv' | 'csv_with_linecount_per_file' | 'markdown' | 'ai' | 'shields';
 interface CliOptions extends CpdOptions {
     paths: string[];
     extensions: Set<string>;
     excludePatterns: string[];
+    respectGitignore: boolean;
+    nonRecursive: boolean;
+    skipDuplicateFiles: boolean;
+    skipLexicalErrors: boolean;
     format: ReportFormat;
     failOnViolation: boolean;
+    baselinePath?: string;
+    updateBaseline: boolean;
 }
 declare function main(argv: string[]): number;
 declare function parseArgs(argv: string[]): CliOptions;
-declare function collectFiles(paths: string[], extensions: Set<string>, excludePatterns?: string[]): string[];
 export { collectFiles, main, parseArgs };