@omfalos/mokosh 0.3.2 → 0.3.3

package/dist/duplication-worker.d.mts

@@ -0,0 +1,25 @@
+import { F as FileType } from './parse-CAKctgb6.mjs';
+
+/**
+ * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language
+ * comment syntax, then splits what remains into a normalized token stream shared by every
+ * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language
+ * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,
+ * LiveScript, Lua, Gherkin, style files, and Markdown.
+ */
+
+/** One normalized token plus the 1-based source line it came from. */
+interface NormalizedToken {
+    text: string;
+    line: number;
+}
+
+/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */
+
+declare function tokenizeInWorker(payload: {
+    source: string;
+    fileType: FileType;
+    ignoreLiterals: boolean;
+}): NormalizedToken[];
+
+export { tokenizeInWorker as default };

package/dist/duplication-worker.d.ts

@@ -0,0 +1,25 @@
+import { F as FileType } from './parse-CAKctgb6.js';
+
+/**
+ * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language
+ * comment syntax, then splits what remains into a normalized token stream shared by every
+ * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language
+ * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,
+ * LiveScript, Lua, Gherkin, style files, and Markdown.
+ */
+
+/** One normalized token plus the 1-based source line it came from. */
+interface NormalizedToken {
+    text: string;
+    line: number;
+}
+
+/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */
+
+declare function tokenizeInWorker(payload: {
+    source: string;
+    fileType: FileType;
+    ignoreLiterals: boolean;
+}): NormalizedToken[];
+
+export { tokenizeInWorker as default };

package/dist/duplication-worker.js

@@ -0,0 +1,5 @@
+"use strict";var p=Object.defineProperty;var u=Object.getOwnPropertyDescriptor;var h=Object.getOwnPropertyNames;var x=Object.prototype.hasOwnProperty;var b=(e,t)=>{for(var n in t)p(e,n,{get:t[n],enumerable:!0})},y=(e,t,n,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of h(t))!x.call(e,o)&&o!==n&&p(e,o,{get:()=>t[o],enumerable:!(i=u(t,o))||i.enumerable});return e};var N=e=>y(p({},"__esModule",{value:!0}),e);var $={};b($,{default:()=>T});module.exports=N($);var R={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function m(e,t,n){let i=e.slice(t,n).replace(/[^\n]/g," ");return e.slice(0,t)+i+e.slice(n)}function z(e,t){if(!t)return e;let n=e;for(let[i,o]of t.block??[]){let c=0;for(;;){let l=n.indexOf(i,c);if(l===-1)break;let r=n.indexOf(o,l+i.length),s=r===-1?n.length:r+o.length;n=m(n,l,s),c=s}}for(let i of t.line??[]){let o=0;for(;;){let c=n.indexOf(i,o);if(c===-1)break;let l=n.indexOf(`
+`,c),r=l===-1?n.length:l;n=m(n,c,r),o=r}}return n}var A=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],d=new RegExp(`${A.map(e=>e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),_=/^[A-Za-z_$][A-Za-z0-9_$]*$/,k=/^\d+(?:\.\d+)?$/,w=/^(".*"|'.*'|`.*`)$/s,S=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);function g(e,t,n=!0){let i=z(e,R[t]),o=[],c=1,l=0;d.lastIndex=0;let r=d.exec(i);for(;r!==null;){for(let a=l;a<r.index;a++)i[a]===`
+`&&c++;l=r.index;let s=r[0],f=s;_.test(s)&&!S.has(s)?f="ID":n&&(k.test(s)||w.test(s))&&(f=k.test(s)?"NUM":"STR"),o.push({text:f,line:c});for(let a=l;a<r.index+s.length;a++)i[a]===`
+`&&c++;l=r.index+s.length,r=d.exec(i)}return o}function T(e){return g(e.source,e.fileType,e.ignoreLiterals)}
+//# sourceMappingURL=duplication-worker.js.map

package/dist/duplication-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/duplication-worker.ts","../src/graph/duplication/tokenizer.ts"],"sourcesContent":["/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n","/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,\n * LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: identifiers always collapse to a single placeholder (so renamed-variable clones\n *   still hash identically), and literals optionally collapse too (`ignoreLiterals`, default\n *   on) so only structural shape — not the specific values used — drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment-stripping rule; unrecognised types skip stripping.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripComments(source, COMMENT_SYNTAX[fileType]);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n"],"mappings":"yaAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,aAAAE,IAAA,eAAAC,EAAAH,GC0BA,IAAMI,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAEA,IAAMO,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EAaM,SAASC,EACdpB,EACAqB,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWnB,EAAcJ,EAAQF,EAAeuB,CAAQ,CAAC,EACzDG,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBZ,EAAc,UAAY,EAC1B,IAAIa,EAAQb,EAAc,KAAKS,CAAQ,EAEvC,KAAOI,IAAU,MAAM,CACrB,QAASC,EAAIF,EAAWE,EAAID,EAAM,MAAOC,IACnCL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPb,EAAmB,KAAKa,CAAG,GAAK,CAACV,EAAS,IAAIU,CAAG,EACnDC,EAAO,KACER,IAAmBL,EAAe,KAAKY,CAAG,GAAKX,EAAe,KAAKW,CAAG,KAC/EC,EAAOb,EAAe,KAAKY,CAAG,EAAI,MAAQ,OAE5CL,EAAO,KAAK,CAAE,KAAAM,EAAM,KAAAL,CAAK,CAAC,EAE1B,QAASG,EAAIF,EAAWE,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAAQE,EAAI,OAC9BF,EAAQb,EAAc,KAAKS,CAAQ,CACrC,CAEA,OAAOC,CACT,CDvOe,SAARO,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["duplication_worker_exports","__export","tokenizeInWorker","__toCommonJS","COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","tokenize","fileType","ignoreLiterals","stripped","tokens","line","lastIndex","match","i","raw","text","tokenizeInWorker","payload","tokenize"]}

package/dist/duplication-worker.mjs

@@ -0,0 +1,5 @@
+var u={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function d(n,s,e){let i=n.slice(s,e).replace(/[^\n]/g," ");return n.slice(0,s)+i+n.slice(e)}function h(n,s){if(!s)return n;let e=n;for(let[i,c]of s.block??[]){let l=0;for(;;){let o=e.indexOf(i,l);if(o===-1)break;let t=e.indexOf(c,o+i.length),r=t===-1?e.length:t+c.length;e=d(e,o,r),l=r}}for(let i of s.line??[]){let c=0;for(;;){let l=e.indexOf(i,c);if(l===-1)break;let o=e.indexOf(`
+`,l),t=o===-1?e.length:o;e=d(e,l,t),c=t}}return e}var x=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],p=new RegExp(`${x.map(n=>n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),b=/^[A-Za-z_$][A-Za-z0-9_$]*$/,m=/^\d+(?:\.\d+)?$/,y=/^(".*"|'.*'|`.*`)$/s,N=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);function k(n,s,e=!0){let i=h(n,u[s]),c=[],l=1,o=0;p.lastIndex=0;let t=p.exec(i);for(;t!==null;){for(let a=o;a<t.index;a++)i[a]===`
+`&&l++;o=t.index;let r=t[0],f=r;b.test(r)&&!N.has(r)?f="ID":e&&(m.test(r)||y.test(r))&&(f=m.test(r)?"NUM":"STR"),c.push({text:f,line:l});for(let a=o;a<t.index+r.length;a++)i[a]===`
+`&&l++;o=t.index+r.length,t=p.exec(i)}return c}function E(n){return k(n.source,n.fileType,n.ignoreLiterals)}export{E as default};
+//# sourceMappingURL=duplication-worker.mjs.map

package/dist/duplication-worker.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/graph/duplication/tokenizer.ts","../src/duplication-worker.ts"],"sourcesContent":["/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,\n * LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: identifiers always collapse to a single placeholder (so renamed-variable clones\n *   still hash identically), and literals optionally collapse too (`ignoreLiterals`, default\n *   on) so only structural shape — not the specific values used — drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment-stripping rule; unrecognised types skip stripping.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripComments(source, COMMENT_SYNTAX[fileType]);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n","/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n"],"mappings":"AA0BA,IAAMA,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAEA,IAAMO,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EAaM,SAASC,EACdpB,EACAqB,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWnB,EAAcJ,EAAQF,EAAeuB,CAAQ,CAAC,EACzDG,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBZ,EAAc,UAAY,EAC1B,IAAIa,EAAQb,EAAc,KAAKS,CAAQ,EAEvC,KAAOI,IAAU,MAAM,CACrB,QAASC,EAAIF,EAAWE,EAAID,EAAM,MAAOC,IACnCL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPb,EAAmB,KAAKa,CAAG,GAAK,CAACV,EAAS,IAAIU,CAAG,EACnDC,EAAO,KACER,IAAmBL,EAAe,KAAKY,CAAG,GAAKX,EAAe,KAAKW,CAAG,KAC/EC,EAAOb,EAAe,KAAKY,CAAG,EAAI,MAAQ,OAE5CL,EAAO,KAAK,CAAE,KAAAM,EAAM,KAAAL,CAAK,CAAC,EAE1B,QAASG,EAAIF,EAAWE,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAAQE,EAAI,OAC9BF,EAAQb,EAAc,KAAKS,CAAQ,CACrC,CAEA,OAAOC,CACT,CCvOe,SAARO,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","tokenize","fileType","ignoreLiterals","stripped","tokens","line","lastIndex","match","i","raw","text","tokenizeInWorker","payload","tokenize"]}

package/dist/index.d.mts

@@ -1,5 +1,7 @@
-import { F as FileNode, C as CallEdge, I as ImportEdge, a as FileType, N as NodeCategory, P as ParseResult, S as StructuredTag } from './types-C9fLCS45.mjs';
-export { E as ExportedSymbol, b as ImportType, T as TagKind } from './types-C9fLCS45.mjs';
+import { F as FileNode, C as CallEdge, I as ImportEdge, P as ParseResult, S as StructuredTag } from './types-BtSqoqbZ.mjs';
+export { E as ExportedSymbol } from './types-BtSqoqbZ.mjs';
+import { F as FileType, N as NodeCategory } from './parse-CAKctgb6.mjs';
+export { I as ImportType, T as TagKind } from './parse-CAKctgb6.mjs';
 
 interface SerializedGraph {
     nodes: FileNode[];
@@ -464,7 +466,8 @@ type DuplicateFamily = "style" | "code";
  * matches into one N-occurrence group instead of reporting C(N,2) near-identical pairs for a block
  * repeated N times. Shared by every language `tokenize()` supports — the shingling step itself has
  * no language awareness at all. See docs/adr-013-duplicate-detection-noise-reduction.md for the
- * noise this addresses.
+ * noise this addresses, and docs/adr-014-duplicate-detection-scale.md for why oversized hash
+ * buckets are capped below.
  */
 
 interface DuplicateOccurrence {
@@ -489,6 +492,13 @@ interface DuplicateGroup {
     family?: DuplicateFamily | undefined;
 }
 
+/** Configures whether/how tokenizing is offloaded to a `piscina` worker pool. `false` always
+ *  tokenizes in-process. */
+type ParallelTokenizingOption = boolean | {
+    minFiles?: number;
+    maxThreads?: number;
+};
+
 interface FindDuplicatesOptions {
     /** Minimum duplicated block size, in source lines, to report (default 6). */
     minLines?: number | undefined;
@@ -504,11 +514,28 @@ interface FindDuplicatesOptions {
      *  CSS/Less/SCSS structural comparator, which already matches on literal declaration content.
      *  Set to 1 to disable. See docs/adr-013-duplicate-detection-noise-reduction.md. */
     maxPunctuationRatio?: number | undefined;
+    /** Skip a hash bucket's O(k²) pairwise comparison once it holds more than this many locations
+     *  (default 400) — bounds worst-case scan time on large repos where a single ubiquitous token
+     *  window (a common import line, a boilerplate header) would otherwise blow past what a single
+     *  scan can finish in. Set `Infinity` to disable. See docs/adr-014-duplicate-detection-scale.md. */
+    maxBucketSize?: number | undefined;
     /** Caps the number of duplicate blocks returned, largest-first (default 50). */
     limit?: number | undefined;
     /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
      *  — `node_modules`, `dist`, `.git`, `mokosh-cache`, `coverage`, etc.). Pass `[]` to disable. */
     ignoreDirs?: readonly string[] | undefined;
+    /** Controls worker-pool offloading of per-file tokenizing (default `true`): offloads once the
+     *  candidate file count reaches `minFiles` (default 20, matching `GraphBuilder`'s parse pool);
+     *  `false` always tokenizes in-process; an object overrides `minFiles`/`maxThreads`. See
+     *  docs/adr-014-duplicate-detection-scale.md. */
+    parallelTokenizing?: ParallelTokenizingOption | undefined;
+}
+interface FindDuplicatesResult {
+    /** Duplicate blocks, largest-first, capped at `limit`. */
+    groups: DuplicateGroup[];
+    /** How many hash buckets were skipped for exceeding `maxBucketSize` — a non-zero count means
+     *  results may under-report duplication that's unusually widespread (see `maxBucketSize`). */
+    skippedBuckets: number;
 }
 /**
  * @description Scans every file already present in `graph` for cross-file (and within-file)
@@ -530,13 +557,17 @@ interface FindDuplicatesOptions {
  *   token-shingle path only (CSS/Less/SCSS always match on literal declaration content);
  *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
  *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
- *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
- *   results. Lock files are always excluded, independent of `ignoreDirs`.
- * @returns Duplicate blocks (each tagged with its `family`), two or more occurrences per block —
- *   every block that pairwise chain-matches another is clustered into one group instead of one
- *   per pair — sorted largest-first across all families.
- */
-declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<DuplicateGroup[]>;
+ *   shared logic; `maxBucketSize` bounds worst-case scan time on large repos by skipping
+ *   pathologically common hash buckets; `ignoreDirs` excludes files under matching directory
+ *   names; `limit` caps results; `parallelTokenizing` offloads per-file tokenizing to a worker
+ *   pool once the candidate file count is large enough to be worth it. Lock files are always
+ *   excluded, independent of `ignoreDirs`.
+ * @returns `groups` — duplicate blocks (each tagged with its `family`), two or more occurrences
+ *   per block, every block that pairwise chain-matches another clustered into one group instead
+ *   of one per pair, sorted largest-first across all families — plus `skippedBuckets` (see
+ *   {@link FindDuplicatesResult}).
+ */
+declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<FindDuplicatesResult>;
 
 /** Controls how aggressively `detectFeatures` promotes files to features. */
 interface FeatureDetectionOptions {

package/dist/index.d.ts

@@ -1,5 +1,7 @@
-import { F as FileNode, C as CallEdge, I as ImportEdge, a as FileType, N as NodeCategory, P as ParseResult, S as StructuredTag } from './types-C9fLCS45.js';
-export { E as ExportedSymbol, b as ImportType, T as TagKind } from './types-C9fLCS45.js';
+import { F as FileNode, C as CallEdge, I as ImportEdge, P as ParseResult, S as StructuredTag } from './types-BlN-U5AM.js';
+export { E as ExportedSymbol } from './types-BlN-U5AM.js';
+import { F as FileType, N as NodeCategory } from './parse-CAKctgb6.js';
+export { I as ImportType, T as TagKind } from './parse-CAKctgb6.js';
 
 interface SerializedGraph {
     nodes: FileNode[];
@@ -464,7 +466,8 @@ type DuplicateFamily = "style" | "code";
  * matches into one N-occurrence group instead of reporting C(N,2) near-identical pairs for a block
  * repeated N times. Shared by every language `tokenize()` supports — the shingling step itself has
  * no language awareness at all. See docs/adr-013-duplicate-detection-noise-reduction.md for the
- * noise this addresses.
+ * noise this addresses, and docs/adr-014-duplicate-detection-scale.md for why oversized hash
+ * buckets are capped below.
  */
 
 interface DuplicateOccurrence {
@@ -489,6 +492,13 @@ interface DuplicateGroup {
     family?: DuplicateFamily | undefined;
 }
 
+/** Configures whether/how tokenizing is offloaded to a `piscina` worker pool. `false` always
+ *  tokenizes in-process. */
+type ParallelTokenizingOption = boolean | {
+    minFiles?: number;
+    maxThreads?: number;
+};
+
 interface FindDuplicatesOptions {
     /** Minimum duplicated block size, in source lines, to report (default 6). */
     minLines?: number | undefined;
@@ -504,11 +514,28 @@ interface FindDuplicatesOptions {
      *  CSS/Less/SCSS structural comparator, which already matches on literal declaration content.
      *  Set to 1 to disable. See docs/adr-013-duplicate-detection-noise-reduction.md. */
     maxPunctuationRatio?: number | undefined;
+    /** Skip a hash bucket's O(k²) pairwise comparison once it holds more than this many locations
+     *  (default 400) — bounds worst-case scan time on large repos where a single ubiquitous token
+     *  window (a common import line, a boilerplate header) would otherwise blow past what a single
+     *  scan can finish in. Set `Infinity` to disable. See docs/adr-014-duplicate-detection-scale.md. */
+    maxBucketSize?: number | undefined;
     /** Caps the number of duplicate blocks returned, largest-first (default 50). */
     limit?: number | undefined;
     /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
      *  — `node_modules`, `dist`, `.git`, `mokosh-cache`, `coverage`, etc.). Pass `[]` to disable. */
     ignoreDirs?: readonly string[] | undefined;
+    /** Controls worker-pool offloading of per-file tokenizing (default `true`): offloads once the
+     *  candidate file count reaches `minFiles` (default 20, matching `GraphBuilder`'s parse pool);
+     *  `false` always tokenizes in-process; an object overrides `minFiles`/`maxThreads`. See
+     *  docs/adr-014-duplicate-detection-scale.md. */
+    parallelTokenizing?: ParallelTokenizingOption | undefined;
+}
+interface FindDuplicatesResult {
+    /** Duplicate blocks, largest-first, capped at `limit`. */
+    groups: DuplicateGroup[];
+    /** How many hash buckets were skipped for exceeding `maxBucketSize` — a non-zero count means
+     *  results may under-report duplication that's unusually widespread (see `maxBucketSize`). */
+    skippedBuckets: number;
 }
 /**
  * @description Scans every file already present in `graph` for cross-file (and within-file)
@@ -530,13 +557,17 @@ interface FindDuplicatesOptions {
  *   token-shingle path only (CSS/Less/SCSS always match on literal declaration content);
  *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
  *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
- *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
- *   results. Lock files are always excluded, independent of `ignoreDirs`.
- * @returns Duplicate blocks (each tagged with its `family`), two or more occurrences per block —
- *   every block that pairwise chain-matches another is clustered into one group instead of one
- *   per pair — sorted largest-first across all families.
- */
-declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<DuplicateGroup[]>;
+ *   shared logic; `maxBucketSize` bounds worst-case scan time on large repos by skipping
+ *   pathologically common hash buckets; `ignoreDirs` excludes files under matching directory
+ *   names; `limit` caps results; `parallelTokenizing` offloads per-file tokenizing to a worker
+ *   pool once the candidate file count is large enough to be worth it. Lock files are always
+ *   excluded, independent of `ignoreDirs`.
+ * @returns `groups` — duplicate blocks (each tagged with its `family`), two or more occurrences
+ *   per block, every block that pairwise chain-matches another clustered into one group instead
+ *   of one per pair, sorted largest-first across all families — plus `skippedBuckets` (see
+ *   {@link FindDuplicatesResult}).
+ */
+declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<FindDuplicatesResult>;
 
 /** Controls how aggressively `detectFeatures` promotes files to features. */
 interface FeatureDetectionOptions {