@omfalos/mokosh 0.3.2 → 0.4.0

package/dist/duplication-worker.d.mts

@@ -0,0 +1,12 @@
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.mjs';
+import { F as FileType } from './parse-CAKctgb6.mjs';
+
+/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */
+
+declare function tokenizeInWorker(payload: {
+    source: string;
+    fileType: FileType;
+    ignoreLiterals: boolean;
+}): NormalizedToken[];
+
+export { tokenizeInWorker as default };

package/dist/duplication-worker.d.ts

@@ -0,0 +1,12 @@
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.js';
+import { F as FileType } from './parse-CAKctgb6.js';
+
+/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */
+
+declare function tokenizeInWorker(payload: {
+    source: string;
+    fileType: FileType;
+    ignoreLiterals: boolean;
+}): NormalizedToken[];
+
+export { tokenizeInWorker as default };

package/dist/duplication-worker.js

@@ -0,0 +1,5 @@
+"use strict";var p=Object.defineProperty;var u=Object.getOwnPropertyDescriptor;var h=Object.getOwnPropertyNames;var x=Object.prototype.hasOwnProperty;var b=(e,t)=>{for(var n in t)p(e,n,{get:t[n],enumerable:!0})},y=(e,t,n,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of h(t))!x.call(e,o)&&o!==n&&p(e,o,{get:()=>t[o],enumerable:!(i=u(t,o))||i.enumerable});return e};var N=e=>y(p({},"__esModule",{value:!0}),e);var $={};b($,{default:()=>T});module.exports=N($);var R={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function m(e,t,n){let i=e.slice(t,n).replace(/[^\n]/g," ");return e.slice(0,t)+i+e.slice(n)}function z(e,t){if(!t)return e;let n=e;for(let[i,o]of t.block??[]){let c=0;for(;;){let l=n.indexOf(i,c);if(l===-1)break;let r=n.indexOf(o,l+i.length),s=r===-1?n.length:r+o.length;n=m(n,l,s),c=s}}for(let i of t.line??[]){let o=0;for(;;){let c=n.indexOf(i,o);if(c===-1)break;let l=n.indexOf(`
+`,c),r=l===-1?n.length:l;n=m(n,c,r),o=r}}return n}var A=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],d=new RegExp(`${A.map(e=>e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),_=/^[A-Za-z_$][A-Za-z0-9_$]*$/,k=/^\d+(?:\.\d+)?$/,w=/^(".*"|'.*'|`.*`)$/s,S=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);function g(e,t,n=!0){let i=z(e,R[t]),o=[],c=1,l=0;d.lastIndex=0;let r=d.exec(i);for(;r!==null;){for(let a=l;a<r.index;a++)i[a]===`
+`&&c++;l=r.index;let s=r[0],f=s;_.test(s)&&!S.has(s)?f="ID":n&&(k.test(s)||w.test(s))&&(f=k.test(s)?"NUM":"STR"),o.push({text:f,line:c});for(let a=l;a<r.index+s.length;a++)i[a]===`
+`&&c++;l=r.index+s.length,r=d.exec(i)}return o}function T(e){return g(e.source,e.fileType,e.ignoreLiterals)}
+//# sourceMappingURL=duplication-worker.js.map

package/dist/duplication-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/duplication-worker.ts","../src/graph/duplication/tokenizer.ts"],"sourcesContent":["/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n","/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,\n * LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: identifiers always collapse to a single placeholder (so renamed-variable clones\n *   still hash identically), and literals optionally collapse too (`ignoreLiterals`, default\n *   on) so only structural shape — not the specific values used — drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment-stripping rule; unrecognised types skip stripping.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripComments(source, COMMENT_SYNTAX[fileType]);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n"],"mappings":"yaAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,aAAAE,IAAA,eAAAC,EAAAH,GC0BA,IAAMI,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAEA,IAAMO,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EAaM,SAASC,EACdpB,EACAqB,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWnB,EAAcJ,EAAQF,EAAeuB,CAAQ,CAAC,EACzDG,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBZ,EAAc,UAAY,EAC1B,IAAIa,EAAQb,EAAc,KAAKS,CAAQ,EAEvC,KAAOI,IAAU,MAAM,CACrB,QAASC,EAAIF,EAAWE,EAAID,EAAM,MAAOC,IACnCL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPb,EAAmB,KAAKa,CAAG,GAAK,CAACV,EAAS,IAAIU,CAAG,EACnDC,EAAO,KACER,IAAmBL,EAAe,KAAKY,CAAG,GAAKX,EAAe,KAAKW,CAAG,KAC/EC,EAAOb,EAAe,KAAKY,CAAG,EAAI,MAAQ,OAE5CL,EAAO,KAAK,CAAE,KAAAM,EAAM,KAAAL,CAAK,CAAC,EAE1B,QAASG,EAAIF,EAAWE,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAAQE,EAAI,OAC9BF,EAAQb,EAAc,KAAKS,CAAQ,CACrC,CAEA,OAAOC,CACT,CDvOe,SAARO,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["duplication_worker_exports","__export","tokenizeInWorker","__toCommonJS","COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","tokenize","fileType","ignoreLiterals","stripped","tokens","line","lastIndex","match","i","raw","text","tokenizeInWorker","payload","tokenize"]}

package/dist/duplication-worker.mjs

@@ -0,0 +1,5 @@
+var u={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function d(n,s,e){let i=n.slice(s,e).replace(/[^\n]/g," ");return n.slice(0,s)+i+n.slice(e)}function h(n,s){if(!s)return n;let e=n;for(let[i,c]of s.block??[]){let l=0;for(;;){let o=e.indexOf(i,l);if(o===-1)break;let t=e.indexOf(c,o+i.length),r=t===-1?e.length:t+c.length;e=d(e,o,r),l=r}}for(let i of s.line??[]){let c=0;for(;;){let l=e.indexOf(i,c);if(l===-1)break;let o=e.indexOf(`
+`,l),t=o===-1?e.length:o;e=d(e,l,t),c=t}}return e}var x=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],p=new RegExp(`${x.map(n=>n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),b=/^[A-Za-z_$][A-Za-z0-9_$]*$/,m=/^\d+(?:\.\d+)?$/,y=/^(".*"|'.*'|`.*`)$/s,N=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);function k(n,s,e=!0){let i=h(n,u[s]),c=[],l=1,o=0;p.lastIndex=0;let t=p.exec(i);for(;t!==null;){for(let a=o;a<t.index;a++)i[a]===`
+`&&l++;o=t.index;let r=t[0],f=r;b.test(r)&&!N.has(r)?f="ID":e&&(m.test(r)||y.test(r))&&(f=m.test(r)?"NUM":"STR"),c.push({text:f,line:l});for(let a=o;a<t.index+r.length;a++)i[a]===`
+`&&l++;o=t.index+r.length,t=p.exec(i)}return c}function E(n){return k(n.source,n.fileType,n.ignoreLiterals)}export{E as default};
+//# sourceMappingURL=duplication-worker.mjs.map

package/dist/duplication-worker.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/graph/duplication/tokenizer.ts","../src/duplication-worker.ts"],"sourcesContent":["/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, CoffeeScript,\n * LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: identifiers always collapse to a single placeholder (so renamed-variable clones\n *   still hash identically), and literals optionally collapse too (`ignoreLiterals`, default\n *   on) so only structural shape — not the specific values used — drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment-stripping rule; unrecognised types skip stripping.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripComments(source, COMMENT_SYNTAX[fileType]);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n","/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n"],"mappings":"AA0BA,IAAMA,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAEA,IAAMO,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EAaM,SAASC,EACdpB,EACAqB,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWnB,EAAcJ,EAAQF,EAAeuB,CAAQ,CAAC,EACzDG,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBZ,EAAc,UAAY,EAC1B,IAAIa,EAAQb,EAAc,KAAKS,CAAQ,EAEvC,KAAOI,IAAU,MAAM,CACrB,QAASC,EAAIF,EAAWE,EAAID,EAAM,MAAOC,IACnCL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPb,EAAmB,KAAKa,CAAG,GAAK,CAACV,EAAS,IAAIU,CAAG,EACnDC,EAAO,KACER,IAAmBL,EAAe,KAAKY,CAAG,GAAKX,EAAe,KAAKW,CAAG,KAC/EC,EAAOb,EAAe,KAAKY,CAAG,EAAI,MAAQ,OAE5CL,EAAO,KAAK,CAAE,KAAAM,EAAM,KAAAL,CAAK,CAAC,EAE1B,QAASG,EAAIF,EAAWE,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAAQE,EAAI,OAC9BF,EAAQb,EAAc,KAAKS,CAAQ,CACrC,CAEA,OAAOC,CACT,CCvOe,SAARO,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","tokenize","fileType","ignoreLiterals","stripped","tokens","line","lastIndex","match","i","raw","text","tokenizeInWorker","payload","tokenize"]}

package/dist/index.d.mts

@@ -1,5 +1,8 @@
-import { F as FileNode, C as CallEdge, I as ImportEdge, a as FileType, N as NodeCategory, P as ParseResult, S as StructuredTag } from './types-C9fLCS45.mjs';
-export { E as ExportedSymbol, b as ImportType, T as TagKind } from './types-C9fLCS45.mjs';
+import { F as FileNode, C as CallEdge, I as ImportEdge, P as ParseResult, S as StructuredTag } from './types-BtSqoqbZ.mjs';
+export { E as ExportedSymbol } from './types-BtSqoqbZ.mjs';
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.mjs';
+import { F as FileType, N as NodeCategory } from './parse-CAKctgb6.mjs';
+export { I as ImportType, T as TagKind } from './parse-CAKctgb6.mjs';
 
 interface SerializedGraph {
     nodes: FileNode[];
@@ -464,7 +467,8 @@ type DuplicateFamily = "style" | "code";
  * matches into one N-occurrence group instead of reporting C(N,2) near-identical pairs for a block
  * repeated N times. Shared by every language `tokenize()` supports — the shingling step itself has
  * no language awareness at all. See docs/adr-013-duplicate-detection-noise-reduction.md for the
- * noise this addresses.
+ * noise this addresses, and docs/adr-014-duplicate-detection-scale.md for why oversized hash
+ * buckets are capped below.
  */
 
 interface DuplicateOccurrence {
@@ -489,6 +493,29 @@ interface DuplicateGroup {
     family?: DuplicateFamily | undefined;
 }
 
+/** Configures whether/how tokenizing is offloaded to a `piscina` worker pool. `false` always
+ *  tokenizes in-process. */
+type ParallelTokenizingOption = boolean | {
+    minFiles?: number;
+    maxThreads?: number;
+};
+
+/** One file's cached tokenize result, fingerprinted by `mtime`/`size`/`ignoreLiterals` — any
+ *  mismatch against the current `FileNode` (or against the `ignoreLiterals` this scan is running
+ *  with) means the entry is stale and must be recomputed, exactly like `GraphBuilder`'s
+ *  mtime+size node reuse for incremental graph builds. */
+interface CachedFileTokens {
+    mtime: number;
+    size: number;
+    ignoreLiterals: boolean;
+    tokens: NormalizedToken[];
+}
+/** Caller-owned cache, keyed by project-relative path, reused across repeated `findDuplicates`
+ *  calls against the same root (e.g. successive MCP tool calls in one session) so unchanged files
+ *  never pay tokenizing cost twice. `findDuplicates` itself is stateless — callers that want this
+ *  benefit own the `Map` and pass it in; the CLI's one-shot process has nothing to gain and omits
+ *  it. See docs/adr-014-duplicate-detection-scale.md. */
+type DuplicationTokenCache = Map<string, CachedFileTokens>;
 interface FindDuplicatesOptions {
     /** Minimum duplicated block size, in source lines, to report (default 6). */
     minLines?: number | undefined;
@@ -509,6 +536,20 @@ interface FindDuplicatesOptions {
     /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
      *  — `node_modules`, `dist`, `.git`, `mokosh-cache`, `coverage`, etc.). Pass `[]` to disable. */
     ignoreDirs?: readonly string[] | undefined;
+    /** Controls worker-pool offloading of per-file tokenizing (default `true`): offloads once the
+     *  candidate file count reaches `minFiles` (default 20, matching `GraphBuilder`'s parse pool);
+     *  `false` always tokenizes in-process; an object overrides `minFiles`/`maxThreads`. See
+     *  docs/adr-014-duplicate-detection-scale.md. */
+    parallelTokenizing?: ParallelTokenizingOption | undefined;
+    /** Optional caller-owned cache reused across calls against the same root — files whose
+     *  `mtime`/`size` are unchanged since the cached entry (and whose `ignoreLiterals` matches this
+     *  call's) skip tokenizing entirely. Mutated in place; omit for one-shot callers (e.g. the CLI).
+     *  See {@link DuplicationTokenCache} and docs/adr-014-duplicate-detection-scale.md. */
+    tokenCache?: DuplicationTokenCache | undefined;
+}
+interface FindDuplicatesResult {
+    /** Duplicate blocks, largest-first, capped at `limit`. */
+    groups: DuplicateGroup[];
 }
 /**
  * @description Scans every file already present in `graph` for cross-file (and within-file)
@@ -531,12 +572,14 @@ interface FindDuplicatesOptions {
  *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
  *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
  *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
- *   results. Lock files are always excluded, independent of `ignoreDirs`.
- * @returns Duplicate blocks (each tagged with its `family`), two or more occurrences per block —
- *   every block that pairwise chain-matches another is clustered into one group instead of one
- *   per pair — sorted largest-first across all families.
- */
-declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<DuplicateGroup[]>;
+ *   results; `parallelTokenizing` offloads per-file tokenizing to a worker pool once the
+ *   candidate file count is large enough to be worth it. Lock files are always excluded,
+ *   independent of `ignoreDirs`.
+ * @returns `groups` — duplicate blocks (each tagged with its `family`), two or more occurrences
+ *   per block, every block that pairwise chain-matches another clustered into one group instead
+ *   of one per pair, sorted largest-first across all families.
+ */
+declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<FindDuplicatesResult>;
 
 /** Controls how aggressively `detectFeatures` promotes files to features. */
 interface FeatureDetectionOptions {
@@ -1399,4 +1442,4 @@ declare function createWorkspaceGraph(rootDir: string, options?: {
  */
 declare function getAllProjectFiles(rootDir: string, options?: ScanOptions): string[];
 
-export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };
+export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, type CachedFileTokens, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, type DuplicationTokenCache, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FindDuplicatesResult, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };

package/dist/index.d.ts

@@ -1,5 +1,8 @@
-import { F as FileNode, C as CallEdge, I as ImportEdge, a as FileType, N as NodeCategory, P as ParseResult, S as StructuredTag } from './types-C9fLCS45.js';
-export { E as ExportedSymbol, b as ImportType, T as TagKind } from './types-C9fLCS45.js';
+import { F as FileNode, C as CallEdge, I as ImportEdge, P as ParseResult, S as StructuredTag } from './types-BlN-U5AM.js';
+export { E as ExportedSymbol } from './types-BlN-U5AM.js';
+import { N as NormalizedToken } from './tokenizer-BaF1eUBQ.js';
+import { F as FileType, N as NodeCategory } from './parse-CAKctgb6.js';
+export { I as ImportType, T as TagKind } from './parse-CAKctgb6.js';
 
 interface SerializedGraph {
     nodes: FileNode[];
@@ -464,7 +467,8 @@ type DuplicateFamily = "style" | "code";
  * matches into one N-occurrence group instead of reporting C(N,2) near-identical pairs for a block
  * repeated N times. Shared by every language `tokenize()` supports — the shingling step itself has
  * no language awareness at all. See docs/adr-013-duplicate-detection-noise-reduction.md for the
- * noise this addresses.
+ * noise this addresses, and docs/adr-014-duplicate-detection-scale.md for why oversized hash
+ * buckets are capped below.
  */
 
 interface DuplicateOccurrence {
@@ -489,6 +493,29 @@ interface DuplicateGroup {
     family?: DuplicateFamily | undefined;
 }
 
+/** Configures whether/how tokenizing is offloaded to a `piscina` worker pool. `false` always
+ *  tokenizes in-process. */
+type ParallelTokenizingOption = boolean | {
+    minFiles?: number;
+    maxThreads?: number;
+};
+
+/** One file's cached tokenize result, fingerprinted by `mtime`/`size`/`ignoreLiterals` — any
+ *  mismatch against the current `FileNode` (or against the `ignoreLiterals` this scan is running
+ *  with) means the entry is stale and must be recomputed, exactly like `GraphBuilder`'s
+ *  mtime+size node reuse for incremental graph builds. */
+interface CachedFileTokens {
+    mtime: number;
+    size: number;
+    ignoreLiterals: boolean;
+    tokens: NormalizedToken[];
+}
+/** Caller-owned cache, keyed by project-relative path, reused across repeated `findDuplicates`
+ *  calls against the same root (e.g. successive MCP tool calls in one session) so unchanged files
+ *  never pay tokenizing cost twice. `findDuplicates` itself is stateless — callers that want this
+ *  benefit own the `Map` and pass it in; the CLI's one-shot process has nothing to gain and omits
+ *  it. See docs/adr-014-duplicate-detection-scale.md. */
+type DuplicationTokenCache = Map<string, CachedFileTokens>;
 interface FindDuplicatesOptions {
     /** Minimum duplicated block size, in source lines, to report (default 6). */
     minLines?: number | undefined;
@@ -509,6 +536,20 @@ interface FindDuplicatesOptions {
     /** Directory names to exclude, matched against any path segment (default `DEFAULT_IGNORE_DIRS`
      *  — `node_modules`, `dist`, `.git`, `mokosh-cache`, `coverage`, etc.). Pass `[]` to disable. */
     ignoreDirs?: readonly string[] | undefined;
+    /** Controls worker-pool offloading of per-file tokenizing (default `true`): offloads once the
+     *  candidate file count reaches `minFiles` (default 20, matching `GraphBuilder`'s parse pool);
+     *  `false` always tokenizes in-process; an object overrides `minFiles`/`maxThreads`. See
+     *  docs/adr-014-duplicate-detection-scale.md. */
+    parallelTokenizing?: ParallelTokenizingOption | undefined;
+    /** Optional caller-owned cache reused across calls against the same root — files whose
+     *  `mtime`/`size` are unchanged since the cached entry (and whose `ignoreLiterals` matches this
+     *  call's) skip tokenizing entirely. Mutated in place; omit for one-shot callers (e.g. the CLI).
+     *  See {@link DuplicationTokenCache} and docs/adr-014-duplicate-detection-scale.md. */
+    tokenCache?: DuplicationTokenCache | undefined;
+}
+interface FindDuplicatesResult {
+    /** Duplicate blocks, largest-first, capped at `limit`. */
+    groups: DuplicateGroup[];
 }
 /**
  * @description Scans every file already present in `graph` for cross-file (and within-file)
@@ -531,12 +572,14 @@ interface FindDuplicatesOptions {
  *   `maxPunctuationRatio` gates out token-shingle blocks that are mostly object/array-literal
  *   structural punctuation (e.g. schema/object-literal boilerplate) rather than substantive
  *   shared logic; `ignoreDirs` excludes files under matching directory names; `limit` caps
- *   results. Lock files are always excluded, independent of `ignoreDirs`.
- * @returns Duplicate blocks (each tagged with its `family`), two or more occurrences per block —
- *   every block that pairwise chain-matches another is clustered into one group instead of one
- *   per pair — sorted largest-first across all families.
- */
-declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<DuplicateGroup[]>;
+ *   results; `parallelTokenizing` offloads per-file tokenizing to a worker pool once the
+ *   candidate file count is large enough to be worth it. Lock files are always excluded,
+ *   independent of `ignoreDirs`.
+ * @returns `groups` — duplicate blocks (each tagged with its `family`), two or more occurrences
+ *   per block, every block that pairwise chain-matches another clustered into one group instead
+ *   of one per pair, sorted largest-first across all families.
+ */
+declare function findDuplicates(graph: Graph, rootDir: string, options?: FindDuplicatesOptions): Promise<FindDuplicatesResult>;
 
 /** Controls how aggressively `detectFeatures` promotes files to features. */
 interface FeatureDetectionOptions {
@@ -1399,4 +1442,4 @@ declare function createWorkspaceGraph(rootDir: string, options?: {
  */
 declare function getAllProjectFiles(rootDir: string, options?: ScanOptions): string[];
 
-export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };
+export { type ApiSurface, type ApplyTagsFileResult, type ApplyTagsResult, CALL_EDGE_TYPES, type CachedFileTokens, CallEdge, type CalleeEntry, type CallerEntry$1 as CallerEntry, type ChangeImpactCache, type ComplexFunctionEntry, DEFAULT_EXTENSIONS, DEFAULT_IGNORE_DIRS, type DependencyGraph, type DuplicateFamily, type DuplicateGroup, type DuplicateOccurrence, type DuplicationTokenCache, EXPORT_TRACKING_TYPES, type ExportKind, type FeatureDetectionOptions, type FeatureDomain, type FeatureGraph, type FeatureGraphOptions, type FeatureInfo, FileNode, FileType, type FindComplexFunctionsOptions, type FindDuplicatesOptions, type FindDuplicatesResult, type FunctionCallInfo, type GetAffectedOptions, type GetCallersOptions, Graph, type CallerEntry as GraphCallerEntry, type GraphExporter, IMPORT_SYMBOL_TYPES, ImportEdge, type LanguageCoverage, MermaidExporter, type ModuleResponsibility, type ModuleRole, type MokoshConfig, type MonorepoDetector, type MonorepoLayout, NodeCategory, type NodeMeta, type NodeQuery, type ParallelParsingOption, type PathWithSymbols, type ProposeTagsOptions, type PublicExport, type ResponsibilityGraph, type ScanOptions, type SerializedGraph, type SerializedWorkspaceGraph, type SlimNode, type SlimSerializedGraph, StructuredTag, type SymbolCaller, type SymbolImporter, type SymbolMatch, type SymbolPrecision, SymbolTraversalContext, type TestNodeIdentifier, type TraversalOptions, type TraversalVisitor, type TypeEdge, type TypeGraph, type TypeKind, type TypeNode, type TypeQueryResult, WorkspaceGraph, type WorkspacePackage, type WorkspacePackageSummary, type WorkspacePackagesSummary, applyConfig, applyTags, buildApiSurface, buildChangeImpactCache, buildFeatureGraph, buildResponsibilityGraph, buildTypeGraph, computeGraphHash, configToGraphOptions, createImportMap, createWorkspaceGraph, detectAllEntryPoints, detectEntryPoint, detectFeatures, detectMonorepo, filterGraph, findComplexFunctions, findDuplicates, findSymbol, getAffected, getAllProjectFiles, getCallers, getDependencies, getDependents, getLanguageCoverage, getNodeMeta, hasCoverageData, isChangeImpactCacheValid, loadChangeImpactCache, loadCoverageMap, loadMokoshConfig, parseQuery, proposeAffectedTests, proposeTags, queryCallGraph, queryChangeImpact, queryTypeGraph, registerConfigMatcher, registerMonorepoDetector, registerParser, registerTestLibrary, registerTestPattern, saveChangeImpactCache, slimSerialize, summarizeWorkspacePackages, toMermaid };