@omfalos/mokosh 0.5.0 → 0.5.2

package/dist/duplication-worker.d.mts

@@ -1,4 +1,4 @@
-import { N as NormalizedToken } from './tokenizer-DIbb0GLe.mjs';
+import { N as NormalizedToken } from './tokenizer-D9P-y7lH.mjs';
 import { F as FileType } from './parse-yamNoaoL.mjs';
 
 /** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */

package/dist/duplication-worker.d.ts

@@ -1,4 +1,4 @@
-import { N as NormalizedToken } from './tokenizer-DIbb0GLe.js';
+import { N as NormalizedToken } from './tokenizer-D9P-y7lH.js';
 import { F as FileType } from './parse-yamNoaoL.js';
 
 /** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */

package/dist/duplication-worker.js

@@ -1,5 +1,7 @@
-"use strict";var p=Object.defineProperty;var u=Object.getOwnPropertyDescriptor;var b=Object.getOwnPropertyNames;var h=Object.prototype.hasOwnProperty;var x=(e,t)=>{for(var n in t)p(e,n,{get:t[n],enumerable:!0})},y=(e,t,n,i)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of b(t))!h.call(e,o)&&o!==n&&p(e,o,{get:()=>t[o],enumerable:!(i=u(t,o))||i.enumerable});return e};var N=e=>y(p({},"__esModule",{value:!0}),e);var $={};x($,{default:()=>T});module.exports=N($);var R={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},java:{line:["//"],block:[["/*","*/"]]},kotlin:{line:["//"],block:[["/*","*/"]]},scala:{line:["//"],block:[["/*","*/"]]},groovy:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function d(e,t,n){let i=e.slice(t,n).replace(/[^\n]/g," ");return e.slice(0,t)+i+e.slice(n)}function z(e,t){if(!t)return e;let n=e;for(let[i,o]of t.block??[]){let c=0;for(;;){let r=n.indexOf(i,c);if(r===-1)break;let l=n.indexOf(o,r+i.length),s=l===-1?n.length:l+o.length;n=d(n,r,s),c=s}}for(let i of t.line??[]){let o=0;for(;;){let c=n.indexOf(i,o);if(c===-1)break;let r=n.indexOf(`
-`,c),l=r===-1?n.length:r;n=d(n,c,l),o=l}}return n}var A=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],k=new RegExp(`${A.map(e=>e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),_=/^[A-Za-z_$][A-Za-z0-9_$]*$/,m=/^\d+(?:\.\d+)?$/,w=/^(".*"|'.*'|`.*`)$/s,S=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);function g(e,t,n=!0){let i=z(e,R[t]),o=[],c=1,r=0;k.lastIndex=0;let l=k.exec(i);for(;l!==null;){for(let a=r;a<l.index;a++)i[a]===`
-`&&c++;r=l.index;let s=l[0],f=s;_.test(s)&&!S.has(s)?f="ID":n&&(m.test(s)||w.test(s))&&(f=m.test(s)?"NUM":"STR"),o.push({text:f,line:c});for(let a=r;a<l.index+s.length;a++)i[a]===`
-`&&c++;r=l.index+s.length,l=k.exec(i)}return o}function T(e){return g(e.source,e.fileType,e.ignoreLiterals)}
+"use strict";var p=Object.defineProperty;var h=Object.getOwnPropertyDescriptor;var R=Object.getOwnPropertyNames;var b=Object.prototype.hasOwnProperty;var y=(e,n)=>{for(var t in n)p(e,t,{get:n[t],enumerable:!0})},E=(e,n,t,s)=>{if(n&&typeof n=="object"||typeof n=="function")for(let r of R(n))!b.call(e,r)&&r!==t&&p(e,r,{get:()=>n[r],enumerable:!(s=h(n,r))||s.enumerable});return e};var N=e=>E(p({},"__esModule",{value:!0}),e);var F={};y(F,{default:()=>T});module.exports=N(F);var x={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},java:{line:["//"],block:[["/*","*/"]]},kotlin:{line:["//"],block:[["/*","*/"]]},scala:{line:["//"],block:[["/*","*/"]]},groovy:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function u(e,n,t){let s=e.slice(n,t).replace(/[^\n]/g," ");return e.slice(0,n)+s+e.slice(t)}function _(e,n){if(!n)return e;let t=e;for(let[s,r]of n.block??[]){let c=0;for(;;){let l=t.indexOf(s,c);if(l===-1)break;let o=t.indexOf(r,l+s.length),i=o===-1?t.length:o+r.length;t=u(t,l,i),c=i}}for(let s of n.line??[]){let r=0;for(;;){let c=t.indexOf(s,r);if(c===-1)break;let l=t.indexOf(`
+`,c),o=l===-1?t.length:l;t=u(t,c,o),r=o}}return t}var A={typescript:[/^\s*import[\s{("*]/,/^\s*export\s.*\sfrom\s/],javascript:[/^\s*import[\s{("*]/,/^\s*export\s.*\sfrom\s/],python:[/^\s*import\s+\S/,/^\s*from\s+\S+\s+import[\s(]/],go:[/^\s*import\s+["(]/],java:[/^\s*import\s+/],kotlin:[/^\s*import\s+/],scala:[/^\s*import\s+/],groovy:[/^\s*import\s+/]};function I(e,n){let t=A[n];if(!t)return e;let s=e.split(`
+`),r=n==="go",c=!1,l=!1;for(let o=0;o<s.length;o++){let i=s[o],f=()=>{s[o]=i.replace(/[^\r]/g," ")};if(c){f(),i.includes(")")&&(c=!1);continue}if(l){f(),(/}\s*from\s/.test(i)||i.includes("}"))&&(l=!1);continue}t.some(a=>a.test(i))&&(f(),r&&/^\s*import\s+\(/.test(i)&&!i.includes(")")?c=!0:(n==="typescript"||n==="javascript")&&/^\s*import\b/.test(i)&&i.includes("{")&&!i.includes("}")&&(l=!0))}return s.join(`
+`)}var k=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],m=new RegExp(`${k.map(e=>e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),O=/^[A-Za-z_$][A-Za-z0-9_$]*$/,g=/^\d+(?:\.\d+)?$/,z=/^(".*"|'.*'|`.*`)$/s,w=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);var $=new Set(k);function d(e,n,t=!0){let s=I(_(e,x[n]),n),r=[],c=1,l=0;m.lastIndex=0;let o=m.exec(s);for(;o!==null;){for(let a=l;a<o.index;a++)s[a]===`
+`&&c++;l=o.index;let i=o[0],f=i;O.test(i)&&!w.has(i)?f="ID":t&&(g.test(i)||z.test(i))&&(f=g.test(i)?"NUM":"STR"),r.push({text:f,line:c});for(let a=l;a<o.index+i.length;a++)s[a]===`
+`&&c++;l=o.index+i.length,o=m.exec(s)}return r}function T(e){return d(e.source,e.fileType,e.ignoreLiterals)}
 //# sourceMappingURL=duplication-worker.js.map

package/dist/duplication-worker.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/duplication-worker.ts","../src/graph/duplication/tokenizer.ts"],"sourcesContent":["/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n","/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, Java, Kotlin, Scala,\n * Groovy, CoffeeScript, LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  java: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  kotlin: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  scala: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  groovy: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: identifiers always collapse to a single placeholder (so renamed-variable clones\n *   still hash identically), and literals optionally collapse too (`ignoreLiterals`, default\n *   on) so only structural shape — not the specific values used — drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment-stripping rule; unrecognised types skip stripping.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripComments(source, COMMENT_SYNTAX[fileType]);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n"],"mappings":"yaAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,aAAAE,IAAA,eAAAC,EAAAH,GC0BA,IAAMI,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,MAAO,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAEA,IAAMO,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EAaM,SAASC,EACdpB,EACAqB,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWnB,EAAcJ,EAAQF,EAAeuB,CAAQ,CAAC,EACzDG,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBZ,EAAc,UAAY,EAC1B,IAAIa,EAAQb,EAAc,KAAKS,CAAQ,EAEvC,KAAOI,IAAU,MAAM,CACrB,QAASC,EAAIF,EAAWE,EAAID,EAAM,MAAOC,IACnCL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPb,EAAmB,KAAKa,CAAG,GAAK,CAACV,EAAS,IAAIU,CAAG,EACnDC,EAAO,KACER,IAAmBL,EAAe,KAAKY,CAAG,GAAKX,EAAe,KAAKW,CAAG,KAC/EC,EAAOb,EAAe,KAAKY,CAAG,EAAI,MAAQ,OAE5CL,EAAO,KAAK,CAAE,KAAAM,EAAM,KAAAL,CAAK,CAAC,EAE1B,QAASG,EAAIF,EAAWE,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAAQE,EAAI,OAC9BF,EAAQb,EAAc,KAAKS,CAAQ,CACrC,CAEA,OAAOC,CACT,CD3Oe,SAARO,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["duplication_worker_exports","__export","tokenizeInWorker","__toCommonJS","COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","tokenize","fileType","ignoreLiterals","stripped","tokens","line","lastIndex","match","i","raw","text","tokenizeInWorker","payload","tokenize"]}
+{"version":3,"sources":["../src/duplication-worker.ts","../src/graph/duplication/tokenizer.ts"],"sourcesContent":["/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n","/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax and import/using statements, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, Java, Kotlin, Scala,\n * Groovy, CoffeeScript, LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  java: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  kotlin: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  scala: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  groovy: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\n/**\n * Per-`FileType` patterns matching a *single* import/using line. Long shared import lists\n * tokenize identically across unrelated files and register as \"duplicates\" that carry no shared\n * logic — same rationale as stripping comments. Masked (space-filled, line breaks kept) before\n * tokenizing. Deliberately coarse and not string-literal-aware, like `stripComments`.\n */\nconst IMPORT_LINE_PATTERNS: Partial<Record<FileType, RegExp[]>> = {\n  typescript: [/^\\s*import[\\s{(\"*]/, /^\\s*export\\s.*\\sfrom\\s/],\n  javascript: [/^\\s*import[\\s{(\"*]/, /^\\s*export\\s.*\\sfrom\\s/],\n  python: [/^\\s*import\\s+\\S/, /^\\s*from\\s+\\S+\\s+import[\\s(]/],\n  go: [/^\\s*import\\s+[\"(]/],\n  java: [/^\\s*import\\s+/],\n  kotlin: [/^\\s*import\\s+/],\n  scala: [/^\\s*import\\s+/],\n  groovy: [/^\\s*import\\s+/],\n};\n\n/**\n * @description Masks import / `from … import` / `using` statements to spaces before tokenizing,\n *   preserving line breaks so later line-number tracking stays accurate. Handles the two\n *   multi-line shapes that matter in practice: a JS/TS `import { … } from \"x\"` whose braces span\n *   lines, and a Go `import ( … )` block. Everything else is matched line-by-line.\n * @param source - File source, already comment-stripped.\n * @param fileType - Selects the import-line patterns; unrecognised types pass through unchanged.\n * @returns The source with import statements masked to spaces.\n */\nfunction stripImports(source: string, fileType: FileType): string {\n  const patterns = IMPORT_LINE_PATTERNS[fileType];\n  if (!patterns) return source;\n\n  const lines = source.split(\"\\n\");\n  const isGo = fileType === \"go\";\n  let inGoBlock = false;\n  let inJsBraces = false;\n\n  for (let i = 0; i < lines.length; i++) {\n    const line = lines[i] as string;\n    const mask = () => {\n      lines[i] = line.replace(/[^\\r]/g, \" \");\n    };\n\n    if (inGoBlock) {\n      mask();\n      if (line.includes(\")\")) inGoBlock = false;\n      continue;\n    }\n    if (inJsBraces) {\n      mask();\n      if (/}\\s*from\\s/.test(line) || line.includes(\"}\")) inJsBraces = false;\n      continue;\n    }\n    if (!patterns.some((pattern) => pattern.test(line))) continue;\n\n    mask();\n    if (isGo && /^\\s*import\\s+\\(/.test(line) && !line.includes(\")\")) inGoBlock = true;\n    else if (\n      (fileType === \"typescript\" || fileType === \"javascript\") &&\n      /^\\s*import\\b/.test(line) &&\n      line.includes(\"{\") &&\n      !line.includes(\"}\")\n    ) {\n      inJsBraces = true;\n    }\n  }\n\n  return lines.join(\"\\n\");\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/** Single-char tokens that carry computation — arithmetic, bitwise, logical-not, optional. The\n *  omissions are deliberate: `( ) { } [ ] ; , : .` are pure structure, and `= < > /` are excluded\n *  because they overwhelmingly appear as JSX attribute/tag punctuation (`attr={…}`, `<Tag/>`)\n *  rather than as assignment or comparison — counting them would let an icon-component wrapper\n *  score like real logic. Multi-char forms (`===`, `<=`, `=>`, …) are caught via\n *  {@link MULTI_CHAR_OPERATORS} instead and stay significant. */\nconst LOGIC_OPERATOR_CHARS = new Set([\"+\", \"-\", \"*\", \"%\", \"&\", \"|\", \"!\", \"^\", \"~\", \"?\"]);\nconst MULTI_CHAR_OPERATOR_SET: ReadonlySet<string> = new Set(MULTI_CHAR_OPERATORS);\n\n/**\n * @description Whether a normalized token text is \"logic-bearing\" — a language keyword or an\n *   operator — as opposed to a normalized identifier/literal (`ID`/`NUM`/`STR`) or structural\n *   punctuation. Used to score a duplicate block by how much of it is actual computation rather\n *   than boilerplate shape (a Flow `type Props` block, a JSX icon wrapper, a schema object all\n *   have very few of these). See `docs/adr-019-logic-token-scoring.md`.\n * @param text - A {@link NormalizedToken}'s `text`.\n * @returns `true` for keywords and operators; `false` for `ID`/`NUM`/`STR` and `( ) { } [ ] ; , : . = < > /`.\n */\nexport function isSignificantToken(text: string): boolean {\n  if (text === \"ID\" || text === \"NUM\" || text === \"STR\") return false;\n  if (KEYWORDS.has(text)) return true;\n  if (text.length === 1) return LOGIC_OPERATOR_CHARS.has(text);\n  return MULTI_CHAR_OPERATOR_SET.has(text);\n}\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: comments and import/using statements are masked out first (shared import lists are\n *   not shared logic), then identifiers always collapse to a single placeholder (so\n *   renamed-variable clones still hash identically), and literals optionally collapse too\n *   (`ignoreLiterals`, default on) so only structural shape — not the specific values used —\n *   drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment- and import-stripping rules; unrecognised types skip both.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripImports(stripComments(source, COMMENT_SYNTAX[fileType]), fileType);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n"],"mappings":"yaAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,aAAAE,IAAA,eAAAC,EAAAH,GC0BA,IAAMI,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,MAAO,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAQA,IAAMO,EAA4D,CAChE,WAAY,CAAC,qBAAsB,wBAAwB,EAC3D,WAAY,CAAC,qBAAsB,wBAAwB,EAC3D,OAAQ,CAAC,kBAAmB,8BAA8B,EAC1D,GAAI,CAAC,mBAAmB,EACxB,KAAM,CAAC,eAAe,EACtB,OAAQ,CAAC,eAAe,EACxB,MAAO,CAAC,eAAe,EACvB,OAAQ,CAAC,eAAe,CAC1B,EAWA,SAASC,EAAad,EAAgBe,EAA4B,CAChE,IAAMC,EAAWH,EAAqBE,CAAQ,EAC9C,GAAI,CAACC,EAAU,OAAOhB,EAEtB,IAAMiB,EAAQjB,EAAO,MAAM;AAAA,CAAI,EACzBkB,EAAOH,IAAa,KACtBI,EAAY,GACZC,EAAa,GAEjB,QAASC,EAAI,EAAGA,EAAIJ,EAAM,OAAQI,IAAK,CACrC,IAAMC,EAAOL,EAAMI,CAAC,EACdE,EAAO,IAAM,CACjBN,EAAMI,CAAC,EAAIC,EAAK,QAAQ,SAAU,GAAG,CACvC,EAEA,GAAIH,EAAW,CACbI,EAAK,EACDD,EAAK,SAAS,GAAG,IAAGH,EAAY,IACpC,QACF,CACA,GAAIC,EAAY,CACdG,EAAK,GACD,aAAa,KAAKD,CAAI,GAAKA,EAAK,SAAS,GAAG,KAAGF,EAAa,IAChE,QACF,CACKJ,EAAS,KAAMQ,GAAYA,EAAQ,KAAKF,CAAI,CAAC,IAElDC,EAAK,EACDL,GAAQ,kBAAkB,KAAKI,CAAI,GAAK,CAACA,EAAK,SAAS,GAAG,EAAGH,EAAY,IAE1EJ,IAAa,cAAgBA,IAAa,eAC3C,eAAe,KAAKO,CAAI,GACxBA,EAAK,SAAS,GAAG,GACjB,CAACA,EAAK,SAAS,GAAG,IAElBF,EAAa,IAEjB,CAEA,OAAOH,EAAM,KAAK;AAAA,CAAI,CACxB,CAEA,IAAMQ,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EASD,IAAMC,EAA+C,IAAI,IAAIC,CAAoB,EA+B1E,SAASC,EACdC,EACAC,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWC,EAAaC,EAAcL,EAAQM,EAAeL,CAAQ,CAAC,EAAGA,CAAQ,EACjFM,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBC,EAAc,UAAY,EAC1B,IAAIC,EAAQD,EAAc,KAAKP,CAAQ,EAEvC,KAAOQ,IAAU,MAAM,CACrB,QAASC,EAAIH,EAAWG,EAAID,EAAM,MAAOC,IACnCT,EAASS,CAAC,IAAM;AAAA,GAAMJ,IAE5BC,EAAYE,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPE,EAAmB,KAAKF,CAAG,GAAK,CAACG,EAAS,IAAIH,CAAG,EACnDC,EAAO,KACEZ,IAAmBe,EAAe,KAAKJ,CAAG,GAAKK,EAAe,KAAKL,CAAG,KAC/EC,EAAOG,EAAe,KAAKJ,CAAG,EAAI,MAAQ,OAE5CN,EAAO,KAAK,CAAE,KAAAO,EAAM,KAAAN,CAAK,CAAC,EAE1B,QAASI,EAAIH,EAAWG,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDT,EAASS,CAAC,IAAM;AAAA,GAAMJ,IAE5BC,EAAYE,EAAM,MAAQE,EAAI,OAC9BF,EAAQD,EAAc,KAAKP,CAAQ,CACrC,CAEA,OAAOI,CACT,CD1Ue,SAARY,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["duplication_worker_exports","__export","tokenizeInWorker","__toCommonJS","COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","IMPORT_LINE_PATTERNS","stripImports","fileType","patterns","lines","isGo","inGoBlock","inJsBraces","i","line","mask","pattern","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","MULTI_CHAR_OPERATOR_SET","MULTI_CHAR_OPERATORS","tokenize","source","fileType","ignoreLiterals","stripped","stripImports","stripComments","COMMENT_SYNTAX","tokens","line","lastIndex","TOKEN_PATTERN","match","i","raw","text","IDENTIFIER_PATTERN","KEYWORDS","NUMBER_PATTERN","STRING_PATTERN","tokenizeInWorker","payload","tokenize"]}

package/dist/duplication-worker.mjs

@@ -1,5 +1,7 @@
-var u={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},java:{line:["//"],block:[["/*","*/"]]},kotlin:{line:["//"],block:[["/*","*/"]]},scala:{line:["//"],block:[["/*","*/"]]},groovy:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function k(n,s,e){let i=n.slice(s,e).replace(/[^\n]/g," ");return n.slice(0,s)+i+n.slice(e)}function b(n,s){if(!s)return n;let e=n;for(let[i,c]of s.block??[]){let r=0;for(;;){let o=e.indexOf(i,r);if(o===-1)break;let t=e.indexOf(c,o+i.length),l=t===-1?e.length:t+c.length;e=k(e,o,l),r=l}}for(let i of s.line??[]){let c=0;for(;;){let r=e.indexOf(i,c);if(r===-1)break;let o=e.indexOf(`
-`,r),t=o===-1?e.length:o;e=k(e,r,t),c=t}}return e}var h=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],p=new RegExp(`${h.map(n=>n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),x=/^[A-Za-z_$][A-Za-z0-9_$]*$/,d=/^\d+(?:\.\d+)?$/,y=/^(".*"|'.*'|`.*`)$/s,N=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);function m(n,s,e=!0){let i=b(n,u[s]),c=[],r=1,o=0;p.lastIndex=0;let t=p.exec(i);for(;t!==null;){for(let a=o;a<t.index;a++)i[a]===`
-`&&r++;o=t.index;let l=t[0],f=l;x.test(l)&&!N.has(l)?f="ID":e&&(d.test(l)||y.test(l))&&(f=d.test(l)?"NUM":"STR"),c.push({text:f,line:r});for(let a=o;a<t.index+l.length;a++)i[a]===`
-`&&r++;o=t.index+l.length,t=p.exec(i)}return c}function E(n){return m(n.source,n.fileType,n.ignoreLiterals)}export{E as default};
+var h={typescript:{line:["//"],block:[["/*","*/"]]},javascript:{line:["//"],block:[["/*","*/"]]},go:{line:["//"],block:[["/*","*/"]]},java:{line:["//"],block:[["/*","*/"]]},kotlin:{line:["//"],block:[["/*","*/"]]},scala:{line:["//"],block:[["/*","*/"]]},groovy:{line:["//"],block:[["/*","*/"]]},css:{block:[["/*","*/"]]},scss:{line:["//"],block:[["/*","*/"]]},less:{line:["//"],block:[["/*","*/"]]},stylus:{line:["//"],block:[["/*","*/"]]},python:{line:["#"]},gherkin:{line:["#"]},coffeescript:{line:["#"],block:[["###","###"]]},livescript:{line:["#"],block:[["###","###"]]},lua:{line:["--"],block:[["--[[","]]"]]},markdown:{block:[["<!--","-->"]]}};function m(i,r,n){let s=i.slice(r,n).replace(/[^\n]/g," ");return i.slice(0,r)+s+i.slice(n)}function R(i,r){if(!r)return i;let n=i;for(let[s,c]of r.block??[]){let l=0;for(;;){let o=n.indexOf(s,l);if(o===-1)break;let t=n.indexOf(c,o+s.length),e=t===-1?n.length:t+c.length;n=m(n,o,e),l=e}}for(let s of r.line??[]){let c=0;for(;;){let l=n.indexOf(s,c);if(l===-1)break;let o=n.indexOf(`
+`,l),t=o===-1?n.length:o;n=m(n,l,t),c=t}}return n}var b={typescript:[/^\s*import[\s{("*]/,/^\s*export\s.*\sfrom\s/],javascript:[/^\s*import[\s{("*]/,/^\s*export\s.*\sfrom\s/],python:[/^\s*import\s+\S/,/^\s*from\s+\S+\s+import[\s(]/],go:[/^\s*import\s+["(]/],java:[/^\s*import\s+/],kotlin:[/^\s*import\s+/],scala:[/^\s*import\s+/],groovy:[/^\s*import\s+/]};function y(i,r){let n=b[r];if(!n)return i;let s=i.split(`
+`),c=r==="go",l=!1,o=!1;for(let t=0;t<s.length;t++){let e=s[t],f=()=>{s[t]=e.replace(/[^\r]/g," ")};if(l){f(),e.includes(")")&&(l=!1);continue}if(o){f(),(/}\s*from\s/.test(e)||e.includes("}"))&&(o=!1);continue}n.some(a=>a.test(e))&&(f(),c&&/^\s*import\s+\(/.test(e)&&!e.includes(")")?l=!0:(r==="typescript"||r==="javascript")&&/^\s*import\b/.test(e)&&e.includes("{")&&!e.includes("}")&&(o=!0))}return s.join(`
+`)}var g=["===","!==","...","=>","==","!=","<=",">=","&&","||","::","->","..","+=","-=","*=","/="],p=new RegExp(`${g.map(i=>i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")).join("|")}|[A-Za-z_$][A-Za-z0-9_$]*|\\d+(?:\\.\\d+)?|"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'|\`(?:\\\\.|[^\`\\\\])*\`|\\S`,"g"),E=/^[A-Za-z_$][A-Za-z0-9_$]*$/,u=/^\d+(?:\.\d+)?$/,N=/^(".*"|'.*'|`.*`)$/s,S=new Set(["if","else","elif","for","while","do","switch","case","default","break","continue","return","function","def","class","struct","interface","const","let","var","local","import","export","from","package","try","catch","except","finally","throw","raise","new","this","self","end","then","yield","async","await","nil","null","None","true","false","True","False","and","or","not","in","of","is"]);var _=new Set(g);function k(i,r,n=!0){let s=y(R(i,h[r]),r),c=[],l=1,o=0;p.lastIndex=0;let t=p.exec(s);for(;t!==null;){for(let a=o;a<t.index;a++)s[a]===`
+`&&l++;o=t.index;let e=t[0],f=e;E.test(e)&&!S.has(e)?f="ID":n&&(u.test(e)||N.test(e))&&(f=u.test(e)?"NUM":"STR"),c.push({text:f,line:l});for(let a=o;a<t.index+e.length;a++)s[a]===`
+`&&l++;o=t.index+e.length,t=p.exec(s)}return c}function x(i){return k(i.source,i.fileType,i.ignoreLiterals)}export{x as default};
 //# sourceMappingURL=duplication-worker.mjs.map

package/dist/duplication-worker.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/graph/duplication/tokenizer.ts","../src/duplication-worker.ts"],"sourcesContent":["/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, Java, Kotlin, Scala,\n * Groovy, CoffeeScript, LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  java: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  kotlin: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  scala: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  groovy: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: identifiers always collapse to a single placeholder (so renamed-variable clones\n *   still hash identically), and literals optionally collapse too (`ignoreLiterals`, default\n *   on) so only structural shape — not the specific values used — drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment-stripping rule; unrecognised types skip stripping.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripComments(source, COMMENT_SYNTAX[fileType]);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n","/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n"],"mappings":"AA0BA,IAAMA,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,MAAO,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAEA,IAAMO,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EAaM,SAASC,EACdpB,EACAqB,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWnB,EAAcJ,EAAQF,EAAeuB,CAAQ,CAAC,EACzDG,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBZ,EAAc,UAAY,EAC1B,IAAIa,EAAQb,EAAc,KAAKS,CAAQ,EAEvC,KAAOI,IAAU,MAAM,CACrB,QAASC,EAAIF,EAAWE,EAAID,EAAM,MAAOC,IACnCL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPb,EAAmB,KAAKa,CAAG,GAAK,CAACV,EAAS,IAAIU,CAAG,EACnDC,EAAO,KACER,IAAmBL,EAAe,KAAKY,CAAG,GAAKX,EAAe,KAAKW,CAAG,KAC/EC,EAAOb,EAAe,KAAKY,CAAG,EAAI,MAAQ,OAE5CL,EAAO,KAAK,CAAE,KAAAM,EAAM,KAAAL,CAAK,CAAC,EAE1B,QAASG,EAAIF,EAAWE,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDL,EAASK,CAAC,IAAM;AAAA,GAAMH,IAE5BC,EAAYC,EAAM,MAAQE,EAAI,OAC9BF,EAAQb,EAAc,KAAKS,CAAQ,CACrC,CAEA,OAAOC,CACT,CC3Oe,SAARO,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","tokenize","fileType","ignoreLiterals","stripped","tokens","line","lastIndex","match","i","raw","text","tokenizeInWorker","payload","tokenize"]}
+{"version":3,"sources":["../src/graph/duplication/tokenizer.ts","../src/duplication-worker.ts"],"sourcesContent":["/**\n * Language-agnostic source tokenizer for duplicate-code detection. Strips per-language\n * comment syntax and import/using statements, then splits what remains into a normalized token stream shared by every\n * language `DEFAULT_EXTENSIONS` covers — one generic tokenizer rather than a per-language\n * lexer, so `findDuplicates` works uniformly across TS/JS, Python, Go, Java, Kotlin, Scala,\n * Groovy, CoffeeScript, LiveScript, Lua, Gherkin, style files, and Markdown.\n */\nimport type { FileType } from \"../../types/parse\";\n\n/** One normalized token plus the 1-based source line it came from. */\nexport interface NormalizedToken {\n  text: string;\n  line: number;\n}\n\ninterface CommentSyntax {\n  line?: string[];\n  block?: Array<[string, string]>;\n}\n\n/**\n * Per-`FileType` comment markers used to mask out comment text before tokenizing, so comment\n * wording never contributes to a duplicate match. Deliberately not string-literal-aware — a\n * `//` inside a string is rare and the cost of occasionally over-stripping is low for a\n * heuristic duplicate finder, same trade-off the complexity scorers make elsewhere.\n */\nconst COMMENT_SYNTAX: Partial<Record<FileType, CommentSyntax>> = {\n  typescript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  javascript: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  go: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  java: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  kotlin: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  scala: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  groovy: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  css: { block: [[\"/*\", \"*/\"]] },\n  scss: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  less: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  stylus: { line: [\"//\"], block: [[\"/*\", \"*/\"]] },\n  python: { line: [\"#\"] },\n  gherkin: { line: [\"#\"] },\n  coffeescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  livescript: { line: [\"#\"], block: [[\"###\", \"###\"]] },\n  lua: { line: [\"--\"], block: [[\"--[[\", \"]]\"]] },\n  markdown: { block: [[\"<!--\", \"-->\"]] },\n};\n\n/**\n * @description Replaces every non-newline character in `source[start, end)` with a space,\n *   so downstream line-number tracking stays correct while the masked text can no longer\n *   match anything.\n * @param source - Full file source.\n * @param start - Start offset (inclusive) of the range to mask.\n * @param end - End offset (exclusive) of the range to mask.\n * @returns `source` with the range masked.\n */\nfunction maskRange(source: string, start: number, end: number): string {\n  const masked = source.slice(start, end).replace(/[^\\n]/g, \" \");\n  return source.slice(0, start) + masked + source.slice(end);\n}\n\n/**\n * @description Masks out every line and block comment matching `syntax`, preserving line\n *   breaks and overall string length so line numbers computed later stay accurate.\n * @param source - Full file source.\n * @param syntax - Comment markers for the file's language; absent for languages with no\n *   configured comment syntax, in which case the source passes through unchanged.\n * @returns The source with comment text masked to spaces.\n */\nfunction stripComments(source: string, syntax: CommentSyntax | undefined): string {\n  if (!syntax) return source;\n  let result = source;\n\n  for (const [open, close] of syntax.block ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(open, searchFrom);\n      if (start === -1) break;\n      const end = result.indexOf(close, start + open.length);\n      const rangeEnd = end === -1 ? result.length : end + close.length;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  for (const marker of syntax.line ?? []) {\n    let searchFrom = 0;\n    for (;;) {\n      const start = result.indexOf(marker, searchFrom);\n      if (start === -1) break;\n      const lineEnd = result.indexOf(\"\\n\", start);\n      const rangeEnd = lineEnd === -1 ? result.length : lineEnd;\n      result = maskRange(result, start, rangeEnd);\n      searchFrom = rangeEnd;\n    }\n  }\n\n  return result;\n}\n\n/**\n * Per-`FileType` patterns matching a *single* import/using line. Long shared import lists\n * tokenize identically across unrelated files and register as \"duplicates\" that carry no shared\n * logic — same rationale as stripping comments. Masked (space-filled, line breaks kept) before\n * tokenizing. Deliberately coarse and not string-literal-aware, like `stripComments`.\n */\nconst IMPORT_LINE_PATTERNS: Partial<Record<FileType, RegExp[]>> = {\n  typescript: [/^\\s*import[\\s{(\"*]/, /^\\s*export\\s.*\\sfrom\\s/],\n  javascript: [/^\\s*import[\\s{(\"*]/, /^\\s*export\\s.*\\sfrom\\s/],\n  python: [/^\\s*import\\s+\\S/, /^\\s*from\\s+\\S+\\s+import[\\s(]/],\n  go: [/^\\s*import\\s+[\"(]/],\n  java: [/^\\s*import\\s+/],\n  kotlin: [/^\\s*import\\s+/],\n  scala: [/^\\s*import\\s+/],\n  groovy: [/^\\s*import\\s+/],\n};\n\n/**\n * @description Masks import / `from … import` / `using` statements to spaces before tokenizing,\n *   preserving line breaks so later line-number tracking stays accurate. Handles the two\n *   multi-line shapes that matter in practice: a JS/TS `import { … } from \"x\"` whose braces span\n *   lines, and a Go `import ( … )` block. Everything else is matched line-by-line.\n * @param source - File source, already comment-stripped.\n * @param fileType - Selects the import-line patterns; unrecognised types pass through unchanged.\n * @returns The source with import statements masked to spaces.\n */\nfunction stripImports(source: string, fileType: FileType): string {\n  const patterns = IMPORT_LINE_PATTERNS[fileType];\n  if (!patterns) return source;\n\n  const lines = source.split(\"\\n\");\n  const isGo = fileType === \"go\";\n  let inGoBlock = false;\n  let inJsBraces = false;\n\n  for (let i = 0; i < lines.length; i++) {\n    const line = lines[i] as string;\n    const mask = () => {\n      lines[i] = line.replace(/[^\\r]/g, \" \");\n    };\n\n    if (inGoBlock) {\n      mask();\n      if (line.includes(\")\")) inGoBlock = false;\n      continue;\n    }\n    if (inJsBraces) {\n      mask();\n      if (/}\\s*from\\s/.test(line) || line.includes(\"}\")) inJsBraces = false;\n      continue;\n    }\n    if (!patterns.some((pattern) => pattern.test(line))) continue;\n\n    mask();\n    if (isGo && /^\\s*import\\s+\\(/.test(line) && !line.includes(\")\")) inGoBlock = true;\n    else if (\n      (fileType === \"typescript\" || fileType === \"javascript\") &&\n      /^\\s*import\\b/.test(line) &&\n      line.includes(\"{\") &&\n      !line.includes(\"}\")\n    ) {\n      inJsBraces = true;\n    }\n  }\n\n  return lines.join(\"\\n\");\n}\n\nconst MULTI_CHAR_OPERATORS = [\n  \"===\",\n  \"!==\",\n  \"...\",\n  \"=>\",\n  \"==\",\n  \"!=\",\n  \"<=\",\n  \">=\",\n  \"&&\",\n  \"||\",\n  \"::\",\n  \"->\",\n  \"..\",\n  \"+=\",\n  \"-=\",\n  \"*=\",\n  \"/=\",\n];\n\n/** Matches, in priority order: multi-char operators, identifiers, numbers, and quoted strings. */\nconst TOKEN_PATTERN = new RegExp(\n  `${MULTI_CHAR_OPERATORS.map((op) => op.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\")).join(\"|\")}|[A-Za-z_$][A-Za-z0-9_$]*|\\\\d+(?:\\\\.\\\\d+)?|\"(?:\\\\\\\\.|[^\"\\\\\\\\])*\"|'(?:\\\\\\\\.|[^'\\\\\\\\])*'|\\`(?:\\\\\\\\.|[^\\`\\\\\\\\])*\\`|\\\\S`,\n  \"g\",\n);\n\nconst IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;\nconst NUMBER_PATTERN = /^\\d+(?:\\.\\d+)?$/;\nconst STRING_PATTERN = /^(\".*\"|'.*'|`.*`)$/s;\n\n/**\n * Keywords kept verbatim rather than collapsed to the `ID` placeholder, shared across every\n * language rather than a per-language keyword table — deliberately coarse. Without this,\n * structurally different code (an `if` vs. a `for`, a `class` vs. a `function`) would tokenize\n * identically once every identifier-shaped word became `ID`, which would make the shingle\n * matcher far too eager. This list only needs to cover the keywords common enough across\n * TS/JS/Python/Go/CoffeeScript/LiveScript/Lua/Gherkin to matter for shape-preservation — it\n * doesn't need to be exhaustive or language-precise, since keeping a non-keyword here just\n * costs a little precision, not correctness.\n */\nconst KEYWORDS = new Set([\n  \"if\",\n  \"else\",\n  \"elif\",\n  \"for\",\n  \"while\",\n  \"do\",\n  \"switch\",\n  \"case\",\n  \"default\",\n  \"break\",\n  \"continue\",\n  \"return\",\n  \"function\",\n  \"def\",\n  \"class\",\n  \"struct\",\n  \"interface\",\n  \"const\",\n  \"let\",\n  \"var\",\n  \"local\",\n  \"import\",\n  \"export\",\n  \"from\",\n  \"package\",\n  \"try\",\n  \"catch\",\n  \"except\",\n  \"finally\",\n  \"throw\",\n  \"raise\",\n  \"new\",\n  \"this\",\n  \"self\",\n  \"end\",\n  \"then\",\n  \"yield\",\n  \"async\",\n  \"await\",\n  \"nil\",\n  \"null\",\n  \"None\",\n  \"true\",\n  \"false\",\n  \"True\",\n  \"False\",\n  \"and\",\n  \"or\",\n  \"not\",\n  \"in\",\n  \"of\",\n  \"is\",\n]);\n\n/** Single-char tokens that carry computation — arithmetic, bitwise, logical-not, optional. The\n *  omissions are deliberate: `( ) { } [ ] ; , : .` are pure structure, and `= < > /` are excluded\n *  because they overwhelmingly appear as JSX attribute/tag punctuation (`attr={…}`, `<Tag/>`)\n *  rather than as assignment or comparison — counting them would let an icon-component wrapper\n *  score like real logic. Multi-char forms (`===`, `<=`, `=>`, …) are caught via\n *  {@link MULTI_CHAR_OPERATORS} instead and stay significant. */\nconst LOGIC_OPERATOR_CHARS = new Set([\"+\", \"-\", \"*\", \"%\", \"&\", \"|\", \"!\", \"^\", \"~\", \"?\"]);\nconst MULTI_CHAR_OPERATOR_SET: ReadonlySet<string> = new Set(MULTI_CHAR_OPERATORS);\n\n/**\n * @description Whether a normalized token text is \"logic-bearing\" — a language keyword or an\n *   operator — as opposed to a normalized identifier/literal (`ID`/`NUM`/`STR`) or structural\n *   punctuation. Used to score a duplicate block by how much of it is actual computation rather\n *   than boilerplate shape (a Flow `type Props` block, a JSX icon wrapper, a schema object all\n *   have very few of these). See `docs/adr-019-logic-token-scoring.md`.\n * @param text - A {@link NormalizedToken}'s `text`.\n * @returns `true` for keywords and operators; `false` for `ID`/`NUM`/`STR` and `( ) { } [ ] ; , : . = < > /`.\n */\nexport function isSignificantToken(text: string): boolean {\n  if (text === \"ID\" || text === \"NUM\" || text === \"STR\") return false;\n  if (KEYWORDS.has(text)) return true;\n  if (text.length === 1) return LOGIC_OPERATOR_CHARS.has(text);\n  return MULTI_CHAR_OPERATOR_SET.has(text);\n}\n\n/**\n * @description Tokenizes source text into a normalized stream for shingle-based duplicate\n *   matching: comments and import/using statements are masked out first (shared import lists are\n *   not shared logic), then identifiers always collapse to a single placeholder (so\n *   renamed-variable clones still hash identically), and literals optionally collapse too\n *   (`ignoreLiterals`, default on) so only structural shape — not the specific values used —\n *   drives the match.\n * @param source - Raw file content.\n * @param fileType - Selects the comment- and import-stripping rules; unrecognised types skip both.\n * @param ignoreLiterals - When true (default), string/number literal tokens are also\n *   normalized to a placeholder rather than compared verbatim.\n * @returns Normalized tokens in source order, each carrying its 1-based source line.\n */\nexport function tokenize(\n  source: string,\n  fileType: FileType,\n  ignoreLiterals = true,\n): NormalizedToken[] {\n  const stripped = stripImports(stripComments(source, COMMENT_SYNTAX[fileType]), fileType);\n  const tokens: NormalizedToken[] = [];\n\n  let line = 1;\n  let lastIndex = 0;\n  TOKEN_PATTERN.lastIndex = 0;\n  let match = TOKEN_PATTERN.exec(stripped);\n\n  while (match !== null) {\n    for (let i = lastIndex; i < match.index; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index;\n\n    const raw = match[0];\n    let text = raw;\n    if (IDENTIFIER_PATTERN.test(raw) && !KEYWORDS.has(raw)) {\n      text = \"ID\";\n    } else if (ignoreLiterals && (NUMBER_PATTERN.test(raw) || STRING_PATTERN.test(raw))) {\n      text = NUMBER_PATTERN.test(raw) ? \"NUM\" : \"STR\";\n    }\n    tokens.push({ text, line });\n\n    for (let i = lastIndex; i < match.index + raw.length; i++) {\n      if (stripped[i] === \"\\n\") line++;\n    }\n    lastIndex = match.index + raw.length;\n    match = TOKEN_PATTERN.exec(stripped);\n  }\n\n  return tokens;\n}\n","/** Piscina task handler: tokenizes a single file's content in a worker thread, for `findDuplicates`. */\n\nimport type { NormalizedToken } from \"./graph/duplication/tokenizer.js\";\nimport { tokenize } from \"./graph/duplication/tokenizer.js\";\nimport type { FileType } from \"./types/parse\";\n\nexport default function tokenizeInWorker(payload: {\n  source: string;\n  fileType: FileType;\n  ignoreLiterals: boolean;\n}): NormalizedToken[] {\n  return tokenize(payload.source, payload.fileType, payload.ignoreLiterals);\n}\n"],"mappings":"AA0BA,IAAMA,EAA2D,CAC/D,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,WAAY,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAClD,GAAI,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC1C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,MAAO,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,IAAK,CAAE,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC7B,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,KAAM,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC5C,OAAQ,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,KAAM,IAAI,CAAC,CAAE,EAC9C,OAAQ,CAAE,KAAM,CAAC,GAAG,CAAE,EACtB,QAAS,CAAE,KAAM,CAAC,GAAG,CAAE,EACvB,aAAc,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACrD,WAAY,CAAE,KAAM,CAAC,GAAG,EAAG,MAAO,CAAC,CAAC,MAAO,KAAK,CAAC,CAAE,EACnD,IAAK,CAAE,KAAM,CAAC,IAAI,EAAG,MAAO,CAAC,CAAC,OAAQ,IAAI,CAAC,CAAE,EAC7C,SAAU,CAAE,MAAO,CAAC,CAAC,OAAQ,KAAK,CAAC,CAAE,CACvC,EAWA,SAASC,EAAUC,EAAgBC,EAAeC,EAAqB,CACrE,IAAMC,EAASH,EAAO,MAAMC,EAAOC,CAAG,EAAE,QAAQ,SAAU,GAAG,EAC7D,OAAOF,EAAO,MAAM,EAAGC,CAAK,EAAIE,EAASH,EAAO,MAAME,CAAG,CAC3D,CAUA,SAASE,EAAcJ,EAAgBK,EAA2C,CAChF,GAAI,CAACA,EAAQ,OAAOL,EACpB,IAAIM,EAASN,EAEb,OAAW,CAACO,EAAMC,CAAK,IAAKH,EAAO,OAAS,CAAC,EAAG,CAC9C,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQC,EAAME,CAAU,EAC7C,GAAIR,IAAU,GAAI,MAClB,IAAMC,EAAMI,EAAO,QAAQE,EAAOP,EAAQM,EAAK,MAAM,EAC/CG,EAAWR,IAAQ,GAAKI,EAAO,OAASJ,EAAMM,EAAM,OAC1DF,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,QAAWC,KAAUN,EAAO,MAAQ,CAAC,EAAG,CACtC,IAAII,EAAa,EACjB,OAAS,CACP,IAAMR,EAAQK,EAAO,QAAQK,EAAQF,CAAU,EAC/C,GAAIR,IAAU,GAAI,MAClB,IAAMW,EAAUN,EAAO,QAAQ;AAAA,EAAML,CAAK,EACpCS,EAAWE,IAAY,GAAKN,EAAO,OAASM,EAClDN,EAASP,EAAUO,EAAQL,EAAOS,CAAQ,EAC1CD,EAAaC,CACf,CACF,CAEA,OAAOJ,CACT,CAQA,IAAMO,EAA4D,CAChE,WAAY,CAAC,qBAAsB,wBAAwB,EAC3D,WAAY,CAAC,qBAAsB,wBAAwB,EAC3D,OAAQ,CAAC,kBAAmB,8BAA8B,EAC1D,GAAI,CAAC,mBAAmB,EACxB,KAAM,CAAC,eAAe,EACtB,OAAQ,CAAC,eAAe,EACxB,MAAO,CAAC,eAAe,EACvB,OAAQ,CAAC,eAAe,CAC1B,EAWA,SAASC,EAAad,EAAgBe,EAA4B,CAChE,IAAMC,EAAWH,EAAqBE,CAAQ,EAC9C,GAAI,CAACC,EAAU,OAAOhB,EAEtB,IAAMiB,EAAQjB,EAAO,MAAM;AAAA,CAAI,EACzBkB,EAAOH,IAAa,KACtBI,EAAY,GACZC,EAAa,GAEjB,QAASC,EAAI,EAAGA,EAAIJ,EAAM,OAAQI,IAAK,CACrC,IAAMC,EAAOL,EAAMI,CAAC,EACdE,EAAO,IAAM,CACjBN,EAAMI,CAAC,EAAIC,EAAK,QAAQ,SAAU,GAAG,CACvC,EAEA,GAAIH,EAAW,CACbI,EAAK,EACDD,EAAK,SAAS,GAAG,IAAGH,EAAY,IACpC,QACF,CACA,GAAIC,EAAY,CACdG,EAAK,GACD,aAAa,KAAKD,CAAI,GAAKA,EAAK,SAAS,GAAG,KAAGF,EAAa,IAChE,QACF,CACKJ,EAAS,KAAMQ,GAAYA,EAAQ,KAAKF,CAAI,CAAC,IAElDC,EAAK,EACDL,GAAQ,kBAAkB,KAAKI,CAAI,GAAK,CAACA,EAAK,SAAS,GAAG,EAAGH,EAAY,IAE1EJ,IAAa,cAAgBA,IAAa,eAC3C,eAAe,KAAKO,CAAI,GACxBA,EAAK,SAAS,GAAG,GACjB,CAACA,EAAK,SAAS,GAAG,IAElBF,EAAa,IAEjB,CAEA,OAAOH,EAAM,KAAK;AAAA,CAAI,CACxB,CAEA,IAAMQ,EAAuB,CAC3B,MACA,MACA,MACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,KACA,IACF,EAGMC,EAAgB,IAAI,OACxB,GAAGD,EAAqB,IAAKE,GAAOA,EAAG,QAAQ,sBAAuB,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,sHACxF,GACF,EAEMC,EAAqB,6BACrBC,EAAiB,kBACjBC,EAAiB,sBAYjBC,EAAW,IAAI,IAAI,CACvB,KACA,OACA,OACA,MACA,QACA,KACA,SACA,OACA,UACA,QACA,WACA,SACA,WACA,MACA,QACA,SACA,YACA,QACA,MACA,MACA,QACA,SACA,SACA,OACA,UACA,MACA,QACA,SACA,UACA,QACA,QACA,MACA,OACA,OACA,MACA,OACA,QACA,QACA,QACA,MACA,OACA,OACA,OACA,QACA,OACA,QACA,MACA,KACA,MACA,KACA,KACA,IACF,CAAC,EASD,IAAMC,EAA+C,IAAI,IAAIC,CAAoB,EA+B1E,SAASC,EACdC,EACAC,EACAC,EAAiB,GACE,CACnB,IAAMC,EAAWC,EAAaC,EAAcL,EAAQM,EAAeL,CAAQ,CAAC,EAAGA,CAAQ,EACjFM,EAA4B,CAAC,EAE/BC,EAAO,EACPC,EAAY,EAChBC,EAAc,UAAY,EAC1B,IAAIC,EAAQD,EAAc,KAAKP,CAAQ,EAEvC,KAAOQ,IAAU,MAAM,CACrB,QAASC,EAAIH,EAAWG,EAAID,EAAM,MAAOC,IACnCT,EAASS,CAAC,IAAM;AAAA,GAAMJ,IAE5BC,EAAYE,EAAM,MAElB,IAAME,EAAMF,EAAM,CAAC,EACfG,EAAOD,EACPE,EAAmB,KAAKF,CAAG,GAAK,CAACG,EAAS,IAAIH,CAAG,EACnDC,EAAO,KACEZ,IAAmBe,EAAe,KAAKJ,CAAG,GAAKK,EAAe,KAAKL,CAAG,KAC/EC,EAAOG,EAAe,KAAKJ,CAAG,EAAI,MAAQ,OAE5CN,EAAO,KAAK,CAAE,KAAAO,EAAM,KAAAN,CAAK,CAAC,EAE1B,QAASI,EAAIH,EAAWG,EAAID,EAAM,MAAQE,EAAI,OAAQD,IAChDT,EAASS,CAAC,IAAM;AAAA,GAAMJ,IAE5BC,EAAYE,EAAM,MAAQE,EAAI,OAC9BF,EAAQD,EAAc,KAAKP,CAAQ,CACrC,CAEA,OAAOI,CACT,CC1Ue,SAARY,EAAkCC,EAInB,CACpB,OAAOC,EAASD,EAAQ,OAAQA,EAAQ,SAAUA,EAAQ,cAAc,CAC1E","names":["COMMENT_SYNTAX","maskRange","source","start","end","masked","stripComments","syntax","result","open","close","searchFrom","rangeEnd","marker","lineEnd","IMPORT_LINE_PATTERNS","stripImports","fileType","patterns","lines","isGo","inGoBlock","inJsBraces","i","line","mask","pattern","MULTI_CHAR_OPERATORS","TOKEN_PATTERN","op","IDENTIFIER_PATTERN","NUMBER_PATTERN","STRING_PATTERN","KEYWORDS","MULTI_CHAR_OPERATOR_SET","MULTI_CHAR_OPERATORS","tokenize","source","fileType","ignoreLiterals","stripped","stripImports","stripComments","COMMENT_SYNTAX","tokens","line","lastIndex","TOKEN_PATTERN","match","i","raw","text","IDENTIFIER_PATTERN","KEYWORDS","NUMBER_PATTERN","STRING_PATTERN","tokenizeInWorker","payload","tokenize"]}