khmer-segment 0.2.0

package/dist/index.js

@@ -0,0 +1,455 @@
+// src/constants/unicode.ts
+var KHMER_RANGE_START = 6016;
+var KHMER_RANGE_END = 6143;
+var CONSONANT_START = 6016;
+var CONSONANT_END = 6050;
+var INDEPENDENT_VOWEL_START = 6051;
+var INDEPENDENT_VOWEL_END = 6067;
+var DEPENDENT_VOWEL_START = 6068;
+var DEPENDENT_VOWEL_END = 6085;
+var SIGN_START = 6086;
+var SIGN_END = 6099;
+var KHMER_COENG = 6098;
+var DIGIT_START = 6112;
+var DIGIT_END = 6121;
+var ASCII_DIGIT_START = 48;
+var ASCII_DIGIT_END = 57;
+
+// src/constants/char-categories.ts
+function isKhmerCodePoint(cp) {
+  return cp >= KHMER_RANGE_START && cp <= KHMER_RANGE_END;
+}
+function isConsonant(cp) {
+  return cp >= CONSONANT_START && cp <= CONSONANT_END;
+}
+function isIndependentVowel(cp) {
+  return cp >= INDEPENDENT_VOWEL_START && cp <= INDEPENDENT_VOWEL_END;
+}
+function isDependentVowel(cp) {
+  return cp >= DEPENDENT_VOWEL_START && cp <= DEPENDENT_VOWEL_END;
+}
+function isSign(cp) {
+  return cp >= SIGN_START && cp <= SIGN_END;
+}
+function isShiftSign(cp) {
+  return cp === 6089 || cp === 6090;
+}
+function isCoeng(cp) {
+  return cp === KHMER_COENG;
+}
+function isKhmerDigit(cp) {
+  return cp >= DIGIT_START && cp <= DIGIT_END;
+}
+function isAsciiDigit(cp) {
+  return cp >= ASCII_DIGIT_START && cp <= ASCII_DIGIT_END;
+}
+function isDigit(cp) {
+  return isKhmerDigit(cp) || isAsciiDigit(cp);
+}
+function isClusterBase(cp) {
+  return isConsonant(cp) || isIndependentVowel(cp);
+}
+
+// src/core/detect.ts
+function isKhmerChar(char) {
+  if (!char) return false;
+  const cp = char.codePointAt(0);
+  return isKhmerCodePoint(cp);
+}
+function containsKhmer(text) {
+  for (const ch of text) {
+    if (isKhmerChar(ch)) return true;
+  }
+  return false;
+}
+function isKhmerText(text) {
+  if (!text.length) return false;
+  let hasKhmer = false;
+  for (const ch of text) {
+    if (/\s/.test(ch)) continue;
+    if (!isKhmerChar(ch)) return false;
+    hasKhmer = true;
+  }
+  return hasKhmer;
+}
+
+// src/core/cluster.ts
+function splitClusters(text) {
+  if (!text) return [];
+  const chars = [...text];
+  const clusters = [];
+  let i = 0;
+  while (i < chars.length) {
+    const cp = chars[i].codePointAt(0);
+    if (isClusterBase(cp)) {
+      let cluster = chars[i];
+      i++;
+      while (i < chars.length) {
+        const nextCp = chars[i].codePointAt(0);
+        if (isCoeng(nextCp)) {
+          cluster += chars[i];
+          i++;
+          if (i < chars.length && isConsonant(chars[i].codePointAt(0))) {
+            cluster += chars[i];
+            i++;
+          }
+        } else if (isDependentVowel(nextCp) || isSign(nextCp)) {
+          cluster += chars[i];
+          i++;
+        } else {
+          break;
+        }
+      }
+      clusters.push(cluster);
+    } else {
+      clusters.push(chars[i]);
+      i++;
+    }
+  }
+  return clusters;
+}
+function countClusters(text) {
+  return splitClusters(text).length;
+}
+function getClusterBoundaries(text) {
+  const clusters = splitClusters(text);
+  const boundaries = [];
+  let offset = 0;
+  for (const cluster of clusters) {
+    boundaries.push({ start: offset, end: offset + cluster.length });
+    offset += cluster.length;
+  }
+  return boundaries;
+}
+
+// src/core/normalize.ts
+var INVISIBLE_CHARS = /[\u200B\u200C\u200D\u2060\u200E\u200F\uFEFF]/g;
+function normalizeKhmerCluster(cluster) {
+  const chars = [...cluster];
+  if (chars.length <= 1) return chars.join("");
+  let i = 0;
+  const base = [];
+  const coengPairs = [];
+  const shiftSigns = [];
+  const vowels = [];
+  const otherSigns = [];
+  const other = [];
+  base.push(chars[i]);
+  i++;
+  while (i < chars.length) {
+    const cp = chars[i].codePointAt(0);
+    if (isCoeng(cp)) {
+      let pair = chars[i];
+      i++;
+      if (i < chars.length && isConsonant(chars[i].codePointAt(0))) {
+        pair += chars[i];
+        i++;
+      }
+      coengPairs.push(pair);
+    } else if (isShiftSign(cp)) {
+      shiftSigns.push(chars[i]);
+      i++;
+    } else if (isDependentVowel(cp)) {
+      vowels.push(chars[i]);
+      i++;
+    } else if (isSign(cp)) {
+      otherSigns.push(chars[i]);
+      i++;
+    } else {
+      other.push(chars[i]);
+      i++;
+    }
+  }
+  return [
+    ...base,
+    ...coengPairs,
+    ...shiftSigns,
+    ...vowels,
+    ...otherSigns,
+    ...other
+  ].join("");
+}
+function normalizeKhmer(text) {
+  const cleaned = text.replace(INVISIBLE_CHARS, "");
+  const clusters = splitClusters(cleaned);
+  return clusters.map((cluster) => {
+    const firstCp = cluster.codePointAt(0);
+    if (isKhmerCodePoint(firstCp)) {
+      return normalizeKhmerCluster(cluster);
+    }
+    return cluster;
+  }).join("");
+}
+
+// src/algorithms/fmm.ts
+function fmmSegment(clusters, dictionary) {
+  const tokens = [];
+  const hasPrefixFn = dictionary.hasPrefix?.bind(dictionary);
+  let i = 0;
+  let offset = 0;
+  while (i < clusters.length) {
+    let matched = false;
+    let maxLen = clusters.length - i;
+    if (hasPrefixFn) {
+      maxLen = 1;
+      let candidate = clusters[i];
+      while (maxLen < clusters.length - i && hasPrefixFn(candidate + clusters[i + maxLen])) {
+        maxLen++;
+        candidate += clusters[i + maxLen - 1];
+      }
+    }
+    for (let len = maxLen; len >= 1; len--) {
+      const word = clusters.slice(i, i + len).join("");
+      if (dictionary.has(word)) {
+        const start = offset;
+        const end = offset + word.length;
+        tokens.push({ value: word, start, end, isKnown: true });
+        offset = end;
+        i += len;
+        matched = true;
+        break;
+      }
+    }
+    if (!matched) {
+      const word = clusters[i];
+      const start = offset;
+      const end = offset + word.length;
+      tokens.push({ value: word, start, end, isKnown: false });
+      offset = end;
+      i++;
+    }
+  }
+  return tokens;
+}
+
+// src/algorithms/bmm.ts
+function bmmSegment(clusters, dictionary) {
+  const tokens = [];
+  const hasSuffixFn = dictionary.hasSuffix?.bind(dictionary);
+  let i = clusters.length - 1;
+  while (i >= 0) {
+    let matched = false;
+    let maxLen = i + 1;
+    if (hasSuffixFn) {
+      maxLen = 1;
+      let candidate = clusters[i];
+      while (maxLen < i + 1 && hasSuffixFn(candidate)) {
+        maxLen++;
+        candidate = clusters[i - maxLen + 1] + candidate;
+      }
+    }
+    for (let len = maxLen; len >= 1; len--) {
+      const word = clusters.slice(i - len + 1, i + 1).join("");
+      if (dictionary.has(word)) {
+        tokens.push({ value: word, start: 0, end: 0, isKnown: true });
+        i -= len;
+        matched = true;
+        break;
+      }
+    }
+    if (!matched) {
+      tokens.push({
+        value: clusters[i],
+        start: 0,
+        end: 0,
+        isKnown: false
+      });
+      i--;
+    }
+  }
+  tokens.reverse();
+  let offset = 0;
+  for (const token of tokens) {
+    token.start = offset;
+    offset += token.value.length;
+    token.end = offset;
+  }
+  return tokens;
+}
+
+// src/algorithms/bimm.ts
+function bimmSegment(clusters, dictionary) {
+  const fmmResult = fmmSegment(clusters, dictionary);
+  const bmmResult = bmmSegment(clusters, dictionary);
+  const fmmUnknowns = fmmResult.filter((t) => !t.isKnown).length;
+  const bmmUnknowns = bmmResult.filter((t) => !t.isKnown).length;
+  if (fmmUnknowns !== bmmUnknowns) {
+    return fmmUnknowns < bmmUnknowns ? fmmResult : bmmResult;
+  }
+  if (fmmResult.length !== bmmResult.length) {
+    return fmmResult.length < bmmResult.length ? fmmResult : bmmResult;
+  }
+  return fmmResult;
+}
+
+// src/algorithms/group-digits.ts
+function isDigitStr(s) {
+  if (s.length !== 1) return false;
+  const cp = s.codePointAt(0);
+  return isDigit(cp);
+}
+function groupDigitTokens(tokens) {
+  if (tokens.length === 0) return [];
+  const result = [];
+  let i = 0;
+  while (i < tokens.length) {
+    if (isDigitStr(tokens[i].value)) {
+      const start = tokens[i].start;
+      let combined = tokens[i].value;
+      let end = tokens[i].end;
+      let known = tokens[i].isKnown;
+      i++;
+      while (i < tokens.length && isDigitStr(tokens[i].value)) {
+        combined += tokens[i].value;
+        end = tokens[i].end;
+        known = known || tokens[i].isKnown;
+        i++;
+      }
+      result.push({ value: combined, start, end, isKnown: known });
+    } else {
+      result.push(tokens[i]);
+      i++;
+    }
+  }
+  return result;
+}
+
+// src/core/segment.ts
+function segmentWords(text, options) {
+  const shouldNormalize = options?.normalize !== false;
+  const normalized = shouldNormalize ? normalizeKhmer(text) : text;
+  const clusters = splitClusters(normalized);
+  const dictionary = options?.dictionary;
+  let tokens;
+  if (dictionary) {
+    const strategy = options?.strategy ?? "fmm";
+    switch (strategy) {
+      case "bmm":
+        tokens = bmmSegment(clusters, dictionary);
+        break;
+      case "bimm":
+        tokens = bimmSegment(clusters, dictionary);
+        break;
+      default:
+        tokens = fmmSegment(clusters, dictionary);
+        break;
+    }
+  } else {
+    let offset = 0;
+    tokens = clusters.map((cluster) => {
+      const start = offset;
+      const end = offset + cluster.length;
+      offset = end;
+      return { value: cluster, start, end, isKnown: false };
+    });
+  }
+  tokens = groupDigitTokens(tokens);
+  return {
+    original: text,
+    normalized,
+    tokens
+  };
+}
+
+// src/dictionary/trie.ts
+var TrieNode = class {
+  constructor() {
+    this.children = /* @__PURE__ */ new Map();
+    this.isEndOfWord = false;
+  }
+};
+var Trie = class {
+  constructor() {
+    this.root = new TrieNode();
+  }
+  insert(word) {
+    let node = this.root;
+    for (const ch of word) {
+      if (!node.children.has(ch)) {
+        node.children.set(ch, new TrieNode());
+      }
+      node = node.children.get(ch);
+    }
+    node.isEndOfWord = true;
+  }
+  has(word) {
+    let node = this.root;
+    for (const ch of word) {
+      if (!node.children.has(ch)) return false;
+      node = node.children.get(ch);
+    }
+    return node.isEndOfWord;
+  }
+  hasPrefix(prefix) {
+    let node = this.root;
+    for (const ch of prefix) {
+      if (!node.children.has(ch)) return false;
+      node = node.children.get(ch);
+    }
+    return true;
+  }
+  hasSuffix(suffix) {
+    return this.hasReverse(suffix, this.root);
+  }
+  hasReverse(suffix, node) {
+    if (suffix.length === 0) return node.isEndOfWord;
+    const lastChar = suffix[suffix.length - 1];
+    for (const [ch, child] of node.children) {
+      if (ch === lastChar && this.hasReverse(suffix.slice(0, -1), child)) {
+        return true;
+      }
+      if (this.hasReverse(suffix, child)) {
+        return true;
+      }
+    }
+    return false;
+  }
+};
+
+// src/dictionary/memory-dictionary.ts
+var MemoryDictionary = class {
+  constructor(words, frequencies) {
+    this.trie = new Trie();
+    this.reverseTrie = new Trie();
+    this.freqMap = frequencies ?? /* @__PURE__ */ new Map();
+    let count = 0;
+    for (const word of words) {
+      if (word.length > 0) {
+        this.trie.insert(word);
+        this.reverseTrie.insert([...word].reverse().join(""));
+        count++;
+      }
+    }
+    this.size = count;
+  }
+  has(word) {
+    return this.trie.has(word);
+  }
+  hasPrefix(value) {
+    return this.trie.hasPrefix(value);
+  }
+  hasSuffix(value) {
+    return this.reverseTrie.hasPrefix([...value].reverse().join(""));
+  }
+  getFrequency(word) {
+    return this.freqMap.get(word);
+  }
+};
+
+// src/dictionary/create-dictionary.ts
+function createDictionary(words, frequencies) {
+  return new MemoryDictionary(words, frequencies);
+}
+export {
+  containsKhmer,
+  countClusters,
+  createDictionary,
+  getClusterBoundaries,
+  isKhmerChar,
+  isKhmerText,
+  normalizeKhmer,
+  normalizeKhmerCluster,
+  segmentWords,
+  splitClusters
+};
+//# sourceMappingURL=index.js.map

package/package.json

@@ -0,0 +1,64 @@
+{
+    "name": "khmer-segment",
+    "version": "0.2.0",
+    "description": "Khmer text segmentation, normalization, and cluster utilities for JavaScript and TypeScript.",
+    "type": "module",
+    "main": "./dist/index.cjs",
+    "module": "./dist/index.js",
+    "types": "./dist/index.d.ts",
+    "exports": {
+        ".": {
+            "types": "./dist/index.d.ts",
+            "import": "./dist/index.js",
+            "require": "./dist/index.cjs"
+        },
+        "./dictionary": {
+            "types": "./dist/dictionary/index.d.ts",
+            "import": "./dist/dictionary/index.js",
+            "require": "./dist/dictionary/index.cjs"
+        }
+    },
+    "files": [
+        "dist",
+        "!dist/**/*.map"
+    ],
+    "sideEffects": false,
+    "scripts": {
+        "build": "tsup",
+        "dev": "tsup --watch",
+        "test": "vitest run",
+        "test:watch": "vitest",
+        "lint": "tsc --noEmit",
+        "format": "prettier --write .",
+        "format:check": "prettier --check .",
+        "prepublishOnly": "npm run build && npm run test && npm run lint",
+        "playground:dev": "npm run dev --prefix playground",
+        "playground:build": "npm run build --prefix playground"
+    },
+    "keywords": [
+        "khmer",
+        "unicode",
+        "segmentation",
+        "nlp",
+        "typescript",
+        "javascript"
+    ],
+    "author": "Phalla Doll",
+    "license": "MIT",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/phalla-doll/khmer-segment-js.git"
+    },
+    "homepage": "https://github.com/phalla-doll/khmer-segment-js#readme",
+    "bugs": {
+        "url": "https://github.com/phalla-doll/khmer-segment-js/issues"
+    },
+    "devDependencies": {
+        "@huggingface/hub": "^2.11.0",
+        "@types/node": "^25.5.2",
+        "prettier": "^3.8.1",
+        "tsup": "^8.0.0",
+        "typescript": "^5.0.0",
+        "vitest": "^3.0.0"
+    }
+}