@lunarisapp/hyphen 0.0.1

package/src/dictionaries/hyph_zu_ZA.dic

@@ -0,0 +1,171 @@
+ISO8859-1
+%  Ukwahlukanisela ngekhonco isiZulu: Ukulandisa kwokusebenza ne-OpenOffice.org
+%  Hyphenation for Zulu: Version for OpenOffice.org
+%  Copyright (C) 2005, 2007 Friedel Wolff
+%
+%  This library is free software; you can redistribute it and/or
+%  modify it under the terms of the GNU Lesser General Public
+%  License as published by the Free Software Foundation;
+%  version 2.1 of the License.
+%
+%  This library is distributed in the hope that it will be useful,
+%  but WITHOUT ANY WARRANTY; without even the implied warranty of
+%  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%  Lesser General Public License for more details.
+%
+%  You should have received a copy of the GNU Lesser General Public
+%  License along with this library; if not, write to the Free Software
+%  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+%
+
+a1
+e1
+i1
+o1
+u1
+%is'thandwa njalonjalo
+'1
+
+%iziphambuko ngenxa yamagama esiBhunu
+1be2rg.
+be1
+1bu2rg.
+bu1
+1da2l.
+da1
+1do2rp.
+do1
+%angazi ngale: Modder-fo-ntein?
+1fonte2i2n.
+fo1
+1ho2e2k.
+1ho2f.
+ho1
+1klo2o2f.
+klo1
+1ko2p.
+ko1
+1kra2ns.
+kra1
+1kro2o2n.
+kro1
+1kru2i2n.
+kru1
+1la2nd.
+la1
+1pa2rk.
+pa1
+1ple2i2n.
+ple1
+1po2o2rt.
+po1
+1ra2nd.
+ra1
+1rivi2er.
+ri1
+1spru2i2t.
+spru1
+1sta2d.
+sta1
+1stra2nd.
+stra1
+
+%ukukhombisa
+1no2o2rd.
+no1
+1o2o2s.
+1su2i2d.
+su1
+1we2s.
+we1
+
+%iziphambuko ngenxa yamagama esiNgisi
+1ba2y.
+ba1
+be2a2ch
+e2a2ch.
+cli2ffe.
+1da2le.
+1fi2e2ld.
+fi1
+%... Hill
+i2ll.
+1me2a2d.
+%1pa2rk. - bona isiBhunu
+1ri2dge.
+%kodwa
+b2ri2dge.
+bri1
+1to2n.
+1to2wn.
+to1
+1vi2e2w.
+1vi2lle.
+vi1
+1wo2o2d.
+wo1
+
+%ukukhombisa
+no2rth.
+e2a2st.
+so2u2th.
+so1
+we2st.
+
+%iziphambuko ngenxa yamagama esiSuthu
+a2ng.
+e2ng.
+i2ng.
+o2ng.
+u2ng.
+
+%iziphambuko ezinhlobonhlobo
+%mhlawumbe amaphutha okupela angazohlupa
+a2a1
+a2e1
+a2i1
+a2o1
+a2u1
+e2a1
+e2e1
+e2i1
+e2o1
+e2u1
+i2a1
+i2e1
+i2i1
+i2o1
+i2u1
+o2a1
+o2e1
+o2i1
+o2o1
+o2u1
+u2a1
+u2e1
+u2i1
+u2o1
+u2u1
+
+2b.
+2c.
+2d.
+2f.
+2g.
+2h.
+2j.
+2k.
+2l.
+2m.
+2n.
+2p.
+2q.
+2r.
+2s.
+2t.
+2v.
+2w.
+2x.
+2z.
+
+

package/src/dictionaries/update.sh

@@ -0,0 +1,7 @@
+git clone https://git.libreoffice.org/dictionaries libreoffice-dictionaries
+cd libreoffice-dictionaries
+git pull
+cd ..
+find libreoffice-dictionaries -name "hyph_*\.dic" | xargs -I '{}' cp '{}' .
+sed -i 's/\r$//' *.dic
+rename -- -Latn _Latn *-Latn.dic

package/src/index.ts

@@ -0,0 +1,235 @@
+import fs from "fs";
+import { join } from "path";
+
+const DICTIONARY_PATH = join(__dirname, "dictionaries");
+const IGNORED = [
+  "%",
+  "#",
+  "LEFTHYPHENMIN",
+  "RIGHTHYPHENMIN",
+  "COMPOUNDLEFTHYPHENMIN",
+  "COMPOUNDRIGHTHYPHENMIN",
+];
+
+class DataInt {
+  value: number;
+  data?: [string, number, number];
+
+  constructor(value: number, data?: [string, number, number]) {
+    this.value = value;
+    this.data = data;
+  }
+}
+
+class HyphenDict {
+  patterns: Map<string, [number, number[]]> = new Map();
+  cache: Map<string, DataInt[]> = new Map();
+  maxLen: number = 0;
+
+  constructor(filePath: string) {
+    if (!fs.existsSync(filePath)) {
+      throw new Error(`Dictionary file not found: ${filePath}`);
+    }
+
+    const lines = fs.readFileSync(filePath, "utf-8").split("\n");
+    lines.forEach((line) => {
+      line = line.trim();
+      if (!line || IGNORED.some((i) => line.startsWith(i))) return;
+
+      const pattern = line.replace(/\^{2}([0-9a-f]{2})/g, (match, hex) =>
+        String.fromCharCode(parseInt(hex, 16)),
+      );
+
+      const parts = Array.from(pattern.matchAll(/(\d?)(\D?)/g));
+      const tags: string[] = [];
+      const values: number[] = [];
+
+      parts.forEach(([, num, char]) => {
+        tags.push(char!);
+        values.push(num ? parseInt(num) : 0);
+      });
+
+      if (Math.max(...values) === 0) return;
+
+      let start = 0,
+        end = values.length;
+      while (!values[start]) start++;
+      while (!values[end - 1]) end--;
+
+      this.patterns.set(tags.join(""), [start, values.slice(start, end)]);
+    });
+
+    this.maxLen = Math.max(
+      ...Array.from(this.patterns.keys()).map((k) => k.length),
+    );
+  }
+
+  positions(word: string): DataInt[] {
+    word = word.toLowerCase();
+    if (this.cache.has(word)) return this.cache.get(word)!;
+
+    const references = new Array(word.length + 2).fill(0);
+    const extendedWord = `.${word}.`;
+
+    for (let i = 0; i < extendedWord.length - 1; i++) {
+      for (
+        let j = i + 1;
+        j < Math.min(i + this.maxLen, extendedWord.length) + 1;
+        j++
+      ) {
+        const pattern = this.patterns.get(extendedWord.slice(i, j));
+        if (!pattern) continue;
+
+        const [offset, values] = pattern;
+        values.forEach((val, idx) => {
+          references[i + offset + idx] = Math.max(
+            val,
+            references[i + offset + idx],
+          );
+        });
+      }
+    }
+
+    const positions = references
+      .map((val, idx) => (val % 2 ? new DataInt(idx - 1) : null))
+      .filter(Boolean) as DataInt[];
+    this.cache.set(word, positions);
+    return positions;
+  }
+}
+
+export class TextHyphen {
+  private readonly left: number;
+  private readonly right: number;
+  private hd: HyphenDict;
+  readonly dictionaries: Record<string, string>;
+  private readonly lowercaseLangs: Record<string, string>;
+
+  constructor(props?: { lang?: string; left?: number; right?: number }) {
+    const { lang = "en_US", left = 2, right = 2 } = props || {};
+    this.left = left;
+    this.right = right;
+    this.dictionaries = this.loadDictionaries();
+    this.lowercaseLangs = Object.keys(this.dictionaries).reduce(
+      (acc, lang) => {
+        acc[lang.toLowerCase()] = lang;
+        return acc;
+      },
+      {} as Record<string, string>,
+    );
+    const fallback = this.getLanguageFallback(lang);
+    if (!fallback) {
+      throw new Error(`Language not found: ${lang}`);
+    }
+    this.hd = new HyphenDict(
+      join(DICTIONARY_PATH, this.dictionaries[fallback]!),
+    );
+  }
+
+  private loadDictionaries() {
+    const dictionaries: Record<string, string> = {};
+    for (const file of fs.readdirSync(DICTIONARY_PATH).sort()) {
+      const [name, ext] = file.split(".");
+      const lang = name!.replace("hyph_", "").replace("-", "_"); // File name format: hyph_LANG-COUNTRY.dic
+      const shortLang = lang.split("_")[0];
+      if (ext === "dic") {
+        dictionaries[lang] = file;
+        if (!dictionaries[shortLang!]) {
+          dictionaries[shortLang!] = file;
+        }
+      }
+    }
+    return dictionaries;
+  }
+
+  /**
+   * Get the fallback language for a given language.
+   * @param language
+   */
+  getLanguageFallback(language: string) {
+    const parts = language.replace("-", "_").toLowerCase().split("_");
+    while (parts.length) {
+      const currentLanguage = parts.join("_");
+      if (this.lowercaseLangs[currentLanguage]) {
+        return this.lowercaseLangs[currentLanguage];
+      }
+      parts.pop();
+    }
+    return undefined;
+  }
+
+  /**
+   * Get the positions of possible hyphenation points in a word
+   * @param word
+   */
+  positions(word: string): number[] {
+    const rightLimit = word.length - this.right;
+    return this.hd
+      .positions(word)
+      .map((pos) => pos.value)
+      .filter((pos) => this.left <= pos && pos <= rightLimit);
+  }
+
+  /**
+   * Get iterator for all possible variants of hyphenating the word.
+   * @param word
+   */
+  *iterate(word: string): Generator<[string, string]> {
+    for (const position of this.hd.positions(word).reverse()) {
+      if (position.data) {
+        const [change, index, cut] = position.data;
+        const updatedIndex = index + position.value;
+        let ch = change;
+        if (word === word.toUpperCase()) {
+          ch = change.toUpperCase();
+        }
+        const [c1, c2] = ch.split("=");
+        yield [
+          word.slice(0, updatedIndex) + c1,
+          c2 + word.slice(updatedIndex + cut),
+        ];
+      } else {
+        yield [word.slice(0, position.value), word.slice(position.value)];
+      }
+    }
+  }
+
+  /**
+   * Get all possible variants for hyphenating the word.
+   * @param word
+   */
+  variants(word: string): string[][] {
+    return this.iterate(word).toArray();
+  }
+
+  /**
+   * Wrap a word at a given width with a hyphen.
+   * @param word
+   * @param width
+   * @param hyphen
+   */
+  wrap(word: string, width: number, hyphen = "-") {
+    width -= hyphen.length;
+    for (const [w1, w2] of this.iterate(word)) {
+      if (w1.length <= width) {
+        return [w1 + hyphen, w2];
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Insert hyphens into a word at possible positions.
+   * @param word
+   * @param hyphen
+   */
+  inserted(word: string, hyphen = "-"): string {
+    const letters = [...word];
+    this.positions(word)
+      .reverse()
+      .forEach((pos) => {
+        letters.splice(pos, 0, hyphen);
+      });
+    return letters.join("");
+  }
+}

package/tests/hyphen.test.ts

@@ -0,0 +1,118 @@
+import { describe, it, expect } from "@jest/globals";
+import { TextHyphen } from "../src";
+
+describe("hyphen tests", () => {
+  it("should correctly hyphenate words using inserted method", () => {
+    const dic = new TextHyphen({ lang: "nl_NL" });
+    expect(dic.inserted("lettergrepen")).toBe("let-ter-gre-pen");
+  });
+
+  it("should correctly wrap words", () => {
+    const dic = new TextHyphen({ lang: "nl_NL" });
+    const actual = dic.wrap("autobandventieldopje", 11);
+    const expected = ["autoband-", "ventieldopje"];
+    expect(actual).toStrictEqual(expected);
+  });
+
+  it("should iterate hyphenation positions correctly", () => {
+    const dic = new TextHyphen({ lang: "nl_NL" });
+    console.log(dic);
+    const actual = dic.variants("Amsterdam");
+    const expected = [
+      ["Amster", "dam"],
+      ["Am", "sterdam"],
+    ];
+    expect(actual).toStrictEqual(expected);
+  });
+
+  it("should use a fallback dictionary", () => {
+    const dic = new TextHyphen({ lang: "nl_NL-variant" });
+    const actual = dic.variants("Amsterdam");
+    const expected = [
+      ["Amster", "dam"],
+      ["Am", "sterdam"],
+    ];
+    expect(actual).toStrictEqual(expected);
+  });
+
+  it("should throw an error for a missing dictionary", () => {
+    expect(() => new TextHyphen({ lang: "mi_SS" })).toThrowError();
+  });
+
+  // it('should use a custom dictionary', () => {
+  //   const dic = new Hyphen('fr');
+  //   expect(dic.inserted('autobandventieldopje')).to.not.equal(
+  //     'au-to-band-ven-tiel-dop-je'
+  //   );
+  //   Hyphen.LANGUAGES['fr'] = Hyphen.LANGUAGES['nl_NL'];
+  //   const dicNew = new Hyphen('fr');
+  //   expect(dicNew.inserted('autobandventieldopje')).to.equal(
+  //     'au-to-band-ven-tiel-dop-je'
+  //   );
+  // });
+
+  describe("should support left and right hyphenation constraints", () => {
+    it("default", () => {
+      const dic = new TextHyphen({ lang: "nl_NL" });
+      expect(dic.inserted("lettergrepen")).toBe("let-ter-gre-pen");
+    });
+    it("left", () => {
+      const dic = new TextHyphen({ lang: "nl_NL", left: 4 });
+      expect(dic.inserted("lettergrepen")).toBe("letter-gre-pen");
+    });
+    it("right", () => {
+      const dic = new TextHyphen({ lang: "nl_NL", right: 4 });
+      expect(dic.inserted("lettergrepen")).toBe("let-ter-grepen");
+    });
+    it("both", () => {
+      const dic = new TextHyphen({ lang: "nl_NL", left: 4, right: 4 });
+      expect(dic.inserted("lettergrepen")).toBe("letter-grepen");
+    });
+  });
+
+  // it("should support alternative parser for hyphenation", () => {
+  //   const dic = new Hyphen({ lang: "hu", left: 1, right: 1 });
+  //   assert.deepStrictEqual(
+  //     [...dic.iterate("kulissza")],
+  //     [
+  //       ["kulisz", "sza"],
+  //       ["ku", "lissza"],
+  //     ],
+  //   );
+  //   assert.strictEqual(dic.inserted("kulissza"), "ku-lisz-sza");
+  // });
+
+  it("should handle uppercase words correctly", () => {
+    const dic = new TextHyphen({ lang: "nl_NL" });
+    expect(dic.inserted("LETTERGREPEN")).toBe("LET-TER-GRE-PEN");
+  });
+
+  // it("should support uppercase alternative parser", () => {
+  //   const dic = new Hyphen({ lang: "hu", left: 1, right: 1 });
+  //   assert.deepStrictEqual(dic.variants("KULISSZA"), [
+  //     ["KULISZ", "SZA"],
+  //     ["KU", "LISSZA"],
+  //   ]);
+  //   assert.strictEqual(dic.inserted("KULISSZA"), "KU-LISZ-SZA");
+  // });
+
+  it("should be able to load all dictionaries", () => {
+    const hyp = new TextHyphen();
+    const dictionaries = hyp.dictionaries;
+    Object.keys(dictionaries).forEach((dict) => {
+      new TextHyphen({ lang: dict });
+    });
+  });
+
+  it("should correctly determine language fallbacks", () => {
+    const hyp = new TextHyphen();
+    expect(hyp.getLanguageFallback("en")).toBe("en");
+    expect(hyp.getLanguageFallback("en_US")).toBe("en_US");
+    expect(hyp.getLanguageFallback("en_FR")).toBe("en");
+    expect(hyp.getLanguageFallback("sr-Latn")).toBe("sr_Latn");
+    expect(hyp.getLanguageFallback("SR-LATN")).toBe("sr_Latn");
+    expect(hyp.getLanguageFallback("sr-Cyrl")).toBe("sr");
+    expect(hyp.getLanguageFallback("fr-Latn-FR")).toBe("fr");
+    expect(hyp.getLanguageFallback("en-US_variant1-x")).toBe("en_US");
+  });
+});

package/tsconfig.pkg.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "dist",
+    "rootDir": "src",
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "tests", "dist"],
+}