@lunarisapp/hyphen 0.0.3 → 0.0.4

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @lunarisapp/hyphen
 
+## 0.0.4
+
+### Patch Changes
+
+- [`cd64da1`](https://github.com/LunarisApp/text-tools/commit/cd64da18993b790ea543286eafcc870bcf7aa4a3) Thanks [@MrLightful](https://github.com/MrLightful)! - Downgrade target to es2020
+
+- [`7e2718f`](https://github.com/LunarisApp/text-tools/commit/7e2718fd9959b66e1fc0912e15b1ad340fde7be3) Thanks [@MrLightful](https://github.com/MrLightful)! - Fix bundling of dictionary assets
+
 ## 0.0.3
 
 ### Patch Changes

package/eslint.config.js

@@ -1,4 +1,4 @@
-import { config } from "../../eslint.config.js";
+const { config } = require("../../eslint.config.js");
 
 /** @type {import("eslint").Linter.Config} */
-export default config;
+module.exports = config;

package/jest.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  transform: {
+    "^.+\\.ts?$": "babel-jest",
+    "^.+\\.dic?$": "<rootDir>/tests/__mocks__/dic.js",
+  },
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@lunarisapp/hyphen",
   "description": "A hyphenation library for JavaScript",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "author": {
     "name": "Roman Sirokov",
     "url": "https://mrlightful.com",
@@ -9,7 +9,7 @@
   },
   "homepage": "https://lunaris.app",
   "repository": {
-    "url": "https://github.com/LunarisApp/text-tools/tree/main/packages/hyphen"
+    "url": "https://github.com/LunarisApp/text-tools"
   },
   "publishConfig": {
     "access": "public"
@@ -18,31 +18,33 @@
     "hyphenation",
     "text",
     "linguistics",
-    "language"
+    "language",
+    "pronunciation",
+    "transcription",
+    "syllables"
   ],
-  "License": "Apache-2.0",
+  "license": "Apache-2.0",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",
   "types": "./dist/index.d.ts",
   "devDependencies": {
-    "@babel/core": "^7.26.8",
-    "@babel/preset-env": "^7.26.8",
+    "@babel/core": "^7.26.9",
+    "@babel/preset-env": "^7.26.9",
     "@babel/preset-typescript": "^7.26.0",
     "@eslint/js": "^9.20.0",
     "@jest/globals": "^29.7.0",
-    "@types/node": "^22.13.4",
     "babel-jest": "^29.7.0",
-    "copyfiles": "^2.4.1",
     "eslint": "^9.20.1",
-    "globals": "^15.15.0",
+    "eslint-config-prettier": "^10.0.1",
+    "eslint-plugin-only-warn": "^1.1.0",
+    "eslint-plugin-turbo": "^2.4.2",
     "jest": "^29.7.0",
     "tsup": "^8.3.6",
-    "tsx": "^4.19.2",
     "typescript": "^5.5.3",
     "typescript-eslint": "^8.24.0"
   },
   "scripts": {
-    "build": "tsup src/index.ts --format cjs,esm --dts && copyfiles -u 1 src/dictionaries/*.dic dist",
+    "build": "tsup",
     "lint": "eslint \"{src,tests}/**/*.ts\" --fix",
     "test": "jest"
   }

package/src/dicts.ts

@@ -0,0 +1,201 @@
+export type HyphenDict = string;
+
+import * as afZaDict from "./dictionaries/hyph_af_ZA.dic";
+export const afZa: HyphenDict = afZaDict.default;
+
+import * as asInDict from "./dictionaries/hyph_as_IN.dic";
+export const asIn: HyphenDict = asInDict.default;
+
+import * as beByDict from "./dictionaries/hyph_be_BY.dic";
+export const beBy: HyphenDict = beByDict.default;
+
+import * as bgBGDict from "./dictionaries/hyph_bg_BG.dic";
+export const bgBG: HyphenDict = bgBGDict.default;
+
+import * as caDict from "./dictionaries/hyph_ca.dic";
+export const ca: HyphenDict = caDict.default;
+
+// import * as cs_CZDict from "./dictionaries/hyph_cs_CZ.dic";
+// export const cs_CZ: HyphenDict = cs_CZDict.default;
+
+import * as daDKDict from "./dictionaries/hyph_da_DK.dic";
+export const daDK: HyphenDict = daDKDict.default;
+
+import * as deDTDict from "./dictionaries/hyph_de_AT.dic";
+export const deDT: HyphenDict = deDTDict.default;
+
+import * as deCHDict from "./dictionaries/hyph_de_CH.dic";
+export const deCH: HyphenDict = deCHDict.default;
+//
+import * as deDEDict from "./dictionaries/hyph_de_DE.dic";
+export const deDE: HyphenDict = deDEDict.default;
+
+// import * as elGRDict from "./dictionaries/hyph_el_GR.dic";
+// export const elGR: HyphenDict = elGRDict.default;
+
+import * as enGBDict from "./dictionaries/hyph_en_GB.dic";
+export const enGB: HyphenDict = enGBDict.default;
+//
+import * as enUSDict from "./dictionaries/hyph_en_US.dic";
+export const enUS: HyphenDict = enUSDict.default;
+//
+import * as eoDict from "./dictionaries/hyph_eo.dic";
+export const eo: HyphenDict = eoDict.default;
+
+import * as esDict from "./dictionaries/hyph_es.dic";
+export const es: HyphenDict = esDict.default;
+
+import * as etEEDict from "./dictionaries/hyph_et_EE.dic";
+export const etEE: HyphenDict = etEEDict.default;
+
+import * as frDict from "./dictionaries/hyph_fr.dic";
+export const fr: HyphenDict = frDict.default;
+
+import * as glDict from "./dictionaries/hyph_gl.dic";
+export const gl: HyphenDict = glDict.default;
+
+import * as hrHRDict from "./dictionaries/hyph_hr_HR.dic";
+export const hrHR: HyphenDict = hrHRDict.default;
+
+import * as huHUDict from "./dictionaries/hyph_hu_HU.dic";
+export const huHU: HyphenDict = huHUDict.default;
+
+import * as idIDDict from "./dictionaries/hyph_id_ID.dic";
+export const idID: HyphenDict = idIDDict.default;
+
+import * as isDict from "./dictionaries/hyph_is.dic";
+export const is: HyphenDict = isDict.default;
+
+import * as itITDict from "./dictionaries/hyph_it_IT.dic";
+export const itIT: HyphenDict = itITDict.default;
+
+import * as knINDict from "./dictionaries/hyph_kn_IN.dic";
+export const knIN: HyphenDict = knINDict.default;
+
+import * as ltDict from "./dictionaries/hyph_lt.dic";
+export const lt: HyphenDict = ltDict.default;
+
+import * as lvLVDict from "./dictionaries/hyph_lv_LV.dic";
+export const lvLV: HyphenDict = lvLVDict.default;
+
+import * as mnMNDict from "./dictionaries/hyph_mn_MN.dic";
+export const mnMN: HyphenDict = mnMNDict.default;
+
+import * as mrINDict from "./dictionaries/hyph_mr_IN.dic";
+export const mrIN: HyphenDict = mrINDict.default;
+
+import * as nbNODict from "./dictionaries/hyph_nb_NO.dic";
+export const nbNO: HyphenDict = nbNODict.default;
+
+import * as nlNLDict from "./dictionaries/hyph_nl_NL.dic";
+export const nlNL: HyphenDict = nlNLDict.default;
+
+import * as nnNODict from "./dictionaries/hyph_nn_NO.dic";
+export const nnNO: HyphenDict = nnNODict.default;
+
+import * as paINDict from "./dictionaries/hyph_pa_IN.dic";
+export const paIN: HyphenDict = paINDict.default;
+
+import * as plPLDict from "./dictionaries/hyph_pl_PL.dic";
+export const plPL: HyphenDict = plPLDict.default;
+
+import * as ptBRDict from "./dictionaries/hyph_pt_BR.dic";
+export const ptBR: HyphenDict = ptBRDict.default;
+
+import * as ptPTDict from "./dictionaries/hyph_pt_PT.dic";
+export const ptPT: HyphenDict = ptPTDict.default;
+
+import * as roRODict from "./dictionaries/hyph_ro_RO.dic";
+export const roRO: HyphenDict = roRODict.default;
+
+import * as ruRUDict from "./dictionaries/hyph_ru_RU.dic";
+export const ruRU: HyphenDict = ruRUDict.default;
+
+import * as saINDict from "./dictionaries/hyph_sa_IN.dic";
+export const saIN: HyphenDict = saINDict.default;
+
+import * as skSKDict from "./dictionaries/hyph_sk_SK.dic";
+export const skSK: HyphenDict = skSKDict.default;
+
+import * as slSIDict from "./dictionaries/hyph_sl_SI.dic";
+export const slSI: HyphenDict = slSIDict.default;
+
+import * as sqALDict from "./dictionaries/hyph_sq_AL.dic";
+export const sqAL: HyphenDict = sqALDict.default;
+
+import * as srDict from "./dictionaries/hyph_sr.dic";
+export const sr: HyphenDict = srDict.default;
+
+import * as srLatnDict from "./dictionaries/hyph_sr-Latn.dic";
+export const srLatn: HyphenDict = srLatnDict.default;
+
+import * as svDict from "./dictionaries/hyph_sv.dic";
+export const sv: HyphenDict = svDict.default;
+
+import * as teINDict from "./dictionaries/hyph_te_IN.dic";
+export const teIN: HyphenDict = teINDict.default;
+
+import * as thTHDict from "./dictionaries/hyph_th_TH.dic";
+export const thTH: HyphenDict = thTHDict.default;
+
+import * as ukUADict from "./dictionaries/hyph_uk_UA.dic";
+export const ukUA: HyphenDict = ukUADict.default;
+
+import * as zuZADict from "./dictionaries/hyph_zu_ZA.dic";
+export const zuZA: HyphenDict = zuZADict.default;
+
+export const dictionaries = {
+  af: afZa,
+  as: asIn,
+  be: beBy,
+  bg: bgBG,
+  ca,
+  // cs: cs_CZ,
+  da: daDK,
+  de: deDE,
+  de_DE: deDE,
+  de_CH: deCH,
+  de_AT: deDT,
+  // el: elGR,
+  en: enUS,
+  en_US: enUS,
+  en_GB: enGB,
+  eo,
+  es,
+  et: etEE,
+  fr,
+  gl,
+  hr: hrHR,
+  hu: huHU,
+  id: idID,
+  is,
+  it: itIT,
+  kn: knIN,
+  lt,
+  lv: lvLV,
+  mn: mnMN,
+  mr: mrIN,
+  nb: nbNO,
+  nl: nlNL,
+  nn: nnNO,
+  pa: paIN,
+  pl: plPL,
+  pt: ptBR,
+  ptPT: ptPT,
+  pt_BR: ptBR,
+  ro: roRO,
+  ru: ruRU,
+  sa: saIN,
+  sk: skSK,
+  sl: slSI,
+  sq: sqAL,
+  sr,
+  sr_Latn: srLatn,
+  sv,
+  te: teIN,
+  th: thTH,
+  uk: ukUA,
+  zu: zuZA,
+};
+
+export type Language = keyof typeof dictionaries;

package/src/index.ts

@@ -1,7 +1,11 @@
-import fs from "fs";
-import { join } from "path";
+import { HyphenDict, enUS, dictionaries, Language } from "./dicts";
+
+export * from "./dicts";
+
+// export function getDictionary(lang: string) {
+//   return import(`./dictionaries/hyph_${lang}.dic`);
+// }
 
-const DICTIONARY_PATH = join(__dirname, "dictionaries");
 const IGNORED = [
   "%",
   "#",
@@ -21,17 +25,13 @@ class DataInt {
   }
 }
 
-class HyphenDict {
+class HyphenDictParser {
   patterns: Map<string, [number, number[]]> = new Map();
   cache: Map<string, DataInt[]> = new Map();
   maxLen: number = 0;
 
-  constructor(filePath: string) {
-    if (!fs.existsSync(filePath)) {
-      throw new Error(`Dictionary file not found: ${filePath}`);
-    }
-
-    const lines = fs.readFileSync(filePath, "utf-8").split("\n");
+  constructor(content: string) {
+    const lines = content.split("\n");
     lines.forEach((line) => {
       line = line.trim();
       if (!line || IGNORED.some((i) => line.startsWith(i))) return;
@@ -101,61 +101,15 @@ class HyphenDict {
 export class TextHyphen {
   private readonly left: number;
   private readonly right: number;
-  private hd: HyphenDict;
-  readonly dictionaries: Record<string, string>;
-  private readonly lowercaseLangs: Record<string, string>;
+  private hd!: HyphenDictParser;
 
-  constructor(props?: { lang?: string; left?: number; right?: number }) {
-    const { lang = "en_US", left = 2, right = 2 } = props || {};
+  constructor(props?: { lang?: Language; left?: number; right?: number }) {
+    const { lang = "en", left = 2, right = 2 } = props || {};
     this.left = left;
     this.right = right;
-    this.dictionaries = this.loadDictionaries();
-    this.lowercaseLangs = Object.keys(this.dictionaries).reduce(
-      (acc, lang) => {
-        acc[lang.toLowerCase()] = lang;
-        return acc;
-      },
-      {} as Record<string, string>,
-    );
-    const fallback = this.getLanguageFallback(lang);
-    if (!fallback) {
-      throw new Error(`Language not found: ${lang}`);
-    }
-    this.hd = new HyphenDict(
-      join(DICTIONARY_PATH, this.dictionaries[fallback]!),
-    );
-  }
-
-  private loadDictionaries() {
-    const dictionaries: Record<string, string> = {};
-    for (const file of fs.readdirSync(DICTIONARY_PATH).sort()) {
-      const [name, ext] = file.split(".");
-      const lang = name!.replace("hyph_", "").replace("-", "_"); // File name format: hyph_LANG-COUNTRY.dic
-      const shortLang = lang.split("_")[0];
-      if (ext === "dic") {
-        dictionaries[lang] = file;
-        if (!dictionaries[shortLang!]) {
-          dictionaries[shortLang!] = file;
-        }
-      }
-    }
-    return dictionaries;
-  }
-
-  /**
-   * Get the fallback language for a given language.
-   * @param language
-   */
-  getLanguageFallback(language: string) {
-    const parts = language.replace("-", "_").toLowerCase().split("_");
-    while (parts.length) {
-      const currentLanguage = parts.join("_");
-      if (this.lowercaseLangs[currentLanguage]) {
-        return this.lowercaseLangs[currentLanguage];
-      }
-      parts.pop();
-    }
-    return undefined;
+    const dict = dictionaries[lang] || enUS;
+    // const dictStr = new TextDecoder().decode(dict);
+    this.hd = new HyphenDictParser(dict);
   }
 
   /**
@@ -199,7 +153,7 @@ export class TextHyphen {
    * @param word
    */
   variants(word: string): string[][] {
-    return this.iterate(word).toArray();
+    return Array.from(this.iterate(word));
   }
 
   /**

package/src/types.d.ts

@@ -0,0 +1,4 @@
+declare module "*.dic" {
+  const content: string;
+  export default content;
+}

package/tests/__mocks__/dic.js

@@ -0,0 +1,7 @@
+module.exports = {
+  process(sourceText, sourcePath, options) {
+    return {
+      code: `module.exports = ${JSON.stringify(sourceText)};`,
+    };
+  },
+};

package/tests/hyphen.test.ts

@@ -3,19 +3,19 @@ import { TextHyphen } from "../src";
 
 describe("hyphen tests", () => {
   it("should correctly hyphenate words using inserted method", () => {
-    const dic = new TextHyphen({ lang: "nl_NL" });
+    const dic = new TextHyphen({ lang: "nl" });
     expect(dic.inserted("lettergrepen")).toBe("let-ter-gre-pen");
   });
 
   it("should correctly wrap words", () => {
-    const dic = new TextHyphen({ lang: "nl_NL" });
+    const dic = new TextHyphen({ lang: "nl" });
     const actual = dic.wrap("autobandventieldopje", 11);
     const expected = ["autoband-", "ventieldopje"];
     expect(actual).toStrictEqual(expected);
   });
 
   it("should iterate hyphenation positions correctly", () => {
-    const dic = new TextHyphen({ lang: "nl_NL" });
+    const dic = new TextHyphen({ lang: "nl" });
     console.log(dic);
     const actual = dic.variants("Amsterdam");
     const expected = [
@@ -25,20 +25,6 @@ describe("hyphen tests", () => {
     expect(actual).toStrictEqual(expected);
   });
 
-  it("should use a fallback dictionary", () => {
-    const dic = new TextHyphen({ lang: "nl_NL-variant" });
-    const actual = dic.variants("Amsterdam");
-    const expected = [
-      ["Amster", "dam"],
-      ["Am", "sterdam"],
-    ];
-    expect(actual).toStrictEqual(expected);
-  });
-
-  it("should throw an error for a missing dictionary", () => {
-    expect(() => new TextHyphen({ lang: "mi_SS" })).toThrowError();
-  });
-
   // it('should use a custom dictionary', () => {
   //   const dic = new Hyphen('fr');
   //   expect(dic.inserted('autobandventieldopje')).to.not.equal(
@@ -53,19 +39,19 @@ describe("hyphen tests", () => {
 
   describe("should support left and right hyphenation constraints", () => {
     it("default", () => {
-      const dic = new TextHyphen({ lang: "nl_NL" });
+      const dic = new TextHyphen({ lang: "nl" });
       expect(dic.inserted("lettergrepen")).toBe("let-ter-gre-pen");
     });
     it("left", () => {
-      const dic = new TextHyphen({ lang: "nl_NL", left: 4 });
+      const dic = new TextHyphen({ lang: "nl", left: 4 });
       expect(dic.inserted("lettergrepen")).toBe("letter-gre-pen");
     });
     it("right", () => {
-      const dic = new TextHyphen({ lang: "nl_NL", right: 4 });
+      const dic = new TextHyphen({ lang: "nl", right: 4 });
       expect(dic.inserted("lettergrepen")).toBe("let-ter-grepen");
     });
     it("both", () => {
-      const dic = new TextHyphen({ lang: "nl_NL", left: 4, right: 4 });
+      const dic = new TextHyphen({ lang: "nl", left: 4, right: 4 });
       expect(dic.inserted("lettergrepen")).toBe("letter-grepen");
     });
   });
@@ -83,7 +69,7 @@ describe("hyphen tests", () => {
   // });
 
   it("should handle uppercase words correctly", () => {
-    const dic = new TextHyphen({ lang: "nl_NL" });
+    const dic = new TextHyphen({ lang: "nl" });
     expect(dic.inserted("LETTERGREPEN")).toBe("LET-TER-GRE-PEN");
   });
 
@@ -95,24 +81,4 @@ describe("hyphen tests", () => {
   //   ]);
   //   assert.strictEqual(dic.inserted("KULISSZA"), "KU-LISZ-SZA");
   // });
-
-  it("should be able to load all dictionaries", () => {
-    const hyp = new TextHyphen();
-    const dictionaries = hyp.dictionaries;
-    Object.keys(dictionaries).forEach((dict) => {
-      new TextHyphen({ lang: dict });
-    });
-  });
-
-  it("should correctly determine language fallbacks", () => {
-    const hyp = new TextHyphen();
-    expect(hyp.getLanguageFallback("en")).toBe("en");
-    expect(hyp.getLanguageFallback("en_US")).toBe("en_US");
-    expect(hyp.getLanguageFallback("en_FR")).toBe("en");
-    expect(hyp.getLanguageFallback("sr-Latn")).toBe("sr_Latn");
-    expect(hyp.getLanguageFallback("SR-LATN")).toBe("sr_Latn");
-    expect(hyp.getLanguageFallback("sr-Cyrl")).toBe("sr");
-    expect(hyp.getLanguageFallback("fr-Latn-FR")).toBe("fr");
-    expect(hyp.getLanguageFallback("en-US_variant1-x")).toBe("en_US");
-  });
 });

package/tsup.config.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+  entry: ["src/index.ts"],
+  format: ["cjs", "esm"],
+  dts: true,
+  sourcemap: true,
+  clean: true,
+  loader: {
+    ".dic": "text",
+  },
+});

package/.turbo/turbo-build.log

@@ -1,23 +0,0 @@
-
-
-> @lunarisapp/hyphen@0.0.2 build /Users/rsirokov/Projects/Lunaris/text-tools/packages/hyphen
-> tsup src/index.ts --format cjs,esm --dts && copyfiles -u 1 src/dictionaries/*.dic dist
-
-CLI Building entry: src/index.ts
-CLI Using tsconfig: ../../tsconfig.json
-CLI tsup v8.3.6
-CLI Target: esnext
-CJS Build start
-ESM Build start
-CJS You have emitDecoratorMetadata enabled but @swc/core was not installed, skipping swc plugin
-ESM You have emitDecoratorMetadata enabled but @swc/core was not installed, skipping swc plugin
-ESM dist/index.mjs 5.59 KB
-ESM ⚡️ Build success in 52ms
-CJS dist/index.js 7.26 KB
-CJS ⚡️ Build success in 52ms
-DTS Build start
-error TS18003: No inputs were found in config file 'tsconfig.json'. Specified 'include' paths were '["src/**/*.ts"]' and 'exclude' paths were '["node_modules","dist"]'.
-
-DTS ⚡️ Build success in 907ms
-DTS dist/index.d.ts  1.27 KB
-DTS dist/index.d.mts 1.27 KB

package/.turbo/turbo-lint.log

@@ -1,12 +0,0 @@
-
-
-> @lunaris/hyphen@0.0.1 lint /Users/rsirokov/Projects/Lunaris/text-tools/packages/hyphen
-> eslint "{src,tests}/**/*.ts" --fix
-
-(node:65030) [MODULE_TYPELESS_PACKAGE_JSON] Warning: Module type of file:///Users/rsirokov/Projects/Lunaris/text-tools/packages/hyphen/eslint.config.js?mtime=1739479673334 is not specified and it doesn't parse as CommonJS.
-Reparsing as ES module because module syntax was detected. This incurs a performance overhead.
-To eliminate this warning, add "type": "module" to /Users/rsirokov/Projects/Lunaris/text-tools/packages/hyphen/package.json.
-(Use `node --trace-warnings ...` to show where the warning was created)
-(node:65030) [MODULE_TYPELESS_PACKAGE_JSON] Warning: Module type of file:///Users/rsirokov/Projects/Lunaris/text-tools/eslint.config.js is not specified and it doesn't parse as CommonJS.
-Reparsing as ES module because module syntax was detected. This incurs a performance overhead.
-To eliminate this warning, add "type": "module" to /Users/rsirokov/Projects/Lunaris/text-tools/package.json.