lingotweaker-data 0.0.1 → 0.2.0

package/README.md

@@ -1,5 +1,53 @@
-# lingotweaker-data
+# LingoTweaker data
 
-Placeholder release. The real package ships one gzipped data pack per language
-for the LingoTweaker proofreading engine. See
-https://github.com/fiduswriter/LingoTweaker
+Runtime data for the LingoTweaker proofreading engine. This package is the
+**code-only loader**: the actual language packs ship in one small package per
+language (`lingotweaker-data-<lang>`, mirroring the per-language PyPI data
+distributions), so npm never has to carry a single multi-language tarball.
+Install the loader plus the languages you need:
+
+```sh
+npm install lingotweaker-data lingotweaker-data-en
+```
+
+```js
+const data = require("lingotweaker-data");
+const { Engine } = require("lingotweaker");
+
+const engine = new Engine("en-US", { dataDir: data.packPath("en") });
+for (const match of engine.check("I can heard you.").matches) {
+  console.log(match.rule_id, match.suggestions);
+}
+```
+
+```js
+import init, { LtEngine } from "lingotweaker-wasm";
+import { packPath } from "lingotweaker-data";
+import { readFileSync } from "node:fs";
+
+await init();
+const pack = readFileSync(packPath("en"));
+const engine = new LtEngine("en-US", pack, JSON.stringify({ today: new Date().toISOString() }));
+```
+
+- `packPath(lang)` — absolute path of a language pack, resolved from the
+  installed `lingotweaker-data-<lang>` package (which must be installed
+  separately; resolution starts next to this package and falls back to the
+  process working directory for pnpm/workspace layouts)
+- `languages()` — the base codes with a published pack
+- `dataManifest()` — version, upstream commit and per-language `bytes`/`sha256`;
+  the `file` paths are relative to the per-language package
+
+The packs are the same artifacts attached to each GitHub Release
+(`packs/<lang>.pack.gz`) and used by the browser demo, so all channels serve
+identical data.
+
+Note for layouts where package resolution cannot reach the per-language
+packages from inside this one (strict pnpm), declare
+`lingotweaker-data-<lang>` as a dependency of the consuming package so the
+package manager links it.
+
+Vendored rule data, dictionaries and models keep their upstream licenses; see
+`THIRD_PARTY_NOTICES.md` in the repository.
+
+License (packaging code): LGPL-2.1-or-later.

package/index.d.ts

@@ -0,0 +1,40 @@
+/** Loader for the LingoTweaker engine's runtime data (code-only package). */
+
+export interface DataFileEntry {
+  /**
+   * Path relative to the per-language package `lingotweaker-data-<lang>`
+   * (e.g. `packs/en.pack.gz`), not to this package.
+   */
+  file: string;
+  bytes: number;
+  sha256: string;
+}
+
+export interface DataLanguageEntry {
+  pack: DataFileEntry;
+}
+
+export interface LingoTweakerDataManifest {
+  version: string;
+  upstream_commit: string;
+  languages: Record<string, DataLanguageEntry>;
+}
+
+/** Languages with a published data pack (base codes: `en`, `de`, `gn`, …). */
+export function languages(): string[];
+
+/**
+ * Absolute path of the gzipped data pack for `lang`.
+ *
+ * The pack lives in the per-language package `lingotweaker-data-<lang>`,
+ * which must be installed separately (`npm install lingotweaker-data-<lang>`).
+ */
+export function packPath(lang: string): string;
+
+/**
+ * The data manifest (version, upstream commit, per-language sha256).
+ *
+ * The `file` paths are relative to the per-language package
+ * `lingotweaker-data-<lang>`, not to this package.
+ */
+export function dataManifest(): LingoTweakerDataManifest;

package/index.js

@@ -0,0 +1,77 @@
+"use strict";
+
+// Loader for the LingoTweaker engine's runtime data.
+//
+// This package is code-only: the packs ship in one small package per language
+// (`lingotweaker-data-<lang>`, mirroring the per-language PyPI data
+// distributions). Install the languages you need alongside it:
+//
+//   npm install lingotweaker-data lingotweaker-data-en
+//
+//   const data = require("lingotweaker-data");
+//   const { Engine } = require("lingotweaker");
+//   const engine = new Engine("en-US", { dataDir: data.packPath("en") });
+
+const { join } = require("node:path");
+const { createRequire } = require("node:module");
+const manifest = require("./manifest.json");
+
+// Resolve from this package's location first (npm's flat node_modules puts the
+// per-language packages next to it); fall back to the process working
+// directory for layouts where they are not siblings (pnpm, workspace hoisting
+// differences), where the per-language package is resolvable from the
+// consuming project but not from inside this package's store path.
+const hereRequire = createRequire(__filename);
+const cwdRequire = createRequire(join(process.cwd(), "index.js"));
+
+/** Languages with a published data pack (base codes: `en`, `de`, `gn`, …). */
+function languages() {
+  return Object.keys(manifest.languages ?? {}).sort();
+}
+
+/**
+ * Absolute path of the gzipped data pack for `lang`.
+ *
+ * The pack lives in the per-language package `lingotweaker-data-<lang>` at
+ * the manifest's `file` path (e.g. `packs/en.pack.gz`); it must be installed
+ * separately (`npm install lingotweaker-data-<lang>`).
+ */
+function packPath(lang) {
+  const entry = manifest.languages?.[lang];
+  if (!entry?.pack?.file) {
+    throw new Error(
+      `lingotweaker-data: no pack for ${JSON.stringify(lang)}; ` +
+        `available: ${languages().join(", ")}`,
+    );
+  }
+  const name = `lingotweaker-data-${lang}`;
+  let cause;
+  for (const req of [hereRequire, cwdRequire]) {
+    try {
+      return req.resolve(`${name}/${entry.pack.file}`);
+    } catch (err) {
+      cause = err;
+    }
+  }
+  throw new Error(
+    `lingotweaker-data: the ${JSON.stringify(lang)} pack is not installed; ` +
+      `run \`npm install ${name}\` (available: ${languages().join(", ")})`,
+    { cause },
+  );
+}
+
+/**
+ * The data manifest (version, upstream commit, per-language sha256).
+ *
+ * The `file` paths are relative to the per-language package
+ * `lingotweaker-data-<lang>`, not to this package.
+ */
+function dataManifest() {
+  return manifest;
+}
+
+module.exports = {
+  languages,
+  packPath,
+  manifest: dataManifest,
+};

package/index.mjs

@@ -0,0 +1,69 @@
+// Loader for the LingoTweaker engine's runtime data.
+//
+// This package is code-only: the packs ship in one small package per language
+// (`lingotweaker-data-<lang>`, mirroring the per-language PyPI data
+// distributions). Install the languages you need alongside it:
+//
+//   npm install lingotweaker-data lingotweaker-data-en
+//
+//   import { packPath } from "lingotweaker-data";
+//   import { Engine } from "lingotweaker";
+//   const engine = new Engine("en-US", { dataDir: packPath("en") });
+
+import { join } from "node:path";
+import { createRequire } from "node:module";
+
+// Resolve from this package's location first (npm's flat node_modules puts the
+// per-language packages next to it); fall back to the process working
+// directory for layouts where they are not siblings (pnpm, workspace hoisting
+// differences), where the per-language package is resolvable from the
+// consuming project but not from inside this package's store path.
+const hereRequire = createRequire(import.meta.url);
+const cwdRequire = createRequire(join(process.cwd(), "index.js"));
+const manifest = hereRequire("./manifest.json");
+
+/** Languages with a published data pack (base codes: `en`, `de`, `gn`, …). */
+export function languages() {
+  return Object.keys(manifest.languages ?? {}).sort();
+}
+
+/**
+ * Absolute path of the gzipped data pack for `lang`.
+ *
+ * The pack lives in the per-language package `lingotweaker-data-<lang>` at
+ * the manifest's `file` path (e.g. `packs/en.pack.gz`); it must be installed
+ * separately (`npm install lingotweaker-data-<lang>`).
+ */
+export function packPath(lang) {
+  const entry = manifest.languages?.[lang];
+  if (!entry?.pack?.file) {
+    throw new Error(
+      `lingotweaker-data: no pack for ${JSON.stringify(lang)}; ` +
+        `available: ${languages().join(", ")}`,
+    );
+  }
+  const name = `lingotweaker-data-${lang}`;
+  let cause;
+  for (const req of [hereRequire, cwdRequire]) {
+    try {
+      return req.resolve(`${name}/${entry.pack.file}`);
+    } catch (err) {
+      cause = err;
+    }
+  }
+  throw new Error(
+    `lingotweaker-data: the ${JSON.stringify(lang)} pack is not installed; ` +
+      `run \`npm install ${name}\` (available: ${languages().join(", ")})`,
+    { cause },
+  );
+}
+
+/**
+ * The data manifest (version, upstream commit, per-language sha256).
+ *
+ * The `file` paths are relative to the per-language package
+ * `lingotweaker-data-<lang>`, not to this package.
+ */
+export function dataManifest() {
+  return manifest;
+}

package/manifest.json

@@ -0,0 +1,272 @@
+{
+  "version": "0.2.0",
+  "upstream_commit": "7bd1f99b849b1de66845113010b3c9261c3f37e0",
+  "languages": {
+    "ar": {
+      "pack": {
+        "file": "packs/ar.pack.gz",
+        "bytes": 14796535,
+        "sha256": "49f50889896a59e8c22d4dc2f4403e1715b259acab11cca47a1602dc770dded0"
+      }
+    },
+    "ast": {
+      "pack": {
+        "file": "packs/ast.pack.gz",
+        "bytes": 965053,
+        "sha256": "3ea7e9f5d95e2bd399e9d92556cf7ee555a846d9f26f5fb983d656c59e97680e"
+      }
+    },
+    "be": {
+      "pack": {
+        "file": "packs/be.pack.gz",
+        "bytes": 934715,
+        "sha256": "1527268186811899b8ceb66532353e30094acd64cda1e2d832cc3bb704dd9ec5"
+      }
+    },
+    "br": {
+      "pack": {
+        "file": "packs/br.pack.gz",
+        "bytes": 2004587,
+        "sha256": "26c30386a95b60e049443c61705d30dcb21b85a40bc3a04d331d474244198884"
+      }
+    },
+    "ca": {
+      "pack": {
+        "file": "packs/ca.pack.gz",
+        "bytes": 11454474,
+        "sha256": "966930437bcddc9849f361ada380c854436c155e52d8488abdf902f16ed8e364"
+      }
+    },
+    "crh": {
+      "pack": {
+        "file": "packs/crh.pack.gz",
+        "bytes": 2564074,
+        "sha256": "4edbd8c127f2060ff35614f15f970bf78e1b6a93eba2c9eb70d02bd1c26091cd"
+      }
+    },
+    "da": {
+      "pack": {
+        "file": "packs/da.pack.gz",
+        "bytes": 2425873,
+        "sha256": "0fef17b0b7ecbfa603ec5d61ba0b58eef644b702c440a1fcb1d71b569ba19056"
+      }
+    },
+    "de": {
+      "pack": {
+        "file": "packs/de.pack.gz",
+        "bytes": 21967951,
+        "sha256": "e006baa5667867a3f9b4034d1beae131afc1cf93dc97482e93759f0d1004ffc8"
+      }
+    },
+    "el": {
+      "pack": {
+        "file": "packs/el.pack.gz",
+        "bytes": 830640,
+        "sha256": "3943fd3d379bd0afbcdf02f888007c9a5b57957081c46fcf6b5fc8dce688ab35"
+      }
+    },
+    "en": {
+      "pack": {
+        "file": "packs/en.pack.gz",
+        "bytes": 15250759,
+        "sha256": "4eca72fac3b1905e1f8e7462959016e60ad6933cf122e0472162804d14e2af62"
+      }
+    },
+    "eo": {
+      "pack": {
+        "file": "packs/eo.pack.gz",
+        "bytes": 518057,
+        "sha256": "f5151af3f570c6f7bb344f13d40f12bdfb4e6b4bb111f7207366ae61bf935fb8"
+      }
+    },
+    "es": {
+      "pack": {
+        "file": "packs/es.pack.gz",
+        "bytes": 3773233,
+        "sha256": "a01cb04af857160b94e5d9a23b36f5abec2e4767f161afdc70ae5b458848694e"
+      }
+    },
+    "fa": {
+      "pack": {
+        "file": "packs/fa.pack.gz",
+        "bytes": 338405,
+        "sha256": "d300c27db1677168f7b73f93c260743ef8b236d11f25aa7ec291503cd744d2ca"
+      }
+    },
+    "fr": {
+      "pack": {
+        "file": "packs/fr.pack.gz",
+        "bytes": 3291344,
+        "sha256": "bc112a7731c291be5054de0433246d7c1c9ee4b9aab50ff7016519fc7510a175"
+      }
+    },
+    "gl": {
+      "pack": {
+        "file": "packs/gl.pack.gz",
+        "bytes": 4638344,
+        "sha256": "a6d82aee58fe31e9c5384104b1a0d517d31d3ced1685c517f05f1540819c73b4"
+      }
+    },
+    "gn": {
+      "pack": {
+        "file": "packs/gn.pack.gz",
+        "bytes": 290293,
+        "sha256": "fe60b1c22072799e07cca2c861a045c09699c635add4198f71f9733df6e5f10d"
+      }
+    },
+    "is": {
+      "pack": {
+        "file": "packs/is.pack.gz",
+        "bytes": 927358,
+        "sha256": "2822e39529d854d8635e83250e09f105240a0e4bfb95b7bde30d196d06d035f3"
+      }
+    },
+    "it": {
+      "pack": {
+        "file": "packs/it.pack.gz",
+        "bytes": 1115580,
+        "sha256": "bb3906451dc45bee7b738ef492cc18dff3d7f967e1c902439291af1598a722c2"
+      }
+    },
+    "ja": {
+      "pack": {
+        "file": "packs/ja.pack.gz",
+        "bytes": 10708382,
+        "sha256": "8e9bfc15c94b59a5bbb4270033febed8c895db11f4637d93d71f1a8577fd8b90"
+      }
+    },
+    "km": {
+      "pack": {
+        "file": "packs/km.pack.gz",
+        "bytes": 808549,
+        "sha256": "db450508e2e1d9e8a77de652b0b7d9b0b8cbfa2538de27ba446b620f7f8634cb"
+      }
+    },
+    "lt": {
+      "pack": {
+        "file": "packs/lt.pack.gz",
+        "bytes": 604799,
+        "sha256": "30794e3cd0bd8350420a4f2875bd9e05ffc6c1f4306889feec5d9faebf7831fb"
+      }
+    },
+    "ml": {
+      "pack": {
+        "file": "packs/ml.pack.gz",
+        "bytes": 959275,
+        "sha256": "f8126ec84bc57fb230a3f7256e8d3a9af73044f6e5c0b814852913f6f570e885"
+      }
+    },
+    "nl": {
+      "pack": {
+        "file": "packs/nl.pack.gz",
+        "bytes": 38077206,
+        "sha256": "c1db973c09be9b98e4447323a088be354267dd2edeb5653efa1f67791958caa1"
+      }
+    },
+    "nn": {
+      "pack": {
+        "file": "packs/nn.pack.gz",
+        "bytes": 1827886,
+        "sha256": "d171a11030221db5e035baa3524722af06c4be0c94630df499eba05d16391757"
+      }
+    },
+    "no": {
+      "pack": {
+        "file": "packs/no.pack.gz",
+        "bytes": 6244378,
+        "sha256": "9b2dd53af5632be218961d85ab6b15bf029c5e222e789b11c13fe705a027ea47"
+      }
+    },
+    "nrd": {
+      "pack": {
+        "file": "packs/nrd.pack.gz",
+        "bytes": 5015821,
+        "sha256": "bd0fc69f330c6bc596bef407fabee5384b5b0e49b65bd448becf85fecda4769c"
+      }
+    },
+    "pl": {
+      "pack": {
+        "file": "packs/pl.pack.gz",
+        "bytes": 6799941,
+        "sha256": "73ac39f1e2ec91dbdb08e5615f293afcde22070af11110447909ba194d814623"
+      }
+    },
+    "pt": {
+      "pack": {
+        "file": "packs/pt.pack.gz",
+        "bytes": 6455461,
+        "sha256": "0f7b0a6d6f5c2bea7e1c1d1b21890fb1c81ddc0a4afa051f30cb703b380a3e2f"
+      }
+    },
+    "ro": {
+      "pack": {
+        "file": "packs/ro.pack.gz",
+        "bytes": 1661169,
+        "sha256": "f4994bfad320a0acbb283635ea862bd288132fd0dc8e483acc18325a46fe6917"
+      }
+    },
+    "ru": {
+      "pack": {
+        "file": "packs/ru.pack.gz",
+        "bytes": 6293233,
+        "sha256": "57c5cd17781fd0a74acb8415eaaa3b22598fb2cdbf270e6763f1d023443b04f9"
+      }
+    },
+    "sk": {
+      "pack": {
+        "file": "packs/sk.pack.gz",
+        "bytes": 2859673,
+        "sha256": "30a6f391d6634230f17c7b2456d2215fb82f379a63ccf56b1b1a4b85aaf1be73"
+      }
+    },
+    "sl": {
+      "pack": {
+        "file": "packs/sl.pack.gz",
+        "bytes": 528116,
+        "sha256": "a957b0424e392d7605132d5f977a74d0eed8500e32d20c9fa64a96c8fd31692f"
+      }
+    },
+    "sr": {
+      "pack": {
+        "file": "packs/sr.pack.gz",
+        "bytes": 6662107,
+        "sha256": "82a22f113ab7b34fac084dee6d295a4a967e56a696270172e9f2c9525f077284"
+      }
+    },
+    "sv": {
+      "pack": {
+        "file": "packs/sv.pack.gz",
+        "bytes": 2027483,
+        "sha256": "a4941d8faa6b5cc9c7f5945b5716ab220787474cd3a0c7002f74785cdeb59b34"
+      }
+    },
+    "ta": {
+      "pack": {
+        "file": "packs/ta.pack.gz",
+        "bytes": 314872,
+        "sha256": "db8f327b9a2f927945430612a8f37d4c5db30540061d16da9aadccf879c2b5f0"
+      }
+    },
+    "tl": {
+      "pack": {
+        "file": "packs/tl.pack.gz",
+        "bytes": 401001,
+        "sha256": "5de86a3656536f89f39cce552c8d6857e51a1bfe7c831b6e111875f124409fcd"
+      }
+    },
+    "uk": {
+      "pack": {
+        "file": "packs/uk.pack.gz",
+        "bytes": 7129147,
+        "sha256": "51d196a669494ec273d394d6dfde1796b19e1c2d0478654038caca63864dfb2a"
+      }
+    },
+    "zh": {
+      "pack": {
+        "file": "packs/zh.pack.gz",
+        "bytes": 14113621,
+        "sha256": "4dc30fe21f758bcdad1899dd005c6d049539bb87601b4059a68855231dd72ce2"
+      }
+    }
+  }
+}

package/package.json

@@ -1,11 +1,40 @@
 {
   "name": "lingotweaker-data",
-  "version": "0.0.1",
-  "description": "Runtime language data packs for the LingoTweaker proofreading engine",
+  "version": "0.2.0",
+  "description": "Loader for the LingoTweaker proofreading engine's runtime data packs (the packs ship in per-language lingotweaker-data-<lang> packages)",
   "license": "LGPL-2.1-or-later",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fiduswriter/LingoTweaker.git"
   },
-  "homepage": "https://fiduswriter.github.io/LingoTweaker/"
-}
+  "homepage": "https://fiduswriter.github.io/LingoTweaker/",
+  "keywords": [
+    "proofreading",
+    "grammar",
+    "spellcheck",
+    "nlp",
+    "language",
+    "data"
+  ],
+  "main": "./index.js",
+  "types": "./index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./index.d.ts",
+      "import": "./index.mjs",
+      "require": "./index.js"
+    },
+    "./manifest.json": "./manifest.json",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "manifest.json",
+    "index.js",
+    "index.mjs",
+    "index.d.ts",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">= 18"
+  }
+}