@gukhanmun/opendict-cdb 0.2.0-dev.0 → 0.2.0-dev.119

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/ATTRIBUTION.md CHANGED
@@ -1,8 +1,8 @@
1
1
  OpenDict CDB package license and attribution
2
2
  ============================================
3
3
 
4
- This package contains Gukhanmun package code and bundled Open Korean Dictionary
5
- (우리말샘) CDB data.
4
+ This package contains Gukhanmun package code and bundled *Open Korean
5
+ Dictionary* (우리말샘) CDB data.
6
6
 
7
7
 
8
8
  Code license
@@ -15,14 +15,14 @@ repository root *LICENSE* file for the full license text.
15
15
  Data attribution
16
16
  ----------------
17
17
 
18
- The bundled dictionary data is derived from the Open Korean Dictionary
18
+ The bundled dictionary data is derived from the *Open Korean Dictionary*
19
19
  (우리말샘) JSON dump published by the National Institute of Korean Language.
20
20
 
21
- - Source: National Institute of Korean Language, Open Korean Dictionary
21
+ - Source: National Institute of Korean Language, *Open Korean Dictionary*
22
22
  (우리말샘), <https://opendict.korean.go.kr/>
23
23
  - Snapshot: *전체 내려받기\_우리말샘\_json\_20260503.zip*
24
24
  - Data license: Creative Commons Attribution-ShareAlike 2.0 Korea
25
25
  (CC BY-SA 2.0 KR), <https://creativecommons.org/licenses/by-sa/2.0/kr/>
26
26
 
27
27
  The generated CDB files in this package are modified dictionary snapshots
28
- produced by Gukhanmun's Open Korean Dictionary extractor.
28
+ produced by Gukhanmun's *Open Korean Dictionary* extractor.
package/README.md CHANGED
@@ -5,8 +5,8 @@
5
5
  [![npm][npm badge]][npm]
6
6
  [![License: GPL-3.0-or-later AND CC BY-SA 2.0 KR][license badge]][GPL]
7
7
 
8
- Open Korean Dictionary (우리말샘) categories bundled as CDB binaries for
9
- `@gukhanmun/wasm` and `@gukhanmun/napi`.
8
+ *Open Korean Dictionary* (우리말샘) categories compiled into CDB binaries for
9
+ use with `@gukhanmun/wasm` or `@gukhanmun/napi`.
10
10
 
11
11
  [JSR badge]: https://jsr.io/badges/@gukhanmun/opendict-cdb
12
12
  [JSR]: https://jsr.io/@gukhanmun/opendict-cdb
@@ -16,6 +16,18 @@ Open Korean Dictionary (우리말샘) categories bundled as CDB binaries for
16
16
  [GPL]: https://www.gnu.org/licenses/gpl-3.0.html
17
17
 
18
18
 
19
+ Installation
20
+ ------------
21
+
22
+ ~~~~ bash
23
+ npm add @gukhanmun/opendict-cdb
24
+ pnpm add @gukhanmun/opendict-cdb
25
+ yarn add @gukhanmun/opendict-cdb
26
+ bun add @gukhanmun/opendict-cdb
27
+ deno add jsr:@gukhanmun/opendict-cdb
28
+ ~~~~
29
+
30
+
19
31
  Usage
20
32
  -----
21
33
 
@@ -27,18 +39,61 @@ const g = await load({
27
39
  preset: "ko-kp",
28
40
  dictionaries: [await opendictNorthKoreanCdb()],
29
41
  });
30
- console.log(g.convert("歷史와 來日"));
42
+ console.log(g.convert("歷史와 來日")); // "력사와 래일"
31
43
  ~~~~
32
44
 
33
- The package exports separate helpers for the 일반어, 북한어, 방언, and 옛말
34
- categories so applications can choose only the dictionaries they need.
45
+ The package exports separate helper functions for the 一般語, 北韓語, 方言,
46
+ and 옛말 categories, so you can select and load only the categories you need:
47
+
48
+ - `opendictGeneralCdb()`
49
+ - `opendictNorthKoreanCdb()`
50
+ - `opendictDialectCdb()`
51
+ - `opendictArchaicCdb()`
52
+
53
+ Each function returns a `DictionarySource` record (a `{ format: "cdb", bytes: Uint8Array }`
54
+ object).
55
+
56
+ ### Raw bytes and URLs
57
+
58
+ If you only need raw `Uint8Array` bytes, use the `*Bytes()` helpers:
59
+
60
+ - `opendictGeneralCdbBytes()`
61
+ - `opendictNorthKoreanCdbBytes()`
62
+ - `opendictDialectCdbBytes()`
63
+ - `opendictArchaicCdbBytes()`
64
+
65
+ To get the package URL of a CDB binary, use the `*Url` constants:
66
+
67
+ - `opendictGeneralCdbUrl`
68
+ - `opendictNorthKoreanCdbUrl`
69
+ - `opendictDialectCdbUrl`
70
+ - `opendictArchaicCdbUrl`
71
+
72
+ The `*Bytes()` helpers read the binary from disk using `node:fs/promises` if loaded from
73
+ a `file:` URL (default for npm installations), and fall back to `fetch` for other protocols.
74
+
75
+ The bundled binaries ship gzip-compressed (the `*Url` constants point at
76
+ *\*.cdb.gz* files) to stay within the JSR per-file size limit. The `*Bytes()`
77
+ helpers inflate them transparently, so their return value is always the raw
78
+ CDB; only reach past them to the `*Url` constants if you intend to handle the
79
+ gzip yourself.
80
+
81
+
82
+ Relation to `@gukhanmun/opendict-fst`
83
+ -------------------------------------
84
+
85
+ Both packages contain the same dictionary data in different binary formats. The
86
+ CDB format has O(1) lookup and a layout that is straightforward to inspect
87
+ manually. The FST format supports prefix streaming and is generally preferred
88
+ for the lattice segmenter. Either works; choose based on your performance
89
+ profile or integration constraints.
35
90
 
36
91
 
37
92
  Data attribution
38
93
  ----------------
39
94
 
40
- The bundled dictionary data is extracted from the National Institute of Korean
41
- Language's Open Korean Dictionary (우리말샘) JSON dump dated 2026-05-03. See
95
+ The bundled dictionary data is derived from the National Institute of Korean
96
+ Language's *Open Korean Dictionary* (우리말샘) JSON dump dated 2026-05-03. See
42
97
  *ATTRIBUTION.md* for source and license details.
43
98
 
44
99
 
Binary file
Binary file
Binary file
package/dist/index.d.ts CHANGED
@@ -53,13 +53,31 @@ interface FileDictionarySource {
53
53
  //#endregion
54
54
  //#region index.d.ts
55
55
 
56
- /** URL of the bundled Open Korean Dictionary 일반어 CDB binary. */
56
+ /**
57
+ * URL of the bundled Open Korean Dictionary 一般語 CDB binary.
58
+ *
59
+ * The bundled artifact is stored gzip-compressed (hence the `.cdb.gz`
60
+ * extension) to stay within registry per-file size limits;
61
+ * {@link opendictCdbBytes} inflates it transparently.
62
+ */
57
63
  declare const opendictGeneralCdbUrl: URL;
58
- /** URL of the bundled Open Korean Dictionary 북한어 CDB binary. */
64
+ /**
65
+ * URL of the bundled Open Korean Dictionary 北韓語 CDB binary.
66
+ *
67
+ * Stored gzip-compressed; {@link opendictCdbBytes} inflates it transparently.
68
+ */
59
69
  declare const opendictNorthKoreanCdbUrl: URL;
60
- /** URL of the bundled Open Korean Dictionary 방언 CDB binary. */
70
+ /**
71
+ * URL of the bundled Open Korean Dictionary 方言 CDB binary.
72
+ *
73
+ * Stored gzip-compressed; {@link opendictCdbBytes} inflates it transparently.
74
+ */
61
75
  declare const opendictDialectCdbUrl: URL;
62
- /** URL of the bundled Open Korean Dictionary 옛말 CDB binary. */
76
+ /**
77
+ * URL of the bundled Open Korean Dictionary 옛말 CDB binary.
78
+ *
79
+ * Stored gzip-compressed; {@link opendictCdbBytes} inflates it transparently.
80
+ */
63
81
  declare const opendictArchaicCdbUrl: URL;
64
82
  /**
65
83
  * Loads an Open Korean Dictionary CDB binary as raw bytes.
@@ -68,6 +86,12 @@ declare const opendictArchaicCdbUrl: URL;
68
86
  * Node.js, Deno, or Bun; in other runtimes (e.g. browsers) and for all
69
87
  * other schemes the bytes are retrieved with `fetch`.
70
88
  *
89
+ * The bundled binaries are stored gzip-compressed to stay within registry
90
+ * per-file size limits. Bytes that begin with the gzip magic number are
91
+ * therefore inflated transparently, so the returned value is always the raw
92
+ * CDB ready to hand to `load`; bytes that are not gzip-compressed are returned
93
+ * unchanged.
94
+ *
71
95
  * @param url Location of the CDB binary to read.
72
96
  * @returns The CDB binary as a `Uint8Array`.
73
97
  */
package/dist/index.js CHANGED
@@ -1,16 +1,57 @@
1
1
  //#region index.ts
2
- /** URL of the bundled Open Korean Dictionary 일반어 CDB binary. */
3
- const opendictGeneralCdbUrl = new URL("./general.cdb", import.meta.url);
4
- /** URL of the bundled Open Korean Dictionary 북한어 CDB binary. */
5
- const opendictNorthKoreanCdbUrl = new URL("./north-korean.cdb", import.meta.url);
6
- /** URL of the bundled Open Korean Dictionary 방언 CDB binary. */
7
- const opendictDialectCdbUrl = new URL("./dialect.cdb", import.meta.url);
8
- /** URL of the bundled Open Korean Dictionary 옛말 CDB binary. */
9
- const opendictArchaicCdbUrl = new URL("./archaic.cdb", import.meta.url);
2
+ /**
3
+ * URL of the bundled Open Korean Dictionary 一般語 CDB binary.
4
+ *
5
+ * The bundled artifact is stored gzip-compressed (hence the `.cdb.gz`
6
+ * extension) to stay within registry per-file size limits;
7
+ * {@link opendictCdbBytes} inflates it transparently.
8
+ */
9
+ const opendictGeneralCdbUrl = new URL("./general.cdb.gz", import.meta.url);
10
+ /**
11
+ * URL of the bundled Open Korean Dictionary 北韓語 CDB binary.
12
+ *
13
+ * Stored gzip-compressed; {@link opendictCdbBytes} inflates it transparently.
14
+ */
15
+ const opendictNorthKoreanCdbUrl = new URL("./north-korean.cdb.gz", import.meta.url);
16
+ /**
17
+ * URL of the bundled Open Korean Dictionary 方言 CDB binary.
18
+ *
19
+ * Stored gzip-compressed; {@link opendictCdbBytes} inflates it transparently.
20
+ */
21
+ const opendictDialectCdbUrl = new URL("./dialect.cdb.gz", import.meta.url);
22
+ /**
23
+ * URL of the bundled Open Korean Dictionary 옛말 CDB binary.
24
+ *
25
+ * Stored gzip-compressed; {@link opendictCdbBytes} inflates it transparently.
26
+ */
27
+ const opendictArchaicCdbUrl = new URL("./archaic.cdb.gz", import.meta.url);
10
28
  function canReadLocalFile() {
11
29
  const globals = globalThis;
12
30
  return globals.Deno !== void 0 || typeof globals.process?.versions?.bun === "string" || typeof globals.process?.versions?.node === "string";
13
31
  }
32
+ async function readBytes(url) {
33
+ if (url.protocol === "file:" && canReadLocalFile()) {
34
+ const buf = await (await import(
35
+ /* webpackIgnore: true */
36
+ "node:fs/promises"
37
+ )).readFile(url);
38
+ return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
39
+ }
40
+ const response = await fetch(url);
41
+ if (!response.ok) throw new Error(`Failed to fetch opendict CDB: HTTP ${response.status}`);
42
+ return new Uint8Array(await response.arrayBuffer());
43
+ }
44
+ function isGzip(bytes) {
45
+ return bytes.length >= 2 && bytes[0] === 31 && bytes[1] === 139;
46
+ }
47
+ async function gunzip(bytes) {
48
+ const decompressor = new DecompressionStream("gzip");
49
+ const inflated = new Response(decompressor.readable).arrayBuffer();
50
+ const writer = decompressor.writable.getWriter();
51
+ await writer.write(bytes);
52
+ await writer.close();
53
+ return new Uint8Array(await inflated);
54
+ }
14
55
  /**
15
56
  * Loads an Open Korean Dictionary CDB binary as raw bytes.
16
57
  *
@@ -18,20 +59,18 @@ function canReadLocalFile() {
18
59
  * Node.js, Deno, or Bun; in other runtimes (e.g. browsers) and for all
19
60
  * other schemes the bytes are retrieved with `fetch`.
20
61
  *
62
+ * The bundled binaries are stored gzip-compressed to stay within registry
63
+ * per-file size limits. Bytes that begin with the gzip magic number are
64
+ * therefore inflated transparently, so the returned value is always the raw
65
+ * CDB ready to hand to `load`; bytes that are not gzip-compressed are returned
66
+ * unchanged.
67
+ *
21
68
  * @param url Location of the CDB binary to read.
22
69
  * @returns The CDB binary as a `Uint8Array`.
23
70
  */
24
71
  async function opendictCdbBytes(url) {
25
- if (url.protocol === "file:" && canReadLocalFile()) {
26
- const buf = await (await import(
27
- /* webpackIgnore: true */
28
- "node:fs/promises"
29
- )).readFile(url);
30
- return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
31
- }
32
- const response = await fetch(url);
33
- if (!response.ok) throw new Error(`Failed to fetch opendict CDB: HTTP ${response.status}`);
34
- return new Uint8Array(await response.arrayBuffer());
72
+ const bytes = await readBytes(url);
73
+ return isGzip(bytes) ? await gunzip(bytes) : bytes;
35
74
  }
36
75
  /**
37
76
  * Loads the bundled Open Korean Dictionary 일반어 CDB binary as raw bytes.
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gukhanmun/opendict-cdb",
3
- "version": "0.2.0-dev.0",
3
+ "version": "0.2.0-dev.119",
4
4
  "description": "Open Korean Dictionary categories bundled as CDB binaries for @gukhanmun/wasm and @gukhanmun/napi.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -9,10 +9,10 @@
9
9
  "import": "./dist/index.js",
10
10
  "default": "./dist/index.js"
11
11
  },
12
- "./general.cdb": "./dist/general.cdb",
13
- "./north-korean.cdb": "./dist/north-korean.cdb",
14
- "./dialect.cdb": "./dist/dialect.cdb",
15
- "./archaic.cdb": "./dist/archaic.cdb"
12
+ "./general.cdb.gz": "./dist/general.cdb.gz",
13
+ "./north-korean.cdb.gz": "./dist/north-korean.cdb.gz",
14
+ "./dialect.cdb.gz": "./dist/dialect.cdb.gz",
15
+ "./archaic.cdb.gz": "./dist/archaic.cdb.gz"
16
16
  },
17
17
  "files": [
18
18
  "ATTRIBUTION.md",
@@ -52,10 +52,10 @@
52
52
  "@gukhanmun/types": "*"
53
53
  },
54
54
  "devDependencies": {
55
- "@gukhanmun/types": "0.2.0-dev.0"
55
+ "@gukhanmun/types": "0.2.0-dev.119+50f854d"
56
56
  },
57
57
  "scripts": {
58
58
  "prebuild": "mise run -C ../.. opendict-cdb-data",
59
- "build": "tsdown && node -e \"const fs=require('node:fs');fs.mkdirSync('dist',{recursive:true});for(const f of ['general.cdb','north-korean.cdb','dialect.cdb','archaic.cdb'])fs.copyFileSync(f,'dist/'+f)\""
59
+ "build": "tsdown && node -e \"const fs=require('node:fs');fs.mkdirSync('dist',{recursive:true});for(const f of ['general.cdb.gz','north-korean.cdb.gz','dialect.cdb.gz','archaic.cdb.gz'])fs.copyFileSync(f,'dist/'+f)\""
60
60
  }
61
61
  }
package/dist/archaic.cdb DELETED
Binary file
package/dist/dialect.cdb DELETED
Binary file
package/dist/general.cdb DELETED
Binary file
Binary file