@pipedog/sentencex-wasm 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +25 -0
- package/LICENSE-sentencex +21 -0
- package/NOTICE +9 -0
- package/README.md +131 -2
- package/THIRD_PARTY_LICENSES.md +1060 -0
- package/dist/api.d.ts +11 -0
- package/dist/api.js +24 -0
- package/dist/engine.d.ts +42 -0
- package/dist/engine.js +105 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +49 -0
- package/dist/wasm_bytes.d.ts +5 -0
- package/dist/wasm_bytes.js +9 -0
- package/dist/workerd.d.ts +13 -0
- package/dist/workerd.js +26 -0
- package/package.json +55 -4
- package/rust/Cargo.lock +312 -0
- package/rust/Cargo.toml +27 -0
- package/rust/src/lib.rs +195 -0
- package/wasm/sentencex.wasm +0 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export type { SentencexApi } from "./api.ts";
|
|
2
|
+
export { type SentenceBoundary, SentencexEngine } from "./engine.ts";
|
|
3
|
+
/** Split `text` into sentences (sentencex rules for `language`, e.g. "en"). */
|
|
4
|
+
export declare const segment: (language: string, text: string) => string[];
|
|
5
|
+
/** Detailed sentence boundaries with JavaScript (UTF-16) string indices. */
|
|
6
|
+
export declare const getSentenceBoundaries: (language: string, text: string) => import("./engine.ts").SentenceBoundary[];
|
|
7
|
+
/** Alias with the `sentencex` npm package's name. */
|
|
8
|
+
export declare const get_sentence_boundaries: (language: string, text: string) => import("./engine.ts").SentenceBoundary[];
|
|
9
|
+
/** Replace the bundled module (a `WebAssembly.Module`; bytes cannot be compiled on workerd). */
|
|
10
|
+
export declare const initSentencex: (source?: WebAssembly.Module | BufferSource) => import("./engine.ts").SentencexEngine;
|
|
11
|
+
/** The shared engine. */
|
|
12
|
+
export declare const getSentencex: () => import("./engine.ts").SentencexEngine;
|
|
13
|
+
//# sourceMappingURL=workerd.d.ts.map
|
package/dist/workerd.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// Copyright (c) 2026, pipedog contributors
|
|
2
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
3
|
+
/**
|
|
4
|
+
* Cloudflare Workers entry (the "workerd" export condition).
|
|
5
|
+
*
|
|
6
|
+
* workerd forbids compiling WebAssembly from bytes at runtime, so the module is
|
|
7
|
+
* imported as a pre-compiled `WebAssembly.Module` (wrangler's default rule for
|
|
8
|
+
* `.wasm`) and instantiated synchronously when this module loads.
|
|
9
|
+
* `initSentencex()` accepts only a `WebAssembly.Module` here.
|
|
10
|
+
*/
|
|
11
|
+
import wasm from "../wasm/sentencex.wasm";
|
|
12
|
+
import { createApi } from "./api.js";
|
|
13
|
+
export { SentencexEngine } from "./engine.js";
|
|
14
|
+
const api = createApi(() => wasm);
|
|
15
|
+
api.getSentencex();
|
|
16
|
+
/** Split `text` into sentences (sentencex rules for `language`, e.g. "en"). */
|
|
17
|
+
export const segment = api.segment;
|
|
18
|
+
/** Detailed sentence boundaries with JavaScript (UTF-16) string indices. */
|
|
19
|
+
export const getSentenceBoundaries = api.getSentenceBoundaries;
|
|
20
|
+
/** Alias with the `sentencex` npm package's name. */
|
|
21
|
+
export const get_sentence_boundaries = api.getSentenceBoundaries;
|
|
22
|
+
/** Replace the bundled module (a `WebAssembly.Module`; bytes cannot be compiled on workerd). */
|
|
23
|
+
export const initSentencex = api.initSentencex;
|
|
24
|
+
/** The shared engine. */
|
|
25
|
+
export const getSentencex = api.getSentencex;
|
|
26
|
+
//# sourceMappingURL=workerd.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,57 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pipedog/sentencex-wasm",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Wikimedia sentencex sentence segmentation (the Rust core, compiled to WebAssembly) for pipedog: synchronous segment() and getSentenceBoundaries() on Bun, Node, Deno and Cloudflare Workers.",
|
|
5
|
+
"license": "BSD-2-Clause AND MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/polvallverdu/pipedog.git",
|
|
9
|
+
"directory": "packages/sentencex-wasm"
|
|
10
|
+
},
|
|
11
|
+
"homepage": "https://github.com/polvallverdu/pipedog/tree/main/packages/sentencex-wasm#readme",
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/polvallverdu/pipedog/issues"
|
|
14
|
+
},
|
|
15
|
+
"type": "module",
|
|
16
|
+
"sideEffects": false,
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"types": "./dist/index.d.ts",
|
|
20
|
+
"workerd": "./dist/workerd.js",
|
|
21
|
+
"default": "./dist/index.js"
|
|
22
|
+
},
|
|
23
|
+
"./engine": {
|
|
24
|
+
"types": "./dist/engine.d.ts",
|
|
25
|
+
"default": "./dist/engine.js"
|
|
26
|
+
},
|
|
27
|
+
"./sentencex.wasm": "./wasm/sentencex.wasm"
|
|
28
|
+
},
|
|
29
|
+
"files": [
|
|
30
|
+
"dist",
|
|
31
|
+
"wasm",
|
|
32
|
+
"rust/Cargo.toml",
|
|
33
|
+
"rust/Cargo.lock",
|
|
34
|
+
"rust/src",
|
|
35
|
+
"LICENSE-sentencex",
|
|
36
|
+
"LICENSE",
|
|
37
|
+
"NOTICE",
|
|
38
|
+
"THIRD_PARTY_LICENSES.md"
|
|
39
|
+
],
|
|
40
|
+
"scripts": {
|
|
41
|
+
"build:wasm": "bun scripts/build-wasm.ts",
|
|
42
|
+
"test": "bun test"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"sentencex": "1.0.32"
|
|
46
|
+
},
|
|
47
|
+
"pipecat": {
|
|
48
|
+
"runtimes": [
|
|
49
|
+
"bun",
|
|
50
|
+
"node",
|
|
51
|
+
"deno",
|
|
52
|
+
"workerd"
|
|
53
|
+
]
|
|
54
|
+
},
|
|
55
|
+
"main": "./dist/index.js",
|
|
56
|
+
"types": "./dist/index.d.ts"
|
|
57
|
+
}
|
package/rust/Cargo.lock
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.4"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "anstream"
|
|
16
|
+
version = "1.0.0"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"anstyle",
|
|
21
|
+
"anstyle-parse",
|
|
22
|
+
"anstyle-query",
|
|
23
|
+
"anstyle-wincon",
|
|
24
|
+
"colorchoice",
|
|
25
|
+
"is_terminal_polyfill",
|
|
26
|
+
"utf8parse",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[[package]]
|
|
30
|
+
name = "anstyle"
|
|
31
|
+
version = "1.0.14"
|
|
32
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
33
|
+
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
|
34
|
+
|
|
35
|
+
[[package]]
|
|
36
|
+
name = "anstyle-parse"
|
|
37
|
+
version = "1.0.0"
|
|
38
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
39
|
+
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
|
40
|
+
dependencies = [
|
|
41
|
+
"utf8parse",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[[package]]
|
|
45
|
+
name = "anstyle-query"
|
|
46
|
+
version = "1.1.5"
|
|
47
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
48
|
+
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
|
49
|
+
dependencies = [
|
|
50
|
+
"windows-sys",
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
[[package]]
|
|
54
|
+
name = "anstyle-wincon"
|
|
55
|
+
version = "3.0.11"
|
|
56
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
57
|
+
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
|
58
|
+
dependencies = [
|
|
59
|
+
"anstyle",
|
|
60
|
+
"once_cell_polyfill",
|
|
61
|
+
"windows-sys",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[[package]]
|
|
65
|
+
name = "clap"
|
|
66
|
+
version = "4.6.7"
|
|
67
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
68
|
+
checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
|
|
69
|
+
dependencies = [
|
|
70
|
+
"clap_builder",
|
|
71
|
+
"clap_derive",
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
[[package]]
|
|
75
|
+
name = "clap_builder"
|
|
76
|
+
version = "4.6.7"
|
|
77
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
78
|
+
checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
|
|
79
|
+
dependencies = [
|
|
80
|
+
"anstream",
|
|
81
|
+
"anstyle",
|
|
82
|
+
"clap_lex",
|
|
83
|
+
"strsim",
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
[[package]]
|
|
87
|
+
name = "clap_derive"
|
|
88
|
+
version = "4.6.7"
|
|
89
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
90
|
+
checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"
|
|
91
|
+
dependencies = [
|
|
92
|
+
"heck",
|
|
93
|
+
"proc-macro2",
|
|
94
|
+
"quote",
|
|
95
|
+
"syn 3.0.9",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
[[package]]
|
|
99
|
+
name = "clap_lex"
|
|
100
|
+
version = "1.1.1"
|
|
101
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
102
|
+
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
|
103
|
+
|
|
104
|
+
[[package]]
|
|
105
|
+
name = "colorchoice"
|
|
106
|
+
version = "1.0.5"
|
|
107
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
108
|
+
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
|
109
|
+
|
|
110
|
+
[[package]]
|
|
111
|
+
name = "heck"
|
|
112
|
+
version = "0.5.0"
|
|
113
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
114
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
115
|
+
|
|
116
|
+
[[package]]
|
|
117
|
+
name = "is_terminal_polyfill"
|
|
118
|
+
version = "1.70.2"
|
|
119
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
120
|
+
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
|
121
|
+
|
|
122
|
+
[[package]]
|
|
123
|
+
name = "memchr"
|
|
124
|
+
version = "2.8.0"
|
|
125
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
126
|
+
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
|
127
|
+
|
|
128
|
+
[[package]]
|
|
129
|
+
name = "once_cell_polyfill"
|
|
130
|
+
version = "1.70.2"
|
|
131
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
132
|
+
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
|
133
|
+
|
|
134
|
+
[[package]]
|
|
135
|
+
name = "pipecat-sentencex-wasm"
|
|
136
|
+
version = "0.0.0"
|
|
137
|
+
dependencies = [
|
|
138
|
+
"sentencex",
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
[[package]]
|
|
142
|
+
name = "proc-macro2"
|
|
143
|
+
version = "1.0.107"
|
|
144
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
145
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
146
|
+
dependencies = [
|
|
147
|
+
"unicode-ident",
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
[[package]]
|
|
151
|
+
name = "quote"
|
|
152
|
+
version = "1.0.47"
|
|
153
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
154
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
155
|
+
dependencies = [
|
|
156
|
+
"proc-macro2",
|
|
157
|
+
]
|
|
158
|
+
|
|
159
|
+
[[package]]
|
|
160
|
+
name = "regex"
|
|
161
|
+
version = "1.12.3"
|
|
162
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
163
|
+
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
|
164
|
+
dependencies = [
|
|
165
|
+
"aho-corasick",
|
|
166
|
+
"memchr",
|
|
167
|
+
"regex-automata",
|
|
168
|
+
"regex-syntax",
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
[[package]]
|
|
172
|
+
name = "regex-automata"
|
|
173
|
+
version = "0.4.14"
|
|
174
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
175
|
+
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
|
176
|
+
dependencies = [
|
|
177
|
+
"aho-corasick",
|
|
178
|
+
"memchr",
|
|
179
|
+
"regex-syntax",
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
[[package]]
|
|
183
|
+
name = "regex-syntax"
|
|
184
|
+
version = "0.8.10"
|
|
185
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
186
|
+
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
|
187
|
+
|
|
188
|
+
[[package]]
|
|
189
|
+
name = "rustc-hash"
|
|
190
|
+
version = "2.1.2"
|
|
191
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
192
|
+
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
|
|
193
|
+
|
|
194
|
+
[[package]]
|
|
195
|
+
name = "sentencex"
|
|
196
|
+
version = "0.1.32"
|
|
197
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
198
|
+
checksum = "d8bd5668ebbbbf863e27366ad97579fa59a4bb2d292fd60ad5b817c514e23716"
|
|
199
|
+
dependencies = [
|
|
200
|
+
"clap",
|
|
201
|
+
"memchr",
|
|
202
|
+
"regex",
|
|
203
|
+
"rustc-hash",
|
|
204
|
+
"serde",
|
|
205
|
+
"strum",
|
|
206
|
+
]
|
|
207
|
+
|
|
208
|
+
[[package]]
|
|
209
|
+
name = "serde"
|
|
210
|
+
version = "1.0.229"
|
|
211
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
212
|
+
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
|
213
|
+
dependencies = [
|
|
214
|
+
"serde_core",
|
|
215
|
+
"serde_derive",
|
|
216
|
+
]
|
|
217
|
+
|
|
218
|
+
[[package]]
|
|
219
|
+
name = "serde_core"
|
|
220
|
+
version = "1.0.229"
|
|
221
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
222
|
+
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
|
223
|
+
dependencies = [
|
|
224
|
+
"serde_derive",
|
|
225
|
+
]
|
|
226
|
+
|
|
227
|
+
[[package]]
|
|
228
|
+
name = "serde_derive"
|
|
229
|
+
version = "1.0.229"
|
|
230
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
231
|
+
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
232
|
+
dependencies = [
|
|
233
|
+
"proc-macro2",
|
|
234
|
+
"quote",
|
|
235
|
+
"syn 3.0.9",
|
|
236
|
+
]
|
|
237
|
+
|
|
238
|
+
[[package]]
|
|
239
|
+
name = "strsim"
|
|
240
|
+
version = "0.11.1"
|
|
241
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
242
|
+
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|
243
|
+
|
|
244
|
+
[[package]]
|
|
245
|
+
name = "strum"
|
|
246
|
+
version = "0.28.0"
|
|
247
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
248
|
+
checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
|
|
249
|
+
dependencies = [
|
|
250
|
+
"strum_macros",
|
|
251
|
+
]
|
|
252
|
+
|
|
253
|
+
[[package]]
|
|
254
|
+
name = "strum_macros"
|
|
255
|
+
version = "0.28.0"
|
|
256
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
257
|
+
checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
|
|
258
|
+
dependencies = [
|
|
259
|
+
"heck",
|
|
260
|
+
"proc-macro2",
|
|
261
|
+
"quote",
|
|
262
|
+
"syn 2.0.119",
|
|
263
|
+
]
|
|
264
|
+
|
|
265
|
+
[[package]]
|
|
266
|
+
name = "syn"
|
|
267
|
+
version = "2.0.119"
|
|
268
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
269
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
270
|
+
dependencies = [
|
|
271
|
+
"proc-macro2",
|
|
272
|
+
"quote",
|
|
273
|
+
"unicode-ident",
|
|
274
|
+
]
|
|
275
|
+
|
|
276
|
+
[[package]]
|
|
277
|
+
name = "syn"
|
|
278
|
+
version = "3.0.9"
|
|
279
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
280
|
+
checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b"
|
|
281
|
+
dependencies = [
|
|
282
|
+
"proc-macro2",
|
|
283
|
+
"quote",
|
|
284
|
+
"unicode-ident",
|
|
285
|
+
]
|
|
286
|
+
|
|
287
|
+
[[package]]
|
|
288
|
+
name = "unicode-ident"
|
|
289
|
+
version = "1.0.27"
|
|
290
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
291
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
|
292
|
+
|
|
293
|
+
[[package]]
|
|
294
|
+
name = "utf8parse"
|
|
295
|
+
version = "0.2.2"
|
|
296
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
297
|
+
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
|
298
|
+
|
|
299
|
+
[[package]]
|
|
300
|
+
name = "windows-link"
|
|
301
|
+
version = "0.2.1"
|
|
302
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
303
|
+
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
|
304
|
+
|
|
305
|
+
[[package]]
|
|
306
|
+
name = "windows-sys"
|
|
307
|
+
version = "0.61.2"
|
|
308
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
309
|
+
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
|
310
|
+
dependencies = [
|
|
311
|
+
"windows-link",
|
|
312
|
+
]
|
package/rust/Cargo.toml
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Copyright (c) 2026, pipedog contributors
|
|
2
|
+
# SPDX-License-Identifier: BSD-2-Clause
|
|
3
|
+
|
|
4
|
+
[package]
|
|
5
|
+
name = "pipecat-sentencex-wasm"
|
|
6
|
+
version = "0.0.0"
|
|
7
|
+
edition = "2021"
|
|
8
|
+
license = "BSD-2-Clause"
|
|
9
|
+
publish = false
|
|
10
|
+
description = "Wikimedia sentencex sentence segmentation behind a tiny C ABI, built for wasm32-unknown-unknown."
|
|
11
|
+
|
|
12
|
+
[lib]
|
|
13
|
+
crate-type = ["cdylib"]
|
|
14
|
+
path = "src/lib.rs"
|
|
15
|
+
|
|
16
|
+
[dependencies]
|
|
17
|
+
# MIT (Wikimedia). crates.io 0.1.32 is the same release (git tag v1.0.32, commit 268ff94)
|
|
18
|
+
# as the Python/npm sentencex==1.0.32 that upstream Pipecat pins.
|
|
19
|
+
sentencex = "=0.1.32"
|
|
20
|
+
|
|
21
|
+
[profile.release]
|
|
22
|
+
opt-level = "z"
|
|
23
|
+
lto = true
|
|
24
|
+
codegen-units = 1
|
|
25
|
+
panic = "abort"
|
|
26
|
+
strip = true
|
|
27
|
+
debug = false
|
package/rust/src/lib.rs
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
// Copyright (c) 2026, pipedog contributors
|
|
2
|
+
// SPDX-License-Identifier: BSD-2-Clause
|
|
3
|
+
//
|
|
4
|
+
// Wikimedia's sentencex (MIT) behind a tiny C ABI, for wasm32-unknown-unknown.
|
|
5
|
+
//
|
|
6
|
+
// No wasm-bindgen: the JS side talks to these `extern "C"` functions through
|
|
7
|
+
// `WebAssembly.Instance.exports` and reads/writes linear memory directly.
|
|
8
|
+
//
|
|
9
|
+
// ptr = sx_alloc(n) // JS writes UTF-8 language + text here
|
|
10
|
+
// n = sx_segment(lang, lang_len, text, text_len)
|
|
11
|
+
// n = sx_boundaries(lang, lang_len, text, text_len)
|
|
12
|
+
// out = sx_out() // u32 records, see below
|
|
13
|
+
// sx_free(ptr, n)
|
|
14
|
+
//
|
|
15
|
+
// Offsets are returned as UTF-16 code-unit indices into the text, so JS can
|
|
16
|
+
// `slice()` its original string without decoding anything. TextEncoder maps a
|
|
17
|
+
// lone surrogate to U+FFFD (one UTF-16 unit, three UTF-8 bytes), so the UTF-16
|
|
18
|
+
// positions still line up with the JS string in that case.
|
|
19
|
+
//
|
|
20
|
+
// Records (all u32):
|
|
21
|
+
// segment: [start16, end16] per sentence
|
|
22
|
+
// boundaries: [start_index16, end_index16, text_start16, text_end16,
|
|
23
|
+
// symbol_ptr, symbol_len (u32::MAX = none), is_paragraph_break]
|
|
24
|
+
// start_index/end_index are sentencex's char (Unicode scalar) indices
|
|
25
|
+
// converted to UTF-16; text_start/text_end locate the boundary's `text`.
|
|
26
|
+
//
|
|
27
|
+
// A negative return value means an error (bad UTF-8, or a slice that is not
|
|
28
|
+
// part of the input text, which sentencex never produces).
|
|
29
|
+
|
|
30
|
+
use std::alloc::{alloc, dealloc, Layout};
|
|
31
|
+
|
|
32
|
+
const BOUNDARY_FIELDS: usize = 7;
|
|
33
|
+
const NONE: u32 = u32::MAX;
|
|
34
|
+
|
|
35
|
+
static mut OUT: Vec<u32> = Vec::new();
|
|
36
|
+
|
|
37
|
+
#[allow(static_mut_refs)]
|
|
38
|
+
fn out() -> &'static mut Vec<u32> {
|
|
39
|
+
// Single-threaded wasm: no concurrent access is possible.
|
|
40
|
+
unsafe { &mut OUT }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
#[no_mangle]
|
|
44
|
+
pub extern "C" fn sx_alloc(len: usize) -> *mut u8 {
|
|
45
|
+
if len == 0 {
|
|
46
|
+
return core::ptr::null_mut::<u8>().wrapping_add(1);
|
|
47
|
+
}
|
|
48
|
+
unsafe { alloc(Layout::from_size_align_unchecked(len, 1)) }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
#[no_mangle]
|
|
52
|
+
pub extern "C" fn sx_free(ptr: *mut u8, len: usize) {
|
|
53
|
+
if len != 0 && !ptr.is_null() {
|
|
54
|
+
unsafe { dealloc(ptr, Layout::from_size_align_unchecked(len, 1)) }
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/// Pointer to the records written by the last `sx_segment` / `sx_boundaries`.
|
|
59
|
+
#[no_mangle]
|
|
60
|
+
pub extern "C" fn sx_out() -> *const u32 {
|
|
61
|
+
out().as_ptr()
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/// Maps byte offsets or char indices of `text` to UTF-16 offsets. Queries are
|
|
65
|
+
/// usually increasing, so it walks forward from the last position and only
|
|
66
|
+
/// restarts from 0 when asked about an earlier one.
|
|
67
|
+
struct Utf16Cursor<'a> {
|
|
68
|
+
text: &'a str,
|
|
69
|
+
byte: usize,
|
|
70
|
+
chars: usize,
|
|
71
|
+
units: usize,
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
impl<'a> Utf16Cursor<'a> {
|
|
75
|
+
fn new(text: &'a str) -> Self {
|
|
76
|
+
Self { text, byte: 0, chars: 0, units: 0 }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
fn reset_if_past(&mut self, past: bool) {
|
|
80
|
+
if past {
|
|
81
|
+
self.byte = 0;
|
|
82
|
+
self.chars = 0;
|
|
83
|
+
self.units = 0;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
fn step(&mut self) -> bool {
|
|
88
|
+
match self.text[self.byte..].chars().next() {
|
|
89
|
+
Some(c) => {
|
|
90
|
+
self.byte += c.len_utf8();
|
|
91
|
+
self.chars += 1;
|
|
92
|
+
self.units += c.len_utf16();
|
|
93
|
+
true
|
|
94
|
+
}
|
|
95
|
+
None => false,
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
fn from_byte(&mut self, byte: usize) -> u32 {
|
|
100
|
+
self.reset_if_past(byte < self.byte);
|
|
101
|
+
while self.byte < byte && self.step() {}
|
|
102
|
+
self.units as u32
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
fn from_char(&mut self, chars: usize) -> u32 {
|
|
106
|
+
self.reset_if_past(chars < self.chars);
|
|
107
|
+
while self.chars < chars && self.step() {}
|
|
108
|
+
// Past the end (never expected): extend linearly so the result is still defined.
|
|
109
|
+
(self.units + (chars - self.chars)) as u32
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
unsafe fn input<'a>(ptr: *const u8, len: usize) -> Option<&'a str> {
|
|
114
|
+
if len == 0 {
|
|
115
|
+
return Some("");
|
|
116
|
+
}
|
|
117
|
+
core::str::from_utf8(core::slice::from_raw_parts(ptr, len)).ok()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/// Byte range of `part` inside `text`, if `part` is a subslice of it.
|
|
121
|
+
fn range_in(text: &str, part: &str) -> Option<(usize, usize)> {
|
|
122
|
+
let base = text.as_ptr() as usize;
|
|
123
|
+
let start = (part.as_ptr() as usize).checked_sub(base)?;
|
|
124
|
+
let end = start + part.len();
|
|
125
|
+
if end > text.len() {
|
|
126
|
+
return None;
|
|
127
|
+
}
|
|
128
|
+
Some((start, end))
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/// Segment `text`; returns the sentence count (records in `sx_out()`), or -1.
|
|
132
|
+
#[no_mangle]
|
|
133
|
+
pub unsafe extern "C" fn sx_segment(
|
|
134
|
+
lang: *const u8,
|
|
135
|
+
lang_len: usize,
|
|
136
|
+
text: *const u8,
|
|
137
|
+
text_len: usize,
|
|
138
|
+
) -> i32 {
|
|
139
|
+
let (Some(lang), Some(text)) = (input(lang, lang_len), input(text, text_len)) else {
|
|
140
|
+
return -1;
|
|
141
|
+
};
|
|
142
|
+
let sentences = sentencex::segment(lang, text);
|
|
143
|
+
let buf = out();
|
|
144
|
+
buf.clear();
|
|
145
|
+
buf.reserve(sentences.len() * 2);
|
|
146
|
+
let mut cur = Utf16Cursor::new(text);
|
|
147
|
+
for s in &sentences {
|
|
148
|
+
let Some((start, end)) = range_in(text, s) else {
|
|
149
|
+
return -2;
|
|
150
|
+
};
|
|
151
|
+
buf.push(cur.from_byte(start));
|
|
152
|
+
buf.push(cur.from_byte(end));
|
|
153
|
+
}
|
|
154
|
+
sentences.len() as i32
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/// Sentence boundaries of `text`; returns the boundary count, or -1.
|
|
158
|
+
#[no_mangle]
|
|
159
|
+
pub unsafe extern "C" fn sx_boundaries(
|
|
160
|
+
lang: *const u8,
|
|
161
|
+
lang_len: usize,
|
|
162
|
+
text: *const u8,
|
|
163
|
+
text_len: usize,
|
|
164
|
+
) -> i32 {
|
|
165
|
+
let (Some(lang), Some(text)) = (input(lang, lang_len), input(text, text_len)) else {
|
|
166
|
+
return -1;
|
|
167
|
+
};
|
|
168
|
+
let boundaries = sentencex::get_sentence_boundaries(lang, text);
|
|
169
|
+
let buf = out();
|
|
170
|
+
buf.clear();
|
|
171
|
+
buf.reserve(boundaries.len() * BOUNDARY_FIELDS);
|
|
172
|
+
let mut by_char = Utf16Cursor::new(text);
|
|
173
|
+
let mut by_byte = Utf16Cursor::new(text);
|
|
174
|
+
for b in &boundaries {
|
|
175
|
+
let Some((start, end)) = range_in(text, b.text) else {
|
|
176
|
+
return -2;
|
|
177
|
+
};
|
|
178
|
+
buf.push(by_char.from_char(b.start_index));
|
|
179
|
+
buf.push(by_char.from_char(b.end_index));
|
|
180
|
+
buf.push(by_byte.from_byte(start));
|
|
181
|
+
buf.push(by_byte.from_byte(end));
|
|
182
|
+
match b.boundary_symbol {
|
|
183
|
+
Some(sym) => {
|
|
184
|
+
buf.push(sym.as_ptr() as u32);
|
|
185
|
+
buf.push(sym.len() as u32);
|
|
186
|
+
}
|
|
187
|
+
None => {
|
|
188
|
+
buf.push(0);
|
|
189
|
+
buf.push(NONE);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
buf.push(b.is_paragraph_break as u32);
|
|
193
|
+
}
|
|
194
|
+
boundaries.len() as i32
|
|
195
|
+
}
|
|
Binary file
|