echogarden 2.0.12 → 2.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/data-structures/DynamicTypedArray.d.ts +2 -1
- package/dist/data-structures/DynamicTypedArray.js +14 -9
- package/dist/data-structures/DynamicTypedArray.js.map +1 -1
- package/dist/utilities/WasmMemoryManager.d.ts +1 -1
- package/docs/CLI.md +3 -3
- package/docs/Tasklist.md +12 -17
- package/package.json +9 -9
- package/src/data-structures/DynamicTypedArray.ts +16 -10
- package/dist/encodings/LPVarInt.d.ts +0 -11
- package/dist/encodings/LPVarInt.js +0 -187
- package/dist/encodings/LPVarInt.js.map +0 -1
- package/src/encodings/LPVarInt.ts +0 -292
package/README.md
CHANGED
|
@@ -3,15 +3,16 @@
|
|
|
3
3
|
Echogarden is an easy-to-use speech toolset that includes a variety of speech processing tools.
|
|
4
4
|
|
|
5
5
|
* Easy to install, run, and update
|
|
6
|
-
* Runs on Windows (x64, ARM64), macOS (x64, ARM64) and Linux (x64, ARM64)
|
|
7
6
|
* Written in TypeScript, for the Node.js runtime
|
|
7
|
+
* Can be used either as a command-line utility, or imported as an `npm` package
|
|
8
|
+
* Runs on Windows (x64, ARM64), macOS (x64, ARM64) and Linux (x64, ARM64)
|
|
8
9
|
* Doesn't require Python, Docker, or other system-level dependencies
|
|
9
|
-
* Doesn't rely on essential platform-specific binaries. Engines are either ported via WebAssembly, imported using the [ONNX runtime](https://onnxruntime.ai/), or written in pure
|
|
10
|
+
* Doesn't rely on essential platform-specific binaries. Engines are either ported via WebAssembly, imported using the [ONNX runtime](https://onnxruntime.ai/), or written in pure TypeScript
|
|
10
11
|
|
|
11
12
|
## Features
|
|
12
13
|
|
|
13
14
|
* **Text-to-speech** using the [VITS](https://github.com/jaywalnut310/vits) neural architecture, and [15 other offline and online engines](docs/Engines.md), including cloud services by [Google](https://cloud.google.com/text-to-speech), [Microsoft](https://azure.microsoft.com/en-us/products/ai-services/text-to-speech/), [Amazon](https://aws.amazon.com/polly/), [OpenAI](https://platform.openai.com/) and [Elevenlabs](https://elevenlabs.io/)
|
|
14
|
-
* **Speech-to-text** using a
|
|
15
|
+
* **Speech-to-text** using a custom TypeScript/ONNX port of the [OpenAI Whisper](https://openai.com/research/whisper) speech recognition architecture, [whisper.cpp](https://github.com/ggerganov/whisper.cpp), and [several other engines](docs/Engines.md), including cloud services by [Google](https://cloud.google.com/speech-to-text), [Microsoft](https://azure.microsoft.com/en-us/products/ai-services/speech-to-text/), [Amazon](https://aws.amazon.com/transcribe/) and [OpenAI](https://platform.openai.com/)
|
|
15
16
|
* **Speech-to-transcript alignment** using several variants of [dynamic time warping](https://en.wikipedia.org/wiki/Dynamic_time_warping) (DTW, DTW-RA), including support for multi-pass (hierarchical) processing, or via guided decoding using Whisper recognition models. Supports 100+ languages
|
|
16
17
|
* **Speech-to-text translation**, translates speech in any of the [98 languages](https://platform.openai.com/docs/guides/speech-to-text/supported-languages) supported by Whisper, to English, with near word-level timing for the translated transcript
|
|
17
18
|
* **Speech-to-translated-transcript alignment** synchronizes spoken audio in one language, to a provided English-translated transcript, using the Whisper engine
|
|
@@ -48,17 +49,18 @@ A small sample of command lines:
|
|
|
48
49
|
echogarden speak "Hello World!"
|
|
49
50
|
echogarden speak-file story.txt
|
|
50
51
|
echogarden transcribe speech.mp3
|
|
52
|
+
echogarden translate-speech speech.webm subtitles.srt
|
|
51
53
|
echogarden align speech.opus transcript.txt
|
|
52
54
|
echogarden isolate speech.wav
|
|
53
55
|
```
|
|
54
56
|
|
|
55
|
-
See the [
|
|
57
|
+
See the [command-line interface guide](docs/CLI.md) for more details on the operations supported, and the [configuration options reference](docs/Options.md) for a comprehensive list of all options supported.
|
|
56
58
|
|
|
57
|
-
**Note**: on `v2.0.0`, a [newly developed audio playback library](https://github.com/echogarden-project/audio-io) was integrated
|
|
59
|
+
**Note**: on `v2.0.0`, a [newly developed audio playback library](https://github.com/echogarden-project/audio-io) was integrated into the command line interface. If you're having trouble hearing sound, or the sound is distorted, please [report this as an issue](https://github.com/echogarden-project/audio-io/issues). You can also switch back to the older [SoX](https://sourceforge.net/projects/sox/)-based player by adding `--player=sox` to the command-line. On macOS, you'll need to ensure SoX is available on the system path by installing it with a system package manager like [Homebrew](https://brew.sh/) (`brew install sox`).
|
|
58
60
|
|
|
59
61
|
## Using the Node.js API
|
|
60
62
|
|
|
61
|
-
If you are a developer, you can also [import the package as a
|
|
63
|
+
If you are a developer, you can also [directly import the package as a dependency](docs/API.md). The API operations and options closely mirror the CLI.
|
|
62
64
|
|
|
63
65
|
## Documentation
|
|
64
66
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TypedArray, TypedArrayConstructor } from
|
|
1
|
+
import { TypedArray, TypedArrayConstructor } from '../typings/TypedArray.js';
|
|
2
2
|
export declare class DynamicTypedArray<T extends TypedArray> {
|
|
3
3
|
private TypedArrayConstructor;
|
|
4
4
|
elements: TypedArray;
|
|
@@ -8,6 +8,7 @@ export declare class DynamicTypedArray<T extends TypedArray> {
|
|
|
8
8
|
addMany(...newElements: number[]): void;
|
|
9
9
|
addArray(newElements: ArrayLike<number>): void;
|
|
10
10
|
ensureCapacity(requiredCapacity: number): void;
|
|
11
|
+
get capacity(): number;
|
|
11
12
|
toTypedArray(): T;
|
|
12
13
|
clear(): void;
|
|
13
14
|
}
|
|
@@ -7,30 +7,35 @@ export class DynamicTypedArray {
|
|
|
7
7
|
this.elements = new TypedArrayConstructor(initialCapacity);
|
|
8
8
|
}
|
|
9
9
|
add(newElement) {
|
|
10
|
-
const
|
|
11
|
-
if (
|
|
12
|
-
this.ensureCapacity(
|
|
10
|
+
const newLength = this.length + 1;
|
|
11
|
+
if (newLength > this.capacity) {
|
|
12
|
+
this.ensureCapacity(newLength);
|
|
13
13
|
}
|
|
14
|
-
this.elements[length] = newElement;
|
|
15
|
-
this.length
|
|
14
|
+
this.elements[this.length] = newElement;
|
|
15
|
+
this.length = newLength;
|
|
16
16
|
}
|
|
17
17
|
addMany(...newElements) {
|
|
18
18
|
this.addArray(newElements);
|
|
19
19
|
}
|
|
20
20
|
addArray(newElements) {
|
|
21
|
-
const
|
|
22
|
-
|
|
21
|
+
const newLength = this.length + newElements.length;
|
|
22
|
+
if (newLength > this.capacity) {
|
|
23
|
+
this.ensureCapacity(newLength);
|
|
24
|
+
}
|
|
23
25
|
this.elements.set(newElements, this.length);
|
|
24
|
-
this.length
|
|
26
|
+
this.length = newLength;
|
|
25
27
|
}
|
|
26
28
|
ensureCapacity(requiredCapacity) {
|
|
27
|
-
if (requiredCapacity > this.
|
|
29
|
+
if (requiredCapacity > this.capacity) {
|
|
28
30
|
const newCapacity = requiredCapacity * 2;
|
|
29
31
|
const newElements = new this.TypedArrayConstructor(newCapacity);
|
|
30
32
|
newElements.set(this.toTypedArray());
|
|
31
33
|
this.elements = newElements;
|
|
32
34
|
}
|
|
33
35
|
}
|
|
36
|
+
get capacity() {
|
|
37
|
+
return this.elements.length;
|
|
38
|
+
}
|
|
34
39
|
toTypedArray() {
|
|
35
40
|
return this.elements.subarray(0, this.length);
|
|
36
41
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DynamicTypedArray.js","sourceRoot":"","sources":["../../src/data-structures/DynamicTypedArray.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,iBAAiB;IAIT;IAHpB,QAAQ,CAAY;IACpB,MAAM,GAAG,CAAC,CAAA;IAEV,YAAoB,qBAA+C,EAAE,eAAe,GAAG,CAAC;QAApE,0BAAqB,GAArB,qBAAqB,CAA0B;QAClE,IAAI,CAAC,QAAQ,GAAG,IAAI,qBAAqB,CAAC,eAAe,CAAC,CAAA;IAC3D,CAAC;IAED,GAAG,CAAC,UAAkB;QACrB,MAAM,
|
|
1
|
+
{"version":3,"file":"DynamicTypedArray.js","sourceRoot":"","sources":["../../src/data-structures/DynamicTypedArray.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,iBAAiB;IAIT;IAHpB,QAAQ,CAAY;IACpB,MAAM,GAAG,CAAC,CAAA;IAEV,YAAoB,qBAA+C,EAAE,eAAe,GAAG,CAAC;QAApE,0BAAqB,GAArB,qBAAqB,CAA0B;QAClE,IAAI,CAAC,QAAQ,GAAG,IAAI,qBAAqB,CAAC,eAAe,CAAC,CAAA;IAC3D,CAAC;IAED,GAAG,CAAC,UAAkB;QACrB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAEjC,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;QAC/B,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,UAAU,CAAA;QACvC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,WAAqB;QAC/B,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;IAC3B,CAAC;IAED,QAAQ,CAAC,WAA8B;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAA;QAElD,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAA;QAC/B,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAC3C,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;IACxB,CAAC;IAED,cAAc,CAAC,gBAAwB;QACtC,IAAI,gBAAgB,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,WAAW,GAAG,gBAAgB,GAAG,CAAC,CAAA;YAExC,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAA;YAC/D,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAA;YAEpC,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,IAAI,QAAQ;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAA;IAC5B,CAAC;IAED,YAAY;QACX,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAM,CAAA;IACnD,CAAC;IAED,KAAK;QACJ,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAChB,CAAC;CACD;AAED,MAAM,UAAU,uBAAuB,CAAC,eAAwB;IAC/D,OAAO,IAAI,iBAAiB,CAAa,UAAU,EAAE,eAAe,CAAC,CAAA;AACtE,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,eAAwB;IAChE,OAAO,IAAI,iBAAiB,CAAc,WAAW,EAAE,eAAe,CAAC,CAAA;AACxE,CAAC"}
|
|
@@ -109,7 +109,7 @@ declare abstract class TypedArrayRef<T extends TypedArray> {
|
|
|
109
109
|
constructor(ptr: number, length: number, manager: WasmMemoryManager);
|
|
110
110
|
get view(): T;
|
|
111
111
|
protected abstract getView(): T;
|
|
112
|
-
slice(start?: number, end?: number):
|
|
112
|
+
slice(start?: number, end?: number): Int8Array<ArrayBuffer> | Uint8Array<ArrayBuffer> | Uint8ClampedArray<ArrayBuffer> | Int16Array<ArrayBuffer> | Uint16Array<ArrayBuffer> | Int32Array<ArrayBuffer> | Uint32Array<ArrayBuffer> | Float32Array<ArrayBuffer> | Float64Array<ArrayBuffer>;
|
|
113
113
|
get address(): number;
|
|
114
114
|
clear(): this;
|
|
115
115
|
free(): void;
|
package/docs/CLI.md
CHANGED
|
@@ -362,17 +362,17 @@ Shows a list of available engines for a given command:
|
|
|
362
362
|
echogarden list-engines speak
|
|
363
363
|
```
|
|
364
364
|
|
|
365
|
-
### `list-
|
|
365
|
+
### `list-voices`
|
|
366
366
|
|
|
367
367
|
Shows a list of available TTS voices for a given engine:
|
|
368
368
|
|
|
369
369
|
```bash
|
|
370
|
-
echogarden list-
|
|
370
|
+
echogarden list-voices google-cloud
|
|
371
371
|
```
|
|
372
372
|
|
|
373
373
|
Saves the voice list in a JSON file:
|
|
374
374
|
```bash
|
|
375
|
-
echogarden list-
|
|
375
|
+
echogarden list-voices google-cloud google-cloud-voices.json
|
|
376
376
|
```
|
|
377
377
|
|
|
378
378
|
## Internal package management
|
package/docs/Tasklist.md
CHANGED
|
@@ -2,11 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
## Bugs
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
### eSpeak
|
|
5
|
+
### eSpeak-ng
|
|
7
6
|
|
|
8
7
|
* IPA -> Kirshenbaum translation is still not completely similar to what is output by eSpeak. Also, in rare situations, it outputs characters that are not accepted by eSpeak and eSpeak errors. Investigate when that happens and how to improve on this
|
|
9
8
|
|
|
9
|
+
### Subtitles
|
|
10
|
+
* Phrase splitting may split on number separators like the `,` in `100,000`. The new segmentation library would resolve that
|
|
11
|
+
|
|
10
12
|
### Browser extension
|
|
11
13
|
* Investigate why WebSpeech events sometimes completely stop working in the middle of an utterance for no apparent reason. Sometimes this is permanent, until the extension is restarted. Is this a browser issue?
|
|
12
14
|
* If a request is made and the server takes too much time to respond, the service worker may sleep and the request never canceled
|
|
@@ -16,28 +18,24 @@
|
|
|
16
18
|
|
|
17
19
|
### External bugs
|
|
18
20
|
|
|
19
|
-
* `espeak-ng`: 'Oh dear!”' is read as "oh dear exclamation mark", because of the special quote character following the exclamation mark
|
|
20
21
|
* `espeak-ng`: [Marker right after sentence end is not reported as an event](https://github.com/espeak-ng/espeak-ng/issues/920)
|
|
21
|
-
* `espeak-ng`: On Japanese text, it says "Chinese
|
|
22
|
+
* `espeak-ng`: On Japanese text, it says "Chinese letter" or "Japanese letter" for characters it doesn't support
|
|
22
23
|
* `espeak-ng`: Broken markers on the Korean voice
|
|
23
24
|
* `wtf_wikipedia` Sometimes fails on `getResult.js` without throwing a humanly readable error
|
|
24
25
|
* `wtf_wikipedia` Sometimes captures markup like `.svg` etc.
|
|
25
26
|
* `msspeech`: Initialization fails on Chinese and Japanese voices (but not Korean)
|
|
26
27
|
* `compromise`: Slow initialization time. Currently, it takes more than a second
|
|
27
|
-
* Chromium doesn't fire timer events when cursor is positioned over scrollbar or body margins
|
|
28
|
-
* `whisper.cpp`: Timestamps aren't very accurate when `enableDTW` is set. There's a constant lag
|
|
29
|
-
* Node.js WASI for `flite` on Node `v21.7.2` and `v20.12.1` is intermittently crashing the process when `run` is called
|
|
28
|
+
* Browser extension: Chromium doesn't fire timer events when cursor is positioned over scrollbar or body margins
|
|
30
29
|
|
|
31
30
|
## Features and enhancements
|
|
32
31
|
|
|
33
32
|
### CLI
|
|
34
33
|
* Show names of files written to disk. This is useful for cases where a file is auto-renamed to prevent overwriting existing data
|
|
35
34
|
* Restrict input media file extensions to ensure that invalid files are not passed to FFmpeg
|
|
36
|
-
* Consider what to do with non-supported templates like `[hello]`
|
|
37
35
|
* Show a message when a new version is available
|
|
38
36
|
* Figure out which terminal outputs should go to stdout, or if that's a good idea at all
|
|
39
37
|
* Print available synthesis voices when no voice matches (or suggest near matches)
|
|
40
|
-
* `transcribe
|
|
38
|
+
* `transcribe`: accept `http://` and `https://` URLs and pull the remote media file
|
|
41
39
|
* Make `enum` options case-insensitive if possible
|
|
42
40
|
* More fine-grained intermediate progress report for operations
|
|
43
41
|
* Suggest possible correction on the error of not using `=`, e.g. `speed 0.9` instead of `speed=0.9`
|
|
@@ -93,13 +91,11 @@
|
|
|
93
91
|
### Text language detection
|
|
94
92
|
* Deploy and add the new n-gram based text language detection model
|
|
95
93
|
|
|
96
|
-
### Segmentation
|
|
97
|
-
* See if it's possible or useful to reliably use eSpeak as a segmentation engine.
|
|
98
|
-
|
|
99
94
|
### Subtitles
|
|
100
|
-
* Split long words if needed
|
|
101
|
-
*
|
|
95
|
+
* Split long words if needed
|
|
96
|
+
* Clauses shouldn't be split in the middle of numbers, like the `,` in `123,456`
|
|
102
97
|
* Decide how many punctuation characters to allow before breaking to a new line (currently it's infinite)
|
|
98
|
+
* If a subtitle is too short and at the end of the audio, try to extend it back if possible (for example, if the previous subtitle is already extended, take back from it)
|
|
103
99
|
* Add more clause separators, for even more special cases
|
|
104
100
|
* Add option to output usable word or phoneme-level caption files (investigate how it's done on YouTube auto-captions)
|
|
105
101
|
* Parse VTT's language
|
|
@@ -155,7 +151,6 @@
|
|
|
155
151
|
### Alignment
|
|
156
152
|
|
|
157
153
|
### Alignment / DTW
|
|
158
|
-
* Accept percentages like `20%` in the `windowDuration` option
|
|
159
154
|
* For the `granularity` option, add more granularities like `xxx-low` and `xxxx-low` (should the naming be changed? Maybe transition to a new naming scheme?)
|
|
160
155
|
* Add and test official support for more than 6 hours of audio
|
|
161
156
|
|
|
@@ -255,17 +250,16 @@
|
|
|
255
250
|
* [PlayHT](https://play.ht/) speech synthesis cloud service
|
|
256
251
|
* [Deepgram](https://deepgram.com/) cloud text-to-speech API
|
|
257
252
|
* [Assembly AI](https://www.assemblyai.com/) cloud speech recognition API
|
|
253
|
+
* [Picovoice Orca](https://github.com/Picovoice/orca) local text-to-speech
|
|
258
254
|
* Coqui STT server connection
|
|
259
255
|
* [MarbleNet VAD](https://github.com/NVIDIA/NeMo/blob/main/tutorials/asr/Online_Offline_Microphone_VAD_Demo.ipynb), included of the NVIDIA NeMo framework, can be exported to ONNX
|
|
260
256
|
* Silero text enhancement engine can be ported to ONNX
|
|
261
|
-
* See what can be done for supporting WinRT speech: in particular `windows.media.speechsynthesis` and `windows.media.speechrecognition` support, possibly using NodeRT or some other method
|
|
262
257
|
* Figure out how to support `julius` speech recognition via WASM
|
|
263
258
|
* Any way to support RHVoice?
|
|
264
259
|
|
|
265
260
|
## Maybe?
|
|
266
261
|
|
|
267
262
|
* Using a machine translation model to provide speech translation to languages other than English
|
|
268
|
-
* Is it possible to get sentence boundaries without punctuation using NLP techniques like part-of-speech tagging?
|
|
269
263
|
|
|
270
264
|
## May or may not be good ideas
|
|
271
265
|
|
|
@@ -274,6 +268,7 @@
|
|
|
274
268
|
|
|
275
269
|
## Other ideas
|
|
276
270
|
|
|
271
|
+
* Speaker diarization
|
|
277
272
|
* Support alignment of EPUB 3 eBooks with a corresponding audiobook
|
|
278
273
|
* Voice cloning
|
|
279
274
|
* Speech-to-speech voice conversion
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.13",
|
|
4
4
|
"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "GPL-3.0",
|
|
@@ -52,9 +52,9 @@
|
|
|
52
52
|
"echogarden": "./dist/cli/CLILauncher.js"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@aws-sdk/client-polly": "^3.
|
|
56
|
-
"@aws-sdk/client-transcribe-streaming": "^3.
|
|
57
|
-
"@echogarden/audio-io": "^0.
|
|
55
|
+
"@aws-sdk/client-polly": "^3.716.0",
|
|
56
|
+
"@aws-sdk/client-transcribe-streaming": "^3.716.0",
|
|
57
|
+
"@echogarden/audio-io": "^0.3.0",
|
|
58
58
|
"@echogarden/espeak-ng-emscripten": "^0.3.3",
|
|
59
59
|
"@echogarden/fasttext-wasm": "^0.1.0",
|
|
60
60
|
"@echogarden/flite-wasi": "^0.1.1",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"@echogarden/transformers-nodejs-lite": "^2.17.1-lite.3",
|
|
69
69
|
"@mozilla/readability": "^0.5.0",
|
|
70
70
|
"alawmulaw": "^6.0.0",
|
|
71
|
-
"chalk": "^5.
|
|
71
|
+
"chalk": "^5.4.0",
|
|
72
72
|
"cldr-segmentation": "^2.2.1",
|
|
73
73
|
"command-exists": "^1.2.9",
|
|
74
74
|
"compromise": "^14.14.3",
|
|
@@ -81,14 +81,14 @@
|
|
|
81
81
|
"jsdom": "^25.0.1",
|
|
82
82
|
"json5": "^2.2.3",
|
|
83
83
|
"kuromoji": "^0.1.2",
|
|
84
|
-
"microsoft-cognitiveservices-speech-sdk": "^1.
|
|
84
|
+
"microsoft-cognitiveservices-speech-sdk": "^1.42.0",
|
|
85
85
|
"msgpack-lite": "^0.1.26",
|
|
86
86
|
"onnxruntime-node": "^1.20.1",
|
|
87
|
-
"openai": "^4.
|
|
87
|
+
"openai": "^4.77.0",
|
|
88
88
|
"sam-js": "^0.3.1",
|
|
89
89
|
"strip-ansi": "^7.1.0",
|
|
90
90
|
"tar": "^7.4.3",
|
|
91
|
-
"tiktoken": "^1.0.
|
|
91
|
+
"tiktoken": "^1.0.18",
|
|
92
92
|
"tinyld": "^1.3.4",
|
|
93
93
|
"wasm-feature-detect": "^1.8.0",
|
|
94
94
|
"ws": "^8.18.0",
|
|
@@ -111,7 +111,7 @@
|
|
|
111
111
|
"@types/graceful-fs": "^4.1.9",
|
|
112
112
|
"@types/jsdom": "^21.1.7",
|
|
113
113
|
"@types/msgpack-lite": "^0.1.11",
|
|
114
|
-
"@types/node": "^22.10.
|
|
114
|
+
"@types/node": "^22.10.2",
|
|
115
115
|
"@types/tar": "^6.1.13",
|
|
116
116
|
"@types/ws": "^8.5.13",
|
|
117
117
|
"ts-json-schema-generator": "^2.3.0",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { TypedArray, TypedArrayConstructor } from
|
|
1
|
+
import { TypedArray, TypedArrayConstructor } from '../typings/TypedArray.js'
|
|
2
2
|
|
|
3
3
|
export class DynamicTypedArray<T extends TypedArray> {
|
|
4
4
|
elements: TypedArray
|
|
@@ -9,14 +9,14 @@ export class DynamicTypedArray<T extends TypedArray> {
|
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
add(newElement: number) {
|
|
12
|
-
const
|
|
12
|
+
const newLength = this.length + 1
|
|
13
13
|
|
|
14
|
-
if (
|
|
15
|
-
this.ensureCapacity(
|
|
14
|
+
if (newLength > this.capacity) {
|
|
15
|
+
this.ensureCapacity(newLength)
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
this.elements[length] = newElement
|
|
19
|
-
this.length
|
|
18
|
+
this.elements[this.length] = newElement
|
|
19
|
+
this.length = newLength
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
addMany(...newElements: number[]) {
|
|
@@ -24,16 +24,18 @@ export class DynamicTypedArray<T extends TypedArray> {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
addArray(newElements: ArrayLike<number>) {
|
|
27
|
-
const
|
|
27
|
+
const newLength = this.length + newElements.length
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
if (newLength > this.capacity) {
|
|
30
|
+
this.ensureCapacity(newLength)
|
|
31
|
+
}
|
|
30
32
|
|
|
31
33
|
this.elements.set(newElements, this.length)
|
|
32
|
-
this.length
|
|
34
|
+
this.length = newLength
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
ensureCapacity(requiredCapacity: number) {
|
|
36
|
-
if (requiredCapacity > this.
|
|
38
|
+
if (requiredCapacity > this.capacity) {
|
|
37
39
|
const newCapacity = requiredCapacity * 2
|
|
38
40
|
|
|
39
41
|
const newElements = new this.TypedArrayConstructor(newCapacity)
|
|
@@ -43,6 +45,10 @@ export class DynamicTypedArray<T extends TypedArray> {
|
|
|
43
45
|
}
|
|
44
46
|
}
|
|
45
47
|
|
|
48
|
+
get capacity() {
|
|
49
|
+
return this.elements.length
|
|
50
|
+
}
|
|
51
|
+
|
|
46
52
|
toTypedArray() {
|
|
47
53
|
return this.elements.subarray(0, this.length) as T
|
|
48
54
|
}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { DynamicUint8Array } from "../data-structures/DynamicTypedArray.js";
|
|
2
|
-
export declare function encodeUnsignedInt31(value: number, outEncodedData: DynamicUint8Array): void;
|
|
3
|
-
export declare function decodeUnsignedInt31(encodedData: ArrayLike<number>, readOffset: number): DecodedValueAndReadOffset;
|
|
4
|
-
export declare function encodeSignedInt32(value: number, outEncodedData: DynamicUint8Array): void;
|
|
5
|
-
export declare function decodeSignedInt32(encodedData: ArrayLike<number>, readOffset: number): DecodedValueAndReadOffset;
|
|
6
|
-
export interface DecodedValueAndReadOffset {
|
|
7
|
-
decodedValue: number;
|
|
8
|
-
readOffset: number;
|
|
9
|
-
}
|
|
10
|
-
export declare function testLPVarintSigned(): void;
|
|
11
|
-
export declare function testLPVarintUnsigned(): void;
|
|
@@ -1,187 +0,0 @@
|
|
|
1
|
-
import { createDynamicUint8Array } from "../data-structures/DynamicTypedArray.js";
|
|
2
|
-
import { logToStderr } from "../utilities/Utilities.js";
|
|
3
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
4
|
-
// Encode unsigned integer
|
|
5
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
6
|
-
export function encodeUnsignedInt31(value, outEncodedData) {
|
|
7
|
-
value = value >>> 0;
|
|
8
|
-
if (value < (2 ** 7)) {
|
|
9
|
-
outEncodedData.add(value);
|
|
10
|
-
}
|
|
11
|
-
else if (value < (2 ** 14)) {
|
|
12
|
-
outEncodedData.addMany((value & 0b00111111) | 0b10000000, value >>> 6);
|
|
13
|
-
}
|
|
14
|
-
else if (value < (2 ** 21)) {
|
|
15
|
-
outEncodedData.addMany((value & 0b00011111) | 0b11000000, value >>> 5, value >>> 13);
|
|
16
|
-
}
|
|
17
|
-
else if (value < (2 ** 28)) {
|
|
18
|
-
outEncodedData.addMany((value & 0b00001111) | 0b11100000, value >>> 4, value >>> 12, value >>> 20);
|
|
19
|
-
}
|
|
20
|
-
else {
|
|
21
|
-
outEncodedData.addMany(0b11110000, value, value >>> 8, value >>> 16, value >>> 24);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
25
|
-
// Decode unsigned integer
|
|
26
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
27
|
-
export function decodeUnsignedInt31(encodedData, readOffset) {
|
|
28
|
-
readOffset = readOffset | 0;
|
|
29
|
-
const byte0 = encodedData[readOffset++];
|
|
30
|
-
if ((byte0 & 0b10000000) === 0) {
|
|
31
|
-
// 1 byte
|
|
32
|
-
let decodedValue = byte0;
|
|
33
|
-
return { decodedValue, readOffset };
|
|
34
|
-
}
|
|
35
|
-
const byte1 = encodedData[readOffset++];
|
|
36
|
-
if ((byte0 & 0b01000000) === 0) {
|
|
37
|
-
// 2 bytes
|
|
38
|
-
let decodedValue = (byte0 & 0b00111111) |
|
|
39
|
-
(byte1 << 6);
|
|
40
|
-
return { decodedValue, readOffset };
|
|
41
|
-
}
|
|
42
|
-
const byte2 = encodedData[readOffset++];
|
|
43
|
-
if ((byte0 & 0b00100000) === 0) {
|
|
44
|
-
// 3 bytes
|
|
45
|
-
let decodedValue = (byte0 & 0b00011111) |
|
|
46
|
-
(byte1 << 5) |
|
|
47
|
-
(byte2 << 13);
|
|
48
|
-
return { decodedValue, readOffset };
|
|
49
|
-
}
|
|
50
|
-
const byte3 = encodedData[readOffset++];
|
|
51
|
-
if ((byte0 & 0b00010000) === 0) {
|
|
52
|
-
// 4 bytes
|
|
53
|
-
let decodedValue = (byte0 & 0b00001111) |
|
|
54
|
-
(byte1 << 4) |
|
|
55
|
-
(byte2 << 12) |
|
|
56
|
-
(byte3 << 20);
|
|
57
|
-
return { decodedValue, readOffset };
|
|
58
|
-
}
|
|
59
|
-
const byte4 = encodedData[readOffset++];
|
|
60
|
-
if (byte0 === 0b11110000) {
|
|
61
|
-
// 5 bytes
|
|
62
|
-
let decodedValue = (byte1) |
|
|
63
|
-
(byte2 << 8) |
|
|
64
|
-
(byte3 << 16) |
|
|
65
|
-
(byte4 << 24);
|
|
66
|
-
return { decodedValue, readOffset };
|
|
67
|
-
}
|
|
68
|
-
throw new Error(`Encountered a value that is encoded with more than 5 bytes (beyond Unsigned Int31 range).`);
|
|
69
|
-
}
|
|
70
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
71
|
-
// Encode signed integer
|
|
72
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
73
|
-
export function encodeSignedInt32(value, outEncodedData) {
|
|
74
|
-
const absValue = Math.abs(value | 0);
|
|
75
|
-
if (absValue < (2 ** 6)) {
|
|
76
|
-
outEncodedData.add(value & 0b01111111);
|
|
77
|
-
}
|
|
78
|
-
else if (absValue < (2 ** 13)) {
|
|
79
|
-
outEncodedData.addMany((value & 0b00111111) | 0b10000000, value >> 6);
|
|
80
|
-
}
|
|
81
|
-
else if (absValue < (2 ** 20)) {
|
|
82
|
-
outEncodedData.addMany((value & 0b00011111) | 0b11000000, value >> 5, value >> 13);
|
|
83
|
-
}
|
|
84
|
-
else if (absValue < (2 ** 27)) {
|
|
85
|
-
outEncodedData.addMany((value & 0b00001111) | 0b11100000, value >> 4, value >> 12, value >> 20);
|
|
86
|
-
}
|
|
87
|
-
else {
|
|
88
|
-
outEncodedData.addMany(0b11110000, value, value >> 8, value >> 16, value >> 24);
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
92
|
-
// Decode signed integer
|
|
93
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
94
|
-
export function decodeSignedInt32(encodedData, readOffset) {
|
|
95
|
-
const byte0 = encodedData[readOffset++];
|
|
96
|
-
if ((byte0 & 0b10000000) === 0) {
|
|
97
|
-
// 1 byte
|
|
98
|
-
let decodedValue = byte0;
|
|
99
|
-
if ((byte0 & 0b01000000) !== 0) {
|
|
100
|
-
decodedValue |= (-1 << 7);
|
|
101
|
-
}
|
|
102
|
-
return { decodedValue, readOffset };
|
|
103
|
-
}
|
|
104
|
-
const byte1 = encodedData[readOffset++];
|
|
105
|
-
if ((byte0 & 0b01000000) === 0) {
|
|
106
|
-
// 2 bytes
|
|
107
|
-
let decodedValue = (byte0 & 0b00111111) |
|
|
108
|
-
(byte1 << 6);
|
|
109
|
-
if ((byte1 & 0b10000000) !== 0) {
|
|
110
|
-
decodedValue |= (-1 << 14);
|
|
111
|
-
}
|
|
112
|
-
return { decodedValue, readOffset };
|
|
113
|
-
}
|
|
114
|
-
const byte2 = encodedData[readOffset++];
|
|
115
|
-
if ((byte0 & 0b00100000) === 0) {
|
|
116
|
-
// 3 bytes
|
|
117
|
-
let decodedValue = (byte0 & 0b00011111) |
|
|
118
|
-
(byte1 << 5) |
|
|
119
|
-
(byte2 << 13);
|
|
120
|
-
if ((byte2 & 0b10000000) !== 0) {
|
|
121
|
-
decodedValue |= (-1 << 21);
|
|
122
|
-
}
|
|
123
|
-
return { decodedValue, readOffset };
|
|
124
|
-
}
|
|
125
|
-
const byte3 = encodedData[readOffset++];
|
|
126
|
-
if ((byte0 & 0b00010000) === 0) {
|
|
127
|
-
// 4 bytes
|
|
128
|
-
let decodedValue = (byte0 & 0b00001111) |
|
|
129
|
-
(byte1 << 4) |
|
|
130
|
-
(byte2 << 12) |
|
|
131
|
-
(byte3 << 20);
|
|
132
|
-
if ((byte3 & 0b10000000) !== 0) {
|
|
133
|
-
decodedValue |= (-1 << 28);
|
|
134
|
-
}
|
|
135
|
-
return { decodedValue, readOffset };
|
|
136
|
-
}
|
|
137
|
-
const byte4 = encodedData[readOffset++];
|
|
138
|
-
if (byte0 === 0b11110000) {
|
|
139
|
-
// 5 bytes
|
|
140
|
-
let decodedValue = (byte1) |
|
|
141
|
-
(byte2 << 8) |
|
|
142
|
-
(byte3 << 16) |
|
|
143
|
-
(byte4 << 24);
|
|
144
|
-
return { decodedValue, readOffset };
|
|
145
|
-
}
|
|
146
|
-
throw new Error(`Encountered a value that is encoded with more than 5 bytes (beyond int32 range).`);
|
|
147
|
-
}
|
|
148
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
149
|
-
// Tests
|
|
150
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
151
|
-
export function testLPVarintSigned() {
|
|
152
|
-
const encodedBytes = createDynamicUint8Array();
|
|
153
|
-
function runTest(testValue) {
|
|
154
|
-
encodedBytes.clear();
|
|
155
|
-
encodeSignedInt32(testValue, encodedBytes);
|
|
156
|
-
const { decodedValue } = decodeSignedInt32(encodedBytes.elements, 0);
|
|
157
|
-
if (decodedValue !== testValue) {
|
|
158
|
-
throw new Error(`Expected ${testValue} but got ${decodedValue}`);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
for (let i = -(2 ** 26); i < 2 ** 26; i++) {
|
|
162
|
-
if (i % 1000000 === 0) {
|
|
163
|
-
logToStderr(i);
|
|
164
|
-
}
|
|
165
|
-
runTest(i);
|
|
166
|
-
}
|
|
167
|
-
const x = 1;
|
|
168
|
-
}
|
|
169
|
-
export function testLPVarintUnsigned() {
|
|
170
|
-
const encodedBytes = createDynamicUint8Array();
|
|
171
|
-
function runTest(testValue) {
|
|
172
|
-
encodedBytes.clear();
|
|
173
|
-
encodeUnsignedInt31(testValue, encodedBytes);
|
|
174
|
-
const { decodedValue } = decodeUnsignedInt31(encodedBytes.elements, 0);
|
|
175
|
-
if (decodedValue !== testValue) {
|
|
176
|
-
throw new Error(`Expected ${testValue} but got ${decodedValue}`);
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
for (let i = 0; i < 2 ** 31; i++) {
|
|
180
|
-
if (i % 1000000 === 0) {
|
|
181
|
-
logToStderr(i);
|
|
182
|
-
}
|
|
183
|
-
runTest(i);
|
|
184
|
-
}
|
|
185
|
-
const x = 1;
|
|
186
|
-
}
|
|
187
|
-
//# sourceMappingURL=LPVarInt.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"LPVarInt.js","sourceRoot":"","sources":["../../src/encodings/LPVarInt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAqB,MAAM,yCAAyC,CAAA;AACpG,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAGvD,oFAAoF;AACpF,0BAA0B;AAC1B,oFAAoF;AACpF,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,cAAiC;IACnF,KAAK,GAAG,KAAK,KAAK,CAAC,CAAA;IAEnB,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACtB,cAAc,CAAC,GAAG,CACjB,KAAK,CACL,CAAA;IACF,CAAC;SAAM,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9B,cAAc,CAAC,OAAO,CACrB,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,UAAU,EACjC,KAAK,KAAK,CAAC,CACX,CAAA;IACF,CAAC;SAAM,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9B,cAAc,CAAC,OAAO,CACrB,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,UAAU,EACjC,KAAK,KAAK,CAAC,EACX,KAAK,KAAK,EAAE,CACZ,CAAA;IACF,CAAC;SAAM,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9B,cAAc,CAAC,OAAO,CACrB,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,UAAU,EACjC,KAAK,KAAK,CAAC,EACX,KAAK,KAAK,EAAE,EACZ,KAAK,KAAK,EAAE,CACZ,CAAA;IACF,CAAC;SAAM,CAAC;QACP,cAAc,CAAC,OAAO,CACrB,UAAU,EACV,KAAK,EACL,KAAK,KAAK,CAAC,EACX,KAAK,KAAK,EAAE,EACZ,KAAK,KAAK,EAAE,CACZ,CAAA;IACF,CAAC;AACF,CAAC;AAED,oFAAoF;AACpF,0BAA0B;AAC1B,oFAAoF;AACpF,MAAM,UAAU,mBAAmB,CAAC,WAA8B,EAAE,UAAkB;IACrF,UAAU,GAAG,UAAU,GAAG,CAAC,CAAA;IAE3B,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,SAAS;QACT,IAAI,YAAY,GACf,KAAK,CAAA;QAEN,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,GAAG,UAAU,CAAC;YACpB,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QAEb,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,GAAG,UAAU,CAAC;YACpB,CAAC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QAEd,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,GAAG,UAAU,CAAC;YACpB,CAAC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC,KAAK,IAAI,EAAE,CAAC;YACb,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QAEd,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC1B,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,CAAC;YACP,CAAC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC,KAAK,IAAI,EAAE,CAAC;YACb,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QAEd,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAA;AAC7G,CAAC;AAED,oFAAoF;AACpF,wBAAwB;AACxB,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,cAAiC;IACjF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;IAEpC,IAAI,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACzB,cAAc,CAAC,GAAG,CACjB,KAAK,GAAG,UAAU,CAClB,CAAA;IACF,CAAC;SAAM,IAAI,QAAQ,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACjC,cAAc,CAAC,OAAO,CACrB,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,UAAU,EACjC,KAAK,IAAI,CAAC,CACV,CAAA;IACF,CAAC;SAAM,IAAI,QAAQ,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACjC,cAAc,CAAC,OAAO,CACrB,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,UAAU,EACjC,KAAK,IAAI,CAAC,EACV,KAAK,IAAI,EAAE,CACX,CAAA;IACF,CAAC;SAAM,IAAI,QAAQ,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACjC,cAAc,CAAC,OAAO,CACrB,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,UAAU,EACjC,KAAK,IAAI,CAAC,EACV,KAAK,IAAI,EAAE,EACX,KAAK,IAAI,EAAE,CACX,CAAA;IACF,CAAC;SAAM,CAAC;QACP,cAAc,CAAC,OAAO,CACrB,UAAU,EACV,KAAK,EACL,KAAK,IAAI,CAAC,EACV,KAAK,IAAI,EAAE,EACX,KAAK,IAAI,EAAE,CACX,CAAA;IACF,CAAC;AACF,CAAC;AAED,oFAAoF;AACpF,wBAAwB;AACxB,oFAAoF;AACpF,MAAM,UAAU,iBAAiB,CAAC,WAA8B,EAAE,UAAkB;IACnF,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,SAAS;QACT,IAAI,YAAY,GACf,KAAK,CAAA;QAEN,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,YAAY,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC1B,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,GAAG,UAAU,CAAC;YACpB,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;QAEb,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,YAAY,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC3B,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,GAAG,UAAU,CAAC;YACpB,CAAC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QAEd,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,YAAY,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC3B,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,GAAG,UAAU,CAAC;YACpB,CAAC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC,KAAK,IAAI,EAAE,CAAC;YACb,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QAEd,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,YAAY,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC3B,CAAC;QAED,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IAEvC,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC1B,UAAU;QACV,IAAI,YAAY,GACf,CAAC,KAAK,CAAC;YACP,CAAC,KAAK,IAAI,CAAC,CAAC;YACZ,CAAC,KAAK,IAAI,EAAE,CAAC;YACb,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QAEd,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,CAAA;IACpC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAA;AACpG,CAAC;AAUD,oFAAoF;AACpF,QAAQ;AACR,oFAAoF;AACpF,MAAM,UAAU,kBAAkB;IACjC,MAAM,YAAY,GAAG,uBAAuB,EAAE,CAAA;IAE9C,SAAS,OAAO,CAAC,SAAiB;QACjC,YAAY,CAAC,KAAK,EAAE,CAAA;QAEpB,iBAAiB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAA;QAC1C,MAAM,EAAE,YAAY,EAAE,GAAG,iBAAiB,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;QAEpE,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,YAAY,SAAS,YAAY,YAAY,EAAE,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;YACvB,WAAW,CAAC,CAAC,CAAC,CAAA;QACf,CAAC;QAED,OAAO,CAAC,CAAC,CAAC,CAAA;IACX,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,oBAAoB;IACnC,MAAM,YAAY,GAAG,uBAAuB,EAAE,CAAA;IAE9C,SAAS,OAAO,CAAC,SAAiB;QACjC,YAAY,CAAC,KAAK,EAAE,CAAA;QAEpB,mBAAmB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAA;QAC5C,MAAM,EAAE,YAAY,EAAE,GAAG,mBAAmB,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;QAEtE,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,YAAY,SAAS,YAAY,YAAY,EAAE,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;YACvB,WAAW,CAAC,CAAC,CAAC,CAAA;QACf,CAAC;QAED,OAAO,CAAC,CAAC,CAAC,CAAA;IACX,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC"}
|
|
@@ -1,292 +0,0 @@
|
|
|
1
|
-
import { createDynamicUint8Array, DynamicUint8Array } from "../data-structures/DynamicTypedArray.js"
|
|
2
|
-
import { logToStderr } from "../utilities/Utilities.js"
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
6
|
-
// Encode unsigned integer
|
|
7
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
8
|
-
export function encodeUnsignedInt31(value: number, outEncodedData: DynamicUint8Array) {
|
|
9
|
-
value = value >>> 0
|
|
10
|
-
|
|
11
|
-
if (value < (2 ** 7)) {
|
|
12
|
-
outEncodedData.add(
|
|
13
|
-
value
|
|
14
|
-
)
|
|
15
|
-
} else if (value < (2 ** 14)) {
|
|
16
|
-
outEncodedData.addMany(
|
|
17
|
-
(value & 0b00111111) | 0b10000000,
|
|
18
|
-
value >>> 6,
|
|
19
|
-
)
|
|
20
|
-
} else if (value < (2 ** 21)) {
|
|
21
|
-
outEncodedData.addMany(
|
|
22
|
-
(value & 0b00011111) | 0b11000000,
|
|
23
|
-
value >>> 5,
|
|
24
|
-
value >>> 13,
|
|
25
|
-
)
|
|
26
|
-
} else if (value < (2 ** 28)) {
|
|
27
|
-
outEncodedData.addMany(
|
|
28
|
-
(value & 0b00001111) | 0b11100000,
|
|
29
|
-
value >>> 4,
|
|
30
|
-
value >>> 12,
|
|
31
|
-
value >>> 20,
|
|
32
|
-
)
|
|
33
|
-
} else {
|
|
34
|
-
outEncodedData.addMany(
|
|
35
|
-
0b11110000,
|
|
36
|
-
value,
|
|
37
|
-
value >>> 8,
|
|
38
|
-
value >>> 16,
|
|
39
|
-
value >>> 24,
|
|
40
|
-
)
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
45
|
-
// Decode unsigned integer
|
|
46
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
47
|
-
export function decodeUnsignedInt31(encodedData: ArrayLike<number>, readOffset: number): DecodedValueAndReadOffset {
|
|
48
|
-
readOffset = readOffset | 0
|
|
49
|
-
|
|
50
|
-
const byte0 = encodedData[readOffset++]
|
|
51
|
-
|
|
52
|
-
if ((byte0 & 0b10000000) === 0) {
|
|
53
|
-
// 1 byte
|
|
54
|
-
let decodedValue =
|
|
55
|
-
byte0
|
|
56
|
-
|
|
57
|
-
return { decodedValue, readOffset }
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const byte1 = encodedData[readOffset++]
|
|
61
|
-
|
|
62
|
-
if ((byte0 & 0b01000000) === 0) {
|
|
63
|
-
// 2 bytes
|
|
64
|
-
let decodedValue =
|
|
65
|
-
(byte0 & 0b00111111) |
|
|
66
|
-
(byte1 << 6)
|
|
67
|
-
|
|
68
|
-
return { decodedValue, readOffset }
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
const byte2 = encodedData[readOffset++]
|
|
72
|
-
|
|
73
|
-
if ((byte0 & 0b00100000) === 0) {
|
|
74
|
-
// 3 bytes
|
|
75
|
-
let decodedValue =
|
|
76
|
-
(byte0 & 0b00011111) |
|
|
77
|
-
(byte1 << 5) |
|
|
78
|
-
(byte2 << 13)
|
|
79
|
-
|
|
80
|
-
return { decodedValue, readOffset }
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const byte3 = encodedData[readOffset++]
|
|
84
|
-
|
|
85
|
-
if ((byte0 & 0b00010000) === 0) {
|
|
86
|
-
// 4 bytes
|
|
87
|
-
let decodedValue =
|
|
88
|
-
(byte0 & 0b00001111) |
|
|
89
|
-
(byte1 << 4) |
|
|
90
|
-
(byte2 << 12) |
|
|
91
|
-
(byte3 << 20)
|
|
92
|
-
|
|
93
|
-
return { decodedValue, readOffset }
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
const byte4 = encodedData[readOffset++]
|
|
97
|
-
|
|
98
|
-
if (byte0 === 0b11110000) {
|
|
99
|
-
// 5 bytes
|
|
100
|
-
let decodedValue =
|
|
101
|
-
(byte1) |
|
|
102
|
-
(byte2 << 8) |
|
|
103
|
-
(byte3 << 16) |
|
|
104
|
-
(byte4 << 24)
|
|
105
|
-
|
|
106
|
-
return { decodedValue, readOffset }
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
throw new Error(`Encountered a value that is encoded with more than 5 bytes (beyond Unsigned Int31 range).`)
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
113
|
-
// Encode signed integer
|
|
114
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
115
|
-
export function encodeSignedInt32(value: number, outEncodedData: DynamicUint8Array) {
|
|
116
|
-
const absValue = Math.abs(value | 0)
|
|
117
|
-
|
|
118
|
-
if (absValue < (2 ** 6)) {
|
|
119
|
-
outEncodedData.add(
|
|
120
|
-
value & 0b01111111
|
|
121
|
-
)
|
|
122
|
-
} else if (absValue < (2 ** 13)) {
|
|
123
|
-
outEncodedData.addMany(
|
|
124
|
-
(value & 0b00111111) | 0b10000000,
|
|
125
|
-
value >> 6
|
|
126
|
-
)
|
|
127
|
-
} else if (absValue < (2 ** 20)) {
|
|
128
|
-
outEncodedData.addMany(
|
|
129
|
-
(value & 0b00011111) | 0b11000000,
|
|
130
|
-
value >> 5,
|
|
131
|
-
value >> 13,
|
|
132
|
-
)
|
|
133
|
-
} else if (absValue < (2 ** 27)) {
|
|
134
|
-
outEncodedData.addMany(
|
|
135
|
-
(value & 0b00001111) | 0b11100000,
|
|
136
|
-
value >> 4,
|
|
137
|
-
value >> 12,
|
|
138
|
-
value >> 20,
|
|
139
|
-
)
|
|
140
|
-
} else {
|
|
141
|
-
outEncodedData.addMany(
|
|
142
|
-
0b11110000,
|
|
143
|
-
value,
|
|
144
|
-
value >> 8,
|
|
145
|
-
value >> 16,
|
|
146
|
-
value >> 24,
|
|
147
|
-
)
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
152
|
-
// Decode signed integer
|
|
153
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
154
|
-
export function decodeSignedInt32(encodedData: ArrayLike<number>, readOffset: number): DecodedValueAndReadOffset {
|
|
155
|
-
const byte0 = encodedData[readOffset++]
|
|
156
|
-
|
|
157
|
-
if ((byte0 & 0b10000000) === 0) {
|
|
158
|
-
// 1 byte
|
|
159
|
-
let decodedValue =
|
|
160
|
-
byte0
|
|
161
|
-
|
|
162
|
-
if ((byte0 & 0b01000000) !== 0) {
|
|
163
|
-
decodedValue |= (-1 << 7)
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
return { decodedValue, readOffset }
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
const byte1 = encodedData[readOffset++]
|
|
170
|
-
|
|
171
|
-
if ((byte0 & 0b01000000) === 0) {
|
|
172
|
-
// 2 bytes
|
|
173
|
-
let decodedValue =
|
|
174
|
-
(byte0 & 0b00111111) |
|
|
175
|
-
(byte1 << 6)
|
|
176
|
-
|
|
177
|
-
if ((byte1 & 0b10000000) !== 0) {
|
|
178
|
-
decodedValue |= (-1 << 14)
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
return { decodedValue, readOffset }
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
const byte2 = encodedData[readOffset++]
|
|
185
|
-
|
|
186
|
-
if ((byte0 & 0b00100000) === 0) {
|
|
187
|
-
// 3 bytes
|
|
188
|
-
let decodedValue =
|
|
189
|
-
(byte0 & 0b00011111) |
|
|
190
|
-
(byte1 << 5) |
|
|
191
|
-
(byte2 << 13)
|
|
192
|
-
|
|
193
|
-
if ((byte2 & 0b10000000) !== 0) {
|
|
194
|
-
decodedValue |= (-1 << 21)
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
return { decodedValue, readOffset }
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
const byte3 = encodedData[readOffset++]
|
|
201
|
-
|
|
202
|
-
if ((byte0 & 0b00010000) === 0) {
|
|
203
|
-
// 4 bytes
|
|
204
|
-
let decodedValue =
|
|
205
|
-
(byte0 & 0b00001111) |
|
|
206
|
-
(byte1 << 4) |
|
|
207
|
-
(byte2 << 12) |
|
|
208
|
-
(byte3 << 20)
|
|
209
|
-
|
|
210
|
-
if ((byte3 & 0b10000000) !== 0) {
|
|
211
|
-
decodedValue |= (-1 << 28)
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
return { decodedValue, readOffset }
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
const byte4 = encodedData[readOffset++]
|
|
218
|
-
|
|
219
|
-
if (byte0 === 0b11110000) {
|
|
220
|
-
// 5 bytes
|
|
221
|
-
let decodedValue =
|
|
222
|
-
(byte1) |
|
|
223
|
-
(byte2 << 8) |
|
|
224
|
-
(byte3 << 16) |
|
|
225
|
-
(byte4 << 24)
|
|
226
|
-
|
|
227
|
-
return { decodedValue, readOffset }
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
throw new Error(`Encountered a value that is encoded with more than 5 bytes (beyond int32 range).`)
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
234
|
-
// Types
|
|
235
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
236
|
-
export interface DecodedValueAndReadOffset {
|
|
237
|
-
decodedValue: number
|
|
238
|
-
readOffset: number
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
242
|
-
// Tests
|
|
243
|
-
////////////////////////////////////////////////////////////////////////////////////
|
|
244
|
-
export function testLPVarintSigned() {
|
|
245
|
-
const encodedBytes = createDynamicUint8Array()
|
|
246
|
-
|
|
247
|
-
function runTest(testValue: number) {
|
|
248
|
-
encodedBytes.clear()
|
|
249
|
-
|
|
250
|
-
encodeSignedInt32(testValue, encodedBytes)
|
|
251
|
-
const { decodedValue } = decodeSignedInt32(encodedBytes.elements, 0)
|
|
252
|
-
|
|
253
|
-
if (decodedValue !== testValue) {
|
|
254
|
-
throw new Error(`Expected ${testValue} but got ${decodedValue}`)
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
for (let i = -(2 ** 26); i < 2 ** 26; i++) {
|
|
259
|
-
if (i % 1000000 === 0) {
|
|
260
|
-
logToStderr(i)
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
runTest(i)
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
const x = 1
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
export function testLPVarintUnsigned() {
|
|
270
|
-
const encodedBytes = createDynamicUint8Array()
|
|
271
|
-
|
|
272
|
-
function runTest(testValue: number) {
|
|
273
|
-
encodedBytes.clear()
|
|
274
|
-
|
|
275
|
-
encodeUnsignedInt31(testValue, encodedBytes)
|
|
276
|
-
const { decodedValue } = decodeUnsignedInt31(encodedBytes.elements, 0)
|
|
277
|
-
|
|
278
|
-
if (decodedValue !== testValue) {
|
|
279
|
-
throw new Error(`Expected ${testValue} but got ${decodedValue}`)
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
for (let i = 0; i < 2 ** 31; i++) {
|
|
284
|
-
if (i % 1000000 === 0) {
|
|
285
|
-
logToStderr(i)
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
runTest(i)
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
const x = 1
|
|
292
|
-
}
|