@small-ltsc/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +169 -0
- package/dist/esm/compress.js +160 -0
- package/dist/esm/compress.js.map +1 -0
- package/dist/esm/config.js +47 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/decompress.js +105 -0
- package/dist/esm/decompress.js.map +1 -0
- package/dist/esm/dictionaries/index.js +104 -0
- package/dist/esm/dictionaries/index.js.map +1 -0
- package/dist/esm/dictionaries/json.json +28 -0
- package/dist/esm/dictionaries/markdown.json +28 -0
- package/dist/esm/dictionaries/python.json +28 -0
- package/dist/esm/dictionaries/sql.json +28 -0
- package/dist/esm/dictionaries/typescript.json +28 -0
- package/dist/esm/index.js +26 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/streaming.js +139 -0
- package/dist/esm/streaming.js.map +1 -0
- package/dist/esm/types.js +25 -0
- package/dist/esm/types.js.map +1 -0
- package/dist/esm/wasm/index.js +5 -0
- package/dist/esm/wasm/index.js.map +1 -0
- package/dist/esm/wasm/loader.js +179 -0
- package/dist/esm/wasm/loader.js.map +1 -0
- package/dist/esm/worker.js +304 -0
- package/dist/esm/worker.js.map +1 -0
- package/dist/types/compress.d.ts +39 -0
- package/dist/types/compress.d.ts.map +1 -0
- package/dist/types/config.d.ts +113 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/decompress.d.ts +53 -0
- package/dist/types/decompress.d.ts.map +1 -0
- package/dist/types/dictionaries/index.d.ts +127 -0
- package/dist/types/dictionaries/index.d.ts.map +1 -0
- package/dist/types/index.d.ts +18 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/streaming.d.ts +81 -0
- package/dist/types/streaming.d.ts.map +1 -0
- package/dist/types/types.d.ts +116 -0
- package/dist/types/types.d.ts.map +1 -0
- package/dist/types/wasm/index.d.ts +6 -0
- package/dist/types/wasm/index.d.ts.map +1 -0
- package/dist/types/wasm/loader.d.ts +71 -0
- package/dist/types/wasm/loader.d.ts.map +1 -0
- package/dist/types/worker.d.ts +106 -0
- package/dist/types/worker.d.ts.map +1 -0
- package/package.json +63 -0
- package/src/wasm/small_ltsc_core_bg.wasm +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG3F,OAAO,EACL,yBAAyB,EACzB,cAAc,EACd,eAAe,EACf,KAAK,mBAAmB,GACzB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,KAAK,UAAU,GAChB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,cAAc,EACd,WAAW,GACZ,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,eAAe,EACf,UAAU,GACX,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,GACf,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,OAAO,UAAU,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Streaming compression API for large inputs.
|
|
3
|
+
*
|
|
4
|
+
* Provides a chunk-based interface for compressing token sequences
|
|
5
|
+
* that are too large to fit in memory at once.
|
|
6
|
+
*/
|
|
7
|
+
import { type CompressionConfig } from './config.js';
|
|
8
|
+
import { type CompressionResult, type TokenInput } from './types.js';
|
|
9
|
+
/**
|
|
10
|
+
* Streaming compressor for processing large token sequences.
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import { createStreamingCompressor, initWasm } from '@small-ltsc/sdk';
|
|
15
|
+
*
|
|
16
|
+
* await initWasm();
|
|
17
|
+
*
|
|
18
|
+
* const compressor = createStreamingCompressor({
|
|
19
|
+
* maxSubsequenceLength: 8,
|
|
20
|
+
* });
|
|
21
|
+
*
|
|
22
|
+
* // Add chunks as they become available
|
|
23
|
+
* for await (const chunk of tokenStream) {
|
|
24
|
+
* await compressor.addChunk(chunk);
|
|
25
|
+
* }
|
|
26
|
+
*
|
|
27
|
+
* // Finish and get result
|
|
28
|
+
* const result = await compressor.finish();
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
export interface StreamingCompressor {
|
|
32
|
+
/**
|
|
33
|
+
* Add a chunk of tokens to the compressor.
|
|
34
|
+
*
|
|
35
|
+
* @param tokens - Chunk of tokens to add
|
|
36
|
+
*/
|
|
37
|
+
addChunk(tokens: TokenInput): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Finish compression and return the result.
|
|
40
|
+
*
|
|
41
|
+
* After calling finish(), the compressor cannot be reused.
|
|
42
|
+
*
|
|
43
|
+
* @returns Promise resolving to compression result
|
|
44
|
+
*/
|
|
45
|
+
finish(): Promise<CompressionResult>;
|
|
46
|
+
/**
|
|
47
|
+
* Get the current memory usage in bytes.
|
|
48
|
+
*/
|
|
49
|
+
memoryUsage(): number;
|
|
50
|
+
/**
|
|
51
|
+
* Check if the compressor has been finished.
|
|
52
|
+
*/
|
|
53
|
+
isFinished(): boolean;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Create a new streaming compressor.
|
|
57
|
+
*
|
|
58
|
+
* @param config - Optional compression configuration
|
|
59
|
+
* @returns Streaming compressor instance
|
|
60
|
+
*/
|
|
61
|
+
export declare function createStreamingCompressor(config?: CompressionConfig): Promise<StreamingCompressor>;
|
|
62
|
+
/**
|
|
63
|
+
* Process a token stream with streaming compression.
|
|
64
|
+
*
|
|
65
|
+
* Convenience function that handles creating the compressor and
|
|
66
|
+
* processing an async iterable of token chunks.
|
|
67
|
+
*
|
|
68
|
+
* @param tokenStream - Async iterable of token chunks
|
|
69
|
+
* @param config - Optional compression configuration
|
|
70
|
+
* @returns Promise resolving to compression result
|
|
71
|
+
*/
|
|
72
|
+
export declare function compressStream(tokenStream: AsyncIterable<TokenInput>, config?: CompressionConfig): Promise<CompressionResult>;
|
|
73
|
+
/**
|
|
74
|
+
* Process tokens in chunks with a callback.
|
|
75
|
+
*
|
|
76
|
+
* @param tokens - Full token sequence
|
|
77
|
+
* @param chunkSize - Size of each chunk
|
|
78
|
+
* @param callback - Callback for each chunk
|
|
79
|
+
*/
|
|
80
|
+
export declare function processInChunks<T>(tokens: TokenInput, chunkSize: number, callback: (chunk: Uint32Array, index: number) => Promise<T>): Promise<T[]>;
|
|
81
|
+
//# sourceMappingURL=streaming.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streaming.d.ts","sourceRoot":"","sources":["../../src/streaming.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,KAAK,iBAAiB,EAA6B,MAAM,aAAa,CAAC;AAChF,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,UAAU,EAEhB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,MAAM,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAErC;;OAEG;IACH,WAAW,IAAI,MAAM,CAAC;IAEtB;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC;CACvB;AAED;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAC7C,MAAM,CAAC,EAAE,iBAAiB,GACzB,OAAO,CAAC,mBAAmB,CAAC,CAY9B;AAgGD;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,EACtC,MAAM,CAAC,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,CAAC,CAQ5B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,CAAC,EACrC,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1D,OAAO,CAAC,CAAC,EAAE,CAAC,CAYd"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core type definitions for Small LTSC SDK.
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Token type - represented as unsigned 32-bit integers.
|
|
6
|
+
*/
|
|
7
|
+
export type Token = number;
|
|
8
|
+
/**
|
|
9
|
+
* A sequence of tokens (array of u32).
|
|
10
|
+
*/
|
|
11
|
+
export type TokenSeq = readonly number[] | number[] | Uint32Array;
|
|
12
|
+
/**
|
|
13
|
+
* Result of a compression operation.
|
|
14
|
+
*/
|
|
15
|
+
export interface CompressionResult {
|
|
16
|
+
/**
|
|
17
|
+
* Original tokens that were compressed.
|
|
18
|
+
*/
|
|
19
|
+
readonly originalTokens: readonly number[];
|
|
20
|
+
/**
|
|
21
|
+
* Fully serialized output (dictionary + body).
|
|
22
|
+
*/
|
|
23
|
+
readonly serializedTokens: readonly number[];
|
|
24
|
+
/**
|
|
25
|
+
* Dictionary tokens only.
|
|
26
|
+
*/
|
|
27
|
+
readonly dictionaryTokens: readonly number[];
|
|
28
|
+
/**
|
|
29
|
+
* Body tokens only (with meta-token references).
|
|
30
|
+
*/
|
|
31
|
+
readonly bodyTokens: readonly number[];
|
|
32
|
+
/**
|
|
33
|
+
* Original sequence length.
|
|
34
|
+
*/
|
|
35
|
+
readonly originalLength: number;
|
|
36
|
+
/**
|
|
37
|
+
* Compressed sequence length.
|
|
38
|
+
*/
|
|
39
|
+
readonly compressedLength: number;
|
|
40
|
+
/**
|
|
41
|
+
* Compression ratio (compressed/original).
|
|
42
|
+
* Values < 1.0 indicate compression savings.
|
|
43
|
+
*/
|
|
44
|
+
readonly compressionRatio: number;
|
|
45
|
+
/**
|
|
46
|
+
* Mapping from meta-token to its expansion.
|
|
47
|
+
*/
|
|
48
|
+
readonly dictionaryMap: ReadonlyMap<number, readonly number[]>;
|
|
49
|
+
/**
|
|
50
|
+
* Static dictionary ID if one was used.
|
|
51
|
+
*/
|
|
52
|
+
readonly staticDictionaryId?: string;
|
|
53
|
+
/**
|
|
54
|
+
* Performance metrics from the compression operation.
|
|
55
|
+
*/
|
|
56
|
+
readonly metrics?: CompressionMetrics;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Performance metrics from compression.
|
|
60
|
+
*/
|
|
61
|
+
export interface CompressionMetrics {
|
|
62
|
+
/**
|
|
63
|
+
* Time spent in pattern discovery (ms).
|
|
64
|
+
*/
|
|
65
|
+
readonly discoveryTimeMs: number;
|
|
66
|
+
/**
|
|
67
|
+
* Time spent in pattern selection (ms).
|
|
68
|
+
*/
|
|
69
|
+
readonly selectionTimeMs: number;
|
|
70
|
+
/**
|
|
71
|
+
* Time spent in serialization (ms).
|
|
72
|
+
*/
|
|
73
|
+
readonly serializationTimeMs: number;
|
|
74
|
+
/**
|
|
75
|
+
* Total compression time (ms).
|
|
76
|
+
*/
|
|
77
|
+
readonly totalTimeMs: number;
|
|
78
|
+
/**
|
|
79
|
+
* Peak memory usage (bytes).
|
|
80
|
+
*/
|
|
81
|
+
readonly peakMemoryBytes: number;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* A discovered pattern with its positions.
|
|
85
|
+
*/
|
|
86
|
+
export interface DiscoveredPattern {
|
|
87
|
+
/**
|
|
88
|
+
* The token pattern.
|
|
89
|
+
*/
|
|
90
|
+
readonly pattern: readonly number[];
|
|
91
|
+
/**
|
|
92
|
+
* Length of the pattern.
|
|
93
|
+
*/
|
|
94
|
+
readonly length: number;
|
|
95
|
+
/**
|
|
96
|
+
* Non-overlapping positions where the pattern occurs.
|
|
97
|
+
*/
|
|
98
|
+
readonly positions: readonly number[];
|
|
99
|
+
/**
|
|
100
|
+
* Number of occurrences.
|
|
101
|
+
*/
|
|
102
|
+
readonly count: number;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Input that can be used for compression/decompression.
|
|
106
|
+
*/
|
|
107
|
+
export type TokenInput = TokenSeq;
|
|
108
|
+
/**
|
|
109
|
+
* Normalize any token input to Uint32Array.
|
|
110
|
+
*/
|
|
111
|
+
export declare function normalizeTokens(tokens: TokenInput): Uint32Array;
|
|
112
|
+
/**
|
|
113
|
+
* Check if a value is a valid token sequence.
|
|
114
|
+
*/
|
|
115
|
+
export declare function isTokenSeq(value: unknown): value is TokenSeq;
|
|
116
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAE3C;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAEvC;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAE/D;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,kBAAkB,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC;;OAEG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IAEtC;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,CAAC;AAElC;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,WAAW,CAK/D;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAU5D"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WASM module exports.
|
|
3
|
+
*/
|
|
4
|
+
export { initWasm, initWasmFromModule, initWasmFromBytes, getWasm, isWasmInitialized, resetWasm, getWasmVersion, } from './loader.js';
|
|
5
|
+
export type { WasmExports, CompressionResultWasm, StreamingCompressorWasm, } from './loader.js';
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/wasm/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,iBAAiB,EACjB,OAAO,EACP,iBAAiB,EACjB,SAAS,EACT,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,WAAW,EACX,qBAAqB,EACrB,uBAAuB,GACxB,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WASM module loader with cross-platform support.
|
|
3
|
+
*
|
|
4
|
+
* Handles loading the WebAssembly module in browser, Node.js, and Deno environments.
|
|
5
|
+
*/
|
|
6
|
+
export interface WasmExports {
|
|
7
|
+
memory: WebAssembly.Memory;
|
|
8
|
+
compress: (tokens: Uint32Array, config: unknown) => CompressionResultWasm;
|
|
9
|
+
decompress: (tokens: Uint32Array, config: unknown) => Uint32Array;
|
|
10
|
+
discover_patterns: (tokens: Uint32Array, minLength: number, maxLength: number) => unknown;
|
|
11
|
+
version: () => string;
|
|
12
|
+
StreamingCompressor: new (config: unknown) => StreamingCompressorWasm;
|
|
13
|
+
}
|
|
14
|
+
export interface CompressionResultWasm {
|
|
15
|
+
compression_ratio: number;
|
|
16
|
+
tokens_saved: number;
|
|
17
|
+
original_length: number;
|
|
18
|
+
compressed_length: number;
|
|
19
|
+
getSerializedTokens: () => Uint32Array;
|
|
20
|
+
getDictionaryTokens: () => Uint32Array;
|
|
21
|
+
getBodyTokens: () => Uint32Array;
|
|
22
|
+
getOriginalTokens: () => Uint32Array;
|
|
23
|
+
getStaticDictionaryId: () => string | null;
|
|
24
|
+
}
|
|
25
|
+
export interface StreamingCompressorWasm {
|
|
26
|
+
add_chunk: (tokens: Uint32Array) => void;
|
|
27
|
+
finish: () => CompressionResultWasm;
|
|
28
|
+
memory_usage: () => number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Initialize the WASM module.
|
|
32
|
+
*
|
|
33
|
+
* This function is idempotent - calling it multiple times will only
|
|
34
|
+
* initialize once.
|
|
35
|
+
*
|
|
36
|
+
* @throws Error if WASM loading fails
|
|
37
|
+
*/
|
|
38
|
+
export declare function initWasm(): Promise<void>;
|
|
39
|
+
/**
|
|
40
|
+
* Initialize from pre-compiled WASM module.
|
|
41
|
+
*
|
|
42
|
+
* Useful for environments where the WASM is bundled differently.
|
|
43
|
+
*
|
|
44
|
+
* @param module - Pre-compiled WebAssembly.Module
|
|
45
|
+
*/
|
|
46
|
+
export declare function initWasmFromModule(module: WebAssembly.Module): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Initialize from WASM bytes.
|
|
49
|
+
*
|
|
50
|
+
* @param bytes - WASM binary as ArrayBuffer or Uint8Array
|
|
51
|
+
*/
|
|
52
|
+
export declare function initWasmFromBytes(bytes: ArrayBuffer | Uint8Array): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Get the initialized WASM exports.
|
|
55
|
+
*
|
|
56
|
+
* @throws Error if WASM is not initialized
|
|
57
|
+
*/
|
|
58
|
+
export declare function getWasm(): WasmExports;
|
|
59
|
+
/**
|
|
60
|
+
* Check if WASM is initialized.
|
|
61
|
+
*/
|
|
62
|
+
export declare function isWasmInitialized(): boolean;
|
|
63
|
+
/**
|
|
64
|
+
* Reset the WASM instance (mainly for testing).
|
|
65
|
+
*/
|
|
66
|
+
export declare function resetWasm(): void;
|
|
67
|
+
/**
|
|
68
|
+
* Get WASM module version.
|
|
69
|
+
*/
|
|
70
|
+
export declare function getWasmVersion(): string;
|
|
71
|
+
//# sourceMappingURL=loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/wasm/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;IAC3B,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,KAAK,qBAAqB,CAAC;IAC1E,UAAU,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,KAAK,WAAW,CAAC;IAClE,iBAAiB,EAAE,CACjB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,KACd,OAAO,CAAC;IACb,OAAO,EAAE,MAAM,MAAM,CAAC;IACtB,mBAAmB,EAAE,KAAK,MAAM,EAAE,OAAO,KAAK,uBAAuB,CAAC;CACvE;AAED,MAAM,WAAW,qBAAqB;IACpC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,WAAW,CAAC;IACvC,mBAAmB,EAAE,MAAM,WAAW,CAAC;IACvC,aAAa,EAAE,MAAM,WAAW,CAAC;IACjC,iBAAiB,EAAE,MAAM,WAAW,CAAC;IACrC,qBAAqB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;IACzC,MAAM,EAAE,MAAM,qBAAqB,CAAC;IACpC,YAAY,EAAE,MAAM,MAAM,CAAC;CAC5B;AA6FD;;;;;;;GAOG;AACH,wBAAsB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB9C;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,WAAW,CAAC,MAAM,GACzB,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,WAAW,GAAG,UAAU,GAC9B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;GAIG;AACH,wBAAgB,OAAO,IAAI,WAAW,CAOrC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAED;;GAEG;AACH,wBAAgB,SAAS,IAAI,IAAI,CAIhC;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAGvC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker thread support for non-blocking compression.
|
|
3
|
+
*
|
|
4
|
+
* Provides utilities for running compression in Web Workers (browser)
|
|
5
|
+
* or Worker threads (Node.js) to avoid blocking the main thread.
|
|
6
|
+
*/
|
|
7
|
+
import type { CompressionConfig, DecompressionConfig } from './config.js';
|
|
8
|
+
import type { CompressionResult, TokenInput } from './types.js';
|
|
9
|
+
/**
|
|
10
|
+
* Message types for worker communication.
|
|
11
|
+
*/
|
|
12
|
+
export type WorkerRequest = {
|
|
13
|
+
type: 'compress';
|
|
14
|
+
requestId: string;
|
|
15
|
+
tokens: number[];
|
|
16
|
+
config?: CompressionConfig;
|
|
17
|
+
} | {
|
|
18
|
+
type: 'decompress';
|
|
19
|
+
requestId: string;
|
|
20
|
+
tokens: number[];
|
|
21
|
+
config?: DecompressionConfig;
|
|
22
|
+
} | {
|
|
23
|
+
type: 'init';
|
|
24
|
+
};
|
|
25
|
+
export type WorkerResponse = {
|
|
26
|
+
type: 'compress';
|
|
27
|
+
requestId: string;
|
|
28
|
+
result: CompressionResult;
|
|
29
|
+
error: null;
|
|
30
|
+
} | {
|
|
31
|
+
type: 'compress';
|
|
32
|
+
requestId: string;
|
|
33
|
+
result: null;
|
|
34
|
+
error: string;
|
|
35
|
+
} | {
|
|
36
|
+
type: 'decompress';
|
|
37
|
+
requestId: string;
|
|
38
|
+
result: readonly number[];
|
|
39
|
+
error: null;
|
|
40
|
+
} | {
|
|
41
|
+
type: 'decompress';
|
|
42
|
+
requestId: string;
|
|
43
|
+
result: null;
|
|
44
|
+
error: string;
|
|
45
|
+
} | {
|
|
46
|
+
type: 'init';
|
|
47
|
+
success: boolean;
|
|
48
|
+
error: string | null;
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Worker pool for parallel compression.
|
|
52
|
+
*/
|
|
53
|
+
export interface WorkerPool {
|
|
54
|
+
/**
|
|
55
|
+
* Compress tokens using a worker.
|
|
56
|
+
*/
|
|
57
|
+
compress(tokens: TokenInput, config?: CompressionConfig): Promise<CompressionResult>;
|
|
58
|
+
/**
|
|
59
|
+
* Decompress tokens using a worker.
|
|
60
|
+
*/
|
|
61
|
+
decompress(tokens: TokenInput, config?: DecompressionConfig): Promise<readonly number[]>;
|
|
62
|
+
/**
|
|
63
|
+
* Terminate all workers in the pool.
|
|
64
|
+
*/
|
|
65
|
+
terminate(): void;
|
|
66
|
+
/**
|
|
67
|
+
* Get the number of workers in the pool.
|
|
68
|
+
*/
|
|
69
|
+
size(): number;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Create a worker pool for parallel compression.
|
|
73
|
+
*
|
|
74
|
+
* @param workerCount - Number of workers to create (default: navigator.hardwareConcurrency or 4)
|
|
75
|
+
* @returns Worker pool instance
|
|
76
|
+
*
|
|
77
|
+
* @example
|
|
78
|
+
* ```typescript
|
|
79
|
+
* import { createWorkerPool } from '@small-ltsc/sdk';
|
|
80
|
+
*
|
|
81
|
+
* const pool = await createWorkerPool(4);
|
|
82
|
+
*
|
|
83
|
+
* // Compress without blocking main thread
|
|
84
|
+
* const result = await pool.compress(tokens);
|
|
85
|
+
*
|
|
86
|
+
* // Clean up when done
|
|
87
|
+
* pool.terminate();
|
|
88
|
+
* ```
|
|
89
|
+
*/
|
|
90
|
+
export declare function createWorkerPool(workerCount?: number): Promise<WorkerPool>;
|
|
91
|
+
/**
|
|
92
|
+
* Compress tokens in a worker (single-use helper).
|
|
93
|
+
*
|
|
94
|
+
* Creates a temporary worker, runs compression, and terminates.
|
|
95
|
+
* For multiple compressions, use createWorkerPool instead.
|
|
96
|
+
*
|
|
97
|
+
* @param tokens - Token sequence to compress
|
|
98
|
+
* @param config - Optional compression configuration
|
|
99
|
+
* @returns Promise resolving to compression result
|
|
100
|
+
*/
|
|
101
|
+
export declare function compressInWorker(tokens: TokenInput, config?: CompressionConfig): Promise<CompressionResult>;
|
|
102
|
+
/**
|
|
103
|
+
* Decompress tokens in a worker (single-use helper).
|
|
104
|
+
*/
|
|
105
|
+
export declare function decompressInWorker(tokens: TokenInput, config?: DecompressionConfig): Promise<readonly number[]>;
|
|
106
|
+
//# sourceMappingURL=worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,iBAAiB,CAAA;CAAE,GACrF;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,mBAAmB,CAAA;CAAE,GACzF;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErB,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,IAAI,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACpE;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,CAAA;CAAE,GACjF;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACtE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC;AAU7D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAErF;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;IAEzF;;OAEG;IACH,SAAS,IAAI,IAAI,CAAC;IAElB;;OAEG;IACH,IAAI,IAAI,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,gBAAgB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAchF;AA2QD;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,CAAC,CAQ5B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAQ5B"}
|
package/package.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@small-ltsc/sdk",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "TypeScript SDK for Small LTSC - Lossless Token Sequence Compression",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/esm/index.js",
|
|
7
|
+
"module": "./dist/esm/index.js",
|
|
8
|
+
"types": "./dist/types/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/esm/index.js",
|
|
12
|
+
"types": "./dist/types/index.d.ts"
|
|
13
|
+
},
|
|
14
|
+
"./wasm": {
|
|
15
|
+
"import": "./dist/esm/wasm/index.js",
|
|
16
|
+
"types": "./dist/types/wasm/index.d.ts"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"dist",
|
|
21
|
+
"src/wasm/*.wasm"
|
|
22
|
+
],
|
|
23
|
+
"sideEffects": false,
|
|
24
|
+
"scripts": {
|
|
25
|
+
"clean": "rm -rf dist",
|
|
26
|
+
"build": "npm run build:esm && npm run build:types",
|
|
27
|
+
"build:esm": "tsc -p tsconfig.esm.json",
|
|
28
|
+
"build:types": "tsc -p tsconfig.types.json",
|
|
29
|
+
"test": "vitest",
|
|
30
|
+
"test:browser": "playwright test",
|
|
31
|
+
"serve:test": "node ./scripts/serve-test.mjs",
|
|
32
|
+
"lint": "eslint src --ext .ts",
|
|
33
|
+
"prepublishOnly": "npm run clean && npm run build"
|
|
34
|
+
},
|
|
35
|
+
"keywords": [
|
|
36
|
+
"compression",
|
|
37
|
+
"llm",
|
|
38
|
+
"tokens",
|
|
39
|
+
"transformer",
|
|
40
|
+
"wasm",
|
|
41
|
+
"lossless"
|
|
42
|
+
],
|
|
43
|
+
"author": "",
|
|
44
|
+
"license": "MIT",
|
|
45
|
+
"repository": {
|
|
46
|
+
"type": "git",
|
|
47
|
+
"url": "git+https://github.com/triage-sec/small.git"
|
|
48
|
+
},
|
|
49
|
+
"bugs": {
|
|
50
|
+
"url": "https://github.com/triage-sec/small/issues"
|
|
51
|
+
},
|
|
52
|
+
"homepage": "https://github.com/triage-sec/small#readme",
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"@playwright/test": "^1.40.0",
|
|
55
|
+
"@types/node": "^20.10.0",
|
|
56
|
+
"eslint": "^8.55.0",
|
|
57
|
+
"typescript": "^5.3.0",
|
|
58
|
+
"vitest": "^1.0.0"
|
|
59
|
+
},
|
|
60
|
+
"engines": {
|
|
61
|
+
"node": ">=18.0.0"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
Binary file
|