@small-ltsc/sdk 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +169 -0
  2. package/dist/esm/compress.js +160 -0
  3. package/dist/esm/compress.js.map +1 -0
  4. package/dist/esm/config.js +47 -0
  5. package/dist/esm/config.js.map +1 -0
  6. package/dist/esm/decompress.js +105 -0
  7. package/dist/esm/decompress.js.map +1 -0
  8. package/dist/esm/dictionaries/index.js +104 -0
  9. package/dist/esm/dictionaries/index.js.map +1 -0
  10. package/dist/esm/dictionaries/json.json +28 -0
  11. package/dist/esm/dictionaries/markdown.json +28 -0
  12. package/dist/esm/dictionaries/python.json +28 -0
  13. package/dist/esm/dictionaries/sql.json +28 -0
  14. package/dist/esm/dictionaries/typescript.json +28 -0
  15. package/dist/esm/index.js +26 -0
  16. package/dist/esm/index.js.map +1 -0
  17. package/dist/esm/streaming.js +139 -0
  18. package/dist/esm/streaming.js.map +1 -0
  19. package/dist/esm/types.js +25 -0
  20. package/dist/esm/types.js.map +1 -0
  21. package/dist/esm/wasm/index.js +5 -0
  22. package/dist/esm/wasm/index.js.map +1 -0
  23. package/dist/esm/wasm/loader.js +179 -0
  24. package/dist/esm/wasm/loader.js.map +1 -0
  25. package/dist/esm/worker.js +304 -0
  26. package/dist/esm/worker.js.map +1 -0
  27. package/dist/types/compress.d.ts +39 -0
  28. package/dist/types/compress.d.ts.map +1 -0
  29. package/dist/types/config.d.ts +113 -0
  30. package/dist/types/config.d.ts.map +1 -0
  31. package/dist/types/decompress.d.ts +53 -0
  32. package/dist/types/decompress.d.ts.map +1 -0
  33. package/dist/types/dictionaries/index.d.ts +127 -0
  34. package/dist/types/dictionaries/index.d.ts.map +1 -0
  35. package/dist/types/index.d.ts +18 -0
  36. package/dist/types/index.d.ts.map +1 -0
  37. package/dist/types/streaming.d.ts +81 -0
  38. package/dist/types/streaming.d.ts.map +1 -0
  39. package/dist/types/types.d.ts +116 -0
  40. package/dist/types/types.d.ts.map +1 -0
  41. package/dist/types/wasm/index.d.ts +6 -0
  42. package/dist/types/wasm/index.d.ts.map +1 -0
  43. package/dist/types/wasm/loader.d.ts +71 -0
  44. package/dist/types/wasm/loader.d.ts.map +1 -0
  45. package/dist/types/worker.d.ts +106 -0
  46. package/dist/types/worker.d.ts.map +1 -0
  47. package/package.json +63 -0
  48. package/src/wasm/small_ltsc_core_bg.wasm +0 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAG3F,OAAO,EACL,yBAAyB,EACzB,cAAc,EACd,eAAe,EACf,KAAK,mBAAmB,GACzB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,KAAK,UAAU,GAChB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,cAAc,EACd,WAAW,GACZ,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,eAAe,EACf,UAAU,GACX,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,oBAAoB,EACpB,sBAAsB,EACtB,sBAAsB,EACtB,mBAAmB,EACnB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,GACf,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,OAAO,UAAU,CAAC"}
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Streaming compression API for large inputs.
3
+ *
4
+ * Provides a chunk-based interface for compressing token sequences
5
+ * that are too large to fit in memory at once.
6
+ */
7
+ import { type CompressionConfig } from './config.js';
8
+ import { type CompressionResult, type TokenInput } from './types.js';
9
+ /**
10
+ * Streaming compressor for processing large token sequences.
11
+ *
12
+ * @example
13
+ * ```typescript
14
+ * import { createStreamingCompressor, initWasm } from '@small-ltsc/sdk';
15
+ *
16
+ * await initWasm();
17
+ *
18
+ * const compressor = createStreamingCompressor({
19
+ * maxSubsequenceLength: 8,
20
+ * });
21
+ *
22
+ * // Add chunks as they become available
23
+ * for await (const chunk of tokenStream) {
24
+ * await compressor.addChunk(chunk);
25
+ * }
26
+ *
27
+ * // Finish and get result
28
+ * const result = await compressor.finish();
29
+ * ```
30
+ */
31
+ export interface StreamingCompressor {
32
+ /**
33
+ * Add a chunk of tokens to the compressor.
34
+ *
35
+ * @param tokens - Chunk of tokens to add
36
+ */
37
+ addChunk(tokens: TokenInput): Promise<void>;
38
+ /**
39
+ * Finish compression and return the result.
40
+ *
41
+ * After calling finish(), the compressor cannot be reused.
42
+ *
43
+ * @returns Promise resolving to compression result
44
+ */
45
+ finish(): Promise<CompressionResult>;
46
+ /**
47
+ * Get the current memory usage in bytes.
48
+ */
49
+ memoryUsage(): number;
50
+ /**
51
+ * Check if the compressor has been finished.
52
+ */
53
+ isFinished(): boolean;
54
+ }
55
+ /**
56
+ * Create a new streaming compressor.
57
+ *
58
+ * @param config - Optional compression configuration
59
+ * @returns Streaming compressor instance
60
+ */
61
+ export declare function createStreamingCompressor(config?: CompressionConfig): Promise<StreamingCompressor>;
62
+ /**
63
+ * Process a token stream with streaming compression.
64
+ *
65
+ * Convenience function that handles creating the compressor and
66
+ * processing an async iterable of token chunks.
67
+ *
68
+ * @param tokenStream - Async iterable of token chunks
69
+ * @param config - Optional compression configuration
70
+ * @returns Promise resolving to compression result
71
+ */
72
+ export declare function compressStream(tokenStream: AsyncIterable<TokenInput>, config?: CompressionConfig): Promise<CompressionResult>;
73
+ /**
74
+ * Process tokens in chunks with a callback.
75
+ *
76
+ * @param tokens - Full token sequence
77
+ * @param chunkSize - Size of each chunk
78
+ * @param callback - Callback for each chunk
79
+ */
80
+ export declare function processInChunks<T>(tokens: TokenInput, chunkSize: number, callback: (chunk: Uint32Array, index: number) => Promise<T>): Promise<T[]>;
81
+ //# sourceMappingURL=streaming.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"streaming.d.ts","sourceRoot":"","sources":["../../src/streaming.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,KAAK,iBAAiB,EAA6B,MAAM,aAAa,CAAC;AAChF,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,UAAU,EAEhB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5C;;;;;;OAMG;IACH,MAAM,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAErC;;OAEG;IACH,WAAW,IAAI,MAAM,CAAC;IAEtB;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC;CACvB;AAED;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAC7C,MAAM,CAAC,EAAE,iBAAiB,GACzB,OAAO,CAAC,mBAAmB,CAAC,CAY9B;AAgGD;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,WAAW,EAAE,aAAa,CAAC,UAAU,CAAC,EACtC,MAAM,CAAC,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,CAAC,CAQ5B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,CAAC,EACrC,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAC1D,OAAO,CAAC,CAAC,EAAE,CAAC,CAYd"}
@@ -0,0 +1,116 @@
1
+ /**
2
+ * Core type definitions for Small LTSC SDK.
3
+ */
4
+ /**
5
+ * Token type - represented as unsigned 32-bit integers.
6
+ */
7
+ export type Token = number;
8
+ /**
9
+ * A sequence of tokens (array of u32).
10
+ */
11
+ export type TokenSeq = readonly number[] | number[] | Uint32Array;
12
+ /**
13
+ * Result of a compression operation.
14
+ */
15
+ export interface CompressionResult {
16
+ /**
17
+ * Original tokens that were compressed.
18
+ */
19
+ readonly originalTokens: readonly number[];
20
+ /**
21
+ * Fully serialized output (dictionary + body).
22
+ */
23
+ readonly serializedTokens: readonly number[];
24
+ /**
25
+ * Dictionary tokens only.
26
+ */
27
+ readonly dictionaryTokens: readonly number[];
28
+ /**
29
+ * Body tokens only (with meta-token references).
30
+ */
31
+ readonly bodyTokens: readonly number[];
32
+ /**
33
+ * Original sequence length.
34
+ */
35
+ readonly originalLength: number;
36
+ /**
37
+ * Compressed sequence length.
38
+ */
39
+ readonly compressedLength: number;
40
+ /**
41
+ * Compression ratio (compressed/original).
42
+ * Values < 1.0 indicate compression savings.
43
+ */
44
+ readonly compressionRatio: number;
45
+ /**
46
+ * Mapping from meta-token to its expansion.
47
+ */
48
+ readonly dictionaryMap: ReadonlyMap<number, readonly number[]>;
49
+ /**
50
+ * Static dictionary ID if one was used.
51
+ */
52
+ readonly staticDictionaryId?: string;
53
+ /**
54
+ * Performance metrics from the compression operation.
55
+ */
56
+ readonly metrics?: CompressionMetrics;
57
+ }
58
+ /**
59
+ * Performance metrics from compression.
60
+ */
61
+ export interface CompressionMetrics {
62
+ /**
63
+ * Time spent in pattern discovery (ms).
64
+ */
65
+ readonly discoveryTimeMs: number;
66
+ /**
67
+ * Time spent in pattern selection (ms).
68
+ */
69
+ readonly selectionTimeMs: number;
70
+ /**
71
+ * Time spent in serialization (ms).
72
+ */
73
+ readonly serializationTimeMs: number;
74
+ /**
75
+ * Total compression time (ms).
76
+ */
77
+ readonly totalTimeMs: number;
78
+ /**
79
+ * Peak memory usage (bytes).
80
+ */
81
+ readonly peakMemoryBytes: number;
82
+ }
83
+ /**
84
+ * A discovered pattern with its positions.
85
+ */
86
+ export interface DiscoveredPattern {
87
+ /**
88
+ * The token pattern.
89
+ */
90
+ readonly pattern: readonly number[];
91
+ /**
92
+ * Length of the pattern.
93
+ */
94
+ readonly length: number;
95
+ /**
96
+ * Non-overlapping positions where the pattern occurs.
97
+ */
98
+ readonly positions: readonly number[];
99
+ /**
100
+ * Number of occurrences.
101
+ */
102
+ readonly count: number;
103
+ }
104
+ /**
105
+ * Input that can be used for compression/decompression.
106
+ */
107
+ export type TokenInput = TokenSeq;
108
+ /**
109
+ * Normalize any token input to Uint32Array.
110
+ */
111
+ export declare function normalizeTokens(tokens: TokenInput): Uint32Array;
112
+ /**
113
+ * Check if a value is a valid token sequence.
114
+ */
115
+ export declare function isTokenSeq(value: unknown): value is TokenSeq;
116
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,MAAM,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;IAE3C;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C;;OAEG;IACH,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAEvC;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC;;;OAGG;IACH,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC;;OAEG;IACH,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC;IAE/D;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,kBAAkB,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IAEjC;;OAEG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IAErC;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAEpC;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IAEtC;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,CAAC;AAElC;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,UAAU,GAAG,WAAW,CAK/D;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAU5D"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * WASM module exports.
3
+ */
4
+ export { initWasm, initWasmFromModule, initWasmFromBytes, getWasm, isWasmInitialized, resetWasm, getWasmVersion, } from './loader.js';
5
+ export type { WasmExports, CompressionResultWasm, StreamingCompressorWasm, } from './loader.js';
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/wasm/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,iBAAiB,EACjB,OAAO,EACP,iBAAiB,EACjB,SAAS,EACT,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,WAAW,EACX,qBAAqB,EACrB,uBAAuB,GACxB,MAAM,aAAa,CAAC"}
@@ -0,0 +1,71 @@
1
+ /**
2
+ * WASM module loader with cross-platform support.
3
+ *
4
+ * Handles loading the WebAssembly module in browser, Node.js, and Deno environments.
5
+ */
6
+ export interface WasmExports {
7
+ memory: WebAssembly.Memory;
8
+ compress: (tokens: Uint32Array, config: unknown) => CompressionResultWasm;
9
+ decompress: (tokens: Uint32Array, config: unknown) => Uint32Array;
10
+ discover_patterns: (tokens: Uint32Array, minLength: number, maxLength: number) => unknown;
11
+ version: () => string;
12
+ StreamingCompressor: new (config: unknown) => StreamingCompressorWasm;
13
+ }
14
+ export interface CompressionResultWasm {
15
+ compression_ratio: number;
16
+ tokens_saved: number;
17
+ original_length: number;
18
+ compressed_length: number;
19
+ getSerializedTokens: () => Uint32Array;
20
+ getDictionaryTokens: () => Uint32Array;
21
+ getBodyTokens: () => Uint32Array;
22
+ getOriginalTokens: () => Uint32Array;
23
+ getStaticDictionaryId: () => string | null;
24
+ }
25
+ export interface StreamingCompressorWasm {
26
+ add_chunk: (tokens: Uint32Array) => void;
27
+ finish: () => CompressionResultWasm;
28
+ memory_usage: () => number;
29
+ }
30
+ /**
31
+ * Initialize the WASM module.
32
+ *
33
+ * This function is idempotent - calling it multiple times will only
34
+ * initialize once.
35
+ *
36
+ * @throws Error if WASM loading fails
37
+ */
38
+ export declare function initWasm(): Promise<void>;
39
+ /**
40
+ * Initialize from pre-compiled WASM module.
41
+ *
42
+ * Useful for environments where the WASM is bundled differently.
43
+ *
44
+ * @param module - Pre-compiled WebAssembly.Module
45
+ */
46
+ export declare function initWasmFromModule(module: WebAssembly.Module): Promise<void>;
47
+ /**
48
+ * Initialize from WASM bytes.
49
+ *
50
+ * @param bytes - WASM binary as ArrayBuffer or Uint8Array
51
+ */
52
+ export declare function initWasmFromBytes(bytes: ArrayBuffer | Uint8Array): Promise<void>;
53
+ /**
54
+ * Get the initialized WASM exports.
55
+ *
56
+ * @throws Error if WASM is not initialized
57
+ */
58
+ export declare function getWasm(): WasmExports;
59
+ /**
60
+ * Check if WASM is initialized.
61
+ */
62
+ export declare function isWasmInitialized(): boolean;
63
+ /**
64
+ * Reset the WASM instance (mainly for testing).
65
+ */
66
+ export declare function resetWasm(): void;
67
+ /**
68
+ * Get WASM module version.
69
+ */
70
+ export declare function getWasmVersion(): string;
71
+ //# sourceMappingURL=loader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/wasm/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;IAC3B,QAAQ,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,KAAK,qBAAqB,CAAC;IAC1E,UAAU,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,KAAK,WAAW,CAAC;IAClE,iBAAiB,EAAE,CACjB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,KACd,OAAO,CAAC;IACb,OAAO,EAAE,MAAM,MAAM,CAAC;IACtB,mBAAmB,EAAE,KAAK,MAAM,EAAE,OAAO,KAAK,uBAAuB,CAAC;CACvE;AAED,MAAM,WAAW,qBAAqB;IACpC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,WAAW,CAAC;IACvC,mBAAmB,EAAE,MAAM,WAAW,CAAC;IACvC,aAAa,EAAE,MAAM,WAAW,CAAC;IACjC,iBAAiB,EAAE,MAAM,WAAW,CAAC;IACrC,qBAAqB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,CAAC;IACzC,MAAM,EAAE,MAAM,qBAAqB,CAAC;IACpC,YAAY,EAAE,MAAM,MAAM,CAAC;CAC5B;AA6FD;;;;;;;GAOG;AACH,wBAAsB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAuB9C;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,WAAW,CAAC,MAAM,GACzB,OAAO,CAAC,IAAI,CAAC,CASf;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,WAAW,GAAG,UAAU,GAC9B,OAAO,CAAC,IAAI,CAAC,CAYf;AAED;;;;GAIG;AACH,wBAAgB,OAAO,IAAI,WAAW,CAOrC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,OAAO,CAE3C;AAED;;GAEG;AACH,wBAAgB,SAAS,IAAI,IAAI,CAIhC;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,MAAM,CAGvC"}
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Worker thread support for non-blocking compression.
3
+ *
4
+ * Provides utilities for running compression in Web Workers (browser)
5
+ * or Worker threads (Node.js) to avoid blocking the main thread.
6
+ */
7
+ import type { CompressionConfig, DecompressionConfig } from './config.js';
8
+ import type { CompressionResult, TokenInput } from './types.js';
9
+ /**
10
+ * Message types for worker communication.
11
+ */
12
+ export type WorkerRequest = {
13
+ type: 'compress';
14
+ requestId: string;
15
+ tokens: number[];
16
+ config?: CompressionConfig;
17
+ } | {
18
+ type: 'decompress';
19
+ requestId: string;
20
+ tokens: number[];
21
+ config?: DecompressionConfig;
22
+ } | {
23
+ type: 'init';
24
+ };
25
+ export type WorkerResponse = {
26
+ type: 'compress';
27
+ requestId: string;
28
+ result: CompressionResult;
29
+ error: null;
30
+ } | {
31
+ type: 'compress';
32
+ requestId: string;
33
+ result: null;
34
+ error: string;
35
+ } | {
36
+ type: 'decompress';
37
+ requestId: string;
38
+ result: readonly number[];
39
+ error: null;
40
+ } | {
41
+ type: 'decompress';
42
+ requestId: string;
43
+ result: null;
44
+ error: string;
45
+ } | {
46
+ type: 'init';
47
+ success: boolean;
48
+ error: string | null;
49
+ };
50
+ /**
51
+ * Worker pool for parallel compression.
52
+ */
53
+ export interface WorkerPool {
54
+ /**
55
+ * Compress tokens using a worker.
56
+ */
57
+ compress(tokens: TokenInput, config?: CompressionConfig): Promise<CompressionResult>;
58
+ /**
59
+ * Decompress tokens using a worker.
60
+ */
61
+ decompress(tokens: TokenInput, config?: DecompressionConfig): Promise<readonly number[]>;
62
+ /**
63
+ * Terminate all workers in the pool.
64
+ */
65
+ terminate(): void;
66
+ /**
67
+ * Get the number of workers in the pool.
68
+ */
69
+ size(): number;
70
+ }
71
+ /**
72
+ * Create a worker pool for parallel compression.
73
+ *
74
+ * @param workerCount - Number of workers to create (default: navigator.hardwareConcurrency or 4)
75
+ * @returns Worker pool instance
76
+ *
77
+ * @example
78
+ * ```typescript
79
+ * import { createWorkerPool } from '@small-ltsc/sdk';
80
+ *
81
+ * const pool = await createWorkerPool(4);
82
+ *
83
+ * // Compress without blocking main thread
84
+ * const result = await pool.compress(tokens);
85
+ *
86
+ * // Clean up when done
87
+ * pool.terminate();
88
+ * ```
89
+ */
90
+ export declare function createWorkerPool(workerCount?: number): Promise<WorkerPool>;
91
+ /**
92
+ * Compress tokens in a worker (single-use helper).
93
+ *
94
+ * Creates a temporary worker, runs compression, and terminates.
95
+ * For multiple compressions, use createWorkerPool instead.
96
+ *
97
+ * @param tokens - Token sequence to compress
98
+ * @param config - Optional compression configuration
99
+ * @returns Promise resolving to compression result
100
+ */
101
+ export declare function compressInWorker(tokens: TokenInput, config?: CompressionConfig): Promise<CompressionResult>;
102
+ /**
103
+ * Decompress tokens in a worker (single-use helper).
104
+ */
105
+ export declare function decompressInWorker(tokens: TokenInput, config?: DecompressionConfig): Promise<readonly number[]>;
106
+ //# sourceMappingURL=worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,iBAAiB,CAAA;CAAE,GACrF;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,mBAAmB,CAAA;CAAE,GACzF;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAErB,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,IAAI,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACpE;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,CAAA;CAAE,GACjF;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACtE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC;AAU7D;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAErF;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;IAEzF;;OAEG;IACH,SAAS,IAAI,IAAI,CAAC;IAElB;;OAEG;IACH,IAAI,IAAI,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,gBAAgB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAchF;AA2QD;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,iBAAiB,GACzB,OAAO,CAAC,iBAAiB,CAAC,CAQ5B;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,UAAU,EAClB,MAAM,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAQ5B"}
package/package.json ADDED
@@ -0,0 +1,63 @@
1
+ {
2
+ "name": "@small-ltsc/sdk",
3
+ "version": "0.1.0",
4
+ "description": "TypeScript SDK for Small LTSC - Lossless Token Sequence Compression",
5
+ "type": "module",
6
+ "main": "./dist/esm/index.js",
7
+ "module": "./dist/esm/index.js",
8
+ "types": "./dist/types/index.d.ts",
9
+ "exports": {
10
+ ".": {
11
+ "import": "./dist/esm/index.js",
12
+ "types": "./dist/types/index.d.ts"
13
+ },
14
+ "./wasm": {
15
+ "import": "./dist/esm/wasm/index.js",
16
+ "types": "./dist/types/wasm/index.d.ts"
17
+ }
18
+ },
19
+ "files": [
20
+ "dist",
21
+ "src/wasm/*.wasm"
22
+ ],
23
+ "sideEffects": false,
24
+ "scripts": {
25
+ "clean": "rm -rf dist",
26
+ "build": "npm run build:esm && npm run build:types",
27
+ "build:esm": "tsc -p tsconfig.esm.json",
28
+ "build:types": "tsc -p tsconfig.types.json",
29
+ "test": "vitest",
30
+ "test:browser": "playwright test",
31
+ "serve:test": "node ./scripts/serve-test.mjs",
32
+ "lint": "eslint src --ext .ts",
33
+ "prepublishOnly": "npm run clean && npm run build"
34
+ },
35
+ "keywords": [
36
+ "compression",
37
+ "llm",
38
+ "tokens",
39
+ "transformer",
40
+ "wasm",
41
+ "lossless"
42
+ ],
43
+ "author": "",
44
+ "license": "MIT",
45
+ "repository": {
46
+ "type": "git",
47
+ "url": "git+https://github.com/triage-sec/small.git"
48
+ },
49
+ "bugs": {
50
+ "url": "https://github.com/triage-sec/small/issues"
51
+ },
52
+ "homepage": "https://github.com/triage-sec/small#readme",
53
+ "devDependencies": {
54
+ "@playwright/test": "^1.40.0",
55
+ "@types/node": "^20.10.0",
56
+ "eslint": "^8.55.0",
57
+ "typescript": "^5.3.0",
58
+ "vitest": "^1.0.0"
59
+ },
60
+ "engines": {
61
+ "node": ">=18.0.0"
62
+ }
63
+ }
Binary file