clawpowers 2.2.6 → 2.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/CHANGELOG.md +186 -175
  2. package/COMPATIBILITY.md +48 -13
  3. package/KNOWN_LIMITATIONS.md +20 -19
  4. package/LICENSE +44 -44
  5. package/LICENSING.md +10 -10
  6. package/README.md +486 -462
  7. package/SECURITY.md +52 -52
  8. package/dist/index.d.ts +17 -5
  9. package/dist/index.js +186 -91
  10. package/dist/index.js.map +1 -1
  11. package/native/Cargo.lock +4927 -4927
  12. package/native/Cargo.toml +73 -73
  13. package/native/crates/canonical/Cargo.toml +24 -24
  14. package/native/crates/canonical/src/lib.rs +677 -677
  15. package/native/crates/compression/Cargo.toml +20 -20
  16. package/native/crates/compression/benches/compression_bench.rs +42 -42
  17. package/native/crates/compression/src/lib.rs +393 -393
  18. package/native/crates/evm-eth/Cargo.toml +13 -13
  19. package/native/crates/evm-eth/src/lib.rs +105 -105
  20. package/native/crates/fee/Cargo.toml +15 -15
  21. package/native/crates/fee/src/lib.rs +281 -281
  22. package/native/crates/index/Cargo.toml +16 -16
  23. package/native/crates/index/src/lib.rs +277 -277
  24. package/native/crates/policy/Cargo.toml +17 -17
  25. package/native/crates/policy/src/lib.rs +614 -614
  26. package/native/crates/security/Cargo.toml +22 -22
  27. package/native/crates/security/src/lib.rs +478 -478
  28. package/native/crates/tokens/Cargo.toml +13 -13
  29. package/native/crates/tokens/src/lib.rs +534 -534
  30. package/native/crates/verification/Cargo.toml +23 -23
  31. package/native/crates/verification/src/lib.rs +333 -333
  32. package/native/crates/wallet/Cargo.toml +20 -20
  33. package/native/crates/wallet/src/lib.rs +261 -261
  34. package/native/crates/x402/Cargo.toml +30 -30
  35. package/native/crates/x402/src/lib.rs +423 -423
  36. package/native/ffi/Cargo.toml +34 -34
  37. package/native/ffi/build.rs +4 -4
  38. package/native/ffi/src/lib.rs +352 -352
  39. package/native/ffi/tests/integration.rs +354 -354
  40. package/native/pyo3/Cargo.toml +26 -26
  41. package/native/pyo3/pyproject.toml +16 -16
  42. package/native/pyo3/src/lib.rs +407 -407
  43. package/native/pyo3/tests/test_smoke.py +180 -180
  44. package/native/wasm/Cargo.toml +47 -47
  45. package/native/wasm/pkg/.gitignore +6 -6
  46. package/native/wasm/pkg/clawpowers_wasm.d.ts +208 -208
  47. package/native/wasm/pkg/clawpowers_wasm.js +872 -872
  48. package/native/wasm/pkg/clawpowers_wasm_bg.wasm.d.ts +40 -40
  49. package/native/wasm/pkg/package.json +16 -16
  50. package/native/wasm/pkg-node/clawpowers_wasm.d.ts +143 -143
  51. package/native/wasm/pkg-node/clawpowers_wasm.js +798 -798
  52. package/native/wasm/pkg-node/clawpowers_wasm_bg.wasm.d.ts +40 -40
  53. package/native/wasm/pkg-node/package.json +12 -12
  54. package/native/wasm/src/lib.rs +433 -433
  55. package/package.json +12 -8
  56. package/scripts/build-wasm.mjs +59 -0
  57. package/scripts/generate_hermes_wrappers.py +211 -0
  58. package/scripts/hermes_wrapper_overrides.json +184 -0
  59. package/scripts/run-python-script.mjs +48 -0
  60. package/scripts/verify-consumer-install.mjs +109 -0
  61. package/scripts/verify-wasm-artifacts.mjs +25 -2
  62. package/scripts/verify_hermes_wrappers.py +154 -0
  63. package/skill.json +1 -1
  64. package/skills/1password/SKILL.md +34 -0
  65. package/skills/README.md +44 -0
  66. package/skills/agent-nexus-2/SKILL.md +34 -0
  67. package/skills/apple-notes/SKILL.md +34 -0
  68. package/skills/apple-reminders/SKILL.md +34 -0
  69. package/skills/autoresearch/SKILL.md +43 -0
  70. package/skills/bear-notes/SKILL.md +34 -0
  71. package/skills/blogwatcher/SKILL.md +34 -0
  72. package/skills/blucli/SKILL.md +34 -0
  73. package/skills/bluebubbles/SKILL.md +34 -0
  74. package/skills/business-strategy/SKILL.md +41 -0
  75. package/skills/camsnap/SKILL.md +34 -0
  76. package/skills/canvas/SKILL.md +34 -0
  77. package/skills/clawhub/SKILL.md +34 -0
  78. package/skills/coding-agent/SKILL.md +34 -0
  79. package/skills/coding-discipline.skill/SKILL.md +34 -0
  80. package/skills/content-writer/SKILL.md +41 -0
  81. package/skills/discord/SKILL.md +34 -0
  82. package/skills/eightctl/SKILL.md +34 -0
  83. package/skills/execution-validation.skill/SKILL.md +34 -0
  84. package/skills/gemini/SKILL.md +34 -0
  85. package/skills/gh-issues/SKILL.md +34 -0
  86. package/skills/gifgrep/SKILL.md +34 -0
  87. package/skills/github/SKILL.md +41 -0
  88. package/skills/gog/SKILL.md +34 -0
  89. package/skills/goplaces/SKILL.md +34 -0
  90. package/skills/healthcheck/SKILL.md +34 -0
  91. package/skills/himalaya/SKILL.md +34 -0
  92. package/skills/humanize/SKILL.md +41 -0
  93. package/skills/imsg/SKILL.md +34 -0
  94. package/skills/itp/SKILL.md +112 -0
  95. package/skills/mcporter/SKILL.md +34 -0
  96. package/skills/model-usage/SKILL.md +34 -0
  97. package/skills/nano-pdf/SKILL.md +34 -0
  98. package/skills/node-connect/SKILL.md +34 -0
  99. package/skills/notion/SKILL.md +34 -0
  100. package/skills/obsidian/SKILL.md +34 -0
  101. package/skills/openai-whisper/SKILL.md +34 -0
  102. package/skills/openai-whisper-api/SKILL.md +34 -0
  103. package/skills/openhue/SKILL.md +34 -0
  104. package/skills/oracle/SKILL.md +34 -0
  105. package/skills/ordercli/SKILL.md +34 -0
  106. package/skills/peekaboo/SKILL.md +34 -0
  107. package/skills/polyclaw/SKILL.md +34 -0
  108. package/skills/prospector/SKILL.md +41 -0
  109. package/skills/rsi.skill/SKILL.md +34 -0
  110. package/skills/sag/SKILL.md +34 -0
  111. package/skills/security/SKILL.md +41 -0
  112. package/skills/session-logs/SKILL.md +34 -0
  113. package/skills/sherpa-onnx-tts/SKILL.md +34 -0
  114. package/skills/skill-creator/SKILL.md +34 -0
  115. package/skills/slack/SKILL.md +34 -0
  116. package/skills/songsee/SKILL.md +34 -0
  117. package/skills/sonoscli/SKILL.md +34 -0
  118. package/skills/spotify-player/SKILL.md +34 -0
  119. package/skills/strykr-prism/SKILL.md +41 -0
  120. package/skills/summarize/SKILL.md +34 -0
  121. package/skills/taskbridge/SKILL.md +34 -0
  122. package/skills/things-mac/SKILL.md +34 -0
  123. package/skills/tmux/SKILL.md +34 -0
  124. package/skills/trello/SKILL.md +34 -0
  125. package/skills/validator-agent/SKILL.md +41 -0
  126. package/skills/video-frames/SKILL.md +34 -0
  127. package/skills/voice-call/SKILL.md +34 -0
  128. package/skills/wacli/SKILL.md +34 -0
  129. package/skills/weather/SKILL.md +34 -0
  130. package/skills/webmcp-payments/SKILL.md +41 -0
  131. package/skills/xurl/SKILL.md +34 -0
  132. package/src/skills/catalog.ts +435 -435
  133. package/src/skills/executor.ts +56 -56
  134. package/src/skills/index.ts +3 -3
  135. package/src/skills/itp/SKILL.md +112 -112
  136. package/src/skills/loader.ts +262 -193
@@ -1,393 +1,393 @@
1
- //! TurboQuant vector compression for TurboMemory.
2
- //!
3
- //! Provides a four-times compression of f32 dense vectors via:
4
- //! 1. Deterministic random orthogonal rotation (decorrelation).
5
- //! 2. Min/max scalar quantization to `u8`.
6
- //! 3. A QJL-inspired residual norm sketch.
7
-
8
- use rand::{Rng, SeedableRng, rngs::StdRng};
9
- use serde::{Deserialize, Serialize};
10
- use thiserror::Error;
11
-
12
- /// Errors produced by the compression pipeline.
13
- #[derive(Debug, Error)]
14
- pub enum CompressionError {
15
- /// Input vector has a different length than the configured dimensions.
16
- #[error("dimension mismatch: expected {expected}, got {got}")]
17
- DimensionMismatch {
18
- /// Expected number of dimensions.
19
- expected: usize,
20
- /// Actual number of dimensions.
21
- got: usize,
22
- },
23
- /// The compressed vector contains no data.
24
- #[error("empty compressed vector")]
25
- EmptyVector,
26
- }
27
-
28
- /// A shorthand result type for [`CompressionError`].
29
- pub type Result<T> = std::result::Result<T, CompressionError>;
30
-
31
- /// Configuration for the TurboQuant compressor.
32
- #[derive(Debug, Clone)]
33
- pub struct CompressionConfig {
34
- /// Number of dimensions in the input vectors.
35
- pub dimensions: usize,
36
- /// Bits used for scalar quantization (default 8, meaning `u8`).
37
- pub quantization_bits: u8,
38
- /// Seed for the deterministic rotation matrix.
39
- pub rotation_seed: u64,
40
- }
41
-
42
- impl Default for CompressionConfig {
43
- fn default() -> Self {
44
- Self {
45
- dimensions: 768,
46
- quantization_bits: 8,
47
- rotation_seed: 0xDEAD_BEEF_CAFE_1234,
48
- }
49
- }
50
- }
51
-
52
- /// The result of compressing a dense f32 vector.
53
- #[derive(Debug, Clone, Serialize, Deserialize)]
54
- pub struct CompressedVector {
55
- /// Scalar-quantized values in `u8`.
56
- pub quantized: Vec<u8>,
57
- /// Minimum f32 value seen in the rotated vector.
58
- pub min_val: f32,
59
- /// Maximum f32 value seen in the rotated vector.
60
- pub max_val: f32,
61
- /// L2 norm of the quantization error (QJL residual sketch).
62
- pub residual_norm: f32,
63
- /// Original dimensionality.
64
- pub original_dim: usize,
65
- }
66
-
67
- impl CompressedVector {
68
- /// Return the byte size of the quantized payload.
69
- pub fn byte_size(&self) -> usize {
70
- self.quantized.len()
71
- }
72
- }
73
-
74
- /// TurboQuant compressor: rotation → quantization → residual sketch.
75
- pub struct TurboCompressor {
76
- /// Configuration.
77
- pub config: CompressionConfig,
78
- rotation_matrix: Vec<f32>,
79
- }
80
-
81
- impl TurboCompressor {
82
- /// Create a new compressor, pre-computing the rotation matrix from the seed.
83
- pub fn new(config: CompressionConfig) -> Self {
84
- let rotation_matrix = build_rotation_matrix(config.dimensions, config.rotation_seed);
85
- Self {
86
- config,
87
- rotation_matrix,
88
- }
89
- }
90
-
91
- /// Compress `vector` into a [`CompressedVector`].
92
- pub fn compress(&self, vector: &[f32]) -> Result<CompressedVector> {
93
- let dim = self.config.dimensions;
94
- if vector.len() != dim {
95
- return Err(CompressionError::DimensionMismatch {
96
- expected: dim,
97
- got: vector.len(),
98
- });
99
- }
100
- let rotated = mat_vec_mul(&self.rotation_matrix, vector, dim);
101
- let min_val = rotated.iter().copied().fold(f32::INFINITY, f32::min);
102
- let max_val = rotated.iter().copied().fold(f32::NEG_INFINITY, f32::max);
103
- let range = if (max_val - min_val).abs() < f32::EPSILON {
104
- 1.0_f32
105
- } else {
106
- max_val - min_val
107
- };
108
- let quantized: Vec<u8> = rotated
109
- .iter()
110
- .map(|&v| (((v - min_val) / range) * 255.0).round().clamp(0.0, 255.0) as u8)
111
- .collect();
112
- let residual_norm = {
113
- let sum_sq: f32 = rotated
114
- .iter()
115
- .zip(quantized.iter())
116
- .map(|(&orig, &q)| {
117
- let reconstructed = (q as f32 / 255.0) * range + min_val;
118
- let err = orig - reconstructed;
119
- err * err
120
- })
121
- .sum();
122
- sum_sq.sqrt()
123
- };
124
- Ok(CompressedVector {
125
- quantized,
126
- min_val,
127
- max_val,
128
- residual_norm,
129
- original_dim: dim,
130
- })
131
- }
132
-
133
- /// Reconstruct an approximate f32 vector from a [`CompressedVector`].
134
- pub fn decompress(&self, compressed: &CompressedVector) -> Result<Vec<f32>> {
135
- if compressed.quantized.is_empty() {
136
- return Err(CompressionError::EmptyVector);
137
- }
138
- let range = compressed.max_val - compressed.min_val;
139
- let dequantized: Vec<f32> = compressed
140
- .quantized
141
- .iter()
142
- .map(|&q| (q as f32 / 255.0) * range + compressed.min_val)
143
- .collect();
144
- Ok(mat_t_vec_mul(
145
- &self.rotation_matrix,
146
- &dequantized,
147
- compressed.original_dim,
148
- ))
149
- }
150
-
151
- /// Fast Euclidean distance estimate in the compressed domain.
152
- pub fn approximate_distance(&self, a: &CompressedVector, b: &CompressedVector) -> Result<f32> {
153
- if a.quantized.is_empty() || b.quantized.is_empty() {
154
- return Err(CompressionError::EmptyVector);
155
- }
156
- if a.quantized.len() != b.quantized.len() {
157
- return Err(CompressionError::DimensionMismatch {
158
- expected: a.quantized.len(),
159
- got: b.quantized.len(),
160
- });
161
- }
162
- let range_a = a.max_val - a.min_val;
163
- let range_b = b.max_val - b.min_val;
164
- let sum_sq: f32 = a
165
- .quantized
166
- .iter()
167
- .zip(b.quantized.iter())
168
- .map(|(&qa, &qb)| {
169
- let va = (qa as f32 / 255.0) * range_a + a.min_val;
170
- let vb = (qb as f32 / 255.0) * range_b + b.min_val;
171
- let diff = va - vb;
172
- diff * diff
173
- })
174
- .sum();
175
- Ok(sum_sq.sqrt())
176
- }
177
- }
178
-
179
- fn build_rotation_matrix(dim: usize, seed: u64) -> Vec<f32> {
180
- let mut rng = StdRng::seed_from_u64(seed);
181
- let mut basis: Vec<Vec<f32>> = (0..dim)
182
- .map(|_| (0..dim).map(|_| rng.random::<f32>() * 2.0 - 1.0).collect())
183
- .collect();
184
- for i in 0..dim {
185
- let norm = l2_norm(&basis[i]);
186
- if norm > f32::EPSILON {
187
- for x in basis[i].iter_mut() {
188
- *x /= norm;
189
- }
190
- }
191
- for j in (i + 1)..dim {
192
- let dot: f32 = basis[i]
193
- .iter()
194
- .zip(basis[j].iter())
195
- .map(|(a, b)| a * b)
196
- .sum();
197
- let proj: Vec<f32> = basis[i].iter().map(|&v| v * dot).collect();
198
- for (x, p) in basis[j].iter_mut().zip(proj.iter()) {
199
- *x -= p;
200
- }
201
- }
202
- }
203
- basis.into_iter().flatten().collect()
204
- }
205
-
206
- fn mat_vec_mul(matrix: &[f32], vector: &[f32], dim: usize) -> Vec<f32> {
207
- (0..dim)
208
- .map(|row| {
209
- matrix[row * dim..(row + 1) * dim]
210
- .iter()
211
- .zip(vector.iter())
212
- .map(|(m, v)| m * v)
213
- .sum()
214
- })
215
- .collect()
216
- }
217
-
218
- fn mat_t_vec_mul(matrix: &[f32], vector: &[f32], dim: usize) -> Vec<f32> {
219
- let mut result = vec![0.0_f32; dim];
220
- for row in 0..dim {
221
- for col in 0..dim {
222
- result[col] += matrix[row * dim + col] * vector[row];
223
- }
224
- }
225
- result
226
- }
227
-
228
- fn l2_norm(v: &[f32]) -> f32 {
229
- v.iter().map(|x| x * x).sum::<f32>().sqrt()
230
- }
231
-
232
- /// Compute the L2 distance between two f32 slices.
233
- pub fn l2_distance(a: &[f32], b: &[f32]) -> f32 {
234
- a.iter()
235
- .zip(b.iter())
236
- .map(|(x, y)| (x - y) * (x - y))
237
- .sum::<f32>()
238
- .sqrt()
239
- }
240
-
241
- /// Compute cosine similarity between two f32 slices.
242
- pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
243
- let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
244
- let na = l2_norm(a);
245
- let nb = l2_norm(b);
246
- if na < f32::EPSILON || nb < f32::EPSILON {
247
- 0.0
248
- } else {
249
- dot / (na * nb)
250
- }
251
- }
252
-
253
- #[cfg(test)]
254
- mod tests {
255
- use super::*;
256
-
257
- const DIM: usize = 64;
258
-
259
- fn random_vector(dim: usize, seed: u64) -> Vec<f32> {
260
- let mut rng = StdRng::seed_from_u64(seed);
261
- (0..dim).map(|_| rng.random::<f32>() * 2.0 - 1.0).collect()
262
- }
263
-
264
- fn make_compressor(dim: usize) -> TurboCompressor {
265
- TurboCompressor::new(CompressionConfig {
266
- dimensions: dim,
267
- quantization_bits: 8,
268
- rotation_seed: 42,
269
- })
270
- }
271
-
272
- #[test]
273
- fn test_roundtrip_l2_error_below_threshold() {
274
- let c = make_compressor(DIM);
275
- let v = random_vector(DIM, 1);
276
- let cv = c.compress(&v).expect("compress");
277
- let r = c.decompress(&cv).expect("decompress");
278
- let err = l2_distance(&v, &r);
279
- let norm = l2_norm(&v);
280
- assert!((if norm > f32::EPSILON { err / norm } else { err }) < 0.05);
281
- }
282
-
283
- #[test]
284
- fn test_roundtrip_768_dim() {
285
- let c = make_compressor(768);
286
- let v = random_vector(768, 99);
287
- let cv = c.compress(&v).expect("compress");
288
- let r = c.decompress(&cv).expect("decompress");
289
- let err = l2_distance(&v, &r);
290
- let norm = l2_norm(&v);
291
- assert!((if norm > f32::EPSILON { err / norm } else { err }) < 0.05);
292
- }
293
-
294
- #[test]
295
- fn test_compression_ratio_4x() {
296
- let c = make_compressor(DIM);
297
- let cv = c.compress(&random_vector(DIM, 2)).expect("compress");
298
- assert_eq!(DIM * 4, cv.byte_size() * 4);
299
- }
300
-
301
- #[test]
302
- fn test_approximate_distance_preserves_order() {
303
- let c = make_compressor(DIM);
304
- let q = random_vector(DIM, 10);
305
- let mut near = q.clone();
306
- near[0] += 0.01;
307
- let far = random_vector(DIM, 20);
308
- let cq = c.compress(&q).expect("cq");
309
- let cn = c.compress(&near).expect("cn");
310
- let cf = c.compress(&far).expect("cf");
311
- let dn = c.approximate_distance(&cq, &cn).expect("dn");
312
- let df = c.approximate_distance(&cq, &cf).expect("df");
313
- assert!(dn < df, "near {dn:.4} should < far {df:.4}");
314
- }
315
-
316
- #[test]
317
- fn test_distance_identical_vectors_is_zero() {
318
- let c = make_compressor(DIM);
319
- let v = random_vector(DIM, 3);
320
- let cv = c.compress(&v).expect("cv");
321
- assert!(c.approximate_distance(&cv, &cv).expect("d") < 0.01);
322
- }
323
-
324
- #[test]
325
- fn test_dimension_mismatch_errors() {
326
- let c = make_compressor(DIM);
327
- assert!(matches!(
328
- c.compress(&random_vector(DIM + 1, 4)).expect_err("e"),
329
- CompressionError::DimensionMismatch { .. }
330
- ));
331
- }
332
-
333
- #[test]
334
- fn test_different_seeds_produce_different_outputs() {
335
- let c1 = TurboCompressor::new(CompressionConfig {
336
- dimensions: DIM,
337
- quantization_bits: 8,
338
- rotation_seed: 1,
339
- });
340
- let c2 = TurboCompressor::new(CompressionConfig {
341
- dimensions: DIM,
342
- quantization_bits: 8,
343
- rotation_seed: 2,
344
- });
345
- let v = random_vector(DIM, 5);
346
- assert_ne!(
347
- c1.compress(&v).expect("c1").quantized,
348
- c2.compress(&v).expect("c2").quantized
349
- );
350
- }
351
-
352
- #[test]
353
- fn test_same_seed_is_deterministic() {
354
- let v = random_vector(DIM, 7);
355
- assert_eq!(
356
- make_compressor(DIM).compress(&v).expect("c1").quantized,
357
- make_compressor(DIM).compress(&v).expect("c2").quantized
358
- );
359
- }
360
-
361
- #[test]
362
- fn test_residual_norm_is_nonnegative() {
363
- let cv = make_compressor(DIM)
364
- .compress(&random_vector(DIM, 8))
365
- .expect("cv");
366
- assert!(cv.residual_norm >= 0.0);
367
- }
368
-
369
- #[test]
370
- fn test_decompress_empty_errors() {
371
- let c = make_compressor(DIM);
372
- let empty = CompressedVector {
373
- quantized: vec![],
374
- min_val: 0.0,
375
- max_val: 1.0,
376
- residual_norm: 0.0,
377
- original_dim: DIM,
378
- };
379
- assert!(matches!(
380
- c.decompress(&empty).expect_err("e"),
381
- CompressionError::EmptyVector
382
- ));
383
- }
384
-
385
- #[test]
386
- fn test_all_zero_vector() {
387
- let c = make_compressor(DIM);
388
- let zero = vec![0.0_f32; DIM];
389
- let cv = c.compress(&zero).expect("cv");
390
- let r = c.decompress(&cv).expect("decompress");
391
- assert!(l2_distance(&zero, &r) < 1e-5);
392
- }
393
- }
1
+ //! TurboQuant vector compression for TurboMemory.
2
+ //!
3
+ //! Provides a four-times compression of f32 dense vectors via:
4
+ //! 1. Deterministic random orthogonal rotation (decorrelation).
5
+ //! 2. Min/max scalar quantization to `u8`.
6
+ //! 3. A QJL-inspired residual norm sketch.
7
+
8
+ use rand::{Rng, SeedableRng, rngs::StdRng};
9
+ use serde::{Deserialize, Serialize};
10
+ use thiserror::Error;
11
+
12
+ /// Errors produced by the compression pipeline.
13
+ #[derive(Debug, Error)]
14
+ pub enum CompressionError {
15
+ /// Input vector has a different length than the configured dimensions.
16
+ #[error("dimension mismatch: expected {expected}, got {got}")]
17
+ DimensionMismatch {
18
+ /// Expected number of dimensions.
19
+ expected: usize,
20
+ /// Actual number of dimensions.
21
+ got: usize,
22
+ },
23
+ /// The compressed vector contains no data.
24
+ #[error("empty compressed vector")]
25
+ EmptyVector,
26
+ }
27
+
28
+ /// A shorthand result type for [`CompressionError`].
29
+ pub type Result<T> = std::result::Result<T, CompressionError>;
30
+
31
+ /// Configuration for the TurboQuant compressor.
32
+ #[derive(Debug, Clone)]
33
+ pub struct CompressionConfig {
34
+ /// Number of dimensions in the input vectors.
35
+ pub dimensions: usize,
36
+ /// Bits used for scalar quantization (default 8, meaning `u8`).
37
+ pub quantization_bits: u8,
38
+ /// Seed for the deterministic rotation matrix.
39
+ pub rotation_seed: u64,
40
+ }
41
+
42
+ impl Default for CompressionConfig {
43
+ fn default() -> Self {
44
+ Self {
45
+ dimensions: 768,
46
+ quantization_bits: 8,
47
+ rotation_seed: 0xDEAD_BEEF_CAFE_1234,
48
+ }
49
+ }
50
+ }
51
+
52
+ /// The result of compressing a dense f32 vector.
53
+ #[derive(Debug, Clone, Serialize, Deserialize)]
54
+ pub struct CompressedVector {
55
+ /// Scalar-quantized values in `u8`.
56
+ pub quantized: Vec<u8>,
57
+ /// Minimum f32 value seen in the rotated vector.
58
+ pub min_val: f32,
59
+ /// Maximum f32 value seen in the rotated vector.
60
+ pub max_val: f32,
61
+ /// L2 norm of the quantization error (QJL residual sketch).
62
+ pub residual_norm: f32,
63
+ /// Original dimensionality.
64
+ pub original_dim: usize,
65
+ }
66
+
67
+ impl CompressedVector {
68
+ /// Return the byte size of the quantized payload.
69
+ pub fn byte_size(&self) -> usize {
70
+ self.quantized.len()
71
+ }
72
+ }
73
+
74
+ /// TurboQuant compressor: rotation → quantization → residual sketch.
75
+ pub struct TurboCompressor {
76
+ /// Configuration.
77
+ pub config: CompressionConfig,
78
+ rotation_matrix: Vec<f32>,
79
+ }
80
+
81
+ impl TurboCompressor {
82
+ /// Create a new compressor, pre-computing the rotation matrix from the seed.
83
+ pub fn new(config: CompressionConfig) -> Self {
84
+ let rotation_matrix = build_rotation_matrix(config.dimensions, config.rotation_seed);
85
+ Self {
86
+ config,
87
+ rotation_matrix,
88
+ }
89
+ }
90
+
91
+ /// Compress `vector` into a [`CompressedVector`].
92
+ pub fn compress(&self, vector: &[f32]) -> Result<CompressedVector> {
93
+ let dim = self.config.dimensions;
94
+ if vector.len() != dim {
95
+ return Err(CompressionError::DimensionMismatch {
96
+ expected: dim,
97
+ got: vector.len(),
98
+ });
99
+ }
100
+ let rotated = mat_vec_mul(&self.rotation_matrix, vector, dim);
101
+ let min_val = rotated.iter().copied().fold(f32::INFINITY, f32::min);
102
+ let max_val = rotated.iter().copied().fold(f32::NEG_INFINITY, f32::max);
103
+ let range = if (max_val - min_val).abs() < f32::EPSILON {
104
+ 1.0_f32
105
+ } else {
106
+ max_val - min_val
107
+ };
108
+ let quantized: Vec<u8> = rotated
109
+ .iter()
110
+ .map(|&v| (((v - min_val) / range) * 255.0).round().clamp(0.0, 255.0) as u8)
111
+ .collect();
112
+ let residual_norm = {
113
+ let sum_sq: f32 = rotated
114
+ .iter()
115
+ .zip(quantized.iter())
116
+ .map(|(&orig, &q)| {
117
+ let reconstructed = (q as f32 / 255.0) * range + min_val;
118
+ let err = orig - reconstructed;
119
+ err * err
120
+ })
121
+ .sum();
122
+ sum_sq.sqrt()
123
+ };
124
+ Ok(CompressedVector {
125
+ quantized,
126
+ min_val,
127
+ max_val,
128
+ residual_norm,
129
+ original_dim: dim,
130
+ })
131
+ }
132
+
133
+ /// Reconstruct an approximate f32 vector from a [`CompressedVector`].
134
+ pub fn decompress(&self, compressed: &CompressedVector) -> Result<Vec<f32>> {
135
+ if compressed.quantized.is_empty() {
136
+ return Err(CompressionError::EmptyVector);
137
+ }
138
+ let range = compressed.max_val - compressed.min_val;
139
+ let dequantized: Vec<f32> = compressed
140
+ .quantized
141
+ .iter()
142
+ .map(|&q| (q as f32 / 255.0) * range + compressed.min_val)
143
+ .collect();
144
+ Ok(mat_t_vec_mul(
145
+ &self.rotation_matrix,
146
+ &dequantized,
147
+ compressed.original_dim,
148
+ ))
149
+ }
150
+
151
+ /// Fast Euclidean distance estimate in the compressed domain.
152
+ pub fn approximate_distance(&self, a: &CompressedVector, b: &CompressedVector) -> Result<f32> {
153
+ if a.quantized.is_empty() || b.quantized.is_empty() {
154
+ return Err(CompressionError::EmptyVector);
155
+ }
156
+ if a.quantized.len() != b.quantized.len() {
157
+ return Err(CompressionError::DimensionMismatch {
158
+ expected: a.quantized.len(),
159
+ got: b.quantized.len(),
160
+ });
161
+ }
162
+ let range_a = a.max_val - a.min_val;
163
+ let range_b = b.max_val - b.min_val;
164
+ let sum_sq: f32 = a
165
+ .quantized
166
+ .iter()
167
+ .zip(b.quantized.iter())
168
+ .map(|(&qa, &qb)| {
169
+ let va = (qa as f32 / 255.0) * range_a + a.min_val;
170
+ let vb = (qb as f32 / 255.0) * range_b + b.min_val;
171
+ let diff = va - vb;
172
+ diff * diff
173
+ })
174
+ .sum();
175
+ Ok(sum_sq.sqrt())
176
+ }
177
+ }
178
+
179
+ fn build_rotation_matrix(dim: usize, seed: u64) -> Vec<f32> {
180
+ let mut rng = StdRng::seed_from_u64(seed);
181
+ let mut basis: Vec<Vec<f32>> = (0..dim)
182
+ .map(|_| (0..dim).map(|_| rng.random::<f32>() * 2.0 - 1.0).collect())
183
+ .collect();
184
+ for i in 0..dim {
185
+ let norm = l2_norm(&basis[i]);
186
+ if norm > f32::EPSILON {
187
+ for x in basis[i].iter_mut() {
188
+ *x /= norm;
189
+ }
190
+ }
191
+ for j in (i + 1)..dim {
192
+ let dot: f32 = basis[i]
193
+ .iter()
194
+ .zip(basis[j].iter())
195
+ .map(|(a, b)| a * b)
196
+ .sum();
197
+ let proj: Vec<f32> = basis[i].iter().map(|&v| v * dot).collect();
198
+ for (x, p) in basis[j].iter_mut().zip(proj.iter()) {
199
+ *x -= p;
200
+ }
201
+ }
202
+ }
203
+ basis.into_iter().flatten().collect()
204
+ }
205
+
206
+ fn mat_vec_mul(matrix: &[f32], vector: &[f32], dim: usize) -> Vec<f32> {
207
+ (0..dim)
208
+ .map(|row| {
209
+ matrix[row * dim..(row + 1) * dim]
210
+ .iter()
211
+ .zip(vector.iter())
212
+ .map(|(m, v)| m * v)
213
+ .sum()
214
+ })
215
+ .collect()
216
+ }
217
+
218
+ fn mat_t_vec_mul(matrix: &[f32], vector: &[f32], dim: usize) -> Vec<f32> {
219
+ let mut result = vec![0.0_f32; dim];
220
+ for row in 0..dim {
221
+ for col in 0..dim {
222
+ result[col] += matrix[row * dim + col] * vector[row];
223
+ }
224
+ }
225
+ result
226
+ }
227
+
228
+ fn l2_norm(v: &[f32]) -> f32 {
229
+ v.iter().map(|x| x * x).sum::<f32>().sqrt()
230
+ }
231
+
232
+ /// Compute the L2 distance between two f32 slices.
233
+ pub fn l2_distance(a: &[f32], b: &[f32]) -> f32 {
234
+ a.iter()
235
+ .zip(b.iter())
236
+ .map(|(x, y)| (x - y) * (x - y))
237
+ .sum::<f32>()
238
+ .sqrt()
239
+ }
240
+
241
+ /// Compute cosine similarity between two f32 slices.
242
+ pub fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
243
+ let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
244
+ let na = l2_norm(a);
245
+ let nb = l2_norm(b);
246
+ if na < f32::EPSILON || nb < f32::EPSILON {
247
+ 0.0
248
+ } else {
249
+ dot / (na * nb)
250
+ }
251
+ }
252
+
253
+ #[cfg(test)]
254
+ mod tests {
255
+ use super::*;
256
+
257
+ const DIM: usize = 64;
258
+
259
+ fn random_vector(dim: usize, seed: u64) -> Vec<f32> {
260
+ let mut rng = StdRng::seed_from_u64(seed);
261
+ (0..dim).map(|_| rng.random::<f32>() * 2.0 - 1.0).collect()
262
+ }
263
+
264
+ fn make_compressor(dim: usize) -> TurboCompressor {
265
+ TurboCompressor::new(CompressionConfig {
266
+ dimensions: dim,
267
+ quantization_bits: 8,
268
+ rotation_seed: 42,
269
+ })
270
+ }
271
+
272
+ #[test]
273
+ fn test_roundtrip_l2_error_below_threshold() {
274
+ let c = make_compressor(DIM);
275
+ let v = random_vector(DIM, 1);
276
+ let cv = c.compress(&v).expect("compress");
277
+ let r = c.decompress(&cv).expect("decompress");
278
+ let err = l2_distance(&v, &r);
279
+ let norm = l2_norm(&v);
280
+ assert!((if norm > f32::EPSILON { err / norm } else { err }) < 0.05);
281
+ }
282
+
283
+ #[test]
284
+ fn test_roundtrip_768_dim() {
285
+ let c = make_compressor(768);
286
+ let v = random_vector(768, 99);
287
+ let cv = c.compress(&v).expect("compress");
288
+ let r = c.decompress(&cv).expect("decompress");
289
+ let err = l2_distance(&v, &r);
290
+ let norm = l2_norm(&v);
291
+ assert!((if norm > f32::EPSILON { err / norm } else { err }) < 0.05);
292
+ }
293
+
294
+ #[test]
295
+ fn test_compression_ratio_4x() {
296
+ let c = make_compressor(DIM);
297
+ let cv = c.compress(&random_vector(DIM, 2)).expect("compress");
298
+ assert_eq!(DIM * 4, cv.byte_size() * 4);
299
+ }
300
+
301
+ #[test]
302
+ fn test_approximate_distance_preserves_order() {
303
+ let c = make_compressor(DIM);
304
+ let q = random_vector(DIM, 10);
305
+ let mut near = q.clone();
306
+ near[0] += 0.01;
307
+ let far = random_vector(DIM, 20);
308
+ let cq = c.compress(&q).expect("cq");
309
+ let cn = c.compress(&near).expect("cn");
310
+ let cf = c.compress(&far).expect("cf");
311
+ let dn = c.approximate_distance(&cq, &cn).expect("dn");
312
+ let df = c.approximate_distance(&cq, &cf).expect("df");
313
+ assert!(dn < df, "near {dn:.4} should < far {df:.4}");
314
+ }
315
+
316
+ #[test]
317
+ fn test_distance_identical_vectors_is_zero() {
318
+ let c = make_compressor(DIM);
319
+ let v = random_vector(DIM, 3);
320
+ let cv = c.compress(&v).expect("cv");
321
+ assert!(c.approximate_distance(&cv, &cv).expect("d") < 0.01);
322
+ }
323
+
324
+ #[test]
325
+ fn test_dimension_mismatch_errors() {
326
+ let c = make_compressor(DIM);
327
+ assert!(matches!(
328
+ c.compress(&random_vector(DIM + 1, 4)).expect_err("e"),
329
+ CompressionError::DimensionMismatch { .. }
330
+ ));
331
+ }
332
+
333
+ #[test]
334
+ fn test_different_seeds_produce_different_outputs() {
335
+ let c1 = TurboCompressor::new(CompressionConfig {
336
+ dimensions: DIM,
337
+ quantization_bits: 8,
338
+ rotation_seed: 1,
339
+ });
340
+ let c2 = TurboCompressor::new(CompressionConfig {
341
+ dimensions: DIM,
342
+ quantization_bits: 8,
343
+ rotation_seed: 2,
344
+ });
345
+ let v = random_vector(DIM, 5);
346
+ assert_ne!(
347
+ c1.compress(&v).expect("c1").quantized,
348
+ c2.compress(&v).expect("c2").quantized
349
+ );
350
+ }
351
+
352
+ #[test]
353
+ fn test_same_seed_is_deterministic() {
354
+ let v = random_vector(DIM, 7);
355
+ assert_eq!(
356
+ make_compressor(DIM).compress(&v).expect("c1").quantized,
357
+ make_compressor(DIM).compress(&v).expect("c2").quantized
358
+ );
359
+ }
360
+
361
+ #[test]
362
+ fn test_residual_norm_is_nonnegative() {
363
+ let cv = make_compressor(DIM)
364
+ .compress(&random_vector(DIM, 8))
365
+ .expect("cv");
366
+ assert!(cv.residual_norm >= 0.0);
367
+ }
368
+
369
+ #[test]
370
+ fn test_decompress_empty_errors() {
371
+ let c = make_compressor(DIM);
372
+ let empty = CompressedVector {
373
+ quantized: vec![],
374
+ min_val: 0.0,
375
+ max_val: 1.0,
376
+ residual_norm: 0.0,
377
+ original_dim: DIM,
378
+ };
379
+ assert!(matches!(
380
+ c.decompress(&empty).expect_err("e"),
381
+ CompressionError::EmptyVector
382
+ ));
383
+ }
384
+
385
+ #[test]
386
+ fn test_all_zero_vector() {
387
+ let c = make_compressor(DIM);
388
+ let zero = vec![0.0_f32; DIM];
389
+ let cv = c.compress(&zero).expect("cv");
390
+ let r = c.decompress(&cv).expect("decompress");
391
+ assert!(l2_distance(&zero, &r) < 1e-5);
392
+ }
393
+ }