@hviana/sema 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/AGENTS.md +6 -5
  2. package/CITATION.cff +49 -0
  3. package/HOW_IT_WORKS.md +11 -12
  4. package/README.md +7 -5
  5. package/dist/example/demo.js +25 -25
  6. package/dist/example/train_base.d.ts +19 -21
  7. package/dist/example/train_base.js +1700 -1952
  8. package/dist/src/alphabet.d.ts +3 -3
  9. package/dist/src/alphabet.js +24 -27
  10. package/dist/src/alu/src/alu.d.ts +169 -194
  11. package/dist/src/alu/src/alu.js +374 -414
  12. package/dist/src/alu/src/expr.d.ts +36 -43
  13. package/dist/src/alu/src/expr.js +252 -278
  14. package/dist/src/alu/src/index.d.ts +9 -98
  15. package/dist/src/alu/src/index.js +10 -64
  16. package/dist/src/alu/src/kernel-arith.d.ts +1 -5
  17. package/dist/src/alu/src/kernel-arith.js +195 -285
  18. package/dist/src/alu/src/kernel-bits.d.ts +1 -5
  19. package/dist/src/alu/src/kernel-bits.js +81 -115
  20. package/dist/src/alu/src/kernel-logic.js +33 -63
  21. package/dist/src/alu/src/kernel-nd.js +155 -195
  22. package/dist/src/alu/src/kernel-numeric.d.ts +16 -80
  23. package/dist/src/alu/src/kernel-numeric.js +259 -331
  24. package/dist/src/alu/src/operation.d.ts +112 -132
  25. package/dist/src/alu/src/operation.js +149 -163
  26. package/dist/src/alu/src/parser.d.ts +168 -175
  27. package/dist/src/alu/src/parser.js +382 -421
  28. package/dist/src/alu/src/resonance.d.ts +16 -25
  29. package/dist/src/alu/src/resonance.js +49 -56
  30. package/dist/src/alu/src/text.d.ts +3 -7
  31. package/dist/src/alu/src/text.js +31 -37
  32. package/dist/src/alu/src/value.d.ts +14 -14
  33. package/dist/src/alu/src/value.js +150 -160
  34. package/dist/src/alu/test/alu.test.js +543 -656
  35. package/dist/src/bytes.d.ts +1 -5
  36. package/dist/src/bytes.js +33 -40
  37. package/dist/src/config.d.ts +99 -102
  38. package/dist/src/config.js +78 -83
  39. package/dist/src/derive/src/deduction.d.ts +60 -62
  40. package/dist/src/derive/src/deduction.js +103 -109
  41. package/dist/src/derive/src/index.d.ts +1 -7
  42. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  43. package/dist/src/derive/src/priority-queue.js +57 -61
  44. package/dist/src/derive/src/rewrite.d.ts +15 -18
  45. package/dist/src/derive/src/rewrite.js +59 -67
  46. package/dist/src/derive/src/trie.d.ts +56 -56
  47. package/dist/src/derive/src/trie.js +175 -179
  48. package/dist/src/derive/test/derive.test.js +96 -100
  49. package/dist/src/extension.d.ts +13 -16
  50. package/dist/src/geometry.d.ts +29 -43
  51. package/dist/src/geometry.js +324 -268
  52. package/dist/src/index.d.ts +2 -17
  53. package/dist/src/index.js +2 -12
  54. package/dist/src/ingest-cache.d.ts +25 -30
  55. package/dist/src/ingest-cache.js +108 -127
  56. package/dist/src/mind/articulation.d.ts +1 -5
  57. package/dist/src/mind/articulation.js +77 -112
  58. package/dist/src/mind/attention.d.ts +40 -108
  59. package/dist/src/mind/attention.js +754 -872
  60. package/dist/src/mind/canonical.d.ts +4 -19
  61. package/dist/src/mind/canonical.js +27 -31
  62. package/dist/src/mind/graph-search.d.ts +201 -225
  63. package/dist/src/mind/graph-search.js +742 -821
  64. package/dist/src/mind/index.d.ts +2 -10
  65. package/dist/src/mind/junction.d.ts +31 -58
  66. package/dist/src/mind/junction.js +172 -237
  67. package/dist/src/mind/learning.d.ts +16 -52
  68. package/dist/src/mind/learning.js +143 -165
  69. package/dist/src/mind/match.d.ts +20 -69
  70. package/dist/src/mind/match.js +259 -318
  71. package/dist/src/mind/mechanisms/alu.js +16 -16
  72. package/dist/src/mind/mechanisms/cast.d.ts +9 -13
  73. package/dist/src/mind/mechanisms/cast.js +363 -456
  74. package/dist/src/mind/mechanisms/confluence.d.ts +8 -12
  75. package/dist/src/mind/mechanisms/confluence.js +152 -183
  76. package/dist/src/mind/mechanisms/cover.d.ts +2 -8
  77. package/dist/src/mind/mechanisms/cover.js +148 -210
  78. package/dist/src/mind/mechanisms/extraction.d.ts +8 -34
  79. package/dist/src/mind/mechanisms/extraction.js +234 -288
  80. package/dist/src/mind/mechanisms/recall.d.ts +6 -10
  81. package/dist/src/mind/mechanisms/recall.js +141 -185
  82. package/dist/src/mind/mind.d.ts +194 -155
  83. package/dist/src/mind/mind.js +392 -289
  84. package/dist/src/mind/pipeline-mechanism.d.ts +112 -124
  85. package/dist/src/mind/pipeline-mechanism.js +161 -172
  86. package/dist/src/mind/pipeline.d.ts +4 -14
  87. package/dist/src/mind/pipeline.js +125 -189
  88. package/dist/src/mind/primitives.d.ts +20 -33
  89. package/dist/src/mind/primitives.js +124 -120
  90. package/dist/src/mind/rationale.d.ts +86 -98
  91. package/dist/src/mind/rationale.js +113 -121
  92. package/dist/src/mind/reasoning.d.ts +2 -13
  93. package/dist/src/mind/reasoning.js +129 -166
  94. package/dist/src/mind/recognition.d.ts +1 -4
  95. package/dist/src/mind/recognition.js +198 -210
  96. package/dist/src/mind/resonance.d.ts +5 -22
  97. package/dist/src/mind/resonance.js +0 -0
  98. package/dist/src/mind/trace.d.ts +6 -25
  99. package/dist/src/mind/trace.js +50 -58
  100. package/dist/src/mind/traverse.d.ts +29 -58
  101. package/dist/src/mind/traverse.js +344 -356
  102. package/dist/src/mind/types.d.ts +122 -127
  103. package/dist/src/mind/types.js +60 -69
  104. package/dist/src/rabitq-ivf/src/database.d.ts +113 -0
  105. package/dist/src/rabitq-ivf/src/database.js +201 -0
  106. package/dist/src/rabitq-ivf/src/index.d.ts +7 -0
  107. package/dist/src/{rabitq-hnsw → rabitq-ivf}/src/index.js +1 -3
  108. package/dist/src/rabitq-ivf/src/ivf.d.ts +200 -0
  109. package/dist/src/rabitq-ivf/src/ivf.js +1165 -0
  110. package/dist/src/rabitq-ivf/src/prng.d.ts +19 -0
  111. package/dist/src/rabitq-ivf/src/prng.js +36 -0
  112. package/dist/src/rabitq-ivf/src/rabitq.d.ts +95 -0
  113. package/dist/src/rabitq-ivf/src/rabitq.js +283 -0
  114. package/dist/src/sema.d.ts +9 -13
  115. package/dist/src/sema.js +26 -40
  116. package/dist/src/store-sqlite.d.ts +171 -166
  117. package/dist/src/store-sqlite.js +768 -662
  118. package/dist/src/store.d.ts +597 -630
  119. package/dist/src/store.js +1434 -1581
  120. package/dist/src/vec.d.ts +5 -9
  121. package/dist/src/vec.js +61 -73
  122. package/example/train_base.ts +33 -15
  123. package/index.html +65 -0
  124. package/package.json +1 -1
  125. package/src/alu/README.md +1 -1
  126. package/src/alu/src/index.ts +1 -1
  127. package/src/config.ts +19 -27
  128. package/src/geometry.ts +93 -1
  129. package/src/index.ts +6 -11
  130. package/src/mind/attention.ts +11 -6
  131. package/src/mind/mechanisms/recall.ts +29 -13
  132. package/src/mind/mind.ts +250 -19
  133. package/src/mind/primitives.ts +43 -6
  134. package/src/mind/recognition.ts +26 -8
  135. package/src/mind/traverse.ts +47 -0
  136. package/src/mind/types.ts +20 -21
  137. package/src/rabitq-ivf/README.md +56 -0
  138. package/src/rabitq-ivf/src/database.ts +276 -0
  139. package/src/{rabitq-hnsw → rabitq-ivf}/src/index.ts +2 -5
  140. package/src/rabitq-ivf/src/ivf.ts +1330 -0
  141. package/src/{rabitq-hnsw → rabitq-ivf}/src/prng.ts +1 -1
  142. package/src/store-sqlite.ts +204 -9
  143. package/src/store.ts +27 -34
  144. package/test/08-storage.test.mjs +3 -3
  145. package/test/13-conversation.test.mjs +190 -20
  146. package/test/14-scaling.test.mjs +2 -2
  147. package/test/35-ivf.test.mjs +263 -0
  148. package/test/36-bloom.test.mjs +123 -0
  149. package/dist/src/rabitq-hnsw/src/database.d.ts +0 -202
  150. package/dist/src/rabitq-hnsw/src/database.js +0 -405
  151. package/dist/src/rabitq-hnsw/src/heap.d.ts +0 -22
  152. package/dist/src/rabitq-hnsw/src/heap.js +0 -94
  153. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +0 -125
  154. package/dist/src/rabitq-hnsw/src/hnsw.js +0 -500
  155. package/dist/src/rabitq-hnsw/src/index.d.ts +0 -15
  156. package/dist/src/rabitq-hnsw/src/prng.d.ts +0 -19
  157. package/dist/src/rabitq-hnsw/src/prng.js +0 -38
  158. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +0 -95
  159. package/dist/src/rabitq-hnsw/src/rabitq.js +0 -299
  160. package/dist/src/rabitq-hnsw/src/store.d.ts +0 -162
  161. package/dist/src/rabitq-hnsw/src/store.js +0 -916
  162. package/dist/src/rabitq-hnsw/test/hnsw.test.d.ts +0 -1
  163. package/dist/src/rabitq-hnsw/test/hnsw.test.js +0 -1197
  164. package/src/rabitq-hnsw/README.md +0 -303
  165. package/src/rabitq-hnsw/src/database.ts +0 -492
  166. package/src/rabitq-hnsw/src/heap.ts +0 -90
  167. package/src/rabitq-hnsw/src/hnsw.ts +0 -514
  168. package/src/rabitq-hnsw/src/store.ts +0 -994
  169. package/src/rabitq-hnsw/test/hnsw.test.ts +0 -1213
  170. /package/src/{rabitq-hnsw → rabitq-ivf}/src/rabitq.ts +0 -0
@@ -1,94 +0,0 @@
1
- /**
2
- * Binary heap over two parallel numeric arrays storing (key, value) pairs.
3
- *
4
- * When `minHeap` is true the smallest key sits at the root, otherwise the
5
- * largest. Keys are distances, values are integer node ids. Parallel plain
6
- * arrays are used (rather than an array of objects) to avoid per-element
7
- * allocation in the search hot path.
8
- */
9
- export class Heap {
10
- keys = [];
11
- vals = [];
12
- minHeap;
13
- constructor(minHeap) {
14
- this.minHeap = minHeap;
15
- }
16
- get size() {
17
- return this.keys.length;
18
- }
19
- clear() {
20
- this.keys.length = 0;
21
- this.vals.length = 0;
22
- }
23
- topKey() {
24
- return this.keys[0];
25
- }
26
- topVal() {
27
- return this.vals[0];
28
- }
29
- /** true if `a` belongs closer to the root than `b`. */
30
- higher(a, b) {
31
- return this.minHeap ? a < b : a > b;
32
- }
33
- push(key, val) {
34
- const keys = this.keys;
35
- const vals = this.vals;
36
- let i = keys.length;
37
- keys.push(key);
38
- vals.push(val);
39
- while (i > 0) {
40
- const parent = (i - 1) >> 1;
41
- if (this.higher(keys[i], keys[parent])) {
42
- const tk = keys[i];
43
- keys[i] = keys[parent];
44
- keys[parent] = tk;
45
- const tv = vals[i];
46
- vals[i] = vals[parent];
47
- vals[parent] = tv;
48
- i = parent;
49
- } else {
50
- break;
51
- }
52
- }
53
- }
54
- pop() {
55
- const keys = this.keys;
56
- const vals = this.vals;
57
- const n = keys.length;
58
- if (n === 0) {
59
- return;
60
- }
61
- const lastKey = keys[n - 1];
62
- const lastVal = vals[n - 1];
63
- keys.pop();
64
- vals.pop();
65
- const m = keys.length;
66
- if (m === 0) {
67
- return;
68
- }
69
- keys[0] = lastKey;
70
- vals[0] = lastVal;
71
- let i = 0;
72
- while (true) {
73
- const left = 2 * i + 1;
74
- const right = left + 1;
75
- let best = i;
76
- if (left < m && this.higher(keys[left], keys[best])) {
77
- best = left;
78
- }
79
- if (right < m && this.higher(keys[right], keys[best])) {
80
- best = right;
81
- }
82
- if (best === i) {
83
- break;
84
- }
85
- const tk = keys[i];
86
- keys[i] = keys[best];
87
- keys[best] = tk;
88
- const tv = vals[i];
89
- vals[i] = vals[best];
90
- vals[best] = tv;
91
- i = best;
92
- }
93
- }
94
- }
@@ -1,125 +0,0 @@
1
- import { RaBitQuantizer } from "./rabitq.js";
2
- import { Store } from "./store.js";
3
- export interface HnswParams {
4
- M: number;
5
- efConstruction: number;
6
- efSearch: number;
7
- seed: number;
8
- }
9
- export interface KnnHit {
10
- /** external (user) id */
11
- id: number;
12
- /** estimated cosine distance (1 - cosine) */
13
- distance: number;
14
- }
15
- /**
16
- * Hierarchical Navigable Small World graph (Malkov & Yashunin, 2018) backed by
17
- * 1-bit RaBitQ codes that live in SQLite (see Store) rather than in RAM. The
18
- * algorithm is the textbook one; the only difference is that codes and adjacency
19
- * lists are read and written through the store on demand, so the resident set is
20
- * the working set of the current operation -- never the whole graph.
21
- *
22
- * Distances use the codes throughout:
23
- * - searching with a full-precision query uses RaBitQ's accurate estimator;
24
- * - building the graph and searching by code compare two codes by sign-bit
25
- * Hamming distance.
26
- * Building on codes alone is what lets `compact` rebuild an honest index with no
27
- * access to the original vectors.
28
- */
29
- export declare class HnswIndex {
30
- readonly M: number;
31
- readonly Mmax0: number;
32
- readonly efConstruction: number;
33
- private readonly mL;
34
- private readonly quantizer;
35
- private readonly store;
36
- private readonly seed;
37
- private readonly levelRng;
38
- private entry;
39
- private maxLevel;
40
- private live;
41
- private total;
42
- private _efSearch;
43
- private readonly working;
44
- private readonly visited;
45
- private visitedTag;
46
- private visitedEpoch;
47
- private seenTag;
48
- private markVisited;
49
- private readonly candHeap;
50
- private readonly resHeap;
51
- private readonly singleEp;
52
- private readonly idxScratch;
53
- private readonly distScratch;
54
- private readonly freshScratch;
55
- private qCtx;
56
- private refBytes;
57
- private building;
58
- lastQueryDistComps: number;
59
- /** Storage row reads issued by the most recent query (cache-independent). */
60
- lastQueryStorageReads: number;
61
- constructor(quantizer: RaBitQuantizer, store: Store, params: HnswParams);
62
- get size(): number;
63
- get physicalSize(): number;
64
- get bytesPerVector(): number;
65
- get efSearch(): number;
66
- set efSearch(v: number);
67
- private persistState;
68
- private randomLevel;
69
- /**
70
- * Fetch a node record into the operation's working set. The first touch in an
71
- * op is a storage read; later touches in the same op reuse it. The set is
72
- * cleared per op and bounded by efConstruction / efSearch, so storage reads
73
- * per op equal the number of *distinct* nodes the op visits -- minimal, and
74
- * independent of any cache.
75
- */
76
- private node;
77
- /** Prefetch several nodes into the working set with ONE batched storage
78
- * read for the misses. Point queries per neighbour made statement
79
- * dispatch — not distance arithmetic — the dominant cost of a large
80
- * build; the batch keeps `reads` accounting identical per row. */
81
- private fetchNodes;
82
- /** Distance from the current source (query vector or reference code) to a node. */
83
- private distOf;
84
- /** Code-to-code distance between two stored nodes (graph wiring only). */
85
- private codeDist;
86
- /**
87
- * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
88
- * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
89
- * never returned. `visited` is a per-call set sized by the nodes seen here.
90
- */
91
- private searchLayer;
92
- /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
93
- private greedyDescend;
94
- /**
95
- * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
96
- * neighbours from `candIds`, preferring those closer to `base` than to any
97
- * already-chosen neighbour, then fills remaining slots with the closest
98
- * leftovers (keep-pruned connections). Appends to `out`.
99
- */
100
- private selectNeighbors;
101
- /**
102
- * Insert a vector's code under external id `ext`; returns the internal node id.
103
- * All graph reads/writes go through the store; the caller owns the transaction.
104
- */
105
- insert(ext: number, code: Uint8Array, efC?: number): number;
106
- /** k-NN with a full-precision query vector (accurate estimator). */
107
- searchKnn(vec: ArrayLike<number>, k: number, ef?: number): KnnHit[];
108
- /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
109
- searchKnnByCode(codeBytes: Uint8Array, k: number, ef?: number): KnnHit[];
110
- private collectKnn;
111
- /** Tombstone a live node by internal id. Caller owns the transaction. */
112
- remove(id: number): boolean;
113
- /**
114
- * Reclaim tombstones with a graph-preserving splice (see
115
- * {@link Store.spliceCompact}): live nodes keep their internal ids and their
116
- * wiring; each dead neighbour is replaced by its own live neighbours. Codes
117
- * are untouched, so the index loses nothing beyond what deletion itself
118
- * removes. The previous implementation replayed every live code through a
119
- * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
120
- * multi-million-node store, inside one WAL-bloating transaction. The splice
121
- * is a streaming pass with batched commits, then a VACUUM to return the
122
- * freed pages.
123
- */
124
- compact(): void;
125
- }