@hviana/sema 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/dist/example/demo.js +25 -25
  2. package/dist/example/train_base.d.ts +21 -19
  3. package/dist/example/train_base.js +1952 -1684
  4. package/dist/src/alphabet.d.ts +3 -3
  5. package/dist/src/alphabet.js +27 -24
  6. package/dist/src/alu/src/alu.d.ts +194 -169
  7. package/dist/src/alu/src/alu.js +414 -374
  8. package/dist/src/alu/src/expr.d.ts +43 -36
  9. package/dist/src/alu/src/expr.js +278 -252
  10. package/dist/src/alu/src/index.d.ts +98 -9
  11. package/dist/src/alu/src/index.js +63 -9
  12. package/dist/src/alu/src/kernel-arith.d.ts +5 -1
  13. package/dist/src/alu/src/kernel-arith.js +285 -195
  14. package/dist/src/alu/src/kernel-bits.d.ts +5 -1
  15. package/dist/src/alu/src/kernel-bits.js +115 -81
  16. package/dist/src/alu/src/kernel-logic.js +63 -33
  17. package/dist/src/alu/src/kernel-nd.js +195 -155
  18. package/dist/src/alu/src/kernel-numeric.d.ts +80 -16
  19. package/dist/src/alu/src/kernel-numeric.js +331 -259
  20. package/dist/src/alu/src/operation.d.ts +132 -112
  21. package/dist/src/alu/src/operation.js +163 -149
  22. package/dist/src/alu/src/parser.d.ts +175 -168
  23. package/dist/src/alu/src/parser.js +421 -382
  24. package/dist/src/alu/src/resonance.d.ts +25 -16
  25. package/dist/src/alu/src/resonance.js +56 -49
  26. package/dist/src/alu/src/text.d.ts +7 -3
  27. package/dist/src/alu/src/text.js +37 -31
  28. package/dist/src/alu/src/value.d.ts +14 -14
  29. package/dist/src/alu/src/value.js +160 -150
  30. package/dist/src/alu/test/alu.test.js +656 -543
  31. package/dist/src/bytes.d.ts +5 -1
  32. package/dist/src/bytes.js +40 -33
  33. package/dist/src/config.d.ts +102 -102
  34. package/dist/src/config.js +83 -83
  35. package/dist/src/derive/src/deduction.d.ts +62 -60
  36. package/dist/src/derive/src/deduction.js +109 -103
  37. package/dist/src/derive/src/index.d.ts +7 -1
  38. package/dist/src/derive/src/priority-queue.d.ts +8 -8
  39. package/dist/src/derive/src/priority-queue.js +61 -57
  40. package/dist/src/derive/src/rewrite.d.ts +18 -15
  41. package/dist/src/derive/src/rewrite.js +67 -59
  42. package/dist/src/derive/src/trie.d.ts +56 -56
  43. package/dist/src/derive/src/trie.js +179 -175
  44. package/dist/src/derive/test/derive.test.js +100 -96
  45. package/dist/src/extension.d.ts +16 -13
  46. package/dist/src/geometry.d.ts +42 -17
  47. package/dist/src/geometry.js +266 -235
  48. package/dist/src/index.d.ts +17 -2
  49. package/dist/src/index.js +7 -1
  50. package/dist/src/ingest-cache.d.ts +30 -25
  51. package/dist/src/ingest-cache.js +127 -108
  52. package/dist/src/mind/articulation.d.ts +5 -1
  53. package/dist/src/mind/articulation.js +112 -77
  54. package/dist/src/mind/attention.d.ts +108 -40
  55. package/dist/src/mind/attention.js +871 -749
  56. package/dist/src/mind/canonical.d.ts +19 -4
  57. package/dist/src/mind/canonical.js +31 -27
  58. package/dist/src/mind/graph-search.d.ts +225 -201
  59. package/dist/src/mind/graph-search.js +821 -742
  60. package/dist/src/mind/index.d.ts +10 -2
  61. package/dist/src/mind/junction.d.ts +58 -31
  62. package/dist/src/mind/junction.js +237 -172
  63. package/dist/src/mind/learning.d.ts +52 -16
  64. package/dist/src/mind/learning.js +165 -143
  65. package/dist/src/mind/match.d.ts +69 -20
  66. package/dist/src/mind/match.js +318 -259
  67. package/dist/src/mind/mechanisms/alu.js +16 -16
  68. package/dist/src/mind/mechanisms/cast.d.ts +13 -9
  69. package/dist/src/mind/mechanisms/cast.js +456 -363
  70. package/dist/src/mind/mechanisms/confluence.d.ts +12 -8
  71. package/dist/src/mind/mechanisms/confluence.js +183 -152
  72. package/dist/src/mind/mechanisms/cover.d.ts +8 -2
  73. package/dist/src/mind/mechanisms/cover.js +210 -148
  74. package/dist/src/mind/mechanisms/extraction.d.ts +34 -8
  75. package/dist/src/mind/mechanisms/extraction.js +288 -234
  76. package/dist/src/mind/mechanisms/recall.d.ts +10 -6
  77. package/dist/src/mind/mechanisms/recall.js +185 -126
  78. package/dist/src/mind/mind.d.ts +154 -129
  79. package/dist/src/mind/mind.js +288 -258
  80. package/dist/src/mind/pipeline-mechanism.d.ts +124 -112
  81. package/dist/src/mind/pipeline-mechanism.js +172 -161
  82. package/dist/src/mind/pipeline.d.ts +14 -4
  83. package/dist/src/mind/pipeline.js +189 -125
  84. package/dist/src/mind/primitives.d.ts +32 -8
  85. package/dist/src/mind/primitives.js +117 -99
  86. package/dist/src/mind/rationale.d.ts +98 -86
  87. package/dist/src/mind/rationale.js +121 -113
  88. package/dist/src/mind/reasoning.d.ts +13 -2
  89. package/dist/src/mind/reasoning.js +166 -129
  90. package/dist/src/mind/recognition.d.ts +4 -1
  91. package/dist/src/mind/recognition.js +208 -183
  92. package/dist/src/mind/resonance.d.ts +22 -5
  93. package/dist/src/mind/resonance.js +0 -0
  94. package/dist/src/mind/trace.d.ts +25 -6
  95. package/dist/src/mind/trace.js +58 -50
  96. package/dist/src/mind/traverse.d.ts +58 -16
  97. package/dist/src/mind/traverse.js +357 -304
  98. package/dist/src/mind/types.d.ts +127 -120
  99. package/dist/src/mind/types.js +69 -60
  100. package/dist/src/rabitq-hnsw/src/database.d.ts +179 -177
  101. package/dist/src/rabitq-hnsw/src/database.js +359 -342
  102. package/dist/src/rabitq-hnsw/src/heap.d.ts +12 -12
  103. package/dist/src/rabitq-hnsw/src/heap.js +80 -75
  104. package/dist/src/rabitq-hnsw/src/hnsw.d.ts +103 -103
  105. package/dist/src/rabitq-hnsw/src/hnsw.js +454 -428
  106. package/dist/src/rabitq-hnsw/src/index.d.ts +6 -1
  107. package/dist/src/rabitq-hnsw/src/prng.d.ts +10 -10
  108. package/dist/src/rabitq-hnsw/src/prng.js +27 -25
  109. package/dist/src/rabitq-hnsw/src/rabitq.d.ts +69 -69
  110. package/dist/src/rabitq-hnsw/src/rabitq.js +262 -246
  111. package/dist/src/rabitq-hnsw/src/store.d.ts +132 -132
  112. package/dist/src/rabitq-hnsw/src/store.js +843 -752
  113. package/dist/src/rabitq-hnsw/test/hnsw.test.js +1054 -805
  114. package/dist/src/sema.d.ts +35 -0
  115. package/dist/src/sema.js +77 -0
  116. package/dist/src/store-sqlite.d.ts +166 -144
  117. package/dist/src/store-sqlite.js +660 -578
  118. package/dist/src/store.d.ts +630 -593
  119. package/dist/src/store.js +1581 -1440
  120. package/dist/src/vec.d.ts +9 -5
  121. package/dist/src/vec.js +73 -61
  122. package/package.json +2 -2
  123. package/src/sema.ts +102 -0
@@ -7,16 +7,16 @@
7
7
  * allocation in the search hot path.
8
8
  */
9
9
  export declare class Heap {
10
- readonly keys: number[];
11
- readonly vals: number[];
12
- private readonly minHeap;
13
- constructor(minHeap: boolean);
14
- get size(): number;
15
- clear(): void;
16
- topKey(): number;
17
- topVal(): number;
18
- /** true if `a` belongs closer to the root than `b`. */
19
- private higher;
20
- push(key: number, val: number): void;
21
- pop(): void;
10
+ readonly keys: number[];
11
+ readonly vals: number[];
12
+ private readonly minHeap;
13
+ constructor(minHeap: boolean);
14
+ get size(): number;
15
+ clear(): void;
16
+ topKey(): number;
17
+ topVal(): number;
18
+ /** true if `a` belongs closer to the root than `b`. */
19
+ private higher;
20
+ push(key: number, val: number): void;
21
+ pop(): void;
22
22
  }
@@ -7,83 +7,88 @@
7
7
  * allocation in the search hot path.
8
8
  */
9
9
  export class Heap {
10
- keys = [];
11
- vals = [];
12
- minHeap;
13
- constructor(minHeap) {
14
- this.minHeap = minHeap;
10
+ keys = [];
11
+ vals = [];
12
+ minHeap;
13
+ constructor(minHeap) {
14
+ this.minHeap = minHeap;
15
+ }
16
+ get size() {
17
+ return this.keys.length;
18
+ }
19
+ clear() {
20
+ this.keys.length = 0;
21
+ this.vals.length = 0;
22
+ }
23
+ topKey() {
24
+ return this.keys[0];
25
+ }
26
+ topVal() {
27
+ return this.vals[0];
28
+ }
29
+ /** true if `a` belongs closer to the root than `b`. */
30
+ higher(a, b) {
31
+ return this.minHeap ? a < b : a > b;
32
+ }
33
+ push(key, val) {
34
+ const keys = this.keys;
35
+ const vals = this.vals;
36
+ let i = keys.length;
37
+ keys.push(key);
38
+ vals.push(val);
39
+ while (i > 0) {
40
+ const parent = (i - 1) >> 1;
41
+ if (this.higher(keys[i], keys[parent])) {
42
+ const tk = keys[i];
43
+ keys[i] = keys[parent];
44
+ keys[parent] = tk;
45
+ const tv = vals[i];
46
+ vals[i] = vals[parent];
47
+ vals[parent] = tv;
48
+ i = parent;
49
+ } else {
50
+ break;
51
+ }
15
52
  }
16
- get size() {
17
- return this.keys.length;
53
+ }
54
+ pop() {
55
+ const keys = this.keys;
56
+ const vals = this.vals;
57
+ const n = keys.length;
58
+ if (n === 0) {
59
+ return;
18
60
  }
19
- clear() {
20
- this.keys.length = 0;
21
- this.vals.length = 0;
61
+ const lastKey = keys[n - 1];
62
+ const lastVal = vals[n - 1];
63
+ keys.pop();
64
+ vals.pop();
65
+ const m = keys.length;
66
+ if (m === 0) {
67
+ return;
22
68
  }
23
- topKey() {
24
- return this.keys[0];
25
- }
26
- topVal() {
27
- return this.vals[0];
28
- }
29
- /** true if `a` belongs closer to the root than `b`. */
30
- higher(a, b) {
31
- return this.minHeap ? a < b : a > b;
32
- }
33
- push(key, val) {
34
- const keys = this.keys;
35
- const vals = this.vals;
36
- let i = keys.length;
37
- keys.push(key);
38
- vals.push(val);
39
- while (i > 0) {
40
- const parent = (i - 1) >> 1;
41
- if (this.higher(keys[i], keys[parent])) {
42
- const tk = keys[i];
43
- keys[i] = keys[parent];
44
- keys[parent] = tk;
45
- const tv = vals[i];
46
- vals[i] = vals[parent];
47
- vals[parent] = tv;
48
- i = parent;
49
- }
50
- else
51
- break;
52
- }
53
- }
54
- pop() {
55
- const keys = this.keys;
56
- const vals = this.vals;
57
- const n = keys.length;
58
- if (n === 0)
59
- return;
60
- const lastKey = keys[n - 1];
61
- const lastVal = vals[n - 1];
62
- keys.pop();
63
- vals.pop();
64
- const m = keys.length;
65
- if (m === 0)
66
- return;
67
- keys[0] = lastKey;
68
- vals[0] = lastVal;
69
- let i = 0;
70
- while (true) {
71
- const left = 2 * i + 1;
72
- const right = left + 1;
73
- let best = i;
74
- if (left < m && this.higher(keys[left], keys[best]))
75
- best = left;
76
- if (right < m && this.higher(keys[right], keys[best]))
77
- best = right;
78
- if (best === i)
79
- break;
80
- const tk = keys[i];
81
- keys[i] = keys[best];
82
- keys[best] = tk;
83
- const tv = vals[i];
84
- vals[i] = vals[best];
85
- vals[best] = tv;
86
- i = best;
87
- }
69
+ keys[0] = lastKey;
70
+ vals[0] = lastVal;
71
+ let i = 0;
72
+ while (true) {
73
+ const left = 2 * i + 1;
74
+ const right = left + 1;
75
+ let best = i;
76
+ if (left < m && this.higher(keys[left], keys[best])) {
77
+ best = left;
78
+ }
79
+ if (right < m && this.higher(keys[right], keys[best])) {
80
+ best = right;
81
+ }
82
+ if (best === i) {
83
+ break;
84
+ }
85
+ const tk = keys[i];
86
+ keys[i] = keys[best];
87
+ keys[best] = tk;
88
+ const tv = vals[i];
89
+ vals[i] = vals[best];
90
+ vals[best] = tv;
91
+ i = best;
88
92
  }
93
+ }
89
94
  }
@@ -1,16 +1,16 @@
1
1
  import { RaBitQuantizer } from "./rabitq.js";
2
2
  import { Store } from "./store.js";
3
3
  export interface HnswParams {
4
- M: number;
5
- efConstruction: number;
6
- efSearch: number;
7
- seed: number;
4
+ M: number;
5
+ efConstruction: number;
6
+ efSearch: number;
7
+ seed: number;
8
8
  }
9
9
  export interface KnnHit {
10
- /** external (user) id */
11
- id: number;
12
- /** estimated cosine distance (1 - cosine) */
13
- distance: number;
10
+ /** external (user) id */
11
+ id: number;
12
+ /** estimated cosine distance (1 - cosine) */
13
+ distance: number;
14
14
  }
15
15
  /**
16
16
  * Hierarchical Navigable Small World graph (Malkov & Yashunin, 2018) backed by
@@ -27,99 +27,99 @@ export interface KnnHit {
27
27
  * access to the original vectors.
28
28
  */
29
29
  export declare class HnswIndex {
30
- readonly M: number;
31
- readonly Mmax0: number;
32
- readonly efConstruction: number;
33
- private readonly mL;
34
- private readonly quantizer;
35
- private readonly store;
36
- private readonly seed;
37
- private readonly levelRng;
38
- private entry;
39
- private maxLevel;
40
- private live;
41
- private total;
42
- private _efSearch;
43
- private readonly working;
44
- private readonly visited;
45
- private visitedTag;
46
- private visitedEpoch;
47
- private seenTag;
48
- private markVisited;
49
- private readonly candHeap;
50
- private readonly resHeap;
51
- private readonly singleEp;
52
- private readonly idxScratch;
53
- private readonly distScratch;
54
- private readonly freshScratch;
55
- private qCtx;
56
- private refBytes;
57
- private building;
58
- lastQueryDistComps: number;
59
- /** Storage row reads issued by the most recent query (cache-independent). */
60
- lastQueryStorageReads: number;
61
- constructor(quantizer: RaBitQuantizer, store: Store, params: HnswParams);
62
- get size(): number;
63
- get physicalSize(): number;
64
- get bytesPerVector(): number;
65
- get efSearch(): number;
66
- set efSearch(v: number);
67
- private persistState;
68
- private randomLevel;
69
- /**
70
- * Fetch a node record into the operation's working set. The first touch in an
71
- * op is a storage read; later touches in the same op reuse it. The set is
72
- * cleared per op and bounded by efConstruction / efSearch, so storage reads
73
- * per op equal the number of *distinct* nodes the op visits -- minimal, and
74
- * independent of any cache.
75
- */
76
- private node;
77
- /** Prefetch several nodes into the working set with ONE batched storage
78
- * read for the misses. Point queries per neighbour made statement
79
- * dispatch — not distance arithmetic — the dominant cost of a large
80
- * build; the batch keeps `reads` accounting identical per row. */
81
- private fetchNodes;
82
- /** Distance from the current source (query vector or reference code) to a node. */
83
- private distOf;
84
- /** Code-to-code distance between two stored nodes (graph wiring only). */
85
- private codeDist;
86
- /**
87
- * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
88
- * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
89
- * never returned. `visited` is a per-call set sized by the nodes seen here.
90
- */
91
- private searchLayer;
92
- /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
93
- private greedyDescend;
94
- /**
95
- * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
96
- * neighbours from `candIds`, preferring those closer to `base` than to any
97
- * already-chosen neighbour, then fills remaining slots with the closest
98
- * leftovers (keep-pruned connections). Appends to `out`.
99
- */
100
- private selectNeighbors;
101
- /**
102
- * Insert a vector's code under external id `ext`; returns the internal node id.
103
- * All graph reads/writes go through the store; the caller owns the transaction.
104
- */
105
- insert(ext: number, code: Uint8Array, efC?: number): number;
106
- /** k-NN with a full-precision query vector (accurate estimator). */
107
- searchKnn(vec: ArrayLike<number>, k: number, ef?: number): KnnHit[];
108
- /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
109
- searchKnnByCode(codeBytes: Uint8Array, k: number, ef?: number): KnnHit[];
110
- private collectKnn;
111
- /** Tombstone a live node by internal id. Caller owns the transaction. */
112
- remove(id: number): boolean;
113
- /**
114
- * Reclaim tombstones with a graph-preserving splice (see
115
- * {@link Store.spliceCompact}): live nodes keep their internal ids and their
116
- * wiring; each dead neighbour is replaced by its own live neighbours. Codes
117
- * are untouched, so the index loses nothing beyond what deletion itself
118
- * removes. The previous implementation replayed every live code through a
119
- * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
120
- * multi-million-node store, inside one WAL-bloating transaction. The splice
121
- * is a streaming pass with batched commits, then a VACUUM to return the
122
- * freed pages.
123
- */
124
- compact(): void;
30
+ readonly M: number;
31
+ readonly Mmax0: number;
32
+ readonly efConstruction: number;
33
+ private readonly mL;
34
+ private readonly quantizer;
35
+ private readonly store;
36
+ private readonly seed;
37
+ private readonly levelRng;
38
+ private entry;
39
+ private maxLevel;
40
+ private live;
41
+ private total;
42
+ private _efSearch;
43
+ private readonly working;
44
+ private readonly visited;
45
+ private visitedTag;
46
+ private visitedEpoch;
47
+ private seenTag;
48
+ private markVisited;
49
+ private readonly candHeap;
50
+ private readonly resHeap;
51
+ private readonly singleEp;
52
+ private readonly idxScratch;
53
+ private readonly distScratch;
54
+ private readonly freshScratch;
55
+ private qCtx;
56
+ private refBytes;
57
+ private building;
58
+ lastQueryDistComps: number;
59
+ /** Storage row reads issued by the most recent query (cache-independent). */
60
+ lastQueryStorageReads: number;
61
+ constructor(quantizer: RaBitQuantizer, store: Store, params: HnswParams);
62
+ get size(): number;
63
+ get physicalSize(): number;
64
+ get bytesPerVector(): number;
65
+ get efSearch(): number;
66
+ set efSearch(v: number);
67
+ private persistState;
68
+ private randomLevel;
69
+ /**
70
+ * Fetch a node record into the operation's working set. The first touch in an
71
+ * op is a storage read; later touches in the same op reuse it. The set is
72
+ * cleared per op and bounded by efConstruction / efSearch, so storage reads
73
+ * per op equal the number of *distinct* nodes the op visits -- minimal, and
74
+ * independent of any cache.
75
+ */
76
+ private node;
77
+ /** Prefetch several nodes into the working set with ONE batched storage
78
+ * read for the misses. Point queries per neighbour made statement
79
+ * dispatch — not distance arithmetic — the dominant cost of a large
80
+ * build; the batch keeps `reads` accounting identical per row. */
81
+ private fetchNodes;
82
+ /** Distance from the current source (query vector or reference code) to a node. */
83
+ private distOf;
84
+ /** Code-to-code distance between two stored nodes (graph wiring only). */
85
+ private codeDist;
86
+ /**
87
+ * SEARCH-LAYER (Algorithm 2). Frontier in `candHeap`, bounded result set in
88
+ * `resHeap` (non-deleted only). Deleted nodes are traversed for routing but
89
+ * never returned. `visited` is a per-call set sized by the nodes seen here.
90
+ */
91
+ private searchLayer;
92
+ /** Greedy single-best descent from `fromLayer` down to (but not into) `toLayer`. */
93
+ private greedyDescend;
94
+ /**
95
+ * SELECT-NEIGHBORS-HEURISTIC (Algorithm 4) on codes. Picks up to `M` diverse
96
+ * neighbours from `candIds`, preferring those closer to `base` than to any
97
+ * already-chosen neighbour, then fills remaining slots with the closest
98
+ * leftovers (keep-pruned connections). Appends to `out`.
99
+ */
100
+ private selectNeighbors;
101
+ /**
102
+ * Insert a vector's code under external id `ext`; returns the internal node id.
103
+ * All graph reads/writes go through the store; the caller owns the transaction.
104
+ */
105
+ insert(ext: number, code: Uint8Array, efC?: number): number;
106
+ /** k-NN with a full-precision query vector (accurate estimator). */
107
+ searchKnn(vec: ArrayLike<number>, k: number, ef?: number): KnnHit[];
108
+ /** k-NN with an already-quantized code (sign-bit Hamming / angular distance). */
109
+ searchKnnByCode(codeBytes: Uint8Array, k: number, ef?: number): KnnHit[];
110
+ private collectKnn;
111
+ /** Tombstone a live node by internal id. Caller owns the transaction. */
112
+ remove(id: number): boolean;
113
+ /**
114
+ * Reclaim tombstones with a graph-preserving splice (see
115
+ * {@link Store.spliceCompact}): live nodes keep their internal ids and their
116
+ * wiring; each dead neighbour is replaced by its own live neighbours. Codes
117
+ * are untouched, so the index loses nothing beyond what deletion itself
118
+ * removes. The previous implementation replayed every live code through a
119
+ * full HNSW insert — O(live · ef · log N) storage reads, HOURS on a trained
120
+ * multi-million-node store, inside one WAL-bloating transaction. The splice
121
+ * is a streaming pass with batched commits, then a VACUUM to return the
122
+ * freed pages.
123
+ */
124
+ compact(): void;
125
125
  }