@hviana/sema 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/AGENTS.md +115 -53
  2. package/HOW_IT_WORKS.md +291 -184
  3. package/jsr.json +6 -0
  4. package/package.json +1 -1
  5. package/src/mind/bridge.ts +10 -104
  6. package/src/mind/match.ts +497 -7
  7. package/src/mind/{prefix-completion.ts → mechanisms/prefix-completion.ts} +66 -92
  8. package/src/mind/mechanisms/recall.ts +38 -129
  9. package/src/mind/mechanisms/reference.ts +343 -0
  10. package/src/mind/mind.ts +12 -8
  11. package/src/mind/pipeline-mechanism.ts +120 -3
  12. package/src/mind/pipeline.ts +16 -2
  13. package/src/mind/traverse.ts +92 -1
  14. package/src/store.ts +13 -4
  15. package/test/33-multi-candidate.test.mjs +21 -11
  16. package/test/70-prefix-completion.test.mjs +1 -1
  17. package/test/72-prefix-candidate-supply.test.mjs +7 -9
  18. package/test/74-prefix-trap-not-sprung-early.test.mjs +1 -1
  19. package/test/76-reference-binding.test.mjs +543 -0
  20. package/dist/example/demo.d.ts +0 -1
  21. package/dist/example/demo.js +0 -39
  22. package/dist/example/train_base.d.ts +0 -93
  23. package/dist/example/train_base.js +0 -2269
  24. package/dist/src/alphabet.d.ts +0 -7
  25. package/dist/src/alphabet.js +0 -33
  26. package/dist/src/alu/src/alu.d.ts +0 -185
  27. package/dist/src/alu/src/alu.js +0 -440
  28. package/dist/src/alu/src/expr.d.ts +0 -61
  29. package/dist/src/alu/src/expr.js +0 -318
  30. package/dist/src/alu/src/index.d.ts +0 -11
  31. package/dist/src/alu/src/index.js +0 -19
  32. package/dist/src/alu/src/kernel-arith.d.ts +0 -16
  33. package/dist/src/alu/src/kernel-arith.js +0 -264
  34. package/dist/src/alu/src/kernel-bits.d.ts +0 -19
  35. package/dist/src/alu/src/kernel-bits.js +0 -152
  36. package/dist/src/alu/src/kernel-logic.d.ts +0 -4
  37. package/dist/src/alu/src/kernel-logic.js +0 -60
  38. package/dist/src/alu/src/kernel-nd.d.ts +0 -3
  39. package/dist/src/alu/src/kernel-nd.js +0 -208
  40. package/dist/src/alu/src/kernel-numeric.d.ts +0 -54
  41. package/dist/src/alu/src/kernel-numeric.js +0 -366
  42. package/dist/src/alu/src/operation.d.ts +0 -168
  43. package/dist/src/alu/src/operation.js +0 -189
  44. package/dist/src/alu/src/parser.d.ts +0 -221
  45. package/dist/src/alu/src/parser.js +0 -577
  46. package/dist/src/alu/src/resonance.d.ts +0 -55
  47. package/dist/src/alu/src/resonance.js +0 -126
  48. package/dist/src/alu/src/text.d.ts +0 -31
  49. package/dist/src/alu/src/text.js +0 -73
  50. package/dist/src/alu/src/value.d.ts +0 -109
  51. package/dist/src/alu/src/value.js +0 -300
  52. package/dist/src/alu/test/alu.test.d.ts +0 -1
  53. package/dist/src/alu/test/alu.test.js +0 -764
  54. package/dist/src/bytes.d.ts +0 -14
  55. package/dist/src/bytes.js +0 -59
  56. package/dist/src/canon.d.ts +0 -45
  57. package/dist/src/canon.js +0 -85
  58. package/dist/src/config.d.ts +0 -111
  59. package/dist/src/config.js +0 -91
  60. package/dist/src/derive/src/deduction.d.ts +0 -136
  61. package/dist/src/derive/src/deduction.js +0 -159
  62. package/dist/src/derive/src/index.d.ts +0 -8
  63. package/dist/src/derive/src/index.js +0 -11
  64. package/dist/src/derive/src/priority-queue.d.ts +0 -20
  65. package/dist/src/derive/src/priority-queue.js +0 -73
  66. package/dist/src/derive/src/rewrite.d.ts +0 -56
  67. package/dist/src/derive/src/rewrite.js +0 -100
  68. package/dist/src/derive/src/trie.d.ts +0 -90
  69. package/dist/src/derive/src/trie.js +0 -217
  70. package/dist/src/derive/test/derive.test.d.ts +0 -1
  71. package/dist/src/derive/test/derive.test.js +0 -122
  72. package/dist/src/extension.d.ts +0 -37
  73. package/dist/src/extension.js +0 -7
  74. package/dist/src/geometry.d.ts +0 -233
  75. package/dist/src/geometry.js +0 -1089
  76. package/dist/src/index.d.ts +0 -17
  77. package/dist/src/index.js +0 -19
  78. package/dist/src/ingest-cache.d.ts +0 -41
  79. package/dist/src/ingest-cache.js +0 -165
  80. package/dist/src/meter.d.ts +0 -176
  81. package/dist/src/meter.js +0 -274
  82. package/dist/src/mind/articulation.d.ts +0 -6
  83. package/dist/src/mind/articulation.js +0 -99
  84. package/dist/src/mind/attention.d.ts +0 -430
  85. package/dist/src/mind/attention.js +0 -2418
  86. package/dist/src/mind/bridge.d.ts +0 -39
  87. package/dist/src/mind/bridge.js +0 -998
  88. package/dist/src/mind/canonical.d.ts +0 -34
  89. package/dist/src/mind/canonical.js +0 -93
  90. package/dist/src/mind/frame-filler.d.ts +0 -15
  91. package/dist/src/mind/frame-filler.js +0 -535
  92. package/dist/src/mind/graph-search.d.ts +0 -294
  93. package/dist/src/mind/graph-search.js +0 -996
  94. package/dist/src/mind/index.d.ts +0 -9
  95. package/dist/src/mind/index.js +0 -5
  96. package/dist/src/mind/junction.d.ts +0 -137
  97. package/dist/src/mind/junction.js +0 -358
  98. package/dist/src/mind/learning.d.ts +0 -75
  99. package/dist/src/mind/learning.js +0 -265
  100. package/dist/src/mind/match.d.ts +0 -192
  101. package/dist/src/mind/match.js +0 -666
  102. package/dist/src/mind/mechanisms/alu.d.ts +0 -4
  103. package/dist/src/mind/mechanisms/alu.js +0 -36
  104. package/dist/src/mind/mechanisms/cast.d.ts +0 -89
  105. package/dist/src/mind/mechanisms/cast.js +0 -1186
  106. package/dist/src/mind/mechanisms/confluence.d.ts +0 -24
  107. package/dist/src/mind/mechanisms/confluence.js +0 -271
  108. package/dist/src/mind/mechanisms/cover.d.ts +0 -6
  109. package/dist/src/mind/mechanisms/cover.js +0 -232
  110. package/dist/src/mind/mechanisms/extraction.d.ts +0 -33
  111. package/dist/src/mind/mechanisms/extraction.js +0 -327
  112. package/dist/src/mind/mechanisms/recall.d.ts +0 -16
  113. package/dist/src/mind/mechanisms/recall.js +0 -558
  114. package/dist/src/mind/mind.d.ts +0 -386
  115. package/dist/src/mind/mind.js +0 -754
  116. package/dist/src/mind/pipeline-mechanism.d.ts +0 -183
  117. package/dist/src/mind/pipeline-mechanism.js +0 -521
  118. package/dist/src/mind/pipeline.d.ts +0 -49
  119. package/dist/src/mind/pipeline.js +0 -331
  120. package/dist/src/mind/prefix-completion.d.ts +0 -59
  121. package/dist/src/mind/prefix-completion.js +0 -270
  122. package/dist/src/mind/primitives.d.ts +0 -85
  123. package/dist/src/mind/primitives.js +0 -333
  124. package/dist/src/mind/rationale.d.ts +0 -139
  125. package/dist/src/mind/rationale.js +0 -163
  126. package/dist/src/mind/reasoning.d.ts +0 -42
  127. package/dist/src/mind/reasoning.js +0 -305
  128. package/dist/src/mind/recognition.d.ts +0 -20
  129. package/dist/src/mind/recognition.js +0 -696
  130. package/dist/src/mind/resonance.d.ts +0 -35
  131. package/dist/src/mind/resonance.js +0 -365
  132. package/dist/src/mind/trace.d.ts +0 -15
  133. package/dist/src/mind/trace.js +0 -73
  134. package/dist/src/mind/traverse.d.ts +0 -158
  135. package/dist/src/mind/traverse.js +0 -716
  136. package/dist/src/mind/types.d.ts +0 -418
  137. package/dist/src/mind/types.js +0 -179
  138. package/dist/src/rabitq-ivf/src/database.d.ts +0 -113
  139. package/dist/src/rabitq-ivf/src/database.js +0 -201
  140. package/dist/src/rabitq-ivf/src/index.d.ts +0 -7
  141. package/dist/src/rabitq-ivf/src/index.js +0 -4
  142. package/dist/src/rabitq-ivf/src/ivf.d.ts +0 -200
  143. package/dist/src/rabitq-ivf/src/ivf.js +0 -1165
  144. package/dist/src/rabitq-ivf/src/prng.d.ts +0 -19
  145. package/dist/src/rabitq-ivf/src/prng.js +0 -36
  146. package/dist/src/rabitq-ivf/src/rabitq.d.ts +0 -95
  147. package/dist/src/rabitq-ivf/src/rabitq.js +0 -313
  148. package/dist/src/sema.d.ts +0 -41
  149. package/dist/src/sema.js +0 -77
  150. package/dist/src/store-sqlite.d.ts +0 -184
  151. package/dist/src/store-sqlite.js +0 -942
  152. package/dist/src/store.d.ts +0 -762
  153. package/dist/src/store.js +0 -1823
  154. package/dist/src/vec.d.ts +0 -31
  155. package/dist/src/vec.js +0 -109
  156. package/src/mind/frame-filler.ts +0 -604
  157. package/test/69-frame-filler.test.mjs +0 -115
@@ -1,19 +0,0 @@
1
- /**
2
- * Small deterministic pseudo-random number generator (mulberry32).
3
- *
4
- * Pure ECMAScript. It is used so that the random orthogonal rotation of the
5
- * RaBitQ quantizer rotation is reproducible and can be
6
- * regenerated after (de)serialization without storing large matrices.
7
- */
8
- export declare class Prng {
9
- private state;
10
- constructor(seed: number);
11
- /** Uniform float in [0, 1). */
12
- next(): number;
13
- /** Uniform integer in [0, n). */
14
- int(n: number): number;
15
- /** Current internal state (one uint32), for persisting/resuming the stream. */
16
- snapshot(): number;
17
- /** Resume the stream from a previously snapshotted state. */
18
- restore(state: number): void;
19
- }
@@ -1,36 +0,0 @@
1
- /**
2
- * Small deterministic pseudo-random number generator (mulberry32).
3
- *
4
- * Pure ECMAScript. It is used so that the random orthogonal rotation of the
5
- * RaBitQ quantizer rotation is reproducible and can be
6
- * regenerated after (de)serialization without storing large matrices.
7
- */
8
- export class Prng {
9
- state;
10
- constructor(seed) {
11
- this.state = seed >>> 0;
12
- if (this.state === 0)
13
- this.state = 0x9e3779b9;
14
- }
15
- /** Uniform float in [0, 1). */
16
- next() {
17
- let t = (this.state = (this.state + 0x6d2b79f5) | 0);
18
- t = Math.imul(t ^ (t >>> 15), t | 1);
19
- t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
20
- return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
21
- }
22
- /** Uniform integer in [0, n). */
23
- int(n) {
24
- return (this.next() * n) | 0;
25
- }
26
- /** Current internal state (one uint32), for persisting/resuming the stream. */
27
- snapshot() {
28
- return this.state >>> 0;
29
- }
30
- /** Resume the stream from a previously snapshotted state. */
31
- restore(state) {
32
- this.state = state >>> 0;
33
- if (this.state === 0)
34
- this.state = 0x9e3779b9;
35
- }
36
- }
@@ -1,95 +0,0 @@
1
- export interface QueryContext {
2
- vmin: number;
3
- delta: number;
4
- sumQInt: number;
5
- /**
6
- * Per-query lookup table of length nbytes*256. `qlut[p*256 + b]` is the sum of
7
- * the quantised query values at the (up to 8) coordinates whose code bits are
8
- * set in byte value `b` at code-byte position `p`. The query/code inner product
9
- * for a stored code is then sum over its bytes of qlut[p*256 + codeByte_p].
10
- */
11
- qlut: Uint8Array | Uint16Array;
12
- /** number of bytes per code (paddedDim / 8). */
13
- nbytes: number;
14
- /** true when the (centered) query has zero norm. */
15
- zero: boolean;
16
- }
17
- export interface RaBitQOptions {
18
- queryBits?: number;
19
- rounds?: number;
20
- seed?: number;
21
- centroid?: ArrayLike<number>;
22
- }
23
- /**
24
- * 1-bit RaBitQ quantizer (cosine) -- the ONLY representation of a vector kept by
25
- * the index. A D-dimensional vector collapses to ceil(D/32) 32-bit words of sign
26
- * bits, e.g. a 256-d vector goes from 256*4 = 1024 bytes to 32 bytes of code.
27
- *
28
- * Each vector is centered by an optional centroid, normalised, rotated by a
29
- * fast random orthogonal transform (random sign flips + Walsh-Hadamard,
30
- * O(D log D)) and reduced to one sign bit per padded dimension. The random
31
- * rotation makes the quantisation error essentially uniform across vectors, so
32
- * the cosine estimate needs only a single fixed scale (`cosFactor`) rather than
33
- * any per-vector correction.
34
- *
35
- * Two estimators are provided:
36
- * - `estimate` : full-precision query vs stored code (accurate)
37
- * - `codeDistanceBytes` : stored code vs stored code (Hamming based; used while
38
- * building the graph, where neither side is full precision)
39
- *
40
- * Reference: Gao & Long, "RaBitQ: Quantizing High-Dimensional Vectors with a
41
- * Theoretical Error Bound for Approximate Nearest Neighbor Search", SIGMOD 2024.
42
- */
43
- export declare class RaBitQuantizer {
44
- readonly dim: number;
45
- readonly paddedDim: number;
46
- readonly codeWords: number;
47
- readonly queryBits: number;
48
- readonly rounds: number;
49
- readonly seed: number;
50
- readonly centroid: Float64Array;
51
- private readonly sqrtD;
52
- private readonly maxQInt;
53
- private readonly signs;
54
- private readonly scratch;
55
- /** Fixed cosine inner-product scale (= 1 / E[L1] of a rotated unit vector). */
56
- readonly cosFactor: number;
57
- private readonly nbytes;
58
- /** coordinate index sitting at byte position p, bit k -> bitCoord[p*8 + k]. */
59
- private readonly bitCoord;
60
- /** true when the largest possible LUT entry overflows a Uint8. */
61
- private readonly lutWide;
62
- constructor(dim: number, opts?: RaBitQOptions);
63
- /** In-place fast Walsh-Hadamard transform; `a.length` must be a power of two. */
64
- private fwht;
65
- /** Apply the orthogonal rotation in place (a.length === paddedDim). */
66
- private rotate;
67
- /** Encode a raw vector into its 1-bit sign code (the whole representation). */
68
- encode(vec: ArrayLike<number>): Uint32Array;
69
- /** Pre-process a full-precision query into the structure consumed by `estimate`. */
70
- prepareQuery(vec: ArrayLike<number>): QueryContext;
71
- /**
72
- * Estimate the cosine distance (1 - cosine) between a stored code and a
73
- * full-precision query, reading the code's bytes against the query's byte LUT.
74
- * The code's set-bit count is tallied in the same byte scan, so nothing beyond
75
- * the code itself is needed.
76
- *
77
- * `codeBytes` is a Uint8 view of the packed code buffer and `byteOffset` is the
78
- * code's start byte (id * paddedDim/8).
79
- */
80
- estimate(codeBytes: Uint8Array, byteOffset: number, q: QueryContext): number;
81
- /**
82
- * Cosine distance (1 - cosine, in [0, 2]) between two packed codes, computed
83
- * directly from the BLOB bytes via their sign-bit Hamming distance — no word
84
- * reinterpretation, so it is endianness-agnostic. Identical codes score 0.
85
- *
86
- * This is the distance used to build the graph (code vs code) and to answer a
87
- * query given an already-quantized code. It is coarser than `estimate`, where
88
- * one side is full precision.
89
- */
90
- codeDistanceBytes(a: Uint8Array, b: Uint8Array): number;
91
- /** Pack a code (codeWords 32-bit words) into its little-endian BLOB bytes. */
92
- codeToBytes(code: ArrayLike<number>): Uint8Array;
93
- /** Reinterpret a code BLOB as a copy of codeWords 32-bit words. */
94
- bytesToCode(bytes: Uint8Array): Uint32Array;
95
- }
@@ -1,313 +0,0 @@
1
- import { Prng } from "./prng.js";
2
- function nextPow2(n) {
3
- let p = 1;
4
- while (p < n)
5
- p <<= 1;
6
- return p;
7
- }
8
- /** Set-bit count for every byte value, for counting code bits during the byte scan. */
9
- const POPCOUNT8 = new Uint8Array(256);
10
- for (let i = 1; i < 256; i++)
11
- POPCOUNT8[i] = POPCOUNT8[i >> 1] + (i & 1);
12
- /**
13
- * 1-bit RaBitQ quantizer (cosine) -- the ONLY representation of a vector kept by
14
- * the index. A D-dimensional vector collapses to ceil(D/32) 32-bit words of sign
15
- * bits, e.g. a 256-d vector goes from 256*4 = 1024 bytes to 32 bytes of code.
16
- *
17
- * Each vector is centered by an optional centroid, normalised, rotated by a
18
- * fast random orthogonal transform (random sign flips + Walsh-Hadamard,
19
- * O(D log D)) and reduced to one sign bit per padded dimension. The random
20
- * rotation makes the quantisation error essentially uniform across vectors, so
21
- * the cosine estimate needs only a single fixed scale (`cosFactor`) rather than
22
- * any per-vector correction.
23
- *
24
- * Two estimators are provided:
25
- * - `estimate` : full-precision query vs stored code (accurate)
26
- * - `codeDistanceBytes` : stored code vs stored code (Hamming based; used while
27
- * building the graph, where neither side is full precision)
28
- *
29
- * Reference: Gao & Long, "RaBitQ: Quantizing High-Dimensional Vectors with a
30
- * Theoretical Error Bound for Approximate Nearest Neighbor Search", SIGMOD 2024.
31
- */
32
- export class RaBitQuantizer {
33
- dim;
34
- paddedDim;
35
- codeWords;
36
- queryBits;
37
- rounds;
38
- seed;
39
- centroid;
40
- sqrtD;
41
- maxQInt;
42
- signs;
43
- scratch;
44
- /** Fixed cosine inner-product scale (= 1 / E[L1] of a rotated unit vector). */
45
- cosFactor;
46
- // byte-LUT machinery for the query/code estimator
47
- nbytes;
48
- /** coordinate index sitting at byte position p, bit k -> bitCoord[p*8 + k]. */
49
- bitCoord;
50
- /** true when the largest possible LUT entry overflows a Uint8. */
51
- lutWide;
52
- constructor(dim, opts = {}) {
53
- this.dim = dim;
54
- this.paddedDim = nextPow2(dim);
55
- this.codeWords = Math.ceil(this.paddedDim / 32);
56
- this.queryBits = opts.queryBits ?? 8;
57
- this.rounds = opts.rounds ?? 3;
58
- this.seed = (opts.seed ?? 0x1234abcd) >>> 0;
59
- this.sqrtD = Math.sqrt(this.paddedDim);
60
- this.cosFactor = Math.sqrt(Math.PI / (2 * this.paddedDim));
61
- this.maxQInt = (1 << this.queryBits) - 1;
62
- this.centroid = new Float64Array(dim);
63
- if (opts.centroid) {
64
- for (let i = 0; i < dim; i++)
65
- this.centroid[i] = opts.centroid[i] ?? 0;
66
- }
67
- const prng = new Prng(this.seed);
68
- this.signs = [];
69
- for (let r = 0; r < this.rounds; r++) {
70
- const s = new Float64Array(this.paddedDim);
71
- for (let i = 0; i < this.paddedDim; i++) {
72
- s[i] = prng.next() < 0.5 ? -1 : 1;
73
- }
74
- this.signs.push(s);
75
- }
76
- this.scratch = new Float64Array(this.paddedDim);
77
- // Map (code-byte position, bit-in-byte) -> coordinate, honouring the host
78
- // byte order so the Uint8 view of the (Uint32) code buffer is interpreted
79
- // correctly on both little- and big-endian platforms.
80
- this.nbytes = this.paddedDim >>> 3;
81
- const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1;
82
- this.bitCoord = new Int32Array(this.nbytes * 8);
83
- for (let p = 0; p < this.nbytes; p++) {
84
- const word = p >>> 2;
85
- const localByte = littleEndian ? p & 3 : 3 - (p & 3);
86
- const bitBase = word * 32 + localByte * 8;
87
- for (let k = 0; k < 8; k++)
88
- this.bitCoord[p * 8 + k] = bitBase + k;
89
- }
90
- this.lutWide = 8 * this.maxQInt > 255;
91
- }
92
- /** In-place fast Walsh-Hadamard transform; `a.length` must be a power of two. */
93
- fwht(a) {
94
- const n = a.length;
95
- for (let len = 1; len < n; len <<= 1) {
96
- const span = len << 1;
97
- for (let i = 0; i < n; i += span) {
98
- for (let j = i; j < i + len; j++) {
99
- const u = a[j];
100
- const v = a[j + len];
101
- a[j] = u + v;
102
- a[j + len] = u - v;
103
- }
104
- }
105
- }
106
- }
107
- /** Apply the orthogonal rotation in place (a.length === paddedDim). */
108
- rotate(a) {
109
- const n = this.paddedDim;
110
- const inv = 1 / this.sqrtD;
111
- for (let r = 0; r < this.rounds; r++) {
112
- const s = this.signs[r];
113
- for (let i = 0; i < n; i++)
114
- a[i] *= s[i];
115
- this.fwht(a);
116
- for (let i = 0; i < n; i++)
117
- a[i] *= inv;
118
- }
119
- }
120
- /** Encode a raw vector into its 1-bit sign code (the whole representation). */
121
- encode(vec) {
122
- const dim = this.dim;
123
- const pd = this.paddedDim;
124
- const buf = this.scratch;
125
- let sq = 0;
126
- for (let i = 0; i < dim; i++) {
127
- const v = vec[i] - this.centroid[i];
128
- buf[i] = v;
129
- sq += v * v;
130
- }
131
- for (let i = dim; i < pd; i++)
132
- buf[i] = 0;
133
- const code = new Uint32Array(this.codeWords);
134
- if (sq === 0) {
135
- return code;
136
- }
137
- const invNorm = 1 / Math.sqrt(sq);
138
- for (let i = 0; i < dim; i++)
139
- buf[i] *= invNorm; // unit residual; padded dims stay 0
140
- this.rotate(buf);
141
- for (let i = 0; i < pd; i++) {
142
- if (buf[i] > 0)
143
- code[i >>> 5] |= 1 << (i & 31);
144
- }
145
- return code;
146
- }
147
- /** Pre-process a full-precision query into the structure consumed by `estimate`. */
148
- prepareQuery(vec) {
149
- const dim = this.dim;
150
- const pd = this.paddedDim;
151
- const nb = this.nbytes;
152
- const buf = this.scratch;
153
- let sq = 0;
154
- for (let i = 0; i < dim; i++) {
155
- const v = vec[i] - this.centroid[i];
156
- buf[i] = v;
157
- sq += v * v;
158
- }
159
- for (let i = dim; i < pd; i++)
160
- buf[i] = 0;
161
- const qNorm = Math.sqrt(sq);
162
- const qlut = this.lutWide
163
- ? new Uint16Array(nb * 256)
164
- : new Uint8Array(nb * 256);
165
- if (qNorm === 0) {
166
- return { vmin: 0, delta: 0, sumQInt: 0, qlut, nbytes: nb, zero: true };
167
- }
168
- const invNorm = 1 / qNorm;
169
- for (let i = 0; i < dim; i++)
170
- buf[i] *= invNorm;
171
- this.rotate(buf);
172
- let vmin = Infinity;
173
- let vmax = -Infinity;
174
- for (let i = 0; i < pd; i++) {
175
- const x = buf[i];
176
- if (x < vmin)
177
- vmin = x;
178
- if (x > vmax)
179
- vmax = x;
180
- }
181
- const range = vmax - vmin;
182
- const delta = range > 0 ? range / this.maxQInt : 0;
183
- const invDelta = delta > 0 ? 1 / delta : 0;
184
- // Quantise each (rotated) query coordinate to queryBits bits.
185
- const qint = buf; // reuse: write the integer code back over the float buffer
186
- let sumQInt = 0;
187
- for (let i = 0; i < pd; i++) {
188
- let q = delta > 0 ? Math.round((buf[i] - vmin) * invDelta) : 0;
189
- if (q < 0)
190
- q = 0;
191
- else if (q > this.maxQInt)
192
- q = this.maxQInt;
193
- qint[i] = q;
194
- sumQInt += q;
195
- }
196
- // Build the byte LUT: qlut[p*256 + v] = sum of qint at coords whose bit is
197
- // set in v, grown incrementally as qlut[..(v with lowest set bit cleared)..]
198
- // plus the contribution of that lowest set bit.
199
- const bitCoord = this.bitCoord;
200
- for (let p = 0; p < nb; p++) {
201
- const base = p << 8;
202
- const cb = p << 3;
203
- for (let v = 1; v < 256; v++) {
204
- const low = v & -v;
205
- const k = 31 - Math.clz32(low);
206
- qlut[base + v] = qlut[base + (v & (v - 1))] + qint[bitCoord[cb + k]];
207
- }
208
- }
209
- return { vmin, delta, sumQInt, qlut, nbytes: nb, zero: false };
210
- }
211
- /**
212
- * Estimate the cosine distance (1 - cosine) between a stored code and a
213
- * full-precision query, reading the code's bytes against the query's byte LUT.
214
- * The code's set-bit count is tallied in the same byte scan, so nothing beyond
215
- * the code itself is needed.
216
- *
217
- * `codeBytes` is a Uint8 view of the packed code buffer and `byteOffset` is the
218
- * code's start byte (id * paddedDim/8).
219
- */
220
- estimate(codeBytes, byteOffset, q) {
221
- if (q.zero) {
222
- return 1;
223
- }
224
- const nb = q.nbytes;
225
- const lut = q.qlut;
226
- let dot = 0;
227
- let popcount = 0;
228
- // THE INNERMOST LOOP OF SEARCH. Profiled on the trained store: 170 ANN
229
- // queries scan 8,702,005 slots, and this estimate — inlined by V8 into
230
- // IvfIndex.scanClusters, which is why it does not appear separately — was
231
- // 21% of all inference CPU.
232
- //
233
- // The `dot` half is an irreducible data-dependent LUT probe per byte. The
234
- // `popcount` half is not: it is the same sign-bit count `codeDistanceBytes`
235
- // below already folds into 32-bit words ("~4x fewer loop iterations"), and
236
- // that reasoning applies verbatim here. Four bytes are packed into one
237
- // word and popcounted with the standard SWAR reduction, while the four LUT
238
- // probes are issued together so their loads overlap instead of serialising
239
- // behind the popcount.
240
- //
241
- // BIT-IDENTICAL, not an approximation: popcount over four bytes equals the
242
- // sum of their individual popcounts, and the LUT terms are added in the
243
- // same order at the same indices. Verified by direct comparison, and the
244
- // 445 suite plus the battery's answers are unchanged.
245
- let p = 0;
246
- for (const n4 = nb & ~3; p < n4; p += 4) {
247
- const o = byteOffset + p;
248
- const b0 = codeBytes[o], b1 = codeBytes[o + 1];
249
- const b2 = codeBytes[o + 2], b3 = codeBytes[o + 3];
250
- dot += lut[(p << 8) + b0] + lut[((p + 1) << 8) + b1] +
251
- lut[((p + 2) << 8) + b2] + lut[((p + 3) << 8) + b3];
252
- let x = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
253
- x -= (x >>> 1) & 0x55555555;
254
- x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
255
- x = (x + (x >>> 4)) & 0x0f0f0f0f;
256
- popcount += Math.imul(x, 0x01010101) >>> 24;
257
- }
258
- for (; p < nb; p++) {
259
- const b = codeBytes[byteOffset + p];
260
- dot += lut[(p << 8) + b];
261
- popcount += POPCOUNT8[b];
262
- }
263
- // A = sum_i sign_i * q_rot_i, recovered from the quantised query.
264
- const A = q.vmin * (2 * popcount - this.paddedDim) +
265
- q.delta * (2 * dot - q.sumQInt);
266
- return 1 - this.cosFactor * A;
267
- }
268
- /**
269
- * Cosine distance (1 - cosine, in [0, 2]) between two packed codes, computed
270
- * directly from the BLOB bytes via their sign-bit Hamming distance — no word
271
- * reinterpretation, so it is endianness-agnostic. Identical codes score 0.
272
- *
273
- * This is the distance used to build the graph (code vs code) and to answer a
274
- * query given an already-quantized code. It is coarser than `estimate`, where
275
- * one side is full precision.
276
- */
277
- codeDistanceBytes(a, b) {
278
- const nb = this.nbytes;
279
- let ham = 0;
280
- // 32-bit-word Hamming: this is the hot arithmetic of graph construction
281
- // (every candidate/prune comparison), so fold 4 bytes into one word and
282
- // popcount it — ~4× fewer loop iterations than a byte-LUT scan, with no
283
- // allocation and no dependence on the buffers' alignment or endianness
284
- // (both sides are composed identically, so XOR is order-agnostic).
285
- let p = 0;
286
- for (const n4 = nb & ~3; p < n4; p += 4) {
287
- let x = (a[p] ^ b[p]) |
288
- ((a[p + 1] ^ b[p + 1]) << 8) |
289
- ((a[p + 2] ^ b[p + 2]) << 16) |
290
- ((a[p + 3] ^ b[p + 3]) << 24);
291
- x -= (x >>> 1) & 0x55555555;
292
- x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
293
- x = (x + (x >>> 4)) & 0x0f0f0f0f;
294
- ham += Math.imul(x, 0x01010101) >>> 24;
295
- }
296
- for (; p < nb; p++)
297
- ham += POPCOUNT8[a[p] ^ b[p]];
298
- return (2 * ham) / this.paddedDim;
299
- }
300
- /** Pack a code (codeWords 32-bit words) into its little-endian BLOB bytes. */
301
- codeToBytes(code) {
302
- const u = new Uint32Array(this.codeWords);
303
- for (let i = 0; i < this.codeWords; i++)
304
- u[i] = code[i];
305
- return new Uint8Array(u.buffer, 0, u.byteLength);
306
- }
307
- /** Reinterpret a code BLOB as a copy of codeWords 32-bit words. */
308
- bytesToCode(bytes) {
309
- const u = new Uint32Array(this.codeWords);
310
- new Uint8Array(u.buffer).set(bytes);
311
- return u;
312
- }
313
- }
@@ -1,41 +0,0 @@
1
- import { Permutation, Vec } from "./vec.js";
2
- /** The one structure. A node's vector is the gist of its whole subtree; it also
3
- * carries the structure the DAG store interns — its leaf bytes, or its kids. */
4
- export interface Sema {
5
- v: Vec;
6
- leaf: Uint8Array | null;
7
- kids: Sema[] | null;
8
- }
9
- export declare const sema: (v: Vec, leaf?: Uint8Array | null, kids?: Sema[] | null) => Sema;
10
- /** Whether a node is a CHUNK — a leaf-parent whose children are ALL leaves,
11
- * the perception tree's smallest grouped unit. The one predicate behind
12
- * region collection, canonical segmentation seams, and sub-span indexing;
13
- * named here beside the type so no consumer restates the shape inline. */
14
- export declare const isChunk: (n: Sema) => n is Sema & {
15
- kids: Sema[];
16
- };
17
- /** The medium: dimension, keyring, and noise source. */
18
- export interface Space {
19
- D: number;
20
- seats: Permutation[];
21
- rand: () => number;
22
- maxGroup: number;
23
- }
24
- /** Bind one vector into a seat — the elementary half of fold. Used to index an
25
- * episode from either side and to pour a partner into a form's halo. */
26
- export declare const bindSeat: (space: Space, v: Vec, seat: number) => Vec;
27
- /** The positional coordinate of child `index` inside a node of `size` items.
28
- *
29
- * Positions are anchored at BOTH ends of the full keyring: the left half uses
30
- * the lowest seats and the right half uses the highest seats. Growing a node
31
- * at one edge therefore preserves the coordinates anchored at the other edge,
32
- * while every position remains injective as long as
33
- * `0 <= index < size <= seatCount`. The helper is shared by perception and by
34
- * synthetic/canonical folds so there is exactly one structural algebra. */
35
- export declare function twoEndedSeat(seatCount: number, size: number, index: number): number;
36
- /** The company signature of node `id` — the halo's pour unit (see above). */
37
- export declare function companySignature(space: Space, id: number): Vec;
38
- /** fold — combine ordered children into one gist.
39
- * Each child is turned in the same two-ended coordinate frame used by the
40
- * perception tree, then superposed and normalized. */
41
- export declare function fold(space: Space, kids: Vec[]): Vec;
package/dist/src/sema.js DELETED
@@ -1,77 +0,0 @@
1
- import { addInto, normalize, permute, permuteInto, randomUnit, rng, zeros, } from "./vec.js";
2
- export const sema = (v, leaf = null, kids = null) => ({ v, leaf, kids });
3
- /** Whether a node is a CHUNK — a leaf-parent whose children are ALL leaves,
4
- * the perception tree's smallest grouped unit. The one predicate behind
5
- * region collection, canonical segmentation seams, and sub-span indexing;
6
- * named here beside the type so no consumer restates the shape inline. */
7
- export const isChunk = (n) => n.kids !== null && n.kids.every((k) => k.kids === null);
8
- // Reusable permute buffer for fold.
9
- let _foldBuf = null;
10
- /** Bind one vector into a seat — the elementary half of fold. Used to index an
11
- * episode from either side and to pour a partner into a form's halo. */
12
- export const bindSeat = (space, v, seat) => permute(v, space.seats[seat].fwd);
13
- /** The positional coordinate of child `index` inside a node of `size` items.
14
- *
15
- * Positions are anchored at BOTH ends of the full keyring: the left half uses
16
- * the lowest seats and the right half uses the highest seats. Growing a node
17
- * at one edge therefore preserves the coordinates anchored at the other edge,
18
- * while every position remains injective as long as
19
- * `0 <= index < size <= seatCount`. The helper is shared by perception and by
20
- * synthetic/canonical folds so there is exactly one structural algebra. */
21
- export function twoEndedSeat(seatCount, size, index) {
22
- const front = (size + 1) >> 1;
23
- return index < front ? index : seatCount - size + index;
24
- }
25
- // ── Company signatures ──────────────────────────────────────────────────
26
- //
27
- // A halo is a superposition of EPISODE SIGNATURES: it answers "who does this
28
- // form keep company with", and two forms share a concept when they keep the
29
- // SAME company (the same partner nodes). Pouring the partner's raw GIST was
30
- // an approximation of that: it worked while the hierarchical fold decorrelated
31
- // unrelated gists quickly, but any byte-overlap between partners leaks CONTENT
32
- // similarity into COMPANY similarity, silently shifting the halo null model
33
- // that conceptThreshold's derivation (unrelated halos ⇒ cosine 0 ± 1/√D)
34
- // depends on. A signature makes the semantics exact and fold-independent:
35
- // a deterministic unit vector derived from the partner's IDENTITY, so two
36
- // halos correlate exactly as much as their company overlaps — never because
37
- // their partners merely contain similar bytes.
38
- //
39
- // Seeded by node id: ids are content-addressed mint order, stable for a given
40
- // corpus (including checkpoint/resume, which re-derives identical ids), and
41
- // halos are per-store training artifacts that are never compared across
42
- // stores.
43
- const _sigCache = new WeakMap();
44
- const SIG_CACHE_MAX = 65_536;
45
- /** The company signature of node `id` — the halo's pour unit (see above). */
46
- export function companySignature(space, id) {
47
- let cache = _sigCache.get(space);
48
- if (!cache)
49
- _sigCache.set(space, cache = new Map());
50
- const hit = cache.get(id);
51
- if (hit)
52
- return hit;
53
- const v = randomUnit(space.D, rng((id ^ 0x9e3779b9) >>> 0));
54
- if (cache.size >= SIG_CACHE_MAX)
55
- cache.clear(); // flat cap; regeneration is cheap
56
- cache.set(id, v);
57
- return v;
58
- }
59
- /** fold — combine ordered children into one gist.
60
- * Each child is turned in the same two-ended coordinate frame used by the
61
- * perception tree, then superposed and normalized. */
62
- export function fold(space, kids) {
63
- if (kids.length > space.seats.length) {
64
- throw new Error(`fold: ${kids.length} children but the keyring has only ${space.seats.length} seats`);
65
- }
66
- const out = zeros(space.D);
67
- if (!_foldBuf || _foldBuf.length !== space.D) {
68
- _foldBuf = new Float32Array(space.D);
69
- }
70
- const buf = _foldBuf;
71
- for (let i = 0; i < kids.length; i++) {
72
- const slot = twoEndedSeat(space.seats.length, kids.length, i);
73
- permuteInto(buf, kids[i], space.seats[slot].fwd);
74
- addInto(out, buf);
75
- }
76
- return normalize(out);
77
- }