@hviana/sema 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/AGENTS.md +470 -0
  2. package/AUTHORS.md +12 -0
  3. package/COMMERCIAL-LICENSE.md +19 -0
  4. package/CONTRIBUTING.md +15 -0
  5. package/HOW_IT_WORKS.md +4210 -0
  6. package/LICENSE.md +117 -0
  7. package/README.md +290 -0
  8. package/TRADEMARKS.md +19 -0
  9. package/example/demo.ts +46 -0
  10. package/example/train_base.ts +2675 -0
  11. package/index.html +893 -0
  12. package/package.json +25 -0
  13. package/src/alphabet.ts +34 -0
  14. package/src/alu/README.md +332 -0
  15. package/src/alu/src/alu.ts +541 -0
  16. package/src/alu/src/expr.ts +339 -0
  17. package/src/alu/src/index.ts +115 -0
  18. package/src/alu/src/kernel-arith.ts +377 -0
  19. package/src/alu/src/kernel-bits.ts +199 -0
  20. package/src/alu/src/kernel-logic.ts +102 -0
  21. package/src/alu/src/kernel-nd.ts +235 -0
  22. package/src/alu/src/kernel-numeric.ts +466 -0
  23. package/src/alu/src/operation.ts +344 -0
  24. package/src/alu/src/parser.ts +574 -0
  25. package/src/alu/src/resonance.ts +161 -0
  26. package/src/alu/src/text.ts +83 -0
  27. package/src/alu/src/value.ts +327 -0
  28. package/src/alu/test/alu.test.ts +1004 -0
  29. package/src/bytes.ts +62 -0
  30. package/src/config.ts +227 -0
  31. package/src/derive/README.md +295 -0
  32. package/src/derive/src/deduction.ts +263 -0
  33. package/src/derive/src/index.ts +24 -0
  34. package/src/derive/src/priority-queue.ts +70 -0
  35. package/src/derive/src/rewrite.ts +127 -0
  36. package/src/derive/src/trie.ts +242 -0
  37. package/src/derive/test/derive.test.ts +137 -0
  38. package/src/extension.ts +42 -0
  39. package/src/geometry.ts +511 -0
  40. package/src/index.ts +38 -0
  41. package/src/ingest-cache.ts +224 -0
  42. package/src/mind/articulation.ts +137 -0
  43. package/src/mind/attention.ts +1061 -0
  44. package/src/mind/canonical.ts +107 -0
  45. package/src/mind/graph-search.ts +1100 -0
  46. package/src/mind/index.ts +18 -0
  47. package/src/mind/junction.ts +347 -0
  48. package/src/mind/learning.ts +246 -0
  49. package/src/mind/match.ts +507 -0
  50. package/src/mind/mechanisms/alu.ts +33 -0
  51. package/src/mind/mechanisms/cast.ts +568 -0
  52. package/src/mind/mechanisms/confluence.ts +268 -0
  53. package/src/mind/mechanisms/cover.ts +248 -0
  54. package/src/mind/mechanisms/extraction.ts +422 -0
  55. package/src/mind/mechanisms/recall.ts +241 -0
  56. package/src/mind/mind.ts +483 -0
  57. package/src/mind/pipeline-mechanism.ts +326 -0
  58. package/src/mind/pipeline.ts +300 -0
  59. package/src/mind/primitives.ts +201 -0
  60. package/src/mind/rationale.ts +275 -0
  61. package/src/mind/reasoning.ts +198 -0
  62. package/src/mind/recognition.ts +234 -0
  63. package/src/mind/resonance.ts +0 -0
  64. package/src/mind/trace.ts +108 -0
  65. package/src/mind/traverse.ts +556 -0
  66. package/src/mind/types.ts +281 -0
  67. package/src/rabitq-hnsw/README.md +303 -0
  68. package/src/rabitq-hnsw/src/database.ts +492 -0
  69. package/src/rabitq-hnsw/src/heap.ts +90 -0
  70. package/src/rabitq-hnsw/src/hnsw.ts +514 -0
  71. package/src/rabitq-hnsw/src/index.ts +15 -0
  72. package/src/rabitq-hnsw/src/prng.ts +39 -0
  73. package/src/rabitq-hnsw/src/rabitq.ts +308 -0
  74. package/src/rabitq-hnsw/src/store.ts +994 -0
  75. package/src/rabitq-hnsw/test/hnsw.test.ts +1213 -0
  76. package/src/store-sqlite.ts +928 -0
  77. package/src/store.ts +2148 -0
  78. package/src/vec.ts +124 -0
  79. package/test/00-extract.test.mjs +151 -0
  80. package/test/01-floor.test.mjs +20 -0
  81. package/test/02-roundtrip.test.mjs +83 -0
  82. package/test/03-recall.test.mjs +98 -0
  83. package/test/04-think.test.mjs +627 -0
  84. package/test/05-concepts.test.mjs +84 -0
  85. package/test/08-storage.test.mjs +948 -0
  86. package/test/09-edges.test.mjs +65 -0
  87. package/test/11-universality.test.mjs +180 -0
  88. package/test/13-conversation.test.mjs +228 -0
  89. package/test/14-scaling.test.mjs +503 -0
  90. package/test/15-decomposition-gap.test.mjs +0 -0
  91. package/test/16-bridge.test.mjs +250 -0
  92. package/test/17-intelligence.test.mjs +240 -0
  93. package/test/18-alu.test.mjs +209 -0
  94. package/test/19-nd.test.mjs +254 -0
  95. package/test/20-stability.test.mjs +489 -0
  96. package/test/21-partial.test.mjs +158 -0
  97. package/test/22-multihop.test.mjs +130 -0
  98. package/test/23-rationale.test.mjs +316 -0
  99. package/test/24-generalization.test.mjs +416 -0
  100. package/test/25-embedding.test.mjs +75 -0
  101. package/test/26-cooperative.test.mjs +89 -0
  102. package/test/27-saturation-drop.test.mjs +637 -0
  103. package/test/28-unknowable.test.mjs +88 -0
  104. package/test/29-counterfactual.test.mjs +476 -0
  105. package/test/30-conflict-resolution.test.mjs +166 -0
  106. package/test/31-audit.test.mjs +288 -0
  107. package/test/32-confluence.test.mjs +173 -0
  108. package/test/33-multi-candidate.test.mjs +222 -0
  109. package/test/34-cross-region.test.mjs +252 -0
  110. package/tsconfig.json +16 -0
package/AGENTS.md ADDED
@@ -0,0 +1,470 @@
1
+ # AGENTS.md — the Sema development manual
2
+
3
+ The working manual for anyone (human or AI agent) changing Sema. It is organized
4
+ around the **engineering patterns the codebase runs on**: what each pattern is,
5
+ where it is applied, how to follow it, and what breaks if you don't. You should
6
+ be able to develop against this document alone; read
7
+ [HOW_IT_WORKS.md](HOW_IT_WORKS.md) only when you need the theory behind a
8
+ pattern, not to get work done.
9
+
10
+ ---
11
+
12
+ ## 1. Orientation
13
+
14
+ Sema is a deterministic reasoning engine with no ML runtime: a content-addressed
15
+ graph store (the knowledge), two approximate vector indexes over it (the search
16
+ accelerators), and a cost-based search that composes answers from stored facts
17
+ (the inference). Everything is plain TypeScript, CPU-only, with `node:sqlite` as
18
+ the only runtime dependency of the library.
19
+
20
+ ```bash
21
+ npm install # dev tooling + the parquet reader used by one example
22
+ npm run build # tsc → dist/
23
+ npm test # tsc && node --test test/**/*.test.mjs
24
+ npm run demo # example/demo.ts — the four-note README demo
25
+ ```
26
+
27
+ Hard facts you must not fight:
28
+
29
+ - **Node ≥ 22.5** (`node:sqlite`). No native add-ons. If you want a GPU, you are
30
+ working against the architecture.
31
+ - **ES modules with explicit `.js` extensions** in imports. Keep the convention
32
+ in new files.
33
+ - **Determinism is the product.** Same seed + same deposit order + same query ⇒
34
+ byte-identical answer. Tests pin it.
35
+
36
+ The mental model, top to bottom:
37
+
38
+ ```
39
+ mind/pipeline.ts the grounding decider: mechanisms compete on one cost scale
40
+ mind/mechanisms/* cover · cast · confluence · extraction · recall · alu
41
+ mind/* shared machinery: match/project, attention, recognition,
42
+ junction ascent, graph search, learning, rationale
43
+ store.ts AbstractStore: ALL domain logic of the DAG store
44
+ store-sqlite.ts the one concrete backend (thin SQL wrappers)
45
+ geometry.ts + vec/alphabet/sema
46
+ vectors, the fold, and every derived threshold
47
+ derive/ · alu/ · rabitq-hnsw/
48
+ firewalled sublibraries with their own READMEs and tests
49
+ ```
50
+
51
+ Lower layers never import higher ones. The three sublibraries import nothing
52
+ from the rest of Sema (the ALU may use `../bytes.ts` only); Sema reaches them
53
+ through narrow interfaces (`DeductionSystem`, `PipelineMechanism`,
54
+ `VectorDatabase`). A change that makes a sublibrary import mind code is
55
+ architecturally wrong, full stop.
56
+
57
+ ---
58
+
59
+ ## 2. The patterns
60
+
61
+ Each subsection: what the pattern is → where it is applied → how to follow it.
62
+
63
+ ### 2.1 Determinism as a contract
64
+
65
+ No `Math.random`, no `Date.now` in behaviour, no iteration over unordered
66
+ collections where order can reach output. All randomness flows from the config
67
+ `seed` (alphabet, keyring, index PRNGs). Every tie-break bottoms out in a fixed,
68
+ corpus-determined ordering — insertion order or lowest node id, with
69
+ **first-inserted** as the universal no-evidence fallback (last-inserted was once
70
+ used in one place; it was a bug).
71
+
72
+ _Follow it:_ when you introduce any choice among equals, pick the tie-break
73
+ explicitly and make it corpus-determined. If a test becomes flaky, you broke
74
+ this contract, not the test.
75
+
76
+ ### 2.2 Derived thresholds — never tuned
77
+
78
+ Every similarity/decision threshold is a **formula** over the vector dimension
79
+ D, the perception window W, or the corpus size N, defined once in
80
+ `src/geometry.ts` (`mergeThreshold`, `identityBar`, `reachThreshold`,
81
+ `significanceBar`, `estimatorNoise`, `conceptThreshold`, `consensusFloor`,
82
+ `dominates`, …). `src/config.ts` holds **capacities and budgets only** (cache
83
+ byte budgets, batch sizes, HNSW parameters, query k, ALU precision, the seed).
84
+
85
+ _Follow it:_ if you are about to add a tunable cutoff to config, derive it in
86
+ `geometry.ts` instead. A threshold knob is a design bug here — one was already
87
+ removed once. When a decision needs a scale, express it in D, W, or N.
88
+
89
+ ### 2.3 Exact decides, approximate proposes
90
+
91
+ Every score from the vector indexes (`resonate`, `resonateHalo`) is a RaBitQ
92
+ _estimate_. Identity is decided only by content-addressed lookup (`resolve`,
93
+ `findLeaf`, `findBranch`), never by `score >= threshold`. Scores rank candidates
94
+ and gate broad regions; bytes make decisions.
95
+
96
+ The same principle appears as **graded evidence ladders** — exact tier first,
97
+ distributional second, geometric last — in three places built on one shape:
98
+
99
+ - `locate` in `mind/match.ts`: exact bytes → halo role → gist.
100
+ - `alignGraded` in `mind/match.ts`: literal W-gram runs → halo-matched sites.
101
+ - `bridge` in `mind/resonance.ts`: junction containers by identity → edge
102
+ junctions → synonym junctions → whole-gist resonance as last resort.
103
+
104
+ _Follow it:_ never reorder a ladder's tiers, and never let an approximate tier
105
+ override an exact one. If you need a new matcher, add a tier to the shared
106
+ family (2.5), not a private score check.
107
+
108
+ ### 2.4 One cost currency
109
+
110
+ The graph search's cost ladder (`mind/graph-search.ts`, exported constants) is
111
+ the single pricing scheme of the whole mind:
112
+
113
+ ```
114
+ MICRO (1e-3) advance over recognised material; per-byte unit of the A* heuristic
115
+ STEP (1) follow one learned edge; one computed result; one projection
116
+ CONCEPT (10) a halo-mediated act (synonym hop, consensus climb)
117
+ PASS (1000/byte) carry a byte nothing explains
118
+ ```
119
+
120
+ Only the _ordering_ matters. The pipeline weighs whole mechanisms in the same
121
+ units: `weight = moves + PASS · unaccounted-bytes`, so a mechanism-level choice
122
+ and a byte-level choice are the same kind of decision.
123
+
124
+ _Follow it:_ place any new cost deliberately in the ordering; never make the A\*
125
+ heuristic exceed a real per-byte cost (admissibility breaks silently — answers
126
+ degrade, nothing errors). Never encode _policy_ as cost: "computation always
127
+ wins" is implemented by masking colliding sites in cover, not by pricing. Keep
128
+ policy in callers, the engine neutral.
129
+
130
+ ### 2.5 One factored machinery: match → project, under a gate
131
+
132
+ `mind/match.ts` is the shared family every generalising mechanism configures:
133
+ matchers (`locate`, `alignRuns`, `alignGraded`, `bestHaloMate`, `haloSiblings`,
134
+ `analogyStrength`) and projections (`follow`, `reverseContext`, `project`,
135
+ `conceptHop`). `mind/traverse.ts` owns the graph readings (`edgeAncestors`,
136
+ `reachOf`, `chooseNext`/`chooseAmong`, `guidedFirst`, `leadsSomewhere`) and the
137
+ corpus scale (`corpusN`, `hubBound`, `hubCap`).
138
+
139
+ _Follow it:_ before writing a new generalising mechanism, express it as a
140
+ (matcher, direction, gate) triple. If those already exist, the mechanism is a
141
+ configuration — write only the configuration. If it genuinely needs a new
142
+ matcher or projection, add it **to the shared family** with a derived gate
143
+ (2.2), never as a private helper. A mechanism file that re-implements locating,
144
+ aligning, edge-following to a fixpoint, predecessor-picking, or fan-out capping
145
+ is reintroducing duplication that was deliberately removed.
146
+
147
+ Related single-definition contracts (define once, import everywhere):
148
+
149
+ - `canonical.ts` — the write/read contract for canonical segmentation
150
+ (`canonicalWindows`, `chainReach`, `leafIdRun`, `leafIdPrefix`, `windowIds`).
151
+ Learning writes through it; recognition and attention read through it.
152
+ Changing one side means changing this file — drift between sides breaks
153
+ canonical recognition with **no type error**.
154
+ - `junction.ts` — the content-addressed "which learnt whole contains these two
155
+ forms?" ascent, shared by the bridge and cross-region attention, with its
156
+ per-response `WalkCache` and once-per-candidate seed computation.
157
+ - `joinWithBridge` (`resonance.ts`) — the one out-of-search way to join two
158
+ answer spans; it emits a `bridgeMiss` trace step on a bare join.
159
+ - `guidedFirst` (`traverse.ts`) — the one answer-shaped "what does this lead
160
+ to?" read (guided pick merged with the first-inserted fallback).
161
+ - `leadsSomewhere` (`traverse.ts`) — the one admission predicate for recognition
162
+ sites (edge-or-halo, via existence probes).
163
+ - `isChunk` (`sema.ts`) — the one "children are all leaves" predicate.
164
+
165
+ ### 2.6 The mechanism market (the free-will architecture)
166
+
167
+ Every grounding mechanism — including the ALU and user extensions — implements
168
+ the same interface, `PipelineMechanism` (`mind/pipeline-mechanism.ts`): optional
169
+ `parse` (authoritative computed spans, collected before anything else), `floor`
170
+ (an admissible lower bound, or `null` when the mechanism structurally cannot
171
+ fire), and `run` (candidate answers). The decider in `mind/pipeline.ts`
172
+ (`think`) holds a plain list (`defaultMechanisms`: cover, cast, confluence,
173
+ extraction, recall, plus the ALU and any user mechanisms) and never branches on
174
+ which mechanism it is holding.
175
+
176
+ Four constraints make the market honest — verify all four for anything you add:
177
+
178
+ 1. **Decoupled.** Zero cross-imports between mechanism files. Adding one never
179
+ touches another. No mechanism asks "did an extension already decide?" — it
180
+ only asks "can I still beat the incumbent?".
181
+ 2. **Declared competence.** Gates are binary structural preconditions checked
182
+ inside `floor`/`run` (query length, anchor shape, weave existence), never
183
+ learned scores — so the rationale states exactly why a mechanism abstained.
184
+ 3. **Visible budget.** Every corpus-scale loop is capped at a named constant
185
+ (`√N` via `hubBound`, `k = 2·recallQueryK`), enforced at the store level
186
+ (2.8).
187
+ 4. **Evidence travels.** Every candidate carries `accounted` (query spans its
188
+ structural evidence explains), `moves` (its acts, priced on the ladder), and
189
+ `unexplained` (a diagnostic label). The decider sees only weights.
190
+
191
+ Two disciplines inside the loop:
192
+
193
+ - **Admissible-floor pruning.** `floor` runs for every mechanism in list order,
194
+ before any `run`; `run` fires only if the floor can still beat the incumbent
195
+ (`worthRunning`). Cover runs first so a computed span's near-zero cost prunes
196
+ everything after it through this same mechanism — not a special case.
197
+ - **Investment discipline.** `worthRunning` is also passed _into_ `floor`: a
198
+ floor that would first-touch an expensive shared analysis (the climb, the
199
+ weave) checks its cheapest possible bound against the incumbent _before_
200
+ paying, and returns the uninvested bound when it already loses. Never compute
201
+ a shared analysis just to discard it.
202
+
203
+ Evidence accounting rules that bite:
204
+
205
+ - **Read-out content is selectively accounted.** Extraction's located frames are
206
+ always evidence; the span between them counts only when _both_ borders were
207
+ located. An open-ended read is content-novel and is priced by exclusion
208
+ (PASS/byte), like the cover's carried literals.
209
+ - **Reverse reading is not derivation.** A `reverseContext` projection produces
210
+ bytes but explains nothing forward: `accounted = []`, weight ≈ PASS·|query|.
211
+ It is the designated last resort by arithmetic, not by rule.
212
+ - `unexplained`, `narrowDecision`, and `thinGrounding` are **observational
213
+ only** — they appear in the trace and never alter the decision.
214
+
215
+ ### 2.7 Two measures of commonality — pick the right population
216
+
217
+ "Is this content discriminative?" has two formally independent answers, and
218
+ using the wrong one is a semantic bug the type system cannot catch:
219
+
220
+ - **Corpus-global** — reference set: all learned contexts. Tooling: `reachOf` +
221
+ `dominates` (+ `corpusN`). Used by the climb's IDF weighting and confluence's
222
+ filler/scaffolding gate. Answers "does this discriminate anything in the
223
+ store?"
224
+ - **Weave-local** — reference set: the structures aligned with _this query_.
225
+ Tooling: the `depth[]` array from `alignGraded` + `MIN_WEAVE` + `dominates`.
226
+ Used by CAST's frame gate. Answers "does this discriminate among the
227
+ structures this query activates?"
228
+
229
+ _Follow it:_ when adding a gate on "shared vs. discriminative", write down which
230
+ population your question is about before choosing the tool. Substituting one for
231
+ the other in CAST misfires on reordered single-fact queries (test 17 pins this).
232
+
233
+ ### 2.8 Bounded reads — the cap lives in the store
234
+
235
+ No per-query read may grow with the corpus. The cap is √N (`hubBound(ctx)`,
236
+ derived from `corpusN(ctx)` — the one definition of corpus size, floored at 2).
237
+ Crucially, the cap is enforced **at the store level**:
238
+
239
+ - LIMITed reads: `nextFirst`, `prevFirst`, `parentsFirst`, `containersSlice`. In
240
+ an adapter these must be real `LIMIT ?` statements — never "materialise then
241
+ slice". Reading `hubBound + 1` parents decides "hub or not" exactly.
242
+ - Existence probes: `hasNext`, `hasParents`, `hasContainers`, `hasHalo`,
243
+ `prevCount` — indexed point probes that never decode vectors or unpack blobs.
244
+ Use them for every "does this lead anywhere?" question instead of
245
+ `next(id).length > 0`.
246
+ - `chainRun` climbs transparent scaffolding chains in one bounded read.
247
+ - The full materialising reads (`next`, `prev`, `parents`, `containers`) exist
248
+ for maintenance and inspection only. Keep them off hot paths.
249
+
250
+ _Follow it:_ any new fan-out walk uses `hubBound`/`hubCap` — do not invent a
251
+ second convention, and do not call `edgeSourceCount()` or
252
+ `Math.ceil(Math.sqrt(...))` inline.
253
+
254
+ ### 2.9 Template-method store
255
+
256
+ `store.ts` (`AbstractStore`) owns **all** domain logic: exact dedup,
257
+ byte-verified near-dedup, lazy gist indexing and bridge promotion, halo
258
+ quantization and exact in-session accumulators, containment buffering, write
259
+ batching, LRU budgets, compaction cadence. `store-sqlite.ts` implements only the
260
+ abstract `_db*`/`_vec*` methods as thin statement wrappers.
261
+
262
+ _Follow it:_ a new backend subclasses `AbstractStore` and implements the
263
+ abstract methods — nothing else. If you find yourself re-implementing dedup or
264
+ indexing logic in an adapter, stop. Facts an adapter (and any store caller) must
265
+ respect:
266
+
267
+ - Branch ids are dense non-negative integers minted in order, never deleted.
268
+ Single-byte leaves are **implicit negative ids** (−256…−1) with no DB row —
269
+ id-iterating code must handle both ranges.
270
+ - Flat branches (all-leaf children) are stored as raw bytes with an empty kids
271
+ blob as marker (`flatKidsBytes`/`flatBytesKids`).
272
+ - `bytes()`/`bytesPrefix()` return arrays **shared with caches — never mutate**;
273
+ copy first.
274
+ - `contentLen(id, cap?)` reads a node's byte length; pass `cap` when exact
275
+ length beyond a bound doesn't matter.
276
+ - Maintenance entry points (`compactContentIndex`, `repairContentIndex`) are
277
+ batch operations for checkpoints, never the hot path.
278
+
279
+ ### 2.10 The async/sync seam and pre-resolution
280
+
281
+ Perception, recognition, and the graph search are **synchronous**; anything
282
+ touching the ANN indexes is **async**. A synchronous consumer that needs
283
+ resonance uses _pre-resolution_: gather the async answers first (concept
284
+ siblings, connectors, ALU operand meanings), hand them in as maps
285
+ (`resolveConcepts`/`resolveConnectors` in `mechanisms/cover.ts` are the models).
286
+ Do not try to make the search async.
287
+
288
+ ### 2.11 Per-response memoization
289
+
290
+ Asking never writes, which is the only reason per-response memos are sound.
291
+ `Precomputed` (`pipeline-mechanism.ts`) is the shared response-scoped container:
292
+ eager fields (recognition, computed spans, guide) plus **lazily-cached methods**
293
+ for expensive analyses (`attention()` — the consensus climb, `weave()`,
294
+ `spanShapedOf`, `queryWindows`, `windowsOf`, `reachMemo`) — each computed at
295
+ most once, shared by mechanisms and post-grounding stages, and never computed if
296
+ nobody asks.
297
+
298
+ _Follow it:_ an expensive analysis a new mechanism needs goes on `Precomputed`
299
+ as a lazy method, not inside the mechanism. Mind-level memos (`climbMemo`,
300
+ `recogniseMemo`, `perceiveMemo`, `_edgeChoice`, `_gistCache`) are created in
301
+ `beginResponse()` and torn down in `endResponse()` — a new memo must be added to
302
+ both. `climbMemo`/`recogniseMemo` are bypassed while a rationale trace is
303
+ attached (every mechanism must emit its steps); `perceiveMemo` is not.
304
+ Consequence: **never benchmark with a trace attached.**
305
+
306
+ ### 2.12 Caches are budgets, not correctness
307
+
308
+ Every in-memory acceleration structure is a `BoundedMap` with a byte budget; a
309
+ miss re-derives from durable state. The degradation order is fixed: what is lost
310
+ under pressure is always speed or reach (a re-perception, a duplicate probe,
311
+ reduced resonance until repair), never identity, reconstruction, or a learned
312
+ relation.
313
+
314
+ _Follow it:_ new caches get budgets and a re-derivation path. If memory grows,
315
+ look for something bypassing a budget — not for a leak in the DAG (nodes are
316
+ meant to accumulate).
317
+
318
+ ### 2.13 Honest degradation, visible failure
319
+
320
+ Nothing degrades silently: counters (`danglingReads`, `compactFailures`), trace
321
+ steps (`bridgeMiss`, `narrowDecision`, `thinGrounding`), the `echoed` flag on
322
+ recall's last tier, `recall-echo` provenance. Empty results are legitimate
323
+ outputs (silence), not errors.
324
+
325
+ _Follow it:_ when your code can degrade, emit a counter or trace step. And mind
326
+ the classic trap: **empty bytes are truthy** — `Uint8Array(0)` passes
327
+ `if (answer)`; always test `.length`.
328
+
329
+ ### 2.14 Comment style
330
+
331
+ Comments state _constraints and failure modes_ — "this guard exists because X
332
+ breaks without it", often naming the test that pins the behaviour — never
333
+ narration of what the next line does. Two standing examples in
334
+ `graph-search.ts`: the `hasHalo` guard on fusing completed rewrites (answer
335
+ corruption via phrase-interior chunks) and the `couldGrow` liveness rule (O(N²)
336
+ chart growth). When you fix a subtle bug, leave the constraint behind, not the
337
+ story of the fix.
338
+
339
+ ---
340
+
341
+ ## 3. Where things live
342
+
343
+ | Concept | File(s) |
344
+ | :-------------------------------------------- | :--------------------------------------------------------------------- |
345
+ | Public surface / assembly | `src/index.ts`, `src/mind/mind.ts` |
346
+ | Config (capacities, budgets, seed) | `src/config.ts` |
347
+ | Derived thresholds, river fold, Hilbert | `src/geometry.ts` |
348
+ | Vector primitives, alphabet, node/fold types | `src/vec.ts`, `src/alphabet.ts`, `src/sema.ts` |
349
+ | Store domain logic / SQLite adapter | `src/store.ts`, `src/store-sqlite.ts` |
350
+ | Mechanism contract + shared `Precomputed` | `src/mind/pipeline-mechanism.ts` |
351
+ | The grounding decider (`think`) | `src/mind/pipeline.ts` |
352
+ | Grounding mechanisms (one file each) | `src/mind/mechanisms/{cover,cast,confluence,extraction,recall,alu}.ts` |
353
+ | Weighted deduction system + cost ladder | `src/mind/graph-search.ts` (engine in `src/derive/`) |
354
+ | Match/project family | `src/mind/match.ts` |
355
+ | Graph traversal, corpus scale, disambiguators | `src/mind/traverse.ts` |
356
+ | Consensus climb + cross-region attention | `src/mind/attention.ts` |
357
+ | Recognition / canonical contract | `src/mind/recognition.ts`, `src/mind/canonical.ts` |
358
+ | Junction ascent (bridge + attention share) | `src/mind/junction.ts`, `src/mind/resonance.ts` |
359
+ | Learning / ingestion / training cache | `src/mind/learning.ts`, `src/ingest-cache.ts` |
360
+ | Post-grounding (reason, fuse, articulate) | `src/mind/reasoning.ts`, `src/mind/articulation.ts` |
361
+ | Rationale / trace | `src/mind/rationale.ts`, `src/mind/trace.ts` |
362
+ | Extension host types | `src/extension.ts` |
363
+ | Sublibraries (own READMEs, own tests) | `src/derive/`, `src/alu/`, `src/rabitq-hnsw/` |
364
+
365
+ Mind functions are **free functions over `MindContext`** (`mind/types.ts`), not
366
+ methods — `mind.ts` is a thin assembly that implements the context and
367
+ delegates. Follow that shape: it keeps every mechanism testable in isolation
368
+ with no hidden `this` state.
369
+
370
+ ---
371
+
372
+ ## 4. Recipes
373
+
374
+ ### Add a grounding mechanism or extension
375
+
376
+ Implement `PipelineMechanism`: `floor` returns an admissible bound or `null`
377
+ (structurally can't fire); `run` returns candidates with `bytes`, `accounted`,
378
+ `moves`, `unexplained`; add `parse` only if you compute authoritative spans (the
379
+ ALU's `aluToMechanism` in `mechanisms/alu.ts` is the reference). Register with
380
+ `new Mind({ mechanismFactories: [host => yourMechanism(host)] })` (or
381
+ `mechanisms: [...]` if no host is needed); reach meaning only through the
382
+ `ExtensionHost`. Verify the four market constraints (2.6). You never touch
383
+ `think()` or another mechanism's file.
384
+
385
+ ### Add an ALU operation
386
+
387
+ One declarative `registry.derive(name, arity, surfaceForms, body)` in the
388
+ relevant `src/alu/src/kernel-*.ts`; the body composes existing ops. Scalar ops
389
+ broadcast over n-d automatically. No parser, search, or mind edits — see
390
+ `src/alu/README.md`.
391
+
392
+ ### Add a deduction rule
393
+
394
+ Rules live in `GraphSearch` (`coverRules`/`formRules`/`outRules`/`fuse`). Place
395
+ its cost in the ladder deliberately (2.4), emit it lazily from the item kind
396
+ that triggers it, keep the heuristic admissible, extend `classifyMove` (the
397
+ single rule-shape → move-name mapping for the rationale), and add a
398
+ rationale-visible test. Async data is pre-resolved in the pipeline (2.10).
399
+
400
+ ### Add a store backend
401
+
402
+ Subclass `AbstractStore`; implement the `_db*`/`_vec*` methods as thin statement
403
+ wrappers (`SQliteStore` is the template); LIMITed variants must be real `LIMIT`
404
+ queries and existence probes real point probes (2.8, 2.9). Run the full suite
405
+ with your store substituted.
406
+
407
+ ### Add a modality
408
+
409
+ Perception consumes byte streams. Grid-shaped data: build a `Grid`
410
+ (`{width, height, channels, data}` or n-dimensional `dims`) — `geometry.ts`
411
+ Hilbert-linearizes it; `Grid[]` stacks frames. Anything else: produce a
412
+ `Uint8Array` with a deterministic, locality-preserving ordering. Nothing
413
+ downstream changes.
414
+
415
+ ### Debug an answer
416
+
417
+ ```ts
418
+ const r = await mind.respond(query, (rationale) => {
419
+ console.dir(rationale, { depth: null }); // every step, cost, data-flow edge
420
+ });
421
+ console.log(r.provenance); // cast | join | cover | extract | recall | recall-echo
422
+ ```
423
+
424
+ Read top-down: which mechanism fired (and why the others abstained), what
425
+ recognition found, how the climb voted, which edges were followed
426
+ (`disambiguate` steps carry the evidence). `recall-echo` means "nearest stored
427
+ form, not a derived fact". Remember traced responses bypass memos (2.11).
428
+
429
+ ### Train at scale
430
+
431
+ Use `CachedIngest` (`ingest-cache.ts`) as a drop-in for `mind.ingest` — it
432
+ memoises perceive+intern of repeated inputs and routes through the same
433
+ `dispatchIngest` as the direct path (shape detection can't drift). Call
434
+ `store.commit()` at checkpoints; run `compactContentIndex` /
435
+ `repairContentIndex` post-training if eviction was heavy. See
436
+ `example/train_base.ts`. Profiling note: the first `resonate` after a big ingest
437
+ pays the pending index flush; the dominant query-side ANN cost is connector
438
+ pre-resolution (bounded by recognised-site count — don't add another loop over
439
+ site pairs).
440
+
441
+ ---
442
+
443
+ ## 5. Testing norms
444
+
445
+ Tests are plain `node:test` suites in `test/*.test.mjs`, numbered by theme, run
446
+ against the built `dist/` (`npm test`; one suite:
447
+ `node --test test/22-multihop.test.mjs` after `tsc`).
448
+
449
+ - New behaviour ⇒ a test in the matching numbered suite, or a new numbered
450
+ suite.
451
+ - Many tests pin **contracts that look like implementation details** (the bridge
452
+ tier order, extraction's two span-shape readings, determinism, honest
453
+ silence). A "simplification" that fails an existing test is wrong until you
454
+ can argue the _test_ is wrong — several guards exist precisely because a
455
+ plausible simplification once failed a dozen suites.
456
+ - Sublibraries test themselves (`src/{alu,derive,rabitq-hnsw}/test/`) with zero
457
+ Sema dependency. Keep it so.
458
+ - Performance claims are tested (the rabitq-hnsw benchmark asserts sub-linear
459
+ scaling and compression). Changing index behaviour means running it.
460
+
461
+ ---
462
+
463
+ ## 6. Dependencies and licensing
464
+
465
+ PolyForm Noncommercial 1.0.0 with separate commercial licensing (see
466
+ [LICENSE.md](LICENSE.md), [COMMERCIAL-LICENSE.md](COMMERCIAL-LICENSE.md),
467
+ [TRADEMARKS.md](TRADEMARKS.md), [CONTRIBUTING.md](CONTRIBUTING.md)). Do not
468
+ vendor code under licenses incompatible with dual distribution, and do not add
469
+ runtime dependencies casually — the near-zero-dependency footprint is a product
470
+ feature.
package/AUTHORS.md ADDED
@@ -0,0 +1,12 @@
1
+ # Authors
2
+
3
+ Sema is maintained by:
4
+
5
+ - Henrique Viana (creator)
6
+
7
+ Unless otherwise stated, copyright belongs to Authors.
8
+
9
+ # Supporters
10
+
11
+ - Marcelo Oliveira dos Reis
12
+ - Rogerio Nascimento
@@ -0,0 +1,19 @@
1
+ # Commercial License
2
+
3
+ Commercial use of Sema requires a separate paid license.
4
+
5
+ The public license only allows non-commercial use under the PolyForm
6
+ Noncommercial License 1.0.0.
7
+
8
+ Commercial use includes, but is not limited to:
9
+
10
+ - use by a company, organization, agency, freelancer, or consultant;
11
+ - use to provide paid services;
12
+ - use to serve clients or customers;
13
+ - use in a SaaS, hosted product, or commercial platform;
14
+ - use inside a paid or revenue-generating product;
15
+ - use to reduce business costs or support business operations.
16
+
17
+ To request a commercial license, contact:
18
+
19
+ hv5088@gmail.com
@@ -0,0 +1,15 @@
1
+ # Contributing
2
+
3
+ By contributing to Sema, you agree that:
4
+
5
+ - you have the legal right to submit your contribution;
6
+ - your contribution does not violate third-party rights;
7
+ - your contribution will be licensed under the PolyForm Noncommercial License
8
+ 1.0.0;
9
+ - this project is source-available, not formally open source;
10
+ - commercial use requires a separate paid license;
11
+ - you will not submit secrets, credentials, private keys, or confidential data.
12
+
13
+ All commits should be signed off:
14
+
15
+ git commit -s