@hviana/sema 0.5.7 → 0.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/AGENTS.md +23 -0
  2. package/DATASETS.md +159 -0
  3. package/HOW_IT_WORKS.md +74 -0
  4. package/README.md +12 -0
  5. package/dist/example/train_base.d.ts +73 -3
  6. package/dist/example/train_base.js +1000 -49
  7. package/dist/src/geometry.d.ts +20 -0
  8. package/dist/src/geometry.js +22 -0
  9. package/dist/src/mind/articulation.js +15 -2
  10. package/dist/src/mind/attention.d.ts +6 -0
  11. package/dist/src/mind/attention.js +44 -4
  12. package/dist/src/mind/learning.js +250 -3
  13. package/dist/src/mind/mechanisms/cast.js +45 -1
  14. package/dist/src/mind/mind.d.ts +6 -1
  15. package/dist/src/mind/mind.js +14 -2
  16. package/dist/src/mind/reasoning.js +59 -5
  17. package/dist/src/mind/recognition.js +29 -3
  18. package/dist/src/mind/traverse.d.ts +34 -0
  19. package/dist/src/mind/traverse.js +42 -0
  20. package/dist/src/store-sqlite.d.ts +4 -0
  21. package/dist/src/store-sqlite.js +47 -0
  22. package/dist/src/store.d.ts +7 -0
  23. package/example/train_base.ts +1193 -46
  24. package/jsr.json +1 -1
  25. package/package.json +1 -1
  26. package/src/geometry.ts +23 -0
  27. package/src/mind/articulation.ts +16 -2
  28. package/src/mind/attention.ts +54 -1
  29. package/src/mind/learning.ts +253 -4
  30. package/src/mind/mechanisms/cast.ts +48 -1
  31. package/src/mind/mind.ts +12 -1
  32. package/src/mind/reasoning.ts +64 -5
  33. package/src/mind/recognition.ts +29 -3
  34. package/src/mind/traverse.ts +48 -0
  35. package/src/store-sqlite.ts +53 -0
  36. package/src/store.ts +28 -0
  37. package/test/29-counterfactual.test.mjs +43 -6
  38. package/test/76-type-level-company.test.mjs +342 -0
  39. package/test/77-company-saturation.test.mjs +302 -0
  40. package/test/78-atom-hub-recognition-cliff.test.mjs +135 -0
  41. package/test/84-composed-answer-honesty.test.mjs +136 -0
  42. package/test/85-answered-directly.test.mjs +126 -0
  43. package/test/86-cast-voices-committed.test.mjs +164 -0
  44. package/test/87-codominant-commitment.test.mjs +250 -0
@@ -39,7 +39,6 @@ export async function reason(ctx, query, answer, preConsumed, pre, voiced = [])
39
39
  const qId = pre.queryResolved;
40
40
  if (qId !== null && ctx.store.prevCount(qId) > 0)
41
41
  return answer;
42
- const consumed = new Set();
43
42
  // Consume a node and its neighbours for pivot-cycle prevention — CAPPED at
44
43
  // the hub bound, via the store's LIMITed edge reads: a common continuation's
45
44
  // reverse fan-in (and a hub context's forward fan-out) is corpus-sized, and
@@ -48,11 +47,64 @@ export async function reason(ctx, query, answer, preConsumed, pre, voiced = [])
48
47
  // read order); a pivot suppressed only by a beyond-cap neighbour may now
49
48
  // fire — the same visibility trade chooseNext documents.
50
49
  const bound = hubBound(ctx);
51
- const consumeNode = (id) => {
50
+ // ANSWERED DIRECTLY the echo guard's other half, and the same principle:
51
+ // the QUERY's own position in the graph, not the answer's content, says the
52
+ // read-out is complete. Above: the query is itself a learnt CONTINUATION.
53
+ // Here: the query is a learnt CONTEXT and the grounded answer is one of ITS
54
+ // OWN continuations. Either way the question was answered directly and there
55
+ // is nothing left to chain for.
56
+ //
57
+ // Every stopping condition in the loop below judges the ANSWER (`consumed` /
58
+ // `restatesQuery` / `bytesEqual`); none asks whether the QUESTION was
59
+ // satisfied. So a single-hop question whose answer happens to name another
60
+ // learnt context extends past a correct answer and REPLACES it:
61
+ //
62
+ // asked "<subj> father"
63
+ // hop 1 "The father of <subj> is Ernest I of Anhalt-Dessau." <- correct
64
+ // pivot "Ernest I of Anhalt-Dessau" <- a learnt context too
65
+ // got "The date of death of Ernest I of Anhalt-Dessau is 12 June 1516."
66
+ //
67
+ // Any store holding a bare-entity context alongside a relation fact has that
68
+ // shape; it is not exotic.
69
+ //
70
+ // Checked ONCE, before the loop, and ahead of BOTH extension branches:
71
+ // `absorbForward` extends the answer too, and nothing about the defect is
72
+ // specific to pivoting, so a guard between them would gate one arbitrary half
73
+ // of the same step. Hop 0 is also the only hop at which the question can be
74
+ // answered directly at all — after a hop, `cur` is no longer the query's own
75
+ // continuation, so re-testing per hop could only cost reads.
76
+ //
77
+ // Read from the ANSWER's side (`prevFirst`) rather than the query's
78
+ // (`nextFirst`). Same relation, but a CONTEXT's fan-out is hub-sized while
79
+ // this is one answer's establishing-context fan-in. Both the resolve and the
80
+ // reverse read are exactly what hop 0 of the loop below would perform, so
81
+ // they are computed ONCE here and handed down (`groundedId`, `groundedPrev`)
82
+ // — the guard then costs nothing when it does not fire. Stated because the
83
+ // naive placement does NOT: `resolve` re-folds the answer bytes on every call
84
+ // (no memo) and `prevFirst` is a direct read (no memo), so a guard that
85
+ // recomputed them would add one fold plus one √N-bounded read per ask.
86
+ // The √N cap carries the file-wide visibility trade, and fails SAFE in the
87
+ // direction that matters: a missed guard costs an over-extended answer, never
88
+ // a suppressed chain.
89
+ //
90
+ // A genuine multi-hop query is not a deposited context at all ("What is the
91
+ // capital of the country of Eiffel Tower?" resolves to nothing), so this can
92
+ // never gate a real chain.
93
+ const groundedId = resolve(ctx, answer);
94
+ const groundedPrev = groundedId === null
95
+ ? null
96
+ : ctx.store.prevFirst(groundedId, bound);
97
+ if (qId !== null && groundedPrev !== null && groundedPrev.includes(qId)) {
98
+ return answer;
99
+ }
100
+ const consumed = new Set();
101
+ /** `prev` lets a caller hand in an already-read reverse-edge list — hop 0
102
+ * reuses the guard's, above, instead of re-reading it. */
103
+ const consumeNode = (id, prev) => {
52
104
  if (id === null)
53
105
  return;
54
106
  consumed.add(id);
55
- for (const p of ctx.store.prevFirst(id, bound))
107
+ for (const p of prev ?? ctx.store.prevFirst(id, bound))
56
108
  consumed.add(p);
57
109
  };
58
110
  const consumeAll = (id) => {
@@ -92,8 +144,10 @@ export async function reason(ctx, query, answer, preConsumed, pre, voiced = [])
92
144
  let t;
93
145
  const startedFrom = answer;
94
146
  for (let hop = 0; hop < ctx.cfg.recallQueryK; hop++) {
95
- const curId = resolve(ctx, cur);
96
- consumeNode(curId);
147
+ // Hop 0's `cur` IS `answer`, so the guard above already resolved it and
148
+ // read its reverse edges — reuse both rather than repeat them.
149
+ const curId = hop === 0 ? groundedId : resolve(ctx, cur);
150
+ consumeNode(curId, hop === 0 ? groundedPrev ?? undefined : undefined);
97
151
  // Forward-absorb: follow only UNCONSUMED continuations. The gate below
98
152
  // checks an unconsumed edge EXISTS, but follow()'s chooseNext knows
99
153
  // nothing of `consumed` and may still walk to a consumed fixpoint —
@@ -6,7 +6,7 @@
6
6
  // segment — leaf-parent segmentation using the geometry's own groupings.
7
7
  import { rItem } from "./trace.js";
8
8
  import { canonResolve, foldTree, gistOf, latin1Key, perceive, resolve, } from "./primitives.js";
9
- import { atomIsHub, corpusN, leadsSomewhere } from "./traverse.js";
9
+ import { atomIsHub, bearsEdge, corpusN, leadsSomewhere } from "./traverse.js";
10
10
  import { chainReach, leafIdAt, leafIdRun } from "./canonical.js";
11
11
  import { canonHash } from "../canon.js";
12
12
  import { isChunk } from "../sema.js";
@@ -585,6 +585,31 @@ function recogniseImpl(ctx, bytes) {
585
585
  // "Eiffel Tower" site vanished with it). The premise is wrong but the
586
586
  // trust it stood in for is real; a replacement signal is still open work.
587
587
  // See bench/README.md.
588
+ //
589
+ // THE REPLACEMENT SIGNAL (2026-08-13): `leadsSomewhere` on the BYTE-EXACT
590
+ // branch the chain already found. The blanket off-boundary suppression is
591
+ // a decision that CHANGES WITH CORPUS SIZE — `atomsAreHubs` flips at
592
+ // N = 4096 (atomReach = ⌈N·W/256⌉ exceeds √N there) — so a store crossing
593
+ // that point silently loses interior sites it used to have. Measured: with
594
+ // the two-hop chain deposited, `recognise("The country of Eiffel Tower is
595
+ // France.")` yields 4 sites including `France` at N = 3920 and 2 sites
596
+ // without it at N = 4227; the pivot dies with the site and multi-hop goes
597
+ // silent from there up (the trained store is N = 325,615).
598
+ //
599
+ // The honest gate is the one `emit` already applies, moved EARLIER and paid
600
+ // for with existence probes instead of a fold: `findBranch` has already
601
+ // proved these bytes are a stored branch, so the only remaining question is
602
+ // whether that branch is a deposited whole (bears an edge or a halo) or an
603
+ // interned fragment. "hi" out of "W[hi]ch" leads nowhere and is still
604
+ // suppressed; `France` bears both and is admitted. Structural, not scalar
605
+ // — no constant enters and nothing reads N, so the verdict no longer moves
606
+ // when the corpus grows.
607
+ //
608
+ // COST: `bearsEdge` is the response-MEMOISED edge probe, not the full
609
+ // `leadsSomewhere` — its uncached `hasHalo` tier took haloProbes from 922 to
610
+ // 9,144 on a nine-query battery over the trained store, which is not a price
611
+ // this pass may charge. `emit` still applies the full predicate, so this is
612
+ // a pre-filter that never widens what is admitted.
588
613
  const tryChain = (p, maxIds, boundary) => {
589
614
  const first = leafFrom(p);
590
615
  if (!first)
@@ -599,9 +624,10 @@ function recogniseImpl(ctx, bytes) {
599
624
  break;
600
625
  ids.push(nx.id);
601
626
  pos = nx.end;
602
- if (store.findBranch(ids) === null)
627
+ const branch = store.findBranch(ids);
628
+ if (branch === null)
603
629
  continue;
604
- if (!boundary && atomsAreHubs)
630
+ if (!boundary && atomsAreHubs && !bearsEdge(ctx, branch))
605
631
  continue;
606
632
  const id = resolveSpan(p, pos);
607
633
  if (id === null || id === prevId)
@@ -42,6 +42,22 @@ export declare function atomReach(ctx: MindContext, contextCount: number): numbe
42
42
  * atom votes and is recognised exactly as any stored form; above it the
43
43
  * alphabet is scaffolding everywhere and abstains. */
44
44
  export declare function atomIsHub(ctx: MindContext, contextCount: number): boolean;
45
+ /** Cached "does this node bear a continuation edge?" — the CHEAP half of
46
+ * {@link leadsSomewhere}, exported for hot paths that must PRE-FILTER a
47
+ * candidate before paying for a fold and cannot afford the halo tier.
48
+ *
49
+ * `leadsSomewhere`'s second tier (`hasHalo`) is deliberately uncached — one
50
+ * indexed point probe per candidate, which is right where candidates are
51
+ * already few. On recognition's off-boundary chain pass they are not few:
52
+ * using the full predicate there took haloProbes from 922 to 9,144 on a
53
+ * nine-query battery over the trained store. The edge tier alone is memoised
54
+ * for the response, so it is ~free, and a node bearing an edge is exactly the
55
+ * "deposited whole, not an interned fragment" claim that pass needs.
56
+ *
57
+ * Strictly NARROWER than `leadsSomewhere` — a halo-only node reads false — so
58
+ * it is sound as a pre-filter before a consumer that applies the full
59
+ * predicate, and never as a replacement for it. */
60
+ export declare function bearsEdge(ctx: MindContext, id: number): boolean;
45
61
  /** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
46
62
  * The admission predicate recognition filters sites with (HOW_IT_WORKS
47
63
  * §15.3): a form that leads nowhere contributes nothing to any derivation.
@@ -80,6 +96,24 @@ export declare function hubCap<T>(ctx: MindContext, ids: readonly T[]): readonly
80
96
  * descent. Used by articulation to keep a voice from revoicing a fragment
81
97
  * OF that voice. */
82
98
  export declare function contains(ctx: MindContext, ancestor: number, descendant: number): boolean;
99
+ /** Whether a continuation edge joins the two forms, in either direction —
100
+ * the EXACT half's veto on calling them synonyms.
101
+ *
102
+ * Halos measure company, and the strongest company any two forms can keep is
103
+ * standing next to each other: a question and its answer co-occur in every
104
+ * episode that taught the pair, so their halos SHOULD be similar, and on a
105
+ * conversational store they are (measured on the CONV fixture: consecutive
106
+ * turns at 0.809 against a 0.516 concept threshold). A gate reading halo
107
+ * cosine alone therefore reads adjacency as synonymy and revoices an answer
108
+ * in the words of the question it answers — "it hangs in madrid" spliced back
109
+ * into "where is it kept now". The distributional layer cannot tell the two
110
+ * relations apart, because to it they are the same observation; the exact
111
+ * half can, for free, because it stored the edge. §4.1's division of labour
112
+ * exactly: approximate proposes, exact decides.
113
+ *
114
+ * Read LIMITed in both directions at the hub bound — a common continuation's
115
+ * fan-in is corpus-sized, and no single decision may scale with it. */
116
+ export declare function answers(ctx: MindContext, a: number, b: number): boolean;
83
117
  /** The best-scoring item by cosine against `query`, among items scoring at
84
118
  * or above `threshold` — the shared arg-max every Pattern-A "which of these
85
119
  * resonates best" decision reduces to. `strict` picks the tie-break a
@@ -407,6 +407,24 @@ export function atomReach(ctx, contextCount) {
407
407
  export function atomIsHub(ctx, contextCount) {
408
408
  return atomReach(ctx, contextCount) > boundFor(contextCount);
409
409
  }
410
+ /** Cached "does this node bear a continuation edge?" — the CHEAP half of
411
+ * {@link leadsSomewhere}, exported for hot paths that must PRE-FILTER a
412
+ * candidate before paying for a fold and cannot afford the halo tier.
413
+ *
414
+ * `leadsSomewhere`'s second tier (`hasHalo`) is deliberately uncached — one
415
+ * indexed point probe per candidate, which is right where candidates are
416
+ * already few. On recognition's off-boundary chain pass they are not few:
417
+ * using the full predicate there took haloProbes from 922 to 9,144 on a
418
+ * nine-query battery over the trained store. The edge tier alone is memoised
419
+ * for the response, so it is ~free, and a node bearing an edge is exactly the
420
+ * "deposited whole, not an interned fragment" claim that pass needs.
421
+ *
422
+ * Strictly NARROWER than `leadsSomewhere` — a halo-only node reads false — so
423
+ * it is sound as a pre-filter before a consumer that applies the full
424
+ * predicate, and never as a replacement for it. */
425
+ export function bearsEdge(ctx, id) {
426
+ return cachedHasNext(ctx, id, getStructCache(ctx));
427
+ }
410
428
  /** Whether a node LEADS SOMEWHERE — it bears a continuation edge or a halo.
411
429
  * The admission predicate recognition filters sites with (HOW_IT_WORKS
412
430
  * §15.3): a form that leads nowhere contributes nothing to any derivation.
@@ -490,6 +508,30 @@ export function contains(ctx, ancestor, descendant) {
490
508
  }
491
509
  return false;
492
510
  }
511
+ /** Whether a continuation edge joins the two forms, in either direction —
512
+ * the EXACT half's veto on calling them synonyms.
513
+ *
514
+ * Halos measure company, and the strongest company any two forms can keep is
515
+ * standing next to each other: a question and its answer co-occur in every
516
+ * episode that taught the pair, so their halos SHOULD be similar, and on a
517
+ * conversational store they are (measured on the CONV fixture: consecutive
518
+ * turns at 0.809 against a 0.516 concept threshold). A gate reading halo
519
+ * cosine alone therefore reads adjacency as synonymy and revoices an answer
520
+ * in the words of the question it answers — "it hangs in madrid" spliced back
521
+ * into "where is it kept now". The distributional layer cannot tell the two
522
+ * relations apart, because to it they are the same observation; the exact
523
+ * half can, for free, because it stored the edge. §4.1's division of labour
524
+ * exactly: approximate proposes, exact decides.
525
+ *
526
+ * Read LIMITed in both directions at the hub bound — a common continuation's
527
+ * fan-in is corpus-sized, and no single decision may scale with it. */
528
+ export function answers(ctx, a, b) {
529
+ const bound = hubBound(ctx);
530
+ if (ctx.store.hasNext(a) && ctx.store.nextFirst(a, bound).includes(b)) {
531
+ return true;
532
+ }
533
+ return ctx.store.hasNext(b) && ctx.store.nextFirst(b, bound).includes(a);
534
+ }
493
535
  // ── Edge disambiguation (Section 6) ──────────────────────────────────────
494
536
  /** The best-scoring item by cosine against `query`, among items scoring at
495
537
  * or above `threshold` — the shared arg-max every Pattern-A "which of these
@@ -46,6 +46,8 @@ export declare class SQliteStore extends AbstractStore implements Store {
46
46
  private _insCanon;
47
47
  private _selCanon;
48
48
  private _cntCanon;
49
+ private _insSketch;
50
+ private _selSketch;
49
51
  private _selContentFrom;
50
52
  private _delMeta;
51
53
  private _insSnapshot;
@@ -136,6 +138,8 @@ export declare class SQliteStore extends AbstractStore implements Store {
136
138
  protected _dbDeleteMeta(key: string): void;
137
139
  canonAdd(h: number, id: number): void;
138
140
  canonFind(h: number): number[];
141
+ sketchGet(id: number): number[] | null;
142
+ sketchPut(id: number, ids: readonly number[]): void;
139
143
  canonCount(): number;
140
144
  eachContent(cb: (id: number, bytes: Uint8Array) => void, fromId?: number): void;
141
145
  protected _dbSaveSnapshot(bytes: Uint8Array): void;
@@ -124,6 +124,20 @@ CREATE TABLE IF NOT EXISTS canon (
124
124
  id INTEGER NOT NULL,
125
125
  PRIMARY KEY (h, id)
126
126
  ) WITHOUT ROWID;
127
+ -- CONSTITUENT SKETCH (Store.sketchGet/sketchPut): the bottom-k minimal
128
+ -- constituents of a node's subtree, k = √D, chosen by identity hash. The blob is
129
+ -- a packed int32 little-endian run, already in hash order; an EMPTY blob is a
130
+ -- real answer (a minimal unit has no constituents) and a MISSING ROW means
131
+ -- "not yet computed" — the two must stay distinguishable, which is why absence
132
+ -- is a missing row rather than an empty blob sentinel. (node:sqlite binds a
133
+ -- zero-length Uint8Array as NULL, so the column is nullable and a NULL blob
134
+ -- reads back as the empty sketch — the ROW is what records "computed".)
135
+ -- Measured on the trained
136
+ -- store: 80.8% of nodes sketch EMPTY, mean 1.14 ids, ~72 MB over 15.7M nodes.
137
+ CREATE TABLE IF NOT EXISTS sketch (
138
+ id INTEGER PRIMARY KEY,
139
+ ids BLOB
140
+ );
127
141
  CREATE TABLE IF NOT EXISTS snapshot (
128
142
  id INTEGER PRIMARY KEY CHECK (id = 1),
129
143
  data BLOB NOT NULL
@@ -227,6 +241,8 @@ export class SQliteStore extends AbstractStore {
227
241
  _insCanon = null;
228
242
  _selCanon = null;
229
243
  _cntCanon = null;
244
+ _insSketch = null;
245
+ _selSketch = null;
230
246
  _selContentFrom = null;
231
247
  _delMeta = null;
232
248
  _insSnapshot = null;
@@ -829,6 +845,37 @@ export class SQliteStore extends AbstractStore {
829
845
  }
830
846
  return this._selCanon.all(h).map((r) => r.id);
831
847
  }
848
+ // -- Constituent sketch (Store optional capability) --
849
+ sketchGet(id) {
850
+ if (!this._selSketch) {
851
+ this._selSketch = this.sqlite.prepare("SELECT ids FROM sketch WHERE id = ?");
852
+ }
853
+ const row = this._selSketch.get(id);
854
+ if (row === undefined)
855
+ return null; // never computed — NOT the same as []
856
+ const b = row.ids;
857
+ if (b === null || b.byteLength === 0)
858
+ return []; // computed, no constituents
859
+ const out = [];
860
+ const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
861
+ for (let i = 0; i + 4 <= b.byteLength; i += 4) {
862
+ out.push(dv.getInt32(i, true));
863
+ }
864
+ return out;
865
+ }
866
+ sketchPut(id, ids) {
867
+ if (!this._insSketch) {
868
+ this._insSketch = this.sqlite.prepare("INSERT OR REPLACE INTO sketch (id, ids) VALUES (?, ?)");
869
+ }
870
+ const b = new Uint8Array(ids.length * 4);
871
+ const dv = new DataView(b.buffer);
872
+ for (let i = 0; i < ids.length; i++)
873
+ dv.setInt32(i * 4, ids[i], true);
874
+ // Join the deferred write transaction (committed by flush/commit), like
875
+ // canonAdd — a training run writes these in bulk.
876
+ this._dbBeginTx();
877
+ this._insSketch.run(id, b);
878
+ }
832
879
  canonCount() {
833
880
  if (!this._cntCanon) {
834
881
  this._cntCanon = this.sqlite.prepare("SELECT count(*) AS c FROM canon");
@@ -324,6 +324,13 @@ export interface Store {
324
324
  * restricts the scan to ids ≥ fromId, so an index refresh after further
325
325
  * training only visits the new rows. */
326
326
  eachContent?(cb: (id: NodeId, bytes: Uint8Array) => void, fromId?: NodeId): void;
327
+ /** The stored sketch of `id`, or null when it has never been computed.
328
+ * An empty array is a REAL answer (a minimal unit has no constituents) and
329
+ * must be distinguished from null. */
330
+ sketchGet?(id: NodeId): NodeId[] | null;
331
+ /** Record `ids` as the sketch of `id`. Idempotent; ids are already sorted
332
+ * by the caller's identity hash. */
333
+ sketchPut?(id: NodeId, ids: readonly NodeId[]): void;
327
334
  size(): Promise<number>;
328
335
  saveSnapshot(bytes: Uint8Array): Promise<void>;
329
336
  loadSnapshot(): Promise<Uint8Array | null>;