@polycode-projects/the-mechanical-code-talker 1.8.20 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -6
- package/ROADMAP.md +1 -1
- package/bin/tmct.mjs +167 -20
- package/corpus/generated/README.md +2 -2
- package/corpus/namenet/LICENSE-NOTICE +68 -0
- package/corpus/namenet/generate.mjs +309 -0
- package/corpus/namenet/manifest.json +20 -0
- package/corpus/namenet/namenet.jsonl +7260 -0
- package/corpus/wordnet/LICENSE-NOTICE +49 -0
- package/corpus/wordnet/generate.mjs +333 -0
- package/corpus/wordnet/manifest.json +34 -0
- package/corpus/wordnet/wordnet-full.jsonl +192498 -0
- package/corpus/wordnet/wordnet-xl.jsonl +23805 -0
- package/package.json +6 -2
- package/src/ask-browser-entry.mjs +13 -2
- package/src/ask-browser.bundle.js +272 -19
- package/src/chat.mjs +229 -82
- package/src/cli-args.mjs +20 -1
- package/src/codegraph.mjs +306 -1
- package/src/corpus/conceptnet-map.toml +17 -12
- package/src/corpus/conceptnet.mjs +23 -1
- package/src/extensions.mjs +44 -1
- package/src/init.mjs +99 -46
- package/src/interpret/normalize.mjs +1 -3
- package/src/memory/core.mjs +191 -14
- package/src/memory/trust.mjs +27 -6
- package/src/memory-ask-browser-entry.mjs +36 -0
- package/src/memory-ask-browser.bundle.js +5544 -0
- package/src/toml-config.mjs +11 -1
- package/src/viz.mjs +548 -73
package/src/cli-args.mjs
CHANGED
|
@@ -5,10 +5,14 @@
|
|
|
5
5
|
// precedence chain — built on top of toml-config.mjs's already-tested
|
|
6
6
|
// mergeEffective/normalizeConfig (arg > toml > default), not a rebuild of it.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
8
|
+
// Four tiny flag helpers (pure, no I/O) plus the one async resolver:
|
|
9
9
|
// strFlag(rest, names, dflt) → single value, last flag occurrence wins
|
|
10
10
|
// repeatedFlag(rest, names) → every value for a repeatable flag (e.g. --graph)
|
|
11
11
|
// boolFlag(rest, names) → true if any of `names` appears at all
|
|
12
|
+
// enumFlag(rest, names, choices) → strFlag, validated against a closed set
|
|
13
|
+
// (throws a clear error naming the flag + the choices — the shared shape
|
|
14
|
+
// for a closed-choice option like `--memory-backend default|memory|sqlite`,
|
|
15
|
+
// matching `--with-persona`'s own "unknown name" error style)
|
|
12
16
|
// resolveRuntimeConfig({argv, cwd, env, gitRoot}) → the resolved repo/config
|
|
13
17
|
//
|
|
14
18
|
// Graph-path precedence (documented once, here — every subcommand shares it):
|
|
@@ -69,6 +73,21 @@ export function boolFlag(rest, names) {
|
|
|
69
73
|
return rest.some((r) => list.includes(r));
|
|
70
74
|
}
|
|
71
75
|
|
|
76
|
+
/** Closed-choice single-value flag: `strFlag` plus validation against
|
|
77
|
+
* `choices`. Returns `undefined` when absent (never a default — the caller
|
|
78
|
+
* decides what "absent" means, same as an omitted `strFlag` call). Throws a
|
|
79
|
+
* clear, user-facing error naming the flag and the valid choices when a
|
|
80
|
+
* value IS given but isn't one of them — validate-before-any-disk-write,
|
|
81
|
+
* the same discipline `tmct init --with-persona <unknown>` already uses. */
|
|
82
|
+
export function enumFlag(rest, names, choices) {
|
|
83
|
+
const val = strFlag(rest, names, undefined);
|
|
84
|
+
if (val !== undefined && !choices.includes(val)) {
|
|
85
|
+
const flagName = asList(names)[0];
|
|
86
|
+
throw new Error(`invalid ${flagName} "${val}". Choices: ${choices.join(", ")}.`);
|
|
87
|
+
}
|
|
88
|
+
return val;
|
|
89
|
+
}
|
|
90
|
+
|
|
72
91
|
/**
|
|
73
92
|
* Resolve one subcommand invocation's repo root, tmct.toml, and graph
|
|
74
93
|
* path(s) — the shared precedence chain every subcommand (chat/memory/init/
|
package/src/codegraph.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { cosine } from "./embed.mjs";
|
|
|
3
3
|
// Single-sourced predicate strings (memory/core.mjs owns these constants) — no
|
|
4
4
|
// circular-import risk: core.mjs imports trust.mjs/shacl.mjs/planning.mjs, never
|
|
5
5
|
// codegraph.mjs, in either direction.
|
|
6
|
-
import { CREATED_AT_PROP, UPDATED_AT_PROP } from "./memory/core.mjs";
|
|
6
|
+
import { CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource } from "./memory/core.mjs";
|
|
7
7
|
|
|
8
8
|
// Pure (no-network, no-fs) query logic over the typed `entities` payload that the
|
|
9
9
|
// deterministic indexer writes to <repo>/.tmct/graph.json (shape produced by
|
|
@@ -102,11 +102,29 @@ const PROP_KIND = {
|
|
|
102
102
|
"mgx:inreplyto": "inReplyTo",
|
|
103
103
|
"mgx:statedby": "statedBy",
|
|
104
104
|
"mgx:canonicalisedfrom": "canonicalisedFrom",
|
|
105
|
+
// PLAN_VIZ_MEMORY.md Bug 2 fix: the two FIXED structural link kinds
|
|
106
|
+
// deriveFactTermGraph (below) synthesizes on every Fact — Fact -> its own
|
|
107
|
+
// subject/object Term individual. Without these a walk seeded on a Fact (the
|
|
108
|
+
// default mostRecentIndividual seed right after a teach turn) could never
|
|
109
|
+
// reach the term graph at all. Distinct from the per-predicate kinds below
|
|
110
|
+
// (an open-ended, DYNAMIC set — see relationKind's "factrel:" branch), these
|
|
111
|
+
// two are fixed and few, so a plain PROP_KIND row is the simplest fit.
|
|
112
|
+
"mgx:factsubjectterm": "factSubjectTerm",
|
|
113
|
+
"mgx:factobjectterm": "factObjectTerm",
|
|
105
114
|
};
|
|
106
115
|
|
|
107
116
|
export function relationKind(group) {
|
|
108
117
|
const prop = String(group?.prop || "").toLowerCase();
|
|
109
118
|
if (PROP_KIND[prop]) return PROP_KIND[prop];
|
|
119
|
+
// PLAN_VIZ_MEMORY.md Bug 2 fix: deriveFactTermGraph's per-predicate relation
|
|
120
|
+
// groups (Term -> Term, one group per DISTINCT fact predicate actually
|
|
121
|
+
// present in the data — there is no fixed vocabulary to enumerate here: a
|
|
122
|
+
// freshly taught "mgx:<verb>" predicate (generalVerbTeach) must classify
|
|
123
|
+
// automatically, never requiring a PROP_KIND edit per predicate). Those
|
|
124
|
+
// groups self-namespace their `prop` as `factrel:<predicate>` specifically
|
|
125
|
+
// so they can self-classify here, verbatim, with zero collision risk against
|
|
126
|
+
// any real code-graph or memory-graph prop token (none use this prefix).
|
|
127
|
+
if (prop.startsWith("factrel:")) return group.predicate || null;
|
|
110
128
|
const pred = String(group?.predicate || "").toLowerCase();
|
|
111
129
|
// symbol-granular fallbacks first, so a near-miss token name still classifies to the
|
|
112
130
|
// fine-grained kind rather than collapsing to module-coarse calls/touches.
|
|
@@ -828,6 +846,18 @@ function beamExpand(graph, scored, beamWidth) {
|
|
|
828
846
|
* - `seeds` (default derived from `scored`, as before) — an explicit id iterable, so a caller
|
|
829
847
|
* with no `scored` list at all (e.g. `mostRecentIndividual`'s single seed) can still drive
|
|
830
848
|
* the walk.
|
|
849
|
+
* - `hubDegree` (default `Infinity`, PLAN_VIZ_MEMORY.md's page-size strategy — seonix's own
|
|
850
|
+
* third cap, default 40 there): stop expanding THROUGH a node with MORE than this many
|
|
851
|
+
* in-graph neighbours over `kinds` — the node itself is still popped/emitted normally (still
|
|
852
|
+
* shown), it just contributes no candidates for the NEXT hop. Distinct from `q` (a relative,
|
|
853
|
+
* per-step quantile gate that always keeps at least one candidate) and from `nodeLimit` (a
|
|
854
|
+
* total emit budget): `hubDegree` is an absolute per-node gate that can drop a hub's entire
|
|
855
|
+
* fan-out to zero, so an ultra-common hypernym ("thing", "entity" — reachable from thousands
|
|
856
|
+
* of IsA facts) can't swallow the whole node budget in one hop. `Infinity` (no gate) keeps
|
|
857
|
+
* every existing caller byte-identical. EXEMPTS the seed(s) (hop 0) themselves — a walk
|
|
858
|
+
* started directly ON a hub (e.g. `tmct viz --term tree` where "tree" is a 1,972-fact
|
|
859
|
+
* ConceptNet hub, measured live this session) still shows that hub's own immediate
|
|
860
|
+
* neighbourhood; only a hub reached MID-walk (hop > 0) has its own further fan-out gated.
|
|
831
861
|
* The score-nudge machinery (mutating `scored`/introducing newly-surfaced individuals into it)
|
|
832
862
|
* is gated behind `scored.length > 0 && maxSeed > 0` — the exact condition the original early
|
|
833
863
|
* return checked — so an empty `scored` degrades gracefully into a pure walk rather than erroring.
|
|
@@ -842,6 +872,7 @@ export function spiralExpand(graph, scored = [], {
|
|
|
842
872
|
classPredicate = (ind) => (ind.class || "") === "Module",
|
|
843
873
|
idNormalizer = null,
|
|
844
874
|
seeds: seedsOpt = null,
|
|
875
|
+
hubDegree = Infinity,
|
|
845
876
|
} = {}) {
|
|
846
877
|
const byId = new Map(scored.map((s) => [s.ind.id, s]));
|
|
847
878
|
let maxSeed = 0;
|
|
@@ -912,6 +943,16 @@ export function spiralExpand(graph, scored = [], {
|
|
|
912
943
|
emitted++;
|
|
913
944
|
}
|
|
914
945
|
if (node.hop >= depth) continue;
|
|
946
|
+
// hubDegree gate: a node above the cap is still shown (already emitted above) but never
|
|
947
|
+
// expanded THROUGH — its own neighbours contribute nothing to the next hop. EXEMPTS hop 0
|
|
948
|
+
// (a seed) deliberately: measured live against a real init:xl-scale corpus this session,
|
|
949
|
+
// `tmct viz --term tree` (a 1,972-fact hub term) with the gate applied unconditionally
|
|
950
|
+
// returned a single, useless lone node — seeding directly ON a term the user explicitly
|
|
951
|
+
// asked to centre on must always show ITS OWN immediate neighbourhood, or the whole
|
|
952
|
+
// `--term`/click-to-recentre feature is pointless on exactly the popular, interesting terms
|
|
953
|
+
// it exists for. A hub only reached mid-walk (hop > 0) still gates normally — this only
|
|
954
|
+
// changes the walk's own STARTING point(s), not general hub suppression elsewhere.
|
|
955
|
+
if (node.hop > 0 && degree(node.id) > hubDegree) continue;
|
|
915
956
|
// This step's candidate set = the popped node's unvisited neighbours matching classPredicate;
|
|
916
957
|
// quantile-gate by degree, keeping the lowest-degree ⌊q·n⌋ (drop the densest hubs), never
|
|
917
958
|
// fewer than one.
|
|
@@ -1416,6 +1457,270 @@ export function buildVizNodesAndEdges(graph, walked, { createdAtProp = CREATED_A
|
|
|
1416
1457
|
return { nodes, edges };
|
|
1417
1458
|
}
|
|
1418
1459
|
|
|
1460
|
+
// PLAN_VIZ_MEMORY.md Bug 2 fix: the FACT_CLASS string, mirrored here rather than
|
|
1461
|
+
// imported (memory/core.mjs's own FACT_CLASS export is a plain "Fact" literal —
|
|
1462
|
+
// importing one more binding across this already-imported module isn't worth
|
|
1463
|
+
// it for a single string every reader of this file can eyeball is exactly what
|
|
1464
|
+
// memory/core.mjs's own appendFact writes).
|
|
1465
|
+
const MEMORY_FACT_CLASS = "Fact";
|
|
1466
|
+
const MEMORY_TERM_CLASS = "Term";
|
|
1467
|
+
|
|
1468
|
+
/** Derive a TERM-relation VIEW of a memory graph's reified Facts (Bug 2 fix,
|
|
1469
|
+
* PLAN_VIZ_MEMORY.md) — never mutates `graph`, never persisted, viz-only.
|
|
1470
|
+
*
|
|
1471
|
+
* A Fact individual stores its subject/predicate/object as plain normalized
|
|
1472
|
+
* STRING attributes (`rdf:subject`/`rdf:predicate`/`rdf:object`,
|
|
1473
|
+
* memory/core.mjs's appendFact) — there is no individual node for "dog" at
|
|
1474
|
+
* all. So the real subject->predicate->object concept structure is
|
|
1475
|
+
* structurally invisible to any walk over `graph.relations` as it stands:
|
|
1476
|
+
* every edge there connects two INDIVIDUAL ids (Utterance/Session/Fact/
|
|
1477
|
+
* Source), never a concept term. This function materializes the missing
|
|
1478
|
+
* structure as a NEW, derived graph:
|
|
1479
|
+
* - one synthetic `Term` individual per distinct normalized subject/object
|
|
1480
|
+
* string (id `term:<t>`, label `t`);
|
|
1481
|
+
* - one synthetic relation group per DISTINCT fact predicate actually
|
|
1482
|
+
* present in the data (Term -> Term, `subject`/`object` = the two terms'
|
|
1483
|
+
* ids) — no hardcoded predicate vocabulary: a freshly taught "mgx:<verb>"
|
|
1484
|
+
* predicate (generalVerbTeach) becomes walkable automatically, the same
|
|
1485
|
+
* turn it's asserted;
|
|
1486
|
+
* - two FIXED structural link groups, `factSubjectTerm`/`factObjectTerm`
|
|
1487
|
+
* (Fact -> its own subject/object Term) — without these a walk seeded on
|
|
1488
|
+
* a Fact (the default `mostRecentIndividual` seed right after a teach
|
|
1489
|
+
* turn) could never reach the term graph in the first place; a `--term`
|
|
1490
|
+
* seed reaches the SAME facts via the same links, in reverse.
|
|
1491
|
+
* Returns `{ graph: <augmented graph>, factRelationKinds: [<predicate>, …] }`
|
|
1492
|
+
* — `factRelationKinds` is exactly the dynamic `kinds` list a caller passes
|
|
1493
|
+
* to `spiralExpand` for the "concept relation" walk (see MEMORY_SPIRAL_EXPAND_KINDS
|
|
1494
|
+
* for the sibling "provenance/meta" kinds list). A graph with no Fact
|
|
1495
|
+
* individuals (or a code graph passed in by mistake) is a safe no-op:
|
|
1496
|
+
* `{ graph, factRelationKinds: [] }`, the SAME graph object, unchanged.
|
|
1497
|
+
* Pure; deterministic (Map iteration order = insertion order = first-seen
|
|
1498
|
+
* order over `graph.individuals`, so re-running on the same input is
|
|
1499
|
+
* byte-identical). */
|
|
1500
|
+
export function deriveFactTermGraph(graph) {
|
|
1501
|
+
const termById = new Map(); // term:<t> -> individual
|
|
1502
|
+
const groupByPredicate = new Map(); // predicate -> relation group
|
|
1503
|
+
const subjectLinks = []; // Fact -> its subject Term
|
|
1504
|
+
const objectLinks = []; // Fact -> its object Term
|
|
1505
|
+
const termId = (t) => `term:${t}`;
|
|
1506
|
+
const ensureTerm = (t) => {
|
|
1507
|
+
const id = termId(t);
|
|
1508
|
+
if (!termById.has(id)) termById.set(id, { id, label: t, class: MEMORY_TERM_CLASS, attributes: [] });
|
|
1509
|
+
return id;
|
|
1510
|
+
};
|
|
1511
|
+
|
|
1512
|
+
for (const ind of graph?.individuals || []) {
|
|
1513
|
+
if ((ind?.class || "") !== MEMORY_FACT_CLASS) continue;
|
|
1514
|
+
const attrs = ind.attributes || [];
|
|
1515
|
+
const s = attrs.find((a) => a?.prop === "rdf:subject")?.value;
|
|
1516
|
+
const p = attrs.find((a) => a?.prop === "rdf:predicate")?.value;
|
|
1517
|
+
const o = attrs.find((a) => a?.prop === "rdf:object")?.value;
|
|
1518
|
+
if (!s || !p || !o) continue; // a malformed/legacy Fact — skip, never throw
|
|
1519
|
+
const subjectTermId = ensureTerm(s);
|
|
1520
|
+
const objectTermId = ensureTerm(o);
|
|
1521
|
+
let group = groupByPredicate.get(p);
|
|
1522
|
+
if (!group) {
|
|
1523
|
+
// `factrel:` namespace: relationKind's own dedicated branch self-classifies
|
|
1524
|
+
// any group with this prefix to its raw predicate, verbatim — see that
|
|
1525
|
+
// function's comment for why (an open-ended, dynamically-discovered kind
|
|
1526
|
+
// set with no PROP_KIND row to add per predicate).
|
|
1527
|
+
group = { predicate: p, prop: `factrel:${p}`, count: 0, edges: [] };
|
|
1528
|
+
groupByPredicate.set(p, group);
|
|
1529
|
+
}
|
|
1530
|
+
group.edges.push({ subject: subjectTermId, object: objectTermId, subjectLabel: s, objectLabel: o });
|
|
1531
|
+
group.count = group.edges.length;
|
|
1532
|
+
subjectLinks.push({ subject: ind.id, object: subjectTermId, subjectLabel: ind.label, objectLabel: s });
|
|
1533
|
+
objectLinks.push({ subject: ind.id, object: objectTermId, subjectLabel: ind.label, objectLabel: o });
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
if (!termById.size) return { graph, factRelationKinds: [] };
|
|
1537
|
+
|
|
1538
|
+
const individuals = [...(graph.individuals || []), ...termById.values()];
|
|
1539
|
+
const byId = new Map(graph.byId);
|
|
1540
|
+
for (const term of termById.values()) byId.set(term.id, term);
|
|
1541
|
+
const relations = [
|
|
1542
|
+
...(graph.relations || []),
|
|
1543
|
+
...groupByPredicate.values(),
|
|
1544
|
+
{ predicate: "factSubjectTerm", prop: "mgx:factSubjectTerm", count: subjectLinks.length, edges: subjectLinks },
|
|
1545
|
+
{ predicate: "factObjectTerm", prop: "mgx:factObjectTerm", count: objectLinks.length, edges: objectLinks },
|
|
1546
|
+
];
|
|
1547
|
+
|
|
1548
|
+
return {
|
|
1549
|
+
graph: { ...graph, individuals, byId, relations },
|
|
1550
|
+
factRelationKinds: [...groupByPredicate.keys()],
|
|
1551
|
+
};
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
/** The two FIXED structural link kinds `deriveFactTermGraph` always emits
|
|
1555
|
+
* (Fact -> its own subject/object Term) — see that function's own doc for
|
|
1556
|
+
* why they're needed at all. Bundled into the "relation" (concept) walk, not
|
|
1557
|
+
* the "meta" (provenance) one: a user who toggles to meta-only still gets
|
|
1558
|
+
* today's exact byte-identical provenance-only view (see viz.mjs's edge-kind
|
|
1559
|
+
* toggle). */
|
|
1560
|
+
export const MEMORY_FACT_LINK_KINDS = ["factSubjectTerm", "factObjectTerm"];
|
|
1561
|
+
|
|
1562
|
+
/** Bug 2 fix: the combined kinds list a memory-graph walk actually uses, for a
|
|
1563
|
+
* given edge-kind MODE — "meta" (today's exact provenance-only walk, kept as
|
|
1564
|
+
* a filterable toggle, never deleted), "relation" (the NEW concept view —
|
|
1565
|
+
* Bug 2's fix), or "both" (the default: meta AND relation kinds together, so
|
|
1566
|
+
* a click on a recently-taught Fact reaches its own concept neighbourhood the
|
|
1567
|
+
* same turn). `factRelationKinds` is the dynamic per-predicate list
|
|
1568
|
+
* `deriveFactTermGraph` discovered in THIS graph — there is no fixed
|
|
1569
|
+
* vocabulary, a freshly taught predicate is walkable the same turn it's
|
|
1570
|
+
* asserted. Lives here (not viz.mjs) and is re-exported through
|
|
1571
|
+
* ask-browser-entry.mjs specifically so BOTH the CLI's own generation-time
|
|
1572
|
+
* walk (viz.mjs's computeVizGraph) AND the browser bundle's client-side
|
|
1573
|
+
* re-walk (a recentre or an edge-kind-toggle change) combine kinds via the
|
|
1574
|
+
* SAME function — viz.mjs itself can't be bundled for the browser (it does
|
|
1575
|
+
* real fs I/O), so this had to live in the shared, browser-safe module. */
|
|
1576
|
+
export function edgeKindsFor(mode, factRelationKinds) {
|
|
1577
|
+
const relationKinds = [...factRelationKinds, ...MEMORY_FACT_LINK_KINDS];
|
|
1578
|
+
if (mode === "meta") return [...MEMORY_SPIRAL_EXPAND_KINDS];
|
|
1579
|
+
if (mode === "relation") return relationKinds;
|
|
1580
|
+
return [...MEMORY_SPIRAL_EXPAND_KINDS, ...relationKinds]; // "both" (default)
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
const LEGEND_MAX_BUCKETS = 20; // seonix precedent, PLAN_VIZ_MEMORY.md: too many chips to be usable
|
|
1584
|
+
const LEGEND_MIN_BUCKETS = 2; // nothing to filter with only one bucket
|
|
1585
|
+
const LEGEND_COLLAPSE_TOP_N = 15; // "top 15 by count, rest grouped as Other" — stays under the max
|
|
1586
|
+
|
|
1587
|
+
/** Normalized Shannon entropy (`H / log2(k)`, `k` = bucket count) of a
|
|
1588
|
+
* {value, count} bucket list — 1.0 for a perfectly even split, ~0 for one
|
|
1589
|
+
* dominant bucket swallowing everything, undefined (returns 0) for k<2. */
|
|
1590
|
+
function normalizedEntropy(buckets) {
|
|
1591
|
+
const k = buckets.length;
|
|
1592
|
+
if (k < 2) return 0;
|
|
1593
|
+
const total = buckets.reduce((sum, b) => sum + b.count, 0);
|
|
1594
|
+
if (!total) return 0;
|
|
1595
|
+
let h = 0;
|
|
1596
|
+
for (const b of buckets) {
|
|
1597
|
+
if (!b.count) continue;
|
|
1598
|
+
const p = b.count / total;
|
|
1599
|
+
h -= p * Math.log2(p);
|
|
1600
|
+
}
|
|
1601
|
+
return h / Math.log2(k);
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
/** Collapse a raw {value,count} bucket list down to at most LEGEND_MAX_BUCKETS
|
|
1605
|
+
* entries: keep the top LEGEND_COLLAPSE_TOP_N by count, fold the rest into a
|
|
1606
|
+
* single "Other" bucket — the plan's own "may need a top-15-by-count, rest
|
|
1607
|
+
* grouped as Other" escape hatch, applied generically (not predicate-only) so
|
|
1608
|
+
* any dimension that happens to be high-cardinality degrades the same way.
|
|
1609
|
+
* A no-op (returns `buckets` unchanged, same array) when already <= the cap.
|
|
1610
|
+
* Exported (not just used internally by pickLegendDimension) so the browser
|
|
1611
|
+
* bundle's client-side legend (live per-view recomputation, viz.mjs's own
|
|
1612
|
+
* computeLegendBuckets) collapses high-cardinality dimensions the SAME way,
|
|
1613
|
+
* never a second hand-rolled copy that could drift. */
|
|
1614
|
+
export function collapseToTopN(buckets) {
|
|
1615
|
+
if (buckets.length <= LEGEND_MAX_BUCKETS) return buckets;
|
|
1616
|
+
const sorted = [...buckets].sort((a, b) => b.count - a.count || (a.value < b.value ? -1 : 1));
|
|
1617
|
+
const kept = sorted.slice(0, LEGEND_COLLAPSE_TOP_N);
|
|
1618
|
+
const restCount = sorted.slice(LEGEND_COLLAPSE_TOP_N).reduce((sum, b) => sum + b.count, 0);
|
|
1619
|
+
return restCount ? [...kept, { value: "Other", count: restCount }] : kept;
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
function bucketCounts(values) {
|
|
1623
|
+
const counts = new Map();
|
|
1624
|
+
for (const v of values) {
|
|
1625
|
+
if (v == null || v === "") continue;
|
|
1626
|
+
counts.set(v, (counts.get(v) || 0) + 1);
|
|
1627
|
+
}
|
|
1628
|
+
return [...counts.entries()].map(([value, count]) => ({ value, count }));
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
/** The normalized provenance-prefix label for a Fact's FIRST recorded
|
|
1632
|
+
* provenance tag (a Fact may carry a " | "-joined union of several; the
|
|
1633
|
+
* legend buckets on the primary/first-recorded one, not a multiset) —
|
|
1634
|
+
* reuses memory/core.mjs's own provenanceTagToSource parser (the SAME
|
|
1635
|
+
* collapse-the-session-id/timestamp-suffix, keep-the-corpus/source-name
|
|
1636
|
+
* logic the trust layer already relies on) rather than re-deriving it.
|
|
1637
|
+
* Null when the Fact carries no provenance tag at all (a legacy/malformed
|
|
1638
|
+
* row) or the tag doesn't parse to a known Source kind. */
|
|
1639
|
+
function provenanceBucketLabel(rawTag) {
|
|
1640
|
+
const tag = String(rawTag || "").split(" | ")[0].trim();
|
|
1641
|
+
if (!tag) return null;
|
|
1642
|
+
const src = provenanceTagToSource(tag);
|
|
1643
|
+
if (!src) return null;
|
|
1644
|
+
if (src.kind === "corpus" || src.kind === "corpusWeak") {
|
|
1645
|
+
return `${src.kind === "corpusWeak" ? "corpus-weak" : "corpus"}:${src.name || "unknown"}`;
|
|
1646
|
+
}
|
|
1647
|
+
if (src.kind === "extracted") return `extracted:${src.name || "unknown"}`;
|
|
1648
|
+
if (src.kind === "entailed") return `entailed:${src.rule || "unknown"}`;
|
|
1649
|
+
if (src.kind === "operator") return "ace:chat";
|
|
1650
|
+
if (src.kind === "teach") return "teach:chat";
|
|
1651
|
+
if (src.kind === "web") return "web";
|
|
1652
|
+
return src.kind;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
/** A single walked NODE's bucket value under one legend dimension — the
|
|
1656
|
+
* per-node counterpart to `pickLegendDimension`'s aggregate bucket counts,
|
|
1657
|
+
* exported so both the CLI's own legend computation AND the browser bundle's
|
|
1658
|
+
* client-side dimension-switcher (a user flipping from "split by predicate"
|
|
1659
|
+
* to "split by trust source" without regenerating the page,
|
|
1660
|
+
* PLAN_VIZ_MEMORY.md's Controls section) filter/color by the SAME derivation,
|
|
1661
|
+
* never a second hand-rolled copy. `"class"` reads every node; `"predicate"`/
|
|
1662
|
+
* `"provenance"` only ever return non-null for a Fact-class node (any other
|
|
1663
|
+
* class simply has no predicate/provenance of its own to bucket on). */
|
|
1664
|
+
export function legendValueFor(graph, node, dimension) {
|
|
1665
|
+
if (dimension === "class") return node?.class || "(none)";
|
|
1666
|
+
if (!node || node.class !== MEMORY_FACT_CLASS) return null;
|
|
1667
|
+
const attrs = graph?.byId?.get?.(node.id)?.attributes || [];
|
|
1668
|
+
if (dimension === "predicate") return attrs.find((a) => a?.prop === "rdf:predicate")?.value || null;
|
|
1669
|
+
if (dimension === "provenance") return provenanceBucketLabel(attrs.find((a) => a?.prop === "mgx:factProvenance")?.value);
|
|
1670
|
+
return null;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
/** Auto-pick the filter/legend dimension at generation time (PLAN_VIZ_MEMORY.md
|
|
1674
|
+
* "Auto-picking the filter/legend dimension" section — full algorithm/
|
|
1675
|
+
* rationale there). seonix hardcodes its legend dimension (a small, near-
|
|
1676
|
+
* uniform set of code-graph classes) — tmct's memory graph does NOT have that
|
|
1677
|
+
* property: `class` is `{Fact, Session, Source, Utterance, Term}` and once
|
|
1678
|
+
* real data is seeded, Fact dominates so heavily that a class-based legend
|
|
1679
|
+
* filters almost nothing. This scores three candidate dimensions by
|
|
1680
|
+
* normalized Shannon entropy over their bucket-size distribution (rewards an
|
|
1681
|
+
* even-ish split, penalizes one dominant bucket) and picks the best-scoring
|
|
1682
|
+
* QUALIFYING one (`LEGEND_MIN_BUCKETS <= k <= LEGEND_MAX_BUCKETS`, after a
|
|
1683
|
+
* top-15+Other collapse for anything over the cap) as the PRIMARY legend:
|
|
1684
|
+
* 1. `class` — every walked node's own `.class`.
|
|
1685
|
+
* 2. `predicate` — every walked Fact node's `rdf:predicate` attribute (a
|
|
1686
|
+
* relation-shaped split: "show me only IsA facts").
|
|
1687
|
+
* 3. `provenance` — every walked Fact node's provenance prefix, collapsed
|
|
1688
|
+
* (see provenanceBucketLabel) — a TRUST-shaped split.
|
|
1689
|
+
* Pure, one pass over the already-walked `nodes` (no new graph traversal) —
|
|
1690
|
+
* computed ONCE at generation time and embedded into the page's JSON, never
|
|
1691
|
+
* recomputed client-side. `graph` supplies the per-Fact attribute lookups
|
|
1692
|
+
* `nodes` itself doesn't carry (predicate/provenance are Fact ATTRIBUTES,
|
|
1693
|
+
* not part of the {id,hop,label,class,createdAt,updatedAt} viz node shape).
|
|
1694
|
+
* Returns `{ primary, dimensions: { class, predicate, provenance } }`, each
|
|
1695
|
+
* entry `{ score, qualifies, buckets: [{value, count}] }`. When nothing
|
|
1696
|
+
* qualifies (e.g. a tiny 1-2-node walk), `primary` falls back to `"class"` —
|
|
1697
|
+
* today's behavior — so the legend is never simply empty. */
|
|
1698
|
+
export function pickLegendDimension(graph, nodes) {
|
|
1699
|
+
const classBuckets = bucketCounts((nodes || []).map((n) => legendValueFor(graph, n, "class")));
|
|
1700
|
+
const predicateBuckets = bucketCounts((nodes || []).map((n) => legendValueFor(graph, n, "predicate")));
|
|
1701
|
+
const provenanceBuckets = bucketCounts((nodes || []).map((n) => legendValueFor(graph, n, "provenance")));
|
|
1702
|
+
|
|
1703
|
+
const score = (rawBuckets) => {
|
|
1704
|
+
const buckets = collapseToTopN(rawBuckets);
|
|
1705
|
+
const qualifies = buckets.length >= LEGEND_MIN_BUCKETS && buckets.length <= LEGEND_MAX_BUCKETS;
|
|
1706
|
+
return { score: normalizedEntropy(buckets), qualifies, buckets };
|
|
1707
|
+
};
|
|
1708
|
+
|
|
1709
|
+
const dimensions = {
|
|
1710
|
+
class: score(classBuckets),
|
|
1711
|
+
predicate: score(predicateBuckets),
|
|
1712
|
+
provenance: score(provenanceBuckets),
|
|
1713
|
+
};
|
|
1714
|
+
|
|
1715
|
+
let primary = "class";
|
|
1716
|
+
let bestScore = -1;
|
|
1717
|
+
for (const [name, d] of Object.entries(dimensions)) {
|
|
1718
|
+
if (!d.qualifies) continue;
|
|
1719
|
+
if (d.score > bestScore) { bestScore = d.score; primary = name; }
|
|
1720
|
+
}
|
|
1721
|
+
return { primary, dimensions };
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1419
1724
|
/** moduleIdOf by raw edge-endpoint id: resolves through byId when the individual exists,
|
|
1420
1725
|
* else falls back to parsing an `fn:<path>#name` id directly (callsSymbol objects may name
|
|
1421
1726
|
* symbols with no individual of their own). Null if it cannot be mapped. */
|
|
@@ -187,44 +187,49 @@ note = "mutual exclusion; ACE pattern 6 ('no N1 is a N2')"
|
|
|
187
187
|
[[relation]]
|
|
188
188
|
rel = "/r/RelatedTo"
|
|
189
189
|
surface = "{start} is related to {end}"
|
|
190
|
-
ace = "
|
|
191
|
-
|
|
190
|
+
ace = "ObjectProperty"
|
|
191
|
+
predicate = "mgx:relatedTo"
|
|
192
|
+
note = "weakest, undirected association — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): the surface template above was already fully authored, so the prior 'too vague for an axiom' exclusion was a design call dressed as a technical one, not a real blocker. Emitted at LOWER trust (src/corpus/conceptnet.mjs routes RelatedTo facts through the corpus-weak: provenance prefix -> SOURCE_PRIOR.corpusWeak, memory/trust.mjs) rather than either full-strength or excluded."
|
|
192
193
|
|
|
193
194
|
[[relation]]
|
|
194
195
|
rel = "/r/Synonym"
|
|
195
196
|
surface = "{start} means the same as {end}"
|
|
196
|
-
ace = "
|
|
197
|
-
|
|
197
|
+
ace = "ObjectProperty"
|
|
198
|
+
predicate = "mgx:synonym"
|
|
199
|
+
note = "lexical alias — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): genuinely fact-shaped ('X means the same as Y' is a real, answerable claim), no real reason it was excluded"
|
|
198
200
|
|
|
199
201
|
[[relation]]
|
|
200
202
|
rel = "/r/Antonym"
|
|
201
203
|
surface = "{start} is the opposite of {end}"
|
|
202
|
-
ace = "
|
|
203
|
-
|
|
204
|
+
ace = "ObjectProperty"
|
|
205
|
+
predicate = "mgx:antonym"
|
|
206
|
+
note = "lexical opposition — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): OWL disjointness would over-claim (hot/cold are not disjoint classes), so this does NOT reuse DistinctFrom/owl:disjointWith, but a plain ObjectProperty edge ('X is the opposite of Y') is a real, fact-shaped claim, not an axiom"
|
|
204
207
|
|
|
205
208
|
[[relation]]
|
|
206
209
|
rel = "/r/FormOf"
|
|
207
210
|
surface = "{start} is a form of the word {end}"
|
|
208
211
|
ace = "none"
|
|
209
|
-
note = "inflection → root; lexicon normalization, not knowledge"
|
|
212
|
+
note = "inflection → root; lexicon normalization, not knowledge — genuinely a different kind of thing than RelatedTo/Synonym/Antonym/SimilarTo (morphology, not world-knowledge); stays excluded, re-confirmed 2026-07-12 (TOO_HARD_AUDIT.md)"
|
|
210
213
|
|
|
211
214
|
[[relation]]
|
|
212
215
|
rel = "/r/DerivedFrom"
|
|
213
216
|
surface = "the word {start} is derived from {end}"
|
|
214
217
|
ace = "none"
|
|
215
|
-
note = "word derivation; lexicon material, not an axiom"
|
|
218
|
+
note = "word derivation; lexicon material, not an axiom — genuinely a different kind of thing than RelatedTo/Synonym/Antonym/SimilarTo (morphology, not world-knowledge); stays excluded, re-confirmed 2026-07-12 (TOO_HARD_AUDIT.md)"
|
|
216
219
|
|
|
217
220
|
[[relation]]
|
|
218
221
|
rel = "/r/SymbolOf"
|
|
219
222
|
surface = "{start} is a symbol of {end}"
|
|
220
|
-
ace = "
|
|
221
|
-
|
|
223
|
+
ace = "ObjectProperty"
|
|
224
|
+
predicate = "mgx:symbolOf"
|
|
225
|
+
note = "symbolism — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): same stale shape as the RelatedTo/Synonym/Antonym/SimilarTo exclusions — a fully-authored surface template sitting right next to a 'no OWL fit' exclusion note. Genuinely fact-shaped and precise (more so than RelatedTo), so full corpus trust, not weak."
|
|
222
226
|
|
|
223
227
|
[[relation]]
|
|
224
228
|
rel = "/r/SimilarTo"
|
|
225
229
|
surface = "{start} is similar to {end}"
|
|
226
|
-
ace = "
|
|
227
|
-
|
|
230
|
+
ace = "ObjectProperty"
|
|
231
|
+
predicate = "mgx:similarTo"
|
|
232
|
+
note = "graded similarity — re-examined 2026-07-12 (TOO_HARD_AUDIT.md): genuinely fact-shaped ('X is similar to Y'), no real reason it was excluded"
|
|
228
233
|
|
|
229
234
|
[[relation]]
|
|
230
235
|
rel = "/r/HasContext"
|
|
@@ -43,6 +43,21 @@ export const SEON_DEFINITIONS_FILE = join(PKG_ROOT, "corpus", "seon", "definitio
|
|
|
43
43
|
export const TIER2_DIR = join(PKG_ROOT, "corpus", "tier2");
|
|
44
44
|
export const TIER2_MANIFEST_FILE = join(TIER2_DIR, "manifest.json");
|
|
45
45
|
|
|
46
|
+
// corpus/wordnet/generate.mjs's output: the Open English WordNet ->
|
|
47
|
+
// ConceptNet-shape conversion, same slice shape/loader path as tier-1/tier-2.
|
|
48
|
+
// Lives alongside its own generator (corpus/conceptnet/'s own precedent:
|
|
49
|
+
// fetch-slice.mjs + slice.jsonl share one directory) rather than under a new
|
|
50
|
+
// "tier-3" name — corpus/README.md's tiering policy already uses "tier-3" for
|
|
51
|
+
// something else entirely (runtime-learned facts, NEVER committed), and this
|
|
52
|
+
// bundle is curated + committed, the same shape as a tier-2 corpus, just too
|
|
53
|
+
// large/mechanically-derived to hand-curate. "wordnet-xl"/"wordnet-full" are
|
|
54
|
+
// wired as BUILTIN_EXTENSIONS corpus entries in src/extensions.mjs, so
|
|
55
|
+
// `tmct import --corpus wordnet-xl` resolves directly there rather than
|
|
56
|
+
// through TIER2_MANIFEST_FILE's id lookup — see that module's own
|
|
57
|
+
// BUILTIN_EXTENSIONS comment.
|
|
58
|
+
export const WORDNET_DIR = join(PKG_ROOT, "corpus", "wordnet");
|
|
59
|
+
export const WORDNET_MANIFEST_FILE = join(WORDNET_DIR, "manifest.json");
|
|
60
|
+
|
|
46
61
|
const ACE_PATTERNS = new Set(["subClassOf", "type", "ObjectProperty", "someValuesFrom", "disjointWith", "property", "none"]);
|
|
47
62
|
|
|
48
63
|
/** Load the slice JSONL as a stream (never the whole file as one string) and
|
|
@@ -121,11 +136,18 @@ export function toFacts(assertions, map, provenancePrefix = "corpus:conceptnet")
|
|
|
121
136
|
const subject = termText(a.start);
|
|
122
137
|
const object = termText(a.end);
|
|
123
138
|
if (!subject || !object) continue; // non-en endpoint slipped in — filtered, not fatal
|
|
139
|
+
// mgx:relatedTo is real but low-precision (undirected, ambiguous) — routed
|
|
140
|
+
// through the corpus-weak: prefix so memory/trust.mjs's SOURCE_PRIOR.corpusWeak
|
|
141
|
+
// (below plain corpus, above web) applies, computed from the Source's kind
|
|
142
|
+
// like every other tier, never hand-set on the Fact.
|
|
143
|
+
const prefix = row.predicate === "mgx:relatedTo"
|
|
144
|
+
? provenancePrefix.replace(/^corpus:/, "corpus-weak:")
|
|
145
|
+
: provenancePrefix;
|
|
124
146
|
facts.push({
|
|
125
147
|
subject,
|
|
126
148
|
predicate: row.predicate,
|
|
127
149
|
object,
|
|
128
|
-
provenance: `${
|
|
150
|
+
provenance: `${prefix} ${a.rel}`,
|
|
129
151
|
});
|
|
130
152
|
}
|
|
131
153
|
return facts;
|
package/src/extensions.mjs
CHANGED
|
@@ -50,19 +50,27 @@
|
|
|
50
50
|
// seedBootstrapMemory already establishes (seon's curated facts should win the
|
|
51
51
|
// content-hash idempotency race over general ConceptNet noise).
|
|
52
52
|
|
|
53
|
-
import { isAbsolute, join, resolve } from "node:path";
|
|
53
|
+
import { isAbsolute, join, resolve, dirname } from "node:path";
|
|
54
54
|
import { readFile } from "node:fs/promises";
|
|
55
|
+
import { fileURLToPath } from "node:url";
|
|
55
56
|
import { loadTomlConfig } from "./toml-config.mjs";
|
|
56
57
|
import {
|
|
57
58
|
SEON_CONCEPTS_FILE,
|
|
58
59
|
SLICE_FILE as CONCEPTNET_SLICE_FILE,
|
|
59
60
|
MAP_FILE as CONCEPTNET_MAP_FILE,
|
|
60
61
|
TIER2_DIR,
|
|
62
|
+
WORDNET_DIR,
|
|
61
63
|
loadSlice,
|
|
62
64
|
loadMap,
|
|
63
65
|
toFacts,
|
|
64
66
|
} from "./corpus/conceptnet.mjs";
|
|
65
67
|
|
|
68
|
+
// corpus/namenet/generate.mjs's output — same small-top-up shape as the
|
|
69
|
+
// wordnet-xl/wordnet-full entries below, just a single bundle (not
|
|
70
|
+
// worth PKG_ROOT-style plumbing through corpus/conceptnet.mjs for one
|
|
71
|
+
// directory constant, so computed locally here instead).
|
|
72
|
+
const NAMENET_DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "corpus", "namenet");
|
|
73
|
+
|
|
66
74
|
export const EXTENSION_KINDS = Object.freeze(["corpus", "lexicon", "templates", "pack", "ontology"]);
|
|
67
75
|
|
|
68
76
|
// The definitional-band-first predicate order chat.mjs's bootstrap has always
|
|
@@ -154,6 +162,41 @@ function builtinExtensions() {
|
|
|
154
162
|
corpusPath: join(TIER2_DIR, "general.jsonl"),
|
|
155
163
|
provenancePrefix: "corpus:tier2-general",
|
|
156
164
|
},
|
|
165
|
+
// corpus/wordnet/generate.mjs's output: a mechanical ConceptNet-shape
|
|
166
|
+
// conversion of Open English WordNet's structural relations, not
|
|
167
|
+
// hand-curated like the tier-2 bundles above (NOT called "tier-3" —
|
|
168
|
+
// corpus/README.md's tiering policy already uses that name for something
|
|
169
|
+
// else, runtime-learned facts that are never committed; this bundle is
|
|
170
|
+
// curated + committed, tier-2-shaped, just too large to hand-author).
|
|
171
|
+
// Named directly "wordnet-xl"/"wordnet-full" so `tmct import --corpus
|
|
172
|
+
// wordnet-xl` resolves straight through this BUILTIN_EXTENSIONS lookup,
|
|
173
|
+
// the same seam every other recognized name already uses — no change
|
|
174
|
+
// needed to bin/tmct.mjs's tier-2-manifest-id resolution path. Shipped
|
|
175
|
+
// inactive, like every other opt-in bundle here.
|
|
176
|
+
"wordnet-xl": {
|
|
177
|
+
kind: "corpus",
|
|
178
|
+
active: false,
|
|
179
|
+
corpusPath: join(WORDNET_DIR, "wordnet-xl.jsonl"),
|
|
180
|
+
provenancePrefix: "corpus:wordnet-xl",
|
|
181
|
+
},
|
|
182
|
+
"wordnet-full": {
|
|
183
|
+
kind: "corpus",
|
|
184
|
+
active: false,
|
|
185
|
+
corpusPath: join(WORDNET_DIR, "wordnet-full.jsonl"),
|
|
186
|
+
provenancePrefix: "corpus:wordnet-full",
|
|
187
|
+
},
|
|
188
|
+
// corpus/namenet/generate.mjs's output: species/common-name and
|
|
189
|
+
// Wikidata-label/WordNet-lemma synonym pairs, mechanically derived from
|
|
190
|
+
// three human-reviewed Open English Namenet linking tables. A small,
|
|
191
|
+
// explicitly OPTIONAL top-up bundle (not a primary corpus) — same
|
|
192
|
+
// BUILTIN_EXTENSIONS seam as wordnet-xl/wordnet-full above, so `tmct
|
|
193
|
+
// import --corpus namenet` resolves directly here. Shipped inactive.
|
|
194
|
+
namenet: {
|
|
195
|
+
kind: "corpus",
|
|
196
|
+
active: false,
|
|
197
|
+
corpusPath: join(NAMENET_DIR, "namenet.jsonl"),
|
|
198
|
+
provenancePrefix: "corpus:namenet",
|
|
199
|
+
},
|
|
157
200
|
};
|
|
158
201
|
}
|
|
159
202
|
|