@polycode-projects/the-mechanical-code-talker 2.3.1 → 2.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -32
- package/bin/tmct.mjs +18 -91
- package/corpus/README.md +3 -3
- package/corpus/seon/README.md +1 -0
- package/corpus/tier2/generate.mjs +18 -18
- package/corpus/tier2/manifest.json +3 -3
- package/data/games/hanoi-3.txt +8 -2
- package/package.json +26 -8
- package/src/adapters/corpus-lanes.mjs +13 -0
- package/src/adapters/graph-build.mjs +5 -7
- package/src/adapters/import-closure.mjs +28 -0
- package/src/adapters/memory/blocks.mjs +5 -4
- package/src/adapters/memory/core.mjs +78 -5
- package/src/adapters/memory/shacl.mjs +12 -0
- package/src/adapters/providers/graph-service.mjs +12 -5
- package/src/adapters/tracked-files.mjs +17 -0
- package/src/domain/ask-vocab.mjs +2 -0
- package/src/domain/ask.mjs +225 -13
- package/src/domain/cli-verbs.mjs +201 -0
- package/src/domain/codegraph.mjs +142 -56
- package/src/domain/completions/graph-adapter.mjs +1 -1
- package/src/domain/completions/group.mjs +3 -17
- package/src/domain/completions/infer.mjs +4 -13
- package/src/domain/completions/rank.mjs +6 -19
- package/src/domain/grammar/lexicon-core.json +1 -1
- package/src/domain/hash.mjs +36 -13
- package/src/domain/interpret/fuzzy.mjs +7 -2
- package/src/domain/interpret/normalize.mjs +9 -0
- package/src/domain/interpret/strategies/keywords.mjs +19 -9
- package/src/domain/memory/capability.mjs +22 -3
- package/src/domain/memory/touched-facts.mjs +17 -0
- package/src/domain/module-paths.mjs +9 -0
- package/src/domain/persona/tiers.mjs +1 -1
- package/src/domain/planning.mjs +37 -0
- package/src/domain/prose.mjs +10 -2
- package/src/domain/relative-specifiers.mjs +12 -0
- package/src/domain/router/registry.mjs +3 -2
- package/src/domain/router/results.mjs +5 -18
- package/src/domain/seeded-random.mjs +33 -0
- package/src/domain/syllogise.mjs +10 -7
- package/src/domain/text-stats.mjs +31 -0
- package/src/services/chat.mjs +722 -184
- package/src/services/extract-facts.mjs +155 -0
- package/src/services/import-file.mjs +2 -2
- package/src/services/init.mjs +2 -2
- package/src/services/ledger-viz.mjs +6 -1
- package/src/services/sentences.mjs +26 -0
- package/src/surfaces/web/memory-ask-browser.bundle.js +11390 -360
- package/src/tools/graph-load.mjs +7 -1
- package/src/tools/readme-docs.mjs +113 -0
- package/src/tools/schema-docs.mjs +2 -2
- package/ROADMAP.md +0 -129
- package/corpus/namenet/generate.mjs +0 -309
- package/corpus/wordnet/generate.mjs +0 -332
- package/src/adapters/prose-tokens.mjs +0 -98
- package/src/adapters/wordnet-source.mjs +0 -70
- package/src/domain/corpus-matrix.mjs +0 -87
- package/src/domain/inflect.mjs +0 -67
- package/src/domain/licences.mjs +0 -68
- package/src/domain/markdown-links.mjs +0 -55
- package/src/domain/persona/codegen.mjs +0 -123
- package/src/domain/publish-gate.mjs +0 -41
- package/src/domain/schemaorg/turtle.mjs +0 -25
- package/src/domain/semcor/parse.mjs +0 -87
- package/src/domain/version-stamp.mjs +0 -36
- package/src/domain/wordnet/yaml.mjs +0 -133
package/src/tools/graph-load.mjs
CHANGED
|
@@ -11,10 +11,16 @@ export async function loadGraph(config, source) {
|
|
|
11
11
|
if (!graph.individuals.length) {
|
|
12
12
|
// Honest miss, never a stack: a fresh repo simply has no graph yet (the chat
|
|
13
13
|
// session itself creates one as the conversation folds in).
|
|
14
|
-
|
|
14
|
+
const e = new ToolError(
|
|
15
15
|
`the graph at ${config.graphFile} is empty — no entities to answer from yet ` +
|
|
16
16
|
"(this repo starts with no graph; the chat session folds the conversation into one).",
|
|
17
17
|
);
|
|
18
|
+
// An empty CODE graph is not an empty world: a caller that can still answer
|
|
19
|
+
// from the lexicon, the corpus or taught memory reads this flag and carries
|
|
20
|
+
// on, rather than reporting the graph's emptiness to someone who never
|
|
21
|
+
// asked a structural question.
|
|
22
|
+
e.emptyGraph = true;
|
|
23
|
+
throw e;
|
|
18
24
|
}
|
|
19
25
|
return graph;
|
|
20
26
|
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// Render README.md's tool section from the tool definitions (src/tools/definitions.mjs),
|
|
2
|
+
// which are the only place a tool's name, schema, purpose or example is written down.
|
|
3
|
+
// scripts/generate-tool-docs.mjs reads and writes the README around this;
|
|
4
|
+
// test/estate/tool-docs.test.mjs runs the same comparison, so a definition edited
|
|
5
|
+
// without regenerating turns the suite red.
|
|
6
|
+
import { HOT_TOOLS, COLD_TOOLS, TOOL_DEFINITIONS, askLexicon, toolByName } from "./definitions.mjs";
|
|
7
|
+
|
|
8
|
+
export const BEGIN_MARKER = "<!-- generated by scripts/generate-tool-docs.mjs from src/tools/definitions.mjs — do not edit by hand -->";
|
|
9
|
+
export const END_MARKER = "<!-- end generated tool section -->";
|
|
10
|
+
|
|
11
|
+
/** An argument list read off a tool's schema: "`symbol` (required), `depth`". */
|
|
12
|
+
function argsOf({ inputSchema }) {
|
|
13
|
+
const required = new Set(inputSchema.required || []);
|
|
14
|
+
const names = Object.keys(inputSchema.properties || {});
|
|
15
|
+
if (!names.length) return "none";
|
|
16
|
+
return names.map((n) => `\`${n}\`${required.has(n) ? " (required)" : ""}`).join(", ");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** The README's tool section, rendered. Pure — the definitions are the only input. */
|
|
20
|
+
export function renderToolDocs() {
|
|
21
|
+
const ask = toolByName("tmct_ask");
|
|
22
|
+
const out = [];
|
|
23
|
+
|
|
24
|
+
out.push("## The tool surface");
|
|
25
|
+
out.push("");
|
|
26
|
+
out.push(
|
|
27
|
+
`Everything above runs on the same ${TOOL_DEFINITIONS.length} tools. Each one is read-only, ` +
|
|
28
|
+
"answers one question in a single call, and returns bounded output. None of them " +
|
|
29
|
+
"calls a model. A tool that cannot ground an answer says so — the same honest miss " +
|
|
30
|
+
"you get everywhere else in tmct.",
|
|
31
|
+
);
|
|
32
|
+
out.push("");
|
|
33
|
+
out.push(
|
|
34
|
+
`Three of them are **hot**: their schemas stay resident, so an agent driving tmct sees ` +
|
|
35
|
+
"them every turn and reaches for one call instead of a Read/Grep loop.",
|
|
36
|
+
);
|
|
37
|
+
out.push("");
|
|
38
|
+
for (const t of HOT_TOOLS) {
|
|
39
|
+
out.push(`- **\`${t.name}\`** — ${t.summary}`);
|
|
40
|
+
out.push(` Arguments: ${argsOf(t)}.`);
|
|
41
|
+
}
|
|
42
|
+
out.push("");
|
|
43
|
+
|
|
44
|
+
// ---- the chat-facing tool: its grammar, its lexicon, and worked examples ----
|
|
45
|
+
out.push("### Asking in plain English");
|
|
46
|
+
out.push("");
|
|
47
|
+
out.push(
|
|
48
|
+
"`tmct_ask` is the chat-facing tool. It takes a structural question as you would type " +
|
|
49
|
+
"it and resolves it to a real traversal. These are the shapes it reads:",
|
|
50
|
+
);
|
|
51
|
+
out.push("");
|
|
52
|
+
out.push("| you type | it answers with |");
|
|
53
|
+
out.push("| --- | --- |");
|
|
54
|
+
for (const g of ask.chat.grammar) out.push(`| ${g.example} | ${g.answers} |`);
|
|
55
|
+
out.push("");
|
|
56
|
+
out.push(
|
|
57
|
+
`The \`<where-marker>\` slot takes any of ${ask.chat.whereMarkers.map((m) => `*${m}*`).join(", ")}. ` +
|
|
58
|
+
"The relation verb is the part that carries the meaning, and each relation has its own " +
|
|
59
|
+
"vocabulary rather than one blessed keyword:",
|
|
60
|
+
);
|
|
61
|
+
out.push("");
|
|
62
|
+
for (const { kind, verbs } of askLexicon()) {
|
|
63
|
+
out.push(`- **${kind}** — ${verbs.map((v) => `*${v}*`).join(", ")}, and more`);
|
|
64
|
+
}
|
|
65
|
+
out.push("");
|
|
66
|
+
out.push("Every question in that table runs against the example graph:");
|
|
67
|
+
out.push("");
|
|
68
|
+
out.push("```bash cwd=repo");
|
|
69
|
+
for (const g of ask.chat.grammar) {
|
|
70
|
+
out.push(
|
|
71
|
+
`node bin/tmct.mjs cli tmct_ask '${JSON.stringify({ query: g.example, repo_path: ask.chat.exampleRepo })}'`,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
out.push("```");
|
|
75
|
+
out.push("");
|
|
76
|
+
|
|
77
|
+
// ---- the cold tools ----
|
|
78
|
+
out.push("### The rest of the tools");
|
|
79
|
+
out.push("");
|
|
80
|
+
out.push(
|
|
81
|
+
"The remaining tools are **cold**: still served, but not billed to an agent every turn. " +
|
|
82
|
+
"Reach one through `tmct cli <tool>`, passing its arguments as JSON:",
|
|
83
|
+
);
|
|
84
|
+
out.push("");
|
|
85
|
+
out.push("| tool | what it answers | arguments |");
|
|
86
|
+
out.push("| --- | --- | --- |");
|
|
87
|
+
for (const t of COLD_TOOLS) out.push(`| \`${t.name}\` | ${t.summary} | ${argsOf(t)} |`);
|
|
88
|
+
out.push("");
|
|
89
|
+
out.push(
|
|
90
|
+
"Add `repo_path` to any of them to point at a repository other than the working " +
|
|
91
|
+
"directory. `tmct init` also writes this catalog, with a worked invocation per tool, " +
|
|
92
|
+
"to `.tmct/TOOLS.md` inside the repo it indexed.",
|
|
93
|
+
);
|
|
94
|
+
out.push("");
|
|
95
|
+
out.push("```bash cwd=repo");
|
|
96
|
+
out.push(`node bin/tmct.mjs cli tmct_untested '${JSON.stringify({ repo_path: "examples/mini-webapp" })}'`);
|
|
97
|
+
out.push("```");
|
|
98
|
+
|
|
99
|
+
return out.join("\n");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Splice the rendered section between the markers in `readme`. Throws when the
|
|
103
|
+
* markers are missing, so a lost section fails loudly rather than appending a copy. */
|
|
104
|
+
export function spliceToolDocs(readme, rendered) {
|
|
105
|
+
const begin = readme.indexOf(BEGIN_MARKER);
|
|
106
|
+
const end = readme.indexOf(END_MARKER);
|
|
107
|
+
if (begin === -1 || end === -1 || end < begin) {
|
|
108
|
+
throw new Error(`README.md is missing the generated tool-section markers (${BEGIN_MARKER})`);
|
|
109
|
+
}
|
|
110
|
+
const head = readme.slice(0, begin + BEGIN_MARKER.length);
|
|
111
|
+
const tail = readme.slice(end);
|
|
112
|
+
return `${head}\n\n${rendered}\n\n${tail}`;
|
|
113
|
+
}
|
|
@@ -154,10 +154,10 @@ export const PREDICATE_DOCS = Object.freeze([
|
|
|
154
154
|
{ prop: "seon:isConstant", kind: "attribute", description:
|
|
155
155
|
"An ALL_CAPS module-level GlobalVariable — a naming-convention signal, not enforced " +
|
|
156
156
|
"immutability." },
|
|
157
|
-
{ prop: "
|
|
157
|
+
{ prop: "mgx:subKind", kind: "attribute", description:
|
|
158
158
|
"The flavour of a Class define when it is not a plain class: interface, enum, struct " +
|
|
159
159
|
"or record. The graph keeps kind=class for every type declaration; this attribute " +
|
|
160
|
-
"carries the distinction." },
|
|
160
|
+
"carries the distinction. A tmct extension — SEON has no subKind property." },
|
|
161
161
|
{ prop: "seon:hasAccessModifier", kind: "attribute", description:
|
|
162
162
|
"Visibility inferred from a leading underscore (private/protected); a public member " +
|
|
163
163
|
"carries no value for this attribute." },
|
package/ROADMAP.md
DELETED
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
# ROADMAP — tmct's current shape and what's next
|
|
2
|
-
|
|
3
|
-
Forward-looking at a **feature level**: what tmct is capable of right now, and what's planned next.
|
|
4
|
-
No session narrative, no dated diary, no "shipped/DONE" history — that's what git log and the
|
|
5
|
-
`archive/`/`BENCHMARK_*.md`/`CAPABILITIES_*.md` records are for. For **task-level** pickup (specific
|
|
6
|
-
open items, session-scoped), see `HANDOVER.md` instead — this file doesn't duplicate that list.
|
|
7
|
-
|
|
8
|
-
## What tmct is
|
|
9
|
-
|
|
10
|
-
A tolerant, ELIZA/PARRY-style chat surface over a codebase, obsessed with software the way PARRY was
|
|
11
|
-
obsessed with the mafia — deterministic, zero-cost, **no LLM anywhere in the product path**. Guides a
|
|
12
|
-
user toward precision queries rather than guessing; every answer is grounded, restates every genuine
|
|
13
|
-
reading it finds in full, or is an honest miss when nothing grounds it at all. Its visual surfaces —
|
|
14
|
-
the ledger explorer with its in-browser chat (`tmct viz`), the animated plan page
|
|
15
|
-
(`chat --prompt … --render blocks`), and the Pages homepage hero — are the same graph read out loud:
|
|
16
|
-
same engine, same provenance, no LLM.
|
|
17
|
-
|
|
18
|
-
## Ambition
|
|
19
|
-
|
|
20
|
-
Declared, forward-looking goals — not yet achieved, stated here so they steer future work instead of
|
|
21
|
-
getting silently traded away by inherited caution:
|
|
22
|
-
|
|
23
|
-
- **Reach for Llama-3-level natural language fluency.** by growing rich
|
|
24
|
-
template/surface-realization variety, so an answer shape has many valid phrasings instead of one
|
|
25
|
-
fixed slot-fill.
|
|
26
|
-
- **Resolve ambiguity breadth-first, always.** Every genuinely valid reading gets its own real answer
|
|
27
|
-
restated in full, never a bare "could mean X or Y — try rephrasing" punt, bounded only by existing
|
|
28
|
-
clipping/pagination limits. L
|
|
29
|
-
- **Paraphrase alongside the original, verified, never instead of it.** A surface-realization variant
|
|
30
|
-
sits next to the literal grounded answer, never replacing it, and its accuracy is checked, not
|
|
31
|
-
assumed — by running tmct's own deterministic inference/consistency machinery (`src/domain/syllogise.mjs`)
|
|
32
|
-
against both the original and the paraphrase: they must entail the same conclusions, and neither may
|
|
33
|
-
contradict the other sentence-by-sentence..
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
## What's next (feature-shaped — see `HANDOVER.md` for the current task-level list)
|
|
37
|
-
|
|
38
|
-
- **`PLAN_ADVENTURE.md`** — a text-adventure architectural stretch. Its world-state and
|
|
39
|
-
actions-as-data substrate shipped generically with the planning lane (action rule kinds,
|
|
40
|
-
per-step board snapshots, legal-move enumeration); what remains its own is the imperative
|
|
41
|
-
command grammar ("go north", "take the key"), the NPC turn scheduler, the Ashcombe Hall
|
|
42
|
-
corpus, and the room-look digest.
|
|
43
|
-
- **`PLAN_SYLLOGIST.md`** — the reasoning engine's research horizon. The single-justification
|
|
44
|
-
retraction slice shipped (`retractSubClassOf`, justification persistence and cascade across all
|
|
45
|
-
five rules); still open there: the ATMS generalization (alternate justification sets per fact),
|
|
46
|
-
incremental matching (§2), and relevance under budget (§4).
|
|
47
|
-
- **`PLAN_GUESS_NUMBER.md`** — closed-loop planning over hidden state (belief-interval bisection,
|
|
48
|
-
thinker-mode secret commitment, observation folding) on top of the shipped planner substrate.
|
|
49
|
-
Design-only.
|
|
50
|
-
- **`PLAN_CODE.md`** — small JS-function and HTML/CSS-fragment synthesis, plus goal-directed
|
|
51
|
-
program repair (tests as the goal state, mutation templates as planning actions), via a sandboxed
|
|
52
|
-
headless browser (Track 1, rule/frame synthesis, already shipped). Blocked on a sandbox
|
|
53
|
-
dependency decision.
|
|
54
|
-
- **`PLAN_AGENTS.md`** — the governing plan for tmct's broader multi-repo arc (marginalia, seonix,
|
|
55
|
-
a pluggable LLM rung for Claude Code/Bedrock/Copilot). Check its own sequencing table for current
|
|
56
|
-
phase status, not this file.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
## Research horizon
|
|
60
|
-
|
|
61
|
-
*(2026-07-08 research pass — a direction recorded so it isn't re-discovered from scratch, not a
|
|
62
|
-
committed build plan. Nothing below is scheduled.)*
|
|
63
|
-
|
|
64
|
-
**Before the horizon — known-how, no research risk**, just scheduling: `PLAN_CODE.md` Tracks 2-4
|
|
65
|
-
(mutation search/repair, JS/HTML/CSS synthesis — APR and CEGIS are established techniques);
|
|
66
|
-
RETE/incremental forward-chaining (`PLAN_SYLLOGIST.md` §2 — Forgy 1982, a citable algorithm not yet
|
|
67
|
-
ported); contingent/conformant planning under initial-state uncertainty (Bonet & Geffner 2000,
|
|
68
|
-
Hoffmann & Brafman 2006, Petrick & Bacchus 2002 all have working algorithms, none yet applied here).
|
|
69
|
-
|
|
70
|
-
**After the horizon — genuinely unsolved in the field**, named as real research targets with
|
|
71
|
-
citations, not stop signs:
|
|
72
|
-
- **The frame problem / relevance realization** (open-world planning boundary). McCarthy & Hayes
|
|
73
|
-
1969 named it; Jaeger, Riedl, Djedovic, Vervaeke & Walsh (2024) argue it may not be algorithmically
|
|
74
|
-
solvable in the general case. Speculative angle: bounded (N+1) goal recognition — recognize
|
|
75
|
-
declared goal 1..N, or reject to an explicit "escalate" class, via parse-shape membership.
|
|
76
|
-
- **Bounded, incremental, trust-tiered, retraction-safe justification tracking** — `PLAN_SYLLOGIST.md`
|
|
77
|
-
§3. Doyle's JTMS (1979) and de Kleer's ATMS (1986) solve retraction; DRed/RDFox's Backward-Forward
|
|
78
|
-
solve incremental Datalog maintenance; nobody's published the combination with tmct's
|
|
79
|
-
multi-trust-tier, hard-budget requirement. The JTMS-lite slice shipped (one persisted
|
|
80
|
-
justification per entailed fact, VERIFY-backed retraction, all five rules); the open piece is the
|
|
81
|
-
ATMS generalization — alternate justification SETS per fact (see that doc's 2026-07-15 addendum).
|
|
82
|
-
- **A shared ~2M-word cross-domain ontology** (general-English + technical/scientific/programming).
|
|
83
|
-
Merging collides senses of lexically-shared words (`class`, `cache`, `thread`, `field`, `state`)
|
|
84
|
-
across registers; knowledge-based WSD is real but weaker than supervised/neural WSD (Lesk 1986;
|
|
85
|
-
Raganato, Camacho-Collados & Navigli, EACL 2017). BabelNet proves cross-resource sense merging is
|
|
86
|
-
achievable at scale but solves the cross-*lingual*, not cross-*domain*, axis, and carries a
|
|
87
|
-
non-commercial licence. Speculative angle: mutual disambiguation from already-resolved neighbouring
|
|
88
|
-
terms in tmct's own closed graph (a bounded reading of Gale/Church/Yarowsky's "one sense per
|
|
89
|
-
discourse" regularity) — not published anywhere found for this application. Fresh live instance
|
|
90
|
-
(2026-07-11): `"tail"` (Unix process vs. animal body part) collides under `normFactTerm`'s
|
|
91
|
-
cross-corpus flattening, `src/adapters/memory/core.mjs:1109-1134`.
|
|
92
|
-
|
|
93
|
-
**Tier-4: learn-on-miss acquisition**. The strongest
|
|
94
|
-
miss signal tmct can emit: lexicon term recognized, query built cleanly, zero matches anywhere — the
|
|
95
|
-
question was well-formed and the knowledge is simply absent. Web search on the resolved term → clean
|
|
96
|
-
the fetched text into the ACE-OWL controlled grammar → store with source provenance → answer the
|
|
97
|
-
original question, citing what was just learned. Strictly opt-in, offline default inviolable.
|
|
98
|
-
Prerequisites: the provenance-trust policy must extend to `via:"learned:web"`, never silently
|
|
99
|
-
blending web-sourced facts with graph/operator facts.
|
|
100
|
-
|
|
101
|
-
## Design docs
|
|
102
|
-
|
|
103
|
-
Every substantial design lives in its own `PLAN_*.md` at the repo root; `archive/` holds the shipped
|
|
104
|
-
and closed ones. This file points to them, it doesn't repeat their content. Each plan states its own
|
|
105
|
-
status in its opening lines — read it there, because a status quoted here would rot.
|
|
106
|
-
|
|
107
|
-
| Plan | What it's for |
|
|
108
|
-
| --- | --- |
|
|
109
|
-
| [PLAN_ADVENTURE.md](PLAN_ADVENTURE.md) | a text adventure as an architectural stretch: imperative command grammar, NPC turn scheduler, room-look digest |
|
|
110
|
-
| [PLAN_AGENTS.md](PLAN_AGENTS.md) | the governing plan for the multi-repo arc (marginalia, seonix, a pluggable LLM rung), with its own phase sequencing |
|
|
111
|
-
| [PLAN_CHILD_CORPUS.md](PLAN_CHILD_CORPUS.md) | a wider default seed corpus, chosen by age of acquisition |
|
|
112
|
-
| [PLAN_CLASS_QUERY.md](PLAN_CLASS_QUERY.md) | "list/count all X of class Y", reconciled against what already shipped |
|
|
113
|
-
| [PLAN_CODE.md](PLAN_CODE.md) | program synthesis over tmct's closed DSLs, plus JS/HTML/CSS fragments and goal-directed program repair |
|
|
114
|
-
| [PLAN_CONSISTENCY_CHECK.md](PLAN_CONSISTENCY_CHECK.md) | tmct as a consistency service for an LLM tool loop |
|
|
115
|
-
| [PLAN_EMBEDDINGS.md](PLAN_EMBEDDINGS.md) | the semantic-similarity axis, and the way back to it |
|
|
116
|
-
| [PLAN_GRAPH_SCAN.md](PLAN_GRAPH_SCAN.md) | seed and query cost at `init:xl`/`init:xxl` corpus scale |
|
|
117
|
-
| [PLAN_GUESS_NUMBER.md](PLAN_GUESS_NUMBER.md) | closed-loop planning over hidden state, via belief-interval bisection |
|
|
118
|
-
| [PLAN_MUD.md](PLAN_MUD.md) | persistent, shared tmct worlds over a `server:` memory backend |
|
|
119
|
-
| [PLAN_NLU_BENCHMARKS.md](PLAN_NLU_BENCHMARKS.md) | scoring tmct on the CLINC150 and HWU64 intent sets |
|
|
120
|
-
| [PLAN_OPEN_ITEMS.md](PLAN_OPEN_ITEMS.md) | the build order closing the backlog `HANDOVER.md` carries |
|
|
121
|
-
| [PLAN_PARAPHRASE_VERIFICATION.md](PLAN_PARAPHRASE_VERIFICATION.md) | checking a paraphrase against the graph before it prints |
|
|
122
|
-
| [PLAN_PURGE.md](PLAN_PURGE.md) | promote the load-bearing code, delete the dead weight |
|
|
123
|
-
| [PLAN_REPO_INDEX.md](PLAN_REPO_INDEX.md) | tmct grows its own code parsers, ported from seonix |
|
|
124
|
-
| [PLAN_SYLLOGIST.md](PLAN_SYLLOGIST.md) | the reasoning engine's incrementality and retraction horizon |
|
|
125
|
-
| [PLAN_SYLLOGIST_EL_DL.md](PLAN_SYLLOGIST_EL_DL.md) | beyond OWL 2 RL: an EL classifier, then a DL tableau prover |
|
|
126
|
-
|
|
127
|
-
`SKILL_*.md` docs specify the repeatable measurement and build cycles (the benchmarks, the capability
|
|
128
|
-
audit, the background strategy advisor, plain-prose writing). `HANDOVER.md` is the single
|
|
129
|
-
current-open-items list.
|
|
@@ -1,309 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// corpus/namenet/generate.mjs — converts THREE reviewed CSVs from a LOCAL
|
|
3
|
-
// Open English Namenet checkout into ConceptNet-shape fact rows. Mirrors
|
|
4
|
-
// corpus/wordnet/generate.mjs's structure/conventions exactly (same
|
|
5
|
-
// maintainer-tool framing, same deterministic sorted JSONL + manifest.json
|
|
6
|
-
// output shape) — smaller scope, OPTIONAL top-up, not load-bearing.
|
|
7
|
-
//
|
|
8
|
-
// node corpus/namenet/generate.mjs [namenetDir]
|
|
9
|
-
// TMCT_NAMENET_DIR=/path/to/english-namenet node corpus/namenet/generate.mjs
|
|
10
|
-
//
|
|
11
|
-
// Input: `~/projects/globalwordnet/english-namenet/` by default (a LOCAL
|
|
12
|
-
// clone, never vendored/committed) — three reviewed CSVs, each a
|
|
13
|
-
// human/algorithm-curated LINKING TABLE between a name/label and an Open
|
|
14
|
-
// English WordNet (OEWN) synset or lemma set:
|
|
15
|
-
// - species_reviewed.csv (5,101 rows) — Scientific Name -> SSID
|
|
16
|
-
// - taxon2common_reviewed.csv (2,368 rows) — SSID 1/Lemmas 1 -> SSID 2/Lemmas 2
|
|
17
|
-
// - linked_occupations_reviewed.csv (2,193 rows) — Wikidata Labels -> SSID/Lemma
|
|
18
|
-
// Every one of the three, once you read real rows (not just the header),
|
|
19
|
-
// turns out to be the SAME shape underneath: two name-lists that denote the
|
|
20
|
-
// SAME real-world thing (a species, a folk-taxonomic category, an
|
|
21
|
-
// occupation), reviewed/accepted by a human as a correct link — never a
|
|
22
|
-
// hierarchy (broader/narrower) or capability claim. That is why every fact
|
|
23
|
-
// this converter emits uses ONE relation, /r/Synonym ("X means the same as
|
|
24
|
-
// Y") — confirmed against conceptnet-map.toml (ace != "none", Phase 1's
|
|
25
|
-
// 2026-07-12 widening) rather than invented here (this task's own scope
|
|
26
|
-
// boundary: conversion only, reuse what Phase 1 already mapped, never add a
|
|
27
|
-
// new relation row).
|
|
28
|
-
//
|
|
29
|
-
// Why NOT /r/IsA or /r/CapableOf (the task brief's own initial guesses,
|
|
30
|
-
// before real rows were read):
|
|
31
|
-
// - species_reviewed.csv: real rows show the "Scientific Name" is almost
|
|
32
|
-
// always ALREADY one of the target SSID's own WordNet members (4,885 of
|
|
33
|
-
// 4,897 accepted rows resolve to a real synset; of those, 4,881 have the
|
|
34
|
-
// scientific name as a literal member string) — this is a same-referent
|
|
35
|
-
// alias table (which of several ambiguous WordNet senses a Wikidata
|
|
36
|
-
// taxon QID actually means), not a species/kind subclass relation.
|
|
37
|
-
// - linked_occupations_reviewed.csv: despite the CSV's name, each row
|
|
38
|
-
// links a WIKIDATA OCCUPATION ENTITY's labels to a WORDNET OCCUPATION
|
|
39
|
-
// SYNSET's lemmas (e.g. "politician, political leader" <-> "pol,
|
|
40
|
-
// political leader, politician, politico") — it is NOT a person linked
|
|
41
|
-
// to their job (no person names anywhere in this file), so /r/CapableOf
|
|
42
|
-
// ("a person can politician") would be nonsensical. It is the same
|
|
43
|
-
// alias-table shape as the other two.
|
|
44
|
-
//
|
|
45
|
-
// species_reviewed.csv needs a SECOND local checkout to resolve: its SSID
|
|
46
|
-
// column has no lemma text of its own (unlike the other two, which carry
|
|
47
|
-
// "Lemmas N" columns directly), so this converter reuses
|
|
48
|
-
// corpus/wordnet/generate.mjs's already-proven `loadAllSynsets`/
|
|
49
|
-
// `DEFAULT_YAML_DIR`/`encodeTerm`/`humanize` (imported, never duplicated —
|
|
50
|
-
// same discipline that file's own header comment describes for its
|
|
51
|
-
// hand-rolled YAML reader) to resolve SSID -> representative lemma.
|
|
52
|
-
// corpus/wordnet/generate.mjs itself is never modified (task scope
|
|
53
|
-
// boundary) — only its exported pure functions are called.
|
|
54
|
-
//
|
|
55
|
-
// NOT part of the product path — a maintainer tool, run by hand, offline,
|
|
56
|
-
// $0; its OUTPUT (corpus/namenet/namenet.jsonl + manifest.json) is what gets
|
|
57
|
-
// committed, never the source CSVs themselves.
|
|
58
|
-
//
|
|
59
|
-
// Licence: see LICENSE-NOTICE in this directory — the source repository
|
|
60
|
-
// (globalwordnet/english-namenet) declares NO explicit license of its own
|
|
61
|
-
// (confirmed via GitHub repo metadata, 2026-07-12: `license: null`, no
|
|
62
|
-
// LICENSE file, no license statement in README.md); this bundle is
|
|
63
|
-
// distributed under CC-BY-4.0 as a conservative match to the Open English
|
|
64
|
-
// WordNet data it is built from and links against, pending clarification
|
|
65
|
-
// from the GlobalWordNet team. The code in this file is tmct code under the
|
|
66
|
-
// repository's MPL-2.0; only the generated data
|
|
67
|
-
// (corpus/namenet/namenet.jsonl) carries that CC-BY-4.0 label.
|
|
68
|
-
|
|
69
|
-
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
|
70
|
-
import { homedir } from "node:os";
|
|
71
|
-
import { createHash } from "node:crypto";
|
|
72
|
-
import { fileURLToPath } from "node:url";
|
|
73
|
-
import { dirname, join } from "node:path";
|
|
74
|
-
import { loadAllSynsets, DEFAULT_YAML_DIR, resolveYamlDir, encodeTerm, humanize } from "../wordnet/generate.mjs";
|
|
75
|
-
|
|
76
|
-
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
77
|
-
export const NAMENET_OUT_DIR = HERE;
|
|
78
|
-
|
|
79
|
-
export const DEFAULT_NAMENET_DIR = join(homedir(), "projects", "globalwordnet", "english-namenet");
|
|
80
|
-
|
|
81
|
-
/** Resolve the input namenet directory: CLI positional arg > env var >
|
|
82
|
-
* default. Pure (argv/env injectable), mirrors wordnet's resolveYamlDir. */
|
|
83
|
-
export function resolveNamenetDir(argv = process.argv.slice(2), env = process.env) {
|
|
84
|
-
return argv[0] || env.TMCT_NAMENET_DIR || DEFAULT_NAMENET_DIR;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
// ---- CSV parsing (pure, unit-tested) ---------------------------------------
|
|
88
|
-
// A small hand-rolled RFC4180-ish reader — no dependency added, same house
|
|
89
|
-
// style as corpus/wordnet/generate.mjs reusing a hand-rolled YAML reader
|
|
90
|
-
// rather than pulling in a general parsing library. Handles quoted fields
|
|
91
|
-
// (commas/newlines inside quotes, "" as an escaped literal quote) and both
|
|
92
|
-
// CRLF and LF line endings — all three source CSVs use quoted fields for any
|
|
93
|
-
// value containing a comma (e.g. `"species, by Garsault, 1764..."`), so a
|
|
94
|
-
// naive `.split(",")` silently misaligns columns on those rows.
|
|
95
|
-
|
|
96
|
-
/** Parse CSV text into an array of records (arrays of string fields). */
|
|
97
|
-
export function parseCsvRecords(text) {
|
|
98
|
-
const records = [];
|
|
99
|
-
let row = [];
|
|
100
|
-
let field = "";
|
|
101
|
-
let inQuotes = false;
|
|
102
|
-
const pushField = () => { row.push(field); field = ""; };
|
|
103
|
-
const pushRow = () => { pushField(); records.push(row); row = []; };
|
|
104
|
-
const src = String(text ?? "");
|
|
105
|
-
for (let i = 0; i < src.length; i++) {
|
|
106
|
-
const c = src[i];
|
|
107
|
-
if (inQuotes) {
|
|
108
|
-
if (c === '"') {
|
|
109
|
-
if (src[i + 1] === '"') { field += '"'; i++; }
|
|
110
|
-
else inQuotes = false;
|
|
111
|
-
} else {
|
|
112
|
-
field += c;
|
|
113
|
-
}
|
|
114
|
-
continue;
|
|
115
|
-
}
|
|
116
|
-
if (c === '"') { inQuotes = true; continue; }
|
|
117
|
-
if (c === ",") { pushField(); continue; }
|
|
118
|
-
if (c === "\r") continue; // CRLF -> swallow, \n below ends the row
|
|
119
|
-
if (c === "\n") { pushRow(); continue; }
|
|
120
|
-
field += c;
|
|
121
|
-
}
|
|
122
|
-
// final field/row, if the text didn't end with a newline
|
|
123
|
-
if (field !== "" || row.length) pushRow();
|
|
124
|
-
// drop a single trailing wholly-empty record (trailing newline artifact)
|
|
125
|
-
if (records.length && records[records.length - 1].every((f) => f === "")) records.pop();
|
|
126
|
-
return records;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/** Parse CSV text into an array of row objects keyed by the header row. */
|
|
130
|
-
export function parseCsv(text) {
|
|
131
|
-
const records = parseCsvRecords(text);
|
|
132
|
-
if (!records.length) return [];
|
|
133
|
-
const header = records[0].map((h) => h.trim());
|
|
134
|
-
return records.slice(1).map((rec) => {
|
|
135
|
-
const obj = {};
|
|
136
|
-
for (let i = 0; i < header.length; i++) obj[header[i]] = rec[i] ?? "";
|
|
137
|
-
return obj;
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/** "Plantae, kingdom Plantae, plant kingdom" -> "Plantae" — the first
|
|
142
|
-
* candidate in a comma-separated lemma/label list, the same "first member is
|
|
143
|
-
* representative" convention corpus/wordnet/generate.mjs's repTerm() uses. */
|
|
144
|
-
export function firstOf(commaList) {
|
|
145
|
-
const s = String(commaList ?? "").trim();
|
|
146
|
-
if (!s) return null;
|
|
147
|
-
const first = s.split(",")[0].trim();
|
|
148
|
-
return first || null;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
// ---- shared row builder (pure) ---------------------------------------------
|
|
152
|
-
// Same dedupe-by-key + self-loop-skip discipline as corpus/wordnet/
|
|
153
|
-
// generate.mjs's makeRowBuilder — re-declared locally (not imported; that
|
|
154
|
-
// function isn't exported, and this is a small enough shape to keep local
|
|
155
|
-
// rather than widen wordnet/generate.mjs's exports for a five-line helper).
|
|
156
|
-
|
|
157
|
-
function makeRowBuilder() {
|
|
158
|
-
const rows = new Map();
|
|
159
|
-
const add = (rawSubject, rel, rawObject) => {
|
|
160
|
-
const start = encodeTerm(rawSubject);
|
|
161
|
-
const end = encodeTerm(rawObject);
|
|
162
|
-
if (!start || !end || start === end) return; // self-loop / empty term — noise, not a fact
|
|
163
|
-
const key = `${rel} ${start} ${end}`;
|
|
164
|
-
if (rows.has(key)) return;
|
|
165
|
-
rows.set(key, {
|
|
166
|
-
start,
|
|
167
|
-
rel,
|
|
168
|
-
end,
|
|
169
|
-
weight: 1,
|
|
170
|
-
surfaceText: `[[${humanize(rawSubject)}]] ${rel.replace("/r/", "")} [[${humanize(rawObject)}]]`,
|
|
171
|
-
});
|
|
172
|
-
};
|
|
173
|
-
return { rows, add };
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
const sortRows = (rows) => rows.slice().sort((a, b) => (
|
|
177
|
-
a.rel !== b.rel ? (a.rel < b.rel ? -1 : 1)
|
|
178
|
-
: a.start !== b.start ? (a.start < b.start ? -1 : 1)
|
|
179
|
-
: a.end < b.end ? -1 : a.end > b.end ? 1 : 0
|
|
180
|
-
));
|
|
181
|
-
|
|
182
|
-
// ---- per-source mappers (pure, unit-tested) --------------------------------
|
|
183
|
-
|
|
184
|
-
/** species_reviewed.csv: accepted rows only; the Scientific Name and the
|
|
185
|
-
* SSID's representative WordNet lemma denote the same species -> /r/Synonym.
|
|
186
|
-
* `bySynset` is the same `Map<synsetId, {members}>` shape
|
|
187
|
-
* corpus/wordnet/generate.mjs's loadAllSynsets returns (or a small fixture
|
|
188
|
-
* Map in tests) — a row whose SSID isn't in the map is skipped, not thrown. */
|
|
189
|
-
export function buildSpeciesFacts(rows, bySynset) {
|
|
190
|
-
const { rows: out, add } = makeRowBuilder();
|
|
191
|
-
for (const row of rows) {
|
|
192
|
-
if (row.Accept !== "TRUE") continue;
|
|
193
|
-
const sciName = row["Scientific Name"];
|
|
194
|
-
const ssid = row.SSID;
|
|
195
|
-
if (!sciName || !ssid) continue;
|
|
196
|
-
const synset = bySynset.get(ssid);
|
|
197
|
-
const members = Array.isArray(synset?.members) ? synset.members : [];
|
|
198
|
-
if (!members.length) continue; // unresolved SSID — skip, don't throw
|
|
199
|
-
add(sciName, "/r/Synonym", members[0]);
|
|
200
|
-
}
|
|
201
|
-
return sortRows([...out.values()]);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
/** taxon2common_reviewed.csv: accepted rows only; the first lemma of each
|
|
205
|
-
* side's "Lemmas N" list denotes the same taxonomic/folk category ->
|
|
206
|
-
* /r/Synonym. No cross-reference needed — both lemma lists are already
|
|
207
|
-
* columns in this CSV. */
|
|
208
|
-
export function buildTaxon2CommonFacts(rows) {
|
|
209
|
-
const { rows: out, add } = makeRowBuilder();
|
|
210
|
-
for (const row of rows) {
|
|
211
|
-
if (row.Accept !== "TRUE") continue;
|
|
212
|
-
const a = firstOf(row["Lemmas 1"]);
|
|
213
|
-
const b = firstOf(row["Lemmas 2"]);
|
|
214
|
-
if (!a || !b) continue;
|
|
215
|
-
add(a, "/r/Synonym", b);
|
|
216
|
-
}
|
|
217
|
-
return sortRows([...out.values()]);
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
/** linked_occupations_reviewed.csv: accepted, genuine-occupation rows only
|
|
221
|
-
* (`Accept === "TRUE"` AND `"Not an occupation" !== "TRUE"`); the first
|
|
222
|
-
* Wikidata label and the first WordNet lemma denote the same occupation ->
|
|
223
|
-
* /r/Synonym. */
|
|
224
|
-
export function buildOccupationFacts(rows) {
|
|
225
|
-
const { rows: out, add } = makeRowBuilder();
|
|
226
|
-
for (const row of rows) {
|
|
227
|
-
if (row.Accept !== "TRUE") continue;
|
|
228
|
-
if (row["Not an occupation"] === "TRUE") continue;
|
|
229
|
-
const label = firstOf(row.Labels);
|
|
230
|
-
const lemma = firstOf(row.Lemma);
|
|
231
|
-
if (!label || !lemma) continue;
|
|
232
|
-
add(label, "/r/Synonym", lemma);
|
|
233
|
-
}
|
|
234
|
-
return sortRows([...out.values()]);
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
/** Merge the three per-source fact sets into one deduped, sorted set — the
|
|
238
|
-
* namenet.jsonl content. A pair already emitted by one source (e.g. the
|
|
239
|
-
* same scientific-name/common-name pair surfacing via both
|
|
240
|
-
* species_reviewed.csv and taxon2common_reviewed.csv) is kept once. */
|
|
241
|
-
export function mergeFacts(...factLists) {
|
|
242
|
-
const { rows, add } = makeRowBuilder();
|
|
243
|
-
for (const list of factLists) {
|
|
244
|
-
for (const f of list) add(humanize(f.start.replace(/^\/c\/en\//, "")), f.rel, humanize(f.end.replace(/^\/c\/en\//, "")));
|
|
245
|
-
}
|
|
246
|
-
return sortRows([...rows.values()]);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// ---- output ------------------------------------------------------------
|
|
250
|
-
|
|
251
|
-
const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
252
|
-
const sha256 = (text) => createHash("sha256").update(text).digest("hex");
|
|
253
|
-
|
|
254
|
-
async function readCsv(dir, name) {
|
|
255
|
-
const text = await readFile(join(dir, name), "utf8");
|
|
256
|
-
return parseCsv(text);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
async function main() {
|
|
260
|
-
const namenetDir = resolveNamenetDir();
|
|
261
|
-
const yamlDir = resolveYamlDir([], process.env) || DEFAULT_YAML_DIR;
|
|
262
|
-
process.stderr.write(`corpus/namenet/generate.mjs: reading ${namenetDir}\n`);
|
|
263
|
-
process.stderr.write(` (species_reviewed.csv also needs OEWN synsets from ${yamlDir})\n`);
|
|
264
|
-
|
|
265
|
-
const [speciesRows, taxonRows, occupationRows] = await Promise.all([
|
|
266
|
-
readCsv(namenetDir, "species_reviewed.csv"),
|
|
267
|
-
readCsv(namenetDir, "taxon2common_reviewed.csv"),
|
|
268
|
-
readCsv(namenetDir, "linked_occupations_reviewed.csv"),
|
|
269
|
-
]);
|
|
270
|
-
const { bySynset } = await loadAllSynsets(yamlDir);
|
|
271
|
-
|
|
272
|
-
const speciesFacts = buildSpeciesFacts(speciesRows, bySynset);
|
|
273
|
-
const taxonFacts = buildTaxon2CommonFacts(taxonRows);
|
|
274
|
-
const occupationFacts = buildOccupationFacts(occupationRows);
|
|
275
|
-
const merged = mergeFacts(speciesFacts, taxonFacts, occupationFacts);
|
|
276
|
-
|
|
277
|
-
process.stderr.write(` species_reviewed.csv: ${speciesRows.length} rows -> ${speciesFacts.length} facts\n`);
|
|
278
|
-
process.stderr.write(` taxon2common_reviewed.csv: ${taxonRows.length} rows -> ${taxonFacts.length} facts\n`);
|
|
279
|
-
process.stderr.write(` linked_occupations_reviewed.csv: ${occupationRows.length} rows -> ${occupationFacts.length} facts\n`);
|
|
280
|
-
process.stderr.write(` namenet (merged, deduped): ${merged.length} facts\n`);
|
|
281
|
-
|
|
282
|
-
await mkdir(NAMENET_OUT_DIR, { recursive: true });
|
|
283
|
-
const outText = toJsonl(merged);
|
|
284
|
-
await writeFile(join(NAMENET_OUT_DIR, "namenet.jsonl"), outText);
|
|
285
|
-
|
|
286
|
-
const manifest = {
|
|
287
|
-
version: 1,
|
|
288
|
-
generated: "by corpus/namenet/generate.mjs",
|
|
289
|
-
corpuses: [
|
|
290
|
-
{
|
|
291
|
-
id: "namenet",
|
|
292
|
-
kind: "language",
|
|
293
|
-
description: "Scientific-name/common-name and Wikidata-label/WordNet-lemma synonym pairs, mechanically derived from three human-reviewed Open English Namenet linking tables (species, taxon-to-common-name, occupations). A small top-up bundle, not a primary corpus.",
|
|
294
|
-
source: { kind: "curated", tool: "corpus/namenet/generate.mjs" },
|
|
295
|
-
file: "namenet.jsonl",
|
|
296
|
-
facts: merged.length,
|
|
297
|
-
bytes: Buffer.byteLength(outText),
|
|
298
|
-
sha256: sha256(outText),
|
|
299
|
-
license: "CC-BY-4.0 (source repo declares no explicit license; see LICENSE-NOTICE)",
|
|
300
|
-
},
|
|
301
|
-
],
|
|
302
|
-
};
|
|
303
|
-
const manifestText = JSON.stringify(manifest, null, 2) + "\n";
|
|
304
|
-
await writeFile(join(NAMENET_OUT_DIR, "manifest.json"), manifestText);
|
|
305
|
-
process.stderr.write(`wrote corpus/namenet/manifest.json (${manifest.corpuses.length} corpuses)\n`);
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
const isMain = process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href;
|
|
309
|
-
if (isMain) await main();
|