@polycode-projects/the-mechanical-code-talker 2.3.1 → 2.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +131 -32
  2. package/bin/tmct.mjs +18 -91
  3. package/corpus/README.md +3 -3
  4. package/corpus/seon/README.md +1 -0
  5. package/corpus/tier2/generate.mjs +18 -18
  6. package/corpus/tier2/manifest.json +3 -3
  7. package/data/games/hanoi-3.txt +8 -2
  8. package/package.json +26 -8
  9. package/src/adapters/corpus-lanes.mjs +13 -0
  10. package/src/adapters/graph-build.mjs +5 -7
  11. package/src/adapters/import-closure.mjs +28 -0
  12. package/src/adapters/memory/blocks.mjs +5 -4
  13. package/src/adapters/memory/core.mjs +78 -5
  14. package/src/adapters/memory/shacl.mjs +12 -0
  15. package/src/adapters/providers/graph-service.mjs +12 -5
  16. package/src/adapters/tracked-files.mjs +17 -0
  17. package/src/domain/ask-vocab.mjs +2 -0
  18. package/src/domain/ask.mjs +225 -13
  19. package/src/domain/cli-verbs.mjs +201 -0
  20. package/src/domain/codegraph.mjs +142 -56
  21. package/src/domain/completions/graph-adapter.mjs +1 -1
  22. package/src/domain/completions/group.mjs +3 -17
  23. package/src/domain/completions/infer.mjs +4 -13
  24. package/src/domain/completions/rank.mjs +6 -19
  25. package/src/domain/grammar/lexicon-core.json +1 -1
  26. package/src/domain/hash.mjs +36 -13
  27. package/src/domain/interpret/fuzzy.mjs +7 -2
  28. package/src/domain/interpret/normalize.mjs +9 -0
  29. package/src/domain/interpret/strategies/keywords.mjs +19 -9
  30. package/src/domain/memory/capability.mjs +22 -3
  31. package/src/domain/memory/touched-facts.mjs +17 -0
  32. package/src/domain/module-paths.mjs +9 -0
  33. package/src/domain/persona/tiers.mjs +1 -1
  34. package/src/domain/planning.mjs +37 -0
  35. package/src/domain/prose.mjs +10 -2
  36. package/src/domain/relative-specifiers.mjs +12 -0
  37. package/src/domain/router/registry.mjs +3 -2
  38. package/src/domain/router/results.mjs +5 -18
  39. package/src/domain/seeded-random.mjs +33 -0
  40. package/src/domain/syllogise.mjs +10 -7
  41. package/src/domain/text-stats.mjs +31 -0
  42. package/src/services/chat.mjs +722 -184
  43. package/src/services/extract-facts.mjs +155 -0
  44. package/src/services/import-file.mjs +2 -2
  45. package/src/services/init.mjs +2 -2
  46. package/src/services/ledger-viz.mjs +6 -1
  47. package/src/services/sentences.mjs +26 -0
  48. package/src/surfaces/web/memory-ask-browser.bundle.js +11390 -360
  49. package/src/tools/graph-load.mjs +7 -1
  50. package/src/tools/readme-docs.mjs +113 -0
  51. package/src/tools/schema-docs.mjs +2 -2
  52. package/ROADMAP.md +0 -129
  53. package/corpus/namenet/generate.mjs +0 -309
  54. package/corpus/wordnet/generate.mjs +0 -332
  55. package/src/adapters/prose-tokens.mjs +0 -98
  56. package/src/adapters/wordnet-source.mjs +0 -70
  57. package/src/domain/corpus-matrix.mjs +0 -87
  58. package/src/domain/inflect.mjs +0 -67
  59. package/src/domain/licences.mjs +0 -68
  60. package/src/domain/markdown-links.mjs +0 -55
  61. package/src/domain/persona/codegen.mjs +0 -123
  62. package/src/domain/publish-gate.mjs +0 -41
  63. package/src/domain/schemaorg/turtle.mjs +0 -25
  64. package/src/domain/semcor/parse.mjs +0 -87
  65. package/src/domain/version-stamp.mjs +0 -36
  66. package/src/domain/wordnet/yaml.mjs +0 -133
@@ -11,10 +11,16 @@ export async function loadGraph(config, source) {
11
11
  if (!graph.individuals.length) {
12
12
  // Honest miss, never a stack: a fresh repo simply has no graph yet (the chat
13
13
  // session itself creates one as the conversation folds in).
14
- throw new ToolError(
14
+ const e = new ToolError(
15
15
  `the graph at ${config.graphFile} is empty — no entities to answer from yet ` +
16
16
  "(this repo starts with no graph; the chat session folds the conversation into one).",
17
17
  );
18
+ // An empty CODE graph is not an empty world: a caller that can still answer
19
+ // from the lexicon, the corpus or taught memory reads this flag and carries
20
+ // on, rather than reporting the graph's emptiness to someone who never
21
+ // asked a structural question.
22
+ e.emptyGraph = true;
23
+ throw e;
18
24
  }
19
25
  return graph;
20
26
  }
@@ -0,0 +1,113 @@
1
+ // Render README.md's tool section from the tool definitions (src/tools/definitions.mjs),
2
+ // which are the only place a tool's name, schema, purpose or example is written down.
3
+ // scripts/generate-tool-docs.mjs reads and writes the README around this;
4
+ // test/estate/tool-docs.test.mjs runs the same comparison, so a definition edited
5
+ // without regenerating turns the suite red.
6
+ import { HOT_TOOLS, COLD_TOOLS, TOOL_DEFINITIONS, askLexicon, toolByName } from "./definitions.mjs";
7
+
8
+ export const BEGIN_MARKER = "<!-- generated by scripts/generate-tool-docs.mjs from src/tools/definitions.mjs — do not edit by hand -->";
9
+ export const END_MARKER = "<!-- end generated tool section -->";
10
+
11
+ /** An argument list read off a tool's schema: "`symbol` (required), `depth`". */
12
+ function argsOf({ inputSchema }) {
13
+ const required = new Set(inputSchema.required || []);
14
+ const names = Object.keys(inputSchema.properties || {});
15
+ if (!names.length) return "none";
16
+ return names.map((n) => `\`${n}\`${required.has(n) ? " (required)" : ""}`).join(", ");
17
+ }
18
+
19
+ /** The README's tool section, rendered. Pure — the definitions are the only input. */
20
+ export function renderToolDocs() {
21
+ const ask = toolByName("tmct_ask");
22
+ const out = [];
23
+
24
+ out.push("## The tool surface");
25
+ out.push("");
26
+ out.push(
27
+ `Everything above runs on the same ${TOOL_DEFINITIONS.length} tools. Each one is read-only, ` +
28
+ "answers one question in a single call, and returns bounded output. None of them " +
29
+ "calls a model. A tool that cannot ground an answer says so — the same honest miss " +
30
+ "you get everywhere else in tmct.",
31
+ );
32
+ out.push("");
33
+ out.push(
34
+ `Three of them are **hot**: their schemas stay resident, so an agent driving tmct sees ` +
35
+ "them every turn and reaches for one call instead of a Read/Grep loop.",
36
+ );
37
+ out.push("");
38
+ for (const t of HOT_TOOLS) {
39
+ out.push(`- **\`${t.name}\`** — ${t.summary}`);
40
+ out.push(` Arguments: ${argsOf(t)}.`);
41
+ }
42
+ out.push("");
43
+
44
+ // ---- the chat-facing tool: its grammar, its lexicon, and worked examples ----
45
+ out.push("### Asking in plain English");
46
+ out.push("");
47
+ out.push(
48
+ "`tmct_ask` is the chat-facing tool. It takes a structural question as you would type " +
49
+ "it and resolves it to a real traversal. These are the shapes it reads:",
50
+ );
51
+ out.push("");
52
+ out.push("| you type | it answers with |");
53
+ out.push("| --- | --- |");
54
+ for (const g of ask.chat.grammar) out.push(`| ${g.example} | ${g.answers} |`);
55
+ out.push("");
56
+ out.push(
57
+ `The \`<where-marker>\` slot takes any of ${ask.chat.whereMarkers.map((m) => `*${m}*`).join(", ")}. ` +
58
+ "The relation verb is the part that carries the meaning, and each relation has its own " +
59
+ "vocabulary rather than one blessed keyword:",
60
+ );
61
+ out.push("");
62
+ for (const { kind, verbs } of askLexicon()) {
63
+ out.push(`- **${kind}** — ${verbs.map((v) => `*${v}*`).join(", ")}, and more`);
64
+ }
65
+ out.push("");
66
+ out.push("Every question in that table runs against the example graph:");
67
+ out.push("");
68
+ out.push("```bash cwd=repo");
69
+ for (const g of ask.chat.grammar) {
70
+ out.push(
71
+ `node bin/tmct.mjs cli tmct_ask '${JSON.stringify({ query: g.example, repo_path: ask.chat.exampleRepo })}'`,
72
+ );
73
+ }
74
+ out.push("```");
75
+ out.push("");
76
+
77
+ // ---- the cold tools ----
78
+ out.push("### The rest of the tools");
79
+ out.push("");
80
+ out.push(
81
+ "The remaining tools are **cold**: still served, but not billed to an agent every turn. " +
82
+ "Reach one through `tmct cli <tool>`, passing its arguments as JSON:",
83
+ );
84
+ out.push("");
85
+ out.push("| tool | what it answers | arguments |");
86
+ out.push("| --- | --- | --- |");
87
+ for (const t of COLD_TOOLS) out.push(`| \`${t.name}\` | ${t.summary} | ${argsOf(t)} |`);
88
+ out.push("");
89
+ out.push(
90
+ "Add `repo_path` to any of them to point at a repository other than the working " +
91
+ "directory. `tmct init` also writes this catalog, with a worked invocation per tool, " +
92
+ "to `.tmct/TOOLS.md` inside the repo it indexed.",
93
+ );
94
+ out.push("");
95
+ out.push("```bash cwd=repo");
96
+ out.push(`node bin/tmct.mjs cli tmct_untested '${JSON.stringify({ repo_path: "examples/mini-webapp" })}'`);
97
+ out.push("```");
98
+
99
+ return out.join("\n");
100
+ }
101
+
102
+ /** Splice the rendered section between the markers in `readme`. Throws when the
103
+ * markers are missing, so a lost section fails loudly rather than appending a copy. */
104
+ export function spliceToolDocs(readme, rendered) {
105
+ const begin = readme.indexOf(BEGIN_MARKER);
106
+ const end = readme.indexOf(END_MARKER);
107
+ if (begin === -1 || end === -1 || end < begin) {
108
+ throw new Error(`README.md is missing the generated tool-section markers (${BEGIN_MARKER})`);
109
+ }
110
+ const head = readme.slice(0, begin + BEGIN_MARKER.length);
111
+ const tail = readme.slice(end);
112
+ return `${head}\n\n${rendered}\n\n${tail}`;
113
+ }
@@ -154,10 +154,10 @@ export const PREDICATE_DOCS = Object.freeze([
154
154
  { prop: "seon:isConstant", kind: "attribute", description:
155
155
  "An ALL_CAPS module-level GlobalVariable — a naming-convention signal, not enforced " +
156
156
  "immutability." },
157
- { prop: "seon:subKind", kind: "attribute", description:
157
+ { prop: "mgx:subKind", kind: "attribute", description:
158
158
  "The flavour of a Class define when it is not a plain class: interface, enum, struct " +
159
159
  "or record. The graph keeps kind=class for every type declaration; this attribute " +
160
- "carries the distinction." },
160
+ "carries the distinction. A tmct extension — SEON has no subKind property." },
161
161
  { prop: "seon:hasAccessModifier", kind: "attribute", description:
162
162
  "Visibility inferred from a leading underscore (private/protected); a public member " +
163
163
  "carries no value for this attribute." },
package/ROADMAP.md DELETED
@@ -1,129 +0,0 @@
1
- # ROADMAP — tmct's current shape and what's next
2
-
3
- Forward-looking at a **feature level**: what tmct is capable of right now, and what's planned next.
4
- No session narrative, no dated diary, no "shipped/DONE" history — that's what git log and the
5
- `archive/`/`BENCHMARK_*.md`/`CAPABILITIES_*.md` records are for. For **task-level** pickup (specific
6
- open items, session-scoped), see `HANDOVER.md` instead — this file doesn't duplicate that list.
7
-
8
- ## What tmct is
9
-
10
- A tolerant, ELIZA/PARRY-style chat surface over a codebase, obsessed with software the way PARRY was
11
- obsessed with the mafia — deterministic, zero-cost, **no LLM anywhere in the product path**. Guides a
12
- user toward precision queries rather than guessing; every answer is grounded, restates every genuine
13
- reading it finds in full, or is an honest miss when nothing grounds it at all. Its visual surfaces —
14
- the ledger explorer with its in-browser chat (`tmct viz`), the animated plan page
15
- (`chat --prompt … --render blocks`), and the Pages homepage hero — are the same graph read out loud:
16
- same engine, same provenance, no LLM.
17
-
18
- ## Ambition
19
-
20
- Declared, forward-looking goals — not yet achieved, stated here so they steer future work instead of
21
- getting silently traded away by inherited caution:
22
-
23
- - **Reach for Llama-3-level natural language fluency.** by growing rich
24
- template/surface-realization variety, so an answer shape has many valid phrasings instead of one
25
- fixed slot-fill.
26
- - **Resolve ambiguity breadth-first, always.** Every genuinely valid reading gets its own real answer
27
- restated in full, never a bare "could mean X or Y — try rephrasing" punt, bounded only by existing
28
- clipping/pagination limits. L
29
- - **Paraphrase alongside the original, verified, never instead of it.** A surface-realization variant
30
- sits next to the literal grounded answer, never replacing it, and its accuracy is checked, not
31
- assumed — by running tmct's own deterministic inference/consistency machinery (`src/domain/syllogise.mjs`)
32
- against both the original and the paraphrase: they must entail the same conclusions, and neither may
33
- contradict the other sentence-by-sentence..
34
-
35
-
36
- ## What's next (feature-shaped — see `HANDOVER.md` for the current task-level list)
37
-
38
- - **`PLAN_ADVENTURE.md`** — a text-adventure architectural stretch. Its world-state and
39
- actions-as-data substrate shipped generically with the planning lane (action rule kinds,
40
- per-step board snapshots, legal-move enumeration); what remains its own is the imperative
41
- command grammar ("go north", "take the key"), the NPC turn scheduler, the Ashcombe Hall
42
- corpus, and the room-look digest.
43
- - **`PLAN_SYLLOGIST.md`** — the reasoning engine's research horizon. The single-justification
44
- retraction slice shipped (`retractSubClassOf`, justification persistence and cascade across all
45
- five rules); still open there: the ATMS generalization (alternate justification sets per fact),
46
- incremental matching (§2), and relevance under budget (§4).
47
- - **`PLAN_GUESS_NUMBER.md`** — closed-loop planning over hidden state (belief-interval bisection,
48
- thinker-mode secret commitment, observation folding) on top of the shipped planner substrate.
49
- Design-only.
50
- - **`PLAN_CODE.md`** — small JS-function and HTML/CSS-fragment synthesis, plus goal-directed
51
- program repair (tests as the goal state, mutation templates as planning actions), via a sandboxed
52
- headless browser (Track 1, rule/frame synthesis, already shipped). Blocked on a sandbox
53
- dependency decision.
54
- - **`PLAN_AGENTS.md`** — the governing plan for tmct's broader multi-repo arc (marginalia, seonix,
55
- a pluggable LLM rung for Claude Code/Bedrock/Copilot). Check its own sequencing table for current
56
- phase status, not this file.
57
-
58
-
59
- ## Research horizon
60
-
61
- *(2026-07-08 research pass — a direction recorded so it isn't re-discovered from scratch, not a
62
- committed build plan. Nothing below is scheduled.)*
63
-
64
- **Before the horizon — known-how, no research risk**, just scheduling: `PLAN_CODE.md` Tracks 2-4
65
- (mutation search/repair, JS/HTML/CSS synthesis — APR and CEGIS are established techniques);
66
- RETE/incremental forward-chaining (`PLAN_SYLLOGIST.md` §2 — Forgy 1982, a citable algorithm not yet
67
- ported); contingent/conformant planning under initial-state uncertainty (Bonet & Geffner 2000,
68
- Hoffmann & Brafman 2006, Petrick & Bacchus 2002 all have working algorithms, none yet applied here).
69
-
70
- **After the horizon — genuinely unsolved in the field**, named as real research targets with
71
- citations, not stop signs:
72
- - **The frame problem / relevance realization** (open-world planning boundary). McCarthy & Hayes
73
- 1969 named it; Jaeger, Riedl, Djedovic, Vervaeke & Walsh (2024) argue it may not be algorithmically
74
- solvable in the general case. Speculative angle: bounded (N+1) goal recognition — recognize
75
- declared goal 1..N, or reject to an explicit "escalate" class, via parse-shape membership.
76
- - **Bounded, incremental, trust-tiered, retraction-safe justification tracking** — `PLAN_SYLLOGIST.md`
77
- §3. Doyle's JTMS (1979) and de Kleer's ATMS (1986) solve retraction; DRed/RDFox's Backward-Forward
78
- solve incremental Datalog maintenance; nobody's published the combination with tmct's
79
- multi-trust-tier, hard-budget requirement. The JTMS-lite slice shipped (one persisted
80
- justification per entailed fact, VERIFY-backed retraction, all five rules); the open piece is the
81
- ATMS generalization — alternate justification SETS per fact (see that doc's 2026-07-15 addendum).
82
- - **A shared ~2M-word cross-domain ontology** (general-English + technical/scientific/programming).
83
- Merging collides senses of lexically-shared words (`class`, `cache`, `thread`, `field`, `state`)
84
- across registers; knowledge-based WSD is real but weaker than supervised/neural WSD (Lesk 1986;
85
- Raganato, Camacho-Collados & Navigli, EACL 2017). BabelNet proves cross-resource sense merging is
86
- achievable at scale but solves the cross-*lingual*, not cross-*domain*, axis, and carries a
87
- non-commercial licence. Speculative angle: mutual disambiguation from already-resolved neighbouring
88
- terms in tmct's own closed graph (a bounded reading of Gale/Church/Yarowsky's "one sense per
89
- discourse" regularity) — not published anywhere found for this application. Fresh live instance
90
- (2026-07-11): `"tail"` (Unix process vs. animal body part) collides under `normFactTerm`'s
91
- cross-corpus flattening, `src/adapters/memory/core.mjs:1109-1134`.
92
-
93
- **Tier-4: learn-on-miss acquisition**. The strongest
94
- miss signal tmct can emit: lexicon term recognized, query built cleanly, zero matches anywhere — the
95
- question was well-formed and the knowledge is simply absent. Web search on the resolved term → clean
96
- the fetched text into the ACE-OWL controlled grammar → store with source provenance → answer the
97
- original question, citing what was just learned. Strictly opt-in, offline default inviolable.
98
- Prerequisites: the provenance-trust policy must extend to `via:"learned:web"`, never silently
99
- blending web-sourced facts with graph/operator facts.
100
-
101
- ## Design docs
102
-
103
- Every substantial design lives in its own `PLAN_*.md` at the repo root; `archive/` holds the shipped
104
- and closed ones. This file points to them, it doesn't repeat their content. Each plan states its own
105
- status in its opening lines — read it there, because a status quoted here would rot.
106
-
107
- | Plan | What it's for |
108
- | --- | --- |
109
- | [PLAN_ADVENTURE.md](PLAN_ADVENTURE.md) | a text adventure as an architectural stretch: imperative command grammar, NPC turn scheduler, room-look digest |
110
- | [PLAN_AGENTS.md](PLAN_AGENTS.md) | the governing plan for the multi-repo arc (marginalia, seonix, a pluggable LLM rung), with its own phase sequencing |
111
- | [PLAN_CHILD_CORPUS.md](PLAN_CHILD_CORPUS.md) | a wider default seed corpus, chosen by age of acquisition |
112
- | [PLAN_CLASS_QUERY.md](PLAN_CLASS_QUERY.md) | "list/count all X of class Y", reconciled against what already shipped |
113
- | [PLAN_CODE.md](PLAN_CODE.md) | program synthesis over tmct's closed DSLs, plus JS/HTML/CSS fragments and goal-directed program repair |
114
- | [PLAN_CONSISTENCY_CHECK.md](PLAN_CONSISTENCY_CHECK.md) | tmct as a consistency service for an LLM tool loop |
115
- | [PLAN_EMBEDDINGS.md](PLAN_EMBEDDINGS.md) | the semantic-similarity axis, and the way back to it |
116
- | [PLAN_GRAPH_SCAN.md](PLAN_GRAPH_SCAN.md) | seed and query cost at `init:xl`/`init:xxl` corpus scale |
117
- | [PLAN_GUESS_NUMBER.md](PLAN_GUESS_NUMBER.md) | closed-loop planning over hidden state, via belief-interval bisection |
118
- | [PLAN_MUD.md](PLAN_MUD.md) | persistent, shared tmct worlds over a `server:` memory backend |
119
- | [PLAN_NLU_BENCHMARKS.md](PLAN_NLU_BENCHMARKS.md) | scoring tmct on the CLINC150 and HWU64 intent sets |
120
- | [PLAN_OPEN_ITEMS.md](PLAN_OPEN_ITEMS.md) | the build order closing the backlog `HANDOVER.md` carries |
121
- | [PLAN_PARAPHRASE_VERIFICATION.md](PLAN_PARAPHRASE_VERIFICATION.md) | checking a paraphrase against the graph before it prints |
122
- | [PLAN_PURGE.md](PLAN_PURGE.md) | promote the load-bearing code, delete the dead weight |
123
- | [PLAN_REPO_INDEX.md](PLAN_REPO_INDEX.md) | tmct grows its own code parsers, ported from seonix |
124
- | [PLAN_SYLLOGIST.md](PLAN_SYLLOGIST.md) | the reasoning engine's incrementality and retraction horizon |
125
- | [PLAN_SYLLOGIST_EL_DL.md](PLAN_SYLLOGIST_EL_DL.md) | beyond OWL 2 RL: an EL classifier, then a DL tableau prover |
126
-
127
- `SKILL_*.md` docs specify the repeatable measurement and build cycles (the benchmarks, the capability
128
- audit, the background strategy advisor, plain-prose writing). `HANDOVER.md` is the single
129
- current-open-items list.
@@ -1,309 +0,0 @@
1
- #!/usr/bin/env node
2
- // corpus/namenet/generate.mjs — converts THREE reviewed CSVs from a LOCAL
3
- // Open English Namenet checkout into ConceptNet-shape fact rows. Mirrors
4
- // corpus/wordnet/generate.mjs's structure/conventions exactly (same
5
- // maintainer-tool framing, same deterministic sorted JSONL + manifest.json
6
- // output shape) — smaller scope, OPTIONAL top-up, not load-bearing.
7
- //
8
- // node corpus/namenet/generate.mjs [namenetDir]
9
- // TMCT_NAMENET_DIR=/path/to/english-namenet node corpus/namenet/generate.mjs
10
- //
11
- // Input: `~/projects/globalwordnet/english-namenet/` by default (a LOCAL
12
- // clone, never vendored/committed) — three reviewed CSVs, each a
13
- // human/algorithm-curated LINKING TABLE between a name/label and an Open
14
- // English WordNet (OEWN) synset or lemma set:
15
- // - species_reviewed.csv (5,101 rows) — Scientific Name -> SSID
16
- // - taxon2common_reviewed.csv (2,368 rows) — SSID 1/Lemmas 1 -> SSID 2/Lemmas 2
17
- // - linked_occupations_reviewed.csv (2,193 rows) — Wikidata Labels -> SSID/Lemma
18
- // Every one of the three, once you read real rows (not just the header),
19
- // turns out to be the SAME shape underneath: two name-lists that denote the
20
- // SAME real-world thing (a species, a folk-taxonomic category, an
21
- // occupation), reviewed/accepted by a human as a correct link — never a
22
- // hierarchy (broader/narrower) or capability claim. That is why every fact
23
- // this converter emits uses ONE relation, /r/Synonym ("X means the same as
24
- // Y") — confirmed against conceptnet-map.toml (ace != "none", Phase 1's
25
- // 2026-07-12 widening) rather than invented here (this task's own scope
26
- // boundary: conversion only, reuse what Phase 1 already mapped, never add a
27
- // new relation row).
28
- //
29
- // Why NOT /r/IsA or /r/CapableOf (the task brief's own initial guesses,
30
- // before real rows were read):
31
- // - species_reviewed.csv: real rows show the "Scientific Name" is almost
32
- // always ALREADY one of the target SSID's own WordNet members (4,885 of
33
- // 4,897 accepted rows resolve to a real synset; of those, 4,881 have the
34
- // scientific name as a literal member string) — this is a same-referent
35
- // alias table (which of several ambiguous WordNet senses a Wikidata
36
- // taxon QID actually means), not a species/kind subclass relation.
37
- // - linked_occupations_reviewed.csv: despite the CSV's name, each row
38
- // links a WIKIDATA OCCUPATION ENTITY's labels to a WORDNET OCCUPATION
39
- // SYNSET's lemmas (e.g. "politician, political leader" <-> "pol,
40
- // political leader, politician, politico") — it is NOT a person linked
41
- // to their job (no person names anywhere in this file), so /r/CapableOf
42
- // ("a person can politician") would be nonsensical. It is the same
43
- // alias-table shape as the other two.
44
- //
45
- // species_reviewed.csv needs a SECOND local checkout to resolve: its SSID
46
- // column has no lemma text of its own (unlike the other two, which carry
47
- // "Lemmas N" columns directly), so this converter reuses
48
- // corpus/wordnet/generate.mjs's already-proven `loadAllSynsets`/
49
- // `DEFAULT_YAML_DIR`/`encodeTerm`/`humanize` (imported, never duplicated —
50
- // same discipline that file's own header comment describes for its
51
- // hand-rolled YAML reader) to resolve SSID -> representative lemma.
52
- // corpus/wordnet/generate.mjs itself is never modified (task scope
53
- // boundary) — only its exported pure functions are called.
54
- //
55
- // NOT part of the product path — a maintainer tool, run by hand, offline,
56
- // $0; its OUTPUT (corpus/namenet/namenet.jsonl + manifest.json) is what gets
57
- // committed, never the source CSVs themselves.
58
- //
59
- // Licence: see LICENSE-NOTICE in this directory — the source repository
60
- // (globalwordnet/english-namenet) declares NO explicit license of its own
61
- // (confirmed via GitHub repo metadata, 2026-07-12: `license: null`, no
62
- // LICENSE file, no license statement in README.md); this bundle is
63
- // distributed under CC-BY-4.0 as a conservative match to the Open English
64
- // WordNet data it is built from and links against, pending clarification
65
- // from the GlobalWordNet team. The code in this file is tmct code under the
66
- // repository's MPL-2.0; only the generated data
67
- // (corpus/namenet/namenet.jsonl) carries that CC-BY-4.0 label.
68
-
69
- import { readFile, writeFile, mkdir } from "node:fs/promises";
70
- import { homedir } from "node:os";
71
- import { createHash } from "node:crypto";
72
- import { fileURLToPath } from "node:url";
73
- import { dirname, join } from "node:path";
74
- import { loadAllSynsets, DEFAULT_YAML_DIR, resolveYamlDir, encodeTerm, humanize } from "../wordnet/generate.mjs";
75
-
76
- const HERE = dirname(fileURLToPath(import.meta.url));
77
- export const NAMENET_OUT_DIR = HERE;
78
-
79
- export const DEFAULT_NAMENET_DIR = join(homedir(), "projects", "globalwordnet", "english-namenet");
80
-
81
- /** Resolve the input namenet directory: CLI positional arg > env var >
82
- * default. Pure (argv/env injectable), mirrors wordnet's resolveYamlDir. */
83
- export function resolveNamenetDir(argv = process.argv.slice(2), env = process.env) {
84
- return argv[0] || env.TMCT_NAMENET_DIR || DEFAULT_NAMENET_DIR;
85
- }
86
-
87
- // ---- CSV parsing (pure, unit-tested) ---------------------------------------
88
- // A small hand-rolled RFC4180-ish reader — no dependency added, same house
89
- // style as corpus/wordnet/generate.mjs reusing a hand-rolled YAML reader
90
- // rather than pulling in a general parsing library. Handles quoted fields
91
- // (commas/newlines inside quotes, "" as an escaped literal quote) and both
92
- // CRLF and LF line endings — all three source CSVs use quoted fields for any
93
- // value containing a comma (e.g. `"species, by Garsault, 1764..."`), so a
94
- // naive `.split(",")` silently misaligns columns on those rows.
95
-
96
- /** Parse CSV text into an array of records (arrays of string fields). */
97
- export function parseCsvRecords(text) {
98
- const records = [];
99
- let row = [];
100
- let field = "";
101
- let inQuotes = false;
102
- const pushField = () => { row.push(field); field = ""; };
103
- const pushRow = () => { pushField(); records.push(row); row = []; };
104
- const src = String(text ?? "");
105
- for (let i = 0; i < src.length; i++) {
106
- const c = src[i];
107
- if (inQuotes) {
108
- if (c === '"') {
109
- if (src[i + 1] === '"') { field += '"'; i++; }
110
- else inQuotes = false;
111
- } else {
112
- field += c;
113
- }
114
- continue;
115
- }
116
- if (c === '"') { inQuotes = true; continue; }
117
- if (c === ",") { pushField(); continue; }
118
- if (c === "\r") continue; // CRLF -> swallow, \n below ends the row
119
- if (c === "\n") { pushRow(); continue; }
120
- field += c;
121
- }
122
- // final field/row, if the text didn't end with a newline
123
- if (field !== "" || row.length) pushRow();
124
- // drop a single trailing wholly-empty record (trailing newline artifact)
125
- if (records.length && records[records.length - 1].every((f) => f === "")) records.pop();
126
- return records;
127
- }
128
-
129
- /** Parse CSV text into an array of row objects keyed by the header row. */
130
- export function parseCsv(text) {
131
- const records = parseCsvRecords(text);
132
- if (!records.length) return [];
133
- const header = records[0].map((h) => h.trim());
134
- return records.slice(1).map((rec) => {
135
- const obj = {};
136
- for (let i = 0; i < header.length; i++) obj[header[i]] = rec[i] ?? "";
137
- return obj;
138
- });
139
- }
140
-
141
- /** "Plantae, kingdom Plantae, plant kingdom" -> "Plantae" — the first
142
- * candidate in a comma-separated lemma/label list, the same "first member is
143
- * representative" convention corpus/wordnet/generate.mjs's repTerm() uses. */
144
- export function firstOf(commaList) {
145
- const s = String(commaList ?? "").trim();
146
- if (!s) return null;
147
- const first = s.split(",")[0].trim();
148
- return first || null;
149
- }
150
-
151
- // ---- shared row builder (pure) ---------------------------------------------
152
- // Same dedupe-by-key + self-loop-skip discipline as corpus/wordnet/
153
- // generate.mjs's makeRowBuilder — re-declared locally (not imported; that
154
- // function isn't exported, and this is a small enough shape to keep local
155
- // rather than widen wordnet/generate.mjs's exports for a five-line helper).
156
-
157
- function makeRowBuilder() {
158
- const rows = new Map();
159
- const add = (rawSubject, rel, rawObject) => {
160
- const start = encodeTerm(rawSubject);
161
- const end = encodeTerm(rawObject);
162
- if (!start || !end || start === end) return; // self-loop / empty term — noise, not a fact
163
- const key = `${rel} ${start} ${end}`;
164
- if (rows.has(key)) return;
165
- rows.set(key, {
166
- start,
167
- rel,
168
- end,
169
- weight: 1,
170
- surfaceText: `[[${humanize(rawSubject)}]] ${rel.replace("/r/", "")} [[${humanize(rawObject)}]]`,
171
- });
172
- };
173
- return { rows, add };
174
- }
175
-
176
- const sortRows = (rows) => rows.slice().sort((a, b) => (
177
- a.rel !== b.rel ? (a.rel < b.rel ? -1 : 1)
178
- : a.start !== b.start ? (a.start < b.start ? -1 : 1)
179
- : a.end < b.end ? -1 : a.end > b.end ? 1 : 0
180
- ));
181
-
182
- // ---- per-source mappers (pure, unit-tested) --------------------------------
183
-
184
- /** species_reviewed.csv: accepted rows only; the Scientific Name and the
185
- * SSID's representative WordNet lemma denote the same species -> /r/Synonym.
186
- * `bySynset` is the same `Map<synsetId, {members}>` shape
187
- * corpus/wordnet/generate.mjs's loadAllSynsets returns (or a small fixture
188
- * Map in tests) — a row whose SSID isn't in the map is skipped, not thrown. */
189
- export function buildSpeciesFacts(rows, bySynset) {
190
- const { rows: out, add } = makeRowBuilder();
191
- for (const row of rows) {
192
- if (row.Accept !== "TRUE") continue;
193
- const sciName = row["Scientific Name"];
194
- const ssid = row.SSID;
195
- if (!sciName || !ssid) continue;
196
- const synset = bySynset.get(ssid);
197
- const members = Array.isArray(synset?.members) ? synset.members : [];
198
- if (!members.length) continue; // unresolved SSID — skip, don't throw
199
- add(sciName, "/r/Synonym", members[0]);
200
- }
201
- return sortRows([...out.values()]);
202
- }
203
-
204
- /** taxon2common_reviewed.csv: accepted rows only; the first lemma of each
205
- * side's "Lemmas N" list denotes the same taxonomic/folk category ->
206
- * /r/Synonym. No cross-reference needed — both lemma lists are already
207
- * columns in this CSV. */
208
- export function buildTaxon2CommonFacts(rows) {
209
- const { rows: out, add } = makeRowBuilder();
210
- for (const row of rows) {
211
- if (row.Accept !== "TRUE") continue;
212
- const a = firstOf(row["Lemmas 1"]);
213
- const b = firstOf(row["Lemmas 2"]);
214
- if (!a || !b) continue;
215
- add(a, "/r/Synonym", b);
216
- }
217
- return sortRows([...out.values()]);
218
- }
219
-
220
- /** linked_occupations_reviewed.csv: accepted, genuine-occupation rows only
221
- * (`Accept === "TRUE"` AND `"Not an occupation" !== "TRUE"`); the first
222
- * Wikidata label and the first WordNet lemma denote the same occupation ->
223
- * /r/Synonym. */
224
- export function buildOccupationFacts(rows) {
225
- const { rows: out, add } = makeRowBuilder();
226
- for (const row of rows) {
227
- if (row.Accept !== "TRUE") continue;
228
- if (row["Not an occupation"] === "TRUE") continue;
229
- const label = firstOf(row.Labels);
230
- const lemma = firstOf(row.Lemma);
231
- if (!label || !lemma) continue;
232
- add(label, "/r/Synonym", lemma);
233
- }
234
- return sortRows([...out.values()]);
235
- }
236
-
237
- /** Merge the three per-source fact sets into one deduped, sorted set — the
238
- * namenet.jsonl content. A pair already emitted by one source (e.g. the
239
- * same scientific-name/common-name pair surfacing via both
240
- * species_reviewed.csv and taxon2common_reviewed.csv) is kept once. */
241
- export function mergeFacts(...factLists) {
242
- const { rows, add } = makeRowBuilder();
243
- for (const list of factLists) {
244
- for (const f of list) add(humanize(f.start.replace(/^\/c\/en\//, "")), f.rel, humanize(f.end.replace(/^\/c\/en\//, "")));
245
- }
246
- return sortRows([...rows.values()]);
247
- }
248
-
249
- // ---- output ------------------------------------------------------------
250
-
251
- const toJsonl = (rows) => rows.map((r) => JSON.stringify(r)).join("\n") + "\n";
252
- const sha256 = (text) => createHash("sha256").update(text).digest("hex");
253
-
254
- async function readCsv(dir, name) {
255
- const text = await readFile(join(dir, name), "utf8");
256
- return parseCsv(text);
257
- }
258
-
259
- async function main() {
260
- const namenetDir = resolveNamenetDir();
261
- const yamlDir = resolveYamlDir([], process.env) || DEFAULT_YAML_DIR;
262
- process.stderr.write(`corpus/namenet/generate.mjs: reading ${namenetDir}\n`);
263
- process.stderr.write(` (species_reviewed.csv also needs OEWN synsets from ${yamlDir})\n`);
264
-
265
- const [speciesRows, taxonRows, occupationRows] = await Promise.all([
266
- readCsv(namenetDir, "species_reviewed.csv"),
267
- readCsv(namenetDir, "taxon2common_reviewed.csv"),
268
- readCsv(namenetDir, "linked_occupations_reviewed.csv"),
269
- ]);
270
- const { bySynset } = await loadAllSynsets(yamlDir);
271
-
272
- const speciesFacts = buildSpeciesFacts(speciesRows, bySynset);
273
- const taxonFacts = buildTaxon2CommonFacts(taxonRows);
274
- const occupationFacts = buildOccupationFacts(occupationRows);
275
- const merged = mergeFacts(speciesFacts, taxonFacts, occupationFacts);
276
-
277
- process.stderr.write(` species_reviewed.csv: ${speciesRows.length} rows -> ${speciesFacts.length} facts\n`);
278
- process.stderr.write(` taxon2common_reviewed.csv: ${taxonRows.length} rows -> ${taxonFacts.length} facts\n`);
279
- process.stderr.write(` linked_occupations_reviewed.csv: ${occupationRows.length} rows -> ${occupationFacts.length} facts\n`);
280
- process.stderr.write(` namenet (merged, deduped): ${merged.length} facts\n`);
281
-
282
- await mkdir(NAMENET_OUT_DIR, { recursive: true });
283
- const outText = toJsonl(merged);
284
- await writeFile(join(NAMENET_OUT_DIR, "namenet.jsonl"), outText);
285
-
286
- const manifest = {
287
- version: 1,
288
- generated: "by corpus/namenet/generate.mjs",
289
- corpuses: [
290
- {
291
- id: "namenet",
292
- kind: "language",
293
- description: "Scientific-name/common-name and Wikidata-label/WordNet-lemma synonym pairs, mechanically derived from three human-reviewed Open English Namenet linking tables (species, taxon-to-common-name, occupations). A small top-up bundle, not a primary corpus.",
294
- source: { kind: "curated", tool: "corpus/namenet/generate.mjs" },
295
- file: "namenet.jsonl",
296
- facts: merged.length,
297
- bytes: Buffer.byteLength(outText),
298
- sha256: sha256(outText),
299
- license: "CC-BY-4.0 (source repo declares no explicit license; see LICENSE-NOTICE)",
300
- },
301
- ],
302
- };
303
- const manifestText = JSON.stringify(manifest, null, 2) + "\n";
304
- await writeFile(join(NAMENET_OUT_DIR, "manifest.json"), manifestText);
305
- process.stderr.write(`wrote corpus/namenet/manifest.json (${manifest.corpuses.length} corpuses)\n`);
306
- }
307
-
308
- const isMain = process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href;
309
- if (isMain) await main();