@polycode-projects/the-mechanical-code-talker 6.0.18 → 6.0.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +20 -23
  2. package/bin/tmct.mjs +16 -33
  3. package/corpus/LICENSES.json +0 -21
  4. package/corpus/README.md +10 -13
  5. package/corpus/reference/manifest.json +19 -19
  6. package/corpus/reference/shards/ref-01.jsonl.gz +0 -0
  7. package/corpus/reference/shards/ref-04.jsonl.gz +0 -0
  8. package/corpus/reference/shards/ref-08.jsonl.gz +0 -0
  9. package/corpus/reference/shards/ref-10.jsonl.gz +0 -0
  10. package/corpus/reference/shards/ref-11.jsonl.gz +0 -0
  11. package/corpus/reference/shards/ref-17.jsonl.gz +0 -0
  12. package/corpus/reference/shards/ref-20.jsonl.gz +0 -0
  13. package/corpus/reference/shards/ref-25.jsonl.gz +0 -0
  14. package/corpus/reference/shards/ref-2c.jsonl.gz +0 -0
  15. package/corpus/tier2/generate.mjs +6 -142
  16. package/corpus/tier2/manifest.json +0 -42
  17. package/package.json +6 -4
  18. package/src/adapters/corpus/child-seed.mjs +74 -0
  19. package/src/adapters/corpus/conceptnet.mjs +45 -26
  20. package/src/adapters/corpus/research-source.mjs +6 -2
  21. package/src/adapters/corpus/wikidata-live.mjs +92 -51
  22. package/src/adapters/memory/blocks.mjs +7 -1
  23. package/src/adapters/memory/core.mjs +505 -107
  24. package/src/adapters/memory/corpus-bands.mjs +27 -10
  25. package/src/adapters/memory/inspect.mjs +24 -5
  26. package/src/adapters/memory/rows.mjs +359 -30
  27. package/src/adapters/memory/shacl.mjs +10 -3
  28. package/src/domain/ask.mjs +27 -10
  29. package/src/domain/cli-verbs.mjs +3 -4
  30. package/src/domain/completions/group.mjs +8 -3
  31. package/src/domain/completions/infer.mjs +7 -2
  32. package/src/domain/completions/prune.mjs +5 -1
  33. package/src/domain/completions/rank.mjs +7 -2
  34. package/src/domain/digest/compose.mjs +5 -1
  35. package/src/domain/digest/select.mjs +12 -6
  36. package/src/domain/domain.mjs +15 -8
  37. package/src/domain/el-classify.mjs +11 -2
  38. package/src/domain/fact-phrase.mjs +86 -4
  39. package/src/domain/hash.mjs +9 -0
  40. package/src/domain/memory/bias.mjs +8 -4
  41. package/src/domain/memory/capability.mjs +12 -6
  42. package/src/domain/memory/fact-order.mjs +29 -0
  43. package/src/domain/memory/resolution.mjs +3 -0
  44. package/src/domain/news-feed.mjs +862 -92
  45. package/src/domain/reference-pack.mjs +5 -0
  46. package/src/domain/sense-gate.mjs +220 -0
  47. package/src/domain/sense-scope.mjs +116 -0
  48. package/src/domain/sense-split.mjs +1 -1
  49. package/src/domain/syllogise.mjs +60 -21
  50. package/src/domain/tableau.mjs +23 -14
  51. package/src/domain/term-ledger.mjs +16 -1
  52. package/src/domain/worlds-pack.mjs +5 -1
  53. package/src/services/adventure-autoplay.mjs +6 -1
  54. package/src/services/adventure-editor.mjs +43 -21
  55. package/src/services/adventure-viz.mjs +26 -9
  56. package/src/services/adventure.mjs +40 -10
  57. package/src/services/chat.mjs +270 -125
  58. package/src/services/extensions.mjs +51 -58
  59. package/src/services/extract-facts.mjs +906 -66
  60. package/src/services/init.mjs +4 -4
  61. package/src/services/ledger-viz.mjs +9 -4
  62. package/src/services/memory-panel-viz.mjs +4 -5
  63. package/src/services/mud-editor.mjs +40 -16
  64. package/src/services/mud-viz.mjs +8 -2
  65. package/src/services/mudiii-turn.mjs +5 -3
  66. package/src/services/mudiii-viz.mjs +8 -2
  67. package/src/services/news.mjs +306 -21
  68. package/src/services/research-viz.mjs +1 -1
  69. package/src/services/sprite-catalog-viz.mjs +10 -5
  70. package/src/surfaces/web/adventure-browser-entry.mjs +6 -12
  71. package/src/surfaces/web/memory-ask-browser.bundle.js +152 -151
  72. package/src/surfaces/web/mud-browser-entry.mjs +7 -11
  73. package/src/surfaces/web/research-browser-entry.mjs +5 -2
  74. package/corpus/tier2/aws.jsonl +0 -39
  75. package/corpus/tier2/java.jsonl +0 -31
  76. package/corpus/tier2/python.jsonl +0 -30
package/README.md CHANGED
@@ -424,7 +424,7 @@ tmct> /exit
424
424
  **[Try it live in your browser →](https://tmct.polycode.co.uk/)**
425
425
  runs the actual query engine client-side. No server, no install. The landing
426
426
  page answers codebase questions live, and six more pages each ground their
427
- own domain: a full chat seeded with 63,470 facts (the same nine bands as
427
+ own domain: a full chat seeded with 70,453 facts (the same eight bands as
428
428
  `npm run init:xl`), the
429
429
  **memory ledger** (every fact as a readable sentence; drill by clicking the
430
430
  terms inside), and a Towers-of-Hanoi **plan** replayed move by move. A 3D
@@ -580,7 +580,7 @@ question against the same entity always renders the same way.
580
580
  ```session cwd=repo
581
581
  $ node bin/tmct.mjs chat --repo examples/mini-webapp --ephemeral
582
582
  tmct> give me a detailed overview of how the Store works
583
- Attribute: prose_tokens = memory record store [mgx:hasProseTokens]. Attribute: doc = In-memory record store. [seon:hasDoc]. Other matches: src/core/store.mjs (Module), loadStore (Function), saveStore (Function), testLoadStore (Function).
583
+ Attribute: prose_tokens = memory record store [mgx:hasProseTokens]. Attribute: doc = In-memory record store. [seon:hasDoc]. Store — Class (id: fn:src/core/store.mjs#Store).
584
584
  ```
585
585
 
586
586
  Programmatically, the same pipeline is `generateCompletion()`
@@ -1065,15 +1065,13 @@ provenance record. Most of its flags choose what gets seeded and where config is
1065
1065
  tmct init [--repo <abs>] initialize a repo for tmct (default: cwd): .tmct/,
1066
1066
  [--force] tmct.toml, .tmct/TOOLS.md (the cold-tool catalog),
1067
1067
  tier-1 corpus seed, provenance record
1068
- [--corpus <id|path>] also seed a corpus — a tier-2 manifest id (aws|python|java|
1069
- general) or a jsonl file path — opt-in, offline, $0
1068
+ [--corpus <id|path>] also seed a corpus — a bundle name (code|conceptnet|child|
1069
+ namenet|general) or a jsonl file path — opt-in, offline, $0
1070
1070
  [--ontology <name|path>] activate+seed an ontology bundle (a recognized name or a path)
1071
1071
  [--lexicon <name|path>] activate a lexicon bundle (recognized name or a path;
1072
1072
  merged read-time, never seeded — see mergedLexiconExtra)
1073
1073
  [--graph <path>] set graph_file/graph_files in tmct.toml (repeatable)
1074
1074
  [--config <path>] write to an alternate tmct.toml location
1075
- [--detect] suggest a tier-2 corpus from the repo's manifests
1076
- (pyproject.toml → python, pom.xml → java); never seeds unasked
1077
1075
  [--with-persona <name>] write an explicit [extensions]/[bias] preset into tmct.toml
1078
1076
  ("code" — today's implicit default, made explicit)
1079
1077
  [--persona-size <medium|large>] grow the default "human" persona's fact count
@@ -1225,8 +1223,8 @@ row-backend table, or clears one; the deployed turn service reads the loaded
1225
1223
  bands per query:
1226
1224
 
1227
1225
  ```output:help:corpus
1228
- tmct corpus load <band> [--table <name>] [--source <path>] [--dry-run] load a shared, read-only corpus band (wikidata-slice, wordnet-complete,
1229
- [--table <name>] conceptnet-full) into a DynamoDB row-backend table from a jsonl of
1226
+ tmct corpus load <band> [--table <name>] [--source <path>] [--dry-run] load a shared, read-only corpus band (wordnet-complete, or a
1227
+ [--table <name>] consumer's own) into a DynamoDB row-backend table from a jsonl of
1230
1228
  wire-row-shaped facts (default table from TMCT_DYNAMO_TABLE); a
1231
1229
  source whose digest already matches the band's manifest is a no-op
1232
1230
  [--source <path>] the band's jsonl (a scripts/corpus-bands/ build output, or any jsonl
@@ -1259,28 +1257,27 @@ once with `tmct init --memory-backend <...>` and every later `tmct chat` in that
1259
1257
  repo picks it up with no flag needed.
1260
1258
  ```
1261
1259
 
1262
- `npm run init` in `package.json` chains one `init` and five `import --corpus`
1263
- calls to combine every shipped bundle (human persona + seon + conceptnet +
1264
- aws/python/java) into ~37,800 facts on the default sqlite backend, a working
1260
+ `npm run init` in `package.json` chains one `init` and two `import --corpus`
1261
+ calls to combine the human persona, the code domain pack and conceptnet
1262
+ into ~37,700 facts on the default sqlite backend, a working
1265
1263
  example to copy from (`init:large` is now just an alias for it). `npm run
1266
1264
  init:small` is the lighter variant: a bare `tmct init`, default persona only,
1267
1265
  no big corpora, 688 facts. It's what a first `npm run chat` in an
1268
1266
  uninitialized repo bootstraps automatically, so no init command is required
1269
1267
  just to start talking; running `init:small` explicitly (after `rm -rf
1270
1268
  tmct.toml .tmct`) gets you back to that same minimal state on purpose.
1271
- `init:xl` starts from the large persona tier and adds the wordnet-xl corpus
1272
- (~72,000 facts); `init:xxl` swaps wordnet-xl for the full WordNet slice plus
1273
- namenet (~239,000 facts, the biggest committed vocabulary, so expect its
1274
- imports to take a minute). The xl chain, spelled out:
1269
+ `init:xl` starts from the large persona tier and adds wordnet-xl, namenet and
1270
+ the child vocabulary pack (~127,000 facts); `init:xxl` swaps wordnet-xl for the
1271
+ full WordNet slice (~296,000 facts, the biggest committed vocabulary, so expect
1272
+ its imports to take a few minutes). The xl chain, spelled out:
1275
1273
 
1276
1274
  ```bash e2e heavy
1277
1275
  npx tmct init --persona-size large # npm run init:xl runs this whole chain from a clone
1278
- npx tmct import --corpus seon
1276
+ npx tmct import --corpus code
1279
1277
  npx tmct import --corpus conceptnet
1280
- npx tmct import --corpus aws
1281
- npx tmct import --corpus python
1282
- npx tmct import --corpus java
1283
- npx tmct import --corpus wordnet-xl # init:xxl instead ends with wordnet-full and namenet
1278
+ npx tmct import --corpus wordnet-xl # init:xxl uses wordnet-full here instead
1279
+ npx tmct import --corpus namenet
1280
+ npx tmct import --corpus child
1284
1281
  ```
1285
1282
 
1286
1283
  ### tmct.toml reference
@@ -1321,8 +1318,8 @@ capture_unknown_context = true # keep the sentence around an unrecognized term
1321
1318
  unknown_context_limit = 200 # cap how many of those contexts are kept
1322
1319
 
1323
1320
  # One [extensions.<name>] table per bundle. A recognized name (human, seon,
1324
- # conceptnet, human-medium, human-large, tier2-aws, tier2-python, tier2-java,
1325
- # tier2-general, wordnet-xl, wordnet-full, namenet) overrides that bundle's
1321
+ # code, conceptnet, human-medium, human-large, tier2-general, wordnet-xl,
1322
+ # wordnet-full, namenet, child) overrides that bundle's
1326
1323
  # shipped defaults. Any other name declares a new bundle and must set `kind`.
1327
1324
  [extensions.human]
1328
1325
  active = true
@@ -1333,7 +1330,7 @@ active = true
1333
1330
  [extensions.conceptnet]
1334
1331
  active = true
1335
1332
 
1336
- [extensions.tier2-aws]
1333
+ [extensions.tier2-general]
1337
1334
  active = true
1338
1335
 
1339
1336
  [extensions.my-custom-pack]
package/bin/tmct.mjs CHANGED
@@ -407,7 +407,7 @@ async function repoRelative(repoRoot, p) {
407
407
  /** Resolve a `--corpus`/`--ontology`/`--lexicon` value to something
408
408
  * activatePluggableInput can act on: either a RECOGNIZED name (a
409
409
  * BUILTIN_EXTENSIONS entry of the matching kind; for `--corpus` specifically,
410
- * also a tier-2 manifest id like "aws" — today's `--corpus <id>` contract,
410
+ * also a tier-2 manifest id like "general" — today's `--corpus <id>` contract,
411
411
  * preserved byte-for-byte) or a filesystem PATH, to be declared as a brand
412
412
  * new `[extensions.<slug>]` entry. Throws the same clear "unknown --corpus"
413
413
  * error the original tier-2-only implementation gave (naming the available
@@ -415,8 +415,13 @@ async function repoRelative(repoRoot, p) {
415
415
  * disk write — callers resolve every pluggable input BEFORE calling
416
416
  * initRepo, so a bad name/path touches nothing. */
417
417
  async function resolvePluggableInput(kind, nameOrPath, { repoRoot }) {
418
- const { BUILTIN_EXTENSIONS } = await import("../src/services/extensions.mjs");
419
- if (Object.prototype.hasOwnProperty.call(BUILTIN_EXTENSIONS, nameOrPath) && BUILTIN_EXTENSIONS[nameOrPath].kind === kind) {
418
+ const { BUILTIN_EXTENSIONS, isSeedableEntry } = await import("../src/services/extensions.mjs");
419
+ const builtin = Object.prototype.hasOwnProperty.call(BUILTIN_EXTENSIONS, nameOrPath)
420
+ ? BUILTIN_EXTENSIONS[nameOrPath]
421
+ : null;
422
+ // A domain pack carries a corpus alongside its vocabulary, so `--corpus <pack>`
423
+ // is the same ask as `--corpus <corpus>` and answers to the same flag.
424
+ if (builtin && (builtin.kind === kind || (kind === "corpus" && builtin.kind === "pack" && isSeedableEntry(builtin)))) {
420
425
  return { known: true, name: nameOrPath };
421
426
  }
422
427
  let manifestIds = null;
@@ -469,11 +474,10 @@ async function activatePluggableInput(repoRoot, resolved) {
469
474
  cfg.extensions = { ...(cfg.extensions || {}), [name]: newEntry };
470
475
  await writeConfig(repoRoot, cfg);
471
476
 
472
- const { resolveExtensions, seedActiveCorpusEntries } = await import("../src/services/extensions.mjs");
477
+ const { resolveExtensions, seedActiveCorpusEntries, isSeedableEntry } = await import("../src/services/extensions.mjs");
473
478
  const { entries } = await resolveExtensions(repoRoot);
474
479
  const entry = entries.get(name);
475
- const seedable = entry.kind === "corpus" || entry.kind === "ontology" || (entry.kind === "pack" && entry.corpusPath);
476
- if (!seedable) {
480
+ if (!isSeedableEntry(entry)) {
477
481
  return `activated "${name}" (${entry.kind}) in tmct.toml — no corpus facts to seed for this kind.\n`;
478
482
  }
479
483
  // Backend-aware seeding (same split-brain bug fix as src/services/init.mjs's own
@@ -867,11 +871,8 @@ async function main() {
867
871
  // record provenance. Idempotent; --force rewrites config + re-records.
868
872
  //
869
873
  // TIERING POLICY: init is OFFLINE, $0 and TIER-1-ONLY by default (seon +
870
- // conceptnet — src/services/extensions.mjs's BUILTIN_EXTENSIONS). A tier-2 domain/
871
- // language corpus (corpus/tier2/: aws, python, java) is added ONLY when
872
- // explicitly asked via `--corpus <id>`. The `--detect` auto-detect is a
873
- // documented STUB: it inspects the repo's manifests (pyproject.toml → python,
874
- // pom.xml → java) and SUGGESTS the matching corpus, but never seeds it unasked.
874
+ // conceptnet — src/services/extensions.mjs's BUILTIN_EXTENSIONS). A tier-2 domain
875
+ // corpus (corpus/tier2/) is added ONLY when explicitly asked via `--corpus <id>`.
875
876
  //
876
877
  // `--repo <abs>` (NEW): init used to always hardcode process.cwd() — the only
877
878
  // subcommand without a --repo flag. It now takes one like every other
@@ -1116,24 +1117,6 @@ async function main() {
1116
1117
 
1117
1118
  if (anyActivation) return;
1118
1119
 
1119
- if (rest.includes("--detect")) {
1120
- // AUTO-DETECT STUB (documented, non-seeding): map a build manifest to the
1121
- // tier-2 corpus that fits, and tell the operator how to add it. Kept a stub on
1122
- // purpose — the $0/offline default never expands the corpus without an ask.
1123
- const { access } = await import("node:fs/promises");
1124
- const has = (f) => access(resolvePath(repoRoot, f)).then(() => true, () => false);
1125
- const DETECT = [["pyproject.toml", "python"], ["pom.xml", "java"]];
1126
- const found = [];
1127
- for (const [file, id] of DETECT) if (await has(file)) found.push([file, id]);
1128
- if (!found.length) {
1129
- process.stdout.write("no tier-2 corpus auto-detected (looked for pyproject.toml → python, pom.xml → java).\n");
1130
- } else {
1131
- for (const [file, id] of found) {
1132
- process.stdout.write(`detected ${file} — run \`tmct init --corpus ${id}\` to add the ${id} tier-2 corpus (offline, $0).\n`);
1133
- }
1134
- }
1135
- return;
1136
- }
1137
1120
  return;
1138
1121
  }
1139
1122
 
@@ -1238,9 +1221,9 @@ async function main() {
1238
1221
  // `--memory-backend` is written FIRST, before any --corpus/--ontology/
1239
1222
  // --lexicon activation below — the same split-brain bug fix as `tmct
1240
1223
  // init`'s ordering: activatePluggableInput seeds into whichever backend
1241
- // tmct.toml names AT THE TIME it runs, so `tmct import --corpus aws
1224
+ // tmct.toml names AT THE TIME it runs, so `tmct import --corpus general
1242
1225
  // --memory-backend sqlite` in one call must have the new backend on disk
1243
- // BEFORE the aws seed step, not after.
1226
+ // BEFORE the seed step, not after.
1244
1227
  if (memoryBackendVal) {
1245
1228
  const { cfg } = await readConfigForRewrite(repoRoot);
1246
1229
  cfg.memory = { ...(cfg.memory || {}), backend: memoryBackendVal };
@@ -1440,7 +1423,7 @@ async function main() {
1440
1423
  const rest = process.argv.slice(3);
1441
1424
  const { resolveRuntimeConfig, strFlag } = await import("../src/services/cli-args.mjs");
1442
1425
  const { newsTurn, resolveNewsConfig, createNewsState } = await import("../src/services/news.mjs");
1443
- const { loadMemory, readFactRows, appendFacts, openMemoryBackend } = await import("../src/adapters/memory/core.mjs");
1426
+ const { loadMemory, readFactRows, appendFacts, removeFacts, openMemoryBackend } = await import("../src/adapters/memory/core.mjs");
1444
1427
  // The same fetcher factory news.html's own session builds from
1445
1428
  // (src/surfaces/web/news-browser-entry.mjs) — real sources over Node's
1446
1429
  // own global fetch, each behind createNewsFetcher's courtesy gate so a
@@ -1453,7 +1436,7 @@ async function main() {
1453
1436
  // news facts must land in the store chat reads back.
1454
1437
  const backendChoice = String(process.env.TMCT_MEMORY_BACKEND || toml?.memory?.backend || "").trim().toLowerCase();
1455
1438
  const { dir: memoryDir, close: closeMemoryStore } = await openMemoryBackend(repo, backendChoice);
1456
- const store = { loadMemory, readFactRows, appendFacts };
1439
+ const store = { loadMemory, readFactRows, appendFacts, removeFacts };
1457
1440
  const memory = await loadMemory(memoryDir);
1458
1441
  const rows = readFactRows(memory);
1459
1442
  // resolveNewsConfig unwraps `toml?.news` itself (the tmct.toml [news]
@@ -30,27 +30,6 @@
30
30
  "shareAlike": false,
31
31
  "notice": "corpus/namenet/LICENSE-NOTICE"
32
32
  },
33
- {
34
- "path": "corpus/tier2/aws.jsonl",
35
- "upstream": "first-party (curated in corpus/tier2/generate.mjs)",
36
- "license": "MPL-2.0",
37
- "shareAlike": false,
38
- "notice": "corpus/tier2/manifest.json"
39
- },
40
- {
41
- "path": "corpus/tier2/python.jsonl",
42
- "upstream": "first-party (curated in corpus/tier2/generate.mjs)",
43
- "license": "MPL-2.0",
44
- "shareAlike": false,
45
- "notice": "corpus/tier2/manifest.json"
46
- },
47
- {
48
- "path": "corpus/tier2/java.jsonl",
49
- "upstream": "first-party (curated in corpus/tier2/generate.mjs)",
50
- "license": "MPL-2.0",
51
- "shareAlike": false,
52
- "notice": "corpus/tier2/manifest.json"
53
- },
54
33
  {
55
34
  "path": "corpus/tier2/general.jsonl",
56
35
  "upstream": "first-party (curated in corpus/tier2/generate.mjs)",
package/corpus/README.md CHANGED
@@ -13,7 +13,7 @@ tmct's knowledge arrives in three tiers, distinguished by **when** it lands and
13
13
  | Tier | What | Ships in the package? | Lands when | Provenance |
14
14
  |---|---|---|---|---|
15
15
  | **1 — base** | the general English/tech ConceptNet slice + the response templates + the SE phrasebook — the vocabulary every tmct has out of the box | **yes**, committed here | `tmct init` seeds `.tmct/` from committed data (offline, $0) | `corpus:conceptnet /r/…` |
16
- | **2 — specialised** | LANGUAGE- or DOMAIN-specific fact sets (`aws`, `python`, `java`) plus one deliberately NON-code-domain "wider general-knowledge" bundle (`general`) so tmct can "expand into a concept for an applicable codebase" or into a seed set that isn't code at all | **no** — selected per repo | activated via `src/services/extensions.mjs`'s `[extensions.tier2-<id>] active = true` (or `tmct init --corpus <id>`), inactive by default | `corpus:tier2-<id> /r/…` |
16
+ | **2 — specialised** | DOMAIN-specific fact sets today the deliberately NON-code-domain "wider general-knowledge" bundle (`general`) plus the `human` persona tiers — so tmct can "expand into a concept for an applicable codebase", or into a seed set that isn't code at all | **no** — selected per repo | activated via `src/services/extensions.mjs`'s `[extensions.tier2-<id>] active = true` (or `tmct init --corpus <id>`), inactive by default | `corpus:tier2-<id> /r/…` |
17
17
  | **3 — learned** | facts tmct writes from the actual conversation / the actual codebase it is pointed at | never committed | at runtime, into `.tmct/memory/` | `chat:…`, `codegraph:…` |
18
18
 
19
19
  **Offline / $0 is the default at every tier.** Tier-1 is committed. Tier-2's
@@ -42,7 +42,7 @@ notice}), guarded by `test/estate/corpus-licences.test.mjs`.
42
42
  | `conceptnet/README.md` | provenance, retrieval date, seed terms, filter rules, row counts | — | — |
43
43
  | `tier2/manifest.json` | tier-2: index of specialised corpuses (id, kind, description, source, sha256, size) | — | MPL-2.0 |
44
44
  | `tier2/generate.mjs` | tier-2: the curated-corpus generator + manifest writer (+ opt-in network-fetch path) | — | MPL-2.0 |
45
- | `tier2/{aws,python,java,general}.jsonl` | tier-2 SAMPLE corpuses — same fact shape as the tier-1 slice, loadable via the same path (`general` is the one deliberately non-code-domain bundle) | ~4-6 KB each | MPL-2.0 |
45
+ | `tier2/general.jsonl` | a tier-2 SAMPLE corpus — same fact shape as the tier-1 slice, loadable via the same path, and deliberately non-code-domain | ~6 KB | MPL-2.0 |
46
46
  | `tier2/human.jsonl` | the DEFAULT active bundle: the everyday "human-world" persona, Small tier (664 facts) — hand-curated from Open English WordNet, bridged to Schema.org's top-level classes | ~80 KB | MPL-2.0 (hand-authored fact set; not a verbatim WordNet/Schema.org excerpt — see `tier2/generate.mjs`'s own header comment) |
47
47
  | `tier2/human-medium.jsonl` / `tier2/human-large.jsonl` | SIZE tiers of the SAME `human` bundle — each holds ONLY the facts that size adds beyond the previous one; both shipped INACTIVE by default, activated via `tmct init --persona-size medium\|large`. Built by `scripts/build-persona-tiers.mjs` from the same WordNet source, automatically curated (sense-ranked, blocklist/denylist-filtered) rather than hand-typed one at a time, given the scale (944 / ~12,000 facts) | ~111 KB / ~1.4 MB | MPL-2.0 (same "hand-authored in homage to the source's shape" basis as `human.jsonl`) |
48
48
  | `prose/sqlite/*.txt` | the frozen prose corpus, code half: plain-text extractions of 12 SQLite documentation pages | ~375 KB | **public domain** (see `prose/sqlite/LICENSE-NOTICE`) |
@@ -136,11 +136,9 @@ loads and seeds through the very same `loadSlice()`/`toFacts()` path as the
136
136
  tier-1 slice — `tier2/generate.mjs --verify` proves it (each sample loads and
137
137
  all its facts seed cleanly, no `ace=none` dead rows).
138
138
 
139
- The idea: a Python repo pulls the `python` corpus so tmct knows "a dict is a
140
- kind of hash table"; an AWS project pulls `aws` so it knows "S3 is object
141
- storage, a bucket is part of S3". Language terms unify onto the **shared CS
142
- concept vocabulary** (`list → array`, `hashmap → hash table`) so specialised
143
- knowledge connects to the tier-1 graph instead of floating apart.
139
+ The idea: a repo pulls the bundle that matches what it is about, and that
140
+ bundle's terms unify onto the **shared concept vocabulary** already in the
141
+ tier-1 graph, so specialised knowledge connects instead of floating apart.
144
142
 
145
143
  **To add a corpus:** add an entry to `CORPUSES` in `tier2/generate.mjs` (a list
146
144
  of `[subject, relation, concept]` triples) and run `node tier2/generate.mjs
@@ -150,9 +148,9 @@ stays reviewable; a corpus too big to hand-curate is a `fetch` manifest entry
150
148
  (URL + sha256, opt-in network — `fetchCorpus()` is the reference downloader).
151
149
 
152
150
  **How tier-2 wires into `tmct init` (done — `src/services/extensions.mjs`):**
153
- `resolveExtensions(repoRoot)` ships all four tier-2 bundles as
154
- shipped-but-inactive `BUILTIN_EXTENSIONS` entries (`tier2-aws`/`tier2-python`/
155
- `tier2-java`/`tier2-general`); `[extensions.tier2-<id>] active = true` in
151
+ `resolveExtensions(repoRoot)` ships every tier-2 bundle as a
152
+ shipped-but-inactive `BUILTIN_EXTENSIONS` entry (`tier2-general`);
153
+ `[extensions.tier2-<id>] active = true` in
156
154
  `tmct.toml` (or `tmct init --corpus <id>`) flips one on, and
157
155
  `seedActiveCorpusEntries` runs it through the exact same
158
156
  `loadSlice → toFacts → appendFacts` pipeline as tier-1, stamped
@@ -160,9 +158,8 @@ shipped-but-inactive `BUILTIN_EXTENSIONS` entries (`tier2-aws`/`tier2-python`/
160
158
  separate tier-2 code path. Idempotency is free (`seedMemory`'s content-hashed
161
159
  fact ids + pre-read skip). None of this touches `package.json` or the tier-1
162
160
  budget; tier-2 files are not shipped, so they do not count against the
163
- ≤ 1.5 MB slice budget. Codebase auto-detection (a `requirements.txt` →
164
- `python`, a `pom.xml`/`build.gradle` `java`, an AWS SDK dep → `aws`) is
165
- still unbuilt — activation today is config-only, never automatic.
161
+ ≤ 1.5 MB slice budget. Activation today is config-only: nothing reads a repo's
162
+ own build manifests to pick a bundle for you.
166
163
 
167
164
  A term that would otherwise be silently dropped when a bundle is seeded (an
168
165
  `ace = "none"` relation like RelatedTo/HasContext, e.g. from a broader slice)
@@ -19,7 +19,7 @@
19
19
  "max": 10485760
20
20
  },
21
21
  "shardsGzBytes": {
22
- "used": 1224167,
22
+ "used": 1224158,
23
23
  "max": 3670016
24
24
  },
25
25
  "indexGzBytes": {
@@ -50,8 +50,8 @@
50
50
  },
51
51
  {
52
52
  "file": "shards/ref-01.jsonl.gz",
53
- "bytes": 15202,
54
- "sha256": "6615f30ae974f043c1ac4f52c9e12c358005272981c492295247bdb387f7941b"
53
+ "bytes": 15193,
54
+ "sha256": "86a07d6f3f9b5ab79ab206e157485d39bb25a996cf3d28d7ca014a45c4686eb8"
55
55
  },
56
56
  {
57
57
  "file": "shards/ref-02.jsonl.gz",
@@ -65,8 +65,8 @@
65
65
  },
66
66
  {
67
67
  "file": "shards/ref-04.jsonl.gz",
68
- "bytes": 21046,
69
- "sha256": "cb09cb6a691c1f14719104867502bb72db43852bdf7165b722fe546690ec9c16"
68
+ "bytes": 21042,
69
+ "sha256": "f8269b02dcbac4d2471043f7e2adc85394ffe61238202f19e54d47a7271943c4"
70
70
  },
71
71
  {
72
72
  "file": "shards/ref-05.jsonl.gz",
@@ -85,8 +85,8 @@
85
85
  },
86
86
  {
87
87
  "file": "shards/ref-08.jsonl.gz",
88
- "bytes": 21630,
89
- "sha256": "f013af59a737e87d1406ca5c19dc09012989fe3ce7d9c97adf6fb7d50ff63369"
88
+ "bytes": 21627,
89
+ "sha256": "21c9cab16fb50ae1183eb25ca9faf2e964b15c7c2220942c25a1567d5cad5428"
90
90
  },
91
91
  {
92
92
  "file": "shards/ref-09.jsonl.gz",
@@ -125,13 +125,13 @@
125
125
  },
126
126
  {
127
127
  "file": "shards/ref-10.jsonl.gz",
128
- "bytes": 17990,
129
- "sha256": "c980e844bc71638111fa3e8ae3c7ca5ebafdfcef74e86e252e1c6858c2778acc"
128
+ "bytes": 17987,
129
+ "sha256": "5a79227e9950e428a046fe580d07fb8529b19c5416fa756b754852f2edbca905"
130
130
  },
131
131
  {
132
132
  "file": "shards/ref-11.jsonl.gz",
133
- "bytes": 17595,
134
- "sha256": "e23ed0bd8fd1f939158705c2624869ef06b89ae5006e1364a7749ad7de1c2cc7"
133
+ "bytes": 17596,
134
+ "sha256": "628fb8699283c3e16703875f58a3bc7d06238477dd6974a2d76da91ad941a6d5"
135
135
  },
136
136
  {
137
137
  "file": "shards/ref-12.jsonl.gz",
@@ -160,8 +160,8 @@
160
160
  },
161
161
  {
162
162
  "file": "shards/ref-17.jsonl.gz",
163
- "bytes": 19992,
164
- "sha256": "ee32aaee2b3fa93085de06ef5fb613a43916de4eaa1efed24bb1e75788a2cbfb"
163
+ "bytes": 19993,
164
+ "sha256": "801799dd9b205e41658fc6074cf9021cab90c2c30e1fc00d0d9bca0d4cd07d76"
165
165
  },
166
166
  {
167
167
  "file": "shards/ref-18.jsonl.gz",
@@ -205,8 +205,8 @@
205
205
  },
206
206
  {
207
207
  "file": "shards/ref-20.jsonl.gz",
208
- "bytes": 22867,
209
- "sha256": "2d131116890dac85725dae54caf7cab62324881a637e354986bbe1bfba4c0ea7"
208
+ "bytes": 22871,
209
+ "sha256": "4f049ef6fb8488fc69eab9cfe826f8dc81cee4293c2f7b96d199752dbb7149ae"
210
210
  },
211
211
  {
212
212
  "file": "shards/ref-21.jsonl.gz",
@@ -230,8 +230,8 @@
230
230
  },
231
231
  {
232
232
  "file": "shards/ref-25.jsonl.gz",
233
- "bytes": 20332,
234
- "sha256": "3656dd63f53fcd502ab385f654488cf291bdb3464f90ca4322799dd569e65e35"
233
+ "bytes": 20328,
234
+ "sha256": "b7ccf2a15d8e9d3f274d35ec7f99d4eb543ed1484aa860c90ca14edaa56e6b4e"
235
235
  },
236
236
  {
237
237
  "file": "shards/ref-26.jsonl.gz",
@@ -265,8 +265,8 @@
265
265
  },
266
266
  {
267
267
  "file": "shards/ref-2c.jsonl.gz",
268
- "bytes": 18088,
269
- "sha256": "28326bc161c881c05bbed6aafaf3c227a3f0abbbc5f873aa62ea11d484b90c19"
268
+ "bytes": 18096,
269
+ "sha256": "0575e2fd83478d64e88f9691f0fbb57a9d913a9d64ace4b1f7f4689300ddbb98"
270
270
  },
271
271
  {
272
272
  "file": "shards/ref-2d.jsonl.gz",
@@ -2,8 +2,8 @@
2
2
  // corpus/tier2/generate.mjs — the TIER-2 specialised-corpus generator + manifest
3
3
  // writer. NOT part of the product path — a maintainer tool. Offline, $0.
4
4
  //
5
- // Tier-2 corpuses are LANGUAGE- or DOMAIN-specific fact sets (aws, python,
6
- // java, …) that tmct fetches/generates into `.tmct/` at init time so it can
5
+ // Tier-2 corpuses are DOMAIN-specific fact sets (general, human, …) that tmct
6
+ // fetches/generates into `.tmct/` at init time so it can
7
7
  // "expand into a concept for an applicable codebase". They are NOT shipped in
8
8
  // the npm package the way the tier-1 ConceptNet slice is — they are opt-in,
9
9
  // selected per repo. See ../README.md for the full tier-1/2/3 policy.
@@ -39,143 +39,7 @@ const HERE = dirname(fileURLToPath(import.meta.url));
39
39
  // turns "/c/en/hash_table" into "hash table". Keep terms <= 3 words (the tier-1
40
40
  // quality-filter rule) — curated data is clean by construction.
41
41
  export const CORPUSES = {
42
- aws: {
43
- kind: "domain",
44
- description: "Amazon Web Services core services and primitives (S3, Lambda, DynamoDB, EC2, IAM, SQS) mapped to general cloud/CS concepts.",
45
- facts: [
46
- ["aws", "/r/IsA", "cloud_platform"],
47
- ["aws", "/r/IsA", "cloud"],
48
- ["aws", "/r/CapableOf", "host_applications"],
49
- // S3
50
- ["s3", "/r/IsA", "object_storage"],
51
- ["s3", "/r/IsA", "storage_service"],
52
- ["s3", "/r/PartOf", "aws"],
53
- ["s3", "/r/HasA", "bucket"],
54
- ["s3", "/r/UsedFor", "storing_files"],
55
- ["s3", "/r/CapableOf", "store_objects"],
56
- ["bucket", "/r/IsA", "container"],
57
- ["bucket", "/r/PartOf", "s3"],
58
- ["bucket", "/r/UsedFor", "storing_objects"],
59
- // Lambda
60
- ["lambda", "/r/IsA", "compute_service"],
61
- ["lambda", "/r/IsA", "function"],
62
- ["lambda", "/r/PartOf", "aws"],
63
- ["lambda", "/r/UsedFor", "running_code"],
64
- ["lambda", "/r/CapableOf", "run_code"],
65
- ["lambda", "/r/HasProperty", "serverless"],
66
- // DynamoDB
67
- ["dynamodb", "/r/IsA", "database"],
68
- ["dynamodb", "/r/IsA", "nosql_database"],
69
- ["dynamodb", "/r/PartOf", "aws"],
70
- ["dynamodb", "/r/HasA", "table"],
71
- ["dynamodb", "/r/UsedFor", "storing_data"],
72
- ["dynamodb", "/r/HasProperty", "managed"],
73
- // EC2
74
- ["ec2", "/r/IsA", "compute_service"],
75
- ["ec2", "/r/IsA", "virtual_machine"],
76
- ["ec2", "/r/PartOf", "aws"],
77
- ["ec2", "/r/HasA", "instance"],
78
- ["ec2", "/r/UsedFor", "running_servers"],
79
- // IAM
80
- ["iam", "/r/IsA", "access_control"],
81
- ["iam", "/r/PartOf", "aws"],
82
- ["iam", "/r/UsedFor", "managing_permissions"],
83
- ["iam", "/r/HasA", "role"],
84
- ["iam", "/r/HasA", "policy"],
85
- // SQS + queue
86
- ["sqs", "/r/IsA", "message_queue"],
87
- ["sqs", "/r/IsA", "queue"],
88
- ["sqs", "/r/PartOf", "aws"],
89
- ["sqs", "/r/UsedFor", "decoupling_services"],
90
- ["queue", "/r/IsA", "data_structure"],
91
- ],
92
- },
93
-
94
- python: {
95
- kind: "language",
96
- description: "Python language constructs and stdlib types mapped to the shared CS concept vocabulary (list->array, dict->hash table, …).",
97
- facts: [
98
- ["python", "/r/IsA", "programming_language"],
99
- ["python", "/r/HasProperty", "interpreted"],
100
- ["python", "/r/HasProperty", "dynamically_typed"],
101
- ["python", "/r/UsedFor", "scripting"],
102
- // built-in types → shared concepts
103
- ["list", "/r/IsA", "array"],
104
- ["list", "/r/IsA", "sequence"],
105
- ["list", "/r/IsA", "data_structure"],
106
- ["dict", "/r/IsA", "hash_table"],
107
- ["dict", "/r/IsA", "dictionary"],
108
- ["dict", "/r/IsA", "mapping"],
109
- ["tuple", "/r/IsA", "sequence"],
110
- ["tuple", "/r/HasProperty", "immutable"],
111
- ["set", "/r/IsA", "collection"],
112
- ["set", "/r/HasProperty", "unordered"],
113
- ["str", "/r/IsA", "string"],
114
- // language constructs
115
- ["decorator", "/r/IsA", "function"],
116
- ["decorator", "/r/UsedFor", "modifying_functions"],
117
- ["generator", "/r/IsA", "iterator"],
118
- ["generator", "/r/UsedFor", "lazy_evaluation"],
119
- ["comprehension", "/r/IsA", "expression"],
120
- ["comprehension", "/r/UsedFor", "building_collections"],
121
- ["exception", "/r/IsA", "error"],
122
- ["module", "/r/IsA", "file"],
123
- ["package", "/r/IsA", "module"],
124
- ["method", "/r/IsA", "function"],
125
- // tooling / runtime
126
- ["pip", "/r/IsA", "package_manager"],
127
- ["pip", "/r/UsedFor", "installing_packages"],
128
- ["gil", "/r/IsA", "lock"],
129
- ["gil", "/r/PartOf", "interpreter"],
130
- ["cpython", "/r/IsA", "interpreter"],
131
- ],
132
- },
133
-
134
- java: {
135
- kind: "language",
136
- description: "Java language and JVM constructs mapped to the shared CS concept vocabulary (ArrayList->list, HashMap->hash table, …).",
137
- facts: [
138
- ["java", "/r/IsA", "programming_language"],
139
- ["java", "/r/HasProperty", "compiled"],
140
- ["java", "/r/HasProperty", "statically_typed"],
141
- ["java", "/r/UsedFor", "building_applications"],
142
- // types → shared concepts
143
- ["arraylist", "/r/IsA", "list"],
144
- ["arraylist", "/r/IsA", "data_structure"],
145
- ["hashmap", "/r/IsA", "hash_table"],
146
- ["hashmap", "/r/IsA", "dictionary"],
147
- ["hashmap", "/r/IsA", "map"],
148
- ["interface", "/r/IsA", "type"],
149
- ["interface", "/r/IsA", "contract"],
150
- ["class", "/r/IsA", "type"],
151
- ["object", "/r/IsA", "instance"],
152
- // JVM / runtime
153
- ["jvm", "/r/IsA", "virtual_machine"],
154
- ["jvm", "/r/UsedFor", "running_bytecode"],
155
- ["jvm", "/r/CapableOf", "execute_bytecode"],
156
- ["bytecode", "/r/IsA", "code"],
157
- ["garbage_collector", "/r/PartOf", "jvm"],
158
- ["garbage_collector", "/r/UsedFor", "freeing_memory"],
159
- // packaging / tooling
160
- ["jar", "/r/IsA", "archive"],
161
- ["jar", "/r/IsA", "file"],
162
- ["jar", "/r/UsedFor", "packaging_classes"],
163
- ["maven", "/r/IsA", "build_tool"],
164
- ["maven", "/r/IsA", "package_manager"],
165
- ["gradle", "/r/IsA", "build_tool"],
166
- // language features
167
- ["thread", "/r/IsA", "process"],
168
- ["thread", "/r/UsedFor", "concurrency"],
169
- ["generics", "/r/UsedFor", "type_safety"],
170
- ["annotation", "/r/IsA", "metadata"],
171
- ["exception", "/r/IsA", "error"],
172
- ["method", "/r/IsA", "function"],
173
- ],
174
- },
175
-
176
- // The wider general-knowledge seed set: the
177
- // three corpuses above are all code-domain-specific (a LANGUAGE or a cloud
178
- // DOMAIN); this one deliberately is NOT — everyday-knowledge concepts (the
42
+ // The wider general-knowledge seed set: everyday-knowledge concepts (the
179
43
  // natural world, weather, food, common objects) with zero code-domain
180
44
  // framing, proving the extension-pack seam generalizes to a seed set that
181
45
  // isn't code at all (the operator's own framing: tmct's code specialization
@@ -14911,9 +14775,9 @@ export async function fetchCorpus(url, expectedSha) {
14911
14775
  * object (e.g. "treating_illness", "growing_into_a_plant") to also become a
14912
14776
  * parseable ACE noun, contradicting the "breadth over depth, no
14913
14777
  * over-modeling" discipline this whole batch was built under, and breaking
14914
- * with EVERY other tier2 corpus's own long-standing precedent (aws.jsonl's
14915
- * "object_storage"/"cloud_platform" concept objects were never lexicon
14916
- * nouns either, and no such stricter check exists for them). The `lexicon`
14778
+ * with EVERY other tier2 corpus's own long-standing precedent (general.jsonl's
14779
+ * concept objects were never lexicon nouns either, and no such stricter check
14780
+ * exists for them). The `lexicon`
14917
14781
  * sub-key's OWN declared words (the genuinely teachable/askable common
14918
14782
  * words this corpus's curation deliberately picked out) are what this
14919
14783
  * check verifies both ends of. */