@polycode-projects/the-mechanical-code-talker 2.10.3 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +68 -12
  2. package/bin/tmct.mjs +5 -2
  3. package/corpus/sprites/src/sprite-facts.jsonl +18 -0
  4. package/corpus/worlds/manifest.json +5 -5
  5. package/corpus/worlds/shards/ashcombe-hall.jsonl.gz +0 -0
  6. package/corpus/worlds/src/ashcombe-hall.jsonl +27 -0
  7. package/data/sprites/book-icon.toml +12 -0
  8. package/data/sprites/cellar-icon.toml +12 -0
  9. package/data/sprites/drawing-room-icon.toml +13 -0
  10. package/data/sprites/garden-icon.toml +12 -0
  11. package/data/sprites/kitchen-icon.toml +13 -0
  12. package/data/sprites/library-icon.toml +12 -0
  13. package/data/sprites/pan-icon.toml +11 -0
  14. package/data/sprites/study-icon.toml +12 -0
  15. package/package.json +7 -2
  16. package/src/adapters/corpus/wikipedia-live.mjs +182 -26
  17. package/src/adapters/corpus/worlds-pack.mjs +8 -2
  18. package/src/adapters/memory/core.mjs +8 -1
  19. package/src/adapters/toml-config.mjs +6 -0
  20. package/src/domain/cli-verbs.mjs +2 -0
  21. package/src/domain/memory/trust.mjs +32 -2
  22. package/src/domain/sense-split.mjs +203 -0
  23. package/src/domain/worlds-pack.mjs +50 -0
  24. package/src/services/adventure-autoplay.mjs +5 -2
  25. package/src/services/adventure-viz.mjs +301 -33
  26. package/src/services/adventure.mjs +162 -14
  27. package/src/services/chat-page-viz.mjs +341 -197
  28. package/src/services/chat-session.mjs +24 -9
  29. package/src/services/chat.mjs +580 -47
  30. package/src/services/code-explorer-viz.mjs +198 -76
  31. package/src/services/extract-facts.mjs +384 -82
  32. package/src/services/fold.mjs +1 -1
  33. package/src/services/ingest-viz.mjs +637 -0
  34. package/src/services/ledger-viz.mjs +209 -0
  35. package/src/services/memory-panel-viz.mjs +159 -0
  36. package/src/services/research.mjs +266 -0
  37. package/src/services/sentences.mjs +19 -0
  38. package/src/services/session-log-format.mjs +64 -0
  39. package/src/services/sessions.mjs +56 -22
  40. package/src/services/spider-fly-turn.mjs +54 -1
  41. package/src/services/spider-fly-viz.mjs +41 -23
  42. package/src/surfaces/web/adventure-browser-entry.mjs +9 -5
  43. package/src/surfaces/web/chat-browser-entry.mjs +32 -11
  44. package/src/surfaces/web/code-explorer-browser-entry.mjs +27 -11
  45. package/src/surfaces/web/ingest-browser-entry.mjs +208 -0
  46. package/src/surfaces/web/ledger-browser-entry.mjs +24 -5
  47. package/src/surfaces/web/memory-ask-browser.bundle.js +134 -125
  48. package/src/surfaces/web/memory-stats.mjs +53 -0
  49. package/src/tools/definitions.mjs +14 -0
  50. package/src/tools/handlers/index.mjs +2 -0
  51. package/src/tools/handlers/tmct-ingest.mjs +43 -0
  52. package/src/tools/server.mjs +5 -2
@@ -304,7 +304,7 @@ ${THEME_TOKENS_CSS}
304
304
  split plus the gap approximates that pair of fractions without them
305
305
  needing to sum to a full width. */
306
306
  .stage { display: grid; grid-template-columns: minmax(0, 8fr) minmax(280px, 3fr); gap: 1.2rem; align-items: start; }
307
- @media (max-width: 760px) { .stage { grid-template-columns: 1fr; } }
307
+ @media (max-width: 760px) { .stage { grid-template-columns: minmax(0, 1fr); } }
308
308
  /* A dusty window corner: a soft light glow near the top-left (WEB_HOME
309
309
  already sits near that corner — spider-fly-world.mjs's own header
310
310
  comment), and a faint diagonal weave standing in for dust/silk caught
@@ -346,10 +346,21 @@ ${THEME_TOKENS_CSS}
346
346
  negative margin equal to this padding, so it reads as one welded unit,
347
347
  not a label floating inside a box. */
348
348
  .hud, .chat, .tuning { background: var(--chrome-face); border: 1px solid var(--chrome-edge-lo); box-shadow: var(--chrome-shadow-raised); border-radius: 2px; padding: .6rem .75rem; }
349
- /* the tuning strip matches the board's own width (not the wider stage-left
350
- column it sits in) so the two line up as one visual stack, the same way
351
- the board-frame below it is already centered at a fixed width. */
352
- .tuning { width: ${BOARD_PX}px; max-width: 100%; margin: 0 auto; box-sizing: border-box; }
349
+ /* Both the controls strip above the board and the tuning strip below it
350
+ match the board's own width (not the wider stage-left column they sit
351
+ in), so all three line up as one visual stack. */
352
+ .tuning, .controls-panel, .head-inner { width: ${BOARD_PX}px; max-width: 100%; margin: 0 auto; box-sizing: border-box; }
353
+ /* the eyebrow+h1 sit in their own board-width column (.head-inner), inside
354
+ a .page-head row that mirrors the real board's own 8fr/3fr split below
355
+ it — the second, empty column just holds that split in place so the
356
+ first column's own left edge lines up with the board's, instead of the
357
+ wider page margin. min-width: 0 matches .stage-left's own override:
358
+ without it, a grid item's default automatic minimum size is its
359
+ content's own max-content width, and at the phone-width single-column
360
+ override below the fixed-width child would force the track (and the
361
+ page) wider than the viewport instead of actually shrinking. */
362
+ .page-head { margin-bottom: 0; }
363
+ .head-inner { min-width: 0; }
353
364
  .hud h2, .chat h2, .tuning h2 {
354
365
  font-family: ${MONO_STACK}; font-size: .68rem; letter-spacing: .1em; text-transform: uppercase; font-weight: 600;
355
366
  margin: -.6rem -.75rem .5rem; padding: .42rem .75rem;
@@ -447,7 +458,7 @@ ${THEME_TOKENS_CSS}
447
458
  .tuning-col.spider input[type="range"]::-moz-range-thumb { border-color: var(--taught); }
448
459
  .tuning-col.fly input[type="range"]::-moz-range-thumb { border-color: var(--fly); }
449
460
  .tuning-col input:disabled { opacity: .45; }
450
- .controls-row { display: flex; align-items: center; gap: .6rem; margin-top: 1rem; flex-wrap: wrap; }
461
+ .controls-row { display: flex; align-items: center; gap: .6rem; flex-wrap: wrap; }
451
462
  /* Chunky keycaps: raised by default, pressed-in on :active — the one place
452
463
  this redesign uses interaction (not animation) to sell "physical
453
464
  control panel", and it costs nothing under reduced-motion since neither
@@ -461,19 +472,39 @@ ${THEME_TOKENS_CSS}
461
472
  page is remembered by. */
462
473
  .controls-row .turn { margin-left: auto; font-family: ${MONO_STACK}; font-size: .82rem; letter-spacing: .05em; text-transform: uppercase; font-variant-numeric: tabular-nums; color: var(--chrome-well-ink); background: var(--chrome-well); border: 1px solid var(--chrome-brass); box-shadow: var(--chrome-shadow-inset); border-radius: 2px; padding: .3rem .7rem; }
463
474
  .status { font-family: ${MONO_STACK}; font-size: .74rem; color: var(--muted); margin-top: .5rem; padding-left: .5rem; border-left: 2px solid var(--chrome-brass); }
464
- body.preview .side, body.preview .controls-row, body.preview .status, body.preview .tuning { display: none; }
475
+ body.preview .side, body.preview .controls-panel, body.preview .tuning { display: none; }
465
476
  body.preview main { padding: 0; max-width: none; }
466
477
  body.preview .stage, body.preview .stage-left { display: block; }
467
478
  body.preview .board-frame { margin: 0; }
468
- body.preview .eyebrow, body.preview h1 { display: none; }
479
+ body.preview .page-head { display: none; }
469
480
  </style>
470
481
  </head>
471
482
  <body>
472
483
  <main>
473
- <div class="eyebrow">tmct &middot; spider and fly</div>
474
- <h1>Multiple competing planning agents</h1>
484
+ <div class="stage page-head">
485
+ <div class="head-inner">
486
+ <div class="eyebrow">tmct &middot; spider and fly</div>
487
+ <h1>Multiple competing planning agents</h1>
488
+ </div>
489
+ <div></div>
490
+ </div>
475
491
  <div class="stage">
476
492
  <div class="stage-left">
493
+ <div class="controls-panel" id="controlsPanel" aria-label="game controls">
494
+ <div class="controls-row">
495
+ <button id="resetBtn" type="button" disabled>reset</button>
496
+ <button id="playBtn" type="button" disabled>&#9654; play</button>
497
+ <button id="stepBtn" type="button" disabled>step</button>
498
+ <span class="turn mono" id="turnLabel">turn: 0</span>
499
+ </div>
500
+ <div class="status" id="status">loading the engine&hellip;</div>
501
+ </div>
502
+ <div class="board-frame" id="boardFrame">
503
+ <canvas id="board" width="${BOARD_PX}" height="${BOARD_PX}" aria-label="the 10x10 board"></canvas>
504
+ <canvas id="pov" width="${BOARD_PX}" height="${BOARD_PX}" aria-hidden="true"></canvas>
505
+ <div class="sprite-layer" id="spriteLayer"></div>
506
+ <div class="thread-tip" id="threadTip"></div>
507
+ </div>
477
508
  <div class="tuning" id="tuning">
478
509
  <h2>live tuning &mdash; mass loss, spawn rate, vision, per class</h2>
479
510
  <div class="tuning-grid">
@@ -497,12 +528,6 @@ ${THEME_TOKENS_CSS}
497
528
  </div>
498
529
  </div>
499
530
  </div>
500
- <div class="board-frame" id="boardFrame">
501
- <canvas id="board" width="${BOARD_PX}" height="${BOARD_PX}" aria-label="the 10x10 board"></canvas>
502
- <canvas id="pov" width="${BOARD_PX}" height="${BOARD_PX}" aria-hidden="true"></canvas>
503
- <div class="sprite-layer" id="spriteLayer"></div>
504
- <div class="thread-tip" id="threadTip"></div>
505
- </div>
506
531
  </div>
507
532
  <aside class="side" aria-label="Chat and agents">
508
533
  <div class="chat">
@@ -528,13 +553,6 @@ ${THEME_TOKENS_CSS}
528
553
  </div>
529
554
  </aside>
530
555
  </div>
531
- <div class="controls-row">
532
- <button id="resetBtn" type="button" disabled>reset</button>
533
- <button id="playBtn" type="button" disabled>&#9654; play</button>
534
- <button id="stepBtn" type="button" disabled>step</button>
535
- <span class="turn mono" id="turnLabel">turn: 0</span>
536
- </div>
537
- <div class="status" id="status">loading the engine&hellip;</div>
538
556
  </main>
539
557
  <script>
540
558
  const SPIDERFLY = ${gridData};
@@ -35,7 +35,7 @@ import {
35
35
  } from "../../adapters/memory/core.mjs";
36
36
  import { parseEntities } from "../../domain/codegraph.mjs";
37
37
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
38
- import { foldWorldState, worldDigestRows, roomAffordances } from "../../services/adventure.mjs";
38
+ import { foldWorldState, worldDigestRows, roomAffordances, worldActionRows } from "../../services/adventure.mjs";
39
39
  import { runAdventureAutoplayTick, exposedFacts } from "../../services/adventure-autoplay.mjs";
40
40
  import { parseWorldEditorText, planWorldEditorSync } from "../../services/adventure-editor.mjs";
41
41
  import { resolveSpriteForClass, SPRITE_REGISTRY, classAncestorChain } from "../../domain/sprite-map.mjs";
@@ -92,7 +92,7 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
92
92
  // bookkeeping either way.
93
93
  let visitedRoomIds = new Set(Array.isArray(restoredVisitedRoomIds) ? restoredVisitedRoomIds : []);
94
94
  const openingRows = readFactRows(await loadMemory(memoryDir));
95
- const openingHere = foldWorldState(openingRows).placements.get("player")?.object ?? null;
95
+ const openingHere = foldWorldState(worldActionRows(openingRows)).placements.get("player")?.object ?? null;
96
96
  if (openingHere) visitedRoomIds.add(openingHere);
97
97
 
98
98
  const graph = parseEntities({ individuals: [], objectProperties: [] });
@@ -126,7 +126,9 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
126
126
  try {
127
127
  result = await runTurn(line, {
128
128
  config: null, source: null, graph, focus, last, memoryDir, sessionId,
129
- env: {}, lexicon, vocabHint: "", planState: planHolder.state,
129
+ env: {}, lexicon, uiContext: "browser",
130
+ vocabHint: 'Try a world question ("where is the key"), or teach me: "remember: the moat is a ditch".',
131
+ planState: planHolder.state,
130
132
  });
131
133
  } catch (e) {
132
134
  const message = e instanceof Error ? e.message : String(e);
@@ -135,7 +137,7 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
135
137
  focus = result.focus;
136
138
  last = result.last;
137
139
  if ("planState" in result) planHolder.state = result.planState;
138
- const here = foldWorldState(readFactRows(await loadMemory(memoryDir))).placements.get("player")?.object ?? null;
140
+ const here = foldWorldState(worldActionRows(readFactRows(await loadMemory(memoryDir)))).placements.get("player")?.object ?? null;
139
141
  if (here) visitedRoomIds.add(here);
140
142
  return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null };
141
143
  },
@@ -148,7 +150,9 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
148
150
  * reference into this closure's own Set. */
149
151
  async snapshot() {
150
152
  const rows = readFactRows(await loadMemory(memoryDir));
151
- const state = foldWorldState(rows);
153
+ // State the page plays and maps from is the game world only; the raw
154
+ // rows still travel for the digest's background colour.
155
+ const state = foldWorldState(worldActionRows(rows));
152
156
  const here = state.placements.get("player")?.object ?? null;
153
157
  return { rows, state, here, turn: state.turnCount, visitedRoomIds: [...visitedRoomIds] };
154
158
  },
@@ -19,6 +19,7 @@
19
19
  import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
20
20
  import { createInMemoryStore, normFactTerm, loadMemory, readFactRows } from "../../adapters/memory/core.mjs";
21
21
  import { serializeFactsJsonl } from "../../adapters/memory/export-jsonl.mjs";
22
+ import { splitSentencesPreservingPaths } from "../../services/sentences.mjs";
22
23
  import { provenanceTagToSource } from "../../domain/memory/trust.mjs";
23
24
  import { parseEntities } from "../../domain/codegraph.mjs";
24
25
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
@@ -32,7 +33,9 @@ import { registerReferencePackProvider } from "../../adapters/corpus/reference-p
32
33
  // The LIVE Wikipedia seam (opt-in, default off): the page's toggle enables it
33
34
  // per session, and e2e tests stub the provider the same way the pack's own
34
35
  // provider is stubbed. The adapter is fetch-only, so it bundles as-is.
35
- import { registerLiveReferenceProvider } from "../../adapters/corpus/wikipedia-live.mjs";
36
+ // registerResearchProvider is the research lane's sibling seam
37
+ // (simple.wikipedia.org) — same stubbing contract for its e2e tests.
38
+ import { registerLiveReferenceProvider, registerResearchProvider } from "../../adapters/corpus/wikipedia-live.mjs";
36
39
  // Best-effort IndexedDB persistence for the page's session store — the page
37
40
  // decides when to save/load/clear; this entry only carries the wrapper
38
41
  // across the bundle boundary.
@@ -51,7 +54,7 @@ import { openPersistedStore } from "./idb-persist.mjs";
51
54
  * { answer, end, record, plan } and threads focus/last/planState between
52
55
  * calls exactly as the CLI session does.
53
56
  */
54
- export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null } = {}) {
57
+ export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null, synthesisBudget = 12 } = {}) {
55
58
  const memoryDir = createInMemoryStore();
56
59
  // Spread onto the store's own empty payload so a partial seed (individuals
57
60
  // and objectProperties only) still carries the classes/prefixes scaffolding
@@ -68,15 +71,28 @@ export function createChatSession({ seedPayload = null, vocabSeeded = false, liv
68
71
  let focus = null;
69
72
  let last = null;
70
73
  let planState = null;
71
- let liveReferenceOn = Boolean(liveReference);
74
+ let researchState = null;
75
+ // Four-state, like the CLI: false (off), true (rescue on a miss),
76
+ // "supplement" (also append a cited read-out under every grounded vocabulary
77
+ // answer), or "always" (widen that to every grounded answer). The two string
78
+ // modes stay strings so runTurn reads them as the supplement/always lanes.
79
+ const normLive = (v) => (v === "always" ? "always" : v === "supplement" ? "supplement" : Boolean(v));
80
+ let liveReferenceOn = normLive(liveReference);
81
+ // The auto-synthesis budget for this session's learn-on-miss loads — the
82
+ // page's slider sets it; 0 stores article facts without any entailed rows.
83
+ let synthesisBudgetOn = Number.isFinite(synthesisBudget) ? synthesisBudget : 12;
72
84
 
73
85
  return {
74
86
  memoryDir,
75
87
  sessionId,
76
88
  get liveReference() { return liveReferenceOn; },
77
- /** The page's toggle seam: flip the live Wikipedia supplement for every
78
- * later turn (the `/wiki on|off` command flips the same state). */
79
- setLiveReference(v) { liveReferenceOn = Boolean(v); },
89
+ /** The page's toggle seam: set the live Wikipedia mode for every later turn
90
+ * (the `/wiki on|off|supplement|always` command sets the same state). */
91
+ setLiveReference(v) { liveReferenceOn = normLive(v); },
92
+ get synthesisBudget() { return synthesisBudgetOn; },
93
+ /** The page's slider seam: set the auto-synthesis budget for every later
94
+ * learn-on-miss load. Clamped to a non-negative integer; 0 disables it. */
95
+ setSynthesisBudget(n) { synthesisBudgetOn = Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; },
80
96
 
81
97
  /** One dispatched turn. A throwing runTurn must never kill the session —
82
98
  * the page has no other chance to show this turn's answer. */
@@ -85,18 +101,23 @@ export function createChatSession({ seedPayload = null, vocabSeeded = false, liv
85
101
  try {
86
102
  result = await runTurn(line, {
87
103
  config: null, source: null, graph, focus, last, memoryDir, sessionId,
88
- env: {}, lexicon, vocabHint, planState,
104
+ env: {}, lexicon, vocabHint, planState, researchState,
89
105
  liveReference: liveReferenceOn, onLiveLookup,
106
+ uiContext: "browser", synthesisBudget: synthesisBudgetOn,
90
107
  });
91
108
  } catch (e) {
92
109
  const message = e instanceof Error ? e.message : String(e);
93
- return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, plan: null };
110
+ return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, plan: null, research: null };
94
111
  }
95
112
  focus = result.focus;
96
113
  last = result.last;
97
114
  if ("planState" in result) planState = result.planState;
98
- if (typeof result.liveReference === "boolean") liveReferenceOn = result.liveReference;
99
- return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null };
115
+ if ("researchState" in result) researchState = result.researchState;
116
+ if (typeof result.liveReference === "boolean" || result.liveReference === "supplement" || result.liveReference === "always") liveReferenceOn = result.liveReference;
117
+ // `research` distinguishes three cases on purpose: a queue snapshot
118
+ // (research turn), null (a research turn that ended the run), and
119
+ // undefined (not a research turn — the page leaves its controls alone).
120
+ return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null, research: result.research };
100
121
  },
101
122
  };
102
123
  }
@@ -158,4 +179,4 @@ export async function exportFactsJsonl(memoryDir) {
158
179
  return serializeFactsJsonl(await loadMemory(memoryDir));
159
180
  }
160
181
 
161
- globalThis.tmctChat = { createChatSession, registerWinkModel, registerReferencePackProvider, registerLiveReferenceProvider, normFactTerm, vocabExampleHint, memoryStats, openPersistedStore, exportFactsJsonl };
182
+ globalThis.tmctChat = { createChatSession, registerWinkModel, registerReferencePackProvider, registerLiveReferenceProvider, registerResearchProvider, normFactTerm, vocabExampleHint, memoryStats, openPersistedStore, exportFactsJsonl, splitSentences: splitSentencesPreservingPaths };
@@ -1,17 +1,19 @@
1
1
  // code-explorer-browser-entry.mjs — the esbuild entry for the code explorer's
2
- // LIVE chat dock (public/code-explorer.bundle.js / electron/renderer/
2
+ // LIVE chat (public/code-explorer.bundle.js / electron/renderer/
3
3
  // code-explorer.bundle.js). It mirrors ledger-browser-entry.mjs, but seeds the
4
- // full runTurn engine from a CODE graph instead of a memory payload: the dock
5
- // answers "what does X import" / "which functions call Y" over the loaded
6
- // graph, the same compositional shapes the hint rail suggests.
4
+ // full runTurn engine from a CODE graph and, when the page hands one over,
5
+ // the same general-knowledge seed payload chat.html boots from so one
6
+ // session answers "which functions call Y" over the loaded graph and "what is
7
+ // a queue" over the seeded memory.
7
8
  //
8
9
  // The graph enters through source.mjs's provider seam — the same seam the CLI
9
10
  // and HTTP surfaces read — so runTurn's symbol-grain lanes see the whole
10
11
  // payload, while parseEntities builds the coarse graph the flat lanes read.
12
+ // The seed enters the in-memory store the same way createChatSession's does.
11
13
  // Gitignored, built fresh by scripts/build-code-explorer-bundle.mjs; the page
12
14
  // degrades to a static view when it is absent (renderCodeExplorerHtml's own
13
15
  // contract), so nothing here is ever published.
14
- import { runTurn } from "../../services/chat.mjs";
16
+ import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
15
17
  import { createInMemoryStore, normFactTerm } from "../../adapters/memory/core.mjs";
16
18
  import { parseEntities } from "../../domain/codegraph.mjs";
17
19
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
@@ -23,17 +25,30 @@ import { generateCodeHints } from "../../domain/code-explorer-hints.mjs";
23
25
  /**
24
26
  * A browser code-explorer session over the real turn engine. Registers the
25
27
  * loaded payload with source.mjs's provider seam, parses the coarse graph, and
26
- * dispatches every turn through the same runTurn the CLI runs. Teaches land in
27
- * an in-memory store so a taught fact never touches disk. Returns
28
- * { sessionId, turn }, the createChatSession shape the page's dock expects.
28
+ * dispatches every turn through the same runTurn the CLI runs.
29
+ *
30
+ * `seedPayload` (optional) is a serialized memory graph (loadMemory's shape,
31
+ * built by scripts/build-chat-seed.mjs) assigned onto the session's fresh
32
+ * in-memory store, so general-knowledge questions ground alongside the code
33
+ * graph's own lanes. `vocabSeeded` tells the vocabulary hint whether that
34
+ * payload carries the starter vocabulary. Teaches land in the same in-memory
35
+ * store, so a taught fact never touches disk.
36
+ *
37
+ * Returns { memoryDir, sessionId, turn }, the createChatSession shape the
38
+ * page's chat expects.
29
39
  */
30
- export function createCodeExplorerSession({ graphPayload } = {}) {
40
+ export function createCodeExplorerSession({ graphPayload, seedPayload = null, vocabSeeded = false } = {}) {
31
41
  const payload = graphPayload || { individuals: [], objectProperties: [] };
32
42
  source.registerProvider(() => payload);
33
43
 
34
44
  const graph = parseEntities(payload);
35
45
  const memoryDir = createInMemoryStore();
46
+ // Spread onto the store's own empty payload so a partial seed (individuals
47
+ // and objectProperties only) still carries the classes/prefixes scaffolding
48
+ // the write path recounts — teach turns must work on any seed.
49
+ if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
36
50
  const lexicon = loadLexicon();
51
+ const vocabHint = vocabExampleHint(vocabSeeded);
37
52
  const sessionId = globalThis.crypto?.randomUUID?.() ?? String(Date.now());
38
53
  // A stable virtual path: the provider answers every fetch, so no file is
39
54
  // ever read, but the code lanes still do path math (join/dirname) on it.
@@ -44,13 +59,14 @@ export function createCodeExplorerSession({ graphPayload } = {}) {
44
59
  let planState = null;
45
60
 
46
61
  return {
62
+ memoryDir,
47
63
  sessionId,
48
64
  async turn(line) {
49
65
  let result;
50
66
  try {
51
67
  result = await runTurn(line, {
52
68
  config, source, graph, focus, last, memoryDir, sessionId,
53
- env: {}, lexicon, vocabHint: "", planState,
69
+ env: {}, lexicon, vocabHint, planState,
54
70
  });
55
71
  } catch (e) {
56
72
  const message = e instanceof Error ? e.message : String(e);
@@ -66,7 +82,7 @@ export function createCodeExplorerSession({ graphPayload } = {}) {
66
82
 
67
83
  // Exposed for the page's inline client: the re-derivation helpers so a graph
68
84
  // swapped through the desktop picker re-renders without duplicating logic, plus
69
- // the wink loader hook registerWinkModel and normFactTerm the dock shares with
85
+ // the wink loader hook registerWinkModel and normFactTerm the chat shares with
70
86
  // the ledger page.
71
87
  globalThis.tmctCodeExplorer = {
72
88
  createCodeExplorerSession,
@@ -0,0 +1,208 @@
1
+ // ingest-browser-entry.mjs — the esbuild entry for the ingest page's engine
2
+ // (public/ingest-browser.bundle.js, built by scripts/build-ingest-bundle.mjs),
3
+ // mirroring ledger-browser-entry.mjs's own createLedgerSession shape.
4
+ //
5
+ // The ingest page turns plain pasted/dropped text into stored facts by
6
+ // running each sentence through the SAME deterministic recognizer the chat
7
+ // teach lane already has (runTurn, src/services/chat.mjs) — no new NLU, no
8
+ // LLM, no guessing, exactly as the `tmct extract` CLI does it
9
+ // (src/services/extract-facts.mjs). A sentence the recognizer turns into a
10
+ // stored assertion is kept; every other sentence is honestly skipped.
11
+ //
12
+ // The recognizer is reached through ONE seam — `groundTextToFacts` — so a
13
+ // wider ingest tier (an optimistic fuzzy-match pass, a canonical/graph-linked
14
+ // output) slots in behind this single function without the page changing.
15
+ // Alongside the strict recognizer, groundTextToFacts runs the same
16
+ // citation-stripping, clause-fallback and bounded pronoun-carry passes
17
+ // `ingestText` (extract-facts.mjs) applies for the CLI, and — opt-in, off by
18
+ // default — the same low-trust optimistic tier, so the browser page and the
19
+ // CLI ground the identical class of sentence.
20
+ //
21
+ // Gitignored, Pages-demo-site-only output (scripts/build-demo-site.mjs builds
22
+ // it fresh on every deploy, never committed) — the same posture
23
+ // ledger-browser-entry.mjs documents for its own bundle. It carries the full
24
+ // runTurn engine, the same weight class as the chat/ledger bundles, and is
25
+ // never published.
26
+ import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
27
+ import { createInMemoryStore, normFactTerm, loadMemory, readFactRows, appendFact } from "../../adapters/memory/core.mjs";
28
+ import { serializeFactsJsonl } from "../../adapters/memory/export-jsonl.mjs";
29
+ import { splitSentencesPreservingPaths, stripCitationResidue } from "../../services/sentences.mjs";
30
+ import { clauseCandidates, optimisticTriples } from "../../services/extract-facts.mjs";
31
+ import { touchedFactRows } from "../../domain/memory/touched-facts.mjs";
32
+ import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
33
+ import { registerWinkModel, winkInstance } from "../../adapters/wink-model.mjs";
34
+ import { memoryStats } from "./memory-stats.mjs";
35
+ import { openPersistedStore } from "./idb-persist.mjs";
36
+
37
+ // The pronoun subjects a bounded carry substitutes with the last unique
38
+ // grounded subject in the SAME paragraph. Reset at every blank line, so a
39
+ // fresh paragraph never resolves against a stale antecedent.
40
+ const PRONOUN_LEAD_RE = /^(?:they|it|these|those|this)\b\s*/i;
41
+
42
+ /**
43
+ * The single recognizer seam. Splits `text` into paragraphs (blank-line
44
+ * separated, so the pronoun carry never bridges a topic break) and each
45
+ * paragraph into sentences (wink's own boundary detection, never a regex).
46
+ * Each sentence's citation residue ("[3]", "[citation needed]") is stripped
47
+ * before grounding; the strict recognizer then tries the whole sentence
48
+ * first, falling back to its clause fragments only on a miss
49
+ * (`clauseCandidates`), and — still on a miss, only for a pronoun-led
50
+ * sentence with a unique adjacent antecedent in this paragraph — one retry
51
+ * with that antecedent substituted in.
52
+ *
53
+ * A sentence still ungrounded after all of that runs the optimistic fuzzy
54
+ * tier when `optimistic` is set (`optimisticTriples`): a copula or known
55
+ * relation verb flanked by two resolvable entities, stored under the
56
+ * low-trust `optimistic-extract:page` tag rather than a teach/assert
57
+ * provenance, so a fuzzy candidate can never corroborate a curated fact.
58
+ *
59
+ * `onFact(fact)` (optional) is awaited after each grounded row so the page can
60
+ * render the canonical facts LIVE as they land, one at a time.
61
+ *
62
+ * Returns { sentences, recognized, skipped, facts } — `facts` an array of
63
+ * { subject, predicate, object, provenance, quantifier, sentence } in the same
64
+ * canonical shape `tmct extract` and the JSONL exporter emit.
65
+ */
66
+ export async function groundTextToFacts(text, { memoryDir, sessionId, lexicon, vocabHint, optimistic = false, onFact = null } = {}) {
67
+ const paragraphs = String(text ?? "").split(/\n[ \t]*\n/);
68
+ const facts = [];
69
+ let focus = null;
70
+ let last = null;
71
+ let planState = null;
72
+ let sentenceCount = 0;
73
+ let recognized = 0;
74
+ // Reused, not reloaded: readFactRows(loadMemory(...)) is O(rows), so a
75
+ // fresh read on every sentence over a large seeded store is exactly the
76
+ // O(sentences × rows) cost this page must avoid. The snapshot only moves
77
+ // forward past a sentence that actually asserted something.
78
+ let beforeRows = readFactRows(await loadMemory(memoryDir));
79
+ const nlp = optimistic ? winkInstance() : null;
80
+
81
+ // One grounding attempt against `memoryDir`: runs `form` through runTurn,
82
+ // threading focus/last/planState the same way whether or not it grounds.
83
+ // No `graph` is passed — this page never offers a code graph to link
84
+ // against, and a non-null one (even an empty one) makes runTurn read an
85
+ // ordinary "the number of X" phrase as a graph count query instead of
86
+ // running the teach cascade, exactly the phrasing the Sales-paragraph
87
+ // sentence this pipeline is built to ground uses.
88
+ // Returns the Fact rows this attempt actually touched, or null — a miss, a
89
+ // non-assert turn, or an assert that touched no Fact row (a Rule teach).
90
+ async function attempt(form) {
91
+ let record;
92
+ try {
93
+ const result = await runTurn(form, {
94
+ config: null, source: null, focus, last, memoryDir, sessionId,
95
+ env: {}, lexicon, vocabHint, planState, uiContext: "browser",
96
+ });
97
+ focus = result.focus;
98
+ last = result.last;
99
+ if ("planState" in result) planState = result.planState;
100
+ record = result.record;
101
+ } catch {
102
+ return null; // a throwing sentence is a skip, never a page-killer
103
+ }
104
+ if (record?.via !== "assert" || record?.miss) return null;
105
+ // The one loadMemory/readFactRows call this attempt pays for, only
106
+ // because it actually asserted something. Equal lengths mean nothing was
107
+ // ADDED — a re-assertion's provenance-only change is the one case this
108
+ // fast path can't see, an accepted trade against paying the full
109
+ // before/after diff on every recognized sentence in a large store.
110
+ const afterRows = readFactRows(await loadMemory(memoryDir));
111
+ const rows = afterRows.length === beforeRows.length ? [] : touchedFactRows(beforeRows, afterRows);
112
+ beforeRows = afterRows;
113
+ return rows.length ? rows : null;
114
+ }
115
+
116
+ for (const paragraph of paragraphs) {
117
+ // The last unique grounded subject in THIS paragraph, carried onto a
118
+ // later pronoun-led sentence the strict recognizer couldn't ground on its
119
+ // own. Cleared at the paragraph boundary.
120
+ let carrySubject = null;
121
+ for (const rawSentence of splitSentencesPreservingPaths(paragraph)) {
122
+ sentenceCount += 1;
123
+ const cleaned = stripCitationResidue(rawSentence);
124
+
125
+ let rows = null;
126
+ for (const candidate of clauseCandidates(cleaned, { nlp })) {
127
+ rows = await attempt(candidate);
128
+ if (rows) break;
129
+ }
130
+ if (!rows && carrySubject && PRONOUN_LEAD_RE.test(cleaned)) {
131
+ rows = await attempt(cleaned.replace(PRONOUN_LEAD_RE, `${carrySubject} `));
132
+ }
133
+
134
+ if (rows) {
135
+ recognized += 1;
136
+ const subjects = new Set(rows.map((r) => r.subject));
137
+ if (subjects.size === 1) carrySubject = [...subjects][0];
138
+ for (const row of rows) {
139
+ const fact = {
140
+ subject: row.subject, predicate: row.predicate, object: row.object,
141
+ provenance: row.provenance || "", quantifier: row.quantifier || "", sentence: rawSentence,
142
+ };
143
+ facts.push(fact);
144
+ if (onFact) await onFact(fact);
145
+ }
146
+ continue;
147
+ }
148
+
149
+ if (!optimistic) continue;
150
+ const candidates = optimisticTriples(cleaned, { lexicon, nlp });
151
+ if (!candidates.length) continue;
152
+ recognized += 1;
153
+ for (const t of candidates) {
154
+ await appendFact(memoryDir, { subject: t.subject, predicate: t.predicate, object: t.object, provenance: "optimistic-extract:page" });
155
+ const fact = {
156
+ subject: t.subject, predicate: t.predicate, object: t.object,
157
+ provenance: "optimistic-extract:page", quantifier: "", sentence: rawSentence,
158
+ };
159
+ facts.push(fact);
160
+ if (onFact) await onFact(fact);
161
+ }
162
+ }
163
+ }
164
+
165
+ return { sentences: sentenceCount, recognized, skipped: sentenceCount - recognized, facts };
166
+ }
167
+
168
+ /**
169
+ * A browser ingest session over the real turn engine — a fresh in-memory
170
+ * Backend-B store, so facts grounded through the page extend one real graph
171
+ * the page can then export. `seedPayload` (optional) pre-loads a graph the
172
+ * recognizer can recall and link against.
173
+ *
174
+ * Returns { memoryDir, sessionId, ingest }. `ingest(text, { onFact,
175
+ * optimistic })` is the one call the page makes; it drives groundTextToFacts
176
+ * against this session's store and returns its { sentences, recognized,
177
+ * skipped, facts } summary.
178
+ */
179
+ export function createIngestSession({ seedPayload = null, vocabSeeded = false } = {}) {
180
+ const memoryDir = createInMemoryStore();
181
+ if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
182
+
183
+ const lexicon = loadLexicon();
184
+ const vocabHint = vocabExampleHint(vocabSeeded);
185
+ const sessionId = globalThis.crypto?.randomUUID?.() ?? String(Date.now());
186
+
187
+ return {
188
+ memoryDir,
189
+ sessionId,
190
+ ingest(text, { onFact = null, optimistic = false } = {}) {
191
+ return groundTextToFacts(text, { memoryDir, sessionId, lexicon, vocabHint, onFact, optimistic });
192
+ },
193
+ };
194
+ }
195
+
196
+ /**
197
+ * The session's whole triple store as JSONL — the same
198
+ * { subject, predicate, object, provenance } shape `tmct extract` and
199
+ * `tmct memory --export` emit, offered to the page as the canonical download.
200
+ */
201
+ export async function exportFactsJsonl(memoryDir) {
202
+ return serializeFactsJsonl(await loadMemory(memoryDir));
203
+ }
204
+
205
+ globalThis.tmctIngest = {
206
+ createIngestSession, groundTextToFacts, exportFactsJsonl, registerWinkModel, normFactTerm,
207
+ memoryStats, openPersistedStore,
208
+ };
@@ -17,10 +17,13 @@
17
17
  // never published — only the hosted demo site's public/ledger.html links to
18
18
  // it, as an optional sibling script the page degrades honestly without.
19
19
  import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
20
- import { createInMemoryStore, normFactTerm } from "../../adapters/memory/core.mjs";
20
+ import { createInMemoryStore, normFactTerm, loadMemory } from "../../adapters/memory/core.mjs";
21
+ import { serializeFactsJsonl } from "../../adapters/memory/export-jsonl.mjs";
22
+ import { splitSentencesPreservingPaths } from "../../services/sentences.mjs";
21
23
  import { parseEntities } from "../../domain/codegraph.mjs";
22
24
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
23
25
  import { registerWinkModel } from "../../adapters/wink-model.mjs";
26
+ import { registerResearchProvider } from "../../adapters/corpus/wikipedia-live.mjs";
24
27
  import { computeLedgerDataFromPayload } from "../../services/ledger-viz.mjs";
25
28
 
26
29
  /**
@@ -52,6 +55,7 @@ export function createLedgerSession({ seedPayload = null, vocabSeeded = false }
52
55
  let focus = null;
53
56
  let last = null;
54
57
  let planState = null;
58
+ let researchState = null;
55
59
 
56
60
  return {
57
61
  memoryDir,
@@ -64,22 +68,37 @@ export function createLedgerSession({ seedPayload = null, vocabSeeded = false }
64
68
  try {
65
69
  result = await runTurn(line, {
66
70
  config: null, source: null, graph, focus, last, memoryDir, sessionId,
67
- env: {}, lexicon, vocabHint, planState,
71
+ env: {}, lexicon, vocabHint, planState, researchState,
68
72
  });
69
73
  } catch (e) {
70
74
  const message = e instanceof Error ? e.message : String(e);
71
- return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null };
75
+ return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, research: null };
72
76
  }
73
77
  focus = result.focus;
74
78
  last = result.last;
75
79
  if ("planState" in result) planState = result.planState;
76
- return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null };
80
+ if ("researchState" in result) researchState = result.researchState;
81
+ // `research` distinguishes a queue snapshot (research turn) from null
82
+ // (a research turn that ended the run) from undefined (not a research
83
+ // turn) — the dock's controls only react to the first two.
84
+ return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, research: result.research };
77
85
  },
78
86
  };
79
87
  }
80
88
 
89
+ /** The session's whole triple store as JSONL — the same
90
+ * { subject, predicate, object, provenance } shape `tmct extract` and
91
+ * `tmct memory --export` emit. The ledger dock's "export facts" control reads
92
+ * this and offers it as a download. */
93
+ export async function exportFactsJsonl(memoryDir) {
94
+ return serializeFactsJsonl(await loadMemory(memoryDir));
95
+ }
96
+
81
97
  // Re-exported so ledger-viz.mjs's own inline script never has to duplicate
82
98
  // the derivation logic that builds rows/terms/edges/contradictions/
83
99
  // worthALook/stats from a payload — the same posture chat-browser-entry.mjs
84
100
  // takes re-exporting registerWinkModel for its own page's CDN wink load.
85
- globalThis.tmctLedger = { createLedgerSession, computeLedgerDataFromPayload, normFactTerm, registerWinkModel };
101
+ // splitSentences + exportFactsJsonl carry the dock's paste-and-drop ingest and
102
+ // its JSONL export across the bundle boundary, the same one-serializer posture
103
+ // chat-browser-entry.mjs holds for its own page.
104
+ globalThis.tmctLedger = { createLedgerSession, computeLedgerDataFromPayload, normFactTerm, registerWinkModel, registerResearchProvider, splitSentences: splitSentencesPreservingPaths, exportFactsJsonl };