@polycode-projects/the-mechanical-code-talker 2.10.5 → 2.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +2 -2
  2. package/corpus/sprites/src/sprite-facts.jsonl +18 -0
  3. package/corpus/worlds/manifest.json +5 -5
  4. package/corpus/worlds/shards/ashcombe-hall.jsonl.gz +0 -0
  5. package/corpus/worlds/src/ashcombe-hall.jsonl +27 -0
  6. package/data/sprites/book-icon.toml +12 -0
  7. package/data/sprites/cellar-icon.toml +12 -0
  8. package/data/sprites/drawing-room-icon.toml +13 -0
  9. package/data/sprites/garden-icon.toml +12 -0
  10. package/data/sprites/kitchen-icon.toml +13 -0
  11. package/data/sprites/library-icon.toml +12 -0
  12. package/data/sprites/pan-icon.toml +11 -0
  13. package/data/sprites/study-icon.toml +12 -0
  14. package/data/templates/responses.jsonl +1 -0
  15. package/package.json +5 -2
  16. package/src/adapters/corpus/wikipedia-live.mjs +182 -26
  17. package/src/adapters/corpus/worlds-pack.mjs +8 -2
  18. package/src/adapters/toml-config.mjs +6 -0
  19. package/src/domain/ask-vocab.mjs +17 -0
  20. package/src/domain/ask.mjs +51 -1
  21. package/src/domain/grammar/ace.mjs +43 -3
  22. package/src/domain/interpret/normalize.mjs +6 -2
  23. package/src/domain/interpret/strategies/grammar.mjs +47 -17
  24. package/src/domain/interpret/strategies/keywords.mjs +53 -4
  25. package/src/domain/memory/trust.mjs +11 -0
  26. package/src/domain/router/registry.mjs +8 -1
  27. package/src/domain/worlds-pack.mjs +50 -0
  28. package/src/services/adventure-autoplay.mjs +5 -2
  29. package/src/services/adventure-viz.mjs +301 -33
  30. package/src/services/adventure.mjs +162 -14
  31. package/src/services/chat-page-viz.mjs +265 -189
  32. package/src/services/chat-session.mjs +15 -5
  33. package/src/services/chat.mjs +471 -79
  34. package/src/services/code-explorer-viz.mjs +183 -75
  35. package/src/services/extract-facts.mjs +118 -28
  36. package/src/services/ingest-viz.mjs +328 -79
  37. package/src/services/ledger-viz.mjs +99 -0
  38. package/src/services/memory-panel-viz.mjs +159 -0
  39. package/src/services/research.mjs +266 -0
  40. package/src/services/sentences.mjs +19 -0
  41. package/src/services/spider-fly-viz.mjs +21 -5
  42. package/src/surfaces/web/adventure-browser-entry.mjs +9 -5
  43. package/src/surfaces/web/chat-browser-entry.mjs +28 -11
  44. package/src/surfaces/web/code-explorer-browser-entry.mjs +27 -11
  45. package/src/surfaces/web/ingest-browser-entry.mjs +123 -41
  46. package/src/surfaces/web/ledger-browser-entry.mjs +10 -4
  47. package/src/surfaces/web/memory-ask-browser.bundle.js +116 -116
  48. package/src/surfaces/web/memory-stats.mjs +53 -0
@@ -35,7 +35,7 @@ import {
35
35
  } from "../../adapters/memory/core.mjs";
36
36
  import { parseEntities } from "../../domain/codegraph.mjs";
37
37
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
38
- import { foldWorldState, worldDigestRows, roomAffordances } from "../../services/adventure.mjs";
38
+ import { foldWorldState, worldDigestRows, roomAffordances, worldActionRows } from "../../services/adventure.mjs";
39
39
  import { runAdventureAutoplayTick, exposedFacts } from "../../services/adventure-autoplay.mjs";
40
40
  import { parseWorldEditorText, planWorldEditorSync } from "../../services/adventure-editor.mjs";
41
41
  import { resolveSpriteForClass, SPRITE_REGISTRY, classAncestorChain } from "../../domain/sprite-map.mjs";
@@ -92,7 +92,7 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
92
92
  // bookkeeping either way.
93
93
  let visitedRoomIds = new Set(Array.isArray(restoredVisitedRoomIds) ? restoredVisitedRoomIds : []);
94
94
  const openingRows = readFactRows(await loadMemory(memoryDir));
95
- const openingHere = foldWorldState(openingRows).placements.get("player")?.object ?? null;
95
+ const openingHere = foldWorldState(worldActionRows(openingRows)).placements.get("player")?.object ?? null;
96
96
  if (openingHere) visitedRoomIds.add(openingHere);
97
97
 
98
98
  const graph = parseEntities({ individuals: [], objectProperties: [] });
@@ -126,7 +126,9 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
126
126
  try {
127
127
  result = await runTurn(line, {
128
128
  config: null, source: null, graph, focus, last, memoryDir, sessionId,
129
- env: {}, lexicon, vocabHint: "", planState: planHolder.state,
129
+ env: {}, lexicon, uiContext: "browser",
130
+ vocabHint: 'Try a world question ("where is the key"), or teach me: "remember: the moat is a ditch".',
131
+ planState: planHolder.state,
130
132
  });
131
133
  } catch (e) {
132
134
  const message = e instanceof Error ? e.message : String(e);
@@ -135,7 +137,7 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
135
137
  focus = result.focus;
136
138
  last = result.last;
137
139
  if ("planState" in result) planHolder.state = result.planState;
138
- const here = foldWorldState(readFactRows(await loadMemory(memoryDir))).placements.get("player")?.object ?? null;
140
+ const here = foldWorldState(worldActionRows(readFactRows(await loadMemory(memoryDir)))).placements.get("player")?.object ?? null;
139
141
  if (here) visitedRoomIds.add(here);
140
142
  return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null };
141
143
  },
@@ -148,7 +150,9 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
148
150
  * reference into this closure's own Set. */
149
151
  async snapshot() {
150
152
  const rows = readFactRows(await loadMemory(memoryDir));
151
- const state = foldWorldState(rows);
153
+ // State the page plays and maps from is the game world only; the raw
154
+ // rows still travel for the digest's background colour.
155
+ const state = foldWorldState(worldActionRows(rows));
152
156
  const here = state.placements.get("player")?.object ?? null;
153
157
  return { rows, state, here, turn: state.turnCount, visitedRoomIds: [...visitedRoomIds] };
154
158
  },
@@ -33,7 +33,9 @@ import { registerReferencePackProvider } from "../../adapters/corpus/reference-p
33
33
  // The LIVE Wikipedia seam (opt-in, default off): the page's toggle enables it
34
34
  // per session, and e2e tests stub the provider the same way the pack's own
35
35
  // provider is stubbed. The adapter is fetch-only, so it bundles as-is.
36
- import { registerLiveReferenceProvider } from "../../adapters/corpus/wikipedia-live.mjs";
36
+ // registerResearchProvider is the research lane's sibling seam
37
+ // (simple.wikipedia.org) — same stubbing contract for its e2e tests.
38
+ import { registerLiveReferenceProvider, registerResearchProvider } from "../../adapters/corpus/wikipedia-live.mjs";
37
39
  // Best-effort IndexedDB persistence for the page's session store — the page
38
40
  // decides when to save/load/clear; this entry only carries the wrapper
39
41
  // across the bundle boundary.
@@ -52,7 +54,7 @@ import { openPersistedStore } from "./idb-persist.mjs";
52
54
  * { answer, end, record, plan } and threads focus/last/planState between
53
55
  * calls exactly as the CLI session does.
54
56
  */
55
- export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null } = {}) {
57
+ export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null, synthesisBudget = 12 } = {}) {
56
58
  const memoryDir = createInMemoryStore();
57
59
  // Spread onto the store's own empty payload so a partial seed (individuals
58
60
  // and objectProperties only) still carries the classes/prefixes scaffolding
@@ -69,18 +71,28 @@ export function createChatSession({ seedPayload = null, vocabSeeded = false, liv
69
71
  let focus = null;
70
72
  let last = null;
71
73
  let planState = null;
72
- // Tri-state, like the TUI: false (off), true (rescue on a miss), or
73
- // "supplement" (also append a cited read-out under every grounded answer).
74
- const normLive = (v) => (v === "supplement" ? "supplement" : Boolean(v));
74
+ let researchState = null;
75
+ // Four-state, like the CLI: false (off), true (rescue on a miss),
76
+ // "supplement" (also append a cited read-out under every grounded vocabulary
77
+ // answer), or "always" (widen that to every grounded answer). The two string
78
+ // modes stay strings so runTurn reads them as the supplement/always lanes.
79
+ const normLive = (v) => (v === "always" ? "always" : v === "supplement" ? "supplement" : Boolean(v));
75
80
  let liveReferenceOn = normLive(liveReference);
81
+ // The auto-synthesis budget for this session's learn-on-miss loads — the
82
+ // page's slider sets it; 0 stores article facts without any entailed rows.
83
+ let synthesisBudgetOn = Number.isFinite(synthesisBudget) ? synthesisBudget : 12;
76
84
 
77
85
  return {
78
86
  memoryDir,
79
87
  sessionId,
80
88
  get liveReference() { return liveReferenceOn; },
81
89
  /** The page's toggle seam: set the live Wikipedia mode for every later turn
82
- * (the `/wiki on|off|supplement` command sets the same state). */
90
+ * (the `/wiki on|off|supplement|always` command sets the same state). */
83
91
  setLiveReference(v) { liveReferenceOn = normLive(v); },
92
+ get synthesisBudget() { return synthesisBudgetOn; },
93
+ /** The page's slider seam: set the auto-synthesis budget for every later
94
+ * learn-on-miss load. Clamped to a non-negative integer; 0 disables it. */
95
+ setSynthesisBudget(n) { synthesisBudgetOn = Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; },
84
96
 
85
97
  /** One dispatched turn. A throwing runTurn must never kill the session —
86
98
  * the page has no other chance to show this turn's answer. */
@@ -89,18 +101,23 @@ export function createChatSession({ seedPayload = null, vocabSeeded = false, liv
89
101
  try {
90
102
  result = await runTurn(line, {
91
103
  config: null, source: null, graph, focus, last, memoryDir, sessionId,
92
- env: {}, lexicon, vocabHint, planState,
104
+ env: {}, lexicon, vocabHint, planState, researchState,
93
105
  liveReference: liveReferenceOn, onLiveLookup,
106
+ uiContext: "browser", synthesisBudget: synthesisBudgetOn,
94
107
  });
95
108
  } catch (e) {
96
109
  const message = e instanceof Error ? e.message : String(e);
97
- return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, plan: null };
110
+ return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, plan: null, research: null };
98
111
  }
99
112
  focus = result.focus;
100
113
  last = result.last;
101
114
  if ("planState" in result) planState = result.planState;
102
- if (typeof result.liveReference === "boolean" || result.liveReference === "supplement") liveReferenceOn = result.liveReference;
103
- return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null };
115
+ if ("researchState" in result) researchState = result.researchState;
116
+ if (typeof result.liveReference === "boolean" || result.liveReference === "supplement" || result.liveReference === "always") liveReferenceOn = result.liveReference;
117
+ // `research` distinguishes three cases on purpose: a queue snapshot
118
+ // (research turn), null (a research turn that ended the run), and
119
+ // undefined (not a research turn — the page leaves its controls alone).
120
+ return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null, research: result.research };
104
121
  },
105
122
  };
106
123
  }
@@ -162,4 +179,4 @@ export async function exportFactsJsonl(memoryDir) {
162
179
  return serializeFactsJsonl(await loadMemory(memoryDir));
163
180
  }
164
181
 
165
- globalThis.tmctChat = { createChatSession, registerWinkModel, registerReferencePackProvider, registerLiveReferenceProvider, normFactTerm, vocabExampleHint, memoryStats, openPersistedStore, exportFactsJsonl, splitSentences: splitSentencesPreservingPaths };
182
+ globalThis.tmctChat = { createChatSession, registerWinkModel, registerReferencePackProvider, registerLiveReferenceProvider, registerResearchProvider, normFactTerm, vocabExampleHint, memoryStats, openPersistedStore, exportFactsJsonl, splitSentences: splitSentencesPreservingPaths };
@@ -1,17 +1,19 @@
1
1
  // code-explorer-browser-entry.mjs — the esbuild entry for the code explorer's
2
- // LIVE chat dock (public/code-explorer.bundle.js / electron/renderer/
2
+ // LIVE chat (public/code-explorer.bundle.js / electron/renderer/
3
3
  // code-explorer.bundle.js). It mirrors ledger-browser-entry.mjs, but seeds the
4
- // full runTurn engine from a CODE graph instead of a memory payload: the dock
5
- // answers "what does X import" / "which functions call Y" over the loaded
6
- // graph, the same compositional shapes the hint rail suggests.
4
+ // full runTurn engine from a CODE graph and, when the page hands one over,
5
+ // the same general-knowledge seed payload chat.html boots from so one
6
+ // session answers "which functions call Y" over the loaded graph and "what is
7
+ // a queue" over the seeded memory.
7
8
  //
8
9
  // The graph enters through source.mjs's provider seam — the same seam the CLI
9
10
  // and HTTP surfaces read — so runTurn's symbol-grain lanes see the whole
10
11
  // payload, while parseEntities builds the coarse graph the flat lanes read.
12
+ // The seed enters the in-memory store the same way createChatSession's does.
11
13
  // Gitignored, built fresh by scripts/build-code-explorer-bundle.mjs; the page
12
14
  // degrades to a static view when it is absent (renderCodeExplorerHtml's own
13
15
  // contract), so nothing here is ever published.
14
- import { runTurn } from "../../services/chat.mjs";
16
+ import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
15
17
  import { createInMemoryStore, normFactTerm } from "../../adapters/memory/core.mjs";
16
18
  import { parseEntities } from "../../domain/codegraph.mjs";
17
19
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
@@ -23,17 +25,30 @@ import { generateCodeHints } from "../../domain/code-explorer-hints.mjs";
23
25
  /**
24
26
  * A browser code-explorer session over the real turn engine. Registers the
25
27
  * loaded payload with source.mjs's provider seam, parses the coarse graph, and
26
- * dispatches every turn through the same runTurn the CLI runs. Teaches land in
27
- * an in-memory store so a taught fact never touches disk. Returns
28
- * { sessionId, turn }, the createChatSession shape the page's dock expects.
28
+ * dispatches every turn through the same runTurn the CLI runs.
29
+ *
30
+ * `seedPayload` (optional) is a serialized memory graph (loadMemory's shape,
31
+ * built by scripts/build-chat-seed.mjs) assigned onto the session's fresh
32
+ * in-memory store, so general-knowledge questions ground alongside the code
33
+ * graph's own lanes. `vocabSeeded` tells the vocabulary hint whether that
34
+ * payload carries the starter vocabulary. Teaches land in the same in-memory
35
+ * store, so a taught fact never touches disk.
36
+ *
37
+ * Returns { memoryDir, sessionId, turn }, the createChatSession shape the
38
+ * page's chat expects.
29
39
  */
30
- export function createCodeExplorerSession({ graphPayload } = {}) {
40
+ export function createCodeExplorerSession({ graphPayload, seedPayload = null, vocabSeeded = false } = {}) {
31
41
  const payload = graphPayload || { individuals: [], objectProperties: [] };
32
42
  source.registerProvider(() => payload);
33
43
 
34
44
  const graph = parseEntities(payload);
35
45
  const memoryDir = createInMemoryStore();
46
+ // Spread onto the store's own empty payload so a partial seed (individuals
47
+ // and objectProperties only) still carries the classes/prefixes scaffolding
48
+ // the write path recounts — teach turns must work on any seed.
49
+ if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
36
50
  const lexicon = loadLexicon();
51
+ const vocabHint = vocabExampleHint(vocabSeeded);
37
52
  const sessionId = globalThis.crypto?.randomUUID?.() ?? String(Date.now());
38
53
  // A stable virtual path: the provider answers every fetch, so no file is
39
54
  // ever read, but the code lanes still do path math (join/dirname) on it.
@@ -44,13 +59,14 @@ export function createCodeExplorerSession({ graphPayload } = {}) {
44
59
  let planState = null;
45
60
 
46
61
  return {
62
+ memoryDir,
47
63
  sessionId,
48
64
  async turn(line) {
49
65
  let result;
50
66
  try {
51
67
  result = await runTurn(line, {
52
68
  config, source, graph, focus, last, memoryDir, sessionId,
53
- env: {}, lexicon, vocabHint: "", planState,
69
+ env: {}, lexicon, vocabHint, planState,
54
70
  });
55
71
  } catch (e) {
56
72
  const message = e instanceof Error ? e.message : String(e);
@@ -66,7 +82,7 @@ export function createCodeExplorerSession({ graphPayload } = {}) {
66
82
 
67
83
  // Exposed for the page's inline client: the re-derivation helpers so a graph
68
84
  // swapped through the desktop picker re-renders without duplicating logic, plus
69
- // the wink loader hook registerWinkModel and normFactTerm the dock shares with
85
+ // the wink loader hook registerWinkModel and normFactTerm the chat shares with
70
86
  // the ledger page.
71
87
  globalThis.tmctCodeExplorer = {
72
88
  createCodeExplorerSession,
@@ -12,7 +12,11 @@
12
12
  // The recognizer is reached through ONE seam — `groundTextToFacts` — so a
13
13
  // wider ingest tier (an optimistic fuzzy-match pass, a canonical/graph-linked
14
14
  // output) slots in behind this single function without the page changing.
15
- // Today the seam is the strict recognizer alone.
15
+ // Alongside the strict recognizer, groundTextToFacts runs the same
16
+ // citation-stripping, clause-fallback and bounded pronoun-carry passes
17
+ // `ingestText` (extract-facts.mjs) applies for the CLI, and — opt-in, off by
18
+ // default — the same low-trust optimistic tier, so the browser page and the
19
+ // CLI ground the identical class of sentence.
16
20
  //
17
21
  // Gitignored, Pages-demo-site-only output (scripts/build-demo-site.mjs builds
18
22
  // it fresh on every deploy, never committed) — the same posture
@@ -20,21 +24,37 @@
20
24
  // runTurn engine, the same weight class as the chat/ledger bundles, and is
21
25
  // never published.
22
26
  import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
23
- import { createInMemoryStore, normFactTerm, loadMemory, readFactRows } from "../../adapters/memory/core.mjs";
27
+ import { createInMemoryStore, normFactTerm, loadMemory, readFactRows, appendFact } from "../../adapters/memory/core.mjs";
24
28
  import { serializeFactsJsonl } from "../../adapters/memory/export-jsonl.mjs";
25
- import { splitSentencesPreservingPaths } from "../../services/sentences.mjs";
29
+ import { splitSentencesPreservingPaths, stripCitationResidue } from "../../services/sentences.mjs";
30
+ import { clauseCandidates, optimisticTriples } from "../../services/extract-facts.mjs";
26
31
  import { touchedFactRows } from "../../domain/memory/touched-facts.mjs";
27
- import { parseEntities } from "../../domain/codegraph.mjs";
28
32
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
29
- import { registerWinkModel } from "../../adapters/wink-model.mjs";
33
+ import { registerWinkModel, winkInstance } from "../../adapters/wink-model.mjs";
34
+ import { memoryStats } from "./memory-stats.mjs";
35
+ import { openPersistedStore } from "./idb-persist.mjs";
36
+
37
+ // The pronoun subjects a bounded carry substitutes with the last unique
38
+ // grounded subject in the SAME paragraph. Reset at every blank line, so a
39
+ // fresh paragraph never resolves against a stale antecedent.
40
+ const PRONOUN_LEAD_RE = /^(?:they|it|these|those|this)\b\s*/i;
30
41
 
31
42
  /**
32
- * The single recognizer seam. Splits `text` into sentences (wink's own
33
- * boundary detection, never a regex), runs each through runTurn against
34
- * `memoryDir`, and keeps a sentence only when runTurn's own record calls it a
35
- * stored assertion (`record.via === "assert"`, `record.miss` false) that
36
- * actually touched a Fact row the identical keep/skip rule
37
- * src/services/extract-facts.mjs's own runSentence holds.
43
+ * The single recognizer seam. Splits `text` into paragraphs (blank-line
44
+ * separated, so the pronoun carry never bridges a topic break) and each
45
+ * paragraph into sentences (wink's own boundary detection, never a regex).
46
+ * Each sentence's citation residue ("[3]", "[citation needed]") is stripped
47
+ * before grounding; the strict recognizer then tries the whole sentence
48
+ * first, falling back to its clause fragments only on a miss
49
+ * (`clauseCandidates`), and — still on a miss, only for a pronoun-led
50
+ * sentence with a unique adjacent antecedent in this paragraph — one retry
51
+ * with that antecedent substituted in.
52
+ *
53
+ * A sentence still ungrounded after all of that runs the optimistic fuzzy
54
+ * tier when `optimistic` is set (`optimisticTriples`): a copula or known
55
+ * relation verb flanked by two resolvable entities, stored under the
56
+ * low-trust `optimistic-extract:page` tag rather than a teach/assert
57
+ * provenance, so a fuzzy candidate can never corroborate a curated fact.
38
58
  *
39
59
  * `onFact(fact)` (optional) is awaited after each grounded row so the page can
40
60
  * render the canonical facts LIVE as they land, one at a time.
@@ -42,48 +62,107 @@ import { registerWinkModel } from "../../adapters/wink-model.mjs";
42
62
  * Returns { sentences, recognized, skipped, facts } — `facts` an array of
43
63
  * { subject, predicate, object, provenance, quantifier, sentence } in the same
44
64
  * canonical shape `tmct extract` and the JSONL exporter emit.
45
- *
46
- * A wider ingest tier plugs in HERE, behind this one function: the page calls
47
- * it and nothing else.
48
65
  */
49
- export async function groundTextToFacts(text, { memoryDir, sessionId, graph, lexicon, vocabHint, onFact = null } = {}) {
50
- const sentences = splitSentencesPreservingPaths(text);
66
+ export async function groundTextToFacts(text, { memoryDir, sessionId, lexicon, vocabHint, optimistic = false, onFact = null } = {}) {
67
+ const paragraphs = String(text ?? "").split(/\n[ \t]*\n/);
51
68
  const facts = [];
52
69
  let focus = null;
53
70
  let last = null;
54
71
  let planState = null;
72
+ let sentenceCount = 0;
55
73
  let recognized = 0;
74
+ // Reused, not reloaded: readFactRows(loadMemory(...)) is O(rows), so a
75
+ // fresh read on every sentence over a large seeded store is exactly the
76
+ // O(sentences × rows) cost this page must avoid. The snapshot only moves
77
+ // forward past a sentence that actually asserted something.
78
+ let beforeRows = readFactRows(await loadMemory(memoryDir));
79
+ const nlp = optimistic ? winkInstance() : null;
56
80
 
57
- for (const sentence of sentences) {
58
- const before = readFactRows(await loadMemory(memoryDir));
81
+ // One grounding attempt against `memoryDir`: runs `form` through runTurn,
82
+ // threading focus/last/planState the same way whether or not it grounds.
83
+ // No `graph` is passed — this page never offers a code graph to link
84
+ // against, and a non-null one (even an empty one) makes runTurn read an
85
+ // ordinary "the number of X" phrase as a graph count query instead of
86
+ // running the teach cascade, exactly the phrasing the Sales-paragraph
87
+ // sentence this pipeline is built to ground uses.
88
+ // Returns the Fact rows this attempt actually touched, or null — a miss, a
89
+ // non-assert turn, or an assert that touched no Fact row (a Rule teach).
90
+ async function attempt(form) {
59
91
  let record;
60
92
  try {
61
- const result = await runTurn(sentence, {
62
- config: null, source: null, graph, focus, last, memoryDir, sessionId,
63
- env: {}, lexicon, vocabHint, planState,
93
+ const result = await runTurn(form, {
94
+ config: null, source: null, focus, last, memoryDir, sessionId,
95
+ env: {}, lexicon, vocabHint, planState, uiContext: "browser",
64
96
  });
65
97
  focus = result.focus;
66
98
  last = result.last;
67
99
  if ("planState" in result) planState = result.planState;
68
100
  record = result.record;
69
101
  } catch {
70
- continue; // a throwing sentence is a skip, never a page-killer
102
+ return null; // a throwing sentence is a skip, never a page-killer
71
103
  }
72
- if (record?.via !== "assert" || record?.miss) continue;
73
- const rows = touchedFactRows(before, readFactRows(await loadMemory(memoryDir)));
74
- if (!rows.length) continue; // a Rule teach touches no Fact row honest skip
75
- recognized += 1;
76
- for (const row of rows) {
77
- const fact = {
78
- subject: row.subject, predicate: row.predicate, object: row.object,
79
- provenance: row.provenance || "", quantifier: row.quantifier || "", sentence,
80
- };
81
- facts.push(fact);
82
- if (onFact) await onFact(fact);
104
+ if (record?.via !== "assert" || record?.miss) return null;
105
+ // The one loadMemory/readFactRows call this attempt pays for, only
106
+ // because it actually asserted something. Equal lengths mean nothing was
107
+ // ADDED — a re-assertion's provenance-only change is the one case this
108
+ // fast path can't see, an accepted trade against paying the full
109
+ // before/after diff on every recognized sentence in a large store.
110
+ const afterRows = readFactRows(await loadMemory(memoryDir));
111
+ const rows = afterRows.length === beforeRows.length ? [] : touchedFactRows(beforeRows, afterRows);
112
+ beforeRows = afterRows;
113
+ return rows.length ? rows : null;
114
+ }
115
+
116
+ for (const paragraph of paragraphs) {
117
+ // The last unique grounded subject in THIS paragraph, carried onto a
118
+ // later pronoun-led sentence the strict recognizer couldn't ground on its
119
+ // own. Cleared at the paragraph boundary.
120
+ let carrySubject = null;
121
+ for (const rawSentence of splitSentencesPreservingPaths(paragraph)) {
122
+ sentenceCount += 1;
123
+ const cleaned = stripCitationResidue(rawSentence);
124
+
125
+ let rows = null;
126
+ for (const candidate of clauseCandidates(cleaned, { nlp })) {
127
+ rows = await attempt(candidate);
128
+ if (rows) break;
129
+ }
130
+ if (!rows && carrySubject && PRONOUN_LEAD_RE.test(cleaned)) {
131
+ rows = await attempt(cleaned.replace(PRONOUN_LEAD_RE, `${carrySubject} `));
132
+ }
133
+
134
+ if (rows) {
135
+ recognized += 1;
136
+ const subjects = new Set(rows.map((r) => r.subject));
137
+ if (subjects.size === 1) carrySubject = [...subjects][0];
138
+ for (const row of rows) {
139
+ const fact = {
140
+ subject: row.subject, predicate: row.predicate, object: row.object,
141
+ provenance: row.provenance || "", quantifier: row.quantifier || "", sentence: rawSentence,
142
+ };
143
+ facts.push(fact);
144
+ if (onFact) await onFact(fact);
145
+ }
146
+ continue;
147
+ }
148
+
149
+ if (!optimistic) continue;
150
+ const candidates = optimisticTriples(cleaned, { lexicon, nlp });
151
+ if (!candidates.length) continue;
152
+ recognized += 1;
153
+ for (const t of candidates) {
154
+ await appendFact(memoryDir, { subject: t.subject, predicate: t.predicate, object: t.object, provenance: "optimistic-extract:page" });
155
+ const fact = {
156
+ subject: t.subject, predicate: t.predicate, object: t.object,
157
+ provenance: "optimistic-extract:page", quantifier: "", sentence: rawSentence,
158
+ };
159
+ facts.push(fact);
160
+ if (onFact) await onFact(fact);
161
+ }
83
162
  }
84
163
  }
85
164
 
86
- return { sentences: sentences.length, recognized, skipped: sentences.length - recognized, facts };
165
+ return { sentences: sentenceCount, recognized, skipped: sentenceCount - recognized, facts };
87
166
  }
88
167
 
89
168
  /**
@@ -92,15 +171,15 @@ export async function groundTextToFacts(text, { memoryDir, sessionId, graph, lex
92
171
  * the page can then export. `seedPayload` (optional) pre-loads a graph the
93
172
  * recognizer can recall and link against.
94
173
  *
95
- * Returns { memoryDir, sessionId, ingest }. `ingest(text, { onFact })` is the
96
- * one call the page makes; it drives groundTextToFacts against this session's
97
- * store and returns its { sentences, recognized, skipped, facts } summary.
174
+ * Returns { memoryDir, sessionId, ingest }. `ingest(text, { onFact,
175
+ * optimistic })` is the one call the page makes; it drives groundTextToFacts
176
+ * against this session's store and returns its { sentences, recognized,
177
+ * skipped, facts } summary.
98
178
  */
99
179
  export function createIngestSession({ seedPayload = null, vocabSeeded = false } = {}) {
100
180
  const memoryDir = createInMemoryStore();
101
181
  if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
102
182
 
103
- const graph = parseEntities({ individuals: [], objectProperties: [] });
104
183
  const lexicon = loadLexicon();
105
184
  const vocabHint = vocabExampleHint(vocabSeeded);
106
185
  const sessionId = globalThis.crypto?.randomUUID?.() ?? String(Date.now());
@@ -108,8 +187,8 @@ export function createIngestSession({ seedPayload = null, vocabSeeded = false }
108
187
  return {
109
188
  memoryDir,
110
189
  sessionId,
111
- ingest(text, { onFact = null } = {}) {
112
- return groundTextToFacts(text, { memoryDir, sessionId, graph, lexicon, vocabHint, onFact });
190
+ ingest(text, { onFact = null, optimistic = false } = {}) {
191
+ return groundTextToFacts(text, { memoryDir, sessionId, lexicon, vocabHint, onFact, optimistic });
113
192
  },
114
193
  };
115
194
  }
@@ -123,4 +202,7 @@ export async function exportFactsJsonl(memoryDir) {
123
202
  return serializeFactsJsonl(await loadMemory(memoryDir));
124
203
  }
125
204
 
126
- globalThis.tmctIngest = { createIngestSession, groundTextToFacts, exportFactsJsonl, registerWinkModel, normFactTerm };
205
+ globalThis.tmctIngest = {
206
+ createIngestSession, groundTextToFacts, exportFactsJsonl, registerWinkModel, normFactTerm,
207
+ memoryStats, openPersistedStore,
208
+ };
@@ -23,6 +23,7 @@ import { splitSentencesPreservingPaths } from "../../services/sentences.mjs";
23
23
  import { parseEntities } from "../../domain/codegraph.mjs";
24
24
  import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
25
25
  import { registerWinkModel } from "../../adapters/wink-model.mjs";
26
+ import { registerResearchProvider } from "../../adapters/corpus/wikipedia-live.mjs";
26
27
  import { computeLedgerDataFromPayload } from "../../services/ledger-viz.mjs";
27
28
 
28
29
  /**
@@ -54,6 +55,7 @@ export function createLedgerSession({ seedPayload = null, vocabSeeded = false }
54
55
  let focus = null;
55
56
  let last = null;
56
57
  let planState = null;
58
+ let researchState = null;
57
59
 
58
60
  return {
59
61
  memoryDir,
@@ -66,16 +68,20 @@ export function createLedgerSession({ seedPayload = null, vocabSeeded = false }
66
68
  try {
67
69
  result = await runTurn(line, {
68
70
  config: null, source: null, graph, focus, last, memoryDir, sessionId,
69
- env: {}, lexicon, vocabHint, planState,
71
+ env: {}, lexicon, vocabHint, planState, researchState,
70
72
  });
71
73
  } catch (e) {
72
74
  const message = e instanceof Error ? e.message : String(e);
73
- return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null };
75
+ return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, research: null };
74
76
  }
75
77
  focus = result.focus;
76
78
  last = result.last;
77
79
  if ("planState" in result) planState = result.planState;
78
- return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null };
80
+ if ("researchState" in result) researchState = result.researchState;
81
+ // `research` distinguishes a queue snapshot (research turn) from null
82
+ // (a research turn that ended the run) from undefined (not a research
83
+ // turn) — the dock's controls only react to the first two.
84
+ return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, research: result.research };
79
85
  },
80
86
  };
81
87
  }
@@ -95,4 +101,4 @@ export async function exportFactsJsonl(memoryDir) {
95
101
  // splitSentences + exportFactsJsonl carry the dock's paste-and-drop ingest and
96
102
  // its JSONL export across the bundle boundary, the same one-serializer posture
97
103
  // chat-browser-entry.mjs holds for its own page.
98
- globalThis.tmctLedger = { createLedgerSession, computeLedgerDataFromPayload, normFactTerm, registerWinkModel, splitSentences: splitSentencesPreservingPaths, exportFactsJsonl };
104
+ globalThis.tmctLedger = { createLedgerSession, computeLedgerDataFromPayload, normFactTerm, registerWinkModel, registerResearchProvider, splitSentences: splitSentencesPreservingPaths, exportFactsJsonl };