@polycode-projects/the-mechanical-code-talker 2.10.5 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/corpus/sprites/src/sprite-facts.jsonl +18 -0
- package/corpus/worlds/manifest.json +5 -5
- package/corpus/worlds/shards/ashcombe-hall.jsonl.gz +0 -0
- package/corpus/worlds/src/ashcombe-hall.jsonl +27 -0
- package/data/sprites/book-icon.toml +12 -0
- package/data/sprites/cellar-icon.toml +12 -0
- package/data/sprites/drawing-room-icon.toml +13 -0
- package/data/sprites/garden-icon.toml +12 -0
- package/data/sprites/kitchen-icon.toml +13 -0
- package/data/sprites/library-icon.toml +12 -0
- package/data/sprites/pan-icon.toml +11 -0
- package/data/sprites/study-icon.toml +12 -0
- package/data/templates/responses.jsonl +1 -0
- package/package.json +5 -2
- package/src/adapters/corpus/wikipedia-live.mjs +182 -26
- package/src/adapters/corpus/worlds-pack.mjs +8 -2
- package/src/adapters/toml-config.mjs +6 -0
- package/src/domain/ask-vocab.mjs +17 -0
- package/src/domain/ask.mjs +51 -1
- package/src/domain/grammar/ace.mjs +43 -3
- package/src/domain/interpret/normalize.mjs +6 -2
- package/src/domain/interpret/strategies/grammar.mjs +47 -17
- package/src/domain/interpret/strategies/keywords.mjs +53 -4
- package/src/domain/memory/trust.mjs +11 -0
- package/src/domain/router/registry.mjs +8 -1
- package/src/domain/worlds-pack.mjs +50 -0
- package/src/services/adventure-autoplay.mjs +5 -2
- package/src/services/adventure-viz.mjs +301 -33
- package/src/services/adventure.mjs +162 -14
- package/src/services/chat-page-viz.mjs +265 -189
- package/src/services/chat-session.mjs +15 -5
- package/src/services/chat.mjs +471 -79
- package/src/services/code-explorer-viz.mjs +183 -75
- package/src/services/extract-facts.mjs +118 -28
- package/src/services/ingest-viz.mjs +328 -79
- package/src/services/ledger-viz.mjs +99 -0
- package/src/services/memory-panel-viz.mjs +159 -0
- package/src/services/research.mjs +266 -0
- package/src/services/sentences.mjs +19 -0
- package/src/services/spider-fly-viz.mjs +21 -5
- package/src/surfaces/web/adventure-browser-entry.mjs +9 -5
- package/src/surfaces/web/chat-browser-entry.mjs +28 -11
- package/src/surfaces/web/code-explorer-browser-entry.mjs +27 -11
- package/src/surfaces/web/ingest-browser-entry.mjs +123 -41
- package/src/surfaces/web/ledger-browser-entry.mjs +10 -4
- package/src/surfaces/web/memory-ask-browser.bundle.js +116 -116
- package/src/surfaces/web/memory-stats.mjs +53 -0
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
} from "../../adapters/memory/core.mjs";
|
|
36
36
|
import { parseEntities } from "../../domain/codegraph.mjs";
|
|
37
37
|
import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
|
|
38
|
-
import { foldWorldState, worldDigestRows, roomAffordances } from "../../services/adventure.mjs";
|
|
38
|
+
import { foldWorldState, worldDigestRows, roomAffordances, worldActionRows } from "../../services/adventure.mjs";
|
|
39
39
|
import { runAdventureAutoplayTick, exposedFacts } from "../../services/adventure-autoplay.mjs";
|
|
40
40
|
import { parseWorldEditorText, planWorldEditorSync } from "../../services/adventure-editor.mjs";
|
|
41
41
|
import { resolveSpriteForClass, SPRITE_REGISTRY, classAncestorChain } from "../../domain/sprite-map.mjs";
|
|
@@ -92,7 +92,7 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
|
|
|
92
92
|
// bookkeeping either way.
|
|
93
93
|
let visitedRoomIds = new Set(Array.isArray(restoredVisitedRoomIds) ? restoredVisitedRoomIds : []);
|
|
94
94
|
const openingRows = readFactRows(await loadMemory(memoryDir));
|
|
95
|
-
const openingHere = foldWorldState(openingRows).placements.get("player")?.object ?? null;
|
|
95
|
+
const openingHere = foldWorldState(worldActionRows(openingRows)).placements.get("player")?.object ?? null;
|
|
96
96
|
if (openingHere) visitedRoomIds.add(openingHere);
|
|
97
97
|
|
|
98
98
|
const graph = parseEntities({ individuals: [], objectProperties: [] });
|
|
@@ -126,7 +126,9 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
|
|
|
126
126
|
try {
|
|
127
127
|
result = await runTurn(line, {
|
|
128
128
|
config: null, source: null, graph, focus, last, memoryDir, sessionId,
|
|
129
|
-
env: {}, lexicon,
|
|
129
|
+
env: {}, lexicon, uiContext: "browser",
|
|
130
|
+
vocabHint: 'Try a world question ("where is the key"), or teach me: "remember: the moat is a ditch".',
|
|
131
|
+
planState: planHolder.state,
|
|
130
132
|
});
|
|
131
133
|
} catch (e) {
|
|
132
134
|
const message = e instanceof Error ? e.message : String(e);
|
|
@@ -135,7 +137,7 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
|
|
|
135
137
|
focus = result.focus;
|
|
136
138
|
last = result.last;
|
|
137
139
|
if ("planState" in result) planHolder.state = result.planState;
|
|
138
|
-
const here = foldWorldState(readFactRows(await loadMemory(memoryDir))).placements.get("player")?.object ?? null;
|
|
140
|
+
const here = foldWorldState(worldActionRows(readFactRows(await loadMemory(memoryDir)))).placements.get("player")?.object ?? null;
|
|
139
141
|
if (here) visitedRoomIds.add(here);
|
|
140
142
|
return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null };
|
|
141
143
|
},
|
|
@@ -148,7 +150,9 @@ export async function createAdventureSession(worldPayload, { restoredPayload = n
|
|
|
148
150
|
* reference into this closure's own Set. */
|
|
149
151
|
async snapshot() {
|
|
150
152
|
const rows = readFactRows(await loadMemory(memoryDir));
|
|
151
|
-
|
|
153
|
+
// State the page plays and maps from is the game world only; the raw
|
|
154
|
+
// rows still travel for the digest's background colour.
|
|
155
|
+
const state = foldWorldState(worldActionRows(rows));
|
|
152
156
|
const here = state.placements.get("player")?.object ?? null;
|
|
153
157
|
return { rows, state, here, turn: state.turnCount, visitedRoomIds: [...visitedRoomIds] };
|
|
154
158
|
},
|
|
@@ -33,7 +33,9 @@ import { registerReferencePackProvider } from "../../adapters/corpus/reference-p
|
|
|
33
33
|
// The LIVE Wikipedia seam (opt-in, default off): the page's toggle enables it
|
|
34
34
|
// per session, and e2e tests stub the provider the same way the pack's own
|
|
35
35
|
// provider is stubbed. The adapter is fetch-only, so it bundles as-is.
|
|
36
|
-
|
|
36
|
+
// registerResearchProvider is the research lane's sibling seam
|
|
37
|
+
// (simple.wikipedia.org) — same stubbing contract for its e2e tests.
|
|
38
|
+
import { registerLiveReferenceProvider, registerResearchProvider } from "../../adapters/corpus/wikipedia-live.mjs";
|
|
37
39
|
// Best-effort IndexedDB persistence for the page's session store — the page
|
|
38
40
|
// decides when to save/load/clear; this entry only carries the wrapper
|
|
39
41
|
// across the bundle boundary.
|
|
@@ -52,7 +54,7 @@ import { openPersistedStore } from "./idb-persist.mjs";
|
|
|
52
54
|
* { answer, end, record, plan } and threads focus/last/planState between
|
|
53
55
|
* calls exactly as the CLI session does.
|
|
54
56
|
*/
|
|
55
|
-
export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null } = {}) {
|
|
57
|
+
export function createChatSession({ seedPayload = null, vocabSeeded = false, liveReference = false, onLiveLookup = null, synthesisBudget = 12 } = {}) {
|
|
56
58
|
const memoryDir = createInMemoryStore();
|
|
57
59
|
// Spread onto the store's own empty payload so a partial seed (individuals
|
|
58
60
|
// and objectProperties only) still carries the classes/prefixes scaffolding
|
|
@@ -69,18 +71,28 @@ export function createChatSession({ seedPayload = null, vocabSeeded = false, liv
|
|
|
69
71
|
let focus = null;
|
|
70
72
|
let last = null;
|
|
71
73
|
let planState = null;
|
|
72
|
-
|
|
73
|
-
//
|
|
74
|
-
|
|
74
|
+
let researchState = null;
|
|
75
|
+
// Four-state, like the CLI: false (off), true (rescue on a miss),
|
|
76
|
+
// "supplement" (also append a cited read-out under every grounded vocabulary
|
|
77
|
+
// answer), or "always" (widen that to every grounded answer). The two string
|
|
78
|
+
// modes stay strings so runTurn reads them as the supplement/always lanes.
|
|
79
|
+
const normLive = (v) => (v === "always" ? "always" : v === "supplement" ? "supplement" : Boolean(v));
|
|
75
80
|
let liveReferenceOn = normLive(liveReference);
|
|
81
|
+
// The auto-synthesis budget for this session's learn-on-miss loads — the
|
|
82
|
+
// page's slider sets it; 0 stores article facts without any entailed rows.
|
|
83
|
+
let synthesisBudgetOn = Number.isFinite(synthesisBudget) ? synthesisBudget : 12;
|
|
76
84
|
|
|
77
85
|
return {
|
|
78
86
|
memoryDir,
|
|
79
87
|
sessionId,
|
|
80
88
|
get liveReference() { return liveReferenceOn; },
|
|
81
89
|
/** The page's toggle seam: set the live Wikipedia mode for every later turn
|
|
82
|
-
* (the `/wiki on|off|supplement` command sets the same state). */
|
|
90
|
+
* (the `/wiki on|off|supplement|always` command sets the same state). */
|
|
83
91
|
setLiveReference(v) { liveReferenceOn = normLive(v); },
|
|
92
|
+
get synthesisBudget() { return synthesisBudgetOn; },
|
|
93
|
+
/** The page's slider seam: set the auto-synthesis budget for every later
|
|
94
|
+
* learn-on-miss load. Clamped to a non-negative integer; 0 disables it. */
|
|
95
|
+
setSynthesisBudget(n) { synthesisBudgetOn = Number.isFinite(n) && n > 0 ? Math.floor(n) : 0; },
|
|
84
96
|
|
|
85
97
|
/** One dispatched turn. A throwing runTurn must never kill the session —
|
|
86
98
|
* the page has no other chance to show this turn's answer. */
|
|
@@ -89,18 +101,23 @@ export function createChatSession({ seedPayload = null, vocabSeeded = false, liv
|
|
|
89
101
|
try {
|
|
90
102
|
result = await runTurn(line, {
|
|
91
103
|
config: null, source: null, graph, focus, last, memoryDir, sessionId,
|
|
92
|
-
env: {}, lexicon, vocabHint, planState,
|
|
104
|
+
env: {}, lexicon, vocabHint, planState, researchState,
|
|
93
105
|
liveReference: liveReferenceOn, onLiveLookup,
|
|
106
|
+
uiContext: "browser", synthesisBudget: synthesisBudgetOn,
|
|
94
107
|
});
|
|
95
108
|
} catch (e) {
|
|
96
109
|
const message = e instanceof Error ? e.message : String(e);
|
|
97
|
-
return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, plan: null };
|
|
110
|
+
return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, plan: null, research: null };
|
|
98
111
|
}
|
|
99
112
|
focus = result.focus;
|
|
100
113
|
last = result.last;
|
|
101
114
|
if ("planState" in result) planState = result.planState;
|
|
102
|
-
if (
|
|
103
|
-
|
|
115
|
+
if ("researchState" in result) researchState = result.researchState;
|
|
116
|
+
if (typeof result.liveReference === "boolean" || result.liveReference === "supplement" || result.liveReference === "always") liveReferenceOn = result.liveReference;
|
|
117
|
+
// `research` distinguishes three cases on purpose: a queue snapshot
|
|
118
|
+
// (research turn), null (a research turn that ended the run), and
|
|
119
|
+
// undefined (not a research turn — the page leaves its controls alone).
|
|
120
|
+
return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, plan: result.plan ?? null, research: result.research };
|
|
104
121
|
},
|
|
105
122
|
};
|
|
106
123
|
}
|
|
@@ -162,4 +179,4 @@ export async function exportFactsJsonl(memoryDir) {
|
|
|
162
179
|
return serializeFactsJsonl(await loadMemory(memoryDir));
|
|
163
180
|
}
|
|
164
181
|
|
|
165
|
-
globalThis.tmctChat = { createChatSession, registerWinkModel, registerReferencePackProvider, registerLiveReferenceProvider, normFactTerm, vocabExampleHint, memoryStats, openPersistedStore, exportFactsJsonl, splitSentences: splitSentencesPreservingPaths };
|
|
182
|
+
globalThis.tmctChat = { createChatSession, registerWinkModel, registerReferencePackProvider, registerLiveReferenceProvider, registerResearchProvider, normFactTerm, vocabExampleHint, memoryStats, openPersistedStore, exportFactsJsonl, splitSentences: splitSentencesPreservingPaths };
|
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
// code-explorer-browser-entry.mjs — the esbuild entry for the code explorer's
|
|
2
|
-
// LIVE chat
|
|
2
|
+
// LIVE chat (public/code-explorer.bundle.js / electron/renderer/
|
|
3
3
|
// code-explorer.bundle.js). It mirrors ledger-browser-entry.mjs, but seeds the
|
|
4
|
-
// full runTurn engine from a CODE graph
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// full runTurn engine from a CODE graph — and, when the page hands one over,
|
|
5
|
+
// the same general-knowledge seed payload chat.html boots from — so one
|
|
6
|
+
// session answers "which functions call Y" over the loaded graph and "what is
|
|
7
|
+
// a queue" over the seeded memory.
|
|
7
8
|
//
|
|
8
9
|
// The graph enters through source.mjs's provider seam — the same seam the CLI
|
|
9
10
|
// and HTTP surfaces read — so runTurn's symbol-grain lanes see the whole
|
|
10
11
|
// payload, while parseEntities builds the coarse graph the flat lanes read.
|
|
12
|
+
// The seed enters the in-memory store the same way createChatSession's does.
|
|
11
13
|
// Gitignored, built fresh by scripts/build-code-explorer-bundle.mjs; the page
|
|
12
14
|
// degrades to a static view when it is absent (renderCodeExplorerHtml's own
|
|
13
15
|
// contract), so nothing here is ever published.
|
|
14
|
-
import { runTurn } from "../../services/chat.mjs";
|
|
16
|
+
import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
|
|
15
17
|
import { createInMemoryStore, normFactTerm } from "../../adapters/memory/core.mjs";
|
|
16
18
|
import { parseEntities } from "../../domain/codegraph.mjs";
|
|
17
19
|
import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
|
|
@@ -23,17 +25,30 @@ import { generateCodeHints } from "../../domain/code-explorer-hints.mjs";
|
|
|
23
25
|
/**
|
|
24
26
|
* A browser code-explorer session over the real turn engine. Registers the
|
|
25
27
|
* loaded payload with source.mjs's provider seam, parses the coarse graph, and
|
|
26
|
-
* dispatches every turn through the same runTurn the CLI runs.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
28
|
+
* dispatches every turn through the same runTurn the CLI runs.
|
|
29
|
+
*
|
|
30
|
+
* `seedPayload` (optional) is a serialized memory graph (loadMemory's shape,
|
|
31
|
+
* built by scripts/build-chat-seed.mjs) assigned onto the session's fresh
|
|
32
|
+
* in-memory store, so general-knowledge questions ground alongside the code
|
|
33
|
+
* graph's own lanes. `vocabSeeded` tells the vocabulary hint whether that
|
|
34
|
+
* payload carries the starter vocabulary. Teaches land in the same in-memory
|
|
35
|
+
* store, so a taught fact never touches disk.
|
|
36
|
+
*
|
|
37
|
+
* Returns { memoryDir, sessionId, turn }, the createChatSession shape the
|
|
38
|
+
* page's chat expects.
|
|
29
39
|
*/
|
|
30
|
-
export function createCodeExplorerSession({ graphPayload } = {}) {
|
|
40
|
+
export function createCodeExplorerSession({ graphPayload, seedPayload = null, vocabSeeded = false } = {}) {
|
|
31
41
|
const payload = graphPayload || { individuals: [], objectProperties: [] };
|
|
32
42
|
source.registerProvider(() => payload);
|
|
33
43
|
|
|
34
44
|
const graph = parseEntities(payload);
|
|
35
45
|
const memoryDir = createInMemoryStore();
|
|
46
|
+
// Spread onto the store's own empty payload so a partial seed (individuals
|
|
47
|
+
// and objectProperties only) still carries the classes/prefixes scaffolding
|
|
48
|
+
// the write path recounts — teach turns must work on any seed.
|
|
49
|
+
if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
|
|
36
50
|
const lexicon = loadLexicon();
|
|
51
|
+
const vocabHint = vocabExampleHint(vocabSeeded);
|
|
37
52
|
const sessionId = globalThis.crypto?.randomUUID?.() ?? String(Date.now());
|
|
38
53
|
// A stable virtual path: the provider answers every fetch, so no file is
|
|
39
54
|
// ever read, but the code lanes still do path math (join/dirname) on it.
|
|
@@ -44,13 +59,14 @@ export function createCodeExplorerSession({ graphPayload } = {}) {
|
|
|
44
59
|
let planState = null;
|
|
45
60
|
|
|
46
61
|
return {
|
|
62
|
+
memoryDir,
|
|
47
63
|
sessionId,
|
|
48
64
|
async turn(line) {
|
|
49
65
|
let result;
|
|
50
66
|
try {
|
|
51
67
|
result = await runTurn(line, {
|
|
52
68
|
config, source, graph, focus, last, memoryDir, sessionId,
|
|
53
|
-
env: {}, lexicon, vocabHint
|
|
69
|
+
env: {}, lexicon, vocabHint, planState,
|
|
54
70
|
});
|
|
55
71
|
} catch (e) {
|
|
56
72
|
const message = e instanceof Error ? e.message : String(e);
|
|
@@ -66,7 +82,7 @@ export function createCodeExplorerSession({ graphPayload } = {}) {
|
|
|
66
82
|
|
|
67
83
|
// Exposed for the page's inline client: the re-derivation helpers so a graph
|
|
68
84
|
// swapped through the desktop picker re-renders without duplicating logic, plus
|
|
69
|
-
// the wink loader hook registerWinkModel and normFactTerm the
|
|
85
|
+
// the wink loader hook registerWinkModel and normFactTerm the chat shares with
|
|
70
86
|
// the ledger page.
|
|
71
87
|
globalThis.tmctCodeExplorer = {
|
|
72
88
|
createCodeExplorerSession,
|
|
@@ -12,7 +12,11 @@
|
|
|
12
12
|
// The recognizer is reached through ONE seam — `groundTextToFacts` — so a
|
|
13
13
|
// wider ingest tier (an optimistic fuzzy-match pass, a canonical/graph-linked
|
|
14
14
|
// output) slots in behind this single function without the page changing.
|
|
15
|
-
//
|
|
15
|
+
// Alongside the strict recognizer, groundTextToFacts runs the same
|
|
16
|
+
// citation-stripping, clause-fallback and bounded pronoun-carry passes
|
|
17
|
+
// `ingestText` (extract-facts.mjs) applies for the CLI, and — opt-in, off by
|
|
18
|
+
// default — the same low-trust optimistic tier, so the browser page and the
|
|
19
|
+
// CLI ground the identical class of sentence.
|
|
16
20
|
//
|
|
17
21
|
// Gitignored, Pages-demo-site-only output (scripts/build-demo-site.mjs builds
|
|
18
22
|
// it fresh on every deploy, never committed) — the same posture
|
|
@@ -20,21 +24,37 @@
|
|
|
20
24
|
// runTurn engine, the same weight class as the chat/ledger bundles, and is
|
|
21
25
|
// never published.
|
|
22
26
|
import { runTurn, vocabExampleHint } from "../../services/chat.mjs";
|
|
23
|
-
import { createInMemoryStore, normFactTerm, loadMemory, readFactRows } from "../../adapters/memory/core.mjs";
|
|
27
|
+
import { createInMemoryStore, normFactTerm, loadMemory, readFactRows, appendFact } from "../../adapters/memory/core.mjs";
|
|
24
28
|
import { serializeFactsJsonl } from "../../adapters/memory/export-jsonl.mjs";
|
|
25
|
-
import { splitSentencesPreservingPaths } from "../../services/sentences.mjs";
|
|
29
|
+
import { splitSentencesPreservingPaths, stripCitationResidue } from "../../services/sentences.mjs";
|
|
30
|
+
import { clauseCandidates, optimisticTriples } from "../../services/extract-facts.mjs";
|
|
26
31
|
import { touchedFactRows } from "../../domain/memory/touched-facts.mjs";
|
|
27
|
-
import { parseEntities } from "../../domain/codegraph.mjs";
|
|
28
32
|
import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
|
|
29
|
-
import { registerWinkModel } from "../../adapters/wink-model.mjs";
|
|
33
|
+
import { registerWinkModel, winkInstance } from "../../adapters/wink-model.mjs";
|
|
34
|
+
import { memoryStats } from "./memory-stats.mjs";
|
|
35
|
+
import { openPersistedStore } from "./idb-persist.mjs";
|
|
36
|
+
|
|
37
|
+
// The pronoun subjects a bounded carry substitutes with the last unique
|
|
38
|
+
// grounded subject in the SAME paragraph. Reset at every blank line, so a
|
|
39
|
+
// fresh paragraph never resolves against a stale antecedent.
|
|
40
|
+
const PRONOUN_LEAD_RE = /^(?:they|it|these|those|this)\b\s*/i;
|
|
30
41
|
|
|
31
42
|
/**
|
|
32
|
-
* The single recognizer seam. Splits `text` into
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
43
|
+
* The single recognizer seam. Splits `text` into paragraphs (blank-line
|
|
44
|
+
* separated, so the pronoun carry never bridges a topic break) and each
|
|
45
|
+
* paragraph into sentences (wink's own boundary detection, never a regex).
|
|
46
|
+
* Each sentence's citation residue ("[3]", "[citation needed]") is stripped
|
|
47
|
+
* before grounding; the strict recognizer then tries the whole sentence
|
|
48
|
+
* first, falling back to its clause fragments only on a miss
|
|
49
|
+
* (`clauseCandidates`), and — still on a miss, only for a pronoun-led
|
|
50
|
+
* sentence with a unique adjacent antecedent in this paragraph — one retry
|
|
51
|
+
* with that antecedent substituted in.
|
|
52
|
+
*
|
|
53
|
+
* A sentence still ungrounded after all of that runs the optimistic fuzzy
|
|
54
|
+
* tier when `optimistic` is set (`optimisticTriples`): a copula or known
|
|
55
|
+
* relation verb flanked by two resolvable entities, stored under the
|
|
56
|
+
* low-trust `optimistic-extract:page` tag rather than a teach/assert
|
|
57
|
+
* provenance, so a fuzzy candidate can never corroborate a curated fact.
|
|
38
58
|
*
|
|
39
59
|
* `onFact(fact)` (optional) is awaited after each grounded row so the page can
|
|
40
60
|
* render the canonical facts LIVE as they land, one at a time.
|
|
@@ -42,48 +62,107 @@ import { registerWinkModel } from "../../adapters/wink-model.mjs";
|
|
|
42
62
|
* Returns { sentences, recognized, skipped, facts } — `facts` an array of
|
|
43
63
|
* { subject, predicate, object, provenance, quantifier, sentence } in the same
|
|
44
64
|
* canonical shape `tmct extract` and the JSONL exporter emit.
|
|
45
|
-
*
|
|
46
|
-
* A wider ingest tier plugs in HERE, behind this one function: the page calls
|
|
47
|
-
* it and nothing else.
|
|
48
65
|
*/
|
|
49
|
-
export async function groundTextToFacts(text, { memoryDir, sessionId,
|
|
50
|
-
const
|
|
66
|
+
export async function groundTextToFacts(text, { memoryDir, sessionId, lexicon, vocabHint, optimistic = false, onFact = null } = {}) {
|
|
67
|
+
const paragraphs = String(text ?? "").split(/\n[ \t]*\n/);
|
|
51
68
|
const facts = [];
|
|
52
69
|
let focus = null;
|
|
53
70
|
let last = null;
|
|
54
71
|
let planState = null;
|
|
72
|
+
let sentenceCount = 0;
|
|
55
73
|
let recognized = 0;
|
|
74
|
+
// Reused, not reloaded: readFactRows(loadMemory(...)) is O(rows), so a
|
|
75
|
+
// fresh read on every sentence over a large seeded store is exactly the
|
|
76
|
+
// O(sentences × rows) cost this page must avoid. The snapshot only moves
|
|
77
|
+
// forward past a sentence that actually asserted something.
|
|
78
|
+
let beforeRows = readFactRows(await loadMemory(memoryDir));
|
|
79
|
+
const nlp = optimistic ? winkInstance() : null;
|
|
56
80
|
|
|
57
|
-
|
|
58
|
-
|
|
81
|
+
// One grounding attempt against `memoryDir`: runs `form` through runTurn,
|
|
82
|
+
// threading focus/last/planState the same way whether or not it grounds.
|
|
83
|
+
// No `graph` is passed — this page never offers a code graph to link
|
|
84
|
+
// against, and a non-null one (even an empty one) makes runTurn read an
|
|
85
|
+
// ordinary "the number of X" phrase as a graph count query instead of
|
|
86
|
+
// running the teach cascade, exactly the phrasing the Sales-paragraph
|
|
87
|
+
// sentence this pipeline is built to ground uses.
|
|
88
|
+
// Returns the Fact rows this attempt actually touched, or null — a miss, a
|
|
89
|
+
// non-assert turn, or an assert that touched no Fact row (a Rule teach).
|
|
90
|
+
async function attempt(form) {
|
|
59
91
|
let record;
|
|
60
92
|
try {
|
|
61
|
-
const result = await runTurn(
|
|
62
|
-
config: null, source: null,
|
|
63
|
-
env: {}, lexicon, vocabHint, planState,
|
|
93
|
+
const result = await runTurn(form, {
|
|
94
|
+
config: null, source: null, focus, last, memoryDir, sessionId,
|
|
95
|
+
env: {}, lexicon, vocabHint, planState, uiContext: "browser",
|
|
64
96
|
});
|
|
65
97
|
focus = result.focus;
|
|
66
98
|
last = result.last;
|
|
67
99
|
if ("planState" in result) planState = result.planState;
|
|
68
100
|
record = result.record;
|
|
69
101
|
} catch {
|
|
70
|
-
|
|
102
|
+
return null; // a throwing sentence is a skip, never a page-killer
|
|
71
103
|
}
|
|
72
|
-
if (record?.via !== "assert" || record?.miss)
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
104
|
+
if (record?.via !== "assert" || record?.miss) return null;
|
|
105
|
+
// The one loadMemory/readFactRows call this attempt pays for, only
|
|
106
|
+
// because it actually asserted something. Equal lengths mean nothing was
|
|
107
|
+
// ADDED — a re-assertion's provenance-only change is the one case this
|
|
108
|
+
// fast path can't see, an accepted trade against paying the full
|
|
109
|
+
// before/after diff on every recognized sentence in a large store.
|
|
110
|
+
const afterRows = readFactRows(await loadMemory(memoryDir));
|
|
111
|
+
const rows = afterRows.length === beforeRows.length ? [] : touchedFactRows(beforeRows, afterRows);
|
|
112
|
+
beforeRows = afterRows;
|
|
113
|
+
return rows.length ? rows : null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
for (const paragraph of paragraphs) {
|
|
117
|
+
// The last unique grounded subject in THIS paragraph, carried onto a
|
|
118
|
+
// later pronoun-led sentence the strict recognizer couldn't ground on its
|
|
119
|
+
// own. Cleared at the paragraph boundary.
|
|
120
|
+
let carrySubject = null;
|
|
121
|
+
for (const rawSentence of splitSentencesPreservingPaths(paragraph)) {
|
|
122
|
+
sentenceCount += 1;
|
|
123
|
+
const cleaned = stripCitationResidue(rawSentence);
|
|
124
|
+
|
|
125
|
+
let rows = null;
|
|
126
|
+
for (const candidate of clauseCandidates(cleaned, { nlp })) {
|
|
127
|
+
rows = await attempt(candidate);
|
|
128
|
+
if (rows) break;
|
|
129
|
+
}
|
|
130
|
+
if (!rows && carrySubject && PRONOUN_LEAD_RE.test(cleaned)) {
|
|
131
|
+
rows = await attempt(cleaned.replace(PRONOUN_LEAD_RE, `${carrySubject} `));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (rows) {
|
|
135
|
+
recognized += 1;
|
|
136
|
+
const subjects = new Set(rows.map((r) => r.subject));
|
|
137
|
+
if (subjects.size === 1) carrySubject = [...subjects][0];
|
|
138
|
+
for (const row of rows) {
|
|
139
|
+
const fact = {
|
|
140
|
+
subject: row.subject, predicate: row.predicate, object: row.object,
|
|
141
|
+
provenance: row.provenance || "", quantifier: row.quantifier || "", sentence: rawSentence,
|
|
142
|
+
};
|
|
143
|
+
facts.push(fact);
|
|
144
|
+
if (onFact) await onFact(fact);
|
|
145
|
+
}
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (!optimistic) continue;
|
|
150
|
+
const candidates = optimisticTriples(cleaned, { lexicon, nlp });
|
|
151
|
+
if (!candidates.length) continue;
|
|
152
|
+
recognized += 1;
|
|
153
|
+
for (const t of candidates) {
|
|
154
|
+
await appendFact(memoryDir, { subject: t.subject, predicate: t.predicate, object: t.object, provenance: "optimistic-extract:page" });
|
|
155
|
+
const fact = {
|
|
156
|
+
subject: t.subject, predicate: t.predicate, object: t.object,
|
|
157
|
+
provenance: "optimistic-extract:page", quantifier: "", sentence: rawSentence,
|
|
158
|
+
};
|
|
159
|
+
facts.push(fact);
|
|
160
|
+
if (onFact) await onFact(fact);
|
|
161
|
+
}
|
|
83
162
|
}
|
|
84
163
|
}
|
|
85
164
|
|
|
86
|
-
return { sentences:
|
|
165
|
+
return { sentences: sentenceCount, recognized, skipped: sentenceCount - recognized, facts };
|
|
87
166
|
}
|
|
88
167
|
|
|
89
168
|
/**
|
|
@@ -92,15 +171,15 @@ export async function groundTextToFacts(text, { memoryDir, sessionId, graph, lex
|
|
|
92
171
|
* the page can then export. `seedPayload` (optional) pre-loads a graph the
|
|
93
172
|
* recognizer can recall and link against.
|
|
94
173
|
*
|
|
95
|
-
* Returns { memoryDir, sessionId, ingest }. `ingest(text, { onFact
|
|
96
|
-
* one call the page makes; it drives groundTextToFacts
|
|
97
|
-
* store and returns its { sentences, recognized,
|
|
174
|
+
* Returns { memoryDir, sessionId, ingest }. `ingest(text, { onFact,
|
|
175
|
+
* optimistic })` is the one call the page makes; it drives groundTextToFacts
|
|
176
|
+
* against this session's store and returns its { sentences, recognized,
|
|
177
|
+
* skipped, facts } summary.
|
|
98
178
|
*/
|
|
99
179
|
export function createIngestSession({ seedPayload = null, vocabSeeded = false } = {}) {
|
|
100
180
|
const memoryDir = createInMemoryStore();
|
|
101
181
|
if (seedPayload) memoryDir.payload = { ...memoryDir.payload, ...seedPayload };
|
|
102
182
|
|
|
103
|
-
const graph = parseEntities({ individuals: [], objectProperties: [] });
|
|
104
183
|
const lexicon = loadLexicon();
|
|
105
184
|
const vocabHint = vocabExampleHint(vocabSeeded);
|
|
106
185
|
const sessionId = globalThis.crypto?.randomUUID?.() ?? String(Date.now());
|
|
@@ -108,8 +187,8 @@ export function createIngestSession({ seedPayload = null, vocabSeeded = false }
|
|
|
108
187
|
return {
|
|
109
188
|
memoryDir,
|
|
110
189
|
sessionId,
|
|
111
|
-
ingest(text, { onFact = null } = {}) {
|
|
112
|
-
return groundTextToFacts(text, { memoryDir, sessionId,
|
|
190
|
+
ingest(text, { onFact = null, optimistic = false } = {}) {
|
|
191
|
+
return groundTextToFacts(text, { memoryDir, sessionId, lexicon, vocabHint, onFact, optimistic });
|
|
113
192
|
},
|
|
114
193
|
};
|
|
115
194
|
}
|
|
@@ -123,4 +202,7 @@ export async function exportFactsJsonl(memoryDir) {
|
|
|
123
202
|
return serializeFactsJsonl(await loadMemory(memoryDir));
|
|
124
203
|
}
|
|
125
204
|
|
|
126
|
-
globalThis.tmctIngest = {
|
|
205
|
+
globalThis.tmctIngest = {
|
|
206
|
+
createIngestSession, groundTextToFacts, exportFactsJsonl, registerWinkModel, normFactTerm,
|
|
207
|
+
memoryStats, openPersistedStore,
|
|
208
|
+
};
|
|
@@ -23,6 +23,7 @@ import { splitSentencesPreservingPaths } from "../../services/sentences.mjs";
|
|
|
23
23
|
import { parseEntities } from "../../domain/codegraph.mjs";
|
|
24
24
|
import { loadLexicon } from "../../domain/grammar/lexicon.mjs";
|
|
25
25
|
import { registerWinkModel } from "../../adapters/wink-model.mjs";
|
|
26
|
+
import { registerResearchProvider } from "../../adapters/corpus/wikipedia-live.mjs";
|
|
26
27
|
import { computeLedgerDataFromPayload } from "../../services/ledger-viz.mjs";
|
|
27
28
|
|
|
28
29
|
/**
|
|
@@ -54,6 +55,7 @@ export function createLedgerSession({ seedPayload = null, vocabSeeded = false }
|
|
|
54
55
|
let focus = null;
|
|
55
56
|
let last = null;
|
|
56
57
|
let planState = null;
|
|
58
|
+
let researchState = null;
|
|
57
59
|
|
|
58
60
|
return {
|
|
59
61
|
memoryDir,
|
|
@@ -66,16 +68,20 @@ export function createLedgerSession({ seedPayload = null, vocabSeeded = false }
|
|
|
66
68
|
try {
|
|
67
69
|
result = await runTurn(line, {
|
|
68
70
|
config: null, source: null, graph, focus, last, memoryDir, sessionId,
|
|
69
|
-
env: {}, lexicon, vocabHint, planState,
|
|
71
|
+
env: {}, lexicon, vocabHint, planState, researchState,
|
|
70
72
|
});
|
|
71
73
|
} catch (e) {
|
|
72
74
|
const message = e instanceof Error ? e.message : String(e);
|
|
73
|
-
return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null };
|
|
75
|
+
return { answer: `Something went wrong answering that (${message}). Try rephrasing, or /help.`, end: false, record: null, research: null };
|
|
74
76
|
}
|
|
75
77
|
focus = result.focus;
|
|
76
78
|
last = result.last;
|
|
77
79
|
if ("planState" in result) planState = result.planState;
|
|
78
|
-
|
|
80
|
+
if ("researchState" in result) researchState = result.researchState;
|
|
81
|
+
// `research` distinguishes a queue snapshot (research turn) from null
|
|
82
|
+
// (a research turn that ended the run) from undefined (not a research
|
|
83
|
+
// turn) — the dock's controls only react to the first two.
|
|
84
|
+
return { answer: result.answer, end: Boolean(result.end), record: result.record ?? null, research: result.research };
|
|
79
85
|
},
|
|
80
86
|
};
|
|
81
87
|
}
|
|
@@ -95,4 +101,4 @@ export async function exportFactsJsonl(memoryDir) {
|
|
|
95
101
|
// splitSentences + exportFactsJsonl carry the dock's paste-and-drop ingest and
|
|
96
102
|
// its JSONL export across the bundle boundary, the same one-serializer posture
|
|
97
103
|
// chat-browser-entry.mjs holds for its own page.
|
|
98
|
-
globalThis.tmctLedger = { createLedgerSession, computeLedgerDataFromPayload, normFactTerm, registerWinkModel, splitSentences: splitSentencesPreservingPaths, exportFactsJsonl };
|
|
104
|
+
globalThis.tmctLedger = { createLedgerSession, computeLedgerDataFromPayload, normFactTerm, registerWinkModel, registerResearchProvider, splitSentences: splitSentencesPreservingPaths, exportFactsJsonl };
|