@polycode-projects/the-mechanical-code-talker 1.11.6 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +244 -48
- package/ROADMAP.md +23 -34
- package/bin/tmct.mjs +117 -81
- package/corpus/LICENSES.json +118 -0
- package/corpus/README.md +17 -13
- package/corpus/conceptnet/README.md +5 -5
- package/corpus/conceptnet/fetch-slice.mjs +1 -1
- package/corpus/conceptnet/filter-dump.mjs +1 -1
- package/corpus/generated/README.md +9 -10
- package/corpus/namenet/README.md +39 -0
- package/corpus/seon/README.md +2 -2
- package/corpus/tier2/generate.mjs +58 -9
- package/corpus/tier2/manifest.json +44 -0
- package/corpus/wordnet/README.md +37 -0
- package/corpus/wordnet/generate.mjs +1 -1
- package/data/templates/constructions/agent-noun-relations.toml +2 -2
- package/data/templates/grammar-rules.toml +1 -1
- package/package.json +13 -22
- package/src/{ask-nlp.mjs → adapters/ask-nlp.mjs} +1 -1
- package/src/{config.mjs → adapters/config.mjs} +1 -1
- package/src/{corpus → adapters/corpus}/conceptnet-map.toml +4 -4
- package/src/{corpus → adapters/corpus}/conceptnet.mjs +3 -3
- package/src/adapters/corpus/construction-banks.mjs +43 -0
- package/src/{corpus → adapters/corpus}/templates.mjs +1 -1
- package/src/{embed.mjs → adapters/embed.mjs} +1 -11
- package/src/{graph-build.mjs → adapters/graph-build.mjs} +6 -6
- package/src/{memory → adapters/memory}/blocks.mjs +2 -2
- package/src/{memory → adapters/memory}/core.mjs +38 -94
- package/src/adapters/prose-tokens.mjs +98 -0
- package/src/{providers → adapters/providers}/bootstrap.mjs +2 -2
- package/src/{providers → adapters/providers}/fixture.mjs +3 -3
- package/src/{providers → adapters/providers}/graph-service.mjs +9 -4
- package/src/{source-slice.mjs → adapters/source-slice.mjs} +2 -2
- package/src/{source.mjs → adapters/source.mjs} +1 -1
- package/src/{toml-config.mjs → adapters/toml-config.mjs} +1 -1
- package/src/{answer-variants.json → domain/answer-variants.json} +1 -1
- package/src/domain/answer-variants.mjs +23 -0
- package/src/{ask-vocab.mjs → domain/ask-vocab.mjs} +37 -5
- package/src/{ask.mjs → domain/ask.mjs} +216 -52
- package/src/{codegraph.mjs → domain/codegraph.mjs} +31 -315
- package/src/{completions → domain/completions}/complete.mjs +16 -10
- package/src/{completions → domain/completions}/graph-adapter.mjs +10 -4
- package/src/{completions → domain/completions}/group.mjs +14 -6
- package/src/{completions → domain/completions}/infer.mjs +57 -39
- package/src/domain/completions/injected.mjs +21 -0
- package/src/{completions → domain/completions}/rank.mjs +15 -8
- package/src/{completions → domain/completions}/search.mjs +4 -2
- package/src/{grammar → domain/grammar}/ace.mjs +3 -3
- package/src/{grammar → domain/grammar}/assert.mjs +12 -8
- package/src/{grammar → domain/grammar}/lexicon-core.json +1 -1
- package/src/{grammar → domain/grammar}/lexicon.mjs +4 -6
- package/src/domain/hash.mjs +147 -0
- package/src/{interpret → domain/interpret}/fuzzy.mjs +42 -4
- package/src/domain/interpret/nlp-registry.mjs +20 -0
- package/src/{interpret → domain/interpret}/normalize.mjs +35 -5
- package/src/{interpret → domain/interpret}/pipeline.mjs +1 -5
- package/src/{interpret → domain/interpret}/strategies/ace.mjs +1 -1
- package/src/{interpret → domain/interpret}/strategies/constructions.mjs +30 -52
- package/src/{interpret → domain/interpret}/strategies/keywords.mjs +48 -26
- package/src/domain/memory/capability.mjs +235 -0
- package/src/domain/memory/fold.mjs +54 -0
- package/src/domain/memory/session-turns.mjs +7 -0
- package/src/{memory → domain/memory}/trust.mjs +48 -0
- package/src/{paraphrase.mjs → domain/paraphrase.mjs} +2 -2
- package/src/{prose.mjs → domain/prose.mjs} +1 -1
- package/src/domain/real-word-collisions.json +1 -0
- package/src/{router → domain/router}/call-validator.mjs +1 -1
- package/src/{router → domain/router}/drive.mjs +34 -25
- package/src/{router → domain/router}/goal-reasoner.mjs +1 -1
- package/src/{router → domain/router}/guardrail.mjs +1 -1
- package/src/{router → domain/router}/planner.mjs +1 -1
- package/src/{router → domain/router}/registry.mjs +5 -5
- package/src/{router → domain/router}/resolver.mjs +16 -13
- package/src/{router → domain/router}/results.mjs +1 -1
- package/src/{router → domain/router}/set-algebra.mjs +1 -1
- package/src/{router → domain/router}/taught.mjs +10 -9
- package/src/{syllogise.mjs → domain/syllogise.mjs} +21 -4
- package/src/domain/vector.mjs +12 -0
- package/src/services/chat-session.mjs +451 -0
- package/src/{chat.mjs → services/chat.mjs} +1238 -680
- package/src/{cli-args.mjs → services/cli-args.mjs} +2 -2
- package/src/services/completions.mjs +55 -0
- package/src/{extensions.mjs → services/extensions.mjs} +7 -7
- package/src/{finish.mjs → services/finish.mjs} +2 -2
- package/src/{memory → services}/fold.mjs +0 -0
- package/src/{import-file.mjs → services/import-file.mjs} +3 -3
- package/src/{index.mjs → services/index.mjs} +21 -12
- package/src/{init.mjs → services/init.mjs} +9 -9
- package/src/{ledger-viz.mjs → services/ledger-viz.mjs} +38 -7
- package/src/{plan-viz.mjs → services/plan-viz.mjs} +65 -26
- package/src/{sentences.mjs → services/sentences.mjs} +1 -1
- package/src/{sessions.mjs → services/sessions.mjs} +4 -5
- package/src/{telemetry.mjs → services/telemetry.mjs} +1 -1
- package/src/{server-http.mjs → surfaces/http/server-http.mjs} +11 -65
- package/src/{tui → surfaces/tui}/app.mjs +3 -3
- package/src/{memory-ask-browser-entry.mjs → surfaces/web/memory-ask-browser-entry.mjs} +5 -5
- package/src/{memory-ask-browser.bundle.js → surfaces/web/memory-ask-browser.bundle.js} +9465 -6366
- package/src/tools/catalog.mjs +29 -0
- package/src/{conformance.mjs → tools/conformance.mjs} +2 -2
- package/src/tools/definitions.mjs +288 -0
- package/src/tools/graph-load.mjs +20 -0
- package/src/tools/handlers/index.mjs +54 -0
- package/src/tools/handlers/kit.mjs +33 -0
- package/src/tools/handlers/tmct-architecture.mjs +7 -0
- package/src/tools/handlers/tmct-ask.mjs +14 -0
- package/src/tools/handlers/tmct-callees.mjs +6 -0
- package/src/tools/handlers/tmct-callers.mjs +6 -0
- package/src/tools/handlers/tmct-calls.mjs +6 -0
- package/src/tools/handlers/tmct-class-history.mjs +6 -0
- package/src/tools/handlers/tmct-cochanges.mjs +6 -0
- package/src/tools/handlers/tmct-context-more.mjs +9 -0
- package/src/tools/handlers/tmct-context.mjs +163 -0
- package/src/tools/handlers/tmct-describe.mjs +15 -0
- package/src/tools/handlers/tmct-exports.mjs +9 -0
- package/src/tools/handlers/tmct-file-history.mjs +6 -0
- package/src/tools/handlers/tmct-history.mjs +6 -0
- package/src/tools/handlers/tmct-impact.mjs +9 -0
- package/src/tools/handlers/tmct-members.mjs +16 -0
- package/src/tools/handlers/tmct-method-history.mjs +6 -0
- package/src/tools/handlers/tmct-search.mjs +22 -0
- package/src/tools/handlers/tmct-signature.mjs +6 -0
- package/src/tools/handlers/tmct-snippet.mjs +37 -0
- package/src/tools/handlers/tmct-subclasses.mjs +16 -0
- package/src/tools/handlers/tmct-tests-for.mjs +6 -0
- package/src/tools/handlers/tmct-untested.mjs +7 -0
- package/src/tools/memory-fallthrough.mjs +65 -0
- package/src/{schema-docs.mjs → tools/schema-docs.mjs} +1 -1
- package/src/tools/server.mjs +61 -0
- package/src/answer-variants.mjs +0 -39
- package/src/hash.mjs +0 -24
- package/src/server.mjs +0 -501
- /package/src/{corpus → adapters/corpus}/unknown-ingest.mjs +0 -0
- /package/src/{graph-merge.mjs → adapters/graph-merge.mjs} +0 -0
- /package/src/{memory → adapters/memory}/inspect.mjs +0 -0
- /package/src/{memory → adapters/memory}/shacl.mjs +0 -0
- /package/src/{prose-nlp.mjs → adapters/prose-nlp.mjs} +0 -0
- /package/src/{repository-interface.mjs → adapters/repository-interface.mjs} +0 -0
- /package/src/{uuid.mjs → adapters/uuid.mjs} +0 -0
- /package/src/{wink-model.mjs → adapters/wink-model.mjs} +0 -0
- /package/src/{completions → domain/completions}/prune.mjs +0 -0
- /package/src/{concept.mjs → domain/concept.mjs} +0 -0
- /package/src/{domain.mjs → domain/domain.mjs} +0 -0
- /package/src/{interpret → domain/interpret}/merge.mjs +0 -0
- /package/src/{interpret → domain/interpret}/strategies/grammar.mjs +0 -0
- /package/src/{interpret → domain/interpret}/strategies/noise-strip.mjs +0 -0
- /package/src/{memory → domain/memory}/bias.mjs +0 -0
- /package/src/{planning.mjs → domain/planning.mjs} +0 -0
- /package/src/{viz-theme.mjs → services/viz-theme.mjs} +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// completions/infer.mjs — Stage 3 ("inference between groups"): applies
|
|
2
|
-
//
|
|
1
|
+
// completions/infer.mjs — Stage 3 ("inference between groups"): applies the injected
|
|
2
|
+
// resolveRelationChase to relationships BETWEEN retrieved text groups, not just graph facts.
|
|
3
3
|
// Four relations (supports/contradicts/elaborates/exemplifies), each with its own named
|
|
4
4
|
// licensing test — see the four test*() functions below. A relation is asserted only when its
|
|
5
5
|
// test concretely licenses it, never from prose similarity.
|
|
@@ -7,18 +7,18 @@
|
|
|
7
7
|
// Entities are a group's content tokens narrowed to graph-known terms (normFactTerm-matched
|
|
8
8
|
// against loaded facts) — sharing an English word alone never licenses a relation.
|
|
9
9
|
|
|
10
|
-
import { normFactTerm, readFactRows, resolveRelationChase } from "../memory/core.mjs";
|
|
11
|
-
import { tokenizeBlock } from "../memory/blocks.mjs";
|
|
12
10
|
import { splitSentences } from "./rank.mjs";
|
|
13
11
|
import { STOPWORDS } from "../prose.mjs";
|
|
12
|
+
import { findActionPath, findReachableSet } from "../planning.mjs";
|
|
13
|
+
import { requireInjected } from "./injected.mjs";
|
|
14
14
|
|
|
15
15
|
// Same content-token filter group.mjs/rank.mjs apply to their own adjacency/ranking; not
|
|
16
16
|
// exported from either, so replicated here rather than reached across files.
|
|
17
17
|
const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
|
|
18
18
|
|
|
19
19
|
/** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
|
|
20
|
-
function
|
|
21
|
-
return tokenizeBlock(text).filter(isContentToken);
|
|
20
|
+
function makeContentTokens(tokenizeBlock) {
|
|
21
|
+
return (text) => tokenizeBlock(text).filter(isContentToken);
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
// Local copy of chat.mjs's private HAS_PROPERTY_PREDICATE constant — not exported, so callers
|
|
@@ -46,7 +46,7 @@ function sentenceIsNegated(sentence) {
|
|
|
46
46
|
|
|
47
47
|
/** Union of contentTokens() over every member's text — a group's own content-token
|
|
48
48
|
* vocabulary, deduped. */
|
|
49
|
-
function groupContentTokenSet(group) {
|
|
49
|
+
function groupContentTokenSet(group, { contentTokens }) {
|
|
50
50
|
const set = new Set();
|
|
51
51
|
for (const m of group?.members || []) {
|
|
52
52
|
for (const t of contentTokens(m?.text || "")) set.add(t);
|
|
@@ -57,7 +57,7 @@ function groupContentTokenSet(group) {
|
|
|
57
57
|
/** Every sentence across a group's members, pre-split (rank.mjs's own splitSentences — reused
|
|
58
58
|
* verbatim, no re-implementation), each carrying its own content-token set and negation flag
|
|
59
59
|
* — the exact per-sentence facts the contradicts test needs. */
|
|
60
|
-
function sentencesOf(group) {
|
|
60
|
+
function sentencesOf(group, { contentTokens }) {
|
|
61
61
|
const out = [];
|
|
62
62
|
for (const m of group?.members || []) {
|
|
63
63
|
for (const sentence of splitSentences(m?.text || "")) {
|
|
@@ -69,7 +69,7 @@ function sentencesOf(group) {
|
|
|
69
69
|
|
|
70
70
|
/** Every normFactTerm-normalized term that appears as SOME fact's subject or object in the
|
|
71
71
|
* loaded memory — the "graph-known term" universe entities are grounded against. */
|
|
72
|
-
function buildGraphTerms(rows) {
|
|
72
|
+
function buildGraphTerms(rows, normFactTerm) {
|
|
73
73
|
const set = new Set();
|
|
74
74
|
for (const r of rows) {
|
|
75
75
|
const s = normFactTerm(r.subject);
|
|
@@ -81,8 +81,8 @@ function buildGraphTerms(rows) {
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
/** A group's GRAPH-GROUNDED entities: content tokens narrowed to graph-known terms, sorted. */
|
|
84
|
-
function entitiesOf(group,
|
|
85
|
-
return [...groupContentTokenSet(group)].filter((t) => graphTerms.has(t)).sort();
|
|
84
|
+
function entitiesOf(group, grounding) {
|
|
85
|
+
return [...groupContentTokenSet(group, grounding)].filter((t) => grounding.graphTerms.has(t)).sort();
|
|
86
86
|
}
|
|
87
87
|
|
|
88
88
|
/** A minimal relationFactsFor(name): direct-predicate match only (`mgx:${name}`), no
|
|
@@ -104,6 +104,8 @@ function makeHelpers(rows) {
|
|
|
104
104
|
byTrust: (a, b) => (b.trust ?? 0) - (a.trust ?? 0),
|
|
105
105
|
rows,
|
|
106
106
|
HAS_PROPERTY_PREDICATE,
|
|
107
|
+
findActionPath,
|
|
108
|
+
findReachableSet,
|
|
107
109
|
};
|
|
108
110
|
}
|
|
109
111
|
|
|
@@ -122,9 +124,10 @@ function relationNameCandidates(rows) {
|
|
|
122
124
|
/** SUPPORTS — groups A/B share >=2 graph-grounded entities AND a taught relation fact
|
|
123
125
|
* connects two of them. Tries every (subject, object) pair against every candidate relation
|
|
124
126
|
* name in fixed sorted order; first hit wins. Returns `{ licensingTest, evidence }` or null. */
|
|
125
|
-
async function testSupports(a, b,
|
|
126
|
-
const
|
|
127
|
-
const
|
|
127
|
+
async function testSupports(a, b, grounding) {
|
|
128
|
+
const { memory, helpers, relationNames, resolveRelationChase } = grounding;
|
|
129
|
+
const entitiesA = entitiesOf(a, grounding);
|
|
130
|
+
const entitiesB = entitiesOf(b, grounding);
|
|
128
131
|
const shared = entitiesA.filter((e) => entitiesB.includes(e));
|
|
129
132
|
if (shared.length < 2) return null;
|
|
130
133
|
for (const subjectTerm of shared) {
|
|
@@ -148,18 +151,18 @@ async function testSupports(a, b, memory, helpers, relationNames, graphTerms) {
|
|
|
148
151
|
/** CONTRADICTS — groups A/B share a graph-grounded entity plus a second co-occurring token
|
|
149
152
|
* ("aspect"), and one side's matching sentence is negated while the other's isn't. Returns
|
|
150
153
|
* `{ licensingTest, evidence }` or null. */
|
|
151
|
-
function testContradicts(a, b,
|
|
152
|
-
const entitiesA = entitiesOf(a,
|
|
153
|
-
const entitiesB = entitiesOf(b,
|
|
154
|
+
function testContradicts(a, b, grounding) {
|
|
155
|
+
const entitiesA = entitiesOf(a, grounding);
|
|
156
|
+
const entitiesB = entitiesOf(b, grounding);
|
|
154
157
|
const sharedEntities = entitiesA.filter((e) => entitiesB.includes(e));
|
|
155
158
|
if (!sharedEntities.length) return null;
|
|
156
159
|
|
|
157
|
-
const tokensA = groupContentTokenSet(a);
|
|
158
|
-
const tokensB = groupContentTokenSet(b);
|
|
160
|
+
const tokensA = groupContentTokenSet(a, grounding);
|
|
161
|
+
const tokensB = groupContentTokenSet(b, grounding);
|
|
159
162
|
const sharedTokens = [...tokensA].filter((t) => tokensB.has(t)).sort();
|
|
160
163
|
|
|
161
|
-
const sentencesA = sentencesOf(a);
|
|
162
|
-
const sentencesB = sentencesOf(b);
|
|
164
|
+
const sentencesA = sentencesOf(a, grounding);
|
|
165
|
+
const sentencesB = sentencesOf(b, grounding);
|
|
163
166
|
|
|
164
167
|
for (const entity of sharedEntities) {
|
|
165
168
|
for (const aspect of sharedTokens) {
|
|
@@ -205,9 +208,9 @@ function isProperSubset(small, big) {
|
|
|
205
208
|
/** ELABORATES — one group's graph-grounded entity set is a PROPER SUBSET of the other's; the
|
|
206
209
|
* wider group elaborates the narrower one. Equal sets never count. Returns
|
|
207
210
|
* `{ wider: "a"|"b", licensingTest, evidence }` or null. */
|
|
208
|
-
function testElaborates(a, b,
|
|
209
|
-
const entitiesA = new Set(entitiesOf(a,
|
|
210
|
-
const entitiesB = new Set(entitiesOf(b,
|
|
211
|
+
function testElaborates(a, b, grounding) {
|
|
212
|
+
const entitiesA = new Set(entitiesOf(a, grounding));
|
|
213
|
+
const entitiesB = new Set(entitiesOf(b, grounding));
|
|
211
214
|
if (!entitiesA.size || !entitiesB.size) return null;
|
|
212
215
|
if (isProperSubset(entitiesB, entitiesA)) {
|
|
213
216
|
return {
|
|
@@ -230,9 +233,10 @@ function testElaborates(a, b, graphTerms) {
|
|
|
230
233
|
* `instance` names an entity taught to BE one directly. Asymmetric: callers probe both
|
|
231
234
|
* directions by calling this twice with groups swapped (see inferRelations below). Returns
|
|
232
235
|
* `{ licensingTest, evidence }` or null. */
|
|
233
|
-
function testExemplifies(general, instance,
|
|
234
|
-
const
|
|
235
|
-
const
|
|
236
|
+
function testExemplifies(general, instance, grounding) {
|
|
237
|
+
const { rows, normFactTerm } = grounding;
|
|
238
|
+
const generalEntities = entitiesOf(general, grounding);
|
|
239
|
+
const instanceEntities = entitiesOf(instance, grounding);
|
|
236
240
|
for (const gA of generalEntities) {
|
|
237
241
|
const isClass = rows.some((r) => ISA_PREDICATES.has(r.predicate) && normFactTerm(r.object) === gA);
|
|
238
242
|
if (!isClass) continue;
|
|
@@ -257,20 +261,34 @@ function testExemplifies(general, instance, rows, graphTerms) {
|
|
|
257
261
|
* closed relations and includes a hit only when its test function fires.
|
|
258
262
|
*
|
|
259
263
|
* @param {Array<{id:string, members:Array<{id:string,text:string}>}>} groups
|
|
260
|
-
* @param {object} memory an already-loaded
|
|
261
|
-
* @param {object} [opts]
|
|
264
|
+
* @param {object} memory an already-loaded loadMemory() payload
|
|
265
|
+
* @param {object} [opts]
|
|
266
|
+
* @param {object} opts.store REQUIRED — the memory store's `{ readFactRows, normFactTerm,
|
|
267
|
+
* resolveRelationChase }` readers plus the block store's `{ tokenizeBlock }` helper
|
|
262
268
|
* @returns {Promise<Array<{from:string, to:string, relation:"supports"|"contradicts"|"elaborates"|"exemplifies", licensingTest:string, evidence:object}>>}
|
|
263
269
|
* deterministic: id-sorted pairwise order, stable-sorted by (from, to, relation).
|
|
264
270
|
*/
|
|
265
|
-
|
|
266
|
-
|
|
271
|
+
export async function inferRelations(groups, memory, { store } = {}) {
|
|
272
|
+
const { readFactRows, normFactTerm, resolveRelationChase, tokenizeBlock } = requireInjected(
|
|
273
|
+
store, ["readFactRows", "normFactTerm", "resolveRelationChase", "tokenizeBlock"],
|
|
274
|
+
{ caller: "inferRelations", option: "store" },
|
|
275
|
+
);
|
|
267
276
|
const list = Array.isArray(groups) ? groups.filter((g) => g && g.id && Array.isArray(g.members)) : [];
|
|
268
277
|
if (list.length < 2) return [];
|
|
269
278
|
|
|
270
279
|
const rows = readFactRows(memory);
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
const
|
|
280
|
+
// Everything the four licensing tests read, resolved once per call: the injected store
|
|
281
|
+
// handles, the loaded fact rows, and the indexes derived from them.
|
|
282
|
+
const grounding = {
|
|
283
|
+
memory,
|
|
284
|
+
rows,
|
|
285
|
+
normFactTerm,
|
|
286
|
+
resolveRelationChase,
|
|
287
|
+
contentTokens: makeContentTokens(tokenizeBlock),
|
|
288
|
+
graphTerms: buildGraphTerms(rows, normFactTerm),
|
|
289
|
+
helpers: makeHelpers(rows),
|
|
290
|
+
relationNames: relationNameCandidates(rows),
|
|
291
|
+
};
|
|
274
292
|
|
|
275
293
|
const sorted = list.slice().sort((x, y) => x.id.localeCompare(y.id));
|
|
276
294
|
const out = [];
|
|
@@ -281,13 +299,13 @@ export async function inferRelations(groups, memory, opts = {}) {
|
|
|
281
299
|
const B = sorted[j];
|
|
282
300
|
|
|
283
301
|
// eslint-disable-next-line no-await-in-loop -- deterministic fixed-order pairwise search
|
|
284
|
-
const sup = await testSupports(A, B,
|
|
302
|
+
const sup = await testSupports(A, B, grounding);
|
|
285
303
|
if (sup) out.push({ from: A.id, to: B.id, relation: "supports", licensingTest: sup.licensingTest, evidence: sup.evidence });
|
|
286
304
|
|
|
287
|
-
const con = testContradicts(A, B,
|
|
305
|
+
const con = testContradicts(A, B, grounding);
|
|
288
306
|
if (con) out.push({ from: A.id, to: B.id, relation: "contradicts", licensingTest: con.licensingTest, evidence: con.evidence });
|
|
289
307
|
|
|
290
|
-
const ela = testElaborates(A, B,
|
|
308
|
+
const ela = testElaborates(A, B, grounding);
|
|
291
309
|
if (ela) {
|
|
292
310
|
const from = ela.wider === "a" ? A.id : B.id;
|
|
293
311
|
const to = ela.wider === "a" ? B.id : A.id;
|
|
@@ -295,9 +313,9 @@ export async function inferRelations(groups, memory, opts = {}) {
|
|
|
295
313
|
}
|
|
296
314
|
|
|
297
315
|
// Both directions probed independently — different claims, each its own test.
|
|
298
|
-
const bExemplifiesA = testExemplifies(A, B,
|
|
316
|
+
const bExemplifiesA = testExemplifies(A, B, grounding);
|
|
299
317
|
if (bExemplifiesA) out.push({ from: B.id, to: A.id, relation: "exemplifies", licensingTest: bExemplifiesA.licensingTest, evidence: bExemplifiesA.evidence });
|
|
300
|
-
const aExemplifiesB = testExemplifies(B, A,
|
|
318
|
+
const aExemplifiesB = testExemplifies(B, A, grounding);
|
|
301
319
|
if (aExemplifiesB) out.push({ from: A.id, to: B.id, relation: "exemplifies", licensingTest: aExemplifiesB.licensingTest, evidence: aExemplifiesB.evidence });
|
|
302
320
|
}
|
|
303
321
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// completions/injected.mjs — the construction check every stage in this directory runs on
|
|
2
|
+
// the handles its caller passes in. The stages are pure: they read the memory store and the
|
|
3
|
+
// prose finisher through an explicit options bag, never by importing either. A missing
|
|
4
|
+
// handle is a loud construction error, never a silent no-op stage.
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @param {object} bag the caller-supplied handle bag (e.g. opts.store)
|
|
8
|
+
* @param {string[]} needed the names the caller must supply
|
|
9
|
+
* @param {{caller: string, option: string}} where the stage's name, and the option the bag
|
|
10
|
+
* arrives on — both quoted back in the error so a miswired call names itself
|
|
11
|
+
* @returns {object} `bag`, once every name is present
|
|
12
|
+
*/
|
|
13
|
+
export function requireInjected(bag, needed, { caller, option }) {
|
|
14
|
+
const missing = needed.filter((name) => bag?.[name] === undefined);
|
|
15
|
+
if (missing.length) {
|
|
16
|
+
throw new TypeError(
|
|
17
|
+
`${caller} needs a \`${option}\` option carrying { ${needed.join(", ")} } — missing ${missing.join(", ")}`,
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
return bag;
|
|
21
|
+
}
|
|
@@ -4,15 +4,15 @@
|
|
|
4
4
|
// way retrieveBlocks() fuses relevance/centrality/hub-dampening into one score.
|
|
5
5
|
// splitSentences() is a simple regex splitter.
|
|
6
6
|
|
|
7
|
-
import { degreeOf, rankBlocks, tokenizeBlock, OVERLAP_MIN } from "../memory/blocks.mjs";
|
|
8
7
|
import { STOPWORDS } from "../prose.mjs";
|
|
8
|
+
import { requireInjected } from "./injected.mjs";
|
|
9
9
|
|
|
10
10
|
// Same content-token filter group.mjs applies (not exported, so replicated here).
|
|
11
11
|
const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
|
|
12
12
|
|
|
13
13
|
/** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
|
|
14
|
-
function
|
|
15
|
-
return tokenizeBlock(text).filter(isContentToken);
|
|
14
|
+
function makeContentTokens(tokenizeBlock) {
|
|
15
|
+
return (text) => tokenizeBlock(text).filter(isContentToken);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
// Sentence boundary: a run of [.!?] followed by whitespace and an uppercase letter or digit
|
|
@@ -46,13 +46,20 @@ export function splitSentences(text) {
|
|
|
46
46
|
*
|
|
47
47
|
* @param {{members: Array<{id:string, text:string}>}} group a group.mjs groupHits() entry
|
|
48
48
|
* @param {object} [opts]
|
|
49
|
-
* @param {number} [opts.overlapMin
|
|
50
|
-
*
|
|
49
|
+
* @param {number} [opts.overlapMin] shared content-token threshold for a sentence-similarity
|
|
50
|
+
* edge; defaults to the store's own OVERLAP_MIN
|
|
51
51
|
* @param {string|null} [opts.query=null] optional query text to focus ranking on
|
|
52
|
+
* @param {object} opts.store REQUIRED — the block store's `{ degreeOf, rankBlocks,
|
|
53
|
+
* tokenizeBlock, OVERLAP_MIN }` ranking handles
|
|
52
54
|
* @returns {Array<{sentence:string, score:number, sourceBlockId:string}>} best-first;
|
|
53
55
|
* deterministic tiebreak (sourceBlockId, then sentence text) on equal score.
|
|
54
56
|
*/
|
|
55
|
-
export function rankSentences(group, { overlapMin
|
|
57
|
+
export function rankSentences(group, { overlapMin, query = null, store } = {}) {
|
|
58
|
+
const { degreeOf, rankBlocks, tokenizeBlock, OVERLAP_MIN } = requireInjected(
|
|
59
|
+
store, ["degreeOf", "rankBlocks", "tokenizeBlock", "OVERLAP_MIN"], { caller: "rankSentences", option: "store" },
|
|
60
|
+
);
|
|
61
|
+
const contentTokens = makeContentTokens(tokenizeBlock);
|
|
62
|
+
const edgeThreshold = overlapMin ?? OVERLAP_MIN;
|
|
56
63
|
const members = Array.isArray(group?.members) ? group.members : [];
|
|
57
64
|
if (!members.length) return [];
|
|
58
65
|
|
|
@@ -69,8 +76,8 @@ export function rankSentences(group, { overlapMin = OVERLAP_MIN, query = null }
|
|
|
69
76
|
const tokensById = {};
|
|
70
77
|
for (const s of sentences) tokensById[s.id] = contentTokens(s.sentence);
|
|
71
78
|
|
|
72
|
-
const ranks = rankBlocks(tokensById, { overlapMin });
|
|
73
|
-
const degrees = degreeOf(tokensById, { overlapMin });
|
|
79
|
+
const ranks = rankBlocks(tokensById, { overlapMin: edgeThreshold });
|
|
80
|
+
const degrees = degreeOf(tokensById, { overlapMin: edgeThreshold });
|
|
74
81
|
|
|
75
82
|
// IDF scoped to THIS group's sentence set (df/N), not the whole corpus.
|
|
76
83
|
const ids = Object.keys(tokensById);
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// and the graph search()/ask() services, asking both sources more widely than a single-answer
|
|
3
3
|
// query. Returns whole-block/whole-result hits only — no sub-block span segmentation.
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { requireInjected } from "./injected.mjs";
|
|
6
6
|
|
|
7
7
|
const DEFAULT_BLOCK_K = 8; // "broad" > chat's narrow single-answer k (typically 3)
|
|
8
8
|
const DEFAULT_GRAPH_LIMIT = 8;
|
|
@@ -20,14 +20,16 @@ const DEFAULT_GRAPH_LIMIT = 8;
|
|
|
20
20
|
* (e.g. createGraphService(graph)). When supplied, its search() and ask() are queried too;
|
|
21
21
|
* omitted -> block-only search.
|
|
22
22
|
* @param {number} [opts.graphLimit=8] graph search()'s result limit
|
|
23
|
+
* @param {object} opts.store REQUIRED — the block store's `{ retrieveBlocks }` reader
|
|
23
24
|
* @returns {Promise<Array<{source:"block"|"graph-search"|"graph-ask", id:string, text:string, score:number}>>}
|
|
24
25
|
* best-first within each source; blocks first, then graph-search, then graph-ask (never
|
|
25
26
|
* shuffled/merged by score across sources — block scores and graph relevance aren't
|
|
26
27
|
* comparable).
|
|
27
28
|
*/
|
|
28
29
|
export async function broadSearch(dir, query, {
|
|
29
|
-
blockK = DEFAULT_BLOCK_K, graphService = null, graphLimit = DEFAULT_GRAPH_LIMIT,
|
|
30
|
+
blockK = DEFAULT_BLOCK_K, graphService = null, graphLimit = DEFAULT_GRAPH_LIMIT, store,
|
|
30
31
|
} = {}) {
|
|
32
|
+
const { retrieveBlocks } = requireInjected(store, ["retrieveBlocks"], { caller: "broadSearch", option: "store" });
|
|
31
33
|
const q = String(query || "").trim();
|
|
32
34
|
if (!q) return [];
|
|
33
35
|
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
// docs/references/schemas/ace-owl-fragment.md and nothing more: fitting the
|
|
4
4
|
// grammar is a strong signal, missing it is a FEATURE — parseAce returns null
|
|
5
5
|
// (or an empty-triples result carrying the unknown words as `residue`) and the
|
|
6
|
-
// interpretation pipeline (src/interpret/) falls through to the tolerant
|
|
6
|
+
// interpretation pipeline (src/domain/interpret/) falls through to the tolerant
|
|
7
7
|
// strategies. No NLP dependency: tokenization is whitespace + trailing
|
|
8
8
|
// punctuation, morphology is the lexicon's suffix fold.
|
|
9
9
|
//
|
|
10
10
|
// parseAce(sentence, lexicon) → { pattern, triples, residue } | null
|
|
11
11
|
// pattern one of the PATTERNS below (also exported individually).
|
|
12
12
|
// triples [{ subject, predicate, object, kind, n? }] — OWL-labelled string
|
|
13
|
-
// triples shaped for src/memory/core.mjs's appendFact (which
|
|
13
|
+
// triples shaped for src/adapters/memory/core.mjs's appendFact (which
|
|
14
14
|
// normalizes subject/object via normFactTerm: "tmct:module" is
|
|
15
15
|
// stored as "module"; the predicate keeps its vocabulary casing).
|
|
16
16
|
// residue [] on a clean parse; the unknown tokens when the sentence FITS a
|
|
@@ -56,7 +56,7 @@ export const PATTERNS = Object.freeze([
|
|
|
56
56
|
const DET = new Set(["a", "an", "the"]);
|
|
57
57
|
// A token SHAPED like a code reference (a path, file, symbol or CURIE) is an
|
|
58
58
|
// individual by form — a deterministic tokenizer rule, not a guess: declared
|
|
59
|
-
// proper names cover words; this covers chat.mjs, src/ask.mjs, Foo#bar.
|
|
59
|
+
// proper names cover words; this covers chat.mjs, src/domain/ask.mjs, Foo#bar.
|
|
60
60
|
const CODE_REF = /[./\\#:@]/;
|
|
61
61
|
|
|
62
62
|
/** Whitespace tokenizer: curly quotes normalized, commas/semicolons dropped,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// grammar/assert.mjs — the grammar→memory bridge: parseAce a sentence and land
|
|
2
|
-
// every emitted triple in tmct's OWN memory graph via
|
|
3
|
-
//
|
|
2
|
+
// every emitted triple in tmct's OWN memory graph via an INJECTED appendFact
|
|
3
|
+
// (memory/core.mjs's, in the live wiring — the grammar never imports the
|
|
4
|
+
// store).
|
|
4
5
|
//
|
|
5
6
|
// appendFact normalizes each triple's subject/object through normFactTerm
|
|
6
7
|
// (tmct:Legacy-module → "legacy-module"; the predicate keeps its vocabulary
|
|
@@ -10,7 +11,6 @@
|
|
|
10
11
|
// ("ace:chat:<sessionId>@<ts>"); core.mjs unions tags "|"-joined when several
|
|
11
12
|
// writers assert the same fact.
|
|
12
13
|
|
|
13
|
-
import { appendFact } from "../memory/core.mjs";
|
|
14
14
|
import { parseAce } from "./ace.mjs";
|
|
15
15
|
import { loadLexicon } from "./lexicon.mjs";
|
|
16
16
|
|
|
@@ -21,11 +21,15 @@ export function provenanceTag({ source = "chat", sessionId = "", ts = "" } = {})
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
/** Parse `sentence` against the ACE-OWL sub-fragment and append every emitted
|
|
24
|
-
* triple to the memory graph under `dir
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
|
|
24
|
+
* triple to the memory graph under `dir`, via the REQUIRED injected
|
|
25
|
+
* `appendFact` (memory/core.mjs's, in the live wiring). Returns the parse
|
|
26
|
+
* result extended with `ids` (one fact id per triple, same order) and the
|
|
27
|
+
* provenance tag — or null (grammar miss, nothing written), or the residue
|
|
28
|
+
* parse (unknown words: triples empty, ids empty, nothing written). */
|
|
29
|
+
export async function assertSentence(dir, sentence, { lexicon, provenance, appendFact } = {}) {
|
|
30
|
+
if (typeof appendFact !== "function") {
|
|
31
|
+
throw new TypeError("assertSentence needs an appendFact option (memory/core.mjs's writer) — the grammar never imports the store");
|
|
32
|
+
}
|
|
29
33
|
const parse = parseAce(sentence, lexicon ?? loadLexicon());
|
|
30
34
|
if (!parse) return null;
|
|
31
35
|
const tag = provenanceTag(provenance);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"comment": "tmct's starter lexicon (ROADMAP Phase 2, item 2), now covering BOTH the original software domain and (PLAN_SEED.md, the default 'human-world' persona) everyday people/places/objects/nature/time/body/mind vocabulary. Every word the ACE-OWL sub-fragment parser (src/grammar/ace.mjs) is allowed to understand is DECLARED here — tmct never guesses a word's category. Nouns may declare a possessive property typing ('data' or 'object', pattern 7) or an irregular 'plural' (WordNet-sourced human nouns like 'man'/'child' need this — the morphology fold only handles regular -s/-ies/-es endings); adjectives MUST declare a type ('subclass' forms a class, 'data' asserts a boolean-ish datatype property, pattern 8); verbs may declare a preposition ('depend' + 'on' → dependsOn). The human-domain additions are flat and simple (breadth over depth — no gender/kinship taxonomy, no formal role hierarchy), matching corpus/tier2/generate.mjs's CORPUSES.human.lexicon sub-key, which --verify drift-checks against. Extend via loadLexicon(extra) with this same shape.",
|
|
2
|
+
"comment": "tmct's starter lexicon (ROADMAP Phase 2, item 2), now covering BOTH the original software domain and (PLAN_SEED.md, the default 'human-world' persona) everyday people/places/objects/nature/time/body/mind vocabulary. Every word the ACE-OWL sub-fragment parser (src/domain/grammar/ace.mjs) is allowed to understand is DECLARED here — tmct never guesses a word's category. Nouns may declare a possessive property typing ('data' or 'object', pattern 7) or an irregular 'plural' (WordNet-sourced human nouns like 'man'/'child' need this — the morphology fold only handles regular -s/-ies/-es endings); adjectives MUST declare a type ('subclass' forms a class, 'data' asserts a boolean-ish datatype property, pattern 8); verbs may declare a preposition ('depend' + 'on' → dependsOn). The human-domain additions are flat and simple (breadth over depth — no gender/kinship taxonomy, no formal role hierarchy), matching corpus/tier2/generate.mjs's CORPUSES.human.lexicon sub-key, which --verify drift-checks against. Extend via loadLexicon(extra) with this same shape.",
|
|
3
3
|
"nouns": {
|
|
4
4
|
"module": {},
|
|
5
5
|
"class": {},
|
|
@@ -20,11 +20,9 @@
|
|
|
20
20
|
// irregular `plural` ("indices"). Anything the fold can't reach is simply not
|
|
21
21
|
// in the lexicon — honest, not clever.
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
import
|
|
25
|
-
import
|
|
26
|
-
|
|
27
|
-
const CORE_FILE = join(dirname(fileURLToPath(import.meta.url)), "lexicon-core.json");
|
|
23
|
+
// The core vocabulary rides in as a JSON module — declarative data, no
|
|
24
|
+
// filesystem read at import or call time, and the browser bundler inlines it.
|
|
25
|
+
import coreLexiconRaw from "./lexicon-core.json" with { type: "json" };
|
|
28
26
|
|
|
29
27
|
/** The CURIE namespace every tmct lexicon mints terms under. */
|
|
30
28
|
export const DEFAULT_NS = "tmct:";
|
|
@@ -133,7 +131,7 @@ const coreCacheByNs = new Map();
|
|
|
133
131
|
* cached per-ns (the JSON is committed, immutable at runtime). */
|
|
134
132
|
export function loadLexicon(extra, ns = DEFAULT_NS) {
|
|
135
133
|
if (!extra && coreCacheByNs.has(ns)) return coreCacheByNs.get(ns);
|
|
136
|
-
const raw =
|
|
134
|
+
const raw = coreLexiconRaw;
|
|
137
135
|
const lex = {
|
|
138
136
|
nouns: new Map(),
|
|
139
137
|
nounPlurals: new Map(),
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// hash.mjs — the single home for tmct's content-address contract: the FNV-1a
|
|
2
|
+
// hash, the fact-term/predicate normalization, and the fact-id derivation.
|
|
3
|
+
//
|
|
4
|
+
// FNV-1a 32-bit is deliberately home-grown: it must be synchronous, browser-safe,
|
|
5
|
+
// dependency-free, and — critically — CROSS-VERSION STABLE, because fact ids are
|
|
6
|
+
// content-addressed by it and a fact's id is its identity across the whole memory
|
|
7
|
+
// graph. Every library candidate fails at least one of those; this eight-line
|
|
8
|
+
// function fails none. Normalization lives beside it because it is PART of that
|
|
9
|
+
// identity: the same (s, p, o) must normalize and hash to the same id from every
|
|
10
|
+
// writer, so the whole contract has exactly one definition.
|
|
11
|
+
|
|
12
|
+
/** FNV-1a 32-bit. Returns the unsigned 32-bit integer (0 … 2^32−1). */
|
|
13
|
+
export function fnv1a32(str) {
|
|
14
|
+
let h = 0x811c9dc5;
|
|
15
|
+
for (let i = 0; i < str.length; i += 1) {
|
|
16
|
+
h ^= str.charCodeAt(i);
|
|
17
|
+
h = Math.imul(h, 0x01000193);
|
|
18
|
+
}
|
|
19
|
+
return h >>> 0;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** FNV-1a 32-bit as a zero-padded 8-char hex string — the stable content-address
|
|
23
|
+
* used for fact ids (`fact:<hex>`). Same (s,p,o) → same id → upsert, never a dup. */
|
|
24
|
+
export function fnv1aHex(str) {
|
|
25
|
+
return fnv1a32(str).toString(16).padStart(8, "0");
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// SHA-256 (FIPS 180-4), home-grown for the same reasons FNV-1a is: it must be
|
|
29
|
+
// synchronous, browser-safe, dependency-free, and cross-version stable —
|
|
30
|
+
// answer-variant selection is keyed on digest bytes, so the digest is part of
|
|
31
|
+
// the answer-text contract. Pinned byte-identical to node:crypto by
|
|
32
|
+
// test/adapters/hash-digest.test.mjs.
|
|
33
|
+
const SHA256_K = new Uint32Array([
|
|
34
|
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
|
35
|
+
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
|
36
|
+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
|
37
|
+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
|
38
|
+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
|
39
|
+
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
40
|
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
|
41
|
+
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
|
+
/** SHA-256 of a string (UTF-8), as a 32-byte Uint8Array. */
|
|
45
|
+
export function sha256Bytes(str) {
|
|
46
|
+
const data = new TextEncoder().encode(String(str));
|
|
47
|
+
const bitLen = data.length * 8;
|
|
48
|
+
const padded = new Uint8Array((((data.length + 8) >> 6) + 1) << 6);
|
|
49
|
+
padded.set(data);
|
|
50
|
+
padded[data.length] = 0x80;
|
|
51
|
+
const dv = new DataView(padded.buffer);
|
|
52
|
+
dv.setUint32(padded.length - 8, Math.floor(bitLen / 0x100000000));
|
|
53
|
+
dv.setUint32(padded.length - 4, bitLen >>> 0);
|
|
54
|
+
|
|
55
|
+
const h = new Uint32Array([0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19]);
|
|
56
|
+
const w = new Uint32Array(64);
|
|
57
|
+
const rotr = (x, n) => (x >>> n) | (x << (32 - n));
|
|
58
|
+
|
|
59
|
+
for (let off = 0; off < padded.length; off += 64) {
|
|
60
|
+
for (let i = 0; i < 16; i += 1) w[i] = dv.getUint32(off + i * 4);
|
|
61
|
+
for (let i = 16; i < 64; i += 1) {
|
|
62
|
+
const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ (w[i - 15] >>> 3);
|
|
63
|
+
const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ (w[i - 2] >>> 10);
|
|
64
|
+
w[i] = (w[i - 16] + s0 + w[i - 7] + s1) >>> 0;
|
|
65
|
+
}
|
|
66
|
+
let [a, b, c, d, e, f, g, hh] = h;
|
|
67
|
+
for (let i = 0; i < 64; i += 1) {
|
|
68
|
+
const S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
|
69
|
+
const ch = (e & f) ^ (~e & g);
|
|
70
|
+
const t1 = (hh + S1 + ch + SHA256_K[i] + w[i]) >>> 0;
|
|
71
|
+
const S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
|
72
|
+
const maj = (a & b) ^ (a & c) ^ (b & c);
|
|
73
|
+
const t2 = (S0 + maj) >>> 0;
|
|
74
|
+
hh = g; g = f; f = e; e = (d + t1) >>> 0; d = c; c = b; b = a; a = (t1 + t2) >>> 0;
|
|
75
|
+
}
|
|
76
|
+
h[0] = (h[0] + a) >>> 0; h[1] = (h[1] + b) >>> 0; h[2] = (h[2] + c) >>> 0; h[3] = (h[3] + d) >>> 0;
|
|
77
|
+
h[4] = (h[4] + e) >>> 0; h[5] = (h[5] + f) >>> 0; h[6] = (h[6] + g) >>> 0; h[7] = (h[7] + hh) >>> 0;
|
|
78
|
+
}
|
|
79
|
+
const out = new Uint8Array(32);
|
|
80
|
+
const ov = new DataView(out.buffer);
|
|
81
|
+
for (let i = 0; i < 8; i += 1) ov.setUint32(i * 4, h[i]);
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const TEXT_CAP = 2000; // an utterance's stored text (a whole answer fits; a pasted book doesn't)
|
|
86
|
+
|
|
87
|
+
/** Whitespace-collapse + cap a stored text/predicate string. Every writer that
|
|
88
|
+
* feeds the fact-id hash uses THIS normalization, so id identity never depends
|
|
89
|
+
* on which module did the writing. */
|
|
90
|
+
export const normText = (t) => String(t ?? "").replace(/\s+/g, " ").trim().slice(0, TEXT_CAP);
|
|
91
|
+
|
|
92
|
+
/** Normalize a fact TERM (subject/object) so every writer converges on one
|
|
93
|
+
* spelling: ConceptNet's /c/en/foo_bar, tmct:Foo_bar, and bare "Foo bar" all
|
|
94
|
+
* become "foo bar". Also strips a leading "the"/"a"/"an" (idempotent — safe
|
|
95
|
+
* for storage too). The predicate is never normalized this way — its casing
|
|
96
|
+
* is meaningful controlled vocabulary. */
|
|
97
|
+
export function normFactTerm(t) {
|
|
98
|
+
let s = normText(t);
|
|
99
|
+
s = s.replace(/^\/c\/[a-z]{2,3}\//i, "");
|
|
100
|
+
s = s.replace(/^[a-z][\w.-]*:/i, "");
|
|
101
|
+
s = s.replace(/_/g, " ").replace(/\s+/g, " ").trim();
|
|
102
|
+
s = s.replace(/^(?:the|an?)\s+/i, "");
|
|
103
|
+
return s.toLowerCase();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The closed taught→corpus predicate table. The teach lane mints a predicate
|
|
107
|
+
* from the verb's LEMMA ("fire causes smoke" → mgx:cause), while the corpus
|
|
108
|
+
* vocabulary spells the same relation in its curated form (mgx:causes). Left
|
|
109
|
+
* apart, a taught fact and the corpus fact for the same triple are two facts
|
|
110
|
+
* with two ids, and neither reader finds the other's. Each row below folds a
|
|
111
|
+
* minted lemma onto the curated predicate that means the SAME thing in the
|
|
112
|
+
* SAME direction, so both spellings converge on one fact id.
|
|
113
|
+
*
|
|
114
|
+
* Closed on purpose — a table, not an inflection rule. Entries are added when
|
|
115
|
+
* a curated predicate's surface phrase is a plain verb ("causes", "wants",
|
|
116
|
+
* "requires", "involves") whose lemma the teach lane would otherwise mint
|
|
117
|
+
* unrelated. Relations that invert their arguments (mgx:ownedBy,
|
|
118
|
+
* mgx:createdBy) are absent: "X owns Y" and "X is owned by Y" are not the
|
|
119
|
+
* same fact, so folding them would store a lie. */
|
|
120
|
+
const CANONICAL_FACT_PREDICATE = new Map([
|
|
121
|
+
["mgx:cause", "mgx:causes"],
|
|
122
|
+
["mgx:desire", "mgx:desires"],
|
|
123
|
+
["mgx:want", "mgx:desires"],
|
|
124
|
+
["mgx:require", "mgx:hasPrerequisite"],
|
|
125
|
+
["mgx:involve", "mgx:hasSubevent"],
|
|
126
|
+
]);
|
|
127
|
+
|
|
128
|
+
/** Normalize a fact PREDICATE: whitespace-collapse + cap (normText's storage
|
|
129
|
+
* contract), then fold a minted lemma onto its curated corpus spelling. Casing
|
|
130
|
+
* is left alone — a predicate is controlled vocabulary, not a term. */
|
|
131
|
+
export function normFactPredicate(p) {
|
|
132
|
+
const t = normText(p);
|
|
133
|
+
return CANONICAL_FACT_PREDICATE.get(t) ?? t;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** A Fact is content-addressed by its NUL-delimited (s, p, o) — NUL never
|
|
137
|
+
* occurs in a normalized term/predicate, so it's collision-proof unlike a
|
|
138
|
+
* space. Takes ALREADY-normalized parts; factIdForTriple normalizes first. */
|
|
139
|
+
export const factIdFor = (s, p, o) => `fact:${fnv1aHex(`${s}\0${p}\0${o}`)}`;
|
|
140
|
+
|
|
141
|
+
/** Content-address a fact's id from (subject, predicate, object) without
|
|
142
|
+
* writing it — same contract as factIdFor. Lets a caller (e.g.
|
|
143
|
+
* syllogise.mjs's retraction machinery) name a not-yet-written fact's id
|
|
144
|
+
* deterministically, without an extra read. Pure, no I/O. */
|
|
145
|
+
export function factIdForTriple(subject, predicate, object) {
|
|
146
|
+
return factIdFor(normFactTerm(subject), normFactPredicate(predicate), normFactTerm(object));
|
|
147
|
+
}
|
|
@@ -3,6 +3,16 @@
|
|
|
3
3
|
// refused or surfaced as ambiguity, never broken by a guess.
|
|
4
4
|
|
|
5
5
|
import { VERB_TO_KIND, ENTITY_TO_TYPE, MODIFIER_TO_KIND } from "../ask-vocab.mjs";
|
|
6
|
+
import collisionData from "../real-word-collisions.json" with { type: "json" };
|
|
7
|
+
|
|
8
|
+
/** Words the LEMMA pass must leave alone even though no parse table spells them.
|
|
9
|
+
* "used" reads as vocabulary, not as a relation: "what is it used for" asks the
|
|
10
|
+
* corpus what a thing is FOR, and the lemma tier would otherwise walk it through
|
|
11
|
+
* "use" to the graph verb "uses" and answer a question about imports and calls
|
|
12
|
+
* instead. The active verb table deliberately has no bare "used" for the same
|
|
13
|
+
* reason. The fuzzy tier reaches "used" too, by one edit, and the collision
|
|
14
|
+
* table below holds it off there — "used" is real English, so no repair applies. */
|
|
15
|
+
const NEVER_CANONICALIZE = ["used"];
|
|
6
16
|
import { STOPWORDS } from "./normalize.mjs";
|
|
7
17
|
|
|
8
18
|
// ---- bounded edit distance — hand-rolled Damerau-Levenshtein, bounded with an
|
|
@@ -38,19 +48,40 @@ export const fuzzyBound = (s) => (s.length <= 5 ? 1 : 2);
|
|
|
38
48
|
* already vocabulary?" gate for the lemma/fuzzy canonicalization passes (an
|
|
39
49
|
* exact vocab word is NEVER rewritten: exact curated match always wins). */
|
|
40
50
|
export const VOCAB_WORDS = new Set(
|
|
41
|
-
[...Object.keys(VERB_TO_KIND), ...Object.keys(ENTITY_TO_TYPE), ...Object.keys(MODIFIER_TO_KIND)
|
|
51
|
+
[...Object.keys(VERB_TO_KIND), ...Object.keys(ENTITY_TO_TYPE), ...Object.keys(MODIFIER_TO_KIND),
|
|
52
|
+
...NEVER_CANONICALIZE]
|
|
42
53
|
.flatMap((p) => p.split(" ")),
|
|
43
54
|
);
|
|
44
55
|
|
|
56
|
+
/** Below this length the edit budget covers half of English, so the repair tier
|
|
57
|
+
* ignores shorter words entirely ("and" is one edit from the "land in"
|
|
58
|
+
* constituent). Shared with the generator that builds the collision table, so
|
|
59
|
+
* both agree on which words the tier can ever reach. */
|
|
60
|
+
export const FUZZY_REPAIR_MIN_LENGTH = 4;
|
|
61
|
+
|
|
45
62
|
/** Fuzzy-correction TARGETS: verb-phrase and modifier constituents only,
|
|
46
63
|
* length ≥4. Entity nouns and short words are excluded — real identifiers
|
|
47
64
|
* collide with them too easily at this distance bound. */
|
|
48
|
-
const FUZZY_TARGET_WORDS = [...new Set(
|
|
65
|
+
export const FUZZY_TARGET_WORDS = [...new Set(
|
|
49
66
|
[...Object.keys(VERB_TO_KIND), ...Object.keys(MODIFIER_TO_KIND)]
|
|
50
67
|
.flatMap((p) => p.split(" "))
|
|
51
|
-
.filter((w) => w.length >=
|
|
68
|
+
.filter((w) => w.length >= FUZZY_REPAIR_MIN_LENGTH),
|
|
52
69
|
)];
|
|
53
70
|
|
|
71
|
+
/** Real English words the repair tier would otherwise rewrite onto a graph verb
|
|
72
|
+
* ("rest" -> "test", "during" -> "using", "bigger" -> "trigger"). Generated
|
|
73
|
+
* offline from this repo's own WordNet corpus by
|
|
74
|
+
* scripts/generate-real-word-collisions.mjs and committed, so the check is a
|
|
75
|
+
* closed table lookup with no dictionary to carry at runtime. */
|
|
76
|
+
const REAL_WORD_COLLISIONS = new Set(collisionData.words);
|
|
77
|
+
|
|
78
|
+
/** Is `w` a real English word that the repair tier attracts? A word already in
|
|
79
|
+
* the language is not a typo, so no repair applies to it — the sentence means
|
|
80
|
+
* what it says, and if we don't record that relation the honest answer is a
|
|
81
|
+
* miss. Only the words the tier could actually reach are tabled; anything else
|
|
82
|
+
* never gets this far. */
|
|
83
|
+
export const isRealEnglishWord = (w) => REAL_WORD_COLLISIONS.has(w);
|
|
84
|
+
|
|
54
85
|
/** A query word may be canonicalized only if it is plain alphabetic, not a
|
|
55
86
|
* stopword, and not already vocabulary. Dotted/digit terms (file names, shas)
|
|
56
87
|
* are never touched. */
|
|
@@ -60,8 +91,15 @@ export function eligibleForCanon(w) {
|
|
|
60
91
|
|
|
61
92
|
/** UNIQUE within-bound fuzzy vocab keyword for `w`, or null — a tie between two
|
|
62
93
|
* distinct target words at the same distance is refused outright (the honest-miss
|
|
63
|
-
* discipline at the vocabulary level; cf. MISSPELLINGS' curated "calss" decision).
|
|
94
|
+
* discipline at the vocabulary level; cf. MISSPELLINGS' curated "calss" decision).
|
|
95
|
+
*
|
|
96
|
+
* This tier repairs TYPOS. A word that is already real English is not a typo,
|
|
97
|
+
* so it is refused before the distance is even measured: "does store.mjs rest
|
|
98
|
+
* on app.mjs" asks about resting, and rewriting it to "tests" answers a
|
|
99
|
+
* question nobody asked. A word outside the language ("impotr") has no such
|
|
100
|
+
* claim, and the repair is the whole point. */
|
|
64
101
|
export function fuzzyVocabWord(w) {
|
|
102
|
+
if (isRealEnglishWord(w)) return null;
|
|
65
103
|
return fuzzyMatchInSet(w, FUZZY_TARGET_WORDS, fuzzyBound(w));
|
|
66
104
|
}
|
|
67
105
|
|