@polycode-projects/the-mechanical-code-talker 1.9.2 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +441 -202
- package/bin/tmct.mjs +126 -1
- package/package.json +4 -2
- package/src/answer-variants.mjs +8 -36
- package/src/ask-browser-entry.mjs +5 -23
- package/src/ask-browser.bundle.js +1 -2
- package/src/ask-nlp.mjs +9 -23
- package/src/ask-vocab.mjs +139 -589
- package/src/ask.mjs +627 -1729
- package/src/chat.mjs +1684 -2872
- package/src/cli-args.mjs +14 -28
- package/src/codegraph.mjs +236 -644
- package/src/completions/complete.mjs +18 -62
- package/src/completions/graph-adapter.mjs +14 -60
- package/src/completions/group.mjs +12 -68
- package/src/completions/infer.mjs +38 -126
- package/src/completions/prune.mjs +17 -70
- package/src/completions/rank.mjs +16 -69
- package/src/completions/search.mjs +8 -31
- package/src/concept.mjs +32 -88
- package/src/conformance.mjs +11 -15
- package/src/corpus/conceptnet.mjs +31 -89
- package/src/corpus/templates.mjs +19 -45
- package/src/corpus/unknown-ingest.mjs +31 -92
- package/src/embed.mjs +10 -22
- package/src/extensions.mjs +50 -154
- package/src/finish.mjs +35 -91
- package/src/grammar/ace.mjs +16 -40
- package/src/grammar/assert.mjs +1 -1
- package/src/grammar/lexicon-core.json +1 -1
- package/src/grammar/lexicon.mjs +9 -27
- package/src/graph-merge.mjs +2 -3
- package/src/hash.mjs +6 -14
- package/src/index.mjs +6 -10
- package/src/init.mjs +38 -125
- package/src/interpret/fuzzy.mjs +10 -29
- package/src/interpret/merge.mjs +9 -27
- package/src/interpret/normalize.mjs +137 -585
- package/src/interpret/pipeline.mjs +23 -71
- package/src/interpret/strategies/ace.mjs +7 -31
- package/src/interpret/strategies/constructions.mjs +14 -41
- package/src/interpret/strategies/grammar.mjs +21 -60
- package/src/interpret/strategies/keywords.mjs +42 -131
- package/src/interpret/strategies/noise-strip.mjs +18 -89
- package/src/memory/bias.mjs +11 -54
- package/src/memory/blocks.mjs +18 -69
- package/src/memory/core.mjs +171 -591
- package/src/memory/fold.mjs +0 -0
- package/src/memory/inspect.mjs +7 -25
- package/src/memory/shacl.mjs +10 -39
- package/src/memory/trust.mjs +26 -127
- package/src/memory-ask-browser-entry.mjs +7 -30
- package/src/memory-ask-browser.bundle.js +1 -1
- package/src/paraphrase.mjs +20 -53
- package/src/planning.mjs +15 -157
- package/src/prose-nlp.mjs +4 -17
- package/src/prose.mjs +19 -67
- package/src/providers/bootstrap.mjs +1 -2
- package/src/providers/fixture.mjs +1 -2
- package/src/providers/graph-service.mjs +28 -59
- package/src/repository-interface.mjs +6 -8
- package/src/router/drive.mjs +183 -0
- package/src/router/goal-reasoner.mjs +66 -231
- package/src/router/guardrail.mjs +20 -58
- package/src/router/planner.mjs +15 -46
- package/src/router/registry.mjs +13 -43
- package/src/router/resolver.mjs +46 -131
- package/src/router/results.mjs +231 -0
- package/src/schema-docs.mjs +10 -27
- package/src/server-http.mjs +10 -19
- package/src/server.mjs +22 -28
- package/src/sessions.mjs +15 -30
- package/src/source-slice.mjs +5 -7
- package/src/source.mjs +10 -20
- package/src/syllogise.mjs +187 -575
- package/src/telemetry.mjs +3 -3
- package/src/toml-config.mjs +4 -4
- package/src/tui/app.mjs +9 -19
- package/src/viz.mjs +66 -123
- package/src/wink-model.mjs +10 -24
|
@@ -1,34 +1,6 @@
|
|
|
1
|
-
// completions/complete.mjs — the
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// sentences into prose -> finish()'s grammar/voice pass (Stage 6). §4's staging table, row 3:
|
|
5
|
-
// "Stage 5+6 — pruning + grammar/voice pass wired end to end... Stage 6 reuses finish.mjs
|
|
6
|
-
// directly... Exit criterion: A full end-to-end completion reads as one consistent voice and
|
|
7
|
-
// every sentence traces to a source span."
|
|
8
|
-
//
|
|
9
|
-
// Extractive, by construction (PLAN_COMPLETIONS.md §3's honest ceiling): the assembled text is
|
|
10
|
-
// the KEPT sentences, in order, joined by a single space — never paraphrased, never reordered
|
|
11
|
-
// beyond what pruning's own group/rank order already decided, never smoothed over a genuine
|
|
12
|
-
// disjointedness in the source material. Stage 6 fixes CAPITALISATION and PUNCTUATION at every
|
|
13
|
-
// sentence boundary (this dispatch's own generalisation of src/finish.mjs's ruleCapitalise/
|
|
14
|
-
// ruleTerminal); it does not invent, reorder, or merge any sentence's WORDS. Where the source
|
|
15
|
-
// material simply doesn't chain into a fluent read, the output reads disjointedly and stays
|
|
16
|
-
// honest about it — that is the plan's own accepted, documented limit, not a bug this module
|
|
17
|
-
// tries to paper over.
|
|
18
|
-
//
|
|
19
|
-
// Traceability: `sourceSpans` is built directly from pruneCompletion()'s own `kept` list (which
|
|
20
|
-
// already carries sourceBlockId + groupId per sentence) BEFORE the grammar pass runs, and the
|
|
21
|
-
// grammar pass never reorders, drops, or merges sentences (only mutates casing/punctuation of
|
|
22
|
-
// prose text) — so sourceSpans stays index-aligned with "one entry per output sentence" even
|
|
23
|
-
// though the FINAL text is produced by finish(), not directly from `kept`. `relations` is
|
|
24
|
-
// infer.mjs's own output, untouched by pruning (relations are never pruned, only the sentences
|
|
25
|
-
// that anchor them are decided) — every relation still carries its own `licensingTest`/
|
|
26
|
-
// `evidence` per infer.mjs's contract.
|
|
27
|
-
//
|
|
28
|
-
// Determinism: every stage this module chains is already deterministic (broadSearch, groupHits,
|
|
29
|
-
// rankSentences, inferRelations, pruneCompletion, finish() are all pure/deterministic functions
|
|
30
|
-
// over their inputs); this module adds no randomness of its own. See
|
|
31
|
-
// test/completions-complete.test.mjs's own double-run diff.
|
|
1
|
+
// completions/complete.mjs — the mechanical-text-generation pipeline: broadSearch ->
|
|
2
|
+
// groupHits -> rankSentences + inferRelations -> pruneCompletion -> assemble -> finish()'s
|
|
3
|
+
// grammar pass. Extractive only: output is the kept sentences, joined, never paraphrased.
|
|
32
4
|
|
|
33
5
|
import { broadSearch } from "./search.mjs";
|
|
34
6
|
import { groupHits } from "./group.mjs";
|
|
@@ -38,15 +10,9 @@ import { pruneCompletion } from "./prune.mjs";
|
|
|
38
10
|
import { loadMemory } from "../memory/core.mjs";
|
|
39
11
|
import { finish, grammarRules } from "../finish.mjs";
|
|
40
12
|
|
|
41
|
-
/** grammarRules() with sentence-capitalisation
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* goldens pinning lowercase openers) — that is a CHAT-voice decision, not a limitation of the
|
|
45
|
-
* rule itself. A genuinely multi-sentence extractive completion has no such single-opener
|
|
46
|
-
* voice to protect and NEEDS every internal sentence boundary capitalised to read as one
|
|
47
|
-
* consistent voice (this pipeline's own exit criterion) — so this module's own Stage 6 call
|
|
48
|
-
* overrides just that one rule's enabled flag, via finish()'s ctx.rules seam, without touching
|
|
49
|
-
* the cached global table or any other caller's behaviour. */
|
|
13
|
+
/** grammarRules() with sentence-capitalisation force-enabled (disabled in live chat only to
|
|
14
|
+
* protect single-answer lowercase-opener goldens, which don't apply to a multi-sentence
|
|
15
|
+
* completion). */
|
|
50
16
|
function completionGrammarRules() {
|
|
51
17
|
return grammarRules().map((r) => (r.id === "sentence-capitalisation" ? { ...r, enabled: true } : r));
|
|
52
18
|
}
|
|
@@ -65,16 +31,13 @@ const DEFAULT_MAX_SENTENCES_PER_GROUP = 3; // see prune.mjs's own file header fo
|
|
|
65
31
|
* passed straight through to broadSearch (see search.mjs)
|
|
66
32
|
* @param {number} [opts.graphLimit] broadSearch's graph search() limit
|
|
67
33
|
* @param {number} [opts.overlapMin] groupHits' shared-token edge threshold
|
|
68
|
-
* @param {object} [opts.memory] an already-loaded memory/core.mjs loadMemory() payload;
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* itself (query-focused summarization, PLAN_COMPLETIONS.md §1.4's own literature framing) —
|
|
73
|
-
* pass `null` explicitly to fall back to self-weighted (LexRank-style) ranking instead
|
|
34
|
+
* @param {object} [opts.memory] an already-loaded memory/core.mjs loadMemory() payload;
|
|
35
|
+
* defaults to loadMemory(dir)
|
|
36
|
+
* @param {string} [opts.query] the query rankSentences()/pruning focus on; defaults to `prompt`;
|
|
37
|
+
* pass `null` for self-weighted (LexRank-style) ranking instead
|
|
74
38
|
* @param {number} [opts.maxSentencesPerGroup=3] prune.mjs's top-K-per-group cutoff
|
|
75
39
|
* @param {object} [opts.graph] optional loaded graph (src/codegraph.mjs parseEntities() shape)
|
|
76
|
-
* handed to finish()'s maskSegments
|
|
77
|
-
* protected during the grammar pass — the same ctx.graph existing finish() call sites pass
|
|
40
|
+
* handed to finish()'s maskSegments to protect known entity labels during the grammar pass
|
|
78
41
|
* @returns {Promise<{
|
|
79
42
|
* text: string,
|
|
80
43
|
* sourceSpans: Array<{sourceBlockId:string, groupId:string, sentence:string}>,
|
|
@@ -97,8 +60,7 @@ export async function generateCompletion(dir, prompt, opts = {}) {
|
|
|
97
60
|
// Stage 2 — grouping
|
|
98
61
|
const groups = groupHits(hits, { overlapMin });
|
|
99
62
|
|
|
100
|
-
// Stage 3 — cross-group inference
|
|
101
|
-
// simply asserts nothing, never fabricates something to compensate)
|
|
63
|
+
// Stage 3 — cross-group inference
|
|
102
64
|
const memory = memoryOpt || await loadMemory(dir);
|
|
103
65
|
const relations = groups.length >= 2 ? await inferRelations(groups, memory) : [];
|
|
104
66
|
|
|
@@ -113,25 +75,19 @@ export async function generateCompletion(dir, prompt, opts = {}) {
|
|
|
113
75
|
);
|
|
114
76
|
|
|
115
77
|
if (!kept.length) {
|
|
116
|
-
// Honest decline
|
|
117
|
-
//
|
|
78
|
+
// Honest decline: nothing cleared the pruning bar for this prompt over this corpus — never
|
|
79
|
+
// fabricate a completion to fill the gap.
|
|
118
80
|
return { text: "", sourceSpans: [], relations, dropped, declined: true, reason: "no source span cleared the pruning bar for this prompt" };
|
|
119
81
|
}
|
|
120
82
|
|
|
121
|
-
// Assemble: kept sentences, in pruneCompletion's own
|
|
122
|
-
// single space — purely extractive, no reordering/paraphrasing beyond that (§3's honest
|
|
123
|
-
// ceiling; see file header).
|
|
83
|
+
// Assemble: kept sentences, in pruneCompletion's own order, joined by a single space.
|
|
124
84
|
const rawText = kept.map((k) => k.sentence).join(" ");
|
|
125
85
|
|
|
126
|
-
// Stage 6 — grammar/voice pass
|
|
127
|
-
// protects any known entity/path/number/receipt/provenance span WITHIN the assembled text,
|
|
128
|
-
// exactly as it does for a single-answer composed answer); only the rule table differs
|
|
129
|
-
// (sentence-capitalisation force-enabled — see completionGrammarRules() above).
|
|
86
|
+
// Stage 6 — grammar/voice pass (reuses finish() verbatim; see completionGrammarRules() above).
|
|
130
87
|
const finished = finish({ answer: rawText, via: "completion" }, { graph, rules: completionGrammarRules() });
|
|
131
88
|
|
|
132
|
-
// sourceSpans traces every
|
|
133
|
-
//
|
|
134
|
-
// grammar rules only mutate casing/punctuation of prose, never reorder/drop/merge sentences.
|
|
89
|
+
// sourceSpans traces every output sentence back to its block id + group id, from `kept`
|
|
90
|
+
// (pre-grammar-pass) — stays index-aligned since finish() only mutates casing/punctuation.
|
|
135
91
|
const sourceSpans = kept.map((k) => ({ sourceBlockId: k.sourceBlockId, groupId: k.groupId, sentence: k.sentence }));
|
|
136
92
|
|
|
137
93
|
return { text: finished.answer, sourceSpans, relations, dropped };
|
|
@@ -1,61 +1,20 @@
|
|
|
1
|
-
// completions/graph-adapter.mjs —
|
|
2
|
-
//
|
|
3
|
-
// docblock names createGraphService(graph) (src/providers/graph-service.mjs) as the
|
|
4
|
-
// reference shape — but until now nothing in live chat ever constructed and passed one
|
|
5
|
-
// through (src/chat.mjs's completionsRescueAnswer called generateCompletion() with no
|
|
6
|
-
// `graphService` at all). Without it, broadSearch could only ever see memory BLOCKS
|
|
7
|
-
// saved via an explicit saveBlock() call — never the already-loaded code graph, and
|
|
8
|
-
// never a taught Fact — so "give me a detailed summary of how X works" declined for any
|
|
9
|
-
// subject on its first real mention in a session, no matter how much the graph or
|
|
10
|
-
// taught Facts actually knew about it.
|
|
1
|
+
// completions/graph-adapter.mjs — a graphService-shaped adapter for broadSearch(), wrapping
|
|
2
|
+
// two already-loaded stores (graph, memory) without re-loading either from disk.
|
|
11
3
|
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
// the same ranked lexical module/symbol search every other Repository-Interface
|
|
18
|
-
// consumer uses (src/codegraph.mjs's searchModulesRanked/scoreSymbolsRanked under
|
|
19
|
-
// the hood). No new search machinery.
|
|
20
|
-
//
|
|
21
|
-
// - .ask(q) does NOT delegate to createGraphService(graph).ask() (src/ask.mjs) —
|
|
22
|
-
// that engine is a mechanical NATURAL-LANGUAGE QUESTION grammar ("which functions
|
|
23
|
-
// call X", "what does X import"), and broadSearch always calls .ask() with the
|
|
24
|
-
// bare SUBJECT TERM itself ("TaskController"), not a question. Tried live: that
|
|
25
|
-
// produces an honest but useless "couldn't parse this as a graph question"
|
|
26
|
-
// rephrase-hint every time — real text, but not about the subject, and it would
|
|
27
|
-
// pollute the completion with noise. Instead .ask() here builds real sentences
|
|
28
|
-
// from two sources that a bare term CAN resolve against directly:
|
|
29
|
-
// 1. resolveSymbol + renderDescribe (src/codegraph.mjs) — the SAME graph-only
|
|
30
|
-
// renderer src/server.mjs's own tmct_describe tool uses: real facts (defining
|
|
31
|
-
// module, contains, inherits, calls, tests, attributes, …), never invented.
|
|
32
|
-
// 2. readFactRows(memory) (src/memory/core.mjs) — any TAUGHT Fact whose subject
|
|
33
|
-
// or object mentions the term. This is the one source the pipeline had NO
|
|
34
|
-
// path to before at all: Stage 3 (inferRelations) only ever augments groups
|
|
35
|
-
// that already exist from Stage 1's hits, so a subject with real taught Facts
|
|
36
|
-
// but zero blocks/code-graph hits still surfaced nothing.
|
|
37
|
-
// svc.ask() is still tried last, but its content is kept ONLY when it genuinely
|
|
38
|
-
// parsed (tmct_ask.miss === false) — e.g. the rare case where the bare term happens
|
|
39
|
-
// to also be a real registered question shape — never its own rephrase-hint noise.
|
|
40
|
-
//
|
|
41
|
-
// Every sentence this adapter returns traces to a real graph edge/attribute or a real
|
|
42
|
-
// taught Fact — never invented, matching src/completions/'s extractive-only discipline
|
|
43
|
-
// (see complete.mjs's own file header).
|
|
4
|
+
// .ask(q) does not delegate to createGraphService(graph).ask(): that expects a natural-
|
|
5
|
+
// language question, but broadSearch calls .ask() with a bare term, which would only produce
|
|
6
|
+
// a "couldn't parse" rephrase-hint. Instead it builds sentences from resolveSymbol/
|
|
7
|
+
// renderDescribe and readFactRows(memory), falling back to svc.ask() only when it genuinely
|
|
8
|
+
// parsed the term.
|
|
44
9
|
|
|
45
10
|
import { createGraphService } from "../providers/graph-service.mjs";
|
|
46
11
|
import { resolveSymbol, renderDescribe } from "../codegraph.mjs";
|
|
47
12
|
import { readFactRows } from "../memory/core.mjs";
|
|
48
13
|
|
|
49
|
-
/** renderDescribe() renders one
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* joins kept sentences with a single space — so two adjacent kept lines without a
|
|
54
|
-
* period between them would otherwise read as one run-on clause. Ensuring every line
|
|
55
|
-
* ends in terminal punctuation here (never rewording/reordering the line itself) is
|
|
56
|
-
* the cheapest fix that stays entirely inside this adapter, touching neither
|
|
57
|
-
* renderDescribe() (server.mjs's tmct_describe tool relies on its current line shape)
|
|
58
|
-
* nor rank.mjs/complete.mjs's own join logic. */
|
|
14
|
+
/** renderDescribe() renders one line per fact with no terminal punctuation of its own;
|
|
15
|
+
* rank.mjs's splitSentences() treats each line as its own candidate sentence, and
|
|
16
|
+
* complete.mjs joins kept sentences with a single space, so two adjacent lines without
|
|
17
|
+
* a period between them would otherwise read as one run-on clause. */
|
|
59
18
|
function withTerminalPunctuation(text) {
|
|
60
19
|
return String(text || "")
|
|
61
20
|
.split("\n")
|
|
@@ -66,14 +25,9 @@ function withTerminalPunctuation(text) {
|
|
|
66
25
|
}
|
|
67
26
|
|
|
68
27
|
/**
|
|
69
|
-
* @param {object|null} graph a parseEntities() result
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* @param {object|null} [memory=null] a loadMemory() payload (src/memory/core.mjs), or
|
|
73
|
-
* null when there's no Fact store to search — the Fact-half of ask() then honestly
|
|
74
|
-
* contributes nothing.
|
|
75
|
-
* @returns {{search: Function, ask: Function}} a Repository-Interface-shaped
|
|
76
|
-
* graphService satisfying src/completions/search.mjs's broadSearch() contract.
|
|
28
|
+
* @param {object|null} graph a parseEntities() result, or null when no code graph is loaded
|
|
29
|
+
* @param {object|null} [memory=null] a loadMemory() payload, or null when there's no Fact store
|
|
30
|
+
* @returns {{search: Function, ask: Function}} a Repository-Interface-shaped graphService
|
|
77
31
|
*/
|
|
78
32
|
export function createCompletionsGraphAdapter(graph, memory = null) {
|
|
79
33
|
const svc = graph ? createGraphService(graph) : null;
|
|
@@ -1,59 +1,15 @@
|
|
|
1
|
-
// completions/group.mjs — Stage 2 ("grouping")
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// happens here.
|
|
6
|
-
//
|
|
7
|
-
// Genuinely greenfield, as the plan's own prior research established: PLAN_COMPLETIONS.md's
|
|
8
|
-
// citation of "insights-panel tag clustering" as prior art turned out to be a different,
|
|
9
|
-
// sibling project's capability, not tmct's — there was no clustering/tag-grouping code
|
|
10
|
-
// anywhere in this repo before this file.
|
|
11
|
-
//
|
|
12
|
-
// Granularity: this module clusters at BLOCK granularity — the exact unit search.mjs's
|
|
13
|
-
// broadSearch() hands it (one node per whole retrieveBlocks hit, or one node per whole
|
|
14
|
-
// graph-search/ask result). PLAN_COMPLETIONS.md's own prose loosely says "spans"; that is
|
|
15
|
-
// resolved explicitly here rather than silently assumed: no sub-block span segmentation is
|
|
16
|
-
// attempted in this build. A finer-grained span-level increment, if the pipeline ever needs
|
|
17
|
-
// it, is future work, out of scope for Stage 0.
|
|
18
|
-
//
|
|
19
|
-
// Algorithm: CONNECTED COMPONENTS over the block-similarity graph (shared-token-overlap
|
|
20
|
-
// edges), reusing memory/blocks.mjs's buildNeighbours()/OVERLAP_MIN — the exact adjacency
|
|
21
|
-
// rankBlocks()/degreeOf() already build for PageRank, exported from blocks.mjs for this
|
|
22
|
-
// purpose rather than re-derived. This is the strategy-advisor's explicit Stage-0
|
|
23
|
-
// recommendation: "the cheapest deterministic starting point" for grouping — genuinely
|
|
24
|
-
// CoRank-adjacent (CoRank = "clustering cum graph ranking", PLAN_COMPLETIONS.md §1.2) since
|
|
25
|
-
// it clusters over the same graph-ranking substrate blocks.mjs's PageRank runs on, and fully
|
|
26
|
-
// deterministic/auditable, unlike any softer/fuzzier clustering. Richer ranking WITHIN a
|
|
27
|
-
// cluster (e.g. running rankBlocks per component, or a true joint clustering+ranking pass)
|
|
28
|
-
// is a future increment if the pipeline needs it later — not attempted here.
|
|
29
|
-
//
|
|
30
|
-
// Labeling: each group carries its member ids AND a human-inspectable label — its top
|
|
31
|
-
// shared-IDF tokens. The only term-frequency machinery this repo ships is the
|
|
32
|
-
// idf = log(1 + N/(1+df)) formula memory/blocks.mjs's retrieveBlocks() already computes
|
|
33
|
-
// inline (there is no standalone exported idf() to import — it lives inline in
|
|
34
|
-
// retrieveBlocks, scoped to a single query's tokens); that exact formula is replicated here,
|
|
35
|
-
// scoped instead to df/N over the hit set passed into groupHits(), rather than inventing a
|
|
36
|
-
// new weighting scheme.
|
|
37
|
-
//
|
|
38
|
-
// Determinism: no randomness anywhere. Same hits in (same ids, same text, same order) ->
|
|
39
|
-
// same groups out (same partition, same member order, same label) — this is exactly what
|
|
40
|
-
// test/completions-stage0.test.mjs's double-run diff asserts as Stage 0's exit criterion.
|
|
1
|
+
// completions/group.mjs — Stage 2 ("grouping"): clusters search.mjs's broadSearch() hits
|
|
2
|
+
// into topical groups via connected components over a shared-token-overlap graph (reusing
|
|
3
|
+
// memory/blocks.mjs's buildNeighbours()/OVERLAP_MIN). Block granularity, no sub-block spans.
|
|
4
|
+
// Each group's label is its top shared-IDF tokens (df/N over the hit set, not the corpus).
|
|
41
5
|
|
|
42
6
|
import { buildNeighbours, OVERLAP_MIN, tokenizeBlock } from "../memory/blocks.mjs";
|
|
43
7
|
import { STOPWORDS } from "../prose.mjs";
|
|
44
8
|
|
|
45
9
|
const LABEL_TOKEN_COUNT = 5;
|
|
46
10
|
|
|
47
|
-
// tokenizeBlock
|
|
48
|
-
//
|
|
49
|
-
// a stopword-shaped fragment is rare, and it doesn't strip trailing sentence punctuation
|
|
50
|
-
// either). For prose-similarity clustering that union re-admits filler ("the", "and", "is",
|
|
51
|
-
// "q:", "a:", "do?", …) as if it were real overlap signal — harmless for retrieveBlocks'
|
|
52
|
-
// IDF-weighted single-query scoring (rare query tokens still dominate), but fatal for
|
|
53
|
-
// unweighted shared-token-overlap clustering: almost any two English sentences share ≥2
|
|
54
|
-
// stopwords, which would collapse everything into one giant component. isContentToken()
|
|
55
|
-
// closes that gap for THIS module's own adjacency/labeling use, without changing
|
|
56
|
-
// tokenizeBlock's shipped behavior (still exactly what retrieveBlocks/rankBlocks use).
|
|
11
|
+
// tokenizeBlock's raw output re-admits stopwords, which would collapse almost any two
|
|
12
|
+
// sentences into one giant component under unweighted overlap clustering; filter them here.
|
|
57
13
|
const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
|
|
58
14
|
|
|
59
15
|
/** tokenizeBlock(text), narrowed to real content tokens (see isContentToken above) — the
|
|
@@ -85,24 +41,17 @@ function unionFind(n) {
|
|
|
85
41
|
*
|
|
86
42
|
* @param {Array<{id:string, text:string}>} hits
|
|
87
43
|
* @param {object} [opts]
|
|
88
|
-
* @param {number} [opts.overlapMin=OVERLAP_MIN] shared-token threshold for a similarity
|
|
89
|
-
* edge — defaults to the exact value memory/blocks.mjs's own PageRank graph uses, so
|
|
90
|
-
* grouping and ranking agree on what "related" means unless the caller deliberately
|
|
91
|
-
* overrides it.
|
|
44
|
+
* @param {number} [opts.overlapMin=OVERLAP_MIN] shared-token threshold for a similarity edge
|
|
92
45
|
* @returns {Array<{ id: string, members: Array<{id:string, text:string}>, memberIds: string[],
|
|
93
46
|
* tokens: string[], label: string }>}
|
|
94
|
-
* One entry per connected component (a singleton hit
|
|
95
|
-
*
|
|
96
|
-
* order: groups sorted by their lowest member id; members within a group sorted by id.
|
|
97
|
-
* `id` is the group's own stable id (`"g:" + memberIds[0]`, deterministic from content).
|
|
47
|
+
* One entry per connected component (a singleton hit is still a one-member group).
|
|
48
|
+
* Deterministic order: groups by lowest member id; members within a group by id.
|
|
98
49
|
*/
|
|
99
50
|
export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
|
|
100
51
|
const list = Array.isArray(hits) ? hits.filter((h) => h && h.id != null) : [];
|
|
101
52
|
if (!list.length) return [];
|
|
102
53
|
|
|
103
|
-
// dedupe by id —
|
|
104
|
-
// and a graph-ask answer both id-tagged distinctly, but a caller re-running broadSearch
|
|
105
|
-
// for an overlapping prompt set could still hand duplicate ids); first occurrence wins.
|
|
54
|
+
// dedupe by id — first occurrence wins.
|
|
106
55
|
const seen = new Set();
|
|
107
56
|
const deduped = [];
|
|
108
57
|
for (const h of list) {
|
|
@@ -128,9 +77,7 @@ export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
|
|
|
128
77
|
componentIdx.get(root).push(i);
|
|
129
78
|
}
|
|
130
79
|
|
|
131
|
-
// IDF over THIS hit set (df/N
|
|
132
|
-
// Stage 1 actually retrieved for this prompt, same discipline retrieveBlocks applies to a
|
|
133
|
-
// single query's tokens.
|
|
80
|
+
// IDF over THIS hit set (df/N), not the whole corpus.
|
|
134
81
|
const N = ids.length;
|
|
135
82
|
const df = new Map();
|
|
136
83
|
for (const id of ids) {
|
|
@@ -145,10 +92,7 @@ export function groupHits(hits, { overlapMin = OVERLAP_MIN } = {}) {
|
|
|
145
92
|
.sort((a, b) => a.id.localeCompare(b.id));
|
|
146
93
|
const memberIds = members.map((m) => m.id);
|
|
147
94
|
|
|
148
|
-
// label tokens: rank by
|
|
149
|
-
// a group's label prefers what its members have IN COMMON over what one member merely
|
|
150
|
-
// contains a lot of — then (b) IDF (rarer across the hit set wins ties), then (c) token
|
|
151
|
-
// text for a fully deterministic order.
|
|
95
|
+
// label tokens: rank by member coverage, then IDF, then token text (deterministic).
|
|
152
96
|
const coverage = new Map();
|
|
153
97
|
for (const i of memberIdx) {
|
|
154
98
|
for (const t of new Set(tokensById[ids[i]])) coverage.set(t, (coverage.get(t) || 0) + 1);
|
|
@@ -1,56 +1,19 @@
|
|
|
1
|
-
// completions/infer.mjs — Stage 3 ("inference between groups")
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// set" — this file, plus test/completions-infer.test.mjs's hand-labeled fixture, is that exit
|
|
7
|
-
// criterion made concrete.
|
|
1
|
+
// completions/infer.mjs — Stage 3 ("inference between groups"): applies resolveRelationChase
|
|
2
|
+
// (src/memory/core.mjs) to relationships BETWEEN retrieved text groups, not just graph facts.
|
|
3
|
+
// Four relations (supports/contradicts/elaborates/exemplifies), each with its own named
|
|
4
|
+
// licensing test — see the four test*() functions below. A relation is asserted only when its
|
|
5
|
+
// test concretely licenses it, never from prose similarity.
|
|
8
6
|
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
// vocabulary, deliberately mirroring marginalia's own TYPED_EDGES closed set... A relationship
|
|
12
|
-
// between two groups is only asserted when a concrete, named test licenses it... never
|
|
13
|
-
// inferred by prose similarity alone" (PLAN_COMPLETIONS.md §1.3). Four relations, each with
|
|
14
|
-
// its own mechanical, named licensing test — see the four test*() functions below, one per
|
|
15
|
-
// relation, each documented at its own definition:
|
|
16
|
-
//
|
|
17
|
-
// supports — resolveRelationChase (src/memory/core.mjs, PLAN_COMPLETIONS.md Stage 1's
|
|
18
|
-
// own prerequisite extraction) confirms a taught relation fact between two
|
|
19
|
-
// entities both groups' text share.
|
|
20
|
-
// contradicts — the two groups' text carry OPPOSITE negation polarity around the SAME
|
|
21
|
-
// shared graph-known entity + shared content token (token-level, closed
|
|
22
|
-
// negation-marker set — no graph fact required).
|
|
23
|
-
// elaborates — one group's graph-known entity set is a PROPER SUBSET of the other's (the
|
|
24
|
-
// wider group elaborates the narrower one).
|
|
25
|
-
// exemplifies — one group names a class-level term (something else is taught
|
|
26
|
-
// rdfs:subClassOf/rdf:type it — checkable via the SAME memory/core.mjs-loaded
|
|
27
|
-
// fact rows), and the other group names a taught INSTANCE of that class.
|
|
28
|
-
//
|
|
29
|
-
// "Entity" grounding: a group's raw content tokens (tokenizeBlock, the same tokenizer
|
|
30
|
-
// group.mjs/rank.mjs use, filtered the same isContentToken way those two files already
|
|
31
|
-
// establish) are narrowed to GRAPH-KNOWN terms only — tokens that normFactTerm-match some
|
|
32
|
-
// fact's subject or object in the loaded memory. This is the concrete grounding that keeps
|
|
33
|
-
// "group A and group B share an entity" a checkable graph fact, not a prose-similarity guess:
|
|
34
|
-
// two groups merely using the same English word never licenses anything on its own unless
|
|
35
|
-
// that word is itself a taught term.
|
|
36
|
-
//
|
|
37
|
-
// Determinism: no randomness anywhere. Groups are processed in a fixed (id-sorted) pairwise
|
|
38
|
-
// order; every internal token/entity set is turned into a sorted array before use; every
|
|
39
|
-
// per-relation test returns at most one hit per (group pair, relation), picked by that fixed
|
|
40
|
-
// order — never "first of an unordered Set/Map iteration". See
|
|
41
|
-
// test/completions-infer.test.mjs's own double-run diff test, the same discipline
|
|
42
|
-
// test/completions-stage0.test.mjs and test/completions-stage2.test.mjs already apply to
|
|
43
|
-
// search+group and to sentence ranking.
|
|
7
|
+
// Entities are a group's content tokens narrowed to graph-known terms (normFactTerm-matched
|
|
8
|
+
// against loaded facts) — sharing an English word alone never licenses a relation.
|
|
44
9
|
|
|
45
10
|
import { normFactTerm, readFactRows, resolveRelationChase } from "../memory/core.mjs";
|
|
46
11
|
import { tokenizeBlock } from "../memory/blocks.mjs";
|
|
47
12
|
import { splitSentences } from "./rank.mjs";
|
|
48
13
|
import { STOPWORDS } from "../prose.mjs";
|
|
49
14
|
|
|
50
|
-
// Same content-token filter group.mjs/rank.mjs
|
|
51
|
-
// exported from either, so replicated here rather than reached across files
|
|
52
|
-
// file's own header for why raw tokenizeBlock output, which re-admits stopword-shaped filler,
|
|
53
|
-
// is unsuitable for unweighted set operations like the ones this file runs).
|
|
15
|
+
// Same content-token filter group.mjs/rank.mjs apply to their own adjacency/ranking; not
|
|
16
|
+
// exported from either, so replicated here rather than reached across files.
|
|
54
17
|
const isContentToken = (t) => /^[a-z0-9]+$/.test(t) && !STOPWORDS.has(t);
|
|
55
18
|
|
|
56
19
|
/** tokenizeBlock(text), narrowed to real content tokens — see isContentToken above. */
|
|
@@ -58,25 +21,15 @@ function contentTokens(text) {
|
|
|
58
21
|
return tokenizeBlock(text).filter(isContentToken);
|
|
59
22
|
}
|
|
60
23
|
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
// resolveRelationChase's own unit tests (test/memory-core.test.mjs) establish the precedent of
|
|
64
|
-
// a caller supplying its own minimal, self-contained copy in its `helpers` bag rather than
|
|
65
|
-
// reaching into chat.mjs (out of scope for this dispatch) for the shared constant.
|
|
24
|
+
// Local copy of chat.mjs's private HAS_PROPERTY_PREDICATE constant — not exported, so callers
|
|
25
|
+
// supply their own copy in the `helpers` bag resolveRelationChase expects.
|
|
66
26
|
const HAS_PROPERTY_PREDICATE = "mgx:hasProperty";
|
|
67
27
|
|
|
68
|
-
// The taught ISA-family predicates (
|
|
69
|
-
// this same pair, elsewhere) — the exemplifies test's "checkable via the taught IsA/subClassOf
|
|
70
|
-
// graph" per PLAN_COMPLETIONS.md §1.3, read directly off readFactRows() rows rather than via
|
|
71
|
-
// syllogise.mjs's fuller OWL 2 RL machinery (out of scope for this dispatch; this file only
|
|
72
|
-
// needs the STORED isa edges, not their transitive closure).
|
|
28
|
+
// The taught ISA-family predicates (stored edges only, not their transitive closure).
|
|
73
29
|
const ISA_PREDICATES = new Set(["rdfs:subClassOf", "rdf:type"]);
|
|
74
30
|
|
|
75
|
-
// The contradicts test's closed negation-marker vocabulary
|
|
76
|
-
//
|
|
77
|
-
// Deliberately checked against the RAW sentence text, not contentTokens()'s output — prose.mjs's
|
|
78
|
-
// STOPWORDS (which isContentToken filters through) already strips "not"/"no" as filler for
|
|
79
|
-
// clustering/ranking purposes, which would silently erase the exact signal this test needs.
|
|
31
|
+
// The contradicts test's closed negation-marker vocabulary, checked against RAW sentence
|
|
32
|
+
// text (contentTokens() strips "not"/"no" as stopword filler, which would erase this signal).
|
|
80
33
|
const NEGATION_MARKERS = new Set([
|
|
81
34
|
"not", "no", "never", "cannot", "none", "nobody", "nothing", "neither", "nor", "without",
|
|
82
35
|
]);
|
|
@@ -127,30 +80,21 @@ function buildGraphTerms(rows) {
|
|
|
127
80
|
return set;
|
|
128
81
|
}
|
|
129
82
|
|
|
130
|
-
/** A group's GRAPH-GROUNDED entities:
|
|
131
|
-
* that are themselves graph-known terms (buildGraphTerms' universe) — sorted for determinism.
|
|
132
|
-
* This is the concrete grounding test/completions-infer.test.mjs's fixture exercises: two
|
|
133
|
-
* groups merely sharing an English word (e.g. both saying "abstraction") never counts unless
|
|
134
|
-
* that word is itself a taught fact term. */
|
|
83
|
+
/** A group's GRAPH-GROUNDED entities: content tokens narrowed to graph-known terms, sorted. */
|
|
135
84
|
function entitiesOf(group, graphTerms) {
|
|
136
85
|
return [...groupContentTokenSet(group)].filter((t) => graphTerms.has(t)).sort();
|
|
137
86
|
}
|
|
138
87
|
|
|
139
|
-
/** A minimal
|
|
140
|
-
*
|
|
141
|
-
* test/memory-core.test.mjs's own testRelationFactsFor precedent exactly: resolveRelationChase
|
|
142
|
-
* never calls the alias substrate itself, that lives entirely inside whatever relationFactsFor
|
|
143
|
-
* the caller supplies, and a direct-only implementation is an honest, valid, simpler one — no
|
|
144
|
-
* alias-chase claim is made or needed for the supports test's own licensing standard. */
|
|
88
|
+
/** A minimal relationFactsFor(name): direct-predicate match only (`mgx:${name}`), no
|
|
89
|
+
* alias/subClassOf chase — sufficient for this test's licensing standard. */
|
|
145
90
|
function makeRelationFactsFor(rows) {
|
|
146
91
|
return (name) => rows
|
|
147
92
|
.filter((f) => f.predicate === `mgx:${name}`)
|
|
148
93
|
.map((f) => ({ fact: f, aliasFacts: [] }));
|
|
149
94
|
}
|
|
150
95
|
|
|
151
|
-
/** The resolveRelationChase/resolveRelationChaseReverse `helpers` bag this file supplies
|
|
152
|
-
*
|
|
153
|
-
* inferRelations() call over the loaded memory's fact rows. */
|
|
96
|
+
/** The resolveRelationChase/resolveRelationChaseReverse `helpers` bag this file supplies,
|
|
97
|
+
* built once per inferRelations() call over the loaded memory's fact rows. */
|
|
154
98
|
function makeHelpers(rows) {
|
|
155
99
|
return {
|
|
156
100
|
relationFactsFor: makeRelationFactsFor(rows),
|
|
@@ -163,10 +107,8 @@ function makeHelpers(rows) {
|
|
|
163
107
|
};
|
|
164
108
|
}
|
|
165
109
|
|
|
166
|
-
/** Every distinct relation NAME
|
|
167
|
-
*
|
|
168
|
-
* (a property-literal marker, not a relation name) — a closed, corpus-derived candidate list,
|
|
169
|
-
* never an open-ended guess at what "a relation" might be named. Sorted for determinism. */
|
|
110
|
+
/** Every distinct relation NAME present among the loaded facts (`mgx:<name>` predicates,
|
|
111
|
+
* minus HAS_PROPERTY_PREDICATE, a property marker not a relation name). Sorted. */
|
|
170
112
|
function relationNameCandidates(rows) {
|
|
171
113
|
const names = new Set();
|
|
172
114
|
for (const r of rows) {
|
|
@@ -177,15 +119,9 @@ function relationNameCandidates(rows) {
|
|
|
177
119
|
return [...names].sort();
|
|
178
120
|
}
|
|
179
121
|
|
|
180
|
-
/**
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
* PLAN_COMPLETIONS.md Stage 1's own prerequisite extraction) confirms a claim connecting two of
|
|
184
|
-
* those shared entities. Tries every (subject, object) ordered pair drawn from the shared
|
|
185
|
-
* entity set, against every candidate relation name, in fixed sorted order; the first hit
|
|
186
|
-
* (deterministic given fixed inputs) is the one asserted. Returns
|
|
187
|
-
* `{ licensingTest, evidence }` or null.
|
|
188
|
-
*/
|
|
122
|
+
/** SUPPORTS — groups A/B share >=2 graph-grounded entities AND a taught relation fact
|
|
123
|
+
* connects two of them. Tries every (subject, object) pair against every candidate relation
|
|
124
|
+
* name in fixed sorted order; first hit wins. Returns `{ licensingTest, evidence }` or null. */
|
|
189
125
|
async function testSupports(a, b, memory, helpers, relationNames, graphTerms) {
|
|
190
126
|
const entitiesA = entitiesOf(a, graphTerms);
|
|
191
127
|
const entitiesB = entitiesOf(b, graphTerms);
|
|
@@ -209,14 +145,9 @@ async function testSupports(a, b, memory, helpers, relationNames, graphTerms) {
|
|
|
209
145
|
return null;
|
|
210
146
|
}
|
|
211
147
|
|
|
212
|
-
/**
|
|
213
|
-
*
|
|
214
|
-
*
|
|
215
|
-
* one sentence on each side — and one side's co-occurring sentence carries a closed-set
|
|
216
|
-
* negation marker while the other side's does not (opposite polarity about the same claim,
|
|
217
|
-
* PLAN_COMPLETIONS.md §1.3's own "simpler token-level polarity check"). Returns
|
|
218
|
-
* `{ licensingTest, evidence }` or null.
|
|
219
|
-
*/
|
|
148
|
+
/** CONTRADICTS — groups A/B share a graph-grounded entity plus a second co-occurring token
|
|
149
|
+
* ("aspect"), and one side's matching sentence is negated while the other's isn't. Returns
|
|
150
|
+
* `{ licensingTest, evidence }` or null. */
|
|
220
151
|
function testContradicts(a, b, graphTerms) {
|
|
221
152
|
const entitiesA = entitiesOf(a, graphTerms);
|
|
222
153
|
const entitiesB = entitiesOf(b, graphTerms);
|
|
@@ -271,14 +202,9 @@ function isProperSubset(small, big) {
|
|
|
271
202
|
return small.size > 0 && small.size < big.size && [...small].every((t) => big.has(t));
|
|
272
203
|
}
|
|
273
204
|
|
|
274
|
-
/**
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
* the narrower group's entities do, plus more. Equal entity sets never count (neither is a
|
|
278
|
-
* *proper* subset of the other) — two groups about exactly the same entities are not in an
|
|
279
|
-
* elaboration relationship by this test. Returns `{ wider: "a"|"b", licensingTest, evidence }`
|
|
280
|
-
* or null.
|
|
281
|
-
*/
|
|
205
|
+
/** ELABORATES — one group's graph-grounded entity set is a PROPER SUBSET of the other's; the
|
|
206
|
+
* wider group elaborates the narrower one. Equal sets never count. Returns
|
|
207
|
+
* `{ wider: "a"|"b", licensingTest, evidence }` or null. */
|
|
282
208
|
function testElaborates(a, b, graphTerms) {
|
|
283
209
|
const entitiesA = new Set(entitiesOf(a, graphTerms));
|
|
284
210
|
const entitiesB = new Set(entitiesOf(b, graphTerms));
|
|
@@ -300,15 +226,10 @@ function testElaborates(a, b, graphTerms) {
|
|
|
300
226
|
return null;
|
|
301
227
|
}
|
|
302
228
|
|
|
303
|
-
/**
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
* general). Asymmetric and directional by construction: `instance`'s group exemplifies
|
|
308
|
-
* `general`'s group, never the reverse in the same call — callers probe both directions by
|
|
309
|
-
* calling this twice with the groups swapped (see inferRelations below). Returns
|
|
310
|
-
* `{ licensingTest, evidence }` or null.
|
|
311
|
-
*/
|
|
229
|
+
/** EXEMPLIFIES — `general` names a class-level term (object of an ISA_PREDICATES edge), and
|
|
230
|
+
* `instance` names an entity taught to BE one directly. Asymmetric: callers probe both
|
|
231
|
+
* directions by calling this twice with groups swapped (see inferRelations below). Returns
|
|
232
|
+
* `{ licensingTest, evidence }` or null. */
|
|
312
233
|
function testExemplifies(general, instance, rows, graphTerms) {
|
|
313
234
|
const generalEntities = entitiesOf(general, graphTerms);
|
|
314
235
|
const instanceEntities = entitiesOf(instance, graphTerms);
|
|
@@ -332,22 +253,14 @@ function testExemplifies(general, instance, rows, graphTerms) {
|
|
|
332
253
|
}
|
|
333
254
|
|
|
334
255
|
/**
|
|
335
|
-
* Stage 3 — cross-group inference. For every unordered pair of groups
|
|
336
|
-
*
|
|
337
|
-
* relations (supports/contradicts/elaborates/exemplifies) via its own concrete, named,
|
|
338
|
-
* mechanical licensing test — never prose similarity. A relation only appears in the output
|
|
339
|
-
* when its own test function returns a hit; every hit carries `licensingTest` (a human-
|
|
340
|
-
* readable description of exactly what fired) and `evidence` (the concrete facts/tokens cited)
|
|
341
|
-
* — PLAN_COMPLETIONS.md §2's auditability bar ("every cross-group claim must cite the
|
|
342
|
-
* two-or-more groups and the inference kind that licensed it").
|
|
256
|
+
* Stage 3 — cross-group inference. For every unordered pair of groups, tests each of the four
|
|
257
|
+
* closed relations and includes a hit only when its test function fires.
|
|
343
258
|
*
|
|
344
259
|
* @param {Array<{id:string, members:Array<{id:string,text:string}>}>} groups
|
|
345
260
|
* @param {object} memory an already-loaded memory/core.mjs loadMemory() payload
|
|
346
261
|
* @param {object} [opts] reserved for future tuning; unused today
|
|
347
262
|
* @returns {Promise<Array<{from:string, to:string, relation:"supports"|"contradicts"|"elaborates"|"exemplifies", licensingTest:string, evidence:object}>>}
|
|
348
|
-
* deterministic:
|
|
349
|
-
* additionally stable-sorted by (from, to, relation) so output order never depends on
|
|
350
|
-
* incidental iteration order anywhere upstream.
|
|
263
|
+
* deterministic: id-sorted pairwise order, stable-sorted by (from, to, relation).
|
|
351
264
|
*/
|
|
352
265
|
// eslint-disable-next-line no-unused-vars -- opts reserved, see docblock
|
|
353
266
|
export async function inferRelations(groups, memory, opts = {}) {
|
|
@@ -381,8 +294,7 @@ export async function inferRelations(groups, memory, opts = {}) {
|
|
|
381
294
|
out.push({ from, to, relation: "elaborates", licensingTest: ela.licensingTest, evidence: ela.evidence });
|
|
382
295
|
}
|
|
383
296
|
|
|
384
|
-
// Both directions probed independently —
|
|
385
|
-
// genuinely different claims, each licensed (or not) by its own class/instance test.
|
|
297
|
+
// Both directions probed independently — different claims, each its own test.
|
|
386
298
|
const bExemplifiesA = testExemplifies(A, B, rows, graphTerms);
|
|
387
299
|
if (bExemplifiesA) out.push({ from: B.id, to: A.id, relation: "exemplifies", licensingTest: bExemplifiesA.licensingTest, evidence: bExemplifiesA.evidence });
|
|
388
300
|
const aExemplifiesB = testExemplifies(B, A, rows, graphTerms);
|