@polycode-projects/the-mechanical-code-talker 4.1.1 → 4.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -18
- package/bin/tmct.mjs +3 -0
- package/data/templates/responses.jsonl +3 -0
- package/package.json +2 -1
- package/src/adapters/memory/core.mjs +1358 -196
- package/src/adapters/memory/inspect.mjs +11 -0
- package/src/adapters/memory/shacl.mjs +38 -0
- package/src/adapters/p2p/webrtc-transport.mjs +28 -5
- package/src/domain/ask-vocab.mjs +39 -0
- package/src/domain/ask.mjs +183 -34
- package/src/domain/grammar/assert.mjs +8 -2
- package/src/domain/hanoi-board.mjs +232 -0
- package/src/domain/ingest-facts.mjs +120 -0
- package/src/domain/interpret/normalize.mjs +49 -0
- package/src/domain/memory/compaction.mjs +284 -0
- package/src/domain/memory/resolution.mjs +171 -0
- package/src/domain/memory/trust.mjs +175 -5
- package/src/domain/memory-facts.mjs +139 -0
- package/src/domain/p2p/facts.mjs +21 -0
- package/src/domain/p2p/peer-id.mjs +15 -0
- package/src/domain/p2p/provenance-relabel.mjs +13 -2
- package/src/domain/p2p/sync-filter.mjs +5 -1
- package/src/domain/p2p/wire.mjs +7 -4
- package/src/domain/scene-compose.mjs +2 -2
- package/src/domain/sprite-facts.mjs +0 -0
- package/src/services/adventure-viz.mjs +5 -1
- package/src/services/adventure.mjs +70 -44
- package/src/services/chat-page-viz.mjs +381 -310
- package/src/services/chat.mjs +273 -155
- package/src/services/code-explorer-viz.mjs +141 -54
- package/src/services/index.mjs +1 -1
- package/src/services/ingest-viz.mjs +134 -9
- package/src/services/ledger-viz.mjs +7 -4
- package/src/services/memory-panel-viz.mjs +8 -3
- package/src/services/mud-turn.mjs +11 -8
- package/src/services/mud-viz.mjs +441 -206
- package/src/services/p2p-room.mjs +110 -23
- package/src/services/plan-viz.mjs +63 -4
- package/src/services/research-viz.mjs +18 -7
- package/src/services/share-overlay-viz.mjs +623 -0
- package/src/services/spider-fly-viz.mjs +2 -2
- package/src/services/sprite-catalog-viz.mjs +303 -78
- package/src/surfaces/web/adventure-browser-entry.mjs +27 -5
- package/src/surfaces/web/chat-browser-entry.mjs +37 -10
- package/src/surfaces/web/code-explorer-browser-entry.mjs +4 -3
- package/src/surfaces/web/ingest-browser-entry.mjs +73 -12
- package/src/surfaces/web/ledger-browser-entry.mjs +32 -7
- package/src/surfaces/web/memory-ask-browser.bundle.js +149 -116
- package/src/surfaces/web/mud-browser-entry.mjs +38 -7
- package/src/surfaces/web/p2p-browser-entry.mjs +1 -1
- package/src/surfaces/web/plan-browser-entry.mjs +33 -2
- package/src/surfaces/web/research-browser-entry.mjs +11 -19
- package/src/surfaces/web/sprites-browser-entry.mjs +39 -8
- package/src/surfaces/web/tmct-surface.mjs +12 -0
- package/src/surfaces/web/turn-session.mjs +10 -3
|
@@ -24,17 +24,39 @@ import { fnv1aHex, normText, normFactTerm, normFactPredicate, factIdFor, factIdF
|
|
|
24
24
|
// a single import site for read/write plus identity.
|
|
25
25
|
export { normFactTerm, normFactPredicate, factIdForTriple } from "../../domain/hash.mjs";
|
|
26
26
|
import {
|
|
27
|
-
computeTrust,
|
|
27
|
+
computeTrust, computeAssertionGroupTrust, computeAssertionGroupTrustBase,
|
|
28
|
+
assertionPrior, sessionReliabilityFrom,
|
|
29
|
+
TRUST_SCORE_PROP, TRUST_INPUTS_PROP, PROV_CLASS_BY_SOURCE_TYPE,
|
|
28
30
|
CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource,
|
|
29
31
|
} from "../../domain/memory/trust.mjs";
|
|
32
|
+
import {
|
|
33
|
+
resolutionStrategyFor, resolveSiblingGroups, MERGE_PREDICATES,
|
|
34
|
+
RESOLUTION_MERGE, RESOLUTION_CONTRADICTION, RESOLUTION_LATEST_OBSERVATION_WINS,
|
|
35
|
+
} from "../../domain/memory/resolution.mjs";
|
|
30
36
|
|
|
31
37
|
// The createdAt/updatedAt vocabulary and the provenance-tag Source parser live
|
|
32
38
|
// with the trust layer (they are its inputs); re-exported here so store
|
|
33
39
|
// consumers keep one import site.
|
|
34
40
|
export { CREATED_AT_PROP, UPDATED_AT_PROP, provenanceTagToSource } from "../../domain/memory/trust.mjs";
|
|
35
41
|
import { NEG_PREDICATE_PREFIX, negatedPredicate } from "../../domain/memory/capability.mjs";
|
|
42
|
+
import {
|
|
43
|
+
planHeadRollup, planChainRollup, mergeRollups,
|
|
44
|
+
isHeadRollupId, isChainRollupId, isRollupId, headRollupTypeOf,
|
|
45
|
+
isAbsorbedSource, absorbedSourceIds,
|
|
46
|
+
ROLLUP_PRIOR_PROP, ROLLUP_EARLIEST_PROP, ROLLUP_LATEST_PROP, ROLLUP_COUNT_PROP,
|
|
47
|
+
CHAIN_ROLLUP_THRESHOLD,
|
|
48
|
+
} from "../../domain/memory/compaction.mjs";
|
|
36
49
|
import { assertIndividualValid } from "./shacl.mjs";
|
|
37
50
|
|
|
51
|
+
// The rollup vocabulary and its tuning constants live with the compaction
|
|
52
|
+
// layer; re-exported here so store consumers keep one import site.
|
|
53
|
+
export {
|
|
54
|
+
GROUP_ROLLUP_THRESHOLD, ROLLUP_KEEP_PER_TYPE, CHAIN_ROLLUP_THRESHOLD, CHAIN_KEEP_DEPTH,
|
|
55
|
+
ROLLUP_SOURCE_IDS_PROP, ROLLUP_RECORD_IDS_PROP, ROLLUP_COUNT_PROP,
|
|
56
|
+
ROLLUP_EARLIEST_PROP, ROLLUP_LATEST_PROP, ROLLUP_PRIOR_PROP,
|
|
57
|
+
headRollupIdFor, chainRollupIdFor, isHeadRollupId, isChainRollupId, isRollupId,
|
|
58
|
+
} from "../../domain/memory/compaction.mjs";
|
|
59
|
+
|
|
38
60
|
export const MEMORY_DIR_REL = join(".tmct", "memory");
|
|
39
61
|
export const MEMORY_GRAPH_REL = join(MEMORY_DIR_REL, "graph.json");
|
|
40
62
|
|
|
@@ -62,6 +84,8 @@ export const SOURCE_RELIABILITY_PROP = "mgx:sourceReliability"; // actor-level (
|
|
|
62
84
|
// per-session Source instead (`${ID}:<sessionId>`, sourceIdFor below).
|
|
63
85
|
export const OPERATOR_SOURCE_ID = "src:operator-chat";
|
|
64
86
|
const TEACH_SOURCE_ID = "src:teach-chat";
|
|
87
|
+
// One Source per peer NODE, keyed by the stable id its relabeled tag carries.
|
|
88
|
+
const TEACH_NODE_SOURCE_ID = "src:teach-node";
|
|
65
89
|
|
|
66
90
|
const ROLES = new Set(["visitor", "tmct"]);
|
|
67
91
|
const LABEL_CAP = 48; // utterance/fact labels stay skimmable in renders
|
|
@@ -80,6 +104,10 @@ const MEMORY_VOCABULARY = [
|
|
|
80
104
|
{ prop: "rdf:predicate", note: "reified fact: the triple's predicate term" },
|
|
81
105
|
{ prop: "rdf:object", note: "reified fact: the triple's object term" },
|
|
82
106
|
{ prop: "mgx:factProvenance", note: "LEGACY COMPAT SHIM: the ' | '-joined provenance tag string a fact came from; the source-of-truth is now the mgx:statedBy edges derived from it" },
|
|
107
|
+
{ prop: "mgx:sourceId", note: "the assertion key a Fact record is filed under — the Source id of the ONE party asserting it, which is also the @-suffix of the record's own id. `src:none` when no tag names a Source, so every record has a key rather than a hole" },
|
|
108
|
+
{ prop: "mgx:observedAt", note: "OPTIONAL valid time: when the asserting party WITNESSED the claim, as against mgx:createdAt's transaction time (when this store recorded it). A stale article read today loses to an eyewitness report from yesterday. Stored only when a caller supplies one — never fabricated, never backfilled" },
|
|
109
|
+
{ prop: "mgx:supersedes", note: "the record id(s) this one replaced when its own source re-asserted the triple with a newer embedded timestamp. A space-joined LIST; absent, never empty, until the first supersession" },
|
|
110
|
+
{ prop: "mgx:supersededBy", note: "the record id(s) that replaced this one. Its presence is what makes a record a demoted leaf rather than a live head, and the group fold skips it: a source's past belief is not a second vote for the present one. A LIST, because one source with two live replicas can fork before they sync" },
|
|
83
111
|
{ prop: "mgx:factQuantifier", note: "OPTIONAL: the quantifier word a plural class-membership teach used ('every'/'some'/'a few'), for literal recall by 'how many Xs are Ys' — never real cardinality counting" },
|
|
84
112
|
{ prop: "mgx:factJustification", note: "an entailed Fact's supporting premise fact ids: ' | '-separated environments, one space-separated premise-id list per independent derivation, capped by syllogise's maxEnvironments knob; a value with no ' | ' is a single environment" },
|
|
85
113
|
{ prop: "mgx:ruleName", note: "a taught Rule's own name (e.g. 'grandparent') — the query-dispatcher's lookup key, PLAN_TAUGHT_RELATIONS.md §2/§3" },
|
|
@@ -204,8 +232,381 @@ CREATE TABLE IF NOT EXISTS individuals (id TEXT PRIMARY KEY, ord INTEGER NOT NUL
|
|
|
204
232
|
CREATE TABLE IF NOT EXISTS relations (prop TEXT PRIMARY KEY, ord INTEGER NOT NULL, predicate TEXT, count INTEGER);
|
|
205
233
|
CREATE TABLE IF NOT EXISTS edges (prop TEXT NOT NULL, subject TEXT NOT NULL, object TEXT NOT NULL, subject_label TEXT, object_label TEXT, extra TEXT, PRIMARY KEY (prop, subject, object));
|
|
206
234
|
CREATE INDEX IF NOT EXISTS edges_by_prop ON edges(prop);
|
|
235
|
+
CREATE TABLE IF NOT EXISTS facts (
|
|
236
|
+
id TEXT PRIMARY KEY,
|
|
237
|
+
triple_hash TEXT NOT NULL,
|
|
238
|
+
subject TEXT NOT NULL,
|
|
239
|
+
predicate TEXT NOT NULL,
|
|
240
|
+
object TEXT NOT NULL,
|
|
241
|
+
source_id TEXT NOT NULL,
|
|
242
|
+
source_type TEXT NOT NULL,
|
|
243
|
+
trust_score REAL NOT NULL,
|
|
244
|
+
created_at TEXT NOT NULL,
|
|
245
|
+
observed_at TEXT,
|
|
246
|
+
superseded_by TEXT,
|
|
247
|
+
json TEXT NOT NULL
|
|
248
|
+
);
|
|
249
|
+
CREATE INDEX IF NOT EXISTS facts_by_triple_hash ON facts(triple_hash);
|
|
250
|
+
CREATE INDEX IF NOT EXISTS facts_by_subject_predicate ON facts(subject, predicate);
|
|
251
|
+
CREATE INDEX IF NOT EXISTS facts_by_predicate_object ON facts(predicate, object);
|
|
252
|
+
CREATE INDEX IF NOT EXISTS facts_current ON facts(triple_hash, source_id, superseded_by);
|
|
253
|
+
CREATE TABLE IF NOT EXISTS fact_heads (
|
|
254
|
+
triple_hash TEXT PRIMARY KEY,
|
|
255
|
+
trust_base REAL NOT NULL,
|
|
256
|
+
inputs_json TEXT NOT NULL,
|
|
257
|
+
updated_at TEXT NOT NULL
|
|
258
|
+
);
|
|
259
|
+
CREATE TABLE IF NOT EXISTS fact_object_supersessions (
|
|
260
|
+
subject TEXT NOT NULL,
|
|
261
|
+
predicate TEXT NOT NULL,
|
|
262
|
+
ord INTEGER NOT NULL,
|
|
263
|
+
from_id TEXT NOT NULL,
|
|
264
|
+
to_id TEXT NOT NULL,
|
|
265
|
+
recorded_at TEXT NOT NULL,
|
|
266
|
+
PRIMARY KEY (subject, predicate, ord)
|
|
267
|
+
);
|
|
207
268
|
`;
|
|
208
269
|
|
|
270
|
+
// ---- The `facts` projection ------------------------------------------------
|
|
271
|
+
// `facts` holds one queryable row per Fact individual, written in the SAME
|
|
272
|
+
// transaction as the `individuals` row it mirrors. The JSON blob stays the
|
|
273
|
+
// single source of truth — the columns exist so a reader can ask the database
|
|
274
|
+
// for "every fact about dog" instead of loading the whole store and scanning
|
|
275
|
+
// it in JS, the same discipline `individuals` already applies to its own
|
|
276
|
+
// `class`/`label` columns. Plain portable SQL: the schema above works
|
|
277
|
+
// unchanged against Postgres/MySQL/Aurora, so a cloud relational backend
|
|
278
|
+
// inherits the read path rather than reinventing it.
|
|
279
|
+
|
|
280
|
+
// A Fact whose provenance tag maps to no Source at all still needs a key, so
|
|
281
|
+
// it projects onto this singleton rather than a NULL.
|
|
282
|
+
const NO_SOURCE_ID = "src:none";
|
|
283
|
+
|
|
284
|
+
const OBSERVED_AT_PROP = "mgx:observedAt"; // valid time: when the asserting party witnessed the claim
|
|
285
|
+
const SOURCE_ID_PROP = "mgx:sourceId"; // the assertion key this record is filed under
|
|
286
|
+
const SUPERSEDES_PROP = "mgx:supersedes"; // the id(s) this record replaced; absent until the first supersession
|
|
287
|
+
const SUPERSEDED_BY_PROP = "mgx:supersededBy"; // the id(s) that replaced this record; absent on a live head
|
|
288
|
+
|
|
289
|
+
/** The Source key a Fact's provenance union projects onto: the first tag that
|
|
290
|
+
* derives one, since the tags of one fact are asserted in arrival order and
|
|
291
|
+
* the earliest is its primary source. `src:none` when no tag parses. */
|
|
292
|
+
function primarySourceOf(provenance) {
|
|
293
|
+
for (const tag of String(provenance || "").split(" | ")) {
|
|
294
|
+
if (!tag) continue;
|
|
295
|
+
const info = sourceIdFor(provenanceTagToSource(tag));
|
|
296
|
+
if (info) return info;
|
|
297
|
+
}
|
|
298
|
+
return { id: NO_SOURCE_ID, type: "" };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// ---- One record per ASSERTION, not one per triple --------------------------
|
|
302
|
+
// A Fact's identity is `<groupId>@<sourceId>`: the content-addressed triple hash
|
|
303
|
+
// every reader still calls "the fact id", plus the Source key of the ONE party
|
|
304
|
+
// asserting it. Two sources asserting the same triple hold two records sharing
|
|
305
|
+
// a group; the same source re-asserting resolves onto its own lineage. The
|
|
306
|
+
// group id stays the public id — it is what a justification premise list, a
|
|
307
|
+
// citation and `tmct inspect` all print — so nothing outside this file has to
|
|
308
|
+
// learn the record id at all.
|
|
309
|
+
|
|
310
|
+
/** The group (triple) id a record id belongs to: everything before the first
|
|
311
|
+
* `@`. A record id with no `@` is its own group, which is what a hand-built
|
|
312
|
+
* fixture and a not-yet-migrated store both still look like. */
|
|
313
|
+
export function factGroupId(recordId) {
|
|
314
|
+
const id = String(recordId || "");
|
|
315
|
+
const at = id.indexOf("@");
|
|
316
|
+
return at < 0 ? id : id.slice(0, at);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** The assertion key one provenance tag derives — the SAME closed derivation
|
|
320
|
+
* Source individuals already use, with the `src:none` singleton standing in
|
|
321
|
+
* for a tag that parses to no Source at all. Under this model every record
|
|
322
|
+
* needs a key, so the null case gets a name rather than a hole. */
|
|
323
|
+
function assertionSourceFor(tag) {
|
|
324
|
+
return sourceIdFor(provenanceTagToSource(tag)) || { id: NO_SOURCE_ID, type: "" };
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** Split a provenance string into one group per Source id, keeping each group's
|
|
328
|
+
* own tags in arrival order. Normally one tag per group; two only when both
|
|
329
|
+
* derive the SAME Source (`corpus:conceptnet /r/IsA` and `child:conceptnet:dog`
|
|
330
|
+
* both key on `src:corpus:conceptnet`). An empty provenance still yields one
|
|
331
|
+
* group, under `src:none`, so the fact keeps a record. */
|
|
332
|
+
function groupTagsBySource(provenance) {
|
|
333
|
+
const groups = new Map();
|
|
334
|
+
for (const raw of String(provenance || "").split(" | ")) {
|
|
335
|
+
const tag = raw.trim();
|
|
336
|
+
if (!tag) continue;
|
|
337
|
+
const { id, type } = assertionSourceFor(tag);
|
|
338
|
+
const group = groups.get(id) || { sourceId: id, sourceType: type, tags: [] };
|
|
339
|
+
if (!group.tags.includes(tag)) group.tags.push(tag);
|
|
340
|
+
groups.set(id, group);
|
|
341
|
+
}
|
|
342
|
+
if (!groups.size) groups.set(NO_SOURCE_ID, { sourceId: NO_SOURCE_ID, sourceType: "", tags: [] });
|
|
343
|
+
return [...groups.values()];
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** The instant a source's own tag(s) EMBED — the origin's assertion moment, and
|
|
347
|
+
* the only clock supersession is allowed to read. `pick` is Math.max for a
|
|
348
|
+
* live write ("when did this source last say it") and Math.min for the
|
|
349
|
+
* migration ("when did this source first say it"). "" when no tag carries one,
|
|
350
|
+
* which is the common corpus case. */
|
|
351
|
+
function embeddedTagTimestamp(tags, pick = Math.max) {
|
|
352
|
+
let best = "";
|
|
353
|
+
let bestAt = null;
|
|
354
|
+
for (const tag of tags || []) {
|
|
355
|
+
const ts = provenanceTagToSource(tag)?.createdAt || "";
|
|
356
|
+
const at = Date.parse(ts);
|
|
357
|
+
if (!Number.isFinite(at)) continue;
|
|
358
|
+
if (bestAt === null || pick(at, bestAt) === at) { best = ts; bestAt = at; }
|
|
359
|
+
}
|
|
360
|
+
return best;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** When a record says it was asserted: its tag's own embedded timestamp, else
|
|
364
|
+
* the caller's explicit createdAt. This is the recency clock — separate from
|
|
365
|
+
* the supersession clock above on purpose. A caller's createdAt is THIS
|
|
366
|
+
* store's transaction stamp; letting it order supersession would turn every
|
|
367
|
+
* re-import that passes a later createdAt into a new version and quietly break
|
|
368
|
+
* first-write-wins. Only what the origin embedded can order the origin. */
|
|
369
|
+
function assertionTimestampFor(tags, fallback = "", pick = Math.max) {
|
|
370
|
+
return embeddedTagTimestamp(tags, pick) || (Number.isFinite(Date.parse(fallback)) ? String(fallback) : "");
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** Does an incoming assertion replace this source's own current record?
|
|
374
|
+
* A genuinely newer embedded timestamp does; an exact re-delivery never does,
|
|
375
|
+
* which is what keeps a re-seed and a duplicate mesh delivery no-ops. Both
|
|
376
|
+
* sides must carry a real embedded timestamp — an unstamped corpus row
|
|
377
|
+
* asserted a second time is the same hop saying the same thing, not a new
|
|
378
|
+
* version. The one tie-break: at equal instants a record carrying an
|
|
379
|
+
* observation time supersedes the same record without one, since only the
|
|
380
|
+
* origin could have supplied that field, so presence can only add information. */
|
|
381
|
+
function supersedesPriorAssertion(incoming, prior) {
|
|
382
|
+
const a = Date.parse(incoming.assertedAt);
|
|
383
|
+
const b = Date.parse(prior.assertedAt);
|
|
384
|
+
if (!Number.isFinite(a) || !Number.isFinite(b)) return false;
|
|
385
|
+
if (a > b) return true;
|
|
386
|
+
return a === b && !!incoming.observedAt && !prior.observedAt;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/** The `facts` bind values for a Fact individual, in column order. `json` is
|
|
390
|
+
* the already-serialized blob the `individuals` row stores, passed in so the
|
|
391
|
+
* two rows can never disagree about the same individual. */
|
|
392
|
+
function factProjectionValues(ind, json) {
|
|
393
|
+
const attr = (prop) => (ind.attributes || []).find((a) => a?.prop === prop)?.value || "";
|
|
394
|
+
// Every tag on one record derives the same Source, so the record's own
|
|
395
|
+
// provenance settles its type; the stored key wins on the id, since a
|
|
396
|
+
// `src:none` record has no tag to re-derive it from.
|
|
397
|
+
const source = primarySourceOf(attr("mgx:factProvenance"));
|
|
398
|
+
// A collection in the blob; the column carries the common-case single
|
|
399
|
+
// successor, so a reader can filter for live heads without opening the JSON.
|
|
400
|
+
const supersededBy = attr(SUPERSEDED_BY_PROP).split(" ").filter(Boolean)[0] || null;
|
|
401
|
+
// A pool-1 summary has no tag to re-derive a type from and no Source of its
|
|
402
|
+
// own: it stands for many absorbed sources of ONE type, which its id carries,
|
|
403
|
+
// and its trust contribution is the noisy-OR base over what it absorbed.
|
|
404
|
+
// Both belong in the columns, so a per-type SQL read sees a compacted group
|
|
405
|
+
// exactly as it sees an uncompacted one.
|
|
406
|
+
const rollupType = headRollupTypeOf(ind.id);
|
|
407
|
+
return [
|
|
408
|
+
ind.id,
|
|
409
|
+
factGroupId(ind.id),
|
|
410
|
+
attr("rdf:subject"), attr("rdf:predicate"), attr("rdf:object"),
|
|
411
|
+
attr(SOURCE_ID_PROP) || source.id, rollupType || source.type,
|
|
412
|
+
Number(attr(rollupType ? ROLLUP_PRIOR_PROP : TRUST_SCORE_PROP)) || 0,
|
|
413
|
+
attr(CREATED_AT_PROP),
|
|
414
|
+
attr(OBSERVED_AT_PROP) || null,
|
|
415
|
+
supersededBy,
|
|
416
|
+
json,
|
|
417
|
+
];
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const FACT_PROJECTION_UPSERT_SQL =
|
|
421
|
+
"INSERT OR REPLACE INTO facts(id, triple_hash, subject, predicate, object, source_id, source_type, trust_score, created_at, observed_at, superseded_by, json)"
|
|
422
|
+
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
|
423
|
+
|
|
424
|
+
/** Project every Fact individual a store already holds into an empty `facts`
|
|
425
|
+
* table — what a store written before the projection existed needs, once.
|
|
426
|
+
* Runs at open so a read-only session gets the columns too. Cheap to skip: a
|
|
427
|
+
* projected store answers the first probe with a row and returns. */
|
|
428
|
+
function backfillFactsProjection(db) {
|
|
429
|
+
if (db.prepare("SELECT id FROM facts LIMIT 1").get()) return;
|
|
430
|
+
if (!db.prepare("SELECT id FROM individuals WHERE class = ? LIMIT 1").get(FACT_CLASS)) return;
|
|
431
|
+
const upsertFact = db.prepare(FACT_PROJECTION_UPSERT_SQL);
|
|
432
|
+
db.exec("BEGIN IMMEDIATE");
|
|
433
|
+
try {
|
|
434
|
+
for (const row of db.prepare("SELECT json FROM individuals WHERE class = ?").all(FACT_CLASS)) {
|
|
435
|
+
upsertFact.run(...factProjectionValues(JSON.parse(row.json), row.json));
|
|
436
|
+
}
|
|
437
|
+
db.exec("COMMIT");
|
|
438
|
+
} catch (e) {
|
|
439
|
+
db.exec("ROLLBACK");
|
|
440
|
+
throw e;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const individualAttr = (ind, prop) => (ind?.attributes || []).find((a) => a?.prop === prop)?.value || "";
|
|
445
|
+
const individualKey = (ind, key) => (ind?.attributes || []).find((a) => a?.key === key)?.value || "";
|
|
446
|
+
const subjectPredicateKey = (subject, predicate) => `${subject}\u0000${predicate}`;
|
|
447
|
+
|
|
448
|
+
// ---- Derived-local tables: `fact_heads` and `fact_object_supersessions` -----
|
|
449
|
+
// Both are breadcrumbs over a computation that is already correct without them,
|
|
450
|
+
// and both are LOCAL: they are never replicated, never exported, and never
|
|
451
|
+
// reachable from a wire fact. Replicating a derived aggregate would manufacture
|
|
452
|
+
// exactly the merge conflicts one-record-per-assertion removes, and recency
|
|
453
|
+
// makes any shipped aggregate stale on arrival.
|
|
454
|
+
//
|
|
455
|
+
// `fact_heads` stores a group's aggregate BASE — the noisy-OR with the time
|
|
456
|
+
// axis removed — plus the per-record audit trail it was folded from. A reader
|
|
457
|
+
// replays that trail through the same aggregate at its own `now`, so the decay
|
|
458
|
+
// lands at the reading moment exactly as it does when the group is folded from
|
|
459
|
+
// scratch. A head that baked recency in would be wrong the moment it was
|
|
460
|
+
// written, which is why the column holds a base and not a score.
|
|
461
|
+
|
|
462
|
+
/** The replayable audit trail a head stores: the four fields the group
|
|
463
|
+
* aggregate actually folds, and nothing that depends on when it was written. */
|
|
464
|
+
function headInputsFrom(assertions) {
|
|
465
|
+
return (assertions || []).map((a) => ({
|
|
466
|
+
sourceId: a.sourceId || "",
|
|
467
|
+
sourceType: a.sourceType || "",
|
|
468
|
+
ownTrust: a.ownTrust,
|
|
469
|
+
assertedAt: a.assertedAt || "",
|
|
470
|
+
}));
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// Symbol-keyed, so it is invisible to JSON.stringify and dropped by
|
|
474
|
+
// structuredClone — a payload carrying the index cannot leak it into a
|
|
475
|
+
// snapshot, an export, or the wire even by accident.
|
|
476
|
+
const FACT_HEADS = Symbol("materialised fact_heads index");
|
|
477
|
+
|
|
478
|
+
/** Attach a materialised head index to a payload, for readFactRows to consume.
|
|
479
|
+
* A payload without one folds every group itself, which is the same answer. */
|
|
480
|
+
export function attachFactHeads(payload, heads) {
|
|
481
|
+
if (payload && heads) payload[FACT_HEADS] = heads;
|
|
482
|
+
return payload;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
export const factHeadsOf = (payload) => payload?.[FACT_HEADS] || null;
|
|
486
|
+
|
|
487
|
+
const FACT_HEAD_UPSERT_SQL =
|
|
488
|
+
"INSERT OR REPLACE INTO fact_heads(triple_hash, trust_base, inputs_json, updated_at) VALUES (?, ?, ?, ?)";
|
|
489
|
+
|
|
490
|
+
/** Every materialised head in a store, as the groupId -> head map
|
|
491
|
+
* readFactRows consumes. */
|
|
492
|
+
function readFactHeadIndex(db) {
|
|
493
|
+
const heads = new Map();
|
|
494
|
+
for (const row of db.prepare("SELECT triple_hash, trust_base, inputs_json FROM fact_heads").all()) {
|
|
495
|
+
heads.set(row.triple_hash, { trustBase: row.trust_base, inputs: JSON.parse(row.inputs_json) });
|
|
496
|
+
}
|
|
497
|
+
return heads;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/** Re-materialise the head of every group this write touched, inside the
|
|
501
|
+
* caller's own open transaction — the same discipline the per-record trust
|
|
502
|
+
* recompute already follows, re-keyed from "the sources on one fact" to "the
|
|
503
|
+
* records in one group".
|
|
504
|
+
*
|
|
505
|
+
* Only touched groups are recomputed, and that is exact rather than thrifty: a
|
|
506
|
+
* base carries no recency, so nothing but a change to a group's own records can
|
|
507
|
+
* move it. Reading the group back inside the transaction is also what makes a
|
|
508
|
+
* second writer correct — it folds over whatever is committed by then, not over
|
|
509
|
+
* the payload it happened to arrive with. */
|
|
510
|
+
function recomputeFactHeads(db, ctx, touchedGroups, headIndex) {
|
|
511
|
+
const upsert = db.prepare(FACT_HEAD_UPSERT_SQL);
|
|
512
|
+
const drop = db.prepare("DELETE FROM fact_heads WHERE triple_hash = ?");
|
|
513
|
+
const updatedAt = nowIso();
|
|
514
|
+
for (const groupId of touchedGroups) {
|
|
515
|
+
const members = ctx.groups.get(groupId);
|
|
516
|
+
// Every record of the group is gone (a retraction) or every one of them is
|
|
517
|
+
// demoted — either way there is no live aggregate left to stand for.
|
|
518
|
+
if (!members?.length) {
|
|
519
|
+
drop.run(groupId);
|
|
520
|
+
headIndex?.delete(groupId);
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
const inputs = headInputsFrom(foldFactGroup(groupId, members, ctx).assertions);
|
|
524
|
+
const trustBase = computeAssertionGroupTrustBase(inputs).score;
|
|
525
|
+
upsert.run(groupId, trustBase, JSON.stringify(inputs), updatedAt);
|
|
526
|
+
headIndex?.set(groupId, { trustBase, inputs });
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/** Materialise a head for every group a store already holds — what a store
|
|
531
|
+
* written before the table existed needs, once. Guarded exactly like the
|
|
532
|
+
* `facts` projection backfill beside it: a store that already has heads, or
|
|
533
|
+
* has no facts at all, returns without touching anything. */
|
|
534
|
+
function backfillFactHeads(handle) {
|
|
535
|
+
const db = handle.db;
|
|
536
|
+
if (db.prepare("SELECT triple_hash FROM fact_heads LIMIT 1").get()) return;
|
|
537
|
+
if (!db.prepare("SELECT id FROM individuals WHERE class = ? LIMIT 1").get(FACT_CLASS)) return;
|
|
538
|
+
const ctx = factFoldContext(buildSqlitePayloadFromRows(handle));
|
|
539
|
+
db.exec("BEGIN IMMEDIATE");
|
|
540
|
+
try {
|
|
541
|
+
recomputeFactHeads(db, ctx, ctx.groups.keys(), null);
|
|
542
|
+
db.exec("COMMIT");
|
|
543
|
+
} catch (e) {
|
|
544
|
+
db.exec("ROLLBACK");
|
|
545
|
+
throw e;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// `fact_object_supersessions` records the OTHER shape supersession takes. A
|
|
550
|
+
// same-source re-assertion supersedes by re-keying its own record, a real
|
|
551
|
+
// replicated chain. A latest-observation-wins winner changing is different in
|
|
552
|
+
// kind: the old room and the new one are different objects, so they are
|
|
553
|
+
// different content-addressed groups, and an id can never be reassigned onto
|
|
554
|
+
// different content the way the same-source case reuses its own slot. What is
|
|
555
|
+
// recordable there is an EDGE between the two groups' own stable ids.
|
|
556
|
+
//
|
|
557
|
+
// It is a breadcrumb, not an authority. Every fact behind it is already fully
|
|
558
|
+
// replicated and the resolution already runs correctly without it, so recording
|
|
559
|
+
// "A used to be current, B is now" changes nothing about how the winner is
|
|
560
|
+
// chosen. It is written lazily, only when a resolution actually finds the winner
|
|
561
|
+
// has moved — a pair that resolves the same way a thousand times running holds
|
|
562
|
+
// one row, and the first row a pair ever gets seeds the chain with an empty
|
|
563
|
+
// `from_id` because nothing preceded it.
|
|
564
|
+
|
|
565
|
+
/** Record the cross-object supersession edge for every (subject, predicate)
|
|
566
|
+
* this write touched whose winner has moved since the last one recorded. */
|
|
567
|
+
function recordObjectSupersessions(db, ctx, touchedPairs) {
|
|
568
|
+
const latest = db.prepare(
|
|
569
|
+
"SELECT ord, to_id FROM fact_object_supersessions WHERE subject = ? AND predicate = ? ORDER BY ord DESC LIMIT 1",
|
|
570
|
+
);
|
|
571
|
+
const insert = db.prepare(
|
|
572
|
+
"INSERT INTO fact_object_supersessions(subject, predicate, ord, from_id, to_id, recorded_at) VALUES (?, ?, ?, ?, ?, ?)",
|
|
573
|
+
);
|
|
574
|
+
const recordedAt = nowIso();
|
|
575
|
+
for (const pairKey of touchedPairs) {
|
|
576
|
+
const [subject, predicate] = pairKey.split("\u0000");
|
|
577
|
+
if (resolutionStrategyFor(predicate) !== RESOLUTION_LATEST_OBSERVATION_WINS) continue;
|
|
578
|
+
const groupIds = ctx.groupsByPair.get(pairKey) || [];
|
|
579
|
+
if (groupIds.length < 2) continue; // one object is a state, not a succession
|
|
580
|
+
const rows = groupIds.map((groupId) => {
|
|
581
|
+
const row = foldFactGroup(groupId, ctx.groups.get(groupId), ctx);
|
|
582
|
+
row.trust = computeAssertionGroupTrust(row.assertions).score;
|
|
583
|
+
return row;
|
|
584
|
+
});
|
|
585
|
+
if (new Set(rows.map((r) => r.object)).size < 2) continue;
|
|
586
|
+
const winner = resolveSiblingGroups(rows, RESOLUTION_LATEST_OBSERVATION_WINS)?.winner?.id;
|
|
587
|
+
if (!winner) continue;
|
|
588
|
+
const prior = latest.get(subject, predicate);
|
|
589
|
+
if (prior?.to_id === winner) continue; // nothing moved — the breadcrumb is already there
|
|
590
|
+
insert.run(subject, predicate, (prior?.ord ?? -1) + 1, prior?.to_id || "", winner, recordedAt);
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
/** The recorded cross-object supersession chain — "what changed to what" for a
|
|
595
|
+
* (subject, predicate), oldest first. The walk a view AT a past instant would
|
|
596
|
+
* consume; empty for any backend that keeps no derived tables. */
|
|
597
|
+
export function readObjectSupersessions(handle, { subject, predicate } = {}) {
|
|
598
|
+
if (!isSqliteHandle(handle)) return [];
|
|
599
|
+
const where = subject && predicate ? " WHERE subject = ? AND predicate = ?" : "";
|
|
600
|
+
const args = where ? [subject, predicate] : [];
|
|
601
|
+
return handle.db
|
|
602
|
+
.prepare(`SELECT subject, predicate, ord, from_id, to_id, recorded_at FROM fact_object_supersessions${where} ORDER BY subject, predicate, ord`)
|
|
603
|
+
.all(...args)
|
|
604
|
+
.map((r) => ({
|
|
605
|
+
subject: r.subject, predicate: r.predicate, ord: r.ord,
|
|
606
|
+
fromId: r.from_id, toId: r.to_id, recordedAt: r.recorded_at,
|
|
607
|
+
}));
|
|
608
|
+
}
|
|
609
|
+
|
|
209
610
|
// Edge keys with dedicated columns; any other key round-trips via `extra`.
|
|
210
611
|
const STD_EDGE_KEYS = new Set(["subject", "object", "subjectLabel", "objectLabel"]);
|
|
211
612
|
|
|
@@ -237,7 +638,10 @@ export async function createSqliteMemoryStore(dbPath) {
|
|
|
237
638
|
db.exec("PRAGMA journal_mode = WAL");
|
|
238
639
|
db.exec("PRAGMA synchronous = NORMAL");
|
|
239
640
|
db.exec(SQLITE_DDL);
|
|
240
|
-
|
|
641
|
+
backfillFactsProjection(db);
|
|
642
|
+
const handle = { backend: BACKEND_SQLITE, db, dbPath };
|
|
643
|
+
backfillFactHeads(handle);
|
|
644
|
+
return handle;
|
|
241
645
|
}
|
|
242
646
|
|
|
243
647
|
/** Close a Backend C handle's connection. A no-op for anything else (so a
|
|
@@ -301,9 +705,15 @@ function readSqlitePayload(handle) {
|
|
|
301
705
|
if (handle.cachedPayload && handle.cachedDataVersion !== dataVersion) handle.cachedPayload = null;
|
|
302
706
|
if (!handle.cachedPayload) {
|
|
303
707
|
handle.cachedPayload = buildSqlitePayloadFromRows(handle);
|
|
708
|
+
// The head index rides the same cache lifecycle as the payload it indexes,
|
|
709
|
+
// so another connection's commit invalidates both together and the two can
|
|
710
|
+
// never describe different stores.
|
|
711
|
+
handle.cachedFactHeads = readFactHeadIndex(handle.db);
|
|
304
712
|
handle.cachedDataVersion = dataVersion;
|
|
305
713
|
}
|
|
306
|
-
|
|
714
|
+
// Attached AFTER the clone: structuredClone drops symbol-keyed properties, so
|
|
715
|
+
// the index has to be put back on each returned payload rather than copied.
|
|
716
|
+
return attachFactHeads(cloneJson(handle.cachedPayload), handle.cachedFactHeads);
|
|
307
717
|
}
|
|
308
718
|
|
|
309
719
|
/** The actual SQL reconstruction — unchanged from the pre-cache implementation,
|
|
@@ -438,7 +848,14 @@ function persistSqlitePayload(handle, payload) {
|
|
|
438
848
|
const maxOrd = db.prepare("SELECT COALESCE(MAX(ord), -1) AS m FROM individuals").get().m;
|
|
439
849
|
let nextOrd = maxOrd + 1;
|
|
440
850
|
const upsertInd = db.prepare("INSERT OR REPLACE INTO individuals(id, ord, class, label, json) VALUES (?, ?, ?, ?, ?)");
|
|
851
|
+
const upsertFact = db.prepare(FACT_PROJECTION_UPSERT_SQL);
|
|
441
852
|
const seenIds = new Set();
|
|
853
|
+
// Every group and every (subject, predicate) this write reaches, so the two
|
|
854
|
+
// derived tables are re-materialised for exactly what moved and nothing
|
|
855
|
+
// else. A group whose records are all untouched cannot have moved: its base
|
|
856
|
+
// carries no recency, so only its own records can change it.
|
|
857
|
+
const touchedGroups = new Set();
|
|
858
|
+
const touchedPairs = new Set();
|
|
442
859
|
for (const ind of payload.individuals || []) {
|
|
443
860
|
seenIds.add(ind.id);
|
|
444
861
|
const json = JSON.stringify(ind);
|
|
@@ -446,14 +863,35 @@ function persistSqlitePayload(handle, payload) {
|
|
|
446
863
|
if (existing && existing.json === json) continue; // unchanged — skip the write entirely (cache already matches)
|
|
447
864
|
const ord = existing ? existing.ord : nextOrd++;
|
|
448
865
|
upsertInd.run(ind.id, ord, ind.class ?? null, ind.label ?? null, json);
|
|
866
|
+
// The queryable projection of the blob just written, same transaction —
|
|
867
|
+
// every write path that reaches a Fact (teach, corpus seed, entailment,
|
|
868
|
+
// migration, a trust recompute) lands here, so none can leave the two
|
|
869
|
+
// out of step.
|
|
870
|
+
if (ind.class === FACT_CLASS) {
|
|
871
|
+
upsertFact.run(...factProjectionValues(ind, json));
|
|
872
|
+
touchedGroups.add(factGroupId(ind.id));
|
|
873
|
+
touchedPairs.add(subjectPredicateKey(individualKey(ind, "subject"), individualKey(ind, "predicate")));
|
|
874
|
+
}
|
|
449
875
|
if (cache) cacheUpsertIndividual(cache, ind);
|
|
450
876
|
}
|
|
451
|
-
// Removal (
|
|
452
|
-
//
|
|
453
|
-
//
|
|
877
|
+
// Removal (what removeFacts' retraction lands as — every append path only
|
|
878
|
+
// ever adds): a cheap index-only scan of the primary-key column, never the
|
|
879
|
+
// JSON payload.
|
|
454
880
|
const deleteInd = db.prepare("DELETE FROM individuals WHERE id = ?");
|
|
881
|
+
const deleteFact = db.prepare("DELETE FROM facts WHERE id = ?");
|
|
882
|
+
const getFact = db.prepare("SELECT triple_hash, subject, predicate FROM facts WHERE id = ?");
|
|
455
883
|
for (const row of db.prepare("SELECT id FROM individuals").all()) {
|
|
456
|
-
if (
|
|
884
|
+
if (seenIds.has(row.id)) continue;
|
|
885
|
+
// Read the projection before dropping it: a retracted record's own group
|
|
886
|
+
// and (subject, predicate) still have to be re-materialised, and once the
|
|
887
|
+
// row is gone there is nothing left to read them off.
|
|
888
|
+
const gone = getFact.get(row.id);
|
|
889
|
+
if (gone) {
|
|
890
|
+
touchedGroups.add(gone.triple_hash);
|
|
891
|
+
touchedPairs.add(subjectPredicateKey(gone.subject, gone.predicate));
|
|
892
|
+
}
|
|
893
|
+
deleteInd.run(row.id);
|
|
894
|
+
deleteFact.run(row.id); // a no-op for a non-Fact id; keeps the projection from outliving its blob
|
|
457
895
|
}
|
|
458
896
|
if (cache) cacheDropIndividualsExcept(cache, seenIds);
|
|
459
897
|
|
|
@@ -508,6 +946,14 @@ function persistSqlitePayload(handle, payload) {
|
|
|
508
946
|
}
|
|
509
947
|
if (cache) cacheDropGroupsExcept(cache, seenProps);
|
|
510
948
|
|
|
949
|
+
// The derived tables, re-materialised in the SAME transaction as the
|
|
950
|
+
// records they summarise, so no reader can ever see one without the other.
|
|
951
|
+
if (touchedGroups.size) {
|
|
952
|
+
const ctx = factFoldContext(payload);
|
|
953
|
+
recomputeFactHeads(db, ctx, touchedGroups, handle.cachedFactHeads);
|
|
954
|
+
recordObjectSupersessions(db, ctx, touchedPairs);
|
|
955
|
+
}
|
|
956
|
+
|
|
511
957
|
db.exec("COMMIT");
|
|
512
958
|
} catch (e) {
|
|
513
959
|
db.exec("ROLLBACK");
|
|
@@ -516,6 +962,7 @@ function persistSqlitePayload(handle, payload) {
|
|
|
516
962
|
// actually committed to SQLite — never trust it silently. Drop it so the
|
|
517
963
|
// next loadMemory() call does an honest full rebuild instead.
|
|
518
964
|
handle.cachedPayload = undefined;
|
|
965
|
+
handle.cachedFactHeads = undefined;
|
|
519
966
|
throw e;
|
|
520
967
|
}
|
|
521
968
|
}
|
|
@@ -603,8 +1050,8 @@ export async function snapshotMemory(dir, { retentionVersions } = {}) {
|
|
|
603
1050
|
* append creates the file). The result is a raw entities payload;
|
|
604
1051
|
* parseEntities() loads it. */
|
|
605
1052
|
export async function loadMemory(dir) {
|
|
606
|
-
if (isMemoryHandle(dir)) return
|
|
607
|
-
if (isSqliteHandle(dir)) return
|
|
1053
|
+
if (isMemoryHandle(dir)) return migrateStoredMemory(dir.payload);
|
|
1054
|
+
if (isSqliteHandle(dir)) return migrateStoredMemory(readSqlitePayload(dir));
|
|
608
1055
|
let text;
|
|
609
1056
|
try {
|
|
610
1057
|
text = await readFile(memoryGraphFile(dir), "utf8");
|
|
@@ -612,9 +1059,14 @@ export async function loadMemory(dir) {
|
|
|
612
1059
|
if (e?.code === "ENOENT") return emptyMemory();
|
|
613
1060
|
throw e;
|
|
614
1061
|
}
|
|
615
|
-
return
|
|
1062
|
+
return migrateStoredMemory(JSON.parse(text));
|
|
616
1063
|
}
|
|
617
1064
|
|
|
1065
|
+
/** The lazy on-load migrations, in order: heal a pre-widening fact id first, so
|
|
1066
|
+
* the assertion re-key that follows content-addresses off the current one.
|
|
1067
|
+
* Both are pure payload transforms and both converge to no-ops. */
|
|
1068
|
+
const migrateStoredMemory = (payload) => migrateFactAssertionKeys(migrateLegacyFactIds(payload));
|
|
1069
|
+
|
|
618
1070
|
// A Fact id written before factIdFor widened to 64 bits — `fact:` + exactly 8
|
|
619
1071
|
// hex. A current id is 16 hex, so this anchored test never matches one, and a
|
|
620
1072
|
// migrated store pays only string checks with no rehash on load.
|
|
@@ -665,6 +1117,138 @@ function migrateLegacyFactIds(payload) {
|
|
|
665
1117
|
return payload;
|
|
666
1118
|
}
|
|
667
1119
|
|
|
1120
|
+
/**
|
|
1121
|
+
* Re-key every pre-assertion-model Fact — one record per TRIPLE, its provenance
|
|
1122
|
+
* a cross-source `" | "` union — into one record per (triple, source), in
|
|
1123
|
+
* place, on load. Same slot and same contract as the two migrations above:
|
|
1124
|
+
* a pure payload transform, deterministic, and a no-op on a migrated store (a
|
|
1125
|
+
* record id carries `@`, so the scan below never picks one up twice).
|
|
1126
|
+
*
|
|
1127
|
+
* Every OTHER reference to the bare id — a mgx:factJustification premise list,
|
|
1128
|
+
* derived_from, an edge endpoint like canonicalisedFrom — is left exactly as it
|
|
1129
|
+
* was. A bare `fact:<hash>` IS the group id under this model, so those
|
|
1130
|
+
* references keep resolving and the public fact id survives the migration.
|
|
1131
|
+
* Only the statedBy edges move, because those are per-record by definition.
|
|
1132
|
+
*/
|
|
1133
|
+
function migrateFactAssertionKeys(payload) {
|
|
1134
|
+
if (!Array.isArray(payload?.individuals)) return payload;
|
|
1135
|
+
const statedGroup = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
|
|
1136
|
+
const legacyStatedBy = new Map(); // legacy fact id -> its own statedBy edges
|
|
1137
|
+
for (const e of statedGroup?.examples || []) {
|
|
1138
|
+
if (!e?.subject) continue;
|
|
1139
|
+
const list = legacyStatedBy.get(e.subject);
|
|
1140
|
+
if (list) list.push(e);
|
|
1141
|
+
else legacyStatedBy.set(e.subject, [e]);
|
|
1142
|
+
}
|
|
1143
|
+
const replaced = new Set();
|
|
1144
|
+
const legacyRecords = new Map(); // legacy fact id -> the record ids it became
|
|
1145
|
+
const records = [];
|
|
1146
|
+
const carriedEdges = [];
|
|
1147
|
+
const individuals = [];
|
|
1148
|
+
for (const ind of payload.individuals) {
|
|
1149
|
+
if (ind?.class !== FACT_CLASS || String(ind.id || "").includes("@")) { individuals.push(ind); continue; }
|
|
1150
|
+
const attr = (prop) => (ind.attributes || []).find((a) => a?.prop === prop)?.value || "";
|
|
1151
|
+
const s = attr("rdf:subject");
|
|
1152
|
+
const p = attr("rdf:predicate");
|
|
1153
|
+
const o = attr("rdf:object");
|
|
1154
|
+
if (!s || !p || !o) { individuals.push(ind); continue; } // no readable triple to re-key on
|
|
1155
|
+
const groupId = factIdFor(s, p, o);
|
|
1156
|
+
const legacyCreated = attr(CREATED_AT_PROP);
|
|
1157
|
+
const provenance = attr("mgx:factProvenance");
|
|
1158
|
+
const edges = legacyStatedBy.get(ind.id) || [];
|
|
1159
|
+
const groups = provenance ? groupTagsBySource(provenance) : [];
|
|
1160
|
+
// A store can carry a statedBy edge whose Source no surviving tag names —
|
|
1161
|
+
// an early write, or a provenance string that was never backfilled. The
|
|
1162
|
+
// edge is the attribution in that case, so it earns its own tagless record
|
|
1163
|
+
// rather than being scrubbed along with the row it hung off.
|
|
1164
|
+
for (const e of edges) {
|
|
1165
|
+
if (!groups.some((g) => g.sourceId === e.object)) groups.push({ sourceId: e.object, sourceType: "", tags: [] });
|
|
1166
|
+
}
|
|
1167
|
+
if (!groups.length) groups.push({ sourceId: NO_SOURCE_ID, sourceType: "", tags: [] });
|
|
1168
|
+
const emitted = [];
|
|
1169
|
+
for (const group of groups) {
|
|
1170
|
+
const record = {
|
|
1171
|
+
id: `${groupId}@${group.sourceId}`,
|
|
1172
|
+
label: ind.label,
|
|
1173
|
+
class: FACT_CLASS,
|
|
1174
|
+
derived_from: cloneJson(ind.derived_from) || [],
|
|
1175
|
+
mentions: cloneJson(ind.mentions) || [],
|
|
1176
|
+
attributes: [
|
|
1177
|
+
{ prop: "rdf:type", key: "type", value: "rdf:Statement" },
|
|
1178
|
+
{ prop: "rdf:subject", key: "subject", value: s },
|
|
1179
|
+
{ prop: "rdf:predicate", key: "predicate", value: p },
|
|
1180
|
+
{ prop: "rdf:object", key: "object", value: o },
|
|
1181
|
+
// This source's own FIRST assertion of the triple, so a record's
|
|
1182
|
+
// createdAt keeps meaning "when this hop said it", never "when the
|
|
1183
|
+
// legacy row happened to be written".
|
|
1184
|
+
{ prop: CREATED_AT_PROP, key: "createdAt", value: assertionTimestampFor(group.tags, legacyCreated, Math.min) || legacyCreated || nowIso() },
|
|
1185
|
+
{ prop: SOURCE_ID_PROP, key: "sourceId", value: group.sourceId },
|
|
1186
|
+
...(group.tags.length ? [{ prop: "mgx:factProvenance", key: "provenance", value: group.tags.join(" | ") }] : []),
|
|
1187
|
+
// Triple-level values, duplicated onto every sibling: a few bytes
|
|
1188
|
+
// each, and it keeps every record readable on its own.
|
|
1189
|
+
...(attr("mgx:hasProseTokens") ? [{ prop: "mgx:hasProseTokens", key: "prose_tokens", value: attr("mgx:hasProseTokens") }] : []),
|
|
1190
|
+
...(attr("mgx:factQuantifier") ? [{ prop: "mgx:factQuantifier", key: "quantifier", value: attr("mgx:factQuantifier") }] : []),
|
|
1191
|
+
// The justification explains the ENTAILMENT, not the corroborators,
|
|
1192
|
+
// so it rides only the entailed record.
|
|
1193
|
+
...(group.sourceType === "entailed" && attr("mgx:factJustification")
|
|
1194
|
+
? [{ prop: "mgx:factJustification", key: "justification", value: attr("mgx:factJustification") }] : []),
|
|
1195
|
+
],
|
|
1196
|
+
};
|
|
1197
|
+
individuals.push(record);
|
|
1198
|
+
records.push(record);
|
|
1199
|
+
emitted.push(record.id);
|
|
1200
|
+
if (group.sourceId !== NO_SOURCE_ID) {
|
|
1201
|
+
const prior = edges.find((e) => e?.object === group.sourceId);
|
|
1202
|
+
carriedEdges.push({
|
|
1203
|
+
...(prior || {}), subject: record.id, object: group.sourceId,
|
|
1204
|
+
subjectLabel: record.label, objectLabel: sourceLabel(group.sourceId),
|
|
1205
|
+
});
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
replaced.add(ind.id);
|
|
1209
|
+
legacyRecords.set(ind.id, emitted);
|
|
1210
|
+
}
|
|
1211
|
+
if (!replaced.size) return payload;
|
|
1212
|
+
|
|
1213
|
+
// The legacy row's statedBy edges named a fact that no longer exists; each
|
|
1214
|
+
// record inherits the one edge that is actually its own, keeping that edge's
|
|
1215
|
+
// original createdAt so a migration never resets when a source first spoke.
|
|
1216
|
+
if (statedGroup) {
|
|
1217
|
+
statedGroup.examples = (statedGroup.examples || []).filter((e) => !replaced.has(e?.subject));
|
|
1218
|
+
statedGroup.examples.push(...carriedEdges);
|
|
1219
|
+
statedGroup.count = statedGroup.examples.length;
|
|
1220
|
+
} else if (carriedEdges.length) {
|
|
1221
|
+
payload.objectProperties = payload.objectProperties || [];
|
|
1222
|
+
payload.objectProperties.push({ predicate: "statedBy", prop: STATED_BY_PROP, count: carriedEdges.length, examples: carriedEdges });
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
// Every OTHER edge that named a re-keyed fact — canonicalisedFrom, derivedFrom
|
|
1226
|
+
// — is redrawn onto the records now holding that triple. An edge endpoint has
|
|
1227
|
+
// to be a node a graph walker can dereference, so unlike a justification
|
|
1228
|
+
// premise list (which is a reference to the TRIPLE, and resolves through the
|
|
1229
|
+
// group id readFactRows still reports) it cannot be left pointing at a group.
|
|
1230
|
+
for (const group of payload.objectProperties || []) {
|
|
1231
|
+
if (group?.prop === STATED_BY_PROP || !group?.examples?.length) continue;
|
|
1232
|
+
const redrawn = [];
|
|
1233
|
+
for (const e of group.examples) {
|
|
1234
|
+
const subjects = replaced.has(e?.subject) ? (legacyRecords.get(e.subject) || []) : [e?.subject];
|
|
1235
|
+
const objects = replaced.has(e?.object) ? (legacyRecords.get(e.object) || []) : [e?.object];
|
|
1236
|
+
for (const subject of subjects) {
|
|
1237
|
+
for (const object of objects) redrawn.push({ ...e, subject, object });
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
group.examples = redrawn;
|
|
1241
|
+
group.count = redrawn.length;
|
|
1242
|
+
}
|
|
1243
|
+
payload.individuals = individuals;
|
|
1244
|
+
buildMemoryIndex(payload);
|
|
1245
|
+
// The add-only, idempotent path that already exists: rebuild each record's
|
|
1246
|
+
// Source, its statedBy edge, and its own single-source trust.
|
|
1247
|
+
for (const record of records) syncFactSources(payload, record);
|
|
1248
|
+
recountClasses(payload);
|
|
1249
|
+
return payload;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
668
1252
|
/** Persist a mutated payload back to `dir`: an atomic file write (Backend A),
|
|
669
1253
|
* a no-op assignment (Backend B, already the live object), or a diffed
|
|
670
1254
|
* per-row SQL write (Backend C, persistSqlitePayload). */
|
|
@@ -712,6 +1296,44 @@ export async function saveSyllogiseState(dir, state) {
|
|
|
712
1296
|
await atomicWriteJson(file, state);
|
|
713
1297
|
}
|
|
714
1298
|
|
|
1299
|
+
// ---- Node id: the stable per-store P2P identity, a second sidecar ----------
|
|
1300
|
+
// 16 hex, minted the first time a store joins a room and never regenerated.
|
|
1301
|
+
// Persisted beside the store rather than inside the graph so it survives a
|
|
1302
|
+
// store that gets re-seeded, and so nothing about it ever replicates: a node
|
|
1303
|
+
// id is this store's own name for itself, not a fact about the world.
|
|
1304
|
+
|
|
1305
|
+
export const NODE_ID_REL = join(MEMORY_DIR_REL, "node-id.json");
|
|
1306
|
+
const SQLITE_NODE_ID_KEY = "nodeId";
|
|
1307
|
+
|
|
1308
|
+
/** This store's node id, or null when it has never joined a room. */
|
|
1309
|
+
export async function loadNodeId(dir) {
|
|
1310
|
+
if (isMemoryHandle(dir)) return dir.nodeId || null;
|
|
1311
|
+
if (isSqliteHandle(dir)) {
|
|
1312
|
+
const row = dir.db.prepare("SELECT v FROM meta WHERE k = ?").get(SQLITE_NODE_ID_KEY);
|
|
1313
|
+
return row?.v ? JSON.parse(row.v).nodeId || null : null;
|
|
1314
|
+
}
|
|
1315
|
+
try {
|
|
1316
|
+
return JSON.parse(await readFile(join(dir, NODE_ID_REL), "utf8")).nodeId || null;
|
|
1317
|
+
} catch (e) {
|
|
1318
|
+
if (e?.code === "ENOENT") return null;
|
|
1319
|
+
throw e;
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
/** Record this store's node id — atomic file write (Backend A), a handle field
|
|
1324
|
+
* (Backend B), or a meta-table row (Backend C). Callers mint through
|
|
1325
|
+
* resolveStoreNodeId, which never overwrites an id a store already holds. */
|
|
1326
|
+
export async function saveNodeId(dir, nodeId) {
|
|
1327
|
+
if (isMemoryHandle(dir)) { dir.nodeId = nodeId; return; }
|
|
1328
|
+
if (isSqliteHandle(dir)) {
|
|
1329
|
+
dir.db.prepare("INSERT OR REPLACE INTO meta(k, v) VALUES (?, ?)").run(SQLITE_NODE_ID_KEY, JSON.stringify({ nodeId }));
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1332
|
+
const file = join(dir, NODE_ID_REL);
|
|
1333
|
+
await mkdir(dirname(file), { recursive: true });
|
|
1334
|
+
await atomicWriteJson(file, { nodeId });
|
|
1335
|
+
}
|
|
1336
|
+
|
|
715
1337
|
/** Fresh read -> mutate -> atomic write. Serialized per call; every public
|
|
716
1338
|
* append goes through here, including the lazy legacy-provenance migration
|
|
717
1339
|
* and actor-level Source reliability recompute. `fn` may be async (the
|
|
@@ -728,10 +1350,20 @@ function buildMemoryIndex(payload) {
|
|
|
728
1350
|
const individualsById = new Map();
|
|
729
1351
|
const sourcesById = new Map();
|
|
730
1352
|
const statedByBySubject = new Map();
|
|
1353
|
+
// groupId -> the record ids asserting that triple, so a write can ask "is
|
|
1354
|
+
// anyone asserting this yet" and an edge can resolve a group id to the real
|
|
1355
|
+
// nodes behind it, both without a scan.
|
|
1356
|
+
const factRecordsByGroup = new Map();
|
|
731
1357
|
for (const ind of payload.individuals || []) {
|
|
732
1358
|
if (!ind?.id) continue;
|
|
733
1359
|
individualsById.set(ind.id, ind);
|
|
734
1360
|
if (ind.class === SOURCE_CLASS) sourcesById.set(ind.id, ind);
|
|
1361
|
+
if (ind.class === FACT_CLASS) {
|
|
1362
|
+
const groupId = factGroupId(ind.id);
|
|
1363
|
+
const held = factRecordsByGroup.get(groupId);
|
|
1364
|
+
if (held) held.push(ind.id);
|
|
1365
|
+
else factRecordsByGroup.set(groupId, [ind.id]);
|
|
1366
|
+
}
|
|
735
1367
|
}
|
|
736
1368
|
const statedGroup = (payload.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
|
|
737
1369
|
for (const e of statedGroup?.examples || []) {
|
|
@@ -740,7 +1372,7 @@ function buildMemoryIndex(payload) {
|
|
|
740
1372
|
if (list) list.push(e.object);
|
|
741
1373
|
else statedByBySubject.set(e.subject, [e.object]);
|
|
742
1374
|
}
|
|
743
|
-
payload[MEMORY_INDEX] = { individualsById, sourcesById, statedByBySubject };
|
|
1375
|
+
payload[MEMORY_INDEX] = { individualsById, sourcesById, statedByBySubject, factRecordsByGroup };
|
|
744
1376
|
return payload[MEMORY_INDEX];
|
|
745
1377
|
}
|
|
746
1378
|
|
|
@@ -786,6 +1418,12 @@ function sourceIdFor(desc) {
|
|
|
786
1418
|
switch (desc?.kind) {
|
|
787
1419
|
case "operator": return { id: desc.sessionId ? `${OPERATOR_SOURCE_ID}:${desc.sessionId}` : OPERATOR_SOURCE_ID, type: "operator" };
|
|
788
1420
|
case "teach": return { id: desc.sessionId ? `${TEACH_SOURCE_ID}:${desc.sessionId}` : TEACH_SOURCE_ID, type: "teach" };
|
|
1421
|
+
// One Source per peer NODE, keyed on the stable id the tag carries rather
|
|
1422
|
+
// than the display name beside it: names are user-chosen and collidable, so
|
|
1423
|
+
// two peers who picked the same one would otherwise collapse into a single
|
|
1424
|
+
// Source and corroborate each other for free. Scores at the teach tier —
|
|
1425
|
+
// a peer teaching is still a person telling us something.
|
|
1426
|
+
case "teachNode": return { id: `${TEACH_NODE_SOURCE_ID}:${desc.nodeId}`, type: "teach" };
|
|
789
1427
|
case "provider": return { id: `src:provider:${desc.name}`, type: "provider" };
|
|
790
1428
|
case "corpus": return { id: `src:corpus:${desc.name}`, type: "corpus" };
|
|
791
1429
|
case "corpusWeak": return { id: `src:corpus-weak:${desc.name}`, type: "corpusWeak" };
|
|
@@ -807,25 +1445,6 @@ function sourceIdFor(desc) {
|
|
|
807
1445
|
|
|
808
1446
|
const sourceLabel = (id) => String(id).replace(/^src:/, "");
|
|
809
1447
|
|
|
810
|
-
// The read-side of the Source split ontology/tmct-core.ttl declares. tmct:Source
|
|
811
|
-
// is one flat class over a sourceType key whose values fall into all three of
|
|
812
|
-
// PROV's disjoint top classes; this maps each stored sourceType to its read-side
|
|
813
|
-
// subclass and PROV top class. Nothing on disk changes — sourceType is already
|
|
814
|
-
// stored on every Source — so this is derivation, not migration.
|
|
815
|
-
const PROV_CLASS_BY_SOURCE_TYPE = Object.freeze({
|
|
816
|
-
operator: { subClass: "tmct:AgentSource", prov: "prov:Agent" },
|
|
817
|
-
teach: { subClass: "tmct:AgentSource", prov: "prov:Agent" },
|
|
818
|
-
provider: { subClass: "tmct:AgentSource", prov: "prov:Agent" },
|
|
819
|
-
corpus: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
820
|
-
corpusWeak: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
821
|
-
reference: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
822
|
-
referenceLive: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
823
|
-
web: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
824
|
-
extracted: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
825
|
-
optimisticExtract: { subClass: "tmct:DocumentSource", prov: "prov:Entity" },
|
|
826
|
-
entailed: { subClass: "tmct:ActivitySource", prov: "prov:Activity" },
|
|
827
|
-
});
|
|
828
|
-
|
|
829
1448
|
/** The read-side PROV subclass and top class a Source's mgx:sourceType maps to,
|
|
830
1449
|
* or null for an unrecognised type (never force-fit). Mirrors the Source-split
|
|
831
1450
|
* subclasses of tmct:Source in ontology/tmct-core.ttl. */
|
|
@@ -898,6 +1517,36 @@ function recomputeFactTrust(payload, fact, nowMs = Date.now(), trustOpts = {}) {
|
|
|
898
1517
|
setAttr(fact, UPDATED_AT_PROP, "updatedAt", new Date(nowMs).toISOString());
|
|
899
1518
|
}
|
|
900
1519
|
|
|
1520
|
+
/** Materialise ONE assertion record's own trust: its single source's effective
|
|
1521
|
+
* prior, and nothing else. No recency and no corroboration are folded in here
|
|
1522
|
+
* — recency belongs to the reading moment, and corroboration is a property of
|
|
1523
|
+
* the GROUP, computed fresh over its live heads by readFactRows. The entailed
|
|
1524
|
+
* hook still lands write-time, because only the writer knows the premises: a
|
|
1525
|
+
* derivation is worth its weakest premise times the rule's confidence. */
|
|
1526
|
+
function recomputeAssertionTrust(payload, record, nowMs = Date.now(), trustOpts = {}) {
|
|
1527
|
+
const [sourceId] = statedByObjectsFor(payload, record.id);
|
|
1528
|
+
const source = sourceId ? sourcesByIdMap(payload)[sourceId] : null;
|
|
1529
|
+
const sourceType = (source?.attributes || []).find((a) => a?.prop === "mgx:sourceType")?.value || "";
|
|
1530
|
+
let own = assertionPrior(sourceType, source);
|
|
1531
|
+
if (sourceType === "entailed" && Array.isArray(trustOpts?.premiseTrusts) && trustOpts.premiseTrusts.length) {
|
|
1532
|
+
const ruleConfidence = typeof trustOpts.ruleConfidence === "number" ? trustOpts.ruleConfidence : 1;
|
|
1533
|
+
own = Math.max(0, Math.min(1, Math.min(...trustOpts.premiseTrusts) * ruleConfidence));
|
|
1534
|
+
}
|
|
1535
|
+
const createdAt = (record.attributes || []).find((a) => a?.prop === CREATED_AT_PROP)?.value || "";
|
|
1536
|
+
setAttr(record, TRUST_SCORE_PROP, "trustScore", String(own));
|
|
1537
|
+
setAttr(record, TRUST_INPUTS_PROP, "trustInputs", JSON.stringify({ sourceType, sourceId: sourceId || "", createdAt }));
|
|
1538
|
+
setAttr(record, UPDATED_AT_PROP, "updatedAt", new Date(nowMs).toISOString());
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
/** Re-materialise one individual's stored trust. A Fact is an assertion record
|
|
1542
|
+
* and carries its own single source's prior; a Rule still carries the blended
|
|
1543
|
+
* multi-source score computeTrust has always given it, since a Rule is not
|
|
1544
|
+
* keyed per assertion and has no group to fold. */
|
|
1545
|
+
function rematerialiseTrust(payload, ind, nowMs, trustOpts) {
|
|
1546
|
+
if (ind?.class === FACT_CLASS) recomputeAssertionTrust(payload, ind, nowMs, trustOpts);
|
|
1547
|
+
else recomputeFactTrust(payload, ind, nowMs, trustOpts);
|
|
1548
|
+
}
|
|
1549
|
+
|
|
901
1550
|
/** Reconcile a Fact's Sources + statedBy edges with its (unchanged, compat)
|
|
902
1551
|
* mgx:factProvenance string, then recompute its trust. ADD-only over
|
|
903
1552
|
* deterministic Source ids and upsertEdge's subject>object dedupe, so it is
|
|
@@ -917,7 +1566,7 @@ function syncFactSources(payload, fact, nowMs = Date.now(), trustOpts = {}) {
|
|
|
917
1566
|
subject: fact.id, object: sid, subjectLabel: fact.label, objectLabel: sourceLabel(sid),
|
|
918
1567
|
});
|
|
919
1568
|
}
|
|
920
|
-
|
|
1569
|
+
rematerialiseTrust(payload, fact, nowMs, trustOpts);
|
|
921
1570
|
}
|
|
922
1571
|
|
|
923
1572
|
/** Lazy, idempotent migration of the legacy provenance union (step (b)): any
|
|
@@ -942,11 +1591,20 @@ function migrateLegacyProvenance(payload) {
|
|
|
942
1591
|
if (changed) recountClasses(payload);
|
|
943
1592
|
}
|
|
944
1593
|
|
|
945
|
-
/** A
|
|
946
|
-
*
|
|
947
|
-
* web/provider/entailed Source
|
|
1594
|
+
/** A Source id that names one actor and can therefore hold a track record —
|
|
1595
|
+
* the `${SINGLETON}:<id>` shape, for a local operator/teach session or for a
|
|
1596
|
+
* peer NODE across the mesh. A corpus/web/provider/entailed Source names a
|
|
1597
|
+
* document or a derivation, so there is no actor to score.
|
|
1598
|
+
*
|
|
1599
|
+
* A peer node counts for the same reason a local session does, and it is the
|
|
1600
|
+
* reason this matters most: a node that asserts junk drags its own every-fact
|
|
1601
|
+
* prior toward half, so minting fresh identities to corroborate yourself stops
|
|
1602
|
+
* being free the moment any of those claims is contradicted. */
|
|
948
1603
|
const isSessionScopedSourceId = (id) =>
|
|
949
|
-
typeof id === "string"
|
|
1604
|
+
typeof id === "string"
|
|
1605
|
+
&& (id.startsWith(`${OPERATOR_SOURCE_ID}:`)
|
|
1606
|
+
|| id.startsWith(`${TEACH_SOURCE_ID}:`)
|
|
1607
|
+
|| id.startsWith(`${TEACH_NODE_SOURCE_ID}:`));
|
|
950
1608
|
|
|
951
1609
|
/**
|
|
952
1610
|
* Recompute + materialise mgx:sourceReliability on every session-scoped
|
|
@@ -990,26 +1648,54 @@ function recomputeSourceReliability(payload) {
|
|
|
990
1648
|
for (const e of statedGroup?.examples || []) if (bySource.has(e?.object)) affected.add(e.subject);
|
|
991
1649
|
for (const id of affected) {
|
|
992
1650
|
const ind = idx ? idx.individualsById.get(id) : payload.individuals.find((i) => i?.id === id);
|
|
993
|
-
if (ind)
|
|
1651
|
+
if (ind) rematerialiseTrust(payload, ind);
|
|
994
1652
|
}
|
|
995
1653
|
}
|
|
996
1654
|
|
|
997
1655
|
/** Upsert an individual by id (replace-in-place keeps ordering stable).
|
|
998
1656
|
* Returns the stored reference — callers should index THAT, not `ind`. */
|
|
1657
|
+
/**
|
|
1658
|
+
* Two rollup summaries at one id JOIN instead of overwriting: union the ids
|
|
1659
|
+
* they absorbed, then re-derive count, bounds and prior from that union. This
|
|
1660
|
+
* is what lets two peers that compacted the same group at different moments
|
|
1661
|
+
* converge — union, min and max are all joins, so the result is the same in
|
|
1662
|
+
* either order and applying it twice changes nothing. Re-writing a summary that
|
|
1663
|
+
* already holds everything the incoming one does is therefore a no-op, which is
|
|
1664
|
+
* why compaction's own write can go through this path unchanged.
|
|
1665
|
+
*
|
|
1666
|
+
* Everything that is not a summary keeps plain last-write-wins.
|
|
1667
|
+
*/
|
|
1668
|
+
function joinIfRollup(payload, prior, incoming) {
|
|
1669
|
+
if (!isRollupId(incoming?.id)) return incoming;
|
|
1670
|
+
const sourceType = headRollupTypeOf(incoming.id);
|
|
1671
|
+
if (!sourceType) return mergeRollups(prior, incoming);
|
|
1672
|
+
const sources = sourcesByIdMap(payload);
|
|
1673
|
+
return mergeRollups(prior, incoming, { priorFor: (sid) => assertionPrior(sourceType, sources[sid]) });
|
|
1674
|
+
}
|
|
1675
|
+
|
|
999
1676
|
function upsertIndividual(payload, ind) {
|
|
1000
1677
|
const idx = memoryIndexOf(payload);
|
|
1001
1678
|
if (idx) {
|
|
1002
1679
|
const prior = idx.individualsById.get(ind.id);
|
|
1003
1680
|
if (prior) {
|
|
1004
|
-
Object.assign(prior, ind);
|
|
1681
|
+
Object.assign(prior, joinIfRollup(payload, prior, ind));
|
|
1005
1682
|
return prior;
|
|
1006
1683
|
}
|
|
1007
1684
|
payload.individuals.push(ind);
|
|
1008
1685
|
idx.individualsById.set(ind.id, ind);
|
|
1686
|
+
if (ind.class === FACT_CLASS) {
|
|
1687
|
+
const groupId = factGroupId(ind.id);
|
|
1688
|
+
const held = idx.factRecordsByGroup.get(groupId);
|
|
1689
|
+
if (held) held.push(ind.id);
|
|
1690
|
+
else idx.factRecordsByGroup.set(groupId, [ind.id]);
|
|
1691
|
+
}
|
|
1009
1692
|
return ind;
|
|
1010
1693
|
}
|
|
1011
1694
|
const i = payload.individuals.findIndex((x) => x?.id === ind.id);
|
|
1012
|
-
if (i >= 0) {
|
|
1695
|
+
if (i >= 0) {
|
|
1696
|
+
payload.individuals[i] = joinIfRollup(payload, payload.individuals[i], ind);
|
|
1697
|
+
return payload.individuals[i];
|
|
1698
|
+
}
|
|
1013
1699
|
payload.individuals.push(ind);
|
|
1014
1700
|
return ind;
|
|
1015
1701
|
}
|
|
@@ -1067,7 +1753,7 @@ function upsertEdge(payload, { predicate, prop }, edge) {
|
|
|
1067
1753
|
* and sampled the way graph-build.mjs counts the code classes. */
|
|
1068
1754
|
function recountClasses(payload) {
|
|
1069
1755
|
const names = [MEMORY_SESSION_CLASS, UTTERANCE_CLASS, FACT_CLASS, SOURCE_CLASS, RULE_CLASS];
|
|
1070
|
-
payload.classes = payload.classes.filter((c) => !names.includes(c?.name));
|
|
1756
|
+
payload.classes = (payload.classes || []).filter((c) => !names.includes(c?.name));
|
|
1071
1757
|
for (const name of names) {
|
|
1072
1758
|
const of = payload.individuals.filter((i) => i?.class === name);
|
|
1073
1759
|
if (of.length) payload.classes.push({ name, count: of.length, sample: of.slice(0, 3).map((i) => i.label) });
|
|
@@ -1166,60 +1852,402 @@ export async function appendCanonicalisedFromEdges(dir, links) {
|
|
|
1166
1852
|
if (!links?.length) return;
|
|
1167
1853
|
await mutateMemory(dir, (payload) => {
|
|
1168
1854
|
for (const l of links) {
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1855
|
+
// Callers name the fact the way every citation does — by its group id.
|
|
1856
|
+
// An edge has to land on a node a walker can dereference, so it resolves
|
|
1857
|
+
// to the records asserting that triple; each one really was canonicalised
|
|
1858
|
+
// from the same utterance.
|
|
1859
|
+
for (const factId of factRecordIdsFor(payload, l.factId)) {
|
|
1860
|
+
upsertEdge(payload, { predicate: "canonicalisedFrom", prop: CANONICALISED_FROM_PROP }, {
|
|
1861
|
+
subject: factId, object: l.uttId, subjectLabel: l.factLabel, objectLabel: l.uttLabel,
|
|
1862
|
+
});
|
|
1863
|
+
}
|
|
1172
1864
|
}
|
|
1173
1865
|
});
|
|
1174
1866
|
}
|
|
1175
1867
|
|
|
1176
|
-
/**
|
|
1177
|
-
*
|
|
1178
|
-
*
|
|
1179
|
-
*
|
|
1180
|
-
|
|
1181
|
-
|
|
1868
|
+
/** Every record id asserting one triple, in payload order. The bridge between
|
|
1869
|
+
* the PUBLIC fact id (the group) and the individuals actually holding it, so a
|
|
1870
|
+
* caller that names a fact the way every citation and premise list does still
|
|
1871
|
+
* reaches real nodes. Empty for a triple nothing asserts. */
|
|
1872
|
+
export function factRecordIdsFor(payload, groupId) {
|
|
1873
|
+
const idx = memoryIndexOf(payload);
|
|
1874
|
+
if (idx) return (idx.factRecordsByGroup.get(groupId) || []).slice();
|
|
1875
|
+
return (payload?.individuals || [])
|
|
1876
|
+
.filter((i) => i?.class === FACT_CLASS && factGroupId(i.id) === groupId)
|
|
1877
|
+
.map((i) => i.id);
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
/** The assertion groups one write lands, given the triple it targets. Normally
|
|
1881
|
+
* one per Source its provenance names. The `src:none` singleton is the
|
|
1882
|
+
* exception: it exists so a fact nobody can be credited for still HAS a
|
|
1883
|
+
* record, so it is minted only when the triple has no record at all yet. A
|
|
1884
|
+
* provenance-less write onto an already-asserted triple names no new source,
|
|
1885
|
+
* so it files no second, unattributable sibling beside the real ones — its
|
|
1886
|
+
* triple-level payload lands through restateFactGroup instead. */
|
|
1887
|
+
function assertionGroupsFor(payload, groupId, provenance) {
|
|
1888
|
+
const groups = groupTagsBySource(provenance);
|
|
1889
|
+
if (groups.length === 1 && groups[0].sourceId === NO_SOURCE_ID) {
|
|
1890
|
+
if (factRecordIdsFor(payload, groupId).length) return [];
|
|
1891
|
+
}
|
|
1892
|
+
// A source this group has already compacted away stays compacted. Every
|
|
1893
|
+
// delivery path for a fact lands here, so this is where a late or re-synced
|
|
1894
|
+
// copy of an absorbed assertion is recognized and dropped rather than
|
|
1895
|
+
// inserted — without it the next sync resurrects everything compaction just
|
|
1896
|
+
// folded, which is what makes deleting from a replicated set hard at all.
|
|
1897
|
+
const rollups = headRollupsFor(payload, groupId);
|
|
1898
|
+
if (!rollups.length) return groups;
|
|
1899
|
+
return groups.filter((group) => !isAbsorbedSource(rollups, group.sourceId));
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
/** A group's pool-1 summaries, one per compacted source type. Reads the index
|
|
1903
|
+
* directly rather than through factRecordIdsFor, because this runs on EVERY
|
|
1904
|
+
* fact write and nearly always finds nothing — the common case must not pay
|
|
1905
|
+
* for an array copy. */
|
|
1906
|
+
function headRollupsFor(payload, groupId) {
|
|
1907
|
+
const idx = memoryIndexOf(payload);
|
|
1908
|
+
const ids = idx ? idx.factRecordsByGroup.get(groupId) : factRecordIdsFor(payload, groupId);
|
|
1909
|
+
const rollups = [];
|
|
1910
|
+
for (const id of ids || []) {
|
|
1911
|
+
if (!isHeadRollupId(id)) continue;
|
|
1912
|
+
const record = storedIndividual(payload, id);
|
|
1913
|
+
if (record) rollups.push(record);
|
|
1914
|
+
}
|
|
1915
|
+
return rollups;
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
/**
|
|
1919
|
+
* Apply a provenance-less write to the records a triple already has. Naming no
|
|
1920
|
+
* source, it asserts nothing new — but it can still carry triple-level payload
|
|
1921
|
+
* that belongs on the records already there, which is how a caller re-states a
|
|
1922
|
+
* derivation's premises without claiming to be a fresh witness.
|
|
1923
|
+
*
|
|
1924
|
+
* Premise environments are a property of the TRIPLE, and readFactRows unions
|
|
1925
|
+
* them across the group, so a re-statement REPLACES the group's whole view of
|
|
1926
|
+
* them. That is what lets a retraction actually prune one: writing to a single
|
|
1927
|
+
* record would leave a sibling rule's copy behind and the union would put it
|
|
1928
|
+
* straight back. They land where they already live when anything holds them,
|
|
1929
|
+
* since a justification explains an entailment, not the corroborators beside it.
|
|
1930
|
+
*
|
|
1931
|
+
* Returns the record ids it touched.
|
|
1932
|
+
*/
|
|
1933
|
+
function restateFactGroup(payload, groupId, { quantifier, environments }) {
|
|
1934
|
+
const live = factRecordIdsFor(payload, groupId)
|
|
1935
|
+
.map((id) => storedIndividual(payload, id))
|
|
1936
|
+
.filter((record) => record && !(record.attributes || []).some((a) => a?.prop === SUPERSEDED_BY_PROP));
|
|
1937
|
+
const touched = new Set();
|
|
1938
|
+
if (environments) {
|
|
1939
|
+
const carriers = live.filter((r) => (r.attributes || []).some((a) => a?.prop === "mgx:factJustification"));
|
|
1940
|
+
for (const record of carriers.length ? carriers : live) {
|
|
1941
|
+
setAttr(record, "mgx:factJustification", "justification", environments.map((e) => e.join(" ")).join(" | "));
|
|
1942
|
+
touched.add(record.id);
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
if (quantifier) {
|
|
1946
|
+
for (const record of live) {
|
|
1947
|
+
// first-write-wins, exactly as a tagged re-assert treats it
|
|
1948
|
+
if ((record.attributes || []).some((a) => a?.prop === "mgx:factQuantifier")) continue;
|
|
1949
|
+
setAttr(record, "mgx:factQuantifier", "quantifier", quantifier);
|
|
1950
|
+
touched.add(record.id);
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
return [...touched];
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
/** How deep this source's own chain for this triple already runs: the version
|
|
1957
|
+
* counts how many times the source has replaced its own record here, so the
|
|
1958
|
+
* first demotion is `#v1` and the oldest leaf keeps the lowest number. Read
|
|
1959
|
+
* off the head's own backward link, which is O(1) and always names the leaf
|
|
1960
|
+
* immediately behind it. */
|
|
1961
|
+
function nextChainVersion(head) {
|
|
1962
|
+
let deepest = 0;
|
|
1963
|
+
const behind = (head.attributes || []).find((a) => a?.prop === SUPERSEDES_PROP)?.value || "";
|
|
1964
|
+
for (const id of behind.split(" ").filter(Boolean)) {
|
|
1965
|
+
const m = /#v([1-9][0-9]*)$/.exec(id);
|
|
1966
|
+
if (m) deepest = Math.max(deepest, Number(m[1]));
|
|
1967
|
+
}
|
|
1968
|
+
return deepest + 1;
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
/**
|
|
1972
|
+
* Plan ONE source's assertion of one triple: the record it wants to write, plus
|
|
1973
|
+
* the demotion that implies when the source is replacing its own earlier belief.
|
|
1974
|
+
* Pure — nothing lands until applyFactAssertion — so the SHACL gate gets to
|
|
1975
|
+
* reject a malformed record while the payload is still untouched.
|
|
1976
|
+
*
|
|
1977
|
+
* Three outcomes, and only the first writes anything new:
|
|
1978
|
+
* - a source this triple has never heard from: a fresh record;
|
|
1979
|
+
* - the same source, genuinely newer: a new HEAD at the same stable id, the
|
|
1980
|
+
* record it replaces kept whole under `#v<n>` and linked both ways;
|
|
1981
|
+
* - the same source saying the same thing again: its tags union onto the head
|
|
1982
|
+
* and its first write's stamps stand. An exact re-delivery changes nothing,
|
|
1983
|
+
* which is what keeps a re-seed and a duplicate mesh path idempotent.
|
|
1984
|
+
*/
|
|
1985
|
+
function planFactAssertion(payload, spec) {
|
|
1986
|
+
const { groupId, s, p, o, label, tokens, group, createdAt, observedAt, quantifier, environments } = spec;
|
|
1987
|
+
const recordId = `${groupId}@${group.sourceId}`;
|
|
1988
|
+
const idx = memoryIndexOf(payload);
|
|
1989
|
+
const head = idx ? idx.individualsById.get(recordId) : payload.individuals.find((x) => x?.id === recordId);
|
|
1990
|
+
const headAttr = (prop) => (head?.attributes || []).find((a) => a?.prop === prop)?.value || "";
|
|
1991
|
+
const headTags = headAttr("mgx:factProvenance").split(" | ").filter(Boolean);
|
|
1992
|
+
const incoming = { assertedAt: embeddedTagTimestamp(group.tags), observedAt };
|
|
1993
|
+
|
|
1994
|
+
let demote = null;
|
|
1995
|
+
let tags = group.tags;
|
|
1996
|
+
let createdAtVal = incoming.assertedAt || createdAt || nowIso();
|
|
1997
|
+
let observedAtVal = observedAt;
|
|
1998
|
+
let supersedes = [];
|
|
1999
|
+
let quantifierVal = quantifier;
|
|
2000
|
+
|
|
2001
|
+
if (head) {
|
|
2002
|
+
const current = { assertedAt: embeddedTagTimestamp(headTags), observedAt: headAttr(OBSERVED_AT_PROP) };
|
|
2003
|
+
if (supersedesPriorAssertion(incoming, current)) {
|
|
2004
|
+
demote = { head, id: `${recordId}#v${nextChainVersion(head)}` };
|
|
2005
|
+
supersedes = [demote.id];
|
|
2006
|
+
} else {
|
|
2007
|
+
tags = [...new Set([...headTags, ...group.tags])];
|
|
2008
|
+
createdAtVal = headAttr(CREATED_AT_PROP) || createdAtVal;
|
|
2009
|
+
observedAtVal = observedAt || headAttr(OBSERVED_AT_PROP);
|
|
2010
|
+
supersedes = headAttr(SUPERSEDES_PROP).split(" ").filter(Boolean);
|
|
2011
|
+
}
|
|
2012
|
+
// A re-assert carrying no quantifier never SILENTLY erases one already
|
|
2013
|
+
// recorded — the same first-write-wins discipline createdAt keeps.
|
|
2014
|
+
quantifierVal = quantifier || headAttr("mgx:factQuantifier");
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
const candidate = {
|
|
2018
|
+
id: recordId, label, class: FACT_CLASS,
|
|
2019
|
+
derived_from: [], mentions: [],
|
|
2020
|
+
attributes: [
|
|
2021
|
+
{ prop: "rdf:type", key: "type", value: "rdf:Statement" },
|
|
2022
|
+
{ prop: "rdf:subject", key: "subject", value: s },
|
|
2023
|
+
{ prop: "rdf:predicate", key: "predicate", value: p },
|
|
2024
|
+
{ prop: "rdf:object", key: "object", value: o },
|
|
2025
|
+
{ prop: CREATED_AT_PROP, key: "createdAt", value: createdAtVal },
|
|
2026
|
+
{ prop: SOURCE_ID_PROP, key: "sourceId", value: group.sourceId },
|
|
2027
|
+
...(tags.length ? [{ prop: "mgx:factProvenance", key: "provenance", value: tags.join(" | ") }] : []),
|
|
2028
|
+
...(observedAtVal ? [{ prop: OBSERVED_AT_PROP, key: "observedAt", value: observedAtVal }] : []),
|
|
2029
|
+
...(tokens.length ? [{ prop: "mgx:hasProseTokens", key: "prose_tokens", value: tokens.join(" ") }] : []),
|
|
2030
|
+
...(quantifierVal ? [{ prop: "mgx:factQuantifier", key: "quantifier", value: quantifierVal }] : []),
|
|
2031
|
+
...(environments ? [{ prop: "mgx:factJustification", key: "justification", value: environments.map((e) => e.join(" ")).join(" | ") }] : []),
|
|
2032
|
+
...(supersedes.length ? [{ prop: SUPERSEDES_PROP, key: "supersedes", value: supersedes.join(" ") }] : []),
|
|
2033
|
+
],
|
|
2034
|
+
};
|
|
2035
|
+
return { candidate, demote };
|
|
2036
|
+
}
|
|
2037
|
+
|
|
2038
|
+
/** Drop a triple's `src:none` record once a real source asserts it. The
|
|
2039
|
+
* singleton exists so a fact nobody can be credited for still HAS a record; it
|
|
2040
|
+
* contributes no Source, no statedBy edge and no trust, so the moment someone
|
|
2041
|
+
* can be credited it is pure clutter. Cheap: the O(1) index check fails for
|
|
2042
|
+
* nearly every write, and only a group that actually holds a placeholder pays
|
|
2043
|
+
* for the removal. */
|
|
2044
|
+
function absorbAnonymousRecord(payload, groupId) {
|
|
2045
|
+
const anonymousId = `${groupId}@${NO_SOURCE_ID}`;
|
|
2046
|
+
const idx = memoryIndexOf(payload);
|
|
2047
|
+
if (idx ? !idx.individualsById.has(anonymousId) : !payload.individuals.some((i) => i?.id === anonymousId)) return;
|
|
2048
|
+
payload.individuals = payload.individuals.filter((i) => i?.id !== anonymousId);
|
|
2049
|
+
if (!idx) return;
|
|
2050
|
+
idx.individualsById.delete(anonymousId);
|
|
2051
|
+
const held = (idx.factRecordsByGroup.get(groupId) || []).filter((id) => id !== anonymousId);
|
|
2052
|
+
if (held.length) idx.factRecordsByGroup.set(groupId, held);
|
|
2053
|
+
else idx.factRecordsByGroup.delete(groupId);
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
/** Land a planned assertion. The demoted record is never deleted and never
|
|
2057
|
+
* rewritten: it moves to its own id carrying the same bytes, plus the forward
|
|
2058
|
+
* link that makes it a leaf rather than a head. Returns every record id this
|
|
2059
|
+
* touched, so the caller reconciles Sources and trust once per record. */
|
|
2060
|
+
function applyFactAssertion(payload, { candidate, demote }) {
|
|
2061
|
+
const touched = [];
|
|
2062
|
+
if (!candidate.id.endsWith(`@${NO_SOURCE_ID}`)) absorbAnonymousRecord(payload, factGroupId(candidate.id));
|
|
2063
|
+
if (demote) {
|
|
2064
|
+
const leaf = { ...demote.head, id: demote.id, attributes: (demote.head.attributes || []).map((a) => ({ ...a })) };
|
|
2065
|
+
setAttr(leaf, SUPERSEDED_BY_PROP, "supersededBy", candidate.id);
|
|
2066
|
+
upsertIndividual(payload, leaf); // before the head is overwritten in place
|
|
2067
|
+
touched.push(leaf.id);
|
|
2068
|
+
}
|
|
2069
|
+
upsertIndividual(payload, candidate);
|
|
2070
|
+
touched.push(candidate.id);
|
|
2071
|
+
return touched;
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
/** The stored individual for a record id — upsertIndividual replaces in place,
|
|
2075
|
+
* so callers must reconcile against what the payload actually holds. */
|
|
2076
|
+
function storedIndividual(payload, id) {
|
|
2077
|
+
const idx = memoryIndexOf(payload);
|
|
2078
|
+
return idx ? idx.individualsById.get(id) : payload.individuals.find((x) => x?.id === id);
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
/** Remove absorbed records and scrub any edge that named them, mirroring
|
|
2082
|
+
* removeFacts' own discipline. An orphaned Source is left in place: a source
|
|
2083
|
+
* whose assertion was compacted is still a real source with a track record. */
|
|
2084
|
+
function dropAbsorbedRecords(payload, ids) {
|
|
2085
|
+
const drop = new Set(ids);
|
|
2086
|
+
if (!drop.size) return;
|
|
2087
|
+
payload.individuals = (payload.individuals || []).filter((ind) => !drop.has(ind?.id));
|
|
2088
|
+
for (const group of payload.objectProperties || []) {
|
|
2089
|
+
const before = group.examples || [];
|
|
2090
|
+
const after = before.filter((e) => !drop.has(e?.subject) && !drop.has(e?.object));
|
|
2091
|
+
if (after.length === before.length) continue;
|
|
2092
|
+
group.examples = after;
|
|
2093
|
+
group.count = after.length;
|
|
2094
|
+
}
|
|
2095
|
+
const idx = memoryIndexOf(payload);
|
|
2096
|
+
if (!idx) return;
|
|
2097
|
+
for (const id of drop) {
|
|
2098
|
+
idx.individualsById.delete(id);
|
|
2099
|
+
idx.statedByBySubject.delete(id);
|
|
2100
|
+
const groupId = factGroupId(id);
|
|
2101
|
+
const held = (idx.factRecordsByGroup.get(groupId) || []).filter((x) => x !== id);
|
|
2102
|
+
if (held.length) idx.factRecordsByGroup.set(groupId, held);
|
|
2103
|
+
else idx.factRecordsByGroup.delete(groupId);
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
/**
|
|
2108
|
+
* Bound one triple's record growth, on the write that grew it. Two pools, two
|
|
2109
|
+
* triggers, two summaries, never mixed:
|
|
2110
|
+
*
|
|
2111
|
+
* - pool 1 absorbs the OLDEST live heads of one source TYPE once that type
|
|
2112
|
+
* holds GROUP_ROLLUP_THRESHOLD of them, keeping the newest
|
|
2113
|
+
* ROLLUP_KEEP_PER_TYPE intact. Its summary carries a prior, because every
|
|
2114
|
+
* head it absorbed was a live vote in the group fold and dropping that
|
|
2115
|
+
* contribution would silently under-trust the compacted answer.
|
|
2116
|
+
* - pool 2 absorbs the OLDEST demoted leaves of ONE source's own chain once
|
|
2117
|
+
* that chain holds CHAIN_ROLLUP_THRESHOLD of them, keeping the newest
|
|
2118
|
+
* CHAIN_KEEP_DEPTH. Its summary carries no prior: a demoted leaf counts for
|
|
2119
|
+
* nothing while it stands, and compacting it must not change that.
|
|
2120
|
+
*
|
|
2121
|
+
* A head absorbed by pool 1 takes its own chain with it. The chain is reachable
|
|
2122
|
+
* only through the head, so once the head is summarized its history answers no
|
|
2123
|
+
* question this model asks, and leaving it behind would orphan it.
|
|
2124
|
+
*
|
|
2125
|
+
* Returns the record ids it absorbed rather than removing them, so a batch that
|
|
2126
|
+
* compacts many groups pays for one sweep of the payload instead of one per
|
|
2127
|
+
* group. Groups are independent, so a still-present absorbed record can never
|
|
2128
|
+
* affect another group's planning.
|
|
2129
|
+
*/
|
|
2130
|
+
function compactFactGroup(payload, groupId) {
|
|
2131
|
+
const idx = memoryIndexOf(payload);
|
|
2132
|
+
const ids = (idx ? idx.factRecordsByGroup.get(groupId) : factRecordIdsFor(payload, groupId)) || [];
|
|
2133
|
+
// Nearly every fact has no summary in either pool, and the record count says
|
|
2134
|
+
// so before anything else is read: pool 2 has the lower trigger, so a group
|
|
2135
|
+
// under it can fire neither pool.
|
|
2136
|
+
if (ids.length < CHAIN_ROLLUP_THRESHOLD) return [];
|
|
2137
|
+
|
|
2138
|
+
const headsByType = new Map();
|
|
2139
|
+
const leavesBySource = new Map();
|
|
2140
|
+
const headRollupByType = new Map();
|
|
2141
|
+
const chainRollupBySource = new Map();
|
|
2142
|
+
for (const id of ids.slice()) {
|
|
2143
|
+
const record = storedIndividual(payload, id);
|
|
2144
|
+
if (!record) continue;
|
|
2145
|
+
const attrOf = (prop) => (record.attributes || []).find((a) => a?.prop === prop)?.value || "";
|
|
2146
|
+
if (isHeadRollupId(id)) { headRollupByType.set(headRollupTypeOf(id), record); continue; }
|
|
2147
|
+
if (isChainRollupId(id)) { chainRollupBySource.set(attrOf(SOURCE_ID_PROP), record); continue; }
|
|
2148
|
+
const provenance = attrOf("mgx:factProvenance");
|
|
2149
|
+
const source = primarySourceOf(provenance);
|
|
2150
|
+
const tags = provenance.split(" | ").filter(Boolean);
|
|
2151
|
+
const entry = {
|
|
2152
|
+
id,
|
|
2153
|
+
sourceId: attrOf(SOURCE_ID_PROP) || source.id,
|
|
2154
|
+
assertedAt: assertionTimestampFor(tags, attrOf(CREATED_AT_PROP)),
|
|
2155
|
+
record,
|
|
2156
|
+
};
|
|
2157
|
+
if (attrOf(SUPERSEDED_BY_PROP)) {
|
|
2158
|
+
const chain = leavesBySource.get(entry.sourceId);
|
|
2159
|
+
if (chain) chain.push(entry);
|
|
2160
|
+
else leavesBySource.set(entry.sourceId, [entry]);
|
|
2161
|
+
continue;
|
|
2162
|
+
}
|
|
2163
|
+
// src:none stands for "no source at all" — it has no type, so it belongs to
|
|
2164
|
+
// no per-type pool and is never compacted.
|
|
2165
|
+
if (!source.type) continue;
|
|
2166
|
+
const heads = headsByType.get(source.type);
|
|
2167
|
+
if (heads) heads.push(entry);
|
|
2168
|
+
else headsByType.set(source.type, [entry]);
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
const sources = sourcesByIdMap(payload);
|
|
2172
|
+
const absorbed = [];
|
|
2173
|
+
|
|
2174
|
+
for (const [sourceType, heads] of headsByType) {
|
|
2175
|
+
const plan = planHeadRollup({
|
|
2176
|
+
groupId, sourceType, heads,
|
|
2177
|
+
existing: headRollupByType.get(sourceType) || null,
|
|
2178
|
+
priorFor: (sid) => assertionPrior(sourceType, sources[sid]),
|
|
2179
|
+
});
|
|
2180
|
+
if (!plan) continue;
|
|
2181
|
+
upsertIndividual(payload, plan.rollup);
|
|
2182
|
+
absorbed.push(...plan.absorbed);
|
|
2183
|
+
for (const sourceId of plan.absorbedSourceIds) {
|
|
2184
|
+
for (const leaf of leavesBySource.get(sourceId) || []) absorbed.push(leaf.id);
|
|
2185
|
+
leavesBySource.delete(sourceId);
|
|
2186
|
+
const chainRollup = chainRollupBySource.get(sourceId);
|
|
2187
|
+
if (chainRollup) absorbed.push(chainRollup.id);
|
|
2188
|
+
chainRollupBySource.delete(sourceId);
|
|
2189
|
+
}
|
|
2190
|
+
}
|
|
2191
|
+
|
|
2192
|
+
for (const [sourceId, leaves] of leavesBySource) {
|
|
2193
|
+
const plan = planChainRollup({
|
|
2194
|
+
groupId, sourceId, leaves,
|
|
2195
|
+
existing: chainRollupBySource.get(sourceId) || null,
|
|
2196
|
+
});
|
|
2197
|
+
if (!plan) continue;
|
|
2198
|
+
upsertIndividual(payload, plan.rollup);
|
|
2199
|
+
absorbed.push(...plan.absorbed);
|
|
2200
|
+
// Keep the chain walkable backward: the oldest leaf still standing points
|
|
2201
|
+
// at the summary rather than at a record that no longer exists. A walk that
|
|
2202
|
+
// reaches it and needs a point inside the absorbed span gets the summary's
|
|
2203
|
+
// own bounds, never a fabricated instant.
|
|
2204
|
+
const rewired = storedIndividual(payload, plan.rewire);
|
|
2205
|
+
if (rewired) setAttr(rewired, SUPERSEDES_PROP, "supersedes", plan.rollup.id);
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
return absorbed;
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
/** Append one grammar-derived OWL triple, RDF-reified as a `Fact` individual —
|
|
2212
|
+
* one record per asserting SOURCE, all of them sharing the content-addressed
|
|
2213
|
+
* group id this returns. Same (s,p,o) from the same source resolves onto that
|
|
2214
|
+
* source's own lineage, never a duplicate. `premiseTrusts`/`ruleConfidence`
|
|
2215
|
+
* optionally engage trust.mjs's entailed hook; `observedAt` records when the
|
|
2216
|
+
* asserting party WITNESSED the claim, which is not when this store heard it.
|
|
2217
|
+
* Validated against ontology/memory-shapes.ttl (memory/shacl.mjs) before the
|
|
2218
|
+
* write. Returns { id } — the group id, the public fact id every reader uses. */
|
|
2219
|
+
export async function appendFact(dir, { subject, predicate, object, provenance = "", createdAt = "", observedAt = "", quantifier = "", premiseTrusts, ruleConfidence } = {}) {
|
|
1182
2220
|
const s = normFactTerm(subject);
|
|
1183
2221
|
const p = normFactPredicate(predicate);
|
|
1184
2222
|
const o = normFactTerm(object);
|
|
1185
2223
|
if (!s || !p || !o) throw new Error("a fact needs subject, predicate and object");
|
|
1186
|
-
const
|
|
2224
|
+
const groupId = factIdFor(s, p, o);
|
|
1187
2225
|
const text = `${s} ${p} ${o}`;
|
|
1188
2226
|
const tokens = proseTokensFor({ doc: text });
|
|
1189
2227
|
const q = normText(quantifier);
|
|
1190
2228
|
await mutateMemory(dir, async (payload) => {
|
|
1191
|
-
const
|
|
1192
|
-
const
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
const
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
...(provs.length ? [{ prop: "mgx:factProvenance", key: "provenance", value: provs.join(" | ") }] : []),
|
|
1211
|
-
...(tokens.length ? [{ prop: "mgx:hasProseTokens", key: "prose_tokens", value: tokens.join(" ") }] : []),
|
|
1212
|
-
...(qVal ? [{ prop: "mgx:factQuantifier", key: "quantifier", value: qVal }] : []),
|
|
1213
|
-
],
|
|
1214
|
-
};
|
|
1215
|
-
await assertIndividualValid(candidate); // the SHACL gate -- throws, never writes, on a violation
|
|
1216
|
-
upsertIndividual(payload, candidate);
|
|
1217
|
-
// Derive Source individuals + statedBy edges from the provenance union and
|
|
1218
|
-
// (re)materialise this fact's trust — the live half of steps (b)/(c).
|
|
1219
|
-
syncFactSources(payload, payload.individuals.find((x) => x?.id === id), undefined, { premiseTrusts, ruleConfidence });
|
|
2229
|
+
const groups = assertionGroupsFor(payload, groupId, normText(provenance));
|
|
2230
|
+
for (const id of groups.length ? [] : restateFactGroup(payload, groupId, { quantifier: q })) {
|
|
2231
|
+
syncFactSources(payload, storedIndividual(payload, id), undefined, { premiseTrusts, ruleConfidence });
|
|
2232
|
+
}
|
|
2233
|
+
for (const group of groups) {
|
|
2234
|
+
const plan = planFactAssertion(payload, {
|
|
2235
|
+
groupId, s, p, o, label: labelOf(text), tokens, group, createdAt, observedAt, quantifier: q,
|
|
2236
|
+
});
|
|
2237
|
+
await assertIndividualValid(plan.candidate); // the SHACL gate -- throws, never writes, on a violation
|
|
2238
|
+
for (const id of applyFactAssertion(payload, plan)) {
|
|
2239
|
+
// Derive the Source individual + statedBy edge from this record's own
|
|
2240
|
+
// tag(s) and materialise its single-source trust. The entailed hook
|
|
2241
|
+
// rides the new head only; a demoted leaf keeps the trust it earned.
|
|
2242
|
+
syncFactSources(payload, storedIndividual(payload, id), undefined,
|
|
2243
|
+
id === plan.candidate.id ? { premiseTrusts, ruleConfidence } : undefined);
|
|
2244
|
+
}
|
|
2245
|
+
}
|
|
2246
|
+
// After the Sources exist, so a summary can price what it absorbs.
|
|
2247
|
+
dropAbsorbedRecords(payload, compactFactGroup(payload, groupId));
|
|
1220
2248
|
recountClasses(payload);
|
|
1221
2249
|
});
|
|
1222
|
-
return { id };
|
|
2250
|
+
return { id: groupId };
|
|
1223
2251
|
}
|
|
1224
2252
|
|
|
1225
2253
|
/** Normalize appendFacts' `justification` input — either a flat premise-id
|
|
@@ -1269,6 +2297,7 @@ export async function appendFacts(dir, facts) {
|
|
|
1269
2297
|
tokens: proseTokensFor({ doc: text }),
|
|
1270
2298
|
provenance: normText(f?.provenance),
|
|
1271
2299
|
createdAt: f?.createdAt || "",
|
|
2300
|
+
observedAt: f?.observedAt || "",
|
|
1272
2301
|
quantifier: normText(f?.quantifier),
|
|
1273
2302
|
premiseTrusts: Array.isArray(f?.premiseTrusts) ? f.premiseTrusts : undefined,
|
|
1274
2303
|
ruleConfidence: typeof f?.ruleConfidence === "number" ? f.ruleConfidence : undefined,
|
|
@@ -1278,66 +2307,48 @@ export async function appendFacts(dir, facts) {
|
|
|
1278
2307
|
const ids = [];
|
|
1279
2308
|
if (!prepared.length) return { ids, appended: 0, skipped };
|
|
1280
2309
|
await mutateMemory(dir, (payload) => {
|
|
1281
|
-
// id → individual index for O(1) upsert (the array grows to thousands).
|
|
1282
|
-
// When mutateMemory already built the Symbol-keyed lookup index, reuse
|
|
1283
|
-
// THAT Map directly (same object) instead of rescanning payload.individuals
|
|
1284
|
-
// a second time — every `byId.set` below then also keeps
|
|
1285
|
-
// idx.individualsById correct for upsertSource/recomputeSourceReliability's
|
|
1286
|
-
// later lookups in this same mutation, with no extra write.
|
|
1287
|
-
const idx = memoryIndexOf(payload);
|
|
1288
|
-
const byId = idx ? idx.individualsById : new Map(payload.individuals.map((i) => [i?.id, i]));
|
|
1289
2310
|
const touched = [];
|
|
1290
2311
|
const seen = new Set();
|
|
1291
2312
|
const trustOptsById = new Map();
|
|
1292
2313
|
for (const f of prepared) {
|
|
1293
|
-
const
|
|
1294
|
-
|
|
1295
|
-
//
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
const
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
// other upsert path now. Previously this did its own inline
|
|
1320
|
-
// `payload.individuals.indexOf(prior)` array scan on a re-assert within
|
|
1321
|
-
// the same batch — an O(n) fallback that could still blow up a batch
|
|
1322
|
-
// heavy with within-file duplicate triples; upsertIndividual has no
|
|
1323
|
-
// such case left.
|
|
1324
|
-
const stored = upsertIndividual(payload, ind);
|
|
1325
|
-
byId.set(f.id, stored);
|
|
1326
|
-
ids.push(f.id);
|
|
1327
|
-
if (!seen.has(f.id)) { seen.add(f.id); touched.push(f.id); }
|
|
1328
|
-
// Last-prepared-row-wins per id for the trust hook opts (mirrors the
|
|
1329
|
-
// provenance/quantifier/ind upsert above, which is also last-wins per id
|
|
1330
|
-
// within one batch — a duplicate id inside the same call is rare, but
|
|
1331
|
-
// when it happens the SAME single-write-per-id discipline applies here).
|
|
1332
|
-
if (f.premiseTrusts !== undefined || f.ruleConfidence !== undefined) {
|
|
1333
|
-
trustOptsById.set(f.id, { premiseTrusts: f.premiseTrusts, ruleConfidence: f.ruleConfidence });
|
|
2314
|
+
const groups = assertionGroupsFor(payload, f.id, f.provenance);
|
|
2315
|
+
// Naming no source, this write asserts nothing new — but its premise
|
|
2316
|
+
// environments and quantifier still belong on the records already there.
|
|
2317
|
+
for (const id of groups.length ? [] : restateFactGroup(payload, f.id, { quantifier: f.quantifier, environments: f.environments })) {
|
|
2318
|
+
if (!seen.has(id)) { seen.add(id); touched.push(id); }
|
|
2319
|
+
if (f.premiseTrusts !== undefined || f.ruleConfidence !== undefined) {
|
|
2320
|
+
trustOptsById.set(id, { premiseTrusts: f.premiseTrusts, ruleConfidence: f.ruleConfidence });
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
for (const group of groups) {
|
|
2324
|
+
const plan = planFactAssertion(payload, {
|
|
2325
|
+
groupId: f.id, s: f.s, p: f.p, o: f.o, label: labelOf(f.text), tokens: f.tokens,
|
|
2326
|
+
group, createdAt: f.createdAt, observedAt: f.observedAt, quantifier: f.quantifier,
|
|
2327
|
+
environments: f.environments,
|
|
2328
|
+
});
|
|
2329
|
+
for (const id of applyFactAssertion(payload, plan)) {
|
|
2330
|
+
if (seen.has(id)) continue;
|
|
2331
|
+
seen.add(id);
|
|
2332
|
+
touched.push(id);
|
|
2333
|
+
}
|
|
2334
|
+
// Last-prepared-row-wins per record for the trust hook opts (mirroring
|
|
2335
|
+
// the upsert above, which is also last-wins within one batch), and only
|
|
2336
|
+
// ever on the head — a demoted leaf keeps the trust it earned.
|
|
2337
|
+
if (f.premiseTrusts !== undefined || f.ruleConfidence !== undefined) {
|
|
2338
|
+
trustOptsById.set(plan.candidate.id, { premiseTrusts: f.premiseTrusts, ruleConfidence: f.ruleConfidence });
|
|
2339
|
+
}
|
|
1334
2340
|
}
|
|
2341
|
+
ids.push(f.id); // the group id — the public fact id, one per prepared triple
|
|
1335
2342
|
}
|
|
1336
|
-
// Reconcile each touched
|
|
1337
|
-
// then recount classes a SINGLE time at the end.
|
|
1338
|
-
|
|
1339
|
-
//
|
|
1340
|
-
|
|
2343
|
+
// Reconcile each touched record's Source + trust once (add-only,
|
|
2344
|
+
// idempotent), then recount classes a SINGLE time at the end.
|
|
2345
|
+
for (const id of touched) syncFactSources(payload, storedIndividual(payload, id), undefined, trustOptsById.get(id));
|
|
2346
|
+
// After the Sources exist, so a summary can price what it absorbs, and once
|
|
2347
|
+
// per touched GROUP rather than once per prepared row — a batch that
|
|
2348
|
+
// asserts the same triple many times compacts it once.
|
|
2349
|
+
const absorbed = [];
|
|
2350
|
+
for (const groupId of new Set(ids)) absorbed.push(...compactFactGroup(payload, groupId));
|
|
2351
|
+
dropAbsorbedRecords(payload, absorbed);
|
|
1341
2352
|
recountClasses(payload);
|
|
1342
2353
|
});
|
|
1343
2354
|
return { ids, appended: ids.length, skipped };
|
|
@@ -1731,79 +2742,225 @@ export async function resolveRelationChaseReverse(memory, name, objectTerm, help
|
|
|
1731
2742
|
// trust and cites provenance WITHOUT re-walking the graph shape.
|
|
1732
2743
|
|
|
1733
2744
|
/**
|
|
1734
|
-
*
|
|
1735
|
-
*
|
|
1736
|
-
*
|
|
1737
|
-
* for trust-weighted fact ranking.
|
|
2745
|
+
* Fold every reified Fact in a loaded memory payload into one row per TRIPLE —
|
|
2746
|
+
* the group of assertion records sharing a content-addressed group id, one
|
|
2747
|
+
* record per asserting source. Pure. The exported seam the chat/answer layer
|
|
2748
|
+
* consumes for trust-weighted fact ranking.
|
|
2749
|
+
*
|
|
2750
|
+
* The row surface is deliberately the one every reader already parses: `id` is
|
|
2751
|
+
* the bare group id, `provenance` the ' | '-joined union of the members' tags,
|
|
2752
|
+
* `sourceIds`/`sourceTypes` the union as before. `assertions` is the addition —
|
|
2753
|
+
* the per-record hop list, for a reader that wants to see WHICH source said it
|
|
2754
|
+
* and when rather than one blended number.
|
|
2755
|
+
*
|
|
2756
|
+
* The fold reads live HEADS only. A record its own source has since superseded
|
|
2757
|
+
* is that source's PAST belief, not a second vote for the present one; folding
|
|
2758
|
+
* it back in would let one source's edit history inflate its own corroboration.
|
|
2759
|
+
* Demoted records stay walkable through mgx:supersedes/mgx:supersededBy, which
|
|
2760
|
+
* answers "what did this source used to say", never "what do I trust now".
|
|
1738
2761
|
*/
|
|
1739
|
-
export function readFactRows(memory) {
|
|
2762
|
+
export function readFactRows(memory, opts = {}) {
|
|
2763
|
+
const ctx = factFoldContext(memory);
|
|
2764
|
+
// A materialised head, when the backend keeps one, replaces the group's own
|
|
2765
|
+
// fold with the audit trail that fold was last built from — the same records,
|
|
2766
|
+
// read back instead of re-derived. It carries no recency by construction, so
|
|
2767
|
+
// the aggregate below still lands at THIS reading moment either way; a store
|
|
2768
|
+
// that has never materialised a head for this group (a fresh in-memory store,
|
|
2769
|
+
// a hand-built fixture, a backend with no head table) simply folds it.
|
|
2770
|
+
const heads = factHeadsOf(memory);
|
|
2771
|
+
const rows = [];
|
|
2772
|
+
for (const [id, members] of ctx.groups) {
|
|
2773
|
+
const row = foldFactGroup(id, members, ctx);
|
|
2774
|
+
const head = heads?.get(id);
|
|
2775
|
+
// Computed fresh, never stored: recency is a function of the reading
|
|
2776
|
+
// moment, so a stored aggregate is stale by pure passage of time.
|
|
2777
|
+
row.trust = computeAssertionGroupTrust(head ? head.inputs : row.assertions, opts).score;
|
|
2778
|
+
rows.push(row);
|
|
2779
|
+
}
|
|
2780
|
+
return rows;
|
|
2781
|
+
}
|
|
2782
|
+
|
|
2783
|
+
/** Everything a group fold reads out of a payload, gathered in one pass: each
|
|
2784
|
+
* triple group's live head records, the groups each (subject, predicate)
|
|
2785
|
+
* carries, and the two lookups a record's own source resolves through.
|
|
2786
|
+
*
|
|
2787
|
+
* Shared by the read fold and by the head materialisation below, deliberately:
|
|
2788
|
+
* a stored aggregate and a computed one folded from different inputs is the
|
|
2789
|
+
* failure a materialised table invites, and one shared builder is what keeps
|
|
2790
|
+
* the two from ever drifting apart. */
|
|
2791
|
+
function factFoldContext(memory) {
|
|
1740
2792
|
const individuals = memory?.individuals || [];
|
|
1741
2793
|
const sourcesById = new Map(individuals.filter((i) => i?.class === SOURCE_CLASS).map((i) => [i.id, i]));
|
|
1742
2794
|
const statedGroup = (memory?.objectProperties || []).find((g) => g?.prop === STATED_BY_PROP);
|
|
1743
|
-
const
|
|
2795
|
+
const statedByRecord = new Map();
|
|
1744
2796
|
for (const e of statedGroup?.examples || []) {
|
|
1745
|
-
if (!
|
|
1746
|
-
|
|
2797
|
+
if (!statedByRecord.has(e.subject)) statedByRecord.set(e.subject, []);
|
|
2798
|
+
statedByRecord.get(e.subject).push(e.object);
|
|
1747
2799
|
}
|
|
1748
|
-
|
|
2800
|
+
|
|
2801
|
+
const groups = new Map();
|
|
1749
2802
|
for (const ind of individuals) {
|
|
1750
2803
|
if (ind?.class !== FACT_CLASS) continue;
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
const
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
2804
|
+
if ((ind.attributes || []).some((a) => a?.prop === SUPERSEDED_BY_PROP)) continue; // a demoted leaf, not a head
|
|
2805
|
+
if (isChainRollupId(ind.id)) continue; // a summary of one source's demoted history, which was never a vote
|
|
2806
|
+
const groupId = factGroupId(ind.id);
|
|
2807
|
+
const group = groups.get(groupId);
|
|
2808
|
+
if (group) group.push(ind);
|
|
2809
|
+
else groups.set(groupId, [ind]);
|
|
2810
|
+
}
|
|
2811
|
+
|
|
2812
|
+
const groupsByPair = new Map();
|
|
2813
|
+
for (const [groupId, members] of groups) {
|
|
2814
|
+
// Codepoint order on the record id, which sorts by source key — the same
|
|
2815
|
+
// locale-free determinism the P2P layer's own sort insists on, so two peers
|
|
2816
|
+
// holding the same records read the same row.
|
|
2817
|
+
members.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
2818
|
+
const key = subjectPredicateKey(individualKey(members[0], "subject"), individualKey(members[0], "predicate"));
|
|
2819
|
+
const held = groupsByPair.get(key);
|
|
2820
|
+
if (held) held.push(groupId);
|
|
2821
|
+
else groupsByPair.set(key, [groupId]);
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
return {
|
|
2825
|
+
groups,
|
|
2826
|
+
groupsByPair,
|
|
2827
|
+
statedByRecord,
|
|
2828
|
+
sourceTypeOf: (id) => (sourcesById.get(id)?.attributes || []).find((a) => a?.prop === "mgx:sourceType")?.value || "",
|
|
2829
|
+
};
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
/** One triple group folded into its row, minus the aggregate trust — that is
|
|
2833
|
+
* the caller's, because it is the only part that depends on when you ask. */
|
|
2834
|
+
function foldFactGroup(id, heads, ctx) {
|
|
2835
|
+
const { statedByRecord, sourceTypeOf } = ctx;
|
|
2836
|
+
const attrOf = individualAttr;
|
|
2837
|
+
const keyOf = individualKey;
|
|
2838
|
+
|
|
2839
|
+
const assertions = [];
|
|
2840
|
+
const sourceIds = [];
|
|
2841
|
+
const sourceTypes = [];
|
|
2842
|
+
const tags = new Set();
|
|
2843
|
+
const environments = [];
|
|
2844
|
+
const seenEnvironment = new Set();
|
|
2845
|
+
let quantifier = "";
|
|
2846
|
+
for (const head of heads) {
|
|
2847
|
+
// A pool-1 summary joins the fold as ONE pseudo-record standing for every
|
|
2848
|
+
// head it absorbed: its noisy-OR base is their combined contribution, and
|
|
2849
|
+
// its recency comes from the newest assertion time it absorbed, so the
|
|
2850
|
+
// decay still happens at the reading moment rather than being baked in.
|
|
2851
|
+
// The sources it absorbed stay in the union a reader renders — they did
|
|
2852
|
+
// vouch for this triple, and the summary is where that record now lives.
|
|
2853
|
+
if (isHeadRollupId(head.id)) {
|
|
2854
|
+
const rollupType = headRollupTypeOf(head.id);
|
|
2855
|
+
const absorbed = absorbedSourceIds(head);
|
|
2856
|
+
for (const sid of absorbed) {
|
|
2857
|
+
if (sourceIds.includes(sid)) continue;
|
|
2858
|
+
sourceIds.push(sid);
|
|
2859
|
+
if (rollupType) sourceTypes.push(rollupType);
|
|
1764
2860
|
}
|
|
2861
|
+
assertions.push({
|
|
2862
|
+
id: head.id, sourceId: "", sourceType: rollupType,
|
|
2863
|
+
provenance: "",
|
|
2864
|
+
createdAt: attrOf(head, ROLLUP_EARLIEST_PROP),
|
|
2865
|
+
ownTrust: Number(attrOf(head, ROLLUP_PRIOR_PROP)) || 0,
|
|
2866
|
+
assertedAt: attrOf(head, ROLLUP_LATEST_PROP),
|
|
2867
|
+
rollup: {
|
|
2868
|
+
count: Number(attrOf(head, ROLLUP_COUNT_PROP)) || absorbed.length,
|
|
2869
|
+
sourceIds: absorbed,
|
|
2870
|
+
earliest: attrOf(head, ROLLUP_EARLIEST_PROP),
|
|
2871
|
+
latest: attrOf(head, ROLLUP_LATEST_PROP),
|
|
2872
|
+
},
|
|
2873
|
+
});
|
|
2874
|
+
continue;
|
|
1765
2875
|
}
|
|
1766
|
-
const
|
|
1767
|
-
const
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
2876
|
+
const headTags = attrOf(head, "mgx:factProvenance").split(" | ").filter(Boolean);
|
|
2877
|
+
for (const tag of headTags) tags.add(tag);
|
|
2878
|
+
const [statedBy] = statedByRecord.get(head.id) || [];
|
|
2879
|
+
const sourceId = statedBy || attrOf(head, SOURCE_ID_PROP);
|
|
2880
|
+
const sourceType = sourceTypeOf(sourceId);
|
|
2881
|
+
// src:none stands for "no Source at all", so it stays out of the union a
|
|
2882
|
+
// reader renders and out of the corroboration count, exactly as an
|
|
2883
|
+
// unattributable fact has always read.
|
|
2884
|
+
if (statedBy && !sourceIds.includes(statedBy)) {
|
|
2885
|
+
sourceIds.push(statedBy);
|
|
2886
|
+
if (sourceType) sourceTypes.push(sourceType);
|
|
1774
2887
|
}
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
// readers that only need "which premises does this fact cite at all".
|
|
1785
|
-
environments,
|
|
1786
|
-
justification,
|
|
2888
|
+
const createdAt = attrOf(head, CREATED_AT_PROP);
|
|
2889
|
+
const observedAt = attrOf(head, OBSERVED_AT_PROP);
|
|
2890
|
+
assertions.push({
|
|
2891
|
+
id: head.id, sourceId, sourceType,
|
|
2892
|
+
provenance: headTags.join(" | "),
|
|
2893
|
+
createdAt,
|
|
2894
|
+
...(observedAt ? { observedAt } : {}),
|
|
2895
|
+
ownTrust: Number(attrOf(head, TRUST_SCORE_PROP)) || 0,
|
|
2896
|
+
assertedAt: assertionTimestampFor(headTags, createdAt),
|
|
1787
2897
|
});
|
|
2898
|
+
quantifier = quantifier || keyOf(head, "quantifier");
|
|
2899
|
+
// ' | '-separated environments, one premise-id list per independent
|
|
2900
|
+
// derivation; a legacy value with no ' | ' parses as one environment.
|
|
2901
|
+
for (const chunk of keyOf(head, "justification").split(" | ")) {
|
|
2902
|
+
const env = chunk.split(" ").filter(Boolean);
|
|
2903
|
+
if (!env.length) continue;
|
|
2904
|
+
const key = env.join(" ");
|
|
2905
|
+
if (seenEnvironment.has(key)) continue;
|
|
2906
|
+
seenEnvironment.add(key);
|
|
2907
|
+
environments.push(env);
|
|
2908
|
+
}
|
|
1788
2909
|
}
|
|
1789
|
-
|
|
2910
|
+
const justification = [];
|
|
2911
|
+
const seenPremise = new Set();
|
|
2912
|
+
for (const env of environments) {
|
|
2913
|
+
for (const premise of env) {
|
|
2914
|
+
if (seenPremise.has(premise)) continue;
|
|
2915
|
+
seenPremise.add(premise);
|
|
2916
|
+
justification.push(premise);
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
return {
|
|
2920
|
+
id,
|
|
2921
|
+
subject: keyOf(heads[0], "subject"), predicate: keyOf(heads[0], "predicate"), object: keyOf(heads[0], "object"),
|
|
2922
|
+
// The compat union string readers already parse, codepoint-sorted so it
|
|
2923
|
+
// does not depend on which order the records happened to arrive in.
|
|
2924
|
+
provenance: [...tags].sort().join(" | "),
|
|
2925
|
+
quantifier, // "" unless a plural class-membership teach set one
|
|
2926
|
+
sourceIds, sourceTypes,
|
|
2927
|
+
// `environments`: every persisted premise set (empty unless entailed);
|
|
2928
|
+
// `justification`: their deduped union in first-occurrence order, for
|
|
2929
|
+
// readers that only need "which premises does this fact cite at all".
|
|
2930
|
+
environments,
|
|
2931
|
+
justification,
|
|
2932
|
+
// The hop list the blended number is folded from — one entry per source
|
|
2933
|
+
// that asserted this triple, each with its own time and its own weight.
|
|
2934
|
+
assertions,
|
|
2935
|
+
};
|
|
1790
2936
|
}
|
|
1791
2937
|
|
|
1792
|
-
/** Retract
|
|
1793
|
-
*
|
|
1794
|
-
*
|
|
1795
|
-
*
|
|
2938
|
+
/** Retract facts by id — a real DELETE (syllogise.mjs's retractability
|
|
2939
|
+
* mechanism). A GROUP id retracts the triple: every source's record for it,
|
|
2940
|
+
* demoted leaves included, since retracting "dogs bark" cannot leave half its
|
|
2941
|
+
* assertions standing. A single record id retracts just that record. Scrubs
|
|
2942
|
+
* any edge referencing a removed id as subject or object; an orphaned Source
|
|
2943
|
+
* is left in place (not a GC pass). Unknown ids are silently skipped. Returns
|
|
2944
|
+
* { removed } — the ids asked for that matched, so it may be smaller than the
|
|
2945
|
+
* input and is never longer than it. */
|
|
1796
2946
|
export async function removeFacts(dir, ids) {
|
|
1797
2947
|
const idSet = new Set((ids || []).filter(Boolean));
|
|
1798
2948
|
const removed = [];
|
|
1799
2949
|
if (!idSet.size) return { removed };
|
|
1800
2950
|
await mutateMemory(dir, (payload) => {
|
|
2951
|
+
const removedSet = new Set();
|
|
2952
|
+
const matched = new Set();
|
|
1801
2953
|
payload.individuals = (payload.individuals || []).filter((ind) => {
|
|
1802
|
-
if (ind?.class
|
|
1803
|
-
|
|
2954
|
+
if (ind?.class !== FACT_CLASS) return true;
|
|
2955
|
+
const groupId = factGroupId(ind.id);
|
|
2956
|
+
const asked = idSet.has(ind.id) ? ind.id : (idSet.has(groupId) ? groupId : "");
|
|
2957
|
+
if (!asked) return true;
|
|
2958
|
+
matched.add(asked);
|
|
2959
|
+
removedSet.add(ind.id);
|
|
2960
|
+
return false;
|
|
1804
2961
|
});
|
|
2962
|
+
for (const id of matched) removed.push(id);
|
|
1805
2963
|
if (!removed.length) return; // honest no-op — nothing matched, no write needed beyond this
|
|
1806
|
-
const removedSet = new Set(removed);
|
|
1807
2964
|
for (const group of payload.objectProperties || []) {
|
|
1808
2965
|
const before = group.examples || [];
|
|
1809
2966
|
group.examples = before.filter((e) => !removedSet.has(e?.subject) && !removedSet.has(e?.object));
|
|
@@ -1823,32 +2980,37 @@ export const CAPABLE_OF_PREDICATE = "mgx:capableOf";
|
|
|
1823
2980
|
|
|
1824
2981
|
/** Predicates whose real-world semantics allow many objects at once ("a dog
|
|
1825
2982
|
* has legs" AND "a dog has a tail"; "a bird can fly" AND "a bird can sing"),
|
|
1826
|
-
* so a second object is a second fact, never a disagreement.
|
|
1827
|
-
*
|
|
1828
|
-
|
|
1829
|
-
* cannot sing" are two claims, not a self-contradiction. */
|
|
1830
|
-
export const MULTI_VALUED_PREDICATES = new Set(
|
|
1831
|
-
[HAS_A_PREDICATE, CAPABLE_OF_PREDICATE].flatMap((p) => [p, negatedPredicate(p)]),
|
|
1832
|
-
);
|
|
2983
|
+
* so a second object is a second fact, never a disagreement. Derived from the
|
|
2984
|
+
* resolver table's `merge` row, so the two can never say different things. */
|
|
2985
|
+
export const MULTI_VALUED_PREDICATES = MERGE_PREDICATES;
|
|
1833
2986
|
|
|
1834
2987
|
/** Facts that CONTRADICT: same (subject, predicate), different object, each
|
|
1835
2988
|
* above the trust floor. Returns groups (trust-desc) so callers surface both,
|
|
1836
|
-
* never silently pick one.
|
|
1837
|
-
*
|
|
2989
|
+
* never silently pick one.
|
|
2990
|
+
*
|
|
2991
|
+
* Two stages, and only the second one is here. Stage 1 — the records inside
|
|
2992
|
+
* one triple group — is readFactRows' own fold: same (s,p,o) is corroboration,
|
|
2993
|
+
* and a group is internally agreeing by construction. Stage 2 is this: across
|
|
2994
|
+
* the OBJECTS one (subject, predicate) carries, under the resolver table.
|
|
2995
|
+
* A merge predicate never reports; a state or registration predicate reports
|
|
2996
|
+
* only what its own clock could not order (the resolver's trust and codepoint
|
|
2997
|
+
* tie-breaks — see resolveSiblingGroups), so ordinary succession stops reading
|
|
2998
|
+
* as disagreement; every other predicate keeps the full keep-both contract. */
|
|
1838
2999
|
export function findContradictions(memory, { floor = CONTRADICTION_TRUST_FLOOR } = {}) {
|
|
1839
3000
|
const rows = readFactRows(memory).filter((r) => r.trust >= floor);
|
|
1840
3001
|
const byKey = new Map();
|
|
1841
3002
|
for (const r of rows) {
|
|
1842
|
-
if (
|
|
3003
|
+
if (resolutionStrategyFor(r.predicate) === RESOLUTION_MERGE) continue;
|
|
1843
3004
|
const key = `${r.subject} ${r.predicate}`;
|
|
1844
3005
|
if (!byKey.has(key)) byKey.set(key, []);
|
|
1845
3006
|
byKey.get(key).push(r);
|
|
1846
3007
|
}
|
|
1847
3008
|
const out = [];
|
|
1848
3009
|
for (const group of byKey.values()) {
|
|
1849
|
-
if (new Set(group.map((r) => r.object)).size
|
|
1850
|
-
|
|
1851
|
-
|
|
3010
|
+
if (new Set(group.map((r) => r.object)).size < 2) continue;
|
|
3011
|
+
const strategy = resolutionStrategyFor(group[0].predicate);
|
|
3012
|
+
if (strategy !== RESOLUTION_CONTRADICTION && !resolveSiblingGroups(group, strategy).contested) continue;
|
|
3013
|
+
out.push(group.slice().sort((a, b) => b.trust - a.trust || a.object.localeCompare(b.object)));
|
|
1852
3014
|
}
|
|
1853
3015
|
return out.sort((a, b) => `${a[0].subject} ${a[0].predicate}`.localeCompare(`${b[0].subject} ${b[0].predicate}`));
|
|
1854
3016
|
}
|