@oortmem/core 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents.d.ts +38 -0
- package/dist/agents.js +67 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.js +7 -0
- package/dist/consolidate.d.ts +36 -7
- package/dist/consolidate.js +114 -39
- package/dist/grant.d.ts +58 -11
- package/dist/grant.js +121 -23
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/isolation.d.ts +26 -3
- package/dist/isolation.js +222 -61
- package/dist/levels.d.ts +79 -0
- package/dist/levels.js +140 -0
- package/dist/observe.d.ts +9 -0
- package/dist/observe.js +27 -3
- package/dist/recall.d.ts +27 -1
- package/dist/recall.js +163 -17
- package/dist/store-conformance.js +69 -16
- package/package.json +1 -1
package/dist/levels.d.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { type Level, type NodeRef } from './address.js';
|
|
2
|
+
import { type Grant } from './grant.js';
|
|
3
|
+
import type { Memory } from './model.js';
|
|
4
|
+
/**
|
|
5
|
+
* The levels a recall can name in `include` (spec 7.1): `self` is the
|
|
6
|
+
* caller's home node, `planet` the person's Planet, `star` the calling agent's
|
|
7
|
+
* own Star, `constellation` and `galaxy` the hive nodes. Always intersected
|
|
8
|
+
* with the grant.
|
|
9
|
+
*/
|
|
10
|
+
export type IncludeLevel = 'self' | 'planet' | 'star' | 'constellation' | 'galaxy';
|
|
11
|
+
export declare const INCLUDE_LEVELS: readonly IncludeLevel[];
|
|
12
|
+
export declare const isIncludeLevel: (s: string) => s is IncludeLevel;
|
|
13
|
+
/** At most this many hive nodes are read by default (spec 7.1: one recall, a bounded fan-out). */
|
|
14
|
+
export declare const DEFAULT_HIVE_NODES = 4;
|
|
15
|
+
/**
|
|
16
|
+
* A recall asked for a level its grant does not cover. Typed, so callers tell
|
|
17
|
+
* "you may not read that" apart from "nothing matched" (spec 7.1).
|
|
18
|
+
*/
|
|
19
|
+
export declare class ScopeDeniedError extends Error {
|
|
20
|
+
readonly code: 'scope_denied';
|
|
21
|
+
readonly levels: IncludeLevel[];
|
|
22
|
+
constructor(levels: IncludeLevel[]);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The nodes one recall reads: with `include`, every readable node a named
|
|
26
|
+
* level covers (a level that covers none rejects with `ScopeDeniedError`);
|
|
27
|
+
* without it, every readable node, hive nodes capped at `DEFAULT_HIVE_NODES`
|
|
28
|
+
* in grant order. Never a node outside `g`.
|
|
29
|
+
*/
|
|
30
|
+
export declare function nodesToRead(g: Grant, include?: readonly IncludeLevel[]): NodeRef[];
|
|
31
|
+
/**
|
|
32
|
+
* How specific a node is, for the specificity prior and the conflict rule:
|
|
33
|
+
* an agent's own memory (Star, Orbit, Flyby) 3, the person's Planet 2, a
|
|
34
|
+
* Constellation 1, a Galaxy 0. Narrower is higher.
|
|
35
|
+
*/
|
|
36
|
+
export declare function specificity(node: Pick<NodeRef, 'level'> & {
|
|
37
|
+
hiveId?: string;
|
|
38
|
+
}): number;
|
|
39
|
+
/**
|
|
40
|
+
* The specificity prior's factor for a node: `1 + ε × specificity / 3`, so
|
|
41
|
+
* with `ε` the narrowest level gains `ε` and a Galaxy nothing. `ε = 0` (the
|
|
42
|
+
* default) leaves every score as it was.
|
|
43
|
+
*/
|
|
44
|
+
export declare function specificityFactor(node: NodeRef, epsilon: number): number;
|
|
45
|
+
/** Brackets in memory text, escaped so the text cannot pass for a code-built label. */
|
|
46
|
+
export declare function escapeLabelText(text: string): string;
|
|
47
|
+
/** Where a recalled memory came from, built by code from its address and author. */
|
|
48
|
+
export interface MemoryOrigin {
|
|
49
|
+
level: Level;
|
|
50
|
+
/** The space name: Planet, Star, Orbit, Flyby, Constellation or Galaxy. */
|
|
51
|
+
space: string;
|
|
52
|
+
/** The node, as a label shows it: `your Planet`, `Star cursor`, `Constellation checkout`. */
|
|
53
|
+
node: string;
|
|
54
|
+
/** For people: `from your Planet`, `from this agent's Star`, `from Star cursor`. */
|
|
55
|
+
from: string;
|
|
56
|
+
/** The memory lives in the caller's home node. */
|
|
57
|
+
home: boolean;
|
|
58
|
+
/** The agent that wrote it. */
|
|
59
|
+
author: string;
|
|
60
|
+
/**
|
|
61
|
+
* A narrower memory in this result contradicts this one (spec 7.2): it ranks
|
|
62
|
+
* first, and this one stays, labelled.
|
|
63
|
+
*/
|
|
64
|
+
overriddenBy?: {
|
|
65
|
+
id: string;
|
|
66
|
+
node: string;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** The node part of a label for `node`, as the caller of grant `g` sees it. */
|
|
70
|
+
export declare function nodeLabel(node: NodeRef, g: Grant): string;
|
|
71
|
+
/** The origin of `m` for the caller of grant `g`. */
|
|
72
|
+
export declare function originOf(m: Pick<Memory, 'scope' | 'address' | 'author'>, g: Grant): MemoryOrigin;
|
|
73
|
+
/** The decision `observe` records on a memory written instead of editing one in another node. */
|
|
74
|
+
export declare const CONTRADICTS: {
|
|
75
|
+
readonly stage: 'level';
|
|
76
|
+
readonly question: 'contradicts';
|
|
77
|
+
};
|
|
78
|
+
/** The ids of memories in other nodes that `m` contradicts (recorded by `observe`). */
|
|
79
|
+
export declare function contradicted(m: Pick<Memory, 'source'>): string[];
|
package/dist/levels.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { addressOf, authorOf, GALAXY_ROOT, nodeKey, nodeOf, spaceName, } from './address.js';
|
|
2
|
+
import { readableNodes } from './grant.js';
|
|
3
|
+
export const INCLUDE_LEVELS = Object.freeze([
|
|
4
|
+
'self',
|
|
5
|
+
'planet',
|
|
6
|
+
'star',
|
|
7
|
+
'constellation',
|
|
8
|
+
'galaxy',
|
|
9
|
+
]);
|
|
10
|
+
export const isIncludeLevel = (s) => INCLUDE_LEVELS.includes(s);
|
|
11
|
+
/** At most this many hive nodes are read by default (spec 7.1: one recall, a bounded fan-out). */
|
|
12
|
+
export const DEFAULT_HIVE_NODES = 4;
|
|
13
|
+
/**
|
|
14
|
+
* A recall asked for a level its grant does not cover. Typed, so callers tell
|
|
15
|
+
* "you may not read that" apart from "nothing matched" (spec 7.1).
|
|
16
|
+
*/
|
|
17
|
+
export class ScopeDeniedError extends Error {
|
|
18
|
+
code = 'scope_denied';
|
|
19
|
+
levels;
|
|
20
|
+
constructor(levels) {
|
|
21
|
+
super(`scope_denied: this grant cannot read ${levels.join(', ')}`);
|
|
22
|
+
this.name = 'ScopeDeniedError';
|
|
23
|
+
this.levels = levels;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
/** True when `node` is what `level` names for a caller whose grant is `g`. */
|
|
27
|
+
function names(level, node, g) {
|
|
28
|
+
switch (level) {
|
|
29
|
+
case 'self':
|
|
30
|
+
return nodeKey(node) === nodeKey(g.home);
|
|
31
|
+
case 'planet':
|
|
32
|
+
return node.level === 'user';
|
|
33
|
+
case 'star':
|
|
34
|
+
return (node.level === 'agent' &&
|
|
35
|
+
node.orgId === g.principal.orgId &&
|
|
36
|
+
node.agentId === g.principal.agentId);
|
|
37
|
+
case 'constellation':
|
|
38
|
+
return node.level === 'hive' && node.hiveId !== GALAXY_ROOT;
|
|
39
|
+
case 'galaxy':
|
|
40
|
+
return node.level === 'hive' && node.hiveId === GALAXY_ROOT;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* The nodes one recall reads: with `include`, every readable node a named
|
|
45
|
+
* level covers (a level that covers none rejects with `ScopeDeniedError`);
|
|
46
|
+
* without it, every readable node, hive nodes capped at `DEFAULT_HIVE_NODES`
|
|
47
|
+
* in grant order. Never a node outside `g`.
|
|
48
|
+
*/
|
|
49
|
+
export function nodesToRead(g, include) {
|
|
50
|
+
const readable = readableNodes(g);
|
|
51
|
+
if (include === undefined || include.length === 0) {
|
|
52
|
+
let hives = 0;
|
|
53
|
+
return readable.filter((n) => n.level !== 'hive' || ++hives <= DEFAULT_HIVE_NODES);
|
|
54
|
+
}
|
|
55
|
+
for (const l of include) {
|
|
56
|
+
if (!isIncludeLevel(l))
|
|
57
|
+
throw new RangeError(`unknown level ${JSON.stringify(l)}`);
|
|
58
|
+
}
|
|
59
|
+
const denied = [...new Set(include)].filter((l) => !readable.some((n) => names(l, n, g)));
|
|
60
|
+
if (denied.length > 0)
|
|
61
|
+
throw new ScopeDeniedError(denied);
|
|
62
|
+
return readable.filter((n) => include.some((l) => names(l, n, g)));
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* How specific a node is, for the specificity prior and the conflict rule:
|
|
66
|
+
* an agent's own memory (Star, Orbit, Flyby) 3, the person's Planet 2, a
|
|
67
|
+
* Constellation 1, a Galaxy 0. Narrower is higher.
|
|
68
|
+
*/
|
|
69
|
+
export function specificity(node) {
|
|
70
|
+
switch (node.level) {
|
|
71
|
+
case 'agent':
|
|
72
|
+
case 'agent-user':
|
|
73
|
+
case 'run':
|
|
74
|
+
return 3;
|
|
75
|
+
case 'user':
|
|
76
|
+
return 2;
|
|
77
|
+
case 'hive':
|
|
78
|
+
return node.hiveId === GALAXY_ROOT ? 0 : 1;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The specificity prior's factor for a node: `1 + ε × specificity / 3`, so
|
|
83
|
+
* with `ε` the narrowest level gains `ε` and a Galaxy nothing. `ε = 0` (the
|
|
84
|
+
* default) leaves every score as it was.
|
|
85
|
+
*/
|
|
86
|
+
export function specificityFactor(node, epsilon) {
|
|
87
|
+
return 1 + (epsilon * specificity(node)) / 3;
|
|
88
|
+
}
|
|
89
|
+
/** Brackets in memory text, escaped so the text cannot pass for a code-built label. */
|
|
90
|
+
export function escapeLabelText(text) {
|
|
91
|
+
return text.replace(/[[\]]/g, (c) => `\\${c}`);
|
|
92
|
+
}
|
|
93
|
+
/** An id shown inside a label: control characters dropped, brackets escaped, clipped. */
|
|
94
|
+
function labelId(id) {
|
|
95
|
+
// oxlint-disable-next-line no-control-regex
|
|
96
|
+
const clean = escapeLabelText(id.replace(/[\u0000-\u001f\u007f]/g, ''));
|
|
97
|
+
return clean.length > 40 ? `${clean.slice(0, 39)}…` : clean;
|
|
98
|
+
}
|
|
99
|
+
/** The node part of a label for `node`, as the caller of grant `g` sees it. */
|
|
100
|
+
export function nodeLabel(node, g) {
|
|
101
|
+
switch (node.level) {
|
|
102
|
+
case 'user':
|
|
103
|
+
return node.orgId === g.principal.orgId && node.customerId === g.principal.customerId
|
|
104
|
+
? 'your Planet'
|
|
105
|
+
: 'a Planet';
|
|
106
|
+
case 'agent':
|
|
107
|
+
return `Star ${labelId(node.agentId)}`;
|
|
108
|
+
case 'agent-user':
|
|
109
|
+
return `Orbit ${labelId(node.agentId)}`;
|
|
110
|
+
case 'run':
|
|
111
|
+
return `Flyby ${labelId(node.runId)}`;
|
|
112
|
+
case 'hive':
|
|
113
|
+
return node.hiveId === GALAXY_ROOT ? 'Galaxy' : `Constellation ${labelId(node.hiveId)}`;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
/** The origin of `m` for the caller of grant `g`. */
|
|
117
|
+
export function originOf(m, g) {
|
|
118
|
+
const node = nodeOf(addressOf(m));
|
|
119
|
+
const label = nodeLabel(node, g);
|
|
120
|
+
const ownStar = node.level === 'agent' &&
|
|
121
|
+
node.orgId === g.principal.orgId &&
|
|
122
|
+
node.agentId === g.principal.agentId;
|
|
123
|
+
return {
|
|
124
|
+
level: node.level,
|
|
125
|
+
space: spaceName(node),
|
|
126
|
+
node: label,
|
|
127
|
+
from: ownStar ? "from this agent's Star" : `from ${label}`,
|
|
128
|
+
home: nodeKey(node) === nodeKey(g.home),
|
|
129
|
+
author: authorOf(m).agentId,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
/** The decision `observe` records on a memory written instead of editing one in another node. */
|
|
133
|
+
export const CONTRADICTS = { stage: 'level', question: 'contradicts' };
|
|
134
|
+
/** The ids of memories in other nodes that `m` contradicts (recorded by `observe`). */
|
|
135
|
+
export function contradicted(m) {
|
|
136
|
+
return m.source.decisions
|
|
137
|
+
.filter((d) => d.stage === CONTRADICTS.stage && d.question === CONTRADICTS.question)
|
|
138
|
+
.map((d) => d.answer);
|
|
139
|
+
}
|
|
140
|
+
//# sourceMappingURL=levels.js.map
|
package/dist/observe.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { DecisionController, Embedder, MemoryStore, MemoryWriter, Observation } from './adapters.js';
|
|
2
2
|
import type { ResolvedConfig } from './config.js';
|
|
3
|
+
import { type Home } from './grant.js';
|
|
3
4
|
import type { Decision, Scope } from './model.js';
|
|
4
5
|
export type MemoryOutcomeKind = 'skipped' | 'reinforced' | 'added' | 'updated' | 'superseded' | 'corrected';
|
|
5
6
|
/** What happened to one memory the turn produced. */
|
|
@@ -29,6 +30,14 @@ export interface ObserveDeps {
|
|
|
29
30
|
now?: Date;
|
|
30
31
|
/** Recorded on `source.harness`; defaults to "unknown". */
|
|
31
32
|
harness?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Where new memories land (spec 5.1): the person's Planet (default) or, for an agent configured
|
|
35
|
+
* as a Star, `scope.agentId`'s own Star. Candidates come from both. A memory is only ever
|
|
36
|
+
* superseded, corrected or refined in the home node, and an agent whose home is its Star never
|
|
37
|
+
* writes the Planet at all: a Planet memory it re-observes gets an access record (activation),
|
|
38
|
+
* never a confirmation or an edit.
|
|
39
|
+
*/
|
|
40
|
+
home?: Home;
|
|
32
41
|
}
|
|
33
42
|
/**
|
|
34
43
|
* Gate-stage reinforcement match floor. When the gate says "do not store", a
|
package/dist/observe.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import {
|
|
2
|
+
import { addressed, addressOf, nodeKey, nodeOf } from './address.js';
|
|
3
|
+
import { grantWrites, homeAddress, localGrant } from './grant.js';
|
|
4
|
+
import { CONTRADICTS } from './levels.js';
|
|
3
5
|
import { capObservation } from './cap.js';
|
|
4
6
|
import { scrubObservation } from './scrub.js';
|
|
5
7
|
import { checkWriterDate, factInterval, utcOffset } from './temporal.js';
|
|
@@ -125,11 +127,30 @@ async function apply(a) {
|
|
|
125
127
|
if (target === undefined)
|
|
126
128
|
return { outcome: 'skipped' };
|
|
127
129
|
await store.recordAccess(grant, [target.id], now);
|
|
128
|
-
|
|
130
|
+
// Confirming writes the memory: only where this agent may write (a Star agent reinforces a
|
|
131
|
+
// Planet memory's activation but never edits it).
|
|
132
|
+
if (grantWrites(grant, target))
|
|
133
|
+
await confirm(store, grant, target.id, now.toISOString());
|
|
129
134
|
return { outcome: 'reinforced', memoryId: target.id };
|
|
130
135
|
}
|
|
131
136
|
let action = best.action === 'UPDATE' ? 'REFINE' : best.action;
|
|
132
137
|
let prior = action === 'ADD' ? undefined : candidates.find((c) => c.id === best.candidateId);
|
|
138
|
+
if (prior !== undefined && nodeKey(nodeOf(addressOf(prior))) !== nodeKey(grant.home)) {
|
|
139
|
+
// The prior lives in another node (the Planet, for an agent writing to its Star, or the
|
|
140
|
+
// reverse). A write never edits a memory outside its home node: the observation becomes a
|
|
141
|
+
// memory of its own there. A contradiction is recorded, so recall can rank the narrower of the
|
|
142
|
+
// two first and label the other (spec 7.2). Reinforcing it (above) stays allowed.
|
|
143
|
+
decisions.push({
|
|
144
|
+
stage: 'level',
|
|
145
|
+
question: 'other-node',
|
|
146
|
+
answer: 'add-instead',
|
|
147
|
+
probability: 1,
|
|
148
|
+
});
|
|
149
|
+
if (action === 'SUPERSEDE' || action === 'CORRECT') {
|
|
150
|
+
decisions.push({ ...CONTRADICTS, answer: prior.id, probability: best.probability });
|
|
151
|
+
}
|
|
152
|
+
prior = undefined;
|
|
153
|
+
}
|
|
133
154
|
if (prior !== undefined && (prior.scope.projectId ?? null) !== (scope.projectId ?? null)) {
|
|
134
155
|
// Candidates include global memories. A project's observation must not end or
|
|
135
156
|
// rewrite a fact another scope (every project, for a global one) relies on, so it
|
|
@@ -235,6 +256,9 @@ async function apply(a) {
|
|
|
235
256
|
const base = {
|
|
236
257
|
id,
|
|
237
258
|
scope,
|
|
259
|
+
// A memory at another level than the Planet carries its address and author.
|
|
260
|
+
...(grant.home.level !== 'user' &&
|
|
261
|
+
addressed(homeAddress(grant, scope.projectId), { agentId: scope.agentId })),
|
|
238
262
|
type: w.type,
|
|
239
263
|
text: w.text,
|
|
240
264
|
stub: w.stub,
|
|
@@ -418,7 +442,7 @@ export async function observe(raw, scope, deps) {
|
|
|
418
442
|
const obs = capObservation(scrubbed.obs);
|
|
419
443
|
const { store, embed, decide, write } = deps;
|
|
420
444
|
const now = deps.now ?? new Date();
|
|
421
|
-
const grant = localGrant(filterFor(scope));
|
|
445
|
+
const grant = localGrant(filterFor(scope), deps.home === undefined ? {} : { home: deps.home });
|
|
422
446
|
const g = await decide.gate(obs);
|
|
423
447
|
const text = `${obs.userText}\n${obs.assistantText}`;
|
|
424
448
|
if (g.shouldStore < GATE_THRESHOLD) {
|
package/dist/recall.d.ts
CHANGED
|
@@ -1,10 +1,25 @@
|
|
|
1
1
|
import type { Embedder, MemoryStore, Reranker, ScopeFilter } from './adapters.js';
|
|
2
2
|
import type { ResolvedConfig } from './config.js';
|
|
3
|
+
import { type Grant, type Home } from './grant.js';
|
|
4
|
+
import { type IncludeLevel, type MemoryOrigin } from './levels.js';
|
|
3
5
|
import { type Memory } from './model.js';
|
|
4
6
|
import { type RecallMode } from './temporal.js';
|
|
5
7
|
export interface RecallQuery {
|
|
6
8
|
text: string;
|
|
7
|
-
|
|
9
|
+
/**
|
|
10
|
+
* Who recalls, as a pre-level filter: read with `localGrant(scope, { home })`, the person's
|
|
11
|
+
* Planet plus this agent's own Star in these projects. Required unless `grant` is given.
|
|
12
|
+
*/
|
|
13
|
+
scope?: ScopeFilter;
|
|
14
|
+
/** The grant to read with, instead of one built from `scope` (multi-node grants, tests). */
|
|
15
|
+
grant?: Grant;
|
|
16
|
+
/** With `scope`: where this agent's memories land (`star` for an agent configured as a Star). */
|
|
17
|
+
home?: Home;
|
|
18
|
+
/**
|
|
19
|
+
* Which levels to read (spec 7.1), intersected with the grant. A level the grant does not cover
|
|
20
|
+
* rejects with `ScopeDeniedError`. Default: every node the grant reads.
|
|
21
|
+
*/
|
|
22
|
+
include?: IncludeLevel[];
|
|
8
23
|
budgetChars?: number;
|
|
9
24
|
/** ISO timestamp: recall what was true then (valid time). Implies `mode: 'as-of'`. */
|
|
10
25
|
asOf?: string;
|
|
@@ -44,6 +59,8 @@ export interface RecalledMemory extends Memory {
|
|
|
44
59
|
/** A volatile fact nobody confirmed within `recall.staleAfterDays`. */
|
|
45
60
|
stale?: true;
|
|
46
61
|
};
|
|
62
|
+
/** Where it came from: level, node, author, and whether a narrower memory overrides it. */
|
|
63
|
+
origin?: MemoryOrigin;
|
|
47
64
|
}
|
|
48
65
|
export type Degradation = 'no-embedder' | 'no-reranker' | 'embed-failed' | 'rerank-failed'
|
|
49
66
|
/** The reranker returned no ids: the fused order stands. */
|
|
@@ -75,6 +92,8 @@ export interface RecallResult {
|
|
|
75
92
|
embedSkipped?: 'lexical-confident';
|
|
76
93
|
/** The temporal view this recall read. Always set by `recall`. */
|
|
77
94
|
temporal?: RecallTemporal;
|
|
95
|
+
/** The nodes this recall read, as labels show them (`your Planet`, `Star cursor`). */
|
|
96
|
+
read?: string[];
|
|
78
97
|
timings: Record<string, number>;
|
|
79
98
|
}
|
|
80
99
|
export interface RecallDeps {
|
|
@@ -107,3 +126,10 @@ export declare function fitForInjection<M extends RecalledMemory>(memories: M[],
|
|
|
107
126
|
kept: M[];
|
|
108
127
|
lines: string[];
|
|
109
128
|
};
|
|
129
|
+
/**
|
|
130
|
+
* Memory text with any `<oort-…` or `</oort-…` tag opening written as `<`, so a memory cannot
|
|
131
|
+
* close the `<oort-memory>` block (or open another) and have what follows read as the prompt.
|
|
132
|
+
*/
|
|
133
|
+
export declare function escapeBlockTags(text: string): string;
|
|
134
|
+
/** The share of an injection budget hive memories may take (spec 7.2). */
|
|
135
|
+
export declare const HIVE_QUOTA = 0.4;
|
package/dist/recall.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { activation } from './activation.js';
|
|
2
2
|
import { fuse, isVisible } from './fusion.js';
|
|
3
|
-
import {
|
|
3
|
+
import { addressOf, nodeKey, nodeOf } from './address.js';
|
|
4
|
+
import { assertGrant, grantWrites, localGrant, narrowGrant, } from './grant.js';
|
|
5
|
+
import { contradicted, escapeLabelText, nodeLabel, nodesToRead, originOf, specificity, specificityFactor, } from './levels.js';
|
|
4
6
|
import { scrub } from './scrub.js';
|
|
5
7
|
import { isCurrent } from './model.js';
|
|
6
8
|
import { temporalIntent } from './temporal.js';
|
|
@@ -17,8 +19,14 @@ const clock = () => performance.now();
|
|
|
17
19
|
*/
|
|
18
20
|
export async function recall(q, deps) {
|
|
19
21
|
const { store, config } = deps;
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
const full = grantOf(q);
|
|
23
|
+
// The nodes to read: the grant intersected with `include` (a level it lacks rejects here,
|
|
24
|
+
// before anything is read). Recall reads and hydrates only inside them; the store enforces it.
|
|
25
|
+
const nodes = nodesToRead(full, q.include);
|
|
26
|
+
const grant = narrowGrant(full, nodes);
|
|
27
|
+
// First stage per node (spec 7.2), then nodes merged by fused score. One node: today's recall.
|
|
28
|
+
const legs = nodes.map((n) => ({ node: n, grant: narrowGrant(full, [n]) }));
|
|
29
|
+
const epsilon = config.recall.specificityPrior;
|
|
22
30
|
const now = deps.now ?? new Date();
|
|
23
31
|
// Scrub once: the query text reaches embedders and rerankers, which may be remote. Temporal
|
|
24
32
|
// words ("on 2 April", "changed over time") pick the view and are not searched for.
|
|
@@ -54,14 +62,14 @@ export async function recall(q, deps) {
|
|
|
54
62
|
prefetched.set(id, m);
|
|
55
63
|
return m;
|
|
56
64
|
};
|
|
57
|
-
const lexicalP = timed('lexicalMs', () => store.lexical(grant, text, config.recall.lexicalK));
|
|
65
|
+
const lexicalP = timed('lexicalMs', () => Promise.all(legs.map((l) => store.lexical(l.grant, text, config.recall.lexicalK))));
|
|
58
66
|
// A failed embed must not leave the lexical promise unobserved.
|
|
59
67
|
lexicalP.catch(() => { });
|
|
60
68
|
// Lexical first (CON-57): when the lexical answer is already clear, skip the embedding round
|
|
61
69
|
// trip. Otherwise the lexical search has run while nothing else waited on it.
|
|
62
70
|
let embedSkipped;
|
|
63
71
|
if (deps.embed && config.recall.lexicalFirst) {
|
|
64
|
-
const first = await lexicalP;
|
|
72
|
+
const first = byRank(await lexicalP);
|
|
65
73
|
if (await lexicalIsConfident(first, queryTokens, read, now, view)) {
|
|
66
74
|
embedSkipped = 'lexical-confident';
|
|
67
75
|
}
|
|
@@ -83,15 +91,24 @@ export async function recall(q, deps) {
|
|
|
83
91
|
timings['embedMs'] = clock() - t;
|
|
84
92
|
}
|
|
85
93
|
const v = vector;
|
|
86
|
-
|
|
94
|
+
let denseFailed = false;
|
|
95
|
+
const [lexicalLegs, denseLegs] = await Promise.all([
|
|
87
96
|
lexicalP,
|
|
88
97
|
v === undefined
|
|
89
|
-
? Promise.resolve([])
|
|
90
|
-
: timed('denseMs', () => store.dense(grant, v, config.recall.denseK).catch(() => {
|
|
91
|
-
|
|
98
|
+
? Promise.resolve(legs.map(() => []))
|
|
99
|
+
: timed('denseMs', () => Promise.all(legs.map((l) => store.dense(l.grant, v, config.recall.denseK).catch(() => {
|
|
100
|
+
denseFailed = true;
|
|
92
101
|
return [];
|
|
93
|
-
})),
|
|
102
|
+
})))),
|
|
94
103
|
]);
|
|
104
|
+
if (denseFailed)
|
|
105
|
+
degraded.push('dense-failed');
|
|
106
|
+
// Across nodes, by rank (one node: its own list). Used where one list is needed.
|
|
107
|
+
const lexical = byRank(lexicalLegs);
|
|
108
|
+
const dense = byRank(denseLegs);
|
|
109
|
+
// Ranks are per node: each node's first stage ranks its own memories.
|
|
110
|
+
const lexicalRank = rankWithin(lexicalLegs);
|
|
111
|
+
const denseRank = rankWithin(denseLegs);
|
|
95
112
|
// Live rows only: a cold row keeps its text, stub and entities, which is all fusion, the gate
|
|
96
113
|
// and the reranker read. Hydration waits until we know what is returned (CON-47).
|
|
97
114
|
const ids = [...new Set([...lexical, ...dense].map((h) => h.id))];
|
|
@@ -101,7 +118,14 @@ export async function recall(q, deps) {
|
|
|
101
118
|
if (m !== null)
|
|
102
119
|
memories.set(m.id, m);
|
|
103
120
|
const tf = clock();
|
|
104
|
-
|
|
121
|
+
// Reciprocal rank fusion within each node as before, then the nodes merged by fused score,
|
|
122
|
+
// each scaled by the specificity prior (off at ε = 0). A stable sort: one node keeps its order.
|
|
123
|
+
const fused = legs
|
|
124
|
+
.flatMap((l, i) => fuse([lexicalLegs[i], denseLegs[i]], memories, now, {
|
|
125
|
+
activation: config.activation,
|
|
126
|
+
view,
|
|
127
|
+
}).map((f) => epsilon === 0 ? f : { ...f, score: f.score * specificityFactor(l.node, epsilon) }))
|
|
128
|
+
.sort((x, y) => y.score - x.score);
|
|
105
129
|
timings['fuseMs'] = clock() - tf;
|
|
106
130
|
const coverage = new Map();
|
|
107
131
|
for (const f of fused) {
|
|
@@ -168,13 +192,13 @@ export async function recall(q, deps) {
|
|
|
168
192
|
};
|
|
169
193
|
const kept = candidates.filter((f) => relevant(f.id));
|
|
170
194
|
const dropped = candidates.length - kept.length;
|
|
171
|
-
|
|
195
|
+
let out = [];
|
|
172
196
|
for (const f of kept) {
|
|
173
197
|
const m = memories.get(f.id);
|
|
174
198
|
if (m === undefined)
|
|
175
199
|
continue;
|
|
176
|
-
const lr =
|
|
177
|
-
const dr =
|
|
200
|
+
const lr = lexicalRank.get(f.id) ?? -1;
|
|
201
|
+
const dr = denseRank.get(f.id) ?? -1;
|
|
178
202
|
const rs = rerankScores.get(f.id);
|
|
179
203
|
const ds = denseScore.get(f.id);
|
|
180
204
|
out.push({
|
|
@@ -190,9 +214,13 @@ export async function recall(q, deps) {
|
|
|
190
214
|
activation: activation(m.accessLog, m.salience, now, config.activation),
|
|
191
215
|
hydrated: false,
|
|
192
216
|
},
|
|
217
|
+
origin: originOf(m, full),
|
|
193
218
|
});
|
|
194
219
|
}
|
|
195
|
-
|
|
220
|
+
out = withConflicts(out);
|
|
221
|
+
let result = view.mode === 'history'
|
|
222
|
+
? await withChains(out, full, read, now, view, queryTokens, config)
|
|
223
|
+
: out;
|
|
196
224
|
for (const m of result) {
|
|
197
225
|
if (isCurrent(m, now) && isStale(m, now, config.recall.staleAfterDays)) {
|
|
198
226
|
m.explanation.stale = true;
|
|
@@ -211,6 +239,10 @@ export async function recall(q, deps) {
|
|
|
211
239
|
result = await Promise.all(result.map(async (m) => {
|
|
212
240
|
if (!touched.has(m.id) || m.tier !== 'cold' || !isCurrent(m, now))
|
|
213
241
|
return m;
|
|
242
|
+
// Hydrating writes the memory back; a node this caller only reads (the Planet, for a Star
|
|
243
|
+
// agent) is served from its live row, cold and lexical-only, and left as it is.
|
|
244
|
+
if (!grantWrites(grant, m))
|
|
245
|
+
return m;
|
|
214
246
|
try {
|
|
215
247
|
const blob = await store.coldGet(grant, m.id);
|
|
216
248
|
// A lost or torn blob: the memory stays cold, lexical-only.
|
|
@@ -264,9 +296,87 @@ export async function recall(q, deps) {
|
|
|
264
296
|
...(rerankSkipped !== undefined ? { rerankSkipped } : {}),
|
|
265
297
|
...(embedSkipped !== undefined ? { embedSkipped } : {}),
|
|
266
298
|
temporal,
|
|
299
|
+
read: nodes.map((n) => nodeLabel(n, full)),
|
|
267
300
|
timings,
|
|
268
301
|
};
|
|
269
302
|
}
|
|
303
|
+
/** The grant a query reads with: its own, or the local grant for its scope and home. */
|
|
304
|
+
function grantOf(q) {
|
|
305
|
+
if (q.grant !== undefined) {
|
|
306
|
+
assertGrant(q.grant);
|
|
307
|
+
return q.grant;
|
|
308
|
+
}
|
|
309
|
+
if (q.scope === undefined)
|
|
310
|
+
throw new TypeError('recall needs a scope or a grant');
|
|
311
|
+
return localGrant(q.scope, q.home === undefined ? {} : { home: q.home });
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Per-node hit lists as one list, by rank: every node's first hit, then every node's second, in
|
|
315
|
+
* node order. With one non-empty node it is exactly that node's list.
|
|
316
|
+
*/
|
|
317
|
+
function byRank(lists) {
|
|
318
|
+
if (lists.length === 1)
|
|
319
|
+
return lists[0];
|
|
320
|
+
const out = [];
|
|
321
|
+
const seen = new Set();
|
|
322
|
+
const longest = Math.max(0, ...lists.map((l) => l.length));
|
|
323
|
+
for (let r = 0; r < longest; r++) {
|
|
324
|
+
for (const l of lists) {
|
|
325
|
+
const h = l[r];
|
|
326
|
+
if (h !== undefined && !seen.has(h.id)) {
|
|
327
|
+
seen.add(h.id);
|
|
328
|
+
out.push(h);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return out;
|
|
333
|
+
}
|
|
334
|
+
/** Each id's 0-based rank within its own node's list. */
|
|
335
|
+
function rankWithin(lists) {
|
|
336
|
+
const out = new Map();
|
|
337
|
+
for (const l of lists) {
|
|
338
|
+
l.forEach((h, i) => {
|
|
339
|
+
if (!out.has(h.id))
|
|
340
|
+
out.set(h.id, i);
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
return out;
|
|
344
|
+
}
|
|
345
|
+
const nodeOfMemory = (m) => nodeOf(addressOf(m));
|
|
346
|
+
/**
|
|
347
|
+
* The conflict rule (spec 7.2), always on: when a returned memory contradicts a returned memory in
|
|
348
|
+
* another node (recorded by observe as a `level`/`contradicts` decision), the narrower of the two
|
|
349
|
+
* ranks first and the wider one stays, its origin naming what overrides it. Equally specific: the
|
|
350
|
+
* memory that recorded the contradiction wins.
|
|
351
|
+
*/
|
|
352
|
+
function withConflicts(out) {
|
|
353
|
+
const byId = new Map(out.map((m) => [m.id, m]));
|
|
354
|
+
const list = [...out];
|
|
355
|
+
for (const m of out) {
|
|
356
|
+
for (const otherId of contradicted(m)) {
|
|
357
|
+
const other = byId.get(otherId);
|
|
358
|
+
if (other === undefined || other === m)
|
|
359
|
+
continue;
|
|
360
|
+
const [a, b] = [nodeOfMemory(m), nodeOfMemory(other)];
|
|
361
|
+
if (nodeKey(a) === nodeKey(b))
|
|
362
|
+
continue;
|
|
363
|
+
const [winner, loser] = specificity(b) > specificity(a) ? [other, m] : [m, other];
|
|
364
|
+
const iw = list.indexOf(winner);
|
|
365
|
+
const il = list.indexOf(loser);
|
|
366
|
+
if (iw > il) {
|
|
367
|
+
list.splice(iw, 1);
|
|
368
|
+
list.splice(il, 0, winner);
|
|
369
|
+
}
|
|
370
|
+
if (loser.origin !== undefined && winner.origin !== undefined) {
|
|
371
|
+
loser.origin = {
|
|
372
|
+
...loser.origin,
|
|
373
|
+
overriddenBy: { id: winner.id, node: winner.origin.node },
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return list;
|
|
379
|
+
}
|
|
270
380
|
/** The view a query asks for: explicit options first, else read from the query in code. */
|
|
271
381
|
function viewFor(q, text, now) {
|
|
272
382
|
const at = (iso) => iso ?? now.toISOString();
|
|
@@ -305,7 +415,7 @@ const CHAIN_MAX = 8;
|
|
|
305
415
|
* first-ranked chain first. Corrected versions are skipped (they were never true) but walked
|
|
306
416
|
* through. Versions only the walk found are marked `chain`.
|
|
307
417
|
*/
|
|
308
|
-
async function withChains(out, read, now, view, queryTokens, config) {
|
|
418
|
+
async function withChains(out, grantFor, read, now, view, queryTokens, config) {
|
|
309
419
|
const byId = new Map(out.map((m) => [m.id, m]));
|
|
310
420
|
const seen = new Set();
|
|
311
421
|
const result = [];
|
|
@@ -341,6 +451,7 @@ async function withChains(out, read, now, view, queryTokens, config) {
|
|
|
341
451
|
hydrated: false,
|
|
342
452
|
chain: true,
|
|
343
453
|
},
|
|
454
|
+
origin: originOf(m, grantFor),
|
|
344
455
|
});
|
|
345
456
|
}
|
|
346
457
|
}
|
|
@@ -453,20 +564,55 @@ const INJECT_CLOSE = '\n</oort-memory>';
|
|
|
453
564
|
*/
|
|
454
565
|
export function fitForInjection(memories, budgetChars, now = new Date()) {
|
|
455
566
|
let total = INJECT_OPEN.length + INJECT_CLOSE.length;
|
|
567
|
+
let hive = 0;
|
|
456
568
|
const lines = [];
|
|
457
569
|
const kept = [];
|
|
570
|
+
// Levels are named only when something comes from outside the reader's home node: a recall of
|
|
571
|
+
// one node reads exactly as before levels existed.
|
|
572
|
+
const labelled = memories.some((m) => m.origin !== undefined && !m.origin.home);
|
|
458
573
|
for (const m of memories) {
|
|
459
574
|
const note = m.explanation.stale === true
|
|
460
575
|
? ` (may be outdated; last confirmed ${age(lastConfirmed(m), now)})`
|
|
461
576
|
: '';
|
|
462
|
-
const
|
|
577
|
+
const text = scrub(escapeBlockTags(escapeLabelText(m.text))).text;
|
|
578
|
+
const line = `- [${labelFor(m, labelled, now)}] ${text}${note}`;
|
|
463
579
|
const cost = line.length + (lines.length > 0 ? 1 : 0);
|
|
580
|
+
// Hive memories (Constellation, Galaxy) take at most HIVE_QUOTA of the budget, so a large team
|
|
581
|
+
// cannot crowd out the person's own memory: one past it is skipped, not the end of the block.
|
|
582
|
+
const fromHive = m.origin?.level === 'hive';
|
|
583
|
+
if (fromHive && hive + cost > HIVE_QUOTA * budgetChars)
|
|
584
|
+
continue;
|
|
464
585
|
if (total + cost > budgetChars)
|
|
465
586
|
break;
|
|
466
587
|
lines.push(line);
|
|
467
588
|
kept.push(m);
|
|
468
589
|
total += cost;
|
|
590
|
+
if (fromHive)
|
|
591
|
+
hive += cost;
|
|
469
592
|
}
|
|
470
593
|
return { kept, lines };
|
|
471
594
|
}
|
|
595
|
+
/**
|
|
596
|
+
* Memory text with any `<oort-…` or `</oort-…` tag opening written as `<`, so a memory cannot
|
|
597
|
+
* close the `<oort-memory>` block (or open another) and have what follows read as the prompt.
|
|
598
|
+
*/
|
|
599
|
+
export function escapeBlockTags(text) {
|
|
600
|
+
return text.replace(/<(?=\s*\/?\s*oort-)/gi, '<');
|
|
601
|
+
}
|
|
602
|
+
/** The share of an injection budget hive memories may take (spec 7.2). */
|
|
603
|
+
export const HIVE_QUOTA = 0.4;
|
|
604
|
+
/**
|
|
605
|
+
* A memory's label, built by code from structured fields only: `semantic, 2d ago`, or with levels
|
|
606
|
+
* named `semantic, Star cursor, 5h ago`, and `, overridden by Star cursor` when a narrower memory
|
|
607
|
+
* in the result contradicts it. Memory text never reaches it (and its brackets are escaped).
|
|
608
|
+
*/
|
|
609
|
+
function labelFor(m, labelled, now) {
|
|
610
|
+
const parts = [m.type];
|
|
611
|
+
if (labelled && m.origin !== undefined)
|
|
612
|
+
parts.push(m.origin.node);
|
|
613
|
+
parts.push(when(m, now));
|
|
614
|
+
if (m.origin?.overriddenBy !== undefined)
|
|
615
|
+
parts.push(`overridden by ${m.origin.overriddenBy.node}`);
|
|
616
|
+
return parts.join(', ');
|
|
617
|
+
}
|
|
472
618
|
//# sourceMappingURL=recall.js.map
|