@gamaze/hicortex 0.13.3 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -1
- package/dist/cli.js +12 -0
- package/dist/db.js +14 -0
- package/dist/init.d.ts +7 -0
- package/dist/init.js +44 -16
- package/dist/lessons-context.d.ts +14 -0
- package/dist/lessons-context.js +3 -0
- package/dist/mcp-server.js +102 -6
- package/dist/nightly.js +5 -0
- package/dist/recall-hook-cli.d.ts +28 -0
- package/dist/recall-hook-cli.js +77 -0
- package/dist/recall-index.d.ts +54 -0
- package/dist/recall-index.js +169 -0
- package/dist/recall-registry.d.ts +48 -0
- package/dist/recall-registry.js +99 -0
- package/dist/retrieval.d.ts +39 -1
- package/dist/retrieval.js +122 -22
- package/dist/storage.d.ts +8 -1
- package/dist/storage.js +27 -1
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* POST /recall-index — pushed recall index (#192).
|
|
4
|
+
*
|
|
5
|
+
* One recall logic for every harness: CC calls it from a UserPromptSubmit
|
|
6
|
+
* hook, the Hermes/OC plugins can call it per turn. The server searches the
|
|
7
|
+
* corpus with the prompt text and returns a COMPACT INDEX (one line per
|
|
8
|
+
* memory — a menu, not the meal); the agent lazy-loads full content with
|
|
9
|
+
* `hicortex_get(id)` only when a line is actually relevant.
|
|
10
|
+
*
|
|
11
|
+
* Strengthening semantics (the recall/decay alignment):
|
|
12
|
+
* - Appearing in the index = exposure: shown_count + last_accessed refresh
|
|
13
|
+
* (mild, temporary strengthen — the decay clock resets) via
|
|
14
|
+
* storage.touchMemoriesShown. NO access_count bump: hardening, the prune
|
|
15
|
+
* shield, and the adoption metric stay driven by real use.
|
|
16
|
+
* - hicortex_get = use: full strengthen (access_count + 1).
|
|
17
|
+
*
|
|
18
|
+
* Anti-bloat gates: relevance floor (measured cosine, or a real BM25 match),
|
|
19
|
+
* per-session TURN-based dedup (SessionRecallRegistry), short-prompt skip,
|
|
20
|
+
* and a hard item cap. On a prompt with no relevant memories the block is
|
|
21
|
+
* null and the hook prints nothing.
|
|
22
|
+
*/
|
|
23
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
24
|
+
if (k2 === undefined) k2 = k;
|
|
25
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
26
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
27
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
28
|
+
}
|
|
29
|
+
Object.defineProperty(o, k2, desc);
|
|
30
|
+
}) : (function(o, m, k, k2) {
|
|
31
|
+
if (k2 === undefined) k2 = k;
|
|
32
|
+
o[k2] = m[k];
|
|
33
|
+
}));
|
|
34
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
35
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
36
|
+
}) : function(o, v) {
|
|
37
|
+
o["default"] = v;
|
|
38
|
+
});
|
|
39
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
40
|
+
var ownKeys = function(o) {
|
|
41
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
42
|
+
var ar = [];
|
|
43
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
44
|
+
return ar;
|
|
45
|
+
};
|
|
46
|
+
return ownKeys(o);
|
|
47
|
+
};
|
|
48
|
+
return function (mod) {
|
|
49
|
+
if (mod && mod.__esModule) return mod;
|
|
50
|
+
var result = {};
|
|
51
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
52
|
+
__setModuleDefault(result, mod);
|
|
53
|
+
return result;
|
|
54
|
+
};
|
|
55
|
+
})();
|
|
56
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
57
|
+
exports.memoryTitle = memoryTitle;
|
|
58
|
+
exports.passesRelevanceGate = passesRelevanceGate;
|
|
59
|
+
exports.handleRecallIndex = handleRecallIndex;
|
|
60
|
+
const storage = __importStar(require("./storage.js"));
|
|
61
|
+
const DEFAULT_MIN_SIMILARITY = 0.55;
|
|
62
|
+
const DEFAULT_MAX_ITEMS = 6;
|
|
63
|
+
const DEFAULT_MIN_PROMPT_LENGTH = 20;
|
|
64
|
+
/** Retrieve more than maxItems so gating + dedup still leave a full menu. */
|
|
65
|
+
const CANDIDATE_MULTIPLIER = 3;
|
|
66
|
+
/** First content line, de-markdowned and truncated — the index line title. */
|
|
67
|
+
function memoryTitle(content, maxLen = 100) {
|
|
68
|
+
const firstLine = content
|
|
69
|
+
.split("\n")
|
|
70
|
+
.map((l) => l.trim())
|
|
71
|
+
.find((l) => l.length > 0) ?? "";
|
|
72
|
+
const title = firstLine
|
|
73
|
+
.replace(/^#+\s*/, "")
|
|
74
|
+
.replace(/^Session Memory:\s*/i, "")
|
|
75
|
+
.replace(/^Lesson:\s*/i, "")
|
|
76
|
+
.trim();
|
|
77
|
+
return title.length > maxLen ? `${title.slice(0, maxLen - 1)}…` : title;
|
|
78
|
+
}
|
|
79
|
+
function formatDate(iso) {
|
|
80
|
+
const d = new Date(iso);
|
|
81
|
+
if (isNaN(d.getTime()))
|
|
82
|
+
return "";
|
|
83
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
84
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
85
|
+
return `${dd}.${mm}.${d.getFullYear()}`;
|
|
86
|
+
}
|
|
87
|
+
function formatIndexLine(r) {
|
|
88
|
+
const meta = [formatDate(r.created_at), r.domain ?? r.project ?? undefined, r.memory_type]
|
|
89
|
+
.filter(Boolean)
|
|
90
|
+
.join(", ");
|
|
91
|
+
return `- [${r.id}] ${memoryTitle(r.content)}${meta ? ` (${meta})` : ""}`;
|
|
92
|
+
}
|
|
93
|
+
/** Relevance gate: real text match, or measured cosine above the floor. */
|
|
94
|
+
function passesRelevanceGate(r, minSimilarity) {
|
|
95
|
+
if (r.source === "fts" || r.source === "both")
|
|
96
|
+
return true;
|
|
97
|
+
return typeof r.similarity === "number" && r.similarity >= minSimilarity;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Handle a /recall-index request body. Thin Express adapter in mcp-server.ts;
|
|
101
|
+
* all behavior lives here so tests exercise it directly.
|
|
102
|
+
*/
|
|
103
|
+
async function handleRecallIndex(deps, body) {
|
|
104
|
+
const req = (body ?? {});
|
|
105
|
+
const sessionId = typeof req.session_id === "string" && req.session_id ? req.session_id : null;
|
|
106
|
+
if (!sessionId) {
|
|
107
|
+
return { status: 400, body: { error: "Missing 'session_id'" } };
|
|
108
|
+
}
|
|
109
|
+
// Reset: SessionStart (startup/resume/clear/compact) — fresh context, so the
|
|
110
|
+
// shown-set is stale by definition.
|
|
111
|
+
if (req.reset === true) {
|
|
112
|
+
deps.registry.reset(sessionId);
|
|
113
|
+
return { status: 200, body: { ok: true, reset: true } };
|
|
114
|
+
}
|
|
115
|
+
const prompt = typeof req.prompt === "string" ? req.prompt.trim() : "";
|
|
116
|
+
const minPromptLength = deps.options?.minPromptLength ?? DEFAULT_MIN_PROMPT_LENGTH;
|
|
117
|
+
if (prompt.length < minPromptLength) {
|
|
118
|
+
return { status: 200, body: { block: null, skipped: "short-prompt" } };
|
|
119
|
+
}
|
|
120
|
+
const maxItems = clampInt(deps.options?.maxItems, DEFAULT_MAX_ITEMS, 1, 20);
|
|
121
|
+
const minSimilarity = clampNumber(deps.options?.minSimilarity, DEFAULT_MIN_SIMILARITY, 0, 1);
|
|
122
|
+
const turn = deps.registry.beginTurn(sessionId);
|
|
123
|
+
let results;
|
|
124
|
+
try {
|
|
125
|
+
results = await deps.retrieveFn(prompt, maxItems * CANDIDATE_MULTIPLIER);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
return {
|
|
129
|
+
status: 500,
|
|
130
|
+
body: { error: err instanceof Error ? err.message : String(err) },
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
const picked = results
|
|
134
|
+
.filter((r) => passesRelevanceGate(r, minSimilarity))
|
|
135
|
+
.filter((r) => deps.registry.isShowable(sessionId, r.id))
|
|
136
|
+
.slice(0, maxItems);
|
|
137
|
+
if (picked.length === 0) {
|
|
138
|
+
return { status: 200, body: { block: null, shown: [], turn } };
|
|
139
|
+
}
|
|
140
|
+
const ids = picked.map((r) => r.id);
|
|
141
|
+
deps.registry.markShown(sessionId, ids);
|
|
142
|
+
// Exposure signal: shown_count + last_accessed refresh, NOT access_count.
|
|
143
|
+
storage.touchMemoriesShown(deps.db, ids, new Date().toISOString());
|
|
144
|
+
const lines = picked.map((r) => formatIndexLine(r));
|
|
145
|
+
const block = [
|
|
146
|
+
"## Memory recall (auto)",
|
|
147
|
+
// Provenance is BUILT IN, split by function (owner decision 27.07,
|
|
148
|
+
// option D): this header carries only the SELECTION-time rules —
|
|
149
|
+
// supersession (the one moment competing dates are visible side by side)
|
|
150
|
+
// and cite-what-you-rely-on (covers snippet-only use, the common case per
|
|
151
|
+
// the 0.14.0 field test). The full citation format + origin agent ride on
|
|
152
|
+
// the hicortex_get response / GET /memory `citation` field (use-time).
|
|
153
|
+
"Possibly relevant memories — dates matter, newer supersedes older. Fetch full content with `hicortex_get(id)` only when relevant; cite any memory you rely on (id, date):",
|
|
154
|
+
...lines,
|
|
155
|
+
].join("\n");
|
|
156
|
+
return { status: 200, body: { block, shown: ids, turn } };
|
|
157
|
+
}
|
|
158
|
+
function clampInt(v, dflt, min, max) {
|
|
159
|
+
const n = Number(v);
|
|
160
|
+
if (!Number.isFinite(n))
|
|
161
|
+
return dflt;
|
|
162
|
+
return Math.max(min, Math.min(max, Math.floor(n)));
|
|
163
|
+
}
|
|
164
|
+
function clampNumber(v, dflt, min, max) {
|
|
165
|
+
const n = Number(v);
|
|
166
|
+
if (!Number.isFinite(n))
|
|
167
|
+
return dflt;
|
|
168
|
+
return Math.max(min, Math.min(max, n));
|
|
169
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SessionRecallRegistry — per-session, TURN-based dedup for pushed recall
|
|
3
|
+
* (#192, POST /recall-index).
|
|
4
|
+
*
|
|
5
|
+
* Why turn-based, not time-based: suppression must track the session's
|
|
6
|
+
* CONTEXT, not the wall clock. A multi-day CC session with a 1M window can
|
|
7
|
+
* hold hundreds of turns; a shown memory is redundant while it is plausibly
|
|
8
|
+
* still in context and useful again once enough turns have passed (or the
|
|
9
|
+
* context was compacted away). Turn count is the proxy the server can own
|
|
10
|
+
* without clients reporting token volumes.
|
|
11
|
+
*
|
|
12
|
+
* Semantics:
|
|
13
|
+
* - Every non-reset /recall-index call for a session advances its turn
|
|
14
|
+
* counter by one.
|
|
15
|
+
* - A memory id shown at turn T is suppressed until turn T + reshowTurns.
|
|
16
|
+
* - reset(sessionId) clears the session's shown-set (fired by the CC
|
|
17
|
+
* SessionStart hook — which includes source=compact, i.e. after
|
|
18
|
+
* compaction the fresh context may legitimately re-receive everything).
|
|
19
|
+
*
|
|
20
|
+
* Purely in-memory: a server restart forgets shown-state, worst case a few
|
|
21
|
+
* early re-shows (~15 tokens each) — harmless by design. Sessions are pruned
|
|
22
|
+
* LRU beyond maxSessions so long-running servers don't accumulate state.
|
|
23
|
+
*/
|
|
24
|
+
export interface RecallRegistryOptions {
|
|
25
|
+
/** Turns a shown id stays suppressed. Config `recallReshowTurns`, default 30. */
|
|
26
|
+
reshowTurns?: number;
|
|
27
|
+
/** Max tracked sessions before LRU eviction. */
|
|
28
|
+
maxSessions?: number;
|
|
29
|
+
}
|
|
30
|
+
export declare const DEFAULT_RESHOW_TURNS = 30;
|
|
31
|
+
export declare class SessionRecallRegistry {
|
|
32
|
+
private readonly reshowTurns;
|
|
33
|
+
private readonly maxSessions;
|
|
34
|
+
private readonly sessions;
|
|
35
|
+
constructor(options?: RecallRegistryOptions);
|
|
36
|
+
/** Advance the session's turn counter (one call = one turn). */
|
|
37
|
+
beginTurn(sessionId: string): number;
|
|
38
|
+
/** True when the id has not been shown within the last reshowTurns turns. */
|
|
39
|
+
isShowable(sessionId: string, memoryId: string): boolean;
|
|
40
|
+
/** Record ids as shown at the session's current turn. */
|
|
41
|
+
markShown(sessionId: string, memoryIds: string[]): void;
|
|
42
|
+
/** Forget a session's shown-set (SessionStart / compaction). */
|
|
43
|
+
reset(sessionId: string): void;
|
|
44
|
+
/** Number of tracked sessions (for /recall-index introspection + tests). */
|
|
45
|
+
size(): number;
|
|
46
|
+
private getOrCreate;
|
|
47
|
+
private evictIfNeeded;
|
|
48
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* SessionRecallRegistry — per-session, TURN-based dedup for pushed recall
|
|
4
|
+
* (#192, POST /recall-index).
|
|
5
|
+
*
|
|
6
|
+
* Why turn-based, not time-based: suppression must track the session's
|
|
7
|
+
* CONTEXT, not the wall clock. A multi-day CC session with a 1M window can
|
|
8
|
+
* hold hundreds of turns; a shown memory is redundant while it is plausibly
|
|
9
|
+
* still in context and useful again once enough turns have passed (or the
|
|
10
|
+
* context was compacted away). Turn count is the proxy the server can own
|
|
11
|
+
* without clients reporting token volumes.
|
|
12
|
+
*
|
|
13
|
+
* Semantics:
|
|
14
|
+
* - Every non-reset /recall-index call for a session advances its turn
|
|
15
|
+
* counter by one.
|
|
16
|
+
* - A memory id shown at turn T is suppressed until turn T + reshowTurns.
|
|
17
|
+
* - reset(sessionId) clears the session's shown-set (fired by the CC
|
|
18
|
+
* SessionStart hook — which includes source=compact, i.e. after
|
|
19
|
+
* compaction the fresh context may legitimately re-receive everything).
|
|
20
|
+
*
|
|
21
|
+
* Purely in-memory: a server restart forgets shown-state, worst case a few
|
|
22
|
+
* early re-shows (~15 tokens each) — harmless by design. Sessions are pruned
|
|
23
|
+
* LRU beyond maxSessions so long-running servers don't accumulate state.
|
|
24
|
+
*/
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.SessionRecallRegistry = exports.DEFAULT_RESHOW_TURNS = void 0;
|
|
27
|
+
exports.DEFAULT_RESHOW_TURNS = 30;
|
|
28
|
+
const DEFAULT_MAX_SESSIONS = 500;
|
|
29
|
+
class SessionRecallRegistry {
|
|
30
|
+
reshowTurns;
|
|
31
|
+
maxSessions;
|
|
32
|
+
sessions = new Map();
|
|
33
|
+
constructor(options) {
|
|
34
|
+
const turns = Number(options?.reshowTurns);
|
|
35
|
+
this.reshowTurns =
|
|
36
|
+
Number.isFinite(turns) && turns > 0 ? Math.floor(turns) : exports.DEFAULT_RESHOW_TURNS;
|
|
37
|
+
const max = Number(options?.maxSessions);
|
|
38
|
+
this.maxSessions =
|
|
39
|
+
Number.isFinite(max) && max > 0 ? Math.floor(max) : DEFAULT_MAX_SESSIONS;
|
|
40
|
+
}
|
|
41
|
+
/** Advance the session's turn counter (one call = one turn). */
|
|
42
|
+
beginTurn(sessionId) {
|
|
43
|
+
const s = this.getOrCreate(sessionId);
|
|
44
|
+
s.turn += 1;
|
|
45
|
+
s.lastUsedAt = Date.now();
|
|
46
|
+
return s.turn;
|
|
47
|
+
}
|
|
48
|
+
/** True when the id has not been shown within the last reshowTurns turns. */
|
|
49
|
+
isShowable(sessionId, memoryId) {
|
|
50
|
+
const s = this.sessions.get(sessionId);
|
|
51
|
+
if (!s)
|
|
52
|
+
return true;
|
|
53
|
+
const shownAt = s.shown.get(memoryId);
|
|
54
|
+
if (shownAt === undefined)
|
|
55
|
+
return true;
|
|
56
|
+
return s.turn - shownAt >= this.reshowTurns;
|
|
57
|
+
}
|
|
58
|
+
/** Record ids as shown at the session's current turn. */
|
|
59
|
+
markShown(sessionId, memoryIds) {
|
|
60
|
+
if (memoryIds.length === 0)
|
|
61
|
+
return;
|
|
62
|
+
const s = this.getOrCreate(sessionId);
|
|
63
|
+
for (const id of memoryIds)
|
|
64
|
+
s.shown.set(id, s.turn);
|
|
65
|
+
s.lastUsedAt = Date.now();
|
|
66
|
+
}
|
|
67
|
+
/** Forget a session's shown-set (SessionStart / compaction). */
|
|
68
|
+
reset(sessionId) {
|
|
69
|
+
this.sessions.delete(sessionId);
|
|
70
|
+
}
|
|
71
|
+
/** Number of tracked sessions (for /recall-index introspection + tests). */
|
|
72
|
+
size() {
|
|
73
|
+
return this.sessions.size;
|
|
74
|
+
}
|
|
75
|
+
getOrCreate(sessionId) {
|
|
76
|
+
let s = this.sessions.get(sessionId);
|
|
77
|
+
if (!s) {
|
|
78
|
+
this.evictIfNeeded();
|
|
79
|
+
s = { turn: 0, shown: new Map(), lastUsedAt: Date.now() };
|
|
80
|
+
this.sessions.set(sessionId, s);
|
|
81
|
+
}
|
|
82
|
+
return s;
|
|
83
|
+
}
|
|
84
|
+
evictIfNeeded() {
|
|
85
|
+
if (this.sessions.size < this.maxSessions)
|
|
86
|
+
return;
|
|
87
|
+
let oldestId = null;
|
|
88
|
+
let oldestAt = Infinity;
|
|
89
|
+
for (const [id, s] of this.sessions) {
|
|
90
|
+
if (s.lastUsedAt < oldestAt) {
|
|
91
|
+
oldestAt = s.lastUsedAt;
|
|
92
|
+
oldestId = id;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (oldestId)
|
|
96
|
+
this.sessions.delete(oldestId);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
exports.SessionRecallRegistry = SessionRecallRegistry;
|
package/dist/retrieval.d.ts
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* score = similarity * 0.4 + effective_strength * 0.3 + connection_score * 0.2 + recency * 0.1
|
|
7
7
|
*
|
|
8
8
|
* Decay model (B+E+D):
|
|
9
|
-
* base_decay =
|
|
9
|
+
* base_decay = derived from decayHalfLifeDays (config; default 365 → ~1-year
|
|
10
|
+
* half-life at importance 0.5, importance-scaled either way)
|
|
10
11
|
* decay_rate = 1 - base_decay * (1 - importance)
|
|
11
12
|
* decay_rate = 1 - (1 - decay_rate) * 0.7^access_count
|
|
12
13
|
* decay_rate = 1 - (1 - decay_rate) * 0.7^link_count
|
|
@@ -15,6 +16,39 @@
|
|
|
15
16
|
*/
|
|
16
17
|
import type Database from "better-sqlite3";
|
|
17
18
|
import type { Memory, MemorySearchResult } from "./types.js";
|
|
19
|
+
/** Default decay half-life (days) at importance 0.5. #192: was 0.0005/h
|
|
20
|
+
* (~115-day half-life at base 0.5) — aggressive enough to bury the long tail
|
|
21
|
+
* in ranking. Long-term remembering is the product; time preference stays,
|
|
22
|
+
* but mild. */
|
|
23
|
+
export declare const DEFAULT_DECAY_HALF_LIFE_DAYS = 365;
|
|
24
|
+
/**
|
|
25
|
+
* Derive the per-hour base decay constant from a half-life target: for the
|
|
26
|
+
* decayable portion, retention^hours = 0.5 at `days`, evaluated at the
|
|
27
|
+
* reference importance 0.5 (the model scales the rate by (1 − importance)).
|
|
28
|
+
* decay_rate = 1 − λ(1 − imp) ⇒ half-life ≈ ln2 / (λ·(1 − imp)), so
|
|
29
|
+
* λ = ln2 / (24·days·0.5).
|
|
30
|
+
*/
|
|
31
|
+
export declare function decayConstantForHalfLife(days: number): number;
|
|
32
|
+
/**
|
|
33
|
+
* Configure the decay speed from config (`decayHalfLifeDays`). Called at boot
|
|
34
|
+
* by the server and the nightly so both processes score with the same clock.
|
|
35
|
+
* Invalid/absent values keep the default. Exported value for tests.
|
|
36
|
+
*/
|
|
37
|
+
export declare function configureDecay(options?: {
|
|
38
|
+
halfLifeDays?: unknown;
|
|
39
|
+
}): number;
|
|
40
|
+
interface RecallDefaults {
|
|
41
|
+
searchLimit: number;
|
|
42
|
+
recentLimit: number;
|
|
43
|
+
recentWindowDays: number;
|
|
44
|
+
coldExposureSlots: number;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Configure recall breadth from config. Called at boot next to
|
|
48
|
+
* configureDecay(); invalid/absent values keep the shipped defaults.
|
|
49
|
+
* Returns the resolved values (for logging + tests).
|
|
50
|
+
*/
|
|
51
|
+
export declare function configureRecall(config?: Record<string, unknown> | null): RecallDefaults;
|
|
18
52
|
/**
|
|
19
53
|
* Convert an L2 distance (as returned by sqlite-vec's vec0 `distance`) to
|
|
20
54
|
* cosine similarity. Valid because our embeddings are L2-normalized
|
|
@@ -51,6 +85,9 @@ export declare function retrieve(db: Database.Database, embedFn: EmbedFn, query:
|
|
|
51
85
|
project?: string | null;
|
|
52
86
|
privacy?: string[];
|
|
53
87
|
sourceAgent?: string;
|
|
88
|
+
/** #192: skip access strengthening — for pushed recall (/recall-index),
|
|
89
|
+
* where appearing in results must not count as use. */
|
|
90
|
+
noStrengthen?: boolean;
|
|
54
91
|
}): Promise<MemorySearchResult[]>;
|
|
55
92
|
/**
|
|
56
93
|
* Get recent context, optionally filtered by project and privacy.
|
|
@@ -60,3 +97,4 @@ export declare function searchRecent(db: Database.Database, options?: {
|
|
|
60
97
|
limit?: number;
|
|
61
98
|
privacy?: string[];
|
|
62
99
|
}): MemorySearchResult[];
|
|
100
|
+
export {};
|
package/dist/retrieval.js
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
* score = similarity * 0.4 + effective_strength * 0.3 + connection_score * 0.2 + recency * 0.1
|
|
8
8
|
*
|
|
9
9
|
* Decay model (B+E+D):
|
|
10
|
-
* base_decay =
|
|
10
|
+
* base_decay = derived from decayHalfLifeDays (config; default 365 → ~1-year
|
|
11
|
+
* half-life at importance 0.5, importance-scaled either way)
|
|
11
12
|
* decay_rate = 1 - base_decay * (1 - importance)
|
|
12
13
|
* decay_rate = 1 - (1 - decay_rate) * 0.7^access_count
|
|
13
14
|
* decay_rate = 1 - (1 - decay_rate) * 0.7^link_count
|
|
@@ -48,13 +49,67 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
48
49
|
};
|
|
49
50
|
})();
|
|
50
51
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
|
+
exports.DEFAULT_DECAY_HALF_LIFE_DAYS = void 0;
|
|
53
|
+
exports.decayConstantForHalfLife = decayConstantForHalfLife;
|
|
54
|
+
exports.configureDecay = configureDecay;
|
|
55
|
+
exports.configureRecall = configureRecall;
|
|
51
56
|
exports.l2ToCosine = l2ToCosine;
|
|
52
57
|
exports.effectiveStrength = effectiveStrength;
|
|
53
58
|
exports.computeScore = computeScore;
|
|
54
59
|
exports.retrieve = retrieve;
|
|
55
60
|
exports.searchRecent = searchRecent;
|
|
56
61
|
const storage = __importStar(require("./storage.js"));
|
|
57
|
-
|
|
62
|
+
/** Default decay half-life (days) at importance 0.5. #192: was 0.0005/h
|
|
63
|
+
* (~115-day half-life at base 0.5) — aggressive enough to bury the long tail
|
|
64
|
+
* in ranking. Long-term remembering is the product; time preference stays,
|
|
65
|
+
* but mild. */
|
|
66
|
+
exports.DEFAULT_DECAY_HALF_LIFE_DAYS = 365;
|
|
67
|
+
/**
|
|
68
|
+
* Derive the per-hour base decay constant from a half-life target: for the
|
|
69
|
+
* decayable portion, retention^hours = 0.5 at `days`, evaluated at the
|
|
70
|
+
* reference importance 0.5 (the model scales the rate by (1 − importance)).
|
|
71
|
+
* decay_rate = 1 − λ(1 − imp) ⇒ half-life ≈ ln2 / (λ·(1 − imp)), so
|
|
72
|
+
* λ = ln2 / (24·days·0.5).
|
|
73
|
+
*/
|
|
74
|
+
function decayConstantForHalfLife(days) {
|
|
75
|
+
return Math.LN2 / (24 * days * 0.5);
|
|
76
|
+
}
|
|
77
|
+
let BASE_DECAY = decayConstantForHalfLife(exports.DEFAULT_DECAY_HALF_LIFE_DAYS);
|
|
78
|
+
/**
|
|
79
|
+
* Configure the decay speed from config (`decayHalfLifeDays`). Called at boot
|
|
80
|
+
* by the server and the nightly so both processes score with the same clock.
|
|
81
|
+
* Invalid/absent values keep the default. Exported value for tests.
|
|
82
|
+
*/
|
|
83
|
+
function configureDecay(options) {
|
|
84
|
+
const days = Number(options?.halfLifeDays);
|
|
85
|
+
BASE_DECAY = decayConstantForHalfLife(Number.isFinite(days) && days > 0 ? days : exports.DEFAULT_DECAY_HALF_LIFE_DAYS);
|
|
86
|
+
return BASE_DECAY;
|
|
87
|
+
}
|
|
88
|
+
const RECALL_DEFAULTS = {
|
|
89
|
+
searchLimit: 8,
|
|
90
|
+
recentLimit: 12,
|
|
91
|
+
recentWindowDays: 180,
|
|
92
|
+
coldExposureSlots: 2,
|
|
93
|
+
};
|
|
94
|
+
let recallDefaults = { ...RECALL_DEFAULTS };
|
|
95
|
+
/**
|
|
96
|
+
* Configure recall breadth from config. Called at boot next to
|
|
97
|
+
* configureDecay(); invalid/absent values keep the shipped defaults.
|
|
98
|
+
* Returns the resolved values (for logging + tests).
|
|
99
|
+
*/
|
|
100
|
+
function configureRecall(config) {
|
|
101
|
+
const pick = (key) => {
|
|
102
|
+
const v = Number(config?.[key]);
|
|
103
|
+
return Number.isFinite(v) && v >= 0 ? Math.floor(v) : RECALL_DEFAULTS[key];
|
|
104
|
+
};
|
|
105
|
+
recallDefaults = {
|
|
106
|
+
searchLimit: Math.max(1, pick("searchLimit")),
|
|
107
|
+
recentLimit: Math.max(1, pick("recentLimit")),
|
|
108
|
+
recentWindowDays: Math.max(1, pick("recentWindowDays")),
|
|
109
|
+
coldExposureSlots: pick("coldExposureSlots"),
|
|
110
|
+
};
|
|
111
|
+
return { ...recallDefaults };
|
|
112
|
+
}
|
|
58
113
|
/**
|
|
59
114
|
* Placeholder L2 distance for candidates that have no measured vector
|
|
60
115
|
* distance (FTS-only hits and graph-discovered neighbors). Chosen so that
|
|
@@ -177,7 +232,7 @@ function collectLinks(db, seedIds, maxHops = 2) {
|
|
|
177
232
|
// ---------------------------------------------------------------------------
|
|
178
233
|
// Formatting
|
|
179
234
|
// ---------------------------------------------------------------------------
|
|
180
|
-
function formatResult(memory, score, effStr, connections) {
|
|
235
|
+
function formatResult(memory, score, effStr, connections, provenance) {
|
|
181
236
|
return {
|
|
182
237
|
id: memory.id,
|
|
183
238
|
content: memory.content ?? "",
|
|
@@ -188,6 +243,8 @@ function formatResult(memory, score, effStr, connections) {
|
|
|
188
243
|
project: memory.project ?? null,
|
|
189
244
|
created_at: memory.created_at ?? "",
|
|
190
245
|
connections,
|
|
246
|
+
similarity: provenance ? provenance.similarity : undefined,
|
|
247
|
+
source: provenance ? provenance.source : undefined,
|
|
191
248
|
};
|
|
192
249
|
}
|
|
193
250
|
// ---------------------------------------------------------------------------
|
|
@@ -224,19 +281,24 @@ function reciprocalRankFusion(rankedLists, k = RRF_K) {
|
|
|
224
281
|
* and composite scoring. Strengthens accessed memories.
|
|
225
282
|
*/
|
|
226
283
|
async function retrieve(db, embedFn, query, options) {
|
|
227
|
-
const limit = options?.limit ??
|
|
284
|
+
const limit = options?.limit ?? recallDefaults.searchLimit;
|
|
228
285
|
const project = options?.project;
|
|
229
286
|
const privacy = options?.privacy;
|
|
230
287
|
const sourceAgent = options?.sourceAgent;
|
|
231
288
|
const now = new Date();
|
|
232
289
|
// 1. Embed
|
|
233
290
|
const queryEmbedding = await embedFn(query);
|
|
234
|
-
// 2. Dual retrieval — vector + BM25
|
|
235
|
-
|
|
291
|
+
// 2. Dual retrieval — vector + BM25.
|
|
292
|
+
// #192: sqlite-vec can't push filters into the KNN, so filtered queries must
|
|
293
|
+
// over-fetch — the old flat limit*3 intersected a global top-15 with (for the
|
|
294
|
+
// median project) ~1% of the corpus, starving every filtered query.
|
|
295
|
+
const filtered = Boolean(project || privacy || sourceAgent);
|
|
296
|
+
const fetchLimit = filtered ? Math.min(limit * 20, 200) : limit * 3;
|
|
236
297
|
let vecCandidates = storage.vectorSearch(db, queryEmbedding, fetchLimit, []);
|
|
237
298
|
let ftsCandidates = [];
|
|
238
299
|
try {
|
|
239
|
-
|
|
300
|
+
// privacy/sourceAgent/project are all pushed into the FTS SQL.
|
|
301
|
+
ftsCandidates = storage.searchFts(db, query, fetchLimit, privacy, sourceAgent, project ?? undefined);
|
|
240
302
|
}
|
|
241
303
|
catch {
|
|
242
304
|
// FTS5 search can fail on special characters; fall back to vector-only
|
|
@@ -247,7 +309,6 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
247
309
|
// Post-filter vector candidates (sqlite-vec can't filter)
|
|
248
310
|
if (project) {
|
|
249
311
|
vecCandidates = vecCandidates.filter((c) => c.project === project);
|
|
250
|
-
ftsCandidates = ftsCandidates.filter((c) => c.project === project);
|
|
251
312
|
}
|
|
252
313
|
if (privacy) {
|
|
253
314
|
vecCandidates = vecCandidates.filter((c) => privacy.includes(c.privacy));
|
|
@@ -259,14 +320,18 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
259
320
|
const vecRanked = vecCandidates.map((c) => c.id);
|
|
260
321
|
const ftsRanked = ftsCandidates.map((c) => c.id);
|
|
261
322
|
const rrfScores = reciprocalRankFusion([vecRanked, ftsRanked]);
|
|
262
|
-
// Build unified candidate map
|
|
323
|
+
// Build unified candidate map (with retrieval-channel provenance, #192)
|
|
263
324
|
const candidateMap = new Map();
|
|
264
325
|
for (const c of vecCandidates) {
|
|
265
|
-
candidateMap.set(c.id, { mem: c, distance: c.distance });
|
|
326
|
+
candidateMap.set(c.id, { mem: c, distance: c.distance, source: "vector" });
|
|
266
327
|
}
|
|
267
328
|
for (const c of ftsCandidates) {
|
|
268
|
-
|
|
269
|
-
|
|
329
|
+
const existing = candidateMap.get(c.id);
|
|
330
|
+
if (existing) {
|
|
331
|
+
existing.source = "both";
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
candidateMap.set(c.id, { mem: c, distance: DEFAULT_GRAPH_DISTANCE, source: "fts" });
|
|
270
335
|
}
|
|
271
336
|
}
|
|
272
337
|
// 4. Graph traversal
|
|
@@ -284,7 +349,7 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
284
349
|
continue;
|
|
285
350
|
if (sourceAgent && mem.source_agent !== sourceAgent)
|
|
286
351
|
continue;
|
|
287
|
-
candidateMap.set(gid, { mem, distance: DEFAULT_GRAPH_DISTANCE });
|
|
352
|
+
candidateMap.set(gid, { mem, distance: DEFAULT_GRAPH_DISTANCE, source: "graph" });
|
|
288
353
|
}
|
|
289
354
|
// 5. Compute composite scores
|
|
290
355
|
const maxConnections = Math.max(...([...connectionCounts.values()].length > 0
|
|
@@ -292,7 +357,7 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
292
357
|
: [0]));
|
|
293
358
|
const scored = [];
|
|
294
359
|
const maxRrf = Math.max(...([...rrfScores.values()].length > 0 ? [...rrfScores.values()] : [1]));
|
|
295
|
-
for (const [mid, { mem, distance }] of candidateMap) {
|
|
360
|
+
for (const [mid, { mem, distance, source }] of candidateMap) {
|
|
296
361
|
const connCount = connectionCounts.get(mid) ?? 0;
|
|
297
362
|
const composite = computeScore(mem, distance, connCount, maxConnections, now);
|
|
298
363
|
const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
|
|
@@ -302,25 +367,60 @@ async function retrieve(db, embedFn, query, options) {
|
|
|
302
367
|
const rrf = rrfScores.get(mid) ?? 0;
|
|
303
368
|
const normalizedRrf = maxRrf > 0 ? rrf / maxRrf : 0;
|
|
304
369
|
const finalScore = composite * 0.8 + normalizedRrf * 0.2;
|
|
305
|
-
|
|
370
|
+
// Measured cosine only for vector-matched candidates; FTS/graph hits carry
|
|
371
|
+
// the neutral placeholder distance, which is not a real similarity.
|
|
372
|
+
const similarity = source === "vector" || source === "both"
|
|
373
|
+
? Math.round(l2ToCosine(distance) * 1e6) / 1e6
|
|
374
|
+
: null;
|
|
375
|
+
scored.push({ mem, finalScore, effStr, connCount, similarity, source });
|
|
306
376
|
}
|
|
307
|
-
// 6. Sort and take top N
|
|
377
|
+
// 6. Sort and take top N — with cold-exposure slots (#192).
|
|
378
|
+
// Access hardening + effective strength make past winners self-reinforcing:
|
|
379
|
+
// 88% of the production corpus had never been returned by any query. Reserve
|
|
380
|
+
// up to 2 of k for the best-scoring never-accessed candidates so the long
|
|
381
|
+
// tail gets nonzero exposure whenever it is semantically in range. Slots are
|
|
382
|
+
// only "reserved" when cold candidates exist; otherwise the top-k is the
|
|
383
|
+
// plain score order.
|
|
308
384
|
scored.sort((a, b) => b.finalScore - a.finalScore);
|
|
309
|
-
const
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
385
|
+
const coldSlots = limit >= 4 ? recallDefaults.coldExposureSlots : 0;
|
|
386
|
+
let top = scored.slice(0, limit);
|
|
387
|
+
if (coldSlots > 0 && scored.length > limit) {
|
|
388
|
+
const coldInTop = top.filter((t) => (t.mem.access_count ?? 0) === 0).length;
|
|
389
|
+
const wanted = coldSlots - coldInTop;
|
|
390
|
+
if (wanted > 0) {
|
|
391
|
+
const coldExtras = scored
|
|
392
|
+
.slice(limit)
|
|
393
|
+
.filter((t) => (t.mem.access_count ?? 0) === 0)
|
|
394
|
+
.slice(0, wanted);
|
|
395
|
+
if (coldExtras.length > 0) {
|
|
396
|
+
top = [...top.slice(0, limit - coldExtras.length), ...coldExtras];
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
const results = top.map((t) => formatResult(t.mem, t.finalScore, t.effStr, t.connCount, {
|
|
401
|
+
similarity: t.similarity,
|
|
402
|
+
source: t.source,
|
|
403
|
+
}));
|
|
404
|
+
// 7. Strengthen — skipped for pushed recall (#192): appearing in a pushed
|
|
405
|
+
// index is exposure, not use; the /recall-index path records shown_count +
|
|
406
|
+
// last_accessed via storage.touchMemoriesShown instead.
|
|
407
|
+
if (!options?.noStrengthen) {
|
|
408
|
+
strengthen(db, top.map((t) => t.mem), now);
|
|
409
|
+
}
|
|
313
410
|
return results;
|
|
314
411
|
}
|
|
315
412
|
/**
|
|
316
413
|
* Get recent context, optionally filtered by project and privacy.
|
|
317
414
|
*/
|
|
318
415
|
function searchRecent(db, options) {
|
|
319
|
-
const limit = options?.limit ??
|
|
416
|
+
const limit = options?.limit ?? recallDefaults.recentLimit;
|
|
320
417
|
const project = options?.project;
|
|
321
418
|
const privacy = options?.privacy;
|
|
322
419
|
const now = new Date();
|
|
323
|
-
|
|
420
|
+
// #192 breadth: 30 → 180-day default window (config recentWindowDays).
|
|
421
|
+
// "Recent" for a long-lived corpus is a season, not a month; the narrow
|
|
422
|
+
// window kept queryless recall re-serving the same few weeks.
|
|
423
|
+
let candidates = storage.getRecentMemories(db, recallDefaults.recentWindowDays, limit * 3);
|
|
324
424
|
if (project) {
|
|
325
425
|
candidates = candidates.filter((c) => c.project === project);
|
|
326
426
|
}
|
package/dist/storage.d.ts
CHANGED
|
@@ -30,6 +30,13 @@ export declare function updateMemory(db: Database.Database, memoryId: string, fi
|
|
|
30
30
|
* Atomically increment access_count and reset last_accessed.
|
|
31
31
|
*/
|
|
32
32
|
export declare function strengthenMemory(db: Database.Database, memoryId: string, nowIsoStr: string): void;
|
|
33
|
+
/**
|
|
34
|
+
* Record that memories appeared in a pushed recall index (#192): bump
|
|
35
|
+
* shown_count and refresh last_accessed (a mild strengthen — the decay clock
|
|
36
|
+
* resets so topically-live memories stop sinking) WITHOUT touching
|
|
37
|
+
* access_count, which stays reserved for real use (hardening + prune shield).
|
|
38
|
+
*/
|
|
39
|
+
export declare function touchMemoriesShown(db: Database.Database, memoryIds: string[], nowIsoStr: string): void;
|
|
33
40
|
/**
|
|
34
41
|
* Delete a memory, its vector, its tags, and all its links.
|
|
35
42
|
*/
|
|
@@ -87,7 +94,7 @@ export declare function vectorSearch(db: Database.Database, queryEmbedding: Floa
|
|
|
87
94
|
* Full-text search using FTS5 BM25 ranking.
|
|
88
95
|
* Returns memories with a rank field (lower is better).
|
|
89
96
|
*/
|
|
90
|
-
export declare function searchFts(db: Database.Database, query: string, limit?: number, privacy?: string[], sourceAgent?: string): Array<Memory & {
|
|
97
|
+
export declare function searchFts(db: Database.Database, query: string, limit?: number, privacy?: string[], sourceAgent?: string, project?: string): Array<Memory & {
|
|
91
98
|
rank: number;
|
|
92
99
|
}>;
|
|
93
100
|
/**
|