@psiclawops/hypermem 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +12 -3
- package/README.md +30 -6
- package/bin/hypermem-status.mjs +166 -0
- package/dist/background-indexer.d.ts +132 -0
- package/dist/background-indexer.d.ts.map +1 -0
- package/dist/background-indexer.js +1044 -0
- package/dist/cache.d.ts +110 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +495 -0
- package/dist/compaction-fence.d.ts +89 -0
- package/dist/compaction-fence.d.ts.map +1 -0
- package/dist/compaction-fence.js +153 -0
- package/dist/compositor.d.ts +226 -0
- package/dist/compositor.d.ts.map +1 -0
- package/dist/compositor.js +2558 -0
- package/dist/content-type-classifier.d.ts +41 -0
- package/dist/content-type-classifier.d.ts.map +1 -0
- package/dist/content-type-classifier.js +181 -0
- package/dist/cross-agent.d.ts +62 -0
- package/dist/cross-agent.d.ts.map +1 -0
- package/dist/cross-agent.js +259 -0
- package/dist/db.d.ts +131 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +402 -0
- package/dist/desired-state-store.d.ts +100 -0
- package/dist/desired-state-store.d.ts.map +1 -0
- package/dist/desired-state-store.js +222 -0
- package/dist/doc-chunk-store.d.ts +140 -0
- package/dist/doc-chunk-store.d.ts.map +1 -0
- package/dist/doc-chunk-store.js +391 -0
- package/dist/doc-chunker.d.ts +99 -0
- package/dist/doc-chunker.d.ts.map +1 -0
- package/dist/doc-chunker.js +324 -0
- package/dist/dreaming-promoter.d.ts +86 -0
- package/dist/dreaming-promoter.d.ts.map +1 -0
- package/dist/dreaming-promoter.js +381 -0
- package/dist/episode-store.d.ts +49 -0
- package/dist/episode-store.d.ts.map +1 -0
- package/dist/episode-store.js +135 -0
- package/dist/fact-store.d.ts +75 -0
- package/dist/fact-store.d.ts.map +1 -0
- package/dist/fact-store.js +236 -0
- package/dist/fleet-store.d.ts +144 -0
- package/dist/fleet-store.d.ts.map +1 -0
- package/dist/fleet-store.js +276 -0
- package/dist/fos-mod.d.ts +178 -0
- package/dist/fos-mod.d.ts.map +1 -0
- package/dist/fos-mod.js +416 -0
- package/dist/hybrid-retrieval.d.ts +64 -0
- package/dist/hybrid-retrieval.d.ts.map +1 -0
- package/dist/hybrid-retrieval.js +344 -0
- package/dist/image-eviction.d.ts +49 -0
- package/dist/image-eviction.d.ts.map +1 -0
- package/dist/image-eviction.js +251 -0
- package/dist/index.d.ts +650 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1072 -0
- package/dist/keystone-scorer.d.ts +51 -0
- package/dist/keystone-scorer.d.ts.map +1 -0
- package/dist/keystone-scorer.js +52 -0
- package/dist/knowledge-graph.d.ts +110 -0
- package/dist/knowledge-graph.d.ts.map +1 -0
- package/dist/knowledge-graph.js +305 -0
- package/dist/knowledge-lint.d.ts +29 -0
- package/dist/knowledge-lint.d.ts.map +1 -0
- package/dist/knowledge-lint.js +116 -0
- package/dist/knowledge-store.d.ts +72 -0
- package/dist/knowledge-store.d.ts.map +1 -0
- package/dist/knowledge-store.js +247 -0
- package/dist/library-schema.d.ts +22 -0
- package/dist/library-schema.d.ts.map +1 -0
- package/dist/library-schema.js +1038 -0
- package/dist/message-store.d.ts +89 -0
- package/dist/message-store.d.ts.map +1 -0
- package/dist/message-store.js +323 -0
- package/dist/metrics-dashboard.d.ts +114 -0
- package/dist/metrics-dashboard.d.ts.map +1 -0
- package/dist/metrics-dashboard.js +260 -0
- package/dist/obsidian-exporter.d.ts +57 -0
- package/dist/obsidian-exporter.d.ts.map +1 -0
- package/dist/obsidian-exporter.js +274 -0
- package/dist/obsidian-watcher.d.ts +147 -0
- package/dist/obsidian-watcher.d.ts.map +1 -0
- package/dist/obsidian-watcher.js +403 -0
- package/dist/open-domain.d.ts +46 -0
- package/dist/open-domain.d.ts.map +1 -0
- package/dist/open-domain.js +125 -0
- package/dist/preference-store.d.ts +54 -0
- package/dist/preference-store.d.ts.map +1 -0
- package/dist/preference-store.js +109 -0
- package/dist/preservation-gate.d.ts +82 -0
- package/dist/preservation-gate.d.ts.map +1 -0
- package/dist/preservation-gate.js +150 -0
- package/dist/proactive-pass.d.ts +63 -0
- package/dist/proactive-pass.d.ts.map +1 -0
- package/dist/proactive-pass.js +239 -0
- package/dist/profiles.d.ts +44 -0
- package/dist/profiles.d.ts.map +1 -0
- package/dist/profiles.js +227 -0
- package/dist/provider-translator.d.ts +50 -0
- package/dist/provider-translator.d.ts.map +1 -0
- package/dist/provider-translator.js +403 -0
- package/dist/rate-limiter.d.ts +76 -0
- package/dist/rate-limiter.d.ts.map +1 -0
- package/dist/rate-limiter.js +179 -0
- package/dist/repair-tool-pairs.d.ts +38 -0
- package/dist/repair-tool-pairs.d.ts.map +1 -0
- package/dist/repair-tool-pairs.js +138 -0
- package/dist/retrieval-policy.d.ts +51 -0
- package/dist/retrieval-policy.d.ts.map +1 -0
- package/dist/retrieval-policy.js +77 -0
- package/dist/schema.d.ts +15 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +229 -0
- package/dist/secret-scanner.d.ts +51 -0
- package/dist/secret-scanner.d.ts.map +1 -0
- package/dist/secret-scanner.js +248 -0
- package/dist/seed.d.ts +108 -0
- package/dist/seed.d.ts.map +1 -0
- package/dist/seed.js +177 -0
- package/dist/session-flusher.d.ts +53 -0
- package/dist/session-flusher.d.ts.map +1 -0
- package/dist/session-flusher.js +69 -0
- package/dist/session-topic-map.d.ts +41 -0
- package/dist/session-topic-map.d.ts.map +1 -0
- package/dist/session-topic-map.js +77 -0
- package/dist/spawn-context.d.ts +54 -0
- package/dist/spawn-context.d.ts.map +1 -0
- package/dist/spawn-context.js +159 -0
- package/dist/system-store.d.ts +73 -0
- package/dist/system-store.d.ts.map +1 -0
- package/dist/system-store.js +182 -0
- package/dist/temporal-store.d.ts +80 -0
- package/dist/temporal-store.d.ts.map +1 -0
- package/dist/temporal-store.js +149 -0
- package/dist/topic-detector.d.ts +35 -0
- package/dist/topic-detector.d.ts.map +1 -0
- package/dist/topic-detector.js +249 -0
- package/dist/topic-store.d.ts +45 -0
- package/dist/topic-store.d.ts.map +1 -0
- package/dist/topic-store.js +136 -0
- package/dist/topic-synthesizer.d.ts +51 -0
- package/dist/topic-synthesizer.d.ts.map +1 -0
- package/dist/topic-synthesizer.js +315 -0
- package/dist/trigger-registry.d.ts +63 -0
- package/dist/trigger-registry.d.ts.map +1 -0
- package/dist/trigger-registry.js +163 -0
- package/dist/types.d.ts +537 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/dist/vector-store.d.ts +170 -0
- package/dist/vector-store.d.ts.map +1 -0
- package/dist/vector-store.js +677 -0
- package/dist/version.d.ts +34 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +34 -0
- package/dist/wiki-page-emitter.d.ts +65 -0
- package/dist/wiki-page-emitter.d.ts.map +1 -0
- package/dist/wiki-page-emitter.js +258 -0
- package/dist/work-store.d.ts +112 -0
- package/dist/work-store.d.ts.map +1 -0
- package/dist/work-store.js +273 -0
- package/package.json +4 -1
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hypermem Compaction Fence
|
|
3
|
+
*
|
|
4
|
+
* Protects the recent conversation tail from being compacted.
|
|
5
|
+
*
|
|
6
|
+
* The fence is a per-conversation high-water mark (message ID) that divides
|
|
7
|
+
* the message timeline into two zones:
|
|
8
|
+
*
|
|
9
|
+
* - ABOVE the fence: messages the LLM can currently see (recent tail).
|
|
10
|
+
* These are off-limits to compaction.
|
|
11
|
+
* - BELOW the fence: older messages eligible for compaction/summarization.
|
|
12
|
+
*
|
|
13
|
+
* The compositor updates the fence every compose cycle by recording the
|
|
14
|
+
* oldest message ID that was included in the composed context. This means
|
|
15
|
+
* the fence automatically advances as conversations grow.
|
|
16
|
+
*
|
|
17
|
+
* Safety defaults:
|
|
18
|
+
* - No fence row = no compaction allowed (explicit opt-in)
|
|
19
|
+
* - Fence never moves backward (monotone progress)
|
|
20
|
+
* - Fence update is idempotent (same value = no-op)
|
|
21
|
+
*
|
|
22
|
+
* Inspired by the continuity model in openclaw-memory-libravdb, which
|
|
23
|
+
* formally proves that compaction must never touch the recent tail.
|
|
24
|
+
*/
|
|
25
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
26
|
+
export interface CompactionFence {
|
|
27
|
+
conversationId: number;
|
|
28
|
+
/** Message ID of the oldest message currently visible to the LLM */
|
|
29
|
+
fenceMessageId: number;
|
|
30
|
+
/** Timestamp of the last fence update */
|
|
31
|
+
updatedAt: string;
|
|
32
|
+
}
|
|
33
|
+
export interface CompactionEligibility {
|
|
34
|
+
conversationId: number;
|
|
35
|
+
/** Total messages below the fence (eligible for compaction) */
|
|
36
|
+
eligibleCount: number;
|
|
37
|
+
/** Message ID of the oldest eligible message */
|
|
38
|
+
oldestEligibleId: number | null;
|
|
39
|
+
/** Message ID of the newest eligible message (just below fence) */
|
|
40
|
+
newestEligibleId: number | null;
|
|
41
|
+
/** The fence itself */
|
|
42
|
+
fence: CompactionFence | null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Add the compaction_fences table to an existing messages.db.
|
|
46
|
+
* Idempotent — safe to call on every startup.
|
|
47
|
+
*/
|
|
48
|
+
export declare function ensureCompactionFenceSchema(db: DatabaseSync): void;
|
|
49
|
+
/**
|
|
50
|
+
* Update the compaction fence for a conversation.
|
|
51
|
+
*
|
|
52
|
+
* Called by the compositor after assembling context, passing the ID of
|
|
53
|
+
* the oldest message that was included in the composed history.
|
|
54
|
+
*
|
|
55
|
+
* The fence only moves forward (monotone progress). If the new fence
|
|
56
|
+
* is lower than the existing one, the update is silently ignored.
|
|
57
|
+
* This prevents a short compose window from accidentally exposing
|
|
58
|
+
* already-compacted messages.
|
|
59
|
+
*/
|
|
60
|
+
export declare function updateCompactionFence(db: DatabaseSync, conversationId: number, oldestVisibleMessageId: number): void;
|
|
61
|
+
/**
|
|
62
|
+
* Get the current compaction fence for a conversation.
|
|
63
|
+
* Returns null if no fence has been set (meaning: no compaction allowed).
|
|
64
|
+
*/
|
|
65
|
+
export declare function getCompactionFence(db: DatabaseSync, conversationId: number): CompactionFence | null;
|
|
66
|
+
/**
|
|
67
|
+
* Query compaction eligibility for a conversation.
|
|
68
|
+
*
|
|
69
|
+
* Returns the count and range of messages that are below the fence
|
|
70
|
+
* and therefore eligible for compaction. If no fence exists, returns
|
|
71
|
+
* zero eligible (safe default: no fence = no compaction).
|
|
72
|
+
*
|
|
73
|
+
* Excludes messages that are already covered by a summary
|
|
74
|
+
* (via the summary_messages junction table).
|
|
75
|
+
*/
|
|
76
|
+
export declare function getCompactionEligibility(db: DatabaseSync, conversationId: number): CompactionEligibility;
|
|
77
|
+
/**
|
|
78
|
+
* Get messages eligible for compaction (below the fence, not yet summarized).
|
|
79
|
+
*
|
|
80
|
+
* Returns messages in chronological order, ready for clustering.
|
|
81
|
+
* Respects the fence boundary and excludes already-summarized messages.
|
|
82
|
+
*/
|
|
83
|
+
export declare function getCompactableMessages(db: DatabaseSync, conversationId: number, limit?: number): Array<{
|
|
84
|
+
id: number;
|
|
85
|
+
role: string;
|
|
86
|
+
textContent: string;
|
|
87
|
+
createdAt: string;
|
|
88
|
+
}>;
|
|
89
|
+
//# sourceMappingURL=compaction-fence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compaction-fence.d.ts","sourceRoot":"","sources":["../src/compaction-fence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIhD,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,cAAc,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,mEAAmE;IACnE,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,uBAAuB;IACvB,KAAK,EAAE,eAAe,GAAG,IAAI,CAAC;CAC/B;AAID;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,EAAE,EAAE,YAAY,GAAG,IAAI,CAQlE;AAID;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CACnC,EAAE,EAAE,YAAY,EAChB,cAAc,EAAE,MAAM,EACtB,sBAAsB,EAAE,MAAM,GAC7B,IAAI,CAqBN;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,YAAY,EAChB,cAAc,EAAE,MAAM,GACrB,eAAe,GAAG,IAAI,CAYxB;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACtC,EAAE,EAAE,YAAY,EAChB,cAAc,EAAE,MAAM,GACrB,qBAAqB,CAwCvB;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,YAAY,EAChB,cAAc,EAAE,MAAM,EACtB,KAAK,GAAE,MAAY,GAClB,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC,CA8B7E"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hypermem Compaction Fence
|
|
3
|
+
*
|
|
4
|
+
* Protects the recent conversation tail from being compacted.
|
|
5
|
+
*
|
|
6
|
+
* The fence is a per-conversation high-water mark (message ID) that divides
|
|
7
|
+
* the message timeline into two zones:
|
|
8
|
+
*
|
|
9
|
+
* - ABOVE the fence: messages the LLM can currently see (recent tail).
|
|
10
|
+
* These are off-limits to compaction.
|
|
11
|
+
* - BELOW the fence: older messages eligible for compaction/summarization.
|
|
12
|
+
*
|
|
13
|
+
* The compositor updates the fence every compose cycle by recording the
|
|
14
|
+
* oldest message ID that was included in the composed context. This means
|
|
15
|
+
* the fence automatically advances as conversations grow.
|
|
16
|
+
*
|
|
17
|
+
* Safety defaults:
|
|
18
|
+
* - No fence row = no compaction allowed (explicit opt-in)
|
|
19
|
+
* - Fence never moves backward (monotone progress)
|
|
20
|
+
* - Fence update is idempotent (same value = no-op)
|
|
21
|
+
*
|
|
22
|
+
* Inspired by the continuity model in openclaw-memory-libravdb, which
|
|
23
|
+
* formally proves that compaction must never touch the recent tail.
|
|
24
|
+
*/
|
|
25
|
+
// ─── Schema ─────────────────────────────────────────────────────
|
|
26
|
+
/**
|
|
27
|
+
* Add the compaction_fences table to an existing messages.db.
|
|
28
|
+
* Idempotent — safe to call on every startup.
|
|
29
|
+
*/
|
|
30
|
+
export function ensureCompactionFenceSchema(db) {
|
|
31
|
+
db.exec(`
|
|
32
|
+
CREATE TABLE IF NOT EXISTS compaction_fences (
|
|
33
|
+
conversation_id INTEGER PRIMARY KEY REFERENCES conversations(id),
|
|
34
|
+
fence_message_id INTEGER NOT NULL,
|
|
35
|
+
updated_at TEXT NOT NULL
|
|
36
|
+
)
|
|
37
|
+
`);
|
|
38
|
+
}
|
|
39
|
+
// ─── Fence Operations ───────────────────────────────────────────
|
|
40
|
+
/**
|
|
41
|
+
* Update the compaction fence for a conversation.
|
|
42
|
+
*
|
|
43
|
+
* Called by the compositor after assembling context, passing the ID of
|
|
44
|
+
* the oldest message that was included in the composed history.
|
|
45
|
+
*
|
|
46
|
+
* The fence only moves forward (monotone progress). If the new fence
|
|
47
|
+
* is lower than the existing one, the update is silently ignored.
|
|
48
|
+
* This prevents a short compose window from accidentally exposing
|
|
49
|
+
* already-compacted messages.
|
|
50
|
+
*/
|
|
51
|
+
export function updateCompactionFence(db, conversationId, oldestVisibleMessageId) {
|
|
52
|
+
const now = new Date().toISOString();
|
|
53
|
+
const existing = db.prepare('SELECT fence_message_id FROM compaction_fences WHERE conversation_id = ?').get(conversationId);
|
|
54
|
+
if (!existing) {
|
|
55
|
+
// First fence for this conversation
|
|
56
|
+
db.prepare('INSERT INTO compaction_fences (conversation_id, fence_message_id, updated_at) VALUES (?, ?, ?)').run(conversationId, oldestVisibleMessageId, now);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
// Monotone progress: fence only moves forward
|
|
60
|
+
if (oldestVisibleMessageId > existing.fence_message_id) {
|
|
61
|
+
db.prepare('UPDATE compaction_fences SET fence_message_id = ?, updated_at = ? WHERE conversation_id = ?').run(oldestVisibleMessageId, now, conversationId);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Get the current compaction fence for a conversation.
|
|
66
|
+
* Returns null if no fence has been set (meaning: no compaction allowed).
|
|
67
|
+
*/
|
|
68
|
+
export function getCompactionFence(db, conversationId) {
|
|
69
|
+
const row = db.prepare('SELECT conversation_id, fence_message_id, updated_at FROM compaction_fences WHERE conversation_id = ?').get(conversationId);
|
|
70
|
+
if (!row)
|
|
71
|
+
return null;
|
|
72
|
+
return {
|
|
73
|
+
conversationId: row.conversation_id,
|
|
74
|
+
fenceMessageId: row.fence_message_id,
|
|
75
|
+
updatedAt: row.updated_at,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Query compaction eligibility for a conversation.
|
|
80
|
+
*
|
|
81
|
+
* Returns the count and range of messages that are below the fence
|
|
82
|
+
* and therefore eligible for compaction. If no fence exists, returns
|
|
83
|
+
* zero eligible (safe default: no fence = no compaction).
|
|
84
|
+
*
|
|
85
|
+
* Excludes messages that are already covered by a summary
|
|
86
|
+
* (via the summary_messages junction table).
|
|
87
|
+
*/
|
|
88
|
+
export function getCompactionEligibility(db, conversationId) {
|
|
89
|
+
const fence = getCompactionFence(db, conversationId);
|
|
90
|
+
if (!fence) {
|
|
91
|
+
return {
|
|
92
|
+
conversationId,
|
|
93
|
+
eligibleCount: 0,
|
|
94
|
+
oldestEligibleId: null,
|
|
95
|
+
newestEligibleId: null,
|
|
96
|
+
fence: null,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
// Messages below the fence that haven't been summarized yet
|
|
100
|
+
const stats = db.prepare(`
|
|
101
|
+
SELECT
|
|
102
|
+
COUNT(*) AS cnt,
|
|
103
|
+
MIN(m.id) AS oldest_id,
|
|
104
|
+
MAX(m.id) AS newest_id
|
|
105
|
+
FROM messages m
|
|
106
|
+
WHERE m.conversation_id = ?
|
|
107
|
+
AND m.id < ?
|
|
108
|
+
AND m.id NOT IN (
|
|
109
|
+
SELECT sm.message_id FROM summary_messages sm
|
|
110
|
+
JOIN summaries s ON sm.summary_id = s.id
|
|
111
|
+
WHERE s.conversation_id = ?
|
|
112
|
+
)
|
|
113
|
+
`).get(conversationId, fence.fenceMessageId, conversationId);
|
|
114
|
+
return {
|
|
115
|
+
conversationId,
|
|
116
|
+
eligibleCount: stats.cnt,
|
|
117
|
+
oldestEligibleId: stats.oldest_id,
|
|
118
|
+
newestEligibleId: stats.newest_id,
|
|
119
|
+
fence,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Get messages eligible for compaction (below the fence, not yet summarized).
|
|
124
|
+
*
|
|
125
|
+
* Returns messages in chronological order, ready for clustering.
|
|
126
|
+
* Respects the fence boundary and excludes already-summarized messages.
|
|
127
|
+
*/
|
|
128
|
+
export function getCompactableMessages(db, conversationId, limit = 100) {
|
|
129
|
+
const fence = getCompactionFence(db, conversationId);
|
|
130
|
+
if (!fence)
|
|
131
|
+
return [];
|
|
132
|
+
const rows = db.prepare(`
|
|
133
|
+
SELECT m.id, m.role, m.text_content, m.created_at
|
|
134
|
+
FROM messages m
|
|
135
|
+
WHERE m.conversation_id = ?
|
|
136
|
+
AND m.id < ?
|
|
137
|
+
AND m.text_content IS NOT NULL
|
|
138
|
+
AND m.id NOT IN (
|
|
139
|
+
SELECT sm.message_id FROM summary_messages sm
|
|
140
|
+
JOIN summaries s ON sm.summary_id = s.id
|
|
141
|
+
WHERE s.conversation_id = ?
|
|
142
|
+
)
|
|
143
|
+
ORDER BY m.id ASC
|
|
144
|
+
LIMIT ?
|
|
145
|
+
`).all(conversationId, fence.fenceMessageId, conversationId, limit);
|
|
146
|
+
return rows.map(r => ({
|
|
147
|
+
id: r.id,
|
|
148
|
+
role: r.role,
|
|
149
|
+
textContent: r.text_content,
|
|
150
|
+
createdAt: r.created_at,
|
|
151
|
+
}));
|
|
152
|
+
}
|
|
153
|
+
//# sourceMappingURL=compaction-fence.js.map
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* hypermem Compositor
|
|
3
|
+
*
|
|
4
|
+
* Assembles context for LLM calls by orchestrating all four memory layers:
|
|
5
|
+
* L1 Redis — hot session working memory (system, identity, recent msgs)
|
|
6
|
+
* L2 Messages — conversation history from messages.db
|
|
7
|
+
* L3 Vectors — semantic search across all indexed content
|
|
8
|
+
* L4 Library — structured knowledge (facts, preferences, knowledge, episodes)
|
|
9
|
+
*
|
|
10
|
+
* Token-budgeted: never exceeds the budget, prioritizes by configured order.
|
|
11
|
+
* Provider-neutral internally, translates at the output boundary.
|
|
12
|
+
*/
|
|
13
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
14
|
+
import type { ComposeRequest, ComposeResult, NeutralMessage, CompositorConfig } from './types.js';
|
|
15
|
+
import { CollectionTrigger } from './trigger-registry.js';
|
|
16
|
+
import { CacheLayer } from './cache.js';
|
|
17
|
+
type AnyCache = CacheLayer;
|
|
18
|
+
import { VectorStore } from './vector-store.js';
|
|
19
|
+
import { type OrgRegistry } from './cross-agent.js';
|
|
20
|
+
export { CollectionTrigger, DEFAULT_TRIGGERS, matchTriggers } from './trigger-registry.js';
|
|
21
|
+
export { getTurnAge, applyToolGradient, appendToolSummary, truncateWithHeadTail, applyTierPayloadCap, evictLargeToolResults };
|
|
22
|
+
/**
|
|
23
|
+
* Public reshape helper: apply tool gradient then trim to fit within a token budget.
|
|
24
|
+
*
|
|
25
|
+
* Used by the plugin's budget-downshift pass to pre-process a Redis history window
|
|
26
|
+
* after a model switch to a smaller context window, before the full compose pipeline
|
|
27
|
+
* runs. Trims from oldest to newest until estimated token cost fits within
|
|
28
|
+
* tokenBudget * 0.65 (using the standard char/4 heuristic).
|
|
29
|
+
*
|
|
30
|
+
* @param messages NeutralMessage array from the Redis hot window
|
|
31
|
+
* @param tokenBudget Effective token budget for this session
|
|
32
|
+
* @returns Trimmed message array ready for setWindow()
|
|
33
|
+
*/
|
|
34
|
+
export declare function applyToolGradientToWindow(messages: NeutralMessage[], tokenBudget: number, totalWindowTokens?: number): NeutralMessage[];
|
|
35
|
+
/**
|
|
36
|
+
* Canonical history must remain lossless for tool turns.
|
|
37
|
+
*
|
|
38
|
+
* If a window contains any structured tool calls or tool results, the caller
|
|
39
|
+
* should treat applyToolGradientToWindow() as a view-only transform for the
|
|
40
|
+
* current compose pass and avoid writing the reshaped messages back into the
|
|
41
|
+
* canonical cache/history store.
|
|
42
|
+
*/
|
|
43
|
+
export declare function canPersistReshapedHistory(messages: NeutralMessage[]): boolean;
|
|
44
|
+
declare function truncateWithHeadTail(content: string, maxChars: number, maxTailChars?: number): string;
|
|
45
|
+
declare function appendToolSummary(textContent: string | null, summary: string): string;
|
|
46
|
+
declare function getTurnAge(messages: NeutralMessage[], index: number): number;
|
|
47
|
+
declare function applyTierPayloadCap(msg: NeutralMessage, perResultCap: number, perTurnCap?: number, usedSoFar?: number, maxTailChars?: number): {
|
|
48
|
+
msg: NeutralMessage;
|
|
49
|
+
usedChars: number;
|
|
50
|
+
};
|
|
51
|
+
declare function evictLargeToolResults<T extends NeutralMessage>(messages: T[]): T[];
|
|
52
|
+
/**
|
|
53
|
+
* Apply gradient tool treatment to a message array.
|
|
54
|
+
*
|
|
55
|
+
* Tiers are based on turn age, where turn age is the number of newer user
|
|
56
|
+
* messages after the current message.
|
|
57
|
+
*/
|
|
58
|
+
declare function applyToolGradient<T extends NeutralMessage>(messages: T[], opts?: {
|
|
59
|
+
totalWindowTokens?: number;
|
|
60
|
+
}): T[];
|
|
61
|
+
export interface CompositorDeps {
|
|
62
|
+
cache: AnyCache;
|
|
63
|
+
vectorStore?: VectorStore | null;
|
|
64
|
+
libraryDb?: DatabaseSync | null;
|
|
65
|
+
/** Custom trigger registry; defaults to DEFAULT_TRIGGERS if not provided */
|
|
66
|
+
triggerRegistry?: CollectionTrigger[];
|
|
67
|
+
}
|
|
68
|
+
export declare class Compositor {
|
|
69
|
+
private readonly config;
|
|
70
|
+
private readonly cache;
|
|
71
|
+
private vectorStore;
|
|
72
|
+
private readonly libraryDb;
|
|
73
|
+
private readonly triggerRegistry;
|
|
74
|
+
/** Cached org registry loaded from fleet_agents at construction time. */
|
|
75
|
+
private _orgRegistry;
|
|
76
|
+
constructor(deps: CompositorDeps, config?: Partial<CompositorConfig>);
|
|
77
|
+
/**
|
|
78
|
+
* Set or replace the vector store after construction.
|
|
79
|
+
* Called by hypermem.create() once sqlite-vec is confirmed available.
|
|
80
|
+
*/
|
|
81
|
+
setVectorStore(vs: VectorStore): void;
|
|
82
|
+
/**
|
|
83
|
+
* Hot-reload the org registry from the fleet_agents table.
|
|
84
|
+
* Call after fleet membership changes (new agent, org restructure)
|
|
85
|
+
* to pick up the latest without a full restart.
|
|
86
|
+
* Falls back to the current cached registry if the DB is unavailable.
|
|
87
|
+
*/
|
|
88
|
+
refreshOrgRegistry(): OrgRegistry;
|
|
89
|
+
/**
|
|
90
|
+
* Return the currently cached org registry.
|
|
91
|
+
*/
|
|
92
|
+
get orgRegistry(): OrgRegistry;
|
|
93
|
+
/**
|
|
94
|
+
* Compose a complete message array for sending to an LLM.
|
|
95
|
+
*
|
|
96
|
+
* Orchestrates all four memory layers:
|
|
97
|
+
* 1. System prompt + identity (never truncated)
|
|
98
|
+
* 2. Conversation history (L1 Redis → L2 messages.db)
|
|
99
|
+
* 3. Active facts from library (L4)
|
|
100
|
+
* 4. Knowledge entries relevant to conversation (L4)
|
|
101
|
+
* 5. User preferences (L4)
|
|
102
|
+
* 6. Semantic recall via vector search (L3)
|
|
103
|
+
* 7. Cross-session context (L2)
|
|
104
|
+
*
|
|
105
|
+
* Each slot respects the remaining token budget.
|
|
106
|
+
*/
|
|
107
|
+
compose(request: ComposeRequest, db: DatabaseSync, libraryDb?: DatabaseSync): Promise<ComposeResult>;
|
|
108
|
+
/**
|
|
109
|
+
* Warm a session from SQLite into Redis.
|
|
110
|
+
* Called on session start or Redis cache miss.
|
|
111
|
+
*/
|
|
112
|
+
warmSession(agentId: string, sessionKey: string, db: DatabaseSync, opts?: {
|
|
113
|
+
systemPrompt?: string;
|
|
114
|
+
identity?: string;
|
|
115
|
+
libraryDb?: DatabaseSync;
|
|
116
|
+
/** Model string for budget resolution. If omitted, falls back to defaultTokenBudget. */
|
|
117
|
+
model?: string;
|
|
118
|
+
}): Promise<void>;
|
|
119
|
+
refreshRedisGradient(agentId: string, sessionKey: string, db: DatabaseSync, tokenBudget?: number): Promise<void>;
|
|
120
|
+
/**
|
|
121
|
+
* Get slot content: try Redis first, fall back to SQLite.
|
|
122
|
+
*/
|
|
123
|
+
private getSlotContent;
|
|
124
|
+
/**
|
|
125
|
+
* Get conversation history: try Redis first, fall back to SQLite.
|
|
126
|
+
*
|
|
127
|
+
* When topicId is provided (P3.4), the SQLite path filters to messages
|
|
128
|
+
* matching that topic OR with topic_id IS NULL (Option B transition safety).
|
|
129
|
+
* The Redis path is unaffected — Redis doesn't index by topic, so topic
|
|
130
|
+
* filtering only applies to the SQLite fallback.
|
|
131
|
+
*/
|
|
132
|
+
private getHistory;
|
|
133
|
+
/**
|
|
134
|
+
* Build facts content from library DB.
|
|
135
|
+
*/
|
|
136
|
+
/**
|
|
137
|
+
* Build facts content from library DB.
|
|
138
|
+
* Applies filterByScope (W1) to enforce retrieval access control.
|
|
139
|
+
* Returns [content, factCount, scopeFilteredCount] or null if DB unavailable.
|
|
140
|
+
*/
|
|
141
|
+
private buildFactsFromDb;
|
|
142
|
+
/**
|
|
143
|
+
* Build knowledge content from library DB.
|
|
144
|
+
* Prioritizes high-confidence, non-superseded entries.
|
|
145
|
+
*/
|
|
146
|
+
private buildKnowledgeFromDb;
|
|
147
|
+
/**
|
|
148
|
+
* Build wiki page context for the active topic.
|
|
149
|
+
* Queries the knowledge table for a synthesized topic page and returns it
|
|
150
|
+
* wrapped with a header. Capped at 600 tokens.
|
|
151
|
+
*/
|
|
152
|
+
private buildWikiPageContext;
|
|
153
|
+
/**
|
|
154
|
+
* Build preferences content from library DB.
|
|
155
|
+
* Shows user/operator preferences relevant to this agent.
|
|
156
|
+
*/
|
|
157
|
+
private buildPreferencesFromDb;
|
|
158
|
+
/**
|
|
159
|
+
* Build semantic recall content using hybrid FTS5+KNN retrieval.
|
|
160
|
+
*
|
|
161
|
+
* Uses Reciprocal Rank Fusion to merge keyword and vector results.
|
|
162
|
+
* Gracefully degrades: FTS5-only when no vector store, KNN-only
|
|
163
|
+
* when FTS query is empty (all stop words), both when available.
|
|
164
|
+
*
|
|
165
|
+
* @param precomputedEmbedding — optional pre-computed embedding for the query.
|
|
166
|
+
* When provided, the Ollama call inside VectorStore.search() is skipped.
|
|
167
|
+
*/
|
|
168
|
+
private buildSemanticRecall;
|
|
169
|
+
/**
|
|
170
|
+
* Format a hybrid search result for injection into context.
|
|
171
|
+
* Shows retrieval source(s) and relevance score.
|
|
172
|
+
*/
|
|
173
|
+
private formatHybridResult;
|
|
174
|
+
/**
|
|
175
|
+
* Format a vector-only search result (legacy fallback).
|
|
176
|
+
*/
|
|
177
|
+
private formatVectorResult;
|
|
178
|
+
/**
|
|
179
|
+
* Build cross-session context by finding recent activity
|
|
180
|
+
* in other sessions for this agent.
|
|
181
|
+
*/
|
|
182
|
+
private buildCrossSessionContext;
|
|
183
|
+
/**
|
|
184
|
+
* Extract the last user message text from the composed messages.
|
|
185
|
+
*/
|
|
186
|
+
private getLastUserMessage;
|
|
187
|
+
/**
|
|
188
|
+
* Truncate text to approximately fit within a token budget.
|
|
189
|
+
* Truncates at line boundaries when possible.
|
|
190
|
+
*/
|
|
191
|
+
private truncateToTokens;
|
|
192
|
+
/**
|
|
193
|
+
* Query and score keystone candidates from before the current history window.
|
|
194
|
+
*
|
|
195
|
+
* Trims the oldest messages from includedHistory to free a keystone budget,
|
|
196
|
+
* then queries the DB for older messages scored by episode significance,
|
|
197
|
+
* FTS5 relevance, and recency.
|
|
198
|
+
*
|
|
199
|
+
* Returns null if keystones cannot be injected (no cutoff ID found,
|
|
200
|
+
* no candidates, or all errors).
|
|
201
|
+
*/
|
|
202
|
+
private buildKeystones;
|
|
203
|
+
/**
|
|
204
|
+
* Pull high-signal messages from OTHER topics in this session when their
|
|
205
|
+
* content is semantically relevant to the current active topic.
|
|
206
|
+
*
|
|
207
|
+
* Heuristic-only: no model calls. Token overlap between the current topic
|
|
208
|
+
* name + last 3 user messages and candidate message content.
|
|
209
|
+
*
|
|
210
|
+
* @param agentId - The agent's ID
|
|
211
|
+
* @param sessionKey - Current session key
|
|
212
|
+
* @param activeTopic - The current active topic (id + name)
|
|
213
|
+
* @param currentMessages - Recently included history messages for query extraction
|
|
214
|
+
* @param db - The messages database
|
|
215
|
+
* @param maxKeystones - Max cross-topic keystones to return (default 3)
|
|
216
|
+
* @returns Scored keystones sorted by score DESC, deduplicated by message id
|
|
217
|
+
*/
|
|
218
|
+
private getKeystonesByTopic;
|
|
219
|
+
/**
|
|
220
|
+
* Extract lowercase key terms from a topic name and the last 3 user messages.
|
|
221
|
+
* Terms are: tokens with ≥4 characters (skip short stop words).
|
|
222
|
+
* Returns a Set for O(1) lookup.
|
|
223
|
+
*/
|
|
224
|
+
private extractQueryTerms;
|
|
225
|
+
}
|
|
226
|
+
//# sourceMappingURL=compositor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compositor.d.ts","sourceRoot":"","sources":["../src/compositor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EAGb,cAAc,EAGd,gBAAgB,EAGjB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,iBAAiB,EAMlB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,KAAK,QAAQ,GAAG,UAAU,CAAC;AAI3B,OAAO,EAAE,WAAW,EAA2B,MAAM,mBAAmB,CAAC;AAKzE,OAAO,EAA8C,KAAK,WAAW,EAAE,MAAM,kBAAkB,CAAC;AA6KhG,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAI3F,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,CAAC;AAgD9H;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,cAAc,EAAE,EAC1B,WAAW,EAAE,MAAM,EACnB,iBAAiB,CAAC,EAAE,MAAM,GACzB,cAAc,EAAE,CAYlB;AAED;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,OAAO,CAE7E;AA4FD,iBAAS,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,SAA+B,GAAG,MAAM,CAKpH;AAqJD,iBAAS,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAI9E;AAED,iBAAS,UAAU,CAAC,QAAQ,EAAE,cAAc,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CASrE;AA6HD,iBAAS,mBAAmB,CAAC,GAAG,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,EAAE,SAAS,GAAE,MAAU,EAAE,YAAY,SAA+B,GAAG;IAAE,GAAG,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CA4C3M;AAYD,iBAAS,qBAAqB,CAAC,CAAC,SAAS,cAAc,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAkB3E;AAED;;;;;GAKG;AACH,iBAAS,iBAAiB,CAAC,CAAC,SAAS,cAAc,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,CAAC,EAAE,CAgD9G;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,QAAQ,CAAC;IAChB,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IACjC,SAAS,CAAC,EAAE,YAAY,GAAG,IAAI,CAAC;IAChC,4EAA4E;IAC5E,eAAe,CAAC,EAAE,iBAAiB,EAAE,CAAC;CACvC;AAKD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAW;IACjC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsB;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsB;IACtD,yEAAyE;IACzE,OAAO,CAAC,YAAY,CAAc;gBAGhC,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAiBpC;;;OAGG;IACH,cAAc,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAIrC;;;;;OAKG;IACH,kBAAkB,IAAI,WAAW;IAOjC;;OAEG;IACH,IAAI,WAAW,IAAI,WAAW,CAE7B;IAED;;;;;;;;;;;;;OAaG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC;IAylC1G;;;OAGG;IACG,WAAW,CACf,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,EAAE,EAAE,YAAY,EAChB,IAAI,CAAC,EAAE;QACL,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,YAAY,CAAC;QACzB,wFAAwF;QACxF,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,GACA,OAAO,CAAC,IAAI,CAAC;IA8DV,oBAAoB,CACxB,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,EAAE,EAAE,YAAY,EAChB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC;IA4ChB;;OAEG;YACW,cAAc;IAsB5B;;;;;;;OAOG;YACW,UAAU;IAyBxB;;OAEG;IACH;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IA8DxB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAyC5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAS5B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IA4C9B;;;;;;;;;OASG;YACW,mBAAmB;IAqGjC;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAiB1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAkB1B;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IA6ChC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAkBxB;;;;;;;;;OASG;YACW,cAAc;IAqN5B;;;;;;;;;;;;;;OAcG;YACW,mBAAmB;IAoHjC;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;CA0B1B"}
|