gnosys 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +1387 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +3753 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2267 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/archive.d.ts +95 -0
- package/dist/lib/archive.d.ts.map +1 -0
- package/dist/lib/archive.js +311 -0
- package/dist/lib/archive.js.map +1 -0
- package/dist/lib/ask.d.ts +77 -0
- package/dist/lib/ask.d.ts.map +1 -0
- package/dist/lib/ask.js +316 -0
- package/dist/lib/ask.js.map +1 -0
- package/dist/lib/audit.d.ts +47 -0
- package/dist/lib/audit.d.ts.map +1 -0
- package/dist/lib/audit.js +136 -0
- package/dist/lib/audit.js.map +1 -0
- package/dist/lib/bootstrap.d.ts +56 -0
- package/dist/lib/bootstrap.d.ts.map +1 -0
- package/dist/lib/bootstrap.js +163 -0
- package/dist/lib/bootstrap.js.map +1 -0
- package/dist/lib/config.d.ts +239 -0
- package/dist/lib/config.d.ts.map +1 -0
- package/dist/lib/config.js +371 -0
- package/dist/lib/config.js.map +1 -0
- package/dist/lib/dashboard.d.ts +81 -0
- package/dist/lib/dashboard.d.ts.map +1 -0
- package/dist/lib/dashboard.js +314 -0
- package/dist/lib/dashboard.js.map +1 -0
- package/dist/lib/db.d.ts +182 -0
- package/dist/lib/db.d.ts.map +1 -0
- package/dist/lib/db.js +620 -0
- package/dist/lib/db.js.map +1 -0
- package/dist/lib/dbSearch.d.ts +65 -0
- package/dist/lib/dbSearch.d.ts.map +1 -0
- package/dist/lib/dbSearch.js +239 -0
- package/dist/lib/dbSearch.js.map +1 -0
- package/dist/lib/dbWrite.d.ts +56 -0
- package/dist/lib/dbWrite.d.ts.map +1 -0
- package/dist/lib/dbWrite.js +171 -0
- package/dist/lib/dbWrite.js.map +1 -0
- package/dist/lib/dream.d.ts +170 -0
- package/dist/lib/dream.d.ts.map +1 -0
- package/dist/lib/dream.js +706 -0
- package/dist/lib/dream.js.map +1 -0
- package/dist/lib/embeddings.d.ts +84 -0
- package/dist/lib/embeddings.d.ts.map +1 -0
- package/dist/lib/embeddings.js +226 -0
- package/dist/lib/embeddings.js.map +1 -0
- package/dist/lib/export.d.ts +92 -0
- package/dist/lib/export.d.ts.map +1 -0
- package/dist/lib/export.js +362 -0
- package/dist/lib/export.js.map +1 -0
- package/dist/lib/federated.d.ts +113 -0
- package/dist/lib/federated.d.ts.map +1 -0
- package/dist/lib/federated.js +346 -0
- package/dist/lib/federated.js.map +1 -0
- package/dist/lib/graph.d.ts +50 -0
- package/dist/lib/graph.d.ts.map +1 -0
- package/dist/lib/graph.js +118 -0
- package/dist/lib/graph.js.map +1 -0
- package/dist/lib/history.d.ts +39 -0
- package/dist/lib/history.d.ts.map +1 -0
- package/dist/lib/history.js +112 -0
- package/dist/lib/history.js.map +1 -0
- package/dist/lib/hybridSearch.d.ts +80 -0
- package/dist/lib/hybridSearch.d.ts.map +1 -0
- package/dist/lib/hybridSearch.js +296 -0
- package/dist/lib/hybridSearch.js.map +1 -0
- package/dist/lib/import.d.ts +52 -0
- package/dist/lib/import.d.ts.map +1 -0
- package/dist/lib/import.js +365 -0
- package/dist/lib/import.js.map +1 -0
- package/dist/lib/ingest.d.ts +51 -0
- package/dist/lib/ingest.d.ts.map +1 -0
- package/dist/lib/ingest.js +144 -0
- package/dist/lib/ingest.js.map +1 -0
- package/dist/lib/lensing.d.ts +35 -0
- package/dist/lib/lensing.d.ts.map +1 -0
- package/dist/lib/lensing.js +85 -0
- package/dist/lib/lensing.js.map +1 -0
- package/dist/lib/llm.d.ts +84 -0
- package/dist/lib/llm.d.ts.map +1 -0
- package/dist/lib/llm.js +386 -0
- package/dist/lib/llm.js.map +1 -0
- package/dist/lib/lock.d.ts +28 -0
- package/dist/lib/lock.d.ts.map +1 -0
- package/dist/lib/lock.js +145 -0
- package/dist/lib/lock.js.map +1 -0
- package/dist/lib/maintenance.d.ts +124 -0
- package/dist/lib/maintenance.d.ts.map +1 -0
- package/dist/lib/maintenance.js +587 -0
- package/dist/lib/maintenance.js.map +1 -0
- package/dist/lib/migrate.d.ts +19 -0
- package/dist/lib/migrate.d.ts.map +1 -0
- package/dist/lib/migrate.js +260 -0
- package/dist/lib/migrate.js.map +1 -0
- package/dist/lib/preferences.d.ts +49 -0
- package/dist/lib/preferences.d.ts.map +1 -0
- package/dist/lib/preferences.js +149 -0
- package/dist/lib/preferences.js.map +1 -0
- package/dist/lib/projectIdentity.d.ts +66 -0
- package/dist/lib/projectIdentity.d.ts.map +1 -0
- package/dist/lib/projectIdentity.js +148 -0
- package/dist/lib/projectIdentity.js.map +1 -0
- package/dist/lib/recall.d.ts +82 -0
- package/dist/lib/recall.d.ts.map +1 -0
- package/dist/lib/recall.js +289 -0
- package/dist/lib/recall.js.map +1 -0
- package/dist/lib/resolver.d.ts +116 -0
- package/dist/lib/resolver.d.ts.map +1 -0
- package/dist/lib/resolver.js +372 -0
- package/dist/lib/resolver.js.map +1 -0
- package/dist/lib/retry.d.ts +24 -0
- package/dist/lib/retry.d.ts.map +1 -0
- package/dist/lib/retry.js +60 -0
- package/dist/lib/retry.js.map +1 -0
- package/dist/lib/rulesGen.d.ts +51 -0
- package/dist/lib/rulesGen.d.ts.map +1 -0
- package/dist/lib/rulesGen.js +167 -0
- package/dist/lib/rulesGen.js.map +1 -0
- package/dist/lib/search.d.ts +51 -0
- package/dist/lib/search.d.ts.map +1 -0
- package/dist/lib/search.js +190 -0
- package/dist/lib/search.js.map +1 -0
- package/dist/lib/staticSearch.d.ts +70 -0
- package/dist/lib/staticSearch.d.ts.map +1 -0
- package/dist/lib/staticSearch.js +162 -0
- package/dist/lib/staticSearch.js.map +1 -0
- package/dist/lib/store.d.ts +79 -0
- package/dist/lib/store.d.ts.map +1 -0
- package/dist/lib/store.js +227 -0
- package/dist/lib/store.js.map +1 -0
- package/dist/lib/structuredIngest.d.ts +37 -0
- package/dist/lib/structuredIngest.d.ts.map +1 -0
- package/dist/lib/structuredIngest.js +208 -0
- package/dist/lib/structuredIngest.js.map +1 -0
- package/dist/lib/tags.d.ts +26 -0
- package/dist/lib/tags.d.ts.map +1 -0
- package/dist/lib/tags.js +109 -0
- package/dist/lib/tags.js.map +1 -0
- package/dist/lib/timeline.d.ts +34 -0
- package/dist/lib/timeline.d.ts.map +1 -0
- package/dist/lib/timeline.js +116 -0
- package/dist/lib/timeline.js.map +1 -0
- package/dist/lib/trace.d.ts +42 -0
- package/dist/lib/trace.d.ts.map +1 -0
- package/dist/lib/trace.js +338 -0
- package/dist/lib/trace.js.map +1 -0
- package/dist/lib/webIndex.d.ts +28 -0
- package/dist/lib/webIndex.d.ts.map +1 -0
- package/dist/lib/webIndex.js +208 -0
- package/dist/lib/webIndex.js.map +1 -0
- package/dist/lib/webIngest.d.ts +51 -0
- package/dist/lib/webIngest.d.ts.map +1 -0
- package/dist/lib/webIngest.js +533 -0
- package/dist/lib/webIngest.js.map +1 -0
- package/dist/lib/wikilinks.d.ts +63 -0
- package/dist/lib/wikilinks.d.ts.map +1 -0
- package/dist/lib/wikilinks.js +146 -0
- package/dist/lib/wikilinks.js.map +1 -0
- package/dist/sandbox/client.d.ts +82 -0
- package/dist/sandbox/client.d.ts.map +1 -0
- package/dist/sandbox/client.js +128 -0
- package/dist/sandbox/client.js.map +1 -0
- package/dist/sandbox/helper-template.d.ts +14 -0
- package/dist/sandbox/helper-template.d.ts.map +1 -0
- package/dist/sandbox/helper-template.js +285 -0
- package/dist/sandbox/helper-template.js.map +1 -0
- package/dist/sandbox/index.d.ts +10 -0
- package/dist/sandbox/index.d.ts.map +1 -0
- package/dist/sandbox/index.js +10 -0
- package/dist/sandbox/index.js.map +1 -0
- package/dist/sandbox/manager.d.ts +40 -0
- package/dist/sandbox/manager.d.ts.map +1 -0
- package/dist/sandbox/manager.js +220 -0
- package/dist/sandbox/manager.js.map +1 -0
- package/dist/sandbox/server.d.ts +44 -0
- package/dist/sandbox/server.d.ts.map +1 -0
- package/dist/sandbox/server.js +661 -0
- package/dist/sandbox/server.js.map +1 -0
- package/package.json +103 -0
- package/prompts/synthesize.md +21 -0
|
@@ -0,0 +1,706 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gnosys Dream Mode — Sleep-time consolidation engine.
|
|
3
|
+
*
|
|
4
|
+
* Metaphor: "closing all the open books overnight."
|
|
5
|
+
*
|
|
6
|
+
* Dream Mode runs during idle periods (configurable threshold) and performs:
|
|
7
|
+
* 1. Confidence decay sweep — apply exponential decay to unreinforced memories
|
|
8
|
+
* 2. Summary generation — hierarchical summaries per category
|
|
9
|
+
* 3. Self-critique — score memories but NEVER delete; produces a review list
|
|
10
|
+
* 4. Relationship discovery — find new links between memories
|
|
11
|
+
* 5. Deduplication pass — flag similar memories for human review
|
|
12
|
+
*
|
|
13
|
+
* Safety guarantees:
|
|
14
|
+
* - Never deletes or archives autonomously
|
|
15
|
+
* - All changes are non-destructive (confidence updates, summaries, review flags)
|
|
16
|
+
* - Configurable max runtime (default 30 min)
|
|
17
|
+
* - Off by default — must be explicitly enabled
|
|
18
|
+
* - Uses cheap/local LLM (configurable, defaults to Ollama)
|
|
19
|
+
*/
|
|
20
|
+
import { getLLMProvider } from "./llm.js";
|
|
21
|
+
import { syncConfidenceToDb, auditToDb } from "./dbWrite.js";
|
|
22
|
+
export const DEFAULT_DREAM_CONFIG = {
|
|
23
|
+
enabled: false,
|
|
24
|
+
idleMinutes: 10,
|
|
25
|
+
maxRuntimeMinutes: 30,
|
|
26
|
+
provider: "ollama",
|
|
27
|
+
model: undefined,
|
|
28
|
+
selfCritique: true,
|
|
29
|
+
generateSummaries: true,
|
|
30
|
+
discoverRelationships: true,
|
|
31
|
+
minMemories: 10,
|
|
32
|
+
};
|
|
33
|
+
// ─── Decay Constants ─────────────────────────────────────────────────────
|
|
34
|
+
const DECAY_LAMBDA = 0.005;
|
|
35
|
+
const STALE_THRESHOLD = 0.3;
|
|
36
|
+
// ─── Dream Engine ────────────────────────────────────────────────────────
|
|
37
|
+
export class GnosysDreamEngine {
|
|
38
|
+
db;
|
|
39
|
+
config;
|
|
40
|
+
dreamConfig;
|
|
41
|
+
provider = null;
|
|
42
|
+
abortRequested = false;
|
|
43
|
+
startTime = 0;
|
|
44
|
+
constructor(db, config, dreamConfig) {
|
|
45
|
+
this.db = db;
|
|
46
|
+
this.config = config;
|
|
47
|
+
this.dreamConfig = { ...DEFAULT_DREAM_CONFIG, ...dreamConfig };
|
|
48
|
+
// Initialize LLM provider for dream operations
|
|
49
|
+
try {
|
|
50
|
+
this.provider = getLLMProvider(this.config, "structuring");
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
this.provider = null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Request abort — dream cycle will stop at the next safe checkpoint.
|
|
58
|
+
*/
|
|
59
|
+
abort() {
|
|
60
|
+
this.abortRequested = true;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Check if we've exceeded max runtime.
|
|
64
|
+
*/
|
|
65
|
+
isOvertime() {
|
|
66
|
+
if (this.dreamConfig.maxRuntimeMinutes <= 0)
|
|
67
|
+
return false;
|
|
68
|
+
const elapsed = Date.now() - this.startTime;
|
|
69
|
+
return elapsed > this.dreamConfig.maxRuntimeMinutes * 60 * 1000;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Check if we should stop (abort requested or overtime).
|
|
73
|
+
*/
|
|
74
|
+
shouldStop() {
|
|
75
|
+
if (this.abortRequested)
|
|
76
|
+
return { stop: true, reason: "abort requested" };
|
|
77
|
+
if (this.isOvertime())
|
|
78
|
+
return { stop: true, reason: `max runtime exceeded (${this.dreamConfig.maxRuntimeMinutes}min)` };
|
|
79
|
+
return { stop: false };
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Run the full dream cycle.
|
|
83
|
+
* This is the main entry point — runs all dream operations in sequence.
|
|
84
|
+
*/
|
|
85
|
+
async dream(onProgress) {
|
|
86
|
+
this.startTime = Date.now();
|
|
87
|
+
this.abortRequested = false;
|
|
88
|
+
const log = onProgress || (() => { });
|
|
89
|
+
const report = {
|
|
90
|
+
startedAt: new Date().toISOString(),
|
|
91
|
+
finishedAt: "",
|
|
92
|
+
durationMs: 0,
|
|
93
|
+
decayUpdated: 0,
|
|
94
|
+
summariesGenerated: 0,
|
|
95
|
+
summariesUpdated: 0,
|
|
96
|
+
reviewSuggestions: [],
|
|
97
|
+
relationshipsDiscovered: 0,
|
|
98
|
+
duplicatesFound: 0,
|
|
99
|
+
errors: [],
|
|
100
|
+
aborted: false,
|
|
101
|
+
};
|
|
102
|
+
if (!this.db.isAvailable() || !this.db.isMigrated()) {
|
|
103
|
+
report.errors.push("gnosys.db not available or not migrated");
|
|
104
|
+
report.finishedAt = new Date().toISOString();
|
|
105
|
+
report.durationMs = Date.now() - this.startTime;
|
|
106
|
+
return report;
|
|
107
|
+
}
|
|
108
|
+
const counts = this.db.getMemoryCount();
|
|
109
|
+
if (counts.active < this.dreamConfig.minMemories) {
|
|
110
|
+
report.errors.push(`Too few memories (${counts.active} < ${this.dreamConfig.minMemories})`);
|
|
111
|
+
report.finishedAt = new Date().toISOString();
|
|
112
|
+
report.durationMs = Date.now() - this.startTime;
|
|
113
|
+
return report;
|
|
114
|
+
}
|
|
115
|
+
log("dream", "Dream cycle starting...");
|
|
116
|
+
// Audit: dream start
|
|
117
|
+
auditToDb(this.db, "dream_start", undefined, {
|
|
118
|
+
config: {
|
|
119
|
+
maxRuntime: this.dreamConfig.maxRuntimeMinutes,
|
|
120
|
+
selfCritique: this.dreamConfig.selfCritique,
|
|
121
|
+
generateSummaries: this.dreamConfig.generateSummaries,
|
|
122
|
+
discoverRelationships: this.dreamConfig.discoverRelationships,
|
|
123
|
+
},
|
|
124
|
+
memoryCount: counts.active,
|
|
125
|
+
});
|
|
126
|
+
// ─── Phase 1: Confidence Decay Sweep ─────────────────────────────────
|
|
127
|
+
log("decay", "Phase 1: Confidence decay sweep...");
|
|
128
|
+
try {
|
|
129
|
+
report.decayUpdated = this.decaySweep();
|
|
130
|
+
log("decay", `Updated ${report.decayUpdated} memories`);
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
report.errors.push(`Decay sweep: ${err instanceof Error ? err.message : String(err)}`);
|
|
134
|
+
}
|
|
135
|
+
let check = this.shouldStop();
|
|
136
|
+
if (check.stop) {
|
|
137
|
+
report.aborted = true;
|
|
138
|
+
report.abortReason = check.reason;
|
|
139
|
+
return this.finalize(report);
|
|
140
|
+
}
|
|
141
|
+
// ─── Phase 2: Self-Critique (Review Suggestions) ─────────────────────
|
|
142
|
+
if (this.dreamConfig.selfCritique) {
|
|
143
|
+
log("critique", "Phase 2: Self-critique...");
|
|
144
|
+
try {
|
|
145
|
+
report.reviewSuggestions = await this.selfCritique(log);
|
|
146
|
+
log("critique", `Generated ${report.reviewSuggestions.length} review suggestions`);
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
report.errors.push(`Self-critique: ${err instanceof Error ? err.message : String(err)}`);
|
|
150
|
+
}
|
|
151
|
+
check = this.shouldStop();
|
|
152
|
+
if (check.stop) {
|
|
153
|
+
report.aborted = true;
|
|
154
|
+
report.abortReason = check.reason;
|
|
155
|
+
return this.finalize(report);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// ─── Phase 3: Summary Generation ─────────────────────────────────────
|
|
159
|
+
if (this.dreamConfig.generateSummaries && this.provider) {
|
|
160
|
+
log("summaries", "Phase 3: Summary generation...");
|
|
161
|
+
try {
|
|
162
|
+
const summaryResult = await this.generateSummaries(log);
|
|
163
|
+
report.summariesGenerated = summaryResult.generated;
|
|
164
|
+
report.summariesUpdated = summaryResult.updated;
|
|
165
|
+
log("summaries", `Generated ${summaryResult.generated}, updated ${summaryResult.updated}`);
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
report.errors.push(`Summary generation: ${err instanceof Error ? err.message : String(err)}`);
|
|
169
|
+
}
|
|
170
|
+
check = this.shouldStop();
|
|
171
|
+
if (check.stop) {
|
|
172
|
+
report.aborted = true;
|
|
173
|
+
report.abortReason = check.reason;
|
|
174
|
+
return this.finalize(report);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
// ─── Phase 4: Relationship Discovery ─────────────────────────────────
|
|
178
|
+
if (this.dreamConfig.discoverRelationships && this.provider) {
|
|
179
|
+
log("relationships", "Phase 4: Relationship discovery...");
|
|
180
|
+
try {
|
|
181
|
+
report.relationshipsDiscovered = await this.discoverRelationships(log);
|
|
182
|
+
log("relationships", `Discovered ${report.relationshipsDiscovered} new relationships`);
|
|
183
|
+
}
|
|
184
|
+
catch (err) {
|
|
185
|
+
report.errors.push(`Relationship discovery: ${err instanceof Error ? err.message : String(err)}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return this.finalize(report);
|
|
189
|
+
}
|
|
190
|
+
finalize(report) {
|
|
191
|
+
report.finishedAt = new Date().toISOString();
|
|
192
|
+
report.durationMs = Date.now() - this.startTime;
|
|
193
|
+
// Audit: dream complete
|
|
194
|
+
auditToDb(this.db, "dream_complete", undefined, {
|
|
195
|
+
durationMs: report.durationMs,
|
|
196
|
+
decayUpdated: report.decayUpdated,
|
|
197
|
+
summariesGenerated: report.summariesGenerated,
|
|
198
|
+
reviewSuggestions: report.reviewSuggestions.length,
|
|
199
|
+
relationshipsDiscovered: report.relationshipsDiscovered,
|
|
200
|
+
errors: report.errors.length,
|
|
201
|
+
aborted: report.aborted,
|
|
202
|
+
}, report.durationMs);
|
|
203
|
+
return report;
|
|
204
|
+
}
|
|
205
|
+
// ─── Phase 1: Decay Sweep ──────────────────────────────────────────────
|
|
206
|
+
/**
|
|
207
|
+
* Apply exponential decay to all active memories.
|
|
208
|
+
* Formula: decayed = confidence * e^(-λ * days_since_reinforced)
|
|
209
|
+
* Only updates if decayed value differs from stored value by > 0.01.
|
|
210
|
+
*/
|
|
211
|
+
decaySweep() {
|
|
212
|
+
const now = new Date();
|
|
213
|
+
const today = now.toISOString().split("T")[0];
|
|
214
|
+
const memories = this.db.getActiveMemories();
|
|
215
|
+
let updated = 0;
|
|
216
|
+
for (const mem of memories) {
|
|
217
|
+
const lastReinforced = mem.last_reinforced || mem.modified || mem.created;
|
|
218
|
+
const lastDate = new Date(lastReinforced);
|
|
219
|
+
const daysSince = Math.max(0, Math.floor((now.getTime() - lastDate.getTime()) / (1000 * 60 * 60 * 24)));
|
|
220
|
+
if (daysSince === 0)
|
|
221
|
+
continue; // Recently active, skip
|
|
222
|
+
const decayed = mem.confidence * Math.exp(-DECAY_LAMBDA * daysSince);
|
|
223
|
+
const rounded = Math.round(decayed * 100) / 100;
|
|
224
|
+
// Only update if meaningful change (>0.01)
|
|
225
|
+
if (Math.abs(rounded - mem.confidence) > 0.01) {
|
|
226
|
+
syncConfidenceToDb(this.db, mem.id, rounded);
|
|
227
|
+
updated++;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return updated;
|
|
231
|
+
}
|
|
232
|
+
// ─── Phase 2: Self-Critique ────────────────────────────────────────────
|
|
233
|
+
/**
|
|
234
|
+
* Score memories and generate review suggestions.
|
|
235
|
+
* NEVER deletes or archives — only flags for human review.
|
|
236
|
+
*/
|
|
237
|
+
async selfCritique(log) {
|
|
238
|
+
const suggestions = [];
|
|
239
|
+
const memories = this.db.getActiveMemories();
|
|
240
|
+
for (const mem of memories) {
|
|
241
|
+
const check = this.shouldStop();
|
|
242
|
+
if (check.stop)
|
|
243
|
+
break;
|
|
244
|
+
// Rule-based critique (no LLM needed)
|
|
245
|
+
const issues = this.critiquMemory(mem);
|
|
246
|
+
if (issues.length > 0) {
|
|
247
|
+
suggestions.push({
|
|
248
|
+
memoryId: mem.id,
|
|
249
|
+
title: mem.title,
|
|
250
|
+
reason: issues.join("; "),
|
|
251
|
+
currentConfidence: mem.confidence,
|
|
252
|
+
suggestedAction: this.suggestAction(mem, issues),
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
// LLM-based critique for borderline memories (if provider available)
|
|
257
|
+
if (this.provider && suggestions.length < 20) {
|
|
258
|
+
const borderline = memories.filter((m) => m.confidence > STALE_THRESHOLD && m.confidence < 0.6);
|
|
259
|
+
for (const mem of borderline.slice(0, 10)) {
|
|
260
|
+
const check = this.shouldStop();
|
|
261
|
+
if (check.stop)
|
|
262
|
+
break;
|
|
263
|
+
try {
|
|
264
|
+
const llmSuggestion = await this.llmCritique(mem);
|
|
265
|
+
if (llmSuggestion) {
|
|
266
|
+
suggestions.push(llmSuggestion);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
catch (err) {
|
|
270
|
+
log("critique", `LLM critique failed for ${mem.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
// Store review suggestions in summaries table
|
|
275
|
+
if (suggestions.length > 0) {
|
|
276
|
+
const today = new Date().toISOString().split("T")[0];
|
|
277
|
+
this.db.upsertSummary({
|
|
278
|
+
id: `review-${today}`,
|
|
279
|
+
scope: "dream",
|
|
280
|
+
scope_key: `review-${today}`,
|
|
281
|
+
content: JSON.stringify(suggestions, null, 2),
|
|
282
|
+
source_ids: JSON.stringify(suggestions.map((s) => s.memoryId)),
|
|
283
|
+
created: today,
|
|
284
|
+
modified: today,
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
return suggestions;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Rule-based critique — fast, no LLM needed.
|
|
291
|
+
*/
|
|
292
|
+
critiquMemory(mem) {
|
|
293
|
+
const issues = [];
|
|
294
|
+
const now = new Date();
|
|
295
|
+
// Low confidence
|
|
296
|
+
if (mem.confidence < STALE_THRESHOLD) {
|
|
297
|
+
issues.push(`Very low confidence (${mem.confidence.toFixed(2)})`);
|
|
298
|
+
}
|
|
299
|
+
// Never reinforced + old
|
|
300
|
+
if (mem.reinforcement_count === 0) {
|
|
301
|
+
const created = new Date(mem.created);
|
|
302
|
+
const daysSince = Math.floor((now.getTime() - created.getTime()) / (1000 * 60 * 60 * 24));
|
|
303
|
+
if (daysSince > 30) {
|
|
304
|
+
issues.push(`Never reinforced in ${daysSince} days`);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
// Very short content (might be incomplete)
|
|
308
|
+
if (mem.content.length < 50) {
|
|
309
|
+
issues.push("Very short content (< 50 chars) — may be incomplete");
|
|
310
|
+
}
|
|
311
|
+
// No tags
|
|
312
|
+
try {
|
|
313
|
+
const tags = JSON.parse(mem.tags || "[]");
|
|
314
|
+
if (Array.isArray(tags) && tags.length === 0) {
|
|
315
|
+
issues.push("No tags — harder to discover");
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
issues.push("Invalid tags format");
|
|
320
|
+
}
|
|
321
|
+
// Empty relevance
|
|
322
|
+
if (!mem.relevance || mem.relevance.trim().length === 0) {
|
|
323
|
+
issues.push("No relevance keywords — invisible to search");
|
|
324
|
+
}
|
|
325
|
+
return issues;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* LLM-based critique for borderline memories.
|
|
329
|
+
*/
|
|
330
|
+
async llmCritique(mem) {
|
|
331
|
+
if (!this.provider)
|
|
332
|
+
return null;
|
|
333
|
+
const prompt = `You are a knowledge management quality reviewer. Evaluate this memory and decide if it needs attention.
|
|
334
|
+
|
|
335
|
+
Title: ${mem.title}
|
|
336
|
+
Category: ${mem.category}
|
|
337
|
+
Confidence: ${mem.confidence}
|
|
338
|
+
Created: ${mem.created}
|
|
339
|
+
Last reinforced: ${mem.last_reinforced || "never"}
|
|
340
|
+
Reinforcement count: ${mem.reinforcement_count}
|
|
341
|
+
|
|
342
|
+
Content:
|
|
343
|
+
${mem.content.substring(0, 500)}
|
|
344
|
+
|
|
345
|
+
Respond with ONLY one of these JSON objects (no explanation):
|
|
346
|
+
- {"action": "ok"} if the memory is fine
|
|
347
|
+
- {"action": "review", "reason": "..."} if it needs human review
|
|
348
|
+
- {"action": "needs-update", "reason": "..."} if content seems outdated
|
|
349
|
+
- {"action": "consider-merge", "reason": "..."} if it seems to overlap with other common knowledge`;
|
|
350
|
+
try {
|
|
351
|
+
const response = await this.provider.generate(prompt, { maxTokens: 200 });
|
|
352
|
+
const jsonMatch = response.match(/\{[^}]+\}/);
|
|
353
|
+
if (!jsonMatch)
|
|
354
|
+
return null;
|
|
355
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
356
|
+
if (parsed.action === "ok")
|
|
357
|
+
return null;
|
|
358
|
+
return {
|
|
359
|
+
memoryId: mem.id,
|
|
360
|
+
title: mem.title,
|
|
361
|
+
reason: parsed.reason || "LLM flagged for review",
|
|
362
|
+
currentConfidence: mem.confidence,
|
|
363
|
+
suggestedAction: parsed.action,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
catch {
|
|
367
|
+
return null;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* Determine suggested action based on issues.
|
|
372
|
+
*/
|
|
373
|
+
suggestAction(mem, issues) {
|
|
374
|
+
if (mem.confidence < STALE_THRESHOLD)
|
|
375
|
+
return "consider-archive";
|
|
376
|
+
if (issues.some((i) => i.includes("incomplete")))
|
|
377
|
+
return "needs-update";
|
|
378
|
+
return "review";
|
|
379
|
+
}
|
|
380
|
+
// ─── Phase 3: Summary Generation ───────────────────────────────────────
|
|
381
|
+
/**
|
|
382
|
+
* Generate or update hierarchical summaries per category.
|
|
383
|
+
* Uses LLM to synthesize category-level overviews.
|
|
384
|
+
*/
|
|
385
|
+
async generateSummaries(log) {
|
|
386
|
+
if (!this.provider)
|
|
387
|
+
return { generated: 0, updated: 0 };
|
|
388
|
+
const categories = this.db.getCategories();
|
|
389
|
+
let generated = 0;
|
|
390
|
+
let updated = 0;
|
|
391
|
+
for (const category of categories) {
|
|
392
|
+
const check = this.shouldStop();
|
|
393
|
+
if (check.stop)
|
|
394
|
+
break;
|
|
395
|
+
const memories = this.db.getMemoriesByCategory(category);
|
|
396
|
+
if (memories.length < 2)
|
|
397
|
+
continue; // Not enough to summarize
|
|
398
|
+
// Check if summary exists and is still current
|
|
399
|
+
const existing = this.db.getSummary("category", category);
|
|
400
|
+
if (existing) {
|
|
401
|
+
const existingIds = JSON.parse(existing.source_ids);
|
|
402
|
+
const currentIds = memories.map((m) => m.id);
|
|
403
|
+
const unchanged = existingIds.length === currentIds.length &&
|
|
404
|
+
existingIds.every((id) => currentIds.includes(id));
|
|
405
|
+
if (unchanged)
|
|
406
|
+
continue; // No new memories in this category
|
|
407
|
+
}
|
|
408
|
+
log("summaries", `Summarizing ${category} (${memories.length} memories)...`);
|
|
409
|
+
try {
|
|
410
|
+
const summary = await this.summarizeCategory(category, memories);
|
|
411
|
+
if (summary) {
|
|
412
|
+
const today = new Date().toISOString().split("T")[0];
|
|
413
|
+
const id = existing?.id || `summary-${category}-${today}`;
|
|
414
|
+
this.db.upsertSummary({
|
|
415
|
+
id,
|
|
416
|
+
scope: "category",
|
|
417
|
+
scope_key: category,
|
|
418
|
+
content: summary,
|
|
419
|
+
source_ids: JSON.stringify(memories.map((m) => m.id)),
|
|
420
|
+
created: existing?.created || today,
|
|
421
|
+
modified: today,
|
|
422
|
+
});
|
|
423
|
+
if (existing) {
|
|
424
|
+
updated++;
|
|
425
|
+
}
|
|
426
|
+
else {
|
|
427
|
+
generated++;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
catch (err) {
|
|
432
|
+
log("summaries", `Failed to summarize ${category}: ${err instanceof Error ? err.message : String(err)}`);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
return { generated, updated };
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Use LLM to generate a category summary.
|
|
439
|
+
*/
|
|
440
|
+
async summarizeCategory(category, memories) {
|
|
441
|
+
if (!this.provider)
|
|
442
|
+
return null;
|
|
443
|
+
// Build context from memories (truncate to fit context window)
|
|
444
|
+
const memoryTexts = memories
|
|
445
|
+
.slice(0, 20) // Max 20 memories per category summary
|
|
446
|
+
.map((m) => `## ${m.title}\n${m.content.substring(0, 300)}`)
|
|
447
|
+
.join("\n\n");
|
|
448
|
+
const prompt = `You are a knowledge management assistant. Create a concise summary of the "${category}" category that captures the key themes, decisions, and patterns across these memories.
|
|
449
|
+
|
|
450
|
+
The summary should:
|
|
451
|
+
- Be 2-4 paragraphs
|
|
452
|
+
- Highlight key themes and patterns
|
|
453
|
+
- Note any contradictions or evolving decisions
|
|
454
|
+
- Be useful as a quick reference for someone new to the project
|
|
455
|
+
|
|
456
|
+
Memories in "${category}":
|
|
457
|
+
|
|
458
|
+
${memoryTexts}
|
|
459
|
+
|
|
460
|
+
Category summary:`;
|
|
461
|
+
try {
|
|
462
|
+
return await this.provider.generate(prompt, { maxTokens: 1024 });
|
|
463
|
+
}
|
|
464
|
+
catch {
|
|
465
|
+
return null;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
// ─── Phase 4: Relationship Discovery ───────────────────────────────────
|
|
469
|
+
/**
|
|
470
|
+
* Use LLM to discover relationships between memories.
|
|
471
|
+
* Only creates new edges — never removes existing ones.
|
|
472
|
+
*/
|
|
473
|
+
async discoverRelationships(log) {
|
|
474
|
+
if (!this.provider)
|
|
475
|
+
return 0;
|
|
476
|
+
const memories = this.db.getActiveMemories();
|
|
477
|
+
if (memories.length < 3)
|
|
478
|
+
return 0;
|
|
479
|
+
let discovered = 0;
|
|
480
|
+
const today = new Date().toISOString().split("T")[0];
|
|
481
|
+
// Get existing relationships to avoid duplicates
|
|
482
|
+
const existingPairs = new Set();
|
|
483
|
+
for (const mem of memories) {
|
|
484
|
+
const rels = this.db.getRelationshipsFrom(mem.id);
|
|
485
|
+
for (const r of rels) {
|
|
486
|
+
existingPairs.add(`${r.source_id}→${r.target_id}→${r.rel_type}`);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
// Build a compact index of all memories for the LLM
|
|
490
|
+
const memoryIndex = memories
|
|
491
|
+
.slice(0, 50) // Limit to 50 most relevant
|
|
492
|
+
.map((m) => `[${m.id}] ${m.title} (${m.category}) — ${(m.relevance || "").substring(0, 80)}`)
|
|
493
|
+
.join("\n");
|
|
494
|
+
// Process in batches of 5 source memories
|
|
495
|
+
const batchSize = 5;
|
|
496
|
+
for (let i = 0; i < Math.min(memories.length, 30); i += batchSize) {
|
|
497
|
+
const check = this.shouldStop();
|
|
498
|
+
if (check.stop)
|
|
499
|
+
break;
|
|
500
|
+
const batch = memories.slice(i, i + batchSize);
|
|
501
|
+
const batchTitles = batch.map((m) => `[${m.id}] ${m.title}`).join(", ");
|
|
502
|
+
log("relationships", `Analyzing relationships for: ${batchTitles}`);
|
|
503
|
+
try {
|
|
504
|
+
const relationships = await this.findRelationships(batch, memoryIndex);
|
|
505
|
+
for (const rel of relationships) {
|
|
506
|
+
const key = `${rel.source_id}→${rel.target_id}→${rel.rel_type}`;
|
|
507
|
+
if (existingPairs.has(key))
|
|
508
|
+
continue;
|
|
509
|
+
this.db.insertRelationship({
|
|
510
|
+
source_id: rel.source_id,
|
|
511
|
+
target_id: rel.target_id,
|
|
512
|
+
rel_type: rel.rel_type,
|
|
513
|
+
label: rel.label,
|
|
514
|
+
confidence: rel.confidence,
|
|
515
|
+
created: today,
|
|
516
|
+
});
|
|
517
|
+
existingPairs.add(key);
|
|
518
|
+
discovered++;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
catch (err) {
|
|
522
|
+
log("relationships", `Failed for batch: ${err instanceof Error ? err.message : String(err)}`);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
return discovered;
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Use LLM to find relationships for a batch of source memories.
|
|
529
|
+
*/
|
|
530
|
+
async findRelationships(sources, memoryIndex) {
|
|
531
|
+
if (!this.provider)
|
|
532
|
+
return [];
|
|
533
|
+
const sourceContext = sources
|
|
534
|
+
.map((m) => `[${m.id}] "${m.title}" — ${m.content.substring(0, 200)}`)
|
|
535
|
+
.join("\n\n");
|
|
536
|
+
const prompt = `You are a knowledge graph assistant. Given these source memories and a full memory index, identify meaningful relationships.
|
|
537
|
+
|
|
538
|
+
Relationship types: references, depends_on, contradicts, extends, supersedes, related_to
|
|
539
|
+
|
|
540
|
+
Source memories:
|
|
541
|
+
${sourceContext}
|
|
542
|
+
|
|
543
|
+
Full memory index:
|
|
544
|
+
${memoryIndex}
|
|
545
|
+
|
|
546
|
+
For each relationship found, output a JSON array of objects with: source_id, target_id, rel_type, label (short description), confidence (0.5-1.0).
|
|
547
|
+
Only output relationships with confidence >= 0.7. Do NOT create self-referencing relationships.
|
|
548
|
+
Output ONLY the JSON array, no explanation.`;
|
|
549
|
+
try {
|
|
550
|
+
const response = await this.provider.generate(prompt, { maxTokens: 1024 });
|
|
551
|
+
// Extract JSON array from response
|
|
552
|
+
const jsonMatch = response.match(/\[[\s\S]*\]/);
|
|
553
|
+
if (!jsonMatch)
|
|
554
|
+
return [];
|
|
555
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
556
|
+
if (!Array.isArray(parsed))
|
|
557
|
+
return [];
|
|
558
|
+
// Validate entries
|
|
559
|
+
return parsed.filter((r) => r.source_id &&
|
|
560
|
+
r.target_id &&
|
|
561
|
+
r.rel_type &&
|
|
562
|
+
r.source_id !== r.target_id &&
|
|
563
|
+
r.confidence >= 0.7);
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
return [];
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
// ─── Dream Scheduler ─────────────────────────────────────────────────────
|
|
571
|
+
/**
|
|
572
|
+
* Manages the dream cycle schedule within a running server.
|
|
573
|
+
* Monitors idle time and triggers dream mode when conditions are met.
|
|
574
|
+
*/
|
|
575
|
+
export class DreamScheduler {
|
|
576
|
+
engine;
|
|
577
|
+
config;
|
|
578
|
+
lastActivity = Date.now();
|
|
579
|
+
checkInterval = null;
|
|
580
|
+
running = false;
|
|
581
|
+
currentDream = null;
|
|
582
|
+
constructor(engine, config) {
|
|
583
|
+
this.engine = engine;
|
|
584
|
+
// Explicitly pick known config keys to prevent prototype pollution
|
|
585
|
+
this.config = { ...DEFAULT_DREAM_CONFIG };
|
|
586
|
+
if (config) {
|
|
587
|
+
for (const key of Object.keys(DEFAULT_DREAM_CONFIG)) {
|
|
588
|
+
if (key in config && config[key] !== undefined) {
|
|
589
|
+
this.config[key] = config[key];
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Record activity — resets the idle timer.
|
|
596
|
+
* Call this on every MCP tool invocation.
|
|
597
|
+
*/
|
|
598
|
+
recordActivity() {
|
|
599
|
+
this.lastActivity = Date.now();
|
|
600
|
+
// If dreaming, abort gracefully
|
|
601
|
+
if (this.running) {
|
|
602
|
+
this.engine.abort();
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Start the scheduler — checks idle time periodically.
|
|
607
|
+
*/
|
|
608
|
+
start() {
|
|
609
|
+
if (!this.config.enabled)
|
|
610
|
+
return;
|
|
611
|
+
if (this.checkInterval)
|
|
612
|
+
return;
|
|
613
|
+
const CHECK_INTERVAL = 60_000; // Check every minute
|
|
614
|
+
this.checkInterval = setInterval(() => this.checkIdle(), CHECK_INTERVAL);
|
|
615
|
+
// Don't prevent Node from exiting
|
|
616
|
+
if (this.checkInterval.unref) {
|
|
617
|
+
this.checkInterval.unref();
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Stop the scheduler and abort any running dream.
|
|
622
|
+
*/
|
|
623
|
+
stop() {
|
|
624
|
+
if (this.checkInterval) {
|
|
625
|
+
clearInterval(this.checkInterval);
|
|
626
|
+
this.checkInterval = null;
|
|
627
|
+
}
|
|
628
|
+
if (this.running) {
|
|
629
|
+
this.engine.abort();
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Check if idle long enough to start dreaming.
|
|
634
|
+
*/
|
|
635
|
+
async checkIdle() {
|
|
636
|
+
if (this.running)
|
|
637
|
+
return;
|
|
638
|
+
const idleMs = Date.now() - this.lastActivity;
|
|
639
|
+
const idleMinutes = idleMs / 60_000;
|
|
640
|
+
if (idleMinutes >= this.config.idleMinutes) {
|
|
641
|
+
this.running = true;
|
|
642
|
+
try {
|
|
643
|
+
this.currentDream = this.engine.dream((phase, detail) => {
|
|
644
|
+
// Log to stderr so it doesn't interfere with MCP stdio
|
|
645
|
+
console.error(`[dream:${phase}] ${detail}`);
|
|
646
|
+
});
|
|
647
|
+
const report = await this.currentDream;
|
|
648
|
+
console.error(`[dream] Complete: ${report.decayUpdated} decay, ${report.summariesGenerated} summaries, ${report.reviewSuggestions.length} reviews, ${report.relationshipsDiscovered} relations (${(report.durationMs / 1000).toFixed(1)}s)`);
|
|
649
|
+
}
|
|
650
|
+
catch (err) {
|
|
651
|
+
console.error(`[dream] Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
652
|
+
}
|
|
653
|
+
finally {
|
|
654
|
+
this.running = false;
|
|
655
|
+
this.currentDream = null;
|
|
656
|
+
this.lastActivity = Date.now(); // Reset idle timer after dream
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
/**
|
|
661
|
+
* Check if currently dreaming.
|
|
662
|
+
*/
|
|
663
|
+
isDreaming() {
|
|
664
|
+
return this.running;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
// ─── Format Helper ───────────────────────────────────────────────────────
|
|
668
|
+
/**
|
|
669
|
+
* Format a dream report as human-readable text.
|
|
670
|
+
*/
|
|
671
|
+
export function formatDreamReport(report) {
|
|
672
|
+
const lines = [];
|
|
673
|
+
lines.push("Gnosys Dream Report");
|
|
674
|
+
lines.push("=".repeat(40));
|
|
675
|
+
lines.push("");
|
|
676
|
+
lines.push(`Duration: ${(report.durationMs / 1000).toFixed(1)}s`);
|
|
677
|
+
lines.push(`Started: ${report.startedAt}`);
|
|
678
|
+
lines.push(`Finished: ${report.finishedAt}`);
|
|
679
|
+
if (report.aborted) {
|
|
680
|
+
lines.push(`⚠ Aborted: ${report.abortReason}`);
|
|
681
|
+
}
|
|
682
|
+
lines.push("");
|
|
683
|
+
lines.push("Results:");
|
|
684
|
+
lines.push(` Confidence decay updates: ${report.decayUpdated}`);
|
|
685
|
+
lines.push(` Summaries generated: ${report.summariesGenerated}`);
|
|
686
|
+
lines.push(` Summaries updated: ${report.summariesUpdated}`);
|
|
687
|
+
lines.push(` Relationships discovered: ${report.relationshipsDiscovered}`);
|
|
688
|
+
lines.push(` Duplicates flagged: ${report.duplicatesFound}`);
|
|
689
|
+
lines.push("");
|
|
690
|
+
if (report.reviewSuggestions.length > 0) {
|
|
691
|
+
lines.push(`Review Suggestions (${report.reviewSuggestions.length}):`);
|
|
692
|
+
for (const s of report.reviewSuggestions) {
|
|
693
|
+
lines.push(` [${s.suggestedAction}] "${s.title}" (confidence: ${s.currentConfidence.toFixed(2)})`);
|
|
694
|
+
lines.push(` Reason: ${s.reason}`);
|
|
695
|
+
}
|
|
696
|
+
lines.push("");
|
|
697
|
+
}
|
|
698
|
+
if (report.errors.length > 0) {
|
|
699
|
+
lines.push(`Errors (${report.errors.length}):`);
|
|
700
|
+
for (const e of report.errors) {
|
|
701
|
+
lines.push(` ! ${e}`);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
return lines.join("\n");
|
|
705
|
+
}
|
|
706
|
+
//# sourceMappingURL=dream.js.map
|