clawmem 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +660 -0
- package/CLAUDE.md +660 -0
- package/LICENSE +21 -0
- package/README.md +993 -0
- package/SKILL.md +717 -0
- package/bin/clawmem +75 -0
- package/package.json +72 -0
- package/src/amem.ts +797 -0
- package/src/beads.ts +263 -0
- package/src/clawmem.ts +1849 -0
- package/src/collections.ts +405 -0
- package/src/config.ts +178 -0
- package/src/consolidation.ts +123 -0
- package/src/directory-context.ts +248 -0
- package/src/errors.ts +41 -0
- package/src/formatter.ts +427 -0
- package/src/graph-traversal.ts +247 -0
- package/src/hooks/context-surfacing.ts +317 -0
- package/src/hooks/curator-nudge.ts +89 -0
- package/src/hooks/decision-extractor.ts +639 -0
- package/src/hooks/feedback-loop.ts +214 -0
- package/src/hooks/handoff-generator.ts +345 -0
- package/src/hooks/postcompact-inject.ts +226 -0
- package/src/hooks/precompact-extract.ts +314 -0
- package/src/hooks/pretool-inject.ts +79 -0
- package/src/hooks/session-bootstrap.ts +324 -0
- package/src/hooks/staleness-check.ts +130 -0
- package/src/hooks.ts +367 -0
- package/src/indexer.ts +327 -0
- package/src/intent.ts +294 -0
- package/src/limits.ts +26 -0
- package/src/llm.ts +1175 -0
- package/src/mcp.ts +2138 -0
- package/src/memory.ts +336 -0
- package/src/mmr.ts +93 -0
- package/src/observer.ts +269 -0
- package/src/openclaw/engine.ts +283 -0
- package/src/openclaw/index.ts +221 -0
- package/src/openclaw/plugin.json +83 -0
- package/src/openclaw/shell.ts +207 -0
- package/src/openclaw/tools.ts +304 -0
- package/src/profile.ts +346 -0
- package/src/promptguard.ts +218 -0
- package/src/retrieval-gate.ts +106 -0
- package/src/search-utils.ts +127 -0
- package/src/server.ts +783 -0
- package/src/splitter.ts +325 -0
- package/src/store.ts +4062 -0
- package/src/validation.ts +67 -0
- package/src/watcher.ts +58 -0
package/src/server.ts
ADDED
|
@@ -0,0 +1,783 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ClawMem HTTP REST API Server
|
|
3
|
+
*
|
|
4
|
+
* REST interface over ClawMem's search, retrieval, and lifecycle operations.
|
|
5
|
+
* Modeled after Engram's server.go — simple JSON handlers, localhost-only by default.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* clawmem serve [--port 7438] [--host 127.0.0.1]
|
|
9
|
+
*
|
|
10
|
+
* All endpoints require Bearer token auth when CLAWMEM_API_TOKEN is set.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { Server } from "bun";
|
|
14
|
+
import type { Store, SearchResult, TimelineResult } from "./store.ts";
|
|
15
|
+
import { enrichResults, reciprocalRankFusion, toRanked } from "./search-utils.ts";
|
|
16
|
+
import { applyCompositeScoring, hasRecencyIntent, type EnrichedResult } from "./memory.ts";
|
|
17
|
+
import { applyMMRDiversity } from "./mmr.ts";
|
|
18
|
+
import { listCollections } from "./collections.ts";
|
|
19
|
+
import { classifyIntent, type IntentType } from "./intent.ts";
|
|
20
|
+
import { getDefaultLlamaCpp } from "./llm.ts";
|
|
21
|
+
import {
|
|
22
|
+
DEFAULT_EMBED_MODEL,
|
|
23
|
+
DEFAULT_QUERY_MODEL,
|
|
24
|
+
DEFAULT_RERANK_MODEL,
|
|
25
|
+
extractSnippet,
|
|
26
|
+
} from "./store.ts";
|
|
27
|
+
|
|
28
|
+
// =============================================================================
|
|
29
|
+
// Types
|
|
30
|
+
// =============================================================================
|
|
31
|
+
|
|
32
|
+
type RouteHandler = (req: Request, url: URL, store: Store) => Promise<Response> | Response;
|
|
33
|
+
|
|
34
|
+
// =============================================================================
|
|
35
|
+
// Auth
|
|
36
|
+
// =============================================================================
|
|
37
|
+
|
|
38
|
+
const API_TOKEN = process.env.CLAWMEM_API_TOKEN || null;
|
|
39
|
+
|
|
40
|
+
function checkAuth(req: Request): Response | null {
|
|
41
|
+
if (!API_TOKEN) return null; // No token configured — open access
|
|
42
|
+
const auth = req.headers.get("authorization");
|
|
43
|
+
if (!auth || auth !== `Bearer ${API_TOKEN}`) {
|
|
44
|
+
return jsonResponse({ error: "Unauthorized" }, 401);
|
|
45
|
+
}
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// =============================================================================
|
|
50
|
+
// JSON Helpers
|
|
51
|
+
// =============================================================================
|
|
52
|
+
|
|
53
|
+
function jsonResponse(data: any, status: number = 200): Response {
|
|
54
|
+
return new Response(JSON.stringify(data), {
|
|
55
|
+
status,
|
|
56
|
+
headers: {
|
|
57
|
+
"Content-Type": "application/json",
|
|
58
|
+
"Access-Control-Allow-Origin": "http://localhost:*",
|
|
59
|
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
|
|
60
|
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function jsonError(message: string, status: number = 400): Response {
|
|
66
|
+
return jsonResponse({ error: message }, status);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function parseBody<T>(req: Request): Promise<T | null> {
|
|
70
|
+
try {
|
|
71
|
+
return await req.json() as T;
|
|
72
|
+
} catch {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function queryParam(url: URL, key: string, defaultValue?: string): string | undefined {
|
|
78
|
+
return url.searchParams.get(key) ?? defaultValue;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function queryInt(url: URL, key: string, defaultValue: number): number {
|
|
82
|
+
const val = url.searchParams.get(key);
|
|
83
|
+
if (!val) return defaultValue;
|
|
84
|
+
const parsed = parseInt(val, 10);
|
|
85
|
+
return isNaN(parsed) ? defaultValue : parsed;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function queryBool(url: URL, key: string, defaultValue: boolean): boolean {
|
|
89
|
+
const val = url.searchParams.get(key);
|
|
90
|
+
if (val === null) return defaultValue;
|
|
91
|
+
return val === "true" || val === "1";
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// =============================================================================
|
|
95
|
+
// Route Handlers
|
|
96
|
+
// =============================================================================
|
|
97
|
+
|
|
98
|
+
// --- Health ---
|
|
99
|
+
|
|
100
|
+
function handleHealth(_req: Request, _url: URL, store: Store): Response {
|
|
101
|
+
const status = store.getStatus();
|
|
102
|
+
return jsonResponse({
|
|
103
|
+
status: "ok",
|
|
104
|
+
service: "clawmem",
|
|
105
|
+
version: "0.2.0",
|
|
106
|
+
database: store.dbPath,
|
|
107
|
+
documents: status.totalDocuments,
|
|
108
|
+
needsEmbedding: status.needsEmbedding,
|
|
109
|
+
hasVectors: status.hasVectorIndex,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// --- Stats ---
|
|
114
|
+
|
|
115
|
+
function handleStats(_req: Request, _url: URL, store: Store): Response {
|
|
116
|
+
const status = store.getStatus();
|
|
117
|
+
const health = store.getIndexHealth();
|
|
118
|
+
return jsonResponse({
|
|
119
|
+
...status,
|
|
120
|
+
health,
|
|
121
|
+
collections: status.collections,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// --- Unified Search ---
|
|
126
|
+
|
|
127
|
+
async function handleSearch(req: Request, _url: URL, store: Store): Promise<Response> {
|
|
128
|
+
const body = await parseBody<{
|
|
129
|
+
query: string;
|
|
130
|
+
mode?: "auto" | "keyword" | "semantic" | "hybrid";
|
|
131
|
+
collection?: string;
|
|
132
|
+
compact?: boolean;
|
|
133
|
+
limit?: number;
|
|
134
|
+
intent?: string;
|
|
135
|
+
}>(req);
|
|
136
|
+
|
|
137
|
+
if (!body?.query) return jsonError("query is required");
|
|
138
|
+
|
|
139
|
+
const query = body.query;
|
|
140
|
+
const mode = body.mode ?? "auto";
|
|
141
|
+
const limit = Math.min(body.limit ?? 10, 50);
|
|
142
|
+
const compact = body.compact ?? true;
|
|
143
|
+
const collections = body.collection ? body.collection.split(",").map(c => c.trim()) : undefined;
|
|
144
|
+
|
|
145
|
+
let results: SearchResult[];
|
|
146
|
+
|
|
147
|
+
if (mode === "keyword" || (mode === "auto" && query.split(/\s+/).length <= 3)) {
|
|
148
|
+
results = store.searchFTS(query, limit * 2, undefined, collections);
|
|
149
|
+
} else if (mode === "semantic") {
|
|
150
|
+
try {
|
|
151
|
+
results = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
|
|
152
|
+
} catch {
|
|
153
|
+
results = store.searchFTS(query, limit * 2, undefined, collections);
|
|
154
|
+
}
|
|
155
|
+
} else {
|
|
156
|
+
// hybrid — BM25 + vector
|
|
157
|
+
const ftsResults = store.searchFTS(query, limit * 2, undefined, collections);
|
|
158
|
+
let vecResults: SearchResult[] = [];
|
|
159
|
+
try {
|
|
160
|
+
vecResults = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
|
|
161
|
+
} catch { /* vector unavailable */ }
|
|
162
|
+
// Simple merge — dedupe by filepath, take max score
|
|
163
|
+
const merged = new Map<string, SearchResult>();
|
|
164
|
+
for (const r of [...ftsResults, ...vecResults]) {
|
|
165
|
+
const existing = merged.get(r.filepath);
|
|
166
|
+
if (!existing || r.score > existing.score) {
|
|
167
|
+
merged.set(r.filepath, r);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
results = Array.from(merged.values());
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Enrich with SAME metadata + composite scoring
|
|
174
|
+
const enriched = enrichResults(store, results, query);
|
|
175
|
+
const scored = applyCompositeScoring(enriched, query, (path) => store.getCoActivated(path));
|
|
176
|
+
const diverse = applyMMRDiversity(scored);
|
|
177
|
+
const final = diverse.slice(0, limit);
|
|
178
|
+
|
|
179
|
+
if (compact) {
|
|
180
|
+
return jsonResponse({
|
|
181
|
+
query,
|
|
182
|
+
mode,
|
|
183
|
+
count: final.length,
|
|
184
|
+
results: final.map(r => ({
|
|
185
|
+
docid: r.docid,
|
|
186
|
+
path: r.displayPath,
|
|
187
|
+
title: r.title,
|
|
188
|
+
score: Math.round(r.compositeScore * 1000) / 1000,
|
|
189
|
+
contentType: r.contentType,
|
|
190
|
+
snippet: extractSnippet(r.body || "", query, 200).snippet,
|
|
191
|
+
})),
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return jsonResponse({
|
|
196
|
+
query,
|
|
197
|
+
mode,
|
|
198
|
+
count: final.length,
|
|
199
|
+
results: final.map(r => ({
|
|
200
|
+
docid: r.docid,
|
|
201
|
+
path: r.displayPath,
|
|
202
|
+
title: r.title,
|
|
203
|
+
score: Math.round(r.compositeScore * 1000) / 1000,
|
|
204
|
+
contentType: r.contentType,
|
|
205
|
+
modifiedAt: r.modifiedAt,
|
|
206
|
+
confidence: r.confidence,
|
|
207
|
+
body: r.body,
|
|
208
|
+
})),
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// --- Document by docid or path ---
|
|
213
|
+
|
|
214
|
+
function handleGetDocument(_req: Request, url: URL, store: Store): Response {
|
|
215
|
+
const docid = url.pathname.split("/").pop();
|
|
216
|
+
if (!docid) return jsonError("docid is required");
|
|
217
|
+
|
|
218
|
+
const result = store.findDocument(docid, { includeBody: true });
|
|
219
|
+
if ("error" in result) {
|
|
220
|
+
return jsonError(`Document not found: ${docid}`, 404);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return jsonResponse({
|
|
224
|
+
docid: result.docid,
|
|
225
|
+
path: result.displayPath,
|
|
226
|
+
title: result.title,
|
|
227
|
+
collection: result.collectionName,
|
|
228
|
+
modifiedAt: result.modifiedAt,
|
|
229
|
+
bodyLength: result.bodyLength,
|
|
230
|
+
body: result.body,
|
|
231
|
+
context: result.context,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// --- Multi-get by pattern ---
|
|
236
|
+
|
|
237
|
+
function handleGetDocuments(_req: Request, url: URL, store: Store): Response {
|
|
238
|
+
const pattern = queryParam(url, "pattern");
|
|
239
|
+
if (!pattern) return jsonError("pattern query parameter is required");
|
|
240
|
+
|
|
241
|
+
const maxBytes = queryInt(url, "max_bytes", 10240);
|
|
242
|
+
const { docs, errors } = store.findDocuments(pattern, { includeBody: true, maxBytes });
|
|
243
|
+
|
|
244
|
+
const resolved = docs.filter(d => !d.skipped).map(d => d.doc);
|
|
245
|
+
return jsonResponse({
|
|
246
|
+
pattern,
|
|
247
|
+
count: resolved.length,
|
|
248
|
+
errors,
|
|
249
|
+
documents: resolved.map(d => ({
|
|
250
|
+
docid: d.docid,
|
|
251
|
+
path: d.displayPath,
|
|
252
|
+
title: d.title,
|
|
253
|
+
collection: d.collectionName,
|
|
254
|
+
modifiedAt: d.modifiedAt,
|
|
255
|
+
bodyLength: d.bodyLength,
|
|
256
|
+
body: d.body,
|
|
257
|
+
})),
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// --- Timeline ---
|
|
262
|
+
|
|
263
|
+
function handleTimeline(_req: Request, url: URL, store: Store): Response {
|
|
264
|
+
const docid = url.pathname.split("/").pop();
|
|
265
|
+
if (!docid) return jsonError("docid is required");
|
|
266
|
+
|
|
267
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
268
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
269
|
+
|
|
270
|
+
const doc = store.db.prepare(
|
|
271
|
+
"SELECT id FROM documents WHERE hash = ? AND active = 1 LIMIT 1"
|
|
272
|
+
).get(resolved.hash) as { id: number } | undefined;
|
|
273
|
+
|
|
274
|
+
if (!doc) return jsonError(`Document not found: ${docid}`, 404);
|
|
275
|
+
|
|
276
|
+
const before = queryInt(url, "before", 5);
|
|
277
|
+
const after = queryInt(url, "after", 5);
|
|
278
|
+
const sameCollection = queryBool(url, "same_collection", false);
|
|
279
|
+
|
|
280
|
+
try {
|
|
281
|
+
const result = store.timeline(doc.id, { before, after, sameCollection });
|
|
282
|
+
return jsonResponse(result);
|
|
283
|
+
} catch (err: any) {
|
|
284
|
+
return jsonError(err.message, 404);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// --- Sessions ---
|
|
289
|
+
|
|
290
|
+
function handleSessions(_req: Request, url: URL, store: Store): Response {
|
|
291
|
+
const limit = queryInt(url, "limit", 10);
|
|
292
|
+
const sessions = store.getRecentSessions(limit);
|
|
293
|
+
return jsonResponse({ count: sessions.length, sessions });
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// --- Collections ---
|
|
297
|
+
|
|
298
|
+
function handleCollections(_req: Request, _url: URL, store: Store): Response {
|
|
299
|
+
const status = store.getStatus();
|
|
300
|
+
return jsonResponse({
|
|
301
|
+
count: status.collections.length,
|
|
302
|
+
collections: status.collections,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// --- Profile ---
|
|
307
|
+
|
|
308
|
+
function handleProfile(_req: Request, _url: URL, store: Store): Response {
|
|
309
|
+
// Search for profile doc
|
|
310
|
+
const profileResults = store.searchFTS("profile", 1);
|
|
311
|
+
if (profileResults.length === 0) {
|
|
312
|
+
return jsonResponse({ profile: null, message: "No profile found" });
|
|
313
|
+
}
|
|
314
|
+
const body = store.getDocumentBody(profileResults[0]!);
|
|
315
|
+
return jsonResponse({
|
|
316
|
+
path: profileResults[0]!.displayPath,
|
|
317
|
+
body,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// --- Causal Links ---
|
|
322
|
+
|
|
323
|
+
function handleCausalLinks(_req: Request, url: URL, store: Store): Response {
|
|
324
|
+
const docid = url.pathname.split("/").pop();
|
|
325
|
+
if (!docid) return jsonError("docid is required");
|
|
326
|
+
|
|
327
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
328
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
329
|
+
|
|
330
|
+
const doc = store.db.prepare(
|
|
331
|
+
"SELECT id FROM documents WHERE hash = ? AND active = 1 LIMIT 1"
|
|
332
|
+
).get(resolved.hash) as { id: number } | undefined;
|
|
333
|
+
if (!doc) return jsonError(`Document not found: ${docid}`, 404);
|
|
334
|
+
|
|
335
|
+
const direction = (queryParam(url, "direction", "both") as "causes" | "caused_by" | "both") || "both";
|
|
336
|
+
const depth = queryInt(url, "depth", 5);
|
|
337
|
+
|
|
338
|
+
const links = store.findCausalLinks(doc.id, direction, depth);
|
|
339
|
+
return jsonResponse({ docid, direction, depth, count: links.length, links });
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// --- Similar Documents ---
|
|
343
|
+
|
|
344
|
+
function handleSimilar(_req: Request, url: URL, store: Store): Response {
|
|
345
|
+
const docid = url.pathname.split("/").pop();
|
|
346
|
+
if (!docid) return jsonError("docid is required");
|
|
347
|
+
|
|
348
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
349
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
350
|
+
|
|
351
|
+
const limit = queryInt(url, "limit", 5);
|
|
352
|
+
const similar = store.findSimilarFiles(docid, undefined, limit);
|
|
353
|
+
|
|
354
|
+
return jsonResponse({ docid, count: similar.length, similar });
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// --- Evolution History ---
|
|
358
|
+
|
|
359
|
+
function handleEvolution(_req: Request, url: URL, store: Store): Response {
|
|
360
|
+
const docid = url.pathname.split("/").pop();
|
|
361
|
+
if (!docid) return jsonError("docid is required");
|
|
362
|
+
|
|
363
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
364
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
365
|
+
|
|
366
|
+
const doc = store.db.prepare(
|
|
367
|
+
"SELECT id, title FROM documents WHERE hash = ? AND active = 1 LIMIT 1"
|
|
368
|
+
).get(resolved.hash) as { id: number; title: string } | undefined;
|
|
369
|
+
if (!doc) return jsonError(`Document not found: ${docid}`, 404);
|
|
370
|
+
|
|
371
|
+
const limit = queryInt(url, "limit", 10);
|
|
372
|
+
const timeline = store.getEvolutionTimeline(doc.id, limit);
|
|
373
|
+
|
|
374
|
+
return jsonResponse({ docid, title: doc.title, count: timeline.length, evolution: timeline });
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// --- Lifecycle Status ---
|
|
378
|
+
|
|
379
|
+
function handleLifecycleStatus(_req: Request, _url: URL, store: Store): Response {
|
|
380
|
+
const stats = store.getLifecycleStats();
|
|
381
|
+
return jsonResponse(stats);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// --- Lifecycle Sweep ---
|
|
385
|
+
|
|
386
|
+
async function handleLifecycleSweep(req: Request, _url: URL, store: Store): Promise<Response> {
|
|
387
|
+
const body = await parseBody<{ dry_run?: boolean }>(req);
|
|
388
|
+
const dryRun = body?.dry_run ?? true;
|
|
389
|
+
|
|
390
|
+
// Load lifecycle policy from config
|
|
391
|
+
const { loadVaultConfig } = await import("./config.ts");
|
|
392
|
+
const config = loadVaultConfig();
|
|
393
|
+
const policy = config.lifecycle
|
|
394
|
+
?? { archive_after_days: 90, type_overrides: {}, purge_after_days: null, exempt_collections: [], dry_run: dryRun };
|
|
395
|
+
|
|
396
|
+
if (!policy) return jsonError("No lifecycle policy configured");
|
|
397
|
+
|
|
398
|
+
const candidates = store.getArchiveCandidates(policy);
|
|
399
|
+
|
|
400
|
+
if (dryRun) {
|
|
401
|
+
return jsonResponse({
|
|
402
|
+
dry_run: true,
|
|
403
|
+
candidates: candidates.length,
|
|
404
|
+
documents: candidates.map(c => ({
|
|
405
|
+
id: c.id,
|
|
406
|
+
path: `${c.collection}/${c.path}`,
|
|
407
|
+
title: c.title,
|
|
408
|
+
content_type: c.content_type,
|
|
409
|
+
modified_at: c.modified_at,
|
|
410
|
+
last_accessed_at: c.last_accessed_at,
|
|
411
|
+
})),
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
const archived = store.archiveDocuments(candidates.map(c => c.id));
|
|
416
|
+
return jsonResponse({ dry_run: false, archived });
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// --- Lifecycle Restore ---
|
|
420
|
+
|
|
421
|
+
async function handleLifecycleRestore(req: Request, _url: URL, store: Store): Promise<Response> {
|
|
422
|
+
const body = await parseBody<{ query?: string; collection?: string }>(req);
|
|
423
|
+
|
|
424
|
+
const filter: { ids?: number[]; collection?: string; sinceDate?: string } = {};
|
|
425
|
+
if (body?.collection) filter.collection = body.collection;
|
|
426
|
+
|
|
427
|
+
const restored = store.restoreArchivedDocuments(filter);
|
|
428
|
+
return jsonResponse({ restored });
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// --- Pin ---
|
|
432
|
+
|
|
433
|
+
async function handlePin(req: Request, url: URL, store: Store): Promise<Response> {
|
|
434
|
+
const docid = url.pathname.split("/").slice(-2, -1)[0];
|
|
435
|
+
if (!docid) return jsonError("docid is required");
|
|
436
|
+
|
|
437
|
+
const body = await parseBody<{ unpin?: boolean }>(req);
|
|
438
|
+
const unpin = body?.unpin ?? false;
|
|
439
|
+
|
|
440
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
441
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
442
|
+
|
|
443
|
+
const doc = store.db.prepare(
|
|
444
|
+
"SELECT id, collection, path FROM documents WHERE hash = ? AND active = 1 LIMIT 1"
|
|
445
|
+
).get(resolved.hash) as { id: number; collection: string; path: string } | undefined;
|
|
446
|
+
if (!doc) return jsonError(`Document not found: ${docid}`, 404);
|
|
447
|
+
|
|
448
|
+
store.pinDocument(doc.collection, doc.path, !unpin);
|
|
449
|
+
return jsonResponse({ docid, pinned: !unpin });
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// --- Snooze ---
|
|
453
|
+
|
|
454
|
+
async function handleSnooze(req: Request, url: URL, store: Store): Promise<Response> {
|
|
455
|
+
const docid = url.pathname.split("/").slice(-2, -1)[0];
|
|
456
|
+
if (!docid) return jsonError("docid is required");
|
|
457
|
+
|
|
458
|
+
const body = await parseBody<{ until?: string; unsnooze?: boolean }>(req);
|
|
459
|
+
|
|
460
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
461
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
462
|
+
|
|
463
|
+
const doc = store.db.prepare(
|
|
464
|
+
"SELECT id, collection, path FROM documents WHERE hash = ? AND active = 1 LIMIT 1"
|
|
465
|
+
).get(resolved.hash) as { id: number; collection: string; path: string } | undefined;
|
|
466
|
+
if (!doc) return jsonError(`Document not found: ${docid}`, 404);
|
|
467
|
+
|
|
468
|
+
const until = body?.unsnooze ? null : (body?.until ?? new Date(Date.now() + 30 * 86400000).toISOString());
|
|
469
|
+
store.snoozeDocument(doc.collection, doc.path, until);
|
|
470
|
+
return jsonResponse({ docid, snoozed: !body?.unsnooze, until });
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// --- Forget ---
|
|
474
|
+
|
|
475
|
+
async function handleForget(_req: Request, url: URL, store: Store): Promise<Response> {
|
|
476
|
+
const docid = url.pathname.split("/").slice(-2, -1)[0];
|
|
477
|
+
if (!docid) return jsonError("docid is required");
|
|
478
|
+
|
|
479
|
+
const resolved = store.findDocumentByDocid(docid);
|
|
480
|
+
if (!resolved) return jsonError(`Document not found: ${docid}`, 404);
|
|
481
|
+
|
|
482
|
+
const doc = store.db.prepare(
|
|
483
|
+
"SELECT id, collection, path FROM documents WHERE hash = ? AND active = 1 LIMIT 1"
|
|
484
|
+
).get(resolved.hash) as { id: number; collection: string; path: string } | undefined;
|
|
485
|
+
if (!doc) return jsonError(`Document not found: ${docid}`, 404);
|
|
486
|
+
|
|
487
|
+
store.deactivateDocument(doc.collection, doc.path);
|
|
488
|
+
return jsonResponse({ docid, forgotten: true });
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// --- Reindex ---
|
|
492
|
+
|
|
493
|
+
async function handleReindex(req: Request, _url: URL, store: Store): Promise<Response> {
|
|
494
|
+
const body = await parseBody<{ collection?: string }>(req);
|
|
495
|
+
|
|
496
|
+
const { indexCollection } = await import("./indexer.ts");
|
|
497
|
+
const collections = listCollections();
|
|
498
|
+
const targetCollections = body?.collection
|
|
499
|
+
? collections.filter(c => c.name === body.collection)
|
|
500
|
+
: collections;
|
|
501
|
+
|
|
502
|
+
if (targetCollections.length === 0) {
|
|
503
|
+
return jsonError(`Collection not found: ${body?.collection}`, 404);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
let totalAdded = 0, totalUpdated = 0, totalRemoved = 0;
|
|
507
|
+
|
|
508
|
+
for (const coll of targetCollections) {
|
|
509
|
+
const stats = await indexCollection(store, coll.name, coll.path, coll.pattern);
|
|
510
|
+
totalAdded += stats.added;
|
|
511
|
+
totalUpdated += stats.updated;
|
|
512
|
+
totalRemoved += stats.removed;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
return jsonResponse({
|
|
516
|
+
collections: targetCollections.length,
|
|
517
|
+
added: totalAdded,
|
|
518
|
+
updated: totalUpdated,
|
|
519
|
+
removed: totalRemoved,
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// --- Export ---
|
|
524
|
+
|
|
525
|
+
function handleExport(_req: Request, _url: URL, store: Store): Response {
|
|
526
|
+
const docs = store.db.prepare(`
|
|
527
|
+
SELECT d.id, d.collection, d.path, d.title, d.content_type, d.confidence,
|
|
528
|
+
d.access_count, d.quality_score, d.pinned, d.created_at, d.modified_at,
|
|
529
|
+
d.duplicate_count, d.revision_count, d.topic_key, d.normalized_hash,
|
|
530
|
+
c.doc as body
|
|
531
|
+
FROM documents d
|
|
532
|
+
JOIN content c ON c.hash = d.hash
|
|
533
|
+
WHERE d.active = 1
|
|
534
|
+
ORDER BY d.collection, d.path
|
|
535
|
+
`).all() as any[];
|
|
536
|
+
|
|
537
|
+
return jsonResponse({
|
|
538
|
+
version: "1.0.0",
|
|
539
|
+
exported_at: new Date().toISOString(),
|
|
540
|
+
count: docs.length,
|
|
541
|
+
documents: docs,
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// --- Build Graphs ---
|
|
546
|
+
|
|
547
|
+
async function handleBuildGraphs(req: Request, _url: URL, store: Store): Promise<Response> {
|
|
548
|
+
const body = await parseBody<{ temporal?: boolean; semantic?: boolean }>(req);
|
|
549
|
+
const doTemporal = body?.temporal ?? true;
|
|
550
|
+
const doSemantic = body?.semantic ?? true;
|
|
551
|
+
|
|
552
|
+
let temporalEdges = 0, semanticEdges = 0;
|
|
553
|
+
|
|
554
|
+
if (doTemporal) {
|
|
555
|
+
temporalEdges = store.buildTemporalBackbone();
|
|
556
|
+
}
|
|
557
|
+
if (doSemantic) {
|
|
558
|
+
semanticEdges = await store.buildSemanticGraph();
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
return jsonResponse({ temporal: temporalEdges, semantic: semanticEdges });
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// --- Unified Retrieve (mirrors memory_retrieve from MCP) ---
|
|
565
|
+
|
|
566
|
+
function classifyRetrievalMode(query: string): "keyword" | "semantic" | "causal" | "timeline" | "hybrid" {
|
|
567
|
+
const q = query.toLowerCase();
|
|
568
|
+
if (/\b(last session|yesterday|prior session|previous session|last time we|handoff|what happened last|what did we do)\b/i.test(q)) return "timeline";
|
|
569
|
+
if (/\b(why did|why was|what caused|what led to|reason for|decided to|decision about|trade.?off|chose to)\b/i.test(q) || /^why\b/i.test(q)) return "causal";
|
|
570
|
+
if (q.length < 50 && (/[A-Z][A-Z0-9_]{2,}/.test(query) || /[\w-]+\.\w{2,4}\b/.test(q.trim()))) return "keyword";
|
|
571
|
+
if (/\b(how does|explain|concept|overview|understand|what is the purpose)\b/i.test(q)) return "semantic";
|
|
572
|
+
return "hybrid";
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
async function handleRetrieve(req: Request, _url: URL, store: Store): Promise<Response> {
|
|
576
|
+
const body = await parseBody<{
|
|
577
|
+
query: string;
|
|
578
|
+
mode?: "auto" | "keyword" | "semantic" | "causal" | "timeline" | "hybrid";
|
|
579
|
+
collection?: string;
|
|
580
|
+
compact?: boolean;
|
|
581
|
+
limit?: number;
|
|
582
|
+
}>(req);
|
|
583
|
+
|
|
584
|
+
if (!body?.query) return jsonError("query is required");
|
|
585
|
+
|
|
586
|
+
const query = body.query;
|
|
587
|
+
const requestedMode = body.mode ?? "auto";
|
|
588
|
+
const mode = requestedMode === "auto" ? classifyRetrievalMode(query) : requestedMode;
|
|
589
|
+
const limit = Math.min(body.limit ?? 10, 50);
|
|
590
|
+
const compact = body.compact ?? true;
|
|
591
|
+
const collections = body.collection ? body.collection.split(",").map(c => c.trim()) : undefined;
|
|
592
|
+
|
|
593
|
+
let results: SearchResult[];
|
|
594
|
+
|
|
595
|
+
if (mode === "timeline") {
|
|
596
|
+
// Delegate to session log
|
|
597
|
+
const sessions = store.getRecentSessions(limit);
|
|
598
|
+
const lines = sessions.map(s => {
|
|
599
|
+
const dur = s.endedAt
|
|
600
|
+
? `${Math.round((new Date(s.endedAt).getTime() - new Date(s.startedAt).getTime()) / 60000)}min`
|
|
601
|
+
: "active";
|
|
602
|
+
return `${s.sessionId.slice(0, 8)} ${s.startedAt} (${dur})${s.summary ? " — " + s.summary.slice(0, 100) : ""}`;
|
|
603
|
+
});
|
|
604
|
+
return jsonResponse({ query, mode: "timeline", count: sessions.length, results: lines });
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
if (mode === "causal") {
|
|
608
|
+
// Intent-aware RRF: boost vector for causal queries
|
|
609
|
+
const llm = getDefaultLlamaCpp();
|
|
610
|
+
const intent = await classifyIntent(query, llm, store.db);
|
|
611
|
+
const bm25 = store.searchFTS(query, limit * 2, undefined, collections);
|
|
612
|
+
let vec: SearchResult[] = [];
|
|
613
|
+
try {
|
|
614
|
+
vec = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
|
|
615
|
+
} catch { /* vector unavailable */ }
|
|
616
|
+
const weights = intent.intent === "WHEN" ? [1.5, 1.0] : [1.0, 1.5];
|
|
617
|
+
const fused = reciprocalRankFusion([bm25.map(toRanked), vec.map(toRanked)], weights);
|
|
618
|
+
const allResults = [...bm25, ...vec];
|
|
619
|
+
results = fused.map(fr => {
|
|
620
|
+
const orig = allResults.find(r => r.filepath === fr.file);
|
|
621
|
+
return orig ? { ...orig, score: fr.score } : null;
|
|
622
|
+
}).filter((r): r is SearchResult => r !== null);
|
|
623
|
+
} else if (mode === "keyword") {
|
|
624
|
+
results = store.searchFTS(query, limit * 2, undefined, collections);
|
|
625
|
+
} else if (mode === "semantic") {
|
|
626
|
+
try {
|
|
627
|
+
results = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
|
|
628
|
+
} catch {
|
|
629
|
+
results = store.searchFTS(query, limit * 2, undefined, collections);
|
|
630
|
+
}
|
|
631
|
+
} else {
|
|
632
|
+
// hybrid
|
|
633
|
+
const fts = store.searchFTS(query, limit * 2, undefined, collections);
|
|
634
|
+
let vec: SearchResult[] = [];
|
|
635
|
+
try {
|
|
636
|
+
vec = await store.searchVec(query, DEFAULT_EMBED_MODEL, limit * 2, undefined, collections);
|
|
637
|
+
} catch { /* vector unavailable */ }
|
|
638
|
+
const merged = new Map<string, SearchResult>();
|
|
639
|
+
for (const r of [...fts, ...vec]) {
|
|
640
|
+
const existing = merged.get(r.filepath);
|
|
641
|
+
if (!existing || r.score > existing.score) merged.set(r.filepath, r);
|
|
642
|
+
}
|
|
643
|
+
results = Array.from(merged.values());
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
const enriched = enrichResults(store, results, query);
|
|
647
|
+
const scored = applyCompositeScoring(enriched, query, (path) => store.getCoActivated(path));
|
|
648
|
+
const diverse = applyMMRDiversity(scored);
|
|
649
|
+
const final = diverse.slice(0, limit);
|
|
650
|
+
|
|
651
|
+
if (compact) {
|
|
652
|
+
return jsonResponse({
|
|
653
|
+
query, mode, count: final.length,
|
|
654
|
+
results: final.map(r => ({
|
|
655
|
+
docid: r.docid, path: r.displayPath, title: r.title,
|
|
656
|
+
score: Math.round(r.compositeScore * 1000) / 1000,
|
|
657
|
+
contentType: r.contentType,
|
|
658
|
+
snippet: extractSnippet(r.body || "", query, 200).snippet,
|
|
659
|
+
})),
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
return jsonResponse({
|
|
664
|
+
query, mode, count: final.length,
|
|
665
|
+
results: final.map(r => ({
|
|
666
|
+
docid: r.docid, path: r.displayPath, title: r.title,
|
|
667
|
+
score: Math.round(r.compositeScore * 1000) / 1000,
|
|
668
|
+
contentType: r.contentType, modifiedAt: r.modifiedAt,
|
|
669
|
+
confidence: r.confidence, body: r.body,
|
|
670
|
+
})),
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// =============================================================================
|
|
675
|
+
// Router
|
|
676
|
+
// =============================================================================
|
|
677
|
+
|
|
678
|
+
type Route = {
|
|
679
|
+
method: string;
|
|
680
|
+
pattern: RegExp;
|
|
681
|
+
handler: RouteHandler;
|
|
682
|
+
};
|
|
683
|
+
|
|
684
|
+
const routes: Route[] = [
|
|
685
|
+
// Health & Stats
|
|
686
|
+
{ method: "GET", pattern: /^\/health$/, handler: handleHealth },
|
|
687
|
+
{ method: "GET", pattern: /^\/stats$/, handler: handleStats },
|
|
688
|
+
|
|
689
|
+
// Search & Retrieve
|
|
690
|
+
{ method: "POST", pattern: /^\/search$/, handler: handleSearch },
|
|
691
|
+
{ method: "POST", pattern: /^\/retrieve$/, handler: handleRetrieve },
|
|
692
|
+
|
|
693
|
+
// Documents
|
|
694
|
+
{ method: "GET", pattern: /^\/documents$/, handler: handleGetDocuments },
|
|
695
|
+
{ method: "GET", pattern: /^\/documents\/([^/]+)$/, handler: handleGetDocument },
|
|
696
|
+
|
|
697
|
+
// Timeline
|
|
698
|
+
{ method: "GET", pattern: /^\/timeline\/([^/]+)$/, handler: handleTimeline },
|
|
699
|
+
|
|
700
|
+
// Sessions
|
|
701
|
+
{ method: "GET", pattern: /^\/sessions$/, handler: handleSessions },
|
|
702
|
+
|
|
703
|
+
// Collections
|
|
704
|
+
{ method: "GET", pattern: /^\/collections$/, handler: handleCollections },
|
|
705
|
+
|
|
706
|
+
// Profile
|
|
707
|
+
{ method: "GET", pattern: /^\/profile$/, handler: handleProfile },
|
|
708
|
+
|
|
709
|
+
// Graph
|
|
710
|
+
{ method: "GET", pattern: /^\/graph\/causal\/([^/]+)$/, handler: handleCausalLinks },
|
|
711
|
+
{ method: "GET", pattern: /^\/graph\/similar\/([^/]+)$/,handler: handleSimilar },
|
|
712
|
+
{ method: "GET", pattern: /^\/graph\/evolution\/([^/]+)$/,handler: handleEvolution },
|
|
713
|
+
|
|
714
|
+
// Lifecycle
|
|
715
|
+
{ method: "GET", pattern: /^\/lifecycle\/status$/, handler: handleLifecycleStatus },
|
|
716
|
+
{ method: "POST", pattern: /^\/lifecycle\/sweep$/, handler: handleLifecycleSweep },
|
|
717
|
+
{ method: "POST", pattern: /^\/lifecycle\/restore$/, handler: handleLifecycleRestore },
|
|
718
|
+
|
|
719
|
+
// Document mutations
|
|
720
|
+
{ method: "POST", pattern: /^\/documents\/([^/]+)\/pin$/, handler: handlePin },
|
|
721
|
+
{ method: "POST", pattern: /^\/documents\/([^/]+)\/snooze$/, handler: handleSnooze },
|
|
722
|
+
{ method: "POST", pattern: /^\/documents\/([^/]+)\/forget$/, handler: handleForget },
|
|
723
|
+
|
|
724
|
+
// Maintenance
|
|
725
|
+
{ method: "POST", pattern: /^\/reindex$/, handler: handleReindex },
|
|
726
|
+
{ method: "POST", pattern: /^\/graphs\/build$/, handler: handleBuildGraphs },
|
|
727
|
+
|
|
728
|
+
// Export
|
|
729
|
+
{ method: "GET", pattern: /^\/export$/, handler: handleExport },
|
|
730
|
+
];
|
|
731
|
+
|
|
732
|
+
function matchRoute(method: string, pathname: string): RouteHandler | null {
|
|
733
|
+
for (const route of routes) {
|
|
734
|
+
if (route.method === method && route.pattern.test(pathname)) {
|
|
735
|
+
return route.handler;
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
return null;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// =============================================================================
|
|
742
|
+
// Server
|
|
743
|
+
// =============================================================================
|
|
744
|
+
|
|
745
|
+
export function startServer(store: Store, port: number = 7438, host: string = "127.0.0.1") {
|
|
746
|
+
return Bun.serve({
|
|
747
|
+
port,
|
|
748
|
+
hostname: host,
|
|
749
|
+
async fetch(req) {
|
|
750
|
+
const url = new URL(req.url);
|
|
751
|
+
|
|
752
|
+
// CORS preflight
|
|
753
|
+
if (req.method === "OPTIONS") {
|
|
754
|
+
return new Response(null, {
|
|
755
|
+
status: 204,
|
|
756
|
+
headers: {
|
|
757
|
+
"Access-Control-Allow-Origin": "*",
|
|
758
|
+
"Access-Control-Allow-Methods": "GET, POST, OPTIONS",
|
|
759
|
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
|
760
|
+
"Access-Control-Max-Age": "86400",
|
|
761
|
+
},
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
// Auth check
|
|
766
|
+
const authError = checkAuth(req);
|
|
767
|
+
if (authError) return authError;
|
|
768
|
+
|
|
769
|
+
// Route matching
|
|
770
|
+
const handler = matchRoute(req.method, url.pathname);
|
|
771
|
+
if (!handler) {
|
|
772
|
+
return jsonError(`Not found: ${req.method} ${url.pathname}`, 404);
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
try {
|
|
776
|
+
return await handler(req, url, store);
|
|
777
|
+
} catch (err: any) {
|
|
778
|
+
console.error(`[clawmem-server] ${req.method} ${url.pathname} error:`, err);
|
|
779
|
+
return jsonError(`Internal error: ${err.message}`, 500);
|
|
780
|
+
}
|
|
781
|
+
},
|
|
782
|
+
});
|
|
783
|
+
}
|