@agfpd/iapeer-memory-core 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +32 -0
- package/src/config.ts +257 -0
- package/src/context-render.ts +185 -0
- package/src/db.ts +550 -0
- package/src/embedding.ts +174 -0
- package/src/fm-update.ts +352 -0
- package/src/frontmatter-fill.ts +529 -0
- package/src/graph.ts +427 -0
- package/src/http-client.ts +129 -0
- package/src/human-edit-detect.ts +213 -0
- package/src/index-render.ts +876 -0
- package/src/index.ts +65 -0
- package/src/indexer.ts +323 -0
- package/src/log.ts +27 -0
- package/src/mcp-tools.ts +468 -0
- package/src/memoryd.ts +680 -0
- package/src/migrate-auto-memory.ts +289 -0
- package/src/parser.ts +269 -0
- package/src/permanent-detect.ts +110 -0
- package/src/render-doctrine.ts +113 -0
- package/src/reranker.ts +162 -0
- package/src/search.ts +806 -0
- package/src/smart-hash.ts +85 -0
- package/src/sqlite-loader.ts +151 -0
- package/src/tags-mirror.ts +47 -0
- package/src/taxonomy.ts +385 -0
- package/src/utils.ts +69 -0
- package/tsconfig.json +24 -0
package/src/memoryd.ts
ADDED
|
@@ -0,0 +1,680 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* memoryd — the single iapeer-memory daemon (ADR-004 + ADR-012).
|
|
3
|
+
*
|
|
4
|
+
* One process owns everything live:
|
|
5
|
+
* - the WRITER role: sole SQLite owner (openDatabase + indexAll), fs.watch
|
|
6
|
+
* over the vault with debounce, incremental re-index by content hash;
|
|
7
|
+
* - the detect subsystems (stage 8 cores): human-edit attribution,
|
|
8
|
+
* tags-dictionary mirror, permanent smart-hash diff with COALESCED
|
|
9
|
+
* events; INBOX_NEW on new inbox files (baseline at start — pre-existing
|
|
10
|
+
* files never replay, parity with the reference monitor);
|
|
11
|
+
* - the event stream: signal lines on stdout (`INBOX_NEW: …`,
|
|
12
|
+
* `PERMANENT_CHANGED: …`) — a notifier watcher forwards each line to the
|
|
13
|
+
* Index as an IAP signal (docs/06-pipelines-and-events.md);
|
|
14
|
+
* - the heartbeat state file (consumer: every peer's SessionStart
|
|
15
|
+
* health-check, ADR-009/010);
|
|
16
|
+
* - the MCP-http endpoint (ADR-012): localhost, port from config, three
|
|
17
|
+
* read-only tools (vault_search / vault_graph / vault_map — ADR-008,
|
|
18
|
+
* vault_read is NOT on the surface), caller identity from the
|
|
19
|
+
* `X-IAPeer-Identity` header per request.
|
|
20
|
+
*
|
|
21
|
+
* The supervision (restart, hang detection, alerts) belongs to the
|
|
22
|
+
* notifier watcher registration owned by the package (ADR-010) — memoryd
|
|
23
|
+
* itself stays a plain long-running process with clean shutdown.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import fs from "node:fs";
|
|
27
|
+
import http from "node:http";
|
|
28
|
+
import os from "node:os";
|
|
29
|
+
import path from "node:path";
|
|
30
|
+
|
|
31
|
+
import corePkg from "../package.json";
|
|
32
|
+
|
|
33
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
34
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
35
|
+
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
|
36
|
+
import { z } from "zod";
|
|
37
|
+
|
|
38
|
+
import type { CoreConfig } from "./config.js";
|
|
39
|
+
import { openDatabase, type CoreDb } from "./db.js";
|
|
40
|
+
import { indexAll } from "./indexer.js";
|
|
41
|
+
import { runSearch, runGraph, runMap } from "./mcp-tools.js";
|
|
42
|
+
import { decideUpdate, getZone } from "./human-edit-detect.js";
|
|
43
|
+
import { decideMirror, tagsDictionarySourceRel } from "./tags-mirror.js";
|
|
44
|
+
import {
|
|
45
|
+
snapshotVault,
|
|
46
|
+
detectPermanentChanges,
|
|
47
|
+
formatEventLines,
|
|
48
|
+
type VaultSnapshot,
|
|
49
|
+
} from "./permanent-detect.js";
|
|
50
|
+
import { makeLogger, type Logger } from "./log.js";
|
|
51
|
+
|
|
52
|
+
// ── identity ────────────────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
const KNOWN_RUNTIME_PREFIXES = ["claude-", "codex-"];
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* `X-IAPeer-Identity: <runtime>-<personality>` → personality. A bare
|
|
58
|
+
* personality (no known runtime prefix) passes through as-is; personalities
|
|
59
|
+
* may themselves contain dashes (`iapeer-memory`), so ONLY the known
|
|
60
|
+
* runtime prefixes are stripped. Missing/empty header → null (admin access,
|
|
61
|
+
* identity-dependent mechanics like the foreign-memory penalty are off).
|
|
62
|
+
*/
|
|
63
|
+
export function parsePersonalityFromIdentity(
|
|
64
|
+
header: string | null | undefined,
|
|
65
|
+
): string | null {
|
|
66
|
+
const v = (header ?? "").trim();
|
|
67
|
+
if (!v) return null;
|
|
68
|
+
for (const prefix of KNOWN_RUNTIME_PREFIXES) {
|
|
69
|
+
if (v.startsWith(prefix) && v.length > prefix.length) {
|
|
70
|
+
return v.slice(prefix.length);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return v;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// ── MCP server (three tools, ADR-008) ───────────────────────────────────────
|
|
77
|
+
|
|
78
|
+
function toResult(payload: unknown): CallToolResult {
|
|
79
|
+
const structured =
|
|
80
|
+
payload && typeof payload === "object" && !Array.isArray(payload)
|
|
81
|
+
? (payload as Record<string, unknown>)
|
|
82
|
+
: undefined;
|
|
83
|
+
return {
|
|
84
|
+
content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
|
|
85
|
+
...(structured ? { structuredContent: structured } : {}),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function toError(toolName: string, err: unknown, logger: Logger): CallToolResult {
|
|
90
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
91
|
+
logger.error(`tool ${toolName} failed: ${msg}`);
|
|
92
|
+
return {
|
|
93
|
+
content: [{ type: "text", text: JSON.stringify({ error: msg }, null, 2) }],
|
|
94
|
+
isError: true,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Zod fragments — ported from the reference server (vault_read schema
|
|
99
|
+
// deliberately absent, ADR-008).
|
|
100
|
+
const relatedItem = z.object({
|
|
101
|
+
path: z.string(),
|
|
102
|
+
title: z.string(),
|
|
103
|
+
direction: z.string(),
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
const searchResultItem = z.object({
|
|
107
|
+
title: z.string(),
|
|
108
|
+
path: z.string(),
|
|
109
|
+
type: z.string().nullable(),
|
|
110
|
+
status: z.string().nullable(),
|
|
111
|
+
score: z.number(),
|
|
112
|
+
snippet: z.string(),
|
|
113
|
+
related: z.array(relatedItem),
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
const pipelineStatusSchema = z.object({
|
|
117
|
+
bm25: z.string(),
|
|
118
|
+
embedding: z.string(),
|
|
119
|
+
reranker: z.string(),
|
|
120
|
+
graph: z.string(),
|
|
121
|
+
caller_agent: z.string().nullable(),
|
|
122
|
+
for_curation: z.boolean(),
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const vaultSearchOutput = z.object({
|
|
126
|
+
query: z.string(),
|
|
127
|
+
results: z.array(searchResultItem),
|
|
128
|
+
pipeline: pipelineStatusSchema,
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const graphNode = z.object({
|
|
132
|
+
path: z.string(),
|
|
133
|
+
title: z.string(),
|
|
134
|
+
type: z.string().nullable(),
|
|
135
|
+
status: z.string().nullable(),
|
|
136
|
+
depth: z.number(),
|
|
137
|
+
direction: z.string(),
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
const graphEdge = z.object({ from: z.string(), to: z.string() });
|
|
141
|
+
|
|
142
|
+
const vaultGraphOutput = z.object({
|
|
143
|
+
center: z
|
|
144
|
+
.object({ path: z.string(), title: z.string(), type: z.string().nullable() })
|
|
145
|
+
.optional(),
|
|
146
|
+
nodes: z.array(graphNode).optional(),
|
|
147
|
+
edges: z.array(graphEdge).optional(),
|
|
148
|
+
stats: z
|
|
149
|
+
.object({ totalNodes: z.number(), totalEdges: z.number(), depth: z.number() })
|
|
150
|
+
.optional(),
|
|
151
|
+
found: z.boolean().optional(),
|
|
152
|
+
error: z.string().optional(),
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
const mapPart = z.enum(["clusters", "hubs", "bridges", "orphans", "orphan_wikilinks"]);
|
|
156
|
+
|
|
157
|
+
const mapClusterFull = z.object({
|
|
158
|
+
name: z.string(),
|
|
159
|
+
size: z.number(),
|
|
160
|
+
hub: z.object({ title: z.string(), degree: z.number() }).nullable(),
|
|
161
|
+
nodes: z.array(z.string()),
|
|
162
|
+
});
|
|
163
|
+
const mapClusterSummary = z.object({
|
|
164
|
+
name: z.string(),
|
|
165
|
+
size: z.number(),
|
|
166
|
+
hub: z.object({ title: z.string(), degree: z.number() }).nullable(),
|
|
167
|
+
top_nodes: z.array(z.string()),
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
const mapHub = z.object({
|
|
171
|
+
title: z.string(),
|
|
172
|
+
in: z.number(),
|
|
173
|
+
out: z.number(),
|
|
174
|
+
total: z.number(),
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
const mapBridge = z.object({ title: z.string(), connects: z.array(z.string()) });
|
|
178
|
+
|
|
179
|
+
const vaultMapOutput = z.object({
|
|
180
|
+
generated: z.string(),
|
|
181
|
+
stats: z.object({
|
|
182
|
+
documents: z.number(),
|
|
183
|
+
edges: z.number(),
|
|
184
|
+
clusters: z.number(),
|
|
185
|
+
hubs: z.number(),
|
|
186
|
+
bridges: z.number(),
|
|
187
|
+
orphans: z.number(),
|
|
188
|
+
orphan_wikilinks: z.number(),
|
|
189
|
+
}),
|
|
190
|
+
detail: z.enum(["summary", "full"]),
|
|
191
|
+
parts: z.array(mapPart),
|
|
192
|
+
clusters: z.array(z.union([mapClusterFull, mapClusterSummary])).optional(),
|
|
193
|
+
hubs: z.array(mapHub).optional(),
|
|
194
|
+
hubs_truncated: z.number().optional(),
|
|
195
|
+
bridges: z.array(mapBridge).optional(),
|
|
196
|
+
orphans: z.array(z.string()).optional(),
|
|
197
|
+
orphan_wikilinks: z
|
|
198
|
+
.array(z.object({ source: z.string(), target: z.string(), reason: z.string() }))
|
|
199
|
+
.optional(),
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
export const MEMORYD_SERVER_NAME = "iapeer-memory";
|
|
203
|
+
|
|
204
|
+
function readCoreVersion(): string {
|
|
205
|
+
// STATIC json import (embedded by the bundler), not a runtime fs read:
|
|
206
|
+
// under `bun build --compile` import.meta.url resolves into /$bunfs/ where
|
|
207
|
+
// ../package.json does not exist (P3a compile fact-check).
|
|
208
|
+
try {
|
|
209
|
+
const v = (corePkg as { version?: unknown }).version;
|
|
210
|
+
if (typeof v === "string" && v.length > 0) return v;
|
|
211
|
+
} catch {
|
|
212
|
+
// fall through
|
|
213
|
+
}
|
|
214
|
+
return "0.0.0";
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Build an MCP server instance with the three-tool surface. `identity` is
|
|
219
|
+
* the caller personality from the http header — it OVERRIDES the config's
|
|
220
|
+
* env-fallback callerAgent for this connection (ADR-012).
|
|
221
|
+
*/
|
|
222
|
+
export function createMcpServer(opts: {
|
|
223
|
+
db: CoreDb;
|
|
224
|
+
config: CoreConfig;
|
|
225
|
+
identity?: string | null;
|
|
226
|
+
logger?: Logger;
|
|
227
|
+
}): McpServer {
|
|
228
|
+
const { db, config } = opts;
|
|
229
|
+
const logger = opts.logger ?? makeLogger("memoryd");
|
|
230
|
+
const effectiveConfig: CoreConfig = {
|
|
231
|
+
...config,
|
|
232
|
+
callerAgent: opts.identity ?? config.callerAgent,
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
const server = new McpServer(
|
|
236
|
+
{ name: MEMORYD_SERVER_NAME, version: readCoreVersion() },
|
|
237
|
+
{
|
|
238
|
+
instructions:
|
|
239
|
+
"iapeer-memory vault — shared team memory. vault_search is the default " +
|
|
240
|
+
"entry point when you don't have an exact path; read notes with the " +
|
|
241
|
+
"native Read tool after search. Stale-status notes are score-deboosted " +
|
|
242
|
+
"but still returned — treat them as history, not current truth.",
|
|
243
|
+
},
|
|
244
|
+
);
|
|
245
|
+
|
|
246
|
+
server.registerTool(
|
|
247
|
+
"vault_search",
|
|
248
|
+
{
|
|
249
|
+
description:
|
|
250
|
+
"Search the vault for knowledge, decisions and context when you don't know the exact note path. " +
|
|
251
|
+
"Hybrid pipeline: BM25 + optional vector/rerank providers → RRF fusion → status boost → 1-hop graph expansion → backlink boost. " +
|
|
252
|
+
"Returns up to max_results items (title, path, type, status, score, snippet, related). " +
|
|
253
|
+
"Read the found note with the native Read tool. " +
|
|
254
|
+
"The `pipeline` object reports per-component status (ok/disabled/skipped/timeout/error/circuit-open).",
|
|
255
|
+
inputSchema: {
|
|
256
|
+
query: z.string().min(1, "vault_search: query is required"),
|
|
257
|
+
forCuration: z.boolean().optional(),
|
|
258
|
+
},
|
|
259
|
+
outputSchema: vaultSearchOutput.shape,
|
|
260
|
+
annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: true },
|
|
261
|
+
},
|
|
262
|
+
async ({ query, forCuration }) => {
|
|
263
|
+
try {
|
|
264
|
+
return toResult(await runSearch(db, effectiveConfig, { query, forCuration }));
|
|
265
|
+
} catch (err) {
|
|
266
|
+
return toError("vault_search", err, logger);
|
|
267
|
+
}
|
|
268
|
+
},
|
|
269
|
+
);
|
|
270
|
+
|
|
271
|
+
server.registerTool(
|
|
272
|
+
"vault_graph",
|
|
273
|
+
{
|
|
274
|
+
description:
|
|
275
|
+
"Walk the wikilink graph outward from a note, 1–3 hops in both directions. " +
|
|
276
|
+
"Returns nodes and edges of the subgraph; agent-memory backlinks of canonical notes are filtered (one-way). " +
|
|
277
|
+
"Unknown center path → {found:false}.",
|
|
278
|
+
inputSchema: {
|
|
279
|
+
path: z.string().min(1, "vault_graph: path is required"),
|
|
280
|
+
depth: z.number().int().min(1).max(3).optional(),
|
|
281
|
+
},
|
|
282
|
+
outputSchema: vaultGraphOutput.shape,
|
|
283
|
+
annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: false },
|
|
284
|
+
},
|
|
285
|
+
async ({ path: p, depth }) => {
|
|
286
|
+
try {
|
|
287
|
+
return toResult(runGraph(db, effectiveConfig, { path: p, depth }));
|
|
288
|
+
} catch (err) {
|
|
289
|
+
return toError("vault_graph", err, logger);
|
|
290
|
+
}
|
|
291
|
+
},
|
|
292
|
+
);
|
|
293
|
+
|
|
294
|
+
server.registerTool(
|
|
295
|
+
"vault_map",
|
|
296
|
+
{
|
|
297
|
+
description:
|
|
298
|
+
"Global topology of the CANONICAL vault graph (agent memory excluded): " +
|
|
299
|
+
"clusters, hubs, bridges, orphans + orphan_wikilinks (opt-in part). " +
|
|
300
|
+
"Stats always returned.",
|
|
301
|
+
inputSchema: {
|
|
302
|
+
detail: z.enum(["summary", "full"]).optional(),
|
|
303
|
+
parts: z.array(mapPart).optional(),
|
|
304
|
+
},
|
|
305
|
+
outputSchema: vaultMapOutput.shape,
|
|
306
|
+
annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: false },
|
|
307
|
+
},
|
|
308
|
+
async ({ detail, parts }) => {
|
|
309
|
+
try {
|
|
310
|
+
return toResult(runMap(db, effectiveConfig, { detail, parts }));
|
|
311
|
+
} catch (err) {
|
|
312
|
+
return toError("vault_map", err, logger);
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
);
|
|
316
|
+
|
|
317
|
+
return server;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Start the MCP-http endpoint (ADR-012): stateless transport, one
|
|
322
|
+
* server+transport pair per request, identity from the
|
|
323
|
+
* `X-IAPeer-Identity` header. Returns the bound port and a closer.
|
|
324
|
+
*/
|
|
325
|
+
export async function startMcpHttp(opts: {
|
|
326
|
+
db: CoreDb;
|
|
327
|
+
config: CoreConfig;
|
|
328
|
+
port?: number;
|
|
329
|
+
host?: string;
|
|
330
|
+
logger?: Logger;
|
|
331
|
+
}): Promise<{ port: number; close: () => Promise<void> }> {
|
|
332
|
+
const logger = opts.logger ?? makeLogger("memoryd");
|
|
333
|
+
const host = opts.host ?? "127.0.0.1";
|
|
334
|
+
|
|
335
|
+
const httpServer = http.createServer((req, res) => {
|
|
336
|
+
void (async () => {
|
|
337
|
+
const url = (req.url ?? "").split("?")[0];
|
|
338
|
+
if (url !== "/mcp") {
|
|
339
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
340
|
+
res.end(JSON.stringify({ error: "not found; MCP endpoint is /mcp" }));
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
const identity = parsePersonalityFromIdentity(
|
|
344
|
+
(req.headers["x-iapeer-identity"] as string | undefined) ?? null,
|
|
345
|
+
);
|
|
346
|
+
const server = createMcpServer({ db: opts.db, config: opts.config, identity, logger });
|
|
347
|
+
const transport = new StreamableHTTPServerTransport({
|
|
348
|
+
sessionIdGenerator: undefined, // stateless: no session tracking
|
|
349
|
+
});
|
|
350
|
+
res.on("close", () => {
|
|
351
|
+
void transport.close();
|
|
352
|
+
void server.close();
|
|
353
|
+
});
|
|
354
|
+
try {
|
|
355
|
+
await server.connect(transport);
|
|
356
|
+
await transport.handleRequest(req, res);
|
|
357
|
+
} catch (err) {
|
|
358
|
+
logger.error(`mcp http request failed: ${String(err)}`);
|
|
359
|
+
if (!res.headersSent) {
|
|
360
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
361
|
+
res.end(JSON.stringify({ error: "internal error" }));
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
})();
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
await new Promise<void>((resolve, reject) => {
|
|
368
|
+
httpServer.once("error", reject);
|
|
369
|
+
httpServer.listen(opts.port ?? 0, host, () => resolve());
|
|
370
|
+
});
|
|
371
|
+
const address = httpServer.address();
|
|
372
|
+
const port = typeof address === "object" && address ? address.port : (opts.port ?? 0);
|
|
373
|
+
logger.info(`MCP http listening on http://${host}:${port}/mcp`);
|
|
374
|
+
|
|
375
|
+
return {
|
|
376
|
+
port,
|
|
377
|
+
close: () =>
|
|
378
|
+
new Promise<void>((resolve) => {
|
|
379
|
+
// Kill keep-alive/SSE connections FIRST — otherwise close() waits
|
|
380
|
+
// for them and a daemon shutdown hangs for the keep-alive timeout.
|
|
381
|
+
httpServer.closeAllConnections?.();
|
|
382
|
+
httpServer.closeIdleConnections?.();
|
|
383
|
+
const fallback = setTimeout(resolve, 1500);
|
|
384
|
+
(fallback as { unref?: () => void }).unref?.();
|
|
385
|
+
httpServer.close(() => {
|
|
386
|
+
clearTimeout(fallback);
|
|
387
|
+
resolve();
|
|
388
|
+
});
|
|
389
|
+
}),
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// ── detect-hash persistence (stage-9 review note 5) ─────────────────────────
|
|
394
|
+
//
|
|
395
|
+
// Without persistence a daemon restart zeroed the human-edit detector's
|
|
396
|
+
// last-seen map: the first event per file skipped the content-hash guard.
|
|
397
|
+
// The risk is LOWER here than in the reference (the startup full-scan
|
|
398
|
+
// catches up indexing; the echo window suppresses spurious re-stamps), but
|
|
399
|
+
// a sync-storm right after a restart could still mis-attribute — so the
|
|
400
|
+
// baseline survives restarts, atomically (same tmp+rename canon).
|
|
401
|
+
|
|
402
|
+
export function loadHashState(filePath: string): Map<string, string> {
|
|
403
|
+
const map = new Map<string, string>();
|
|
404
|
+
try {
|
|
405
|
+
const obj = JSON.parse(fs.readFileSync(filePath, "utf-8")) as Record<string, unknown>;
|
|
406
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
407
|
+
if (typeof v === "string" && /^[a-f0-9]{64}$/.test(v)) map.set(k, v);
|
|
408
|
+
}
|
|
409
|
+
} catch {
|
|
410
|
+
// first run / corrupt file — start empty
|
|
411
|
+
}
|
|
412
|
+
return map;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
export function persistHashState(filePath: string, map: Map<string, string>): void {
|
|
416
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
417
|
+
const tmp = `${filePath}.tmp`;
|
|
418
|
+
fs.writeFileSync(tmp, JSON.stringify(Object.fromEntries(map)), "utf-8");
|
|
419
|
+
fs.renameSync(tmp, filePath);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// ── the daemon ───────────────────────────────────────────────────────────────
|
|
423
|
+
|
|
424
|
+
export type MemorydOptions = {
|
|
425
|
+
config: CoreConfig;
|
|
426
|
+
logger?: Logger;
|
|
427
|
+
/** Event sink; default writes signal lines to stdout. */
|
|
428
|
+
emit?: (line: string) => void;
|
|
429
|
+
/** Debounce for fs events (ms). */
|
|
430
|
+
debounceMs?: number;
|
|
431
|
+
/** Heartbeat period (ms). */
|
|
432
|
+
heartbeatMs?: number;
|
|
433
|
+
/** Heartbeat file; default `<db dir>/memoryd.heartbeat`. */
|
|
434
|
+
heartbeatPath?: string;
|
|
435
|
+
/** Tags mirror target; default `<db dir>/tags-dictionary.md`. */
|
|
436
|
+
tagsMirrorPath?: string;
|
|
437
|
+
/** Detect-hash persistence file; default `<db dir>/memoryd.hashes.json`. */
|
|
438
|
+
hashStatePath?: string;
|
|
439
|
+
/** Periodic hash-persist interval (ms). */
|
|
440
|
+
persistMs?: number;
|
|
441
|
+
/** Human owner name; human-edit detection is OFF when absent (⚖7). */
|
|
442
|
+
humanName?: string | null;
|
|
443
|
+
freshEditWindowS?: number;
|
|
444
|
+
/**
|
|
445
|
+
* MCP http port (0 = ephemeral). Pass null to disable the endpoint;
|
|
446
|
+
* omit to use `config.mcp.port` (the configured default, ADR-012).
|
|
447
|
+
*/
|
|
448
|
+
mcpPort?: number | null;
|
|
449
|
+
};
|
|
450
|
+
|
|
451
|
+
export type MemorydHandle = {
|
|
452
|
+
mcpPort: number | null;
|
|
453
|
+
/** Force one detect pass immediately (used by tests and shutdown flush). */
|
|
454
|
+
runDetectPass: () => Promise<void>;
|
|
455
|
+
close: () => Promise<void>;
|
|
456
|
+
};
|
|
457
|
+
|
|
458
|
+
export async function startMemoryd(opts: MemorydOptions): Promise<MemorydHandle> {
|
|
459
|
+
const { config } = opts;
|
|
460
|
+
const logger = opts.logger ?? makeLogger("memoryd");
|
|
461
|
+
const emit = opts.emit ?? ((line: string) => process.stdout.write(`${line}\n`));
|
|
462
|
+
const debounceMs = opts.debounceMs ?? 1500;
|
|
463
|
+
const heartbeatMs = opts.heartbeatMs ?? 30_000;
|
|
464
|
+
const dbDir = path.dirname(config.index.dbPath);
|
|
465
|
+
const heartbeatPath = opts.heartbeatPath ?? path.join(dbDir, "memoryd.heartbeat");
|
|
466
|
+
const tagsMirrorPath = opts.tagsMirrorPath ?? path.join(dbDir, "tags-dictionary.md");
|
|
467
|
+
const hashStatePath = opts.hashStatePath ?? path.join(dbDir, "memoryd.hashes.json");
|
|
468
|
+
const persistMs = opts.persistMs ?? 60_000;
|
|
469
|
+
const taxonomy = config.taxonomy;
|
|
470
|
+
|
|
471
|
+
const db = openDatabase(config);
|
|
472
|
+
await indexAll({ db, config, logger });
|
|
473
|
+
|
|
474
|
+
// Baselines: detect from NOW on; pre-existing inbox files do not replay
|
|
475
|
+
// (the Index catches them up at its own start — reference semantics).
|
|
476
|
+
let snapshot: VaultSnapshot = snapshotVault(config.vaultPath, taxonomy);
|
|
477
|
+
const inboxDir = path.join(config.vaultPath, taxonomy.folders.inbox);
|
|
478
|
+
const inboxSeen = new Set<string>(
|
|
479
|
+
fs.existsSync(inboxDir) ? fs.readdirSync(inboxDir).filter((f) => f.endsWith(".md")) : [],
|
|
480
|
+
);
|
|
481
|
+
const lastSeenHashes = loadHashState(hashStatePath);
|
|
482
|
+
|
|
483
|
+
syncTagsMirror(); // best-effort materialisation at start
|
|
484
|
+
|
|
485
|
+
function syncTagsMirror(): void {
|
|
486
|
+
const srcPath = path.join(config.vaultPath, tagsDictionarySourceRel(taxonomy));
|
|
487
|
+
let srcContent: string | null = null;
|
|
488
|
+
try {
|
|
489
|
+
srcContent = fs.readFileSync(srcPath, "utf-8");
|
|
490
|
+
} catch {
|
|
491
|
+
srcContent = null;
|
|
492
|
+
}
|
|
493
|
+
let mirrorContent: string | null = null;
|
|
494
|
+
try {
|
|
495
|
+
mirrorContent = fs.readFileSync(tagsMirrorPath, "utf-8");
|
|
496
|
+
} catch {
|
|
497
|
+
mirrorContent = null;
|
|
498
|
+
}
|
|
499
|
+
const decision = decideMirror({ srcContent, mirrorContent });
|
|
500
|
+
if (decision.action !== "write") return;
|
|
501
|
+
fs.mkdirSync(path.dirname(tagsMirrorPath), { recursive: true });
|
|
502
|
+
const tmp = `${tagsMirrorPath}.tmp`;
|
|
503
|
+
fs.writeFileSync(tmp, srcContent!, "utf-8");
|
|
504
|
+
fs.renameSync(tmp, tagsMirrorPath);
|
|
505
|
+
logger.info(`tags mirror updated (${decision.reason})`);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
function humanEditPass(changedAbs: Set<string>): void {
|
|
509
|
+
const human = opts.humanName ?? null;
|
|
510
|
+
if (!human) return; // ⚖7: no human role — detection off
|
|
511
|
+
for (const filePath of changedAbs) {
|
|
512
|
+
const zone = getZone(filePath, config.vaultPath, taxonomy);
|
|
513
|
+
if (!zone) continue;
|
|
514
|
+
let content: string;
|
|
515
|
+
let stat: fs.Stats;
|
|
516
|
+
try {
|
|
517
|
+
content = fs.readFileSync(filePath, "utf-8");
|
|
518
|
+
stat = fs.statSync(filePath);
|
|
519
|
+
} catch {
|
|
520
|
+
continue; // deleted mid-debounce
|
|
521
|
+
}
|
|
522
|
+
const decision = decideUpdate({
|
|
523
|
+
content,
|
|
524
|
+
zone,
|
|
525
|
+
human,
|
|
526
|
+
nowMs: Date.now(),
|
|
527
|
+
birthtimeMs: stat.birthtime ? stat.birthtime.getTime() : 0,
|
|
528
|
+
mtimeMs: stat.mtime.getTime(),
|
|
529
|
+
basename: path.basename(filePath),
|
|
530
|
+
lastHash: lastSeenHashes.get(filePath) ?? null,
|
|
531
|
+
taxonomy,
|
|
532
|
+
freshEditWindowS: opts.freshEditWindowS,
|
|
533
|
+
});
|
|
534
|
+
if (decision.action === "skip") {
|
|
535
|
+
if (decision.recordHash !== null) lastSeenHashes.set(filePath, decision.recordHash);
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
const tmp = `${filePath}.memoryd.tmp`;
|
|
539
|
+
try {
|
|
540
|
+
fs.writeFileSync(tmp, decision.newContent, "utf-8");
|
|
541
|
+
fs.renameSync(tmp, filePath);
|
|
542
|
+
lastSeenHashes.set(filePath, decision.recordHash);
|
|
543
|
+
logger.info(`human-edit ${decision.reason}: ${path.relative(config.vaultPath, filePath)}`);
|
|
544
|
+
} catch (err) {
|
|
545
|
+
try {
|
|
546
|
+
fs.unlinkSync(tmp);
|
|
547
|
+
} catch {
|
|
548
|
+
// best effort
|
|
549
|
+
}
|
|
550
|
+
logger.error(`human-edit write failed for ${filePath}: ${String(err)}`);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// ── fs.watch + debounce ──
|
|
556
|
+
const pending = new Set<string>();
|
|
557
|
+
let flushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
558
|
+
let flushing: Promise<void> = Promise.resolve();
|
|
559
|
+
|
|
560
|
+
async function flush(): Promise<void> {
|
|
561
|
+
const changed = new Set(pending);
|
|
562
|
+
pending.clear();
|
|
563
|
+
|
|
564
|
+
// INBOX_NEW — new inbox files since the baseline.
|
|
565
|
+
for (const abs of changed) {
|
|
566
|
+
const rel = path.relative(config.vaultPath, abs);
|
|
567
|
+
const parts = rel.split(path.sep);
|
|
568
|
+
if (parts[0] === taxonomy.folders.inbox && parts.length === 2) {
|
|
569
|
+
const name = parts[1];
|
|
570
|
+
if (!inboxSeen.has(name) && fs.existsSync(abs)) {
|
|
571
|
+
inboxSeen.add(name);
|
|
572
|
+
emit(`INBOX_NEW: ${name}`);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
humanEditPass(changed);
|
|
578
|
+
syncTagsMirror();
|
|
579
|
+
await indexAll({ db, config, logger }); // incremental by content hash
|
|
580
|
+
|
|
581
|
+
const { event, next } = detectPermanentChanges({
|
|
582
|
+
vault: config.vaultPath,
|
|
583
|
+
taxonomy,
|
|
584
|
+
prev: snapshot,
|
|
585
|
+
});
|
|
586
|
+
snapshot = next;
|
|
587
|
+
if (event) {
|
|
588
|
+
for (const line of formatEventLines(event)) emit(line);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
function schedule(absPath: string): void {
|
|
593
|
+
pending.add(absPath);
|
|
594
|
+
if (flushTimer) clearTimeout(flushTimer);
|
|
595
|
+
flushTimer = setTimeout(() => {
|
|
596
|
+
flushTimer = null;
|
|
597
|
+
flushing = flushing.then(() => flush()).catch((err) => {
|
|
598
|
+
logger.error(`detect pass failed: ${String(err)}`);
|
|
599
|
+
});
|
|
600
|
+
}, debounceMs);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
let watcher: fs.FSWatcher | null = null;
|
|
604
|
+
try {
|
|
605
|
+
watcher = fs.watch(config.vaultPath, { recursive: true }, (_event, filename) => {
|
|
606
|
+
if (!filename || !filename.toString().endsWith(".md")) return;
|
|
607
|
+
const name = filename.toString();
|
|
608
|
+
if (name.includes(".memoryd.tmp")) return;
|
|
609
|
+
schedule(path.join(config.vaultPath, name));
|
|
610
|
+
});
|
|
611
|
+
} catch (err) {
|
|
612
|
+
logger.error(`fs.watch failed (events degraded to manual passes): ${String(err)}`);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
// ── heartbeat ──
|
|
616
|
+
function touchHeartbeat(): void {
|
|
617
|
+
try {
|
|
618
|
+
fs.mkdirSync(path.dirname(heartbeatPath), { recursive: true });
|
|
619
|
+
fs.writeFileSync(heartbeatPath, `${new Date().toISOString()} ${os.hostname()}\n`);
|
|
620
|
+
} catch (err) {
|
|
621
|
+
logger.error(`heartbeat write failed: ${String(err)}`);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
touchHeartbeat();
|
|
625
|
+
const heartbeatTimer = setInterval(touchHeartbeat, heartbeatMs);
|
|
626
|
+
heartbeatTimer.unref?.();
|
|
627
|
+
|
|
628
|
+
// Periodic atomic persist of the detect baseline (only when changed —
|
|
629
|
+
// no churn); a non-graceful exit loses at most one interval.
|
|
630
|
+
let lastPersistedJson: string | null = null;
|
|
631
|
+
function persistQuiet(): void {
|
|
632
|
+
try {
|
|
633
|
+
const json = JSON.stringify(Object.fromEntries(lastSeenHashes));
|
|
634
|
+
if (json === lastPersistedJson) return;
|
|
635
|
+
persistHashState(hashStatePath, lastSeenHashes);
|
|
636
|
+
lastPersistedJson = json;
|
|
637
|
+
} catch (err) {
|
|
638
|
+
logger.error(`hash-state persist failed: ${String(err)}`);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
const persistTimer = setInterval(persistQuiet, persistMs);
|
|
642
|
+
persistTimer.unref?.();
|
|
643
|
+
|
|
644
|
+
// ── MCP http ──
|
|
645
|
+
let mcp: { port: number; close: () => Promise<void> } | null = null;
|
|
646
|
+
if (opts.mcpPort !== null) {
|
|
647
|
+
mcp = await startMcpHttp({ db, config, port: opts.mcpPort ?? config.mcp.port, logger });
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
logger.info(
|
|
651
|
+
`memoryd up: vault=${config.vaultPath}, watch=${watcher ? "on" : "OFF"}, mcp=${mcp ? mcp.port : "disabled"}`,
|
|
652
|
+
);
|
|
653
|
+
|
|
654
|
+
return {
|
|
655
|
+
mcpPort: mcp?.port ?? null,
|
|
656
|
+
runDetectPass: async () => {
|
|
657
|
+
if (flushTimer) {
|
|
658
|
+
clearTimeout(flushTimer);
|
|
659
|
+
flushTimer = null;
|
|
660
|
+
}
|
|
661
|
+
await flushing;
|
|
662
|
+
await flush();
|
|
663
|
+
},
|
|
664
|
+
close: async () => {
|
|
665
|
+
watcher?.close();
|
|
666
|
+
if (flushTimer) clearTimeout(flushTimer);
|
|
667
|
+
clearInterval(heartbeatTimer);
|
|
668
|
+
clearInterval(persistTimer);
|
|
669
|
+
await flushing;
|
|
670
|
+
persistQuiet();
|
|
671
|
+
if (mcp) await mcp.close();
|
|
672
|
+
try {
|
|
673
|
+
fs.unlinkSync(heartbeatPath);
|
|
674
|
+
} catch {
|
|
675
|
+
// best effort
|
|
676
|
+
}
|
|
677
|
+
db.close();
|
|
678
|
+
},
|
|
679
|
+
};
|
|
680
|
+
}
|