@knowall-ai/reverie 0.4.0 → 0.5.0

package/README.md

@@ -523,6 +523,81 @@ The inspector provides a web UI to:
 - Validate your Neo4j connection
 - Debug tool parameters and responses
 
+## HTTP mode (`reverie serve`)
+
+`reverie` on its own is the stdio MCP server. `reverie serve` runs the same server over
+[Streamable HTTP](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports#streamable-http)
+and adds a read-only **Brain API** that the [KnowAll Agents Portal](https://github.com/knowall-ai/agents-portal)
+Brain tab uses to draw the graph live. One bearer token protects both; only the health check is open.
+
+```bash
+REVERIE_SERVE_TOKEN=$(openssl rand -hex 24) NEO4J_PASSWORD=… reverie serve
+# Reverie HTTP server listening on http://127.0.0.1:8643
+```
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `REVERIE_SERVE_TOKEN` | (required) | Bearer token clients must send as `Authorization: Bearer …` |
+| `REVERIE_HTTP_HOST` / `REVERIE_HTTP_PORT` | `127.0.0.1` / `8643` | Bind address; keep it on loopback behind a reverse proxy |
+| `REVERIE_ALLOWED_ORIGINS` | none | Comma-separated browser origins allowed to call the API (they get CORS headers and preflight answers); requests carrying any other `Origin` header get 403. Server-to-server clients send no Origin and need nothing here |
+| `REVERIE_EVENTS_PATH` | `~/.reverie/events.jsonl` | Activation log (see below); empty string disables it |
+| `REVERIE_DREAMS_DIR` | unset | Optional dream diary folder (`*.md`); the newest file names the last dream |
+| `REVERIE_USAGE_STATS_PATH` / `REVERIE_BOOST_STATE_PATH` | `~/call-transcripts/usage-stats.json` / `boost-state.json` | Presence HUD files passed through in `state` (usage only when written within 15 min) |
+| `REVERIE_GRAPH_POLL_SECONDS` / `REVERIE_STATE_SECONDS` / `REVERIE_EVENT_POLL_SECONDS` | `5` / `5` / `1` | How often the event stream re-reads the graph, host state and activation log |
+
+`NEO4J_*` and `REVERIE_EMBEDDINGS` work exactly as in stdio mode.
+
+| Endpoint | Auth | Returns |
+|---|---|---|
+| `GET /health` (alias `/brain/health`) | none | `{ ok, neo4j, ts }` |
+| `POST /mcp` | bearer | MCP Streamable HTTP (stateless; JSON responses). Any MCP client that speaks Streamable HTTP can use it |
+| `GET /brain/graph?limit=400` | bearer | The most connected / most recent nodes (`limit` ≤ 1500), the relationships among them, totals per label and relationship type, and `state` |
+| `GET /brain/state` | bearer | `dreaming`, `lastActivityAt`, `lastDreamAt`, recent read/write counts, host CPU / load / memory, Presence `usage` and `boost` |
+| `GET /brain/events?limit=400` | bearer | Server-Sent Events: the last 30 `activation` events, then `state`; afterwards every new `activation` as it happens, a `graph` diff (`nodesAdded`, `nodesUpdated`, `nodesRemoved`, `relsAdded`, `relsRemoved`, `stats`) whenever the graph changed, `state` every few seconds and `error` if a poll fails |
+
+Node ids in the Brain API are the same numeric ids the tools return, as strings, so activation
+events and snapshot nodes line up. Archived memories are never shown. Embedding vectors never leave
+the server.
+
+**Activation log.** Every `search_memories` (`recall`), `create_memory` / `update_memory`
+(`remember`), `create_connection` / `update_connection` (`connect`), `delete_*` (`forget`) and real
+`dream` run (`dream.start` / `dream.end`) appends one JSON line to `REVERIE_EVENTS_PATH`, whichever
+transport served it. It holds ids, names and search terms only, is private to the user (mode 0600), is capped at
+5 MB / 5,000 lines, and is what makes the Brain view light up. The graph itself never records what was looked at.
+
+Behind Caddy on an agent's VM:
+
+```caddyfile
+agent.example.com {
+    # … the agent's other routes …
+    handle /reverie/* {
+        uri strip_prefix /reverie
+        reverse_proxy 127.0.0.1:8643
+    }
+}
+```
+
+and as a service:
+
+```ini
+[Unit]
+Description=Reverie graph memory (HTTP mode)
+After=network-online.target
+
+[Service]
+User=agent
+EnvironmentFile=/home/agent/.config/reverie.env   # NEO4J_*, REVERIE_SERVE_TOKEN, REVERIE_EMBEDDINGS…
+ExecStart=/usr/bin/reverie serve
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Give the portal `https://<agent host>/reverie` as the agent's `brainUrl` and the same token as
+`REVERIE_TOKEN`.
+
 ## License
 
 MIT

package/build/brain.js

@@ -0,0 +1,334 @@
+/**
+ * Brain view data for the Agents Portal: graph snapshots, snapshot diffs and the agent's
+ * awake/dreaming state. No HTTP here; `http-server.ts` exposes it, which keeps this testable
+ * without a server or a database. Field names are the portal's contract, so they stay camelCase.
+ */
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import neo4j from 'neo4j-driver';
+import { MAX_LINES, eventsPath, expandHome, tail } from './events.js';
+export const MAX_LIMIT = 1500;
+export const DEFAULT_LIMIT = 400;
+const DREAM_WINDOW_SECONDS = 30 * 60;
+const RECENT_WINDOW_SECONDS = 15 * 60;
+const USAGE_MAX_AGE_SECONDS = 900;
+const PROP_MAX_CHARS = 300;
+const PROP_MAX_KEYS = 24;
+const PROP_MAX_ITEMS = 12;
+const HIDDEN_PROPS = new Set(['embedding', 'name_embedding', 'vector', 'embedding_model', 'embedded_at']);
+// Ids are the same numeric ids the MCP tools expose (as strings), so activation events and
+// snapshot nodes line up in the portal.
+const NODE_QUERY = `
+MATCH (n)
+WHERE n.status IS NULL OR n.status <> 'archived'
+OPTIONAL MATCH (n)-[r]-()
+WITH n, count(r) AS degree
+ORDER BY degree DESC, coalesce(n.updated_at, n.created_at) DESC
+LIMIT $limit
+RETURN toString(id(n)) AS id, labels(n) AS labels, n.name AS name, degree,
+       n.updated_at AS updated_at, n.created_at AS created_at, properties(n) AS props`;
+const REL_QUERY = `
+MATCH (a)-[r]->(b)
+WHERE toString(id(a)) IN $ids AND toString(id(b)) IN $ids
+RETURN toString(id(r)) AS id, type(r) AS type, toString(id(a)) AS source, toString(id(b)) AS target,
+       r.updated_at AS updated_at, r.created_at AS created_at`;
+const LABEL_STATS_QUERY = `
+MATCH (n) WHERE n.status IS NULL OR n.status <> 'archived'
+RETURN head(labels(n)) AS label, count(*) AS n`;
+const REL_STATS_QUERY = `
+MATCH (a)-[r]->(b)
+WHERE (a.status IS NULL OR a.status <> 'archived') AND (b.status IS NULL OR b.status <> 'archived')
+RETURN type(r) AS type, count(*) AS n`;
+/** A query-string `limit`: absent → the default; otherwise it must be a base-10 integer in 1..MAX_LIMIT, else null. */
+export function parseLimit(raw) {
+    if (raw === null || raw === undefined || raw === '') {
+        return DEFAULT_LIMIT;
+    }
+    if (!/^\d{1,5}$/.test(raw)) {
+        return null;
+    }
+    const value = Number.parseInt(raw, 10);
+    return value >= 1 && value <= MAX_LIMIT ? value : null;
+}
+export function clampLimit(value, fallback = DEFAULT_LIMIT) {
+    const parsed = typeof value === 'number' ? value : Number.parseInt(String(value ?? ''), 10);
+    if (!Number.isFinite(parsed)) {
+        return fallback;
+    }
+    return Math.max(1, Math.min(Math.trunc(parsed), MAX_LIMIT));
+}
+/** Epoch seconds from whatever a graph stores: epoch seconds or milliseconds, ISO strings, Neo4j temporals (already strings). */
+export function toEpochSeconds(value) {
+    if (typeof value === 'number' && Number.isFinite(value)) {
+        return Math.round(value >= 1e12 ? value / 1000 : value);
+    }
+    if (typeof value === 'string') {
+        const trimmed = value.trim();
+        if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
+            return toEpochSeconds(Number(trimmed));
+        }
+        const parsed = Date.parse(trimmed);
+        return Number.isFinite(parsed) ? Math.round(parsed / 1000) : 0;
+    }
+    if (value && typeof value === 'object' && typeof value.toString === 'function') {
+        const text = String(value);
+        return text && text !== '[object Object]' ? toEpochSeconds(text) : 0;
+    }
+    return 0;
+}
+/** Primitives only, short strings only, a bounded number of keys; vectors never leave the server. */
+export function cleanProps(props) {
+    const out = {};
+    for (const [key, value] of Object.entries(props ?? {})) {
+        if (HIDDEN_PROPS.has(key) || Object.keys(out).length >= PROP_MAX_KEYS) {
+            continue;
+        }
+        if (value === null || typeof value === 'boolean') {
+            out[key] = value;
+        }
+        else if (typeof value === 'number' && Number.isFinite(value)) {
+            out[key] = value;
+        }
+        else if (typeof value === 'string') {
+            out[key] = value.length <= PROP_MAX_CHARS ? value : `${value.slice(0, PROP_MAX_CHARS)}…`;
+        }
+        else if (Array.isArray(value) && value.every((item) => typeof item === 'string' || typeof item === 'number')) {
+            out[key] = value.slice(0, PROP_MAX_ITEMS);
+        }
+    }
+    return out;
+}
+/** Nodes (most connected / most recent first), the relationships among them, and totals. */
+export async function snapshot(run, limit = DEFAULT_LIMIT) {
+    const nodeRows = await run(NODE_QUERY, { limit: neo4j.int(clampLimit(limit)) });
+    const nodes = nodeRows.map((row) => {
+        const labels = Array.isArray(row.labels) ? row.labels.map(String) : [];
+        const id = String(row.id);
+        const createdAt = toEpochSeconds(row.created_at);
+        return {
+            id,
+            labels,
+            label: labels[0] ?? 'Unknown',
+            name: row.name === null || row.name === undefined || row.name === '' ? id : String(row.name),
+            degree: Number(row.degree ?? 0),
+            updatedAt: row.updated_at === null || row.updated_at === undefined ? createdAt : toEpochSeconds(row.updated_at),
+            createdAt,
+            props: cleanProps(row.props ?? {})
+        };
+    });
+    const ids = nodes.map((node) => node.id);
+    const relRows = ids.length > 0 ? await run(REL_QUERY, { ids }) : [];
+    const rels = relRows.map((row) => ({
+        id: String(row.id),
+        type: String(row.type),
+        source: String(row.source),
+        target: String(row.target),
+        updatedAt: toEpochSeconds(row.updated_at ?? row.created_at)
+    }));
+    const labels = {};
+    for (const row of await run(LABEL_STATS_QUERY)) {
+        labels[row.label ? String(row.label) : 'Unknown'] = Number(row.n ?? 0);
+    }
+    const relTypes = {};
+    for (const row of await run(REL_STATS_QUERY)) {
+        relTypes[String(row.type)] = Number(row.n ?? 0);
+    }
+    return {
+        nodes,
+        rels,
+        stats: {
+            nodeCount: Object.values(labels).reduce((sum, n) => sum + n, 0),
+            relCount: Object.values(relTypes).reduce((sum, n) => sum + n, 0),
+            labels,
+            relTypes,
+            shown: nodes.length
+        },
+        generatedAt: Date.now() / 1000
+    };
+}
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+/**
+ * What changed between two snapshots. Nodes count as updated when any exposed field differs
+ * (not just degree or the rounded timestamp); a changed relationship is a removal plus an
+ * addition, since the contract has no "relsUpdated"; `stats` is set only when the totals changed.
+ */
+export function diffSnapshot(prev, curr) {
+    const prevNodes = new Map(prev.nodes.map((node) => [node.id, node]));
+    const currNodes = new Map(curr.nodes.map((node) => [node.id, node]));
+    const prevRels = new Map(prev.rels.map((rel) => [rel.id, rel]));
+    const currRels = new Map(curr.rels.map((rel) => [rel.id, rel]));
+    const changedRels = curr.rels.filter((rel) => {
+        const before = prevRels.get(rel.id);
+        return before !== undefined && !same(before, rel);
+    });
+    const diff = {
+        nodesAdded: curr.nodes.filter((node) => !prevNodes.has(node.id)),
+        nodesUpdated: curr.nodes.filter((node) => {
+            const before = prevNodes.get(node.id);
+            return before !== undefined && !same(before, node);
+        }),
+        nodesRemoved: prev.nodes.filter((node) => !currNodes.has(node.id)).map((node) => node.id),
+        relsAdded: [...curr.rels.filter((rel) => !prevRels.has(rel.id)), ...changedRels],
+        relsRemoved: [...prev.rels.filter((rel) => !currRels.has(rel.id)).map((rel) => rel.id), ...changedRels.map((rel) => rel.id)]
+    };
+    if (!same(prev.stats, curr.stats)) {
+        diff.stats = curr.stats;
+    }
+    return diff;
+}
+/** A diff with nothing to apply: no node or relationship changes and no changed totals. */
+export function isEmptyDiff(diff) {
+    return (diff.nodesAdded.length === 0 &&
+        diff.nodesUpdated.length === 0 &&
+        diff.nodesRemoved.length === 0 &&
+        diff.relsAdded.length === 0 &&
+        diff.relsRemoved.length === 0 &&
+        diff.stats === undefined);
+}
+/** The whole snapshot expressed as a diff against nothing: what a fresh stream sends first. */
+export function snapshotAsDiff(snap) {
+    return { nodesAdded: snap.nodes, nodesUpdated: [], nodesRemoved: [], relsAdded: snap.rels, relsRemoved: [], stats: snap.stats };
+}
+/** Newest dream diary entry (*.md in REVERIE_DREAMS_DIR) by modification time; null when the variable is unset. */
+export function latestDream(env = process.env) {
+    const raw = env.REVERIE_DREAMS_DIR?.trim();
+    if (!raw) {
+        return null;
+    }
+    const dir = expandHome(raw);
+    try {
+        if (!fs.statSync(dir).isDirectory()) {
+            return null;
+        }
+        let newest = null;
+        for (const entry of fs.readdirSync(dir)) {
+            if (!entry.endsWith('.md')) {
+                continue;
+            }
+            const at = fs.statSync(path.join(dir, entry)).mtimeMs / 1000;
+            if (!newest || at > newest.at) {
+                newest = { name: entry.slice(0, -3), at };
+            }
+        }
+        return newest;
+    }
+    catch {
+        return null;
+    }
+}
+let cpuLast = null;
+/** Overall CPU busy percentage since the previous call (Linux /proc/stat); null on the first call or elsewhere. */
+export function cpuPercent() {
+    let fields;
+    try {
+        fields = fs.readFileSync('/proc/stat', 'utf8').split('\n')[0].trim().split(/\s+/).slice(1, 8).map(Number);
+    }
+    catch {
+        return null;
+    }
+    if (fields.length < 5 || fields.some((n) => !Number.isFinite(n))) {
+        return null;
+    }
+    const idle = fields[3] + fields[4];
+    const total = fields.reduce((sum, n) => sum + n, 0);
+    const prev = cpuLast;
+    cpuLast = { idle, total };
+    if (!prev || total === prev.total) {
+        return null;
+    }
+    return round((100 * (1 - (idle - prev.idle) / (total - prev.total))), 1);
+}
+/** CPU %, 1-minute load and memory use of the agent's VM, for the telemetry panel. Nulls where unavailable. */
+export function hostStats() {
+    const out = { cpuPercent: cpuPercent(), load1: null, memPercent: null, memUsedGb: null, memTotalGb: null };
+    try {
+        out.load1 = round(os.loadavg()[0], 2);
+    }
+    catch {
+        out.load1 = null;
+    }
+    try {
+        const info = {};
+        for (const line of fs.readFileSync('/proc/meminfo', 'utf8').split('\n')) {
+            const [key, rest] = line.split(':');
+            if (key && rest) {
+                info[key.trim()] = Number(rest.trim().split(/\s+/)[0]);
+            }
+        }
+        const total = info.MemTotal;
+        const available = info.MemAvailable;
+        if (total > 0 && Number.isFinite(available)) {
+            out.memPercent = round(100 * (1 - available / total), 1);
+            out.memTotalGb = round(total / 1048576, 1);
+            out.memUsedGb = round((total - available) / 1048576, 1);
+        }
+    }
+    catch {
+        // not Linux, or /proc unreadable
+    }
+    return out;
+}
+function readJsonFile(env, variable, fallback, maxAgeSeconds) {
+    const raw = env[variable] === undefined ? fallback : env[variable].trim();
+    if (!raw) {
+        return null;
+    }
+    const file = expandHome(raw);
+    try {
+        const stat = fs.statSync(file);
+        if (!stat.isFile()) {
+            return null;
+        }
+        if (maxAgeSeconds !== undefined && Date.now() / 1000 - stat.mtimeMs / 1000 > maxAgeSeconds) {
+            return null;
+        }
+        const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+        return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** usage-stats.json (fresh within 15 min) and boost-state.json written by the agent's Presence stack. */
+export function presenceStats(env = process.env) {
+    const home = os.homedir();
+    return {
+        usage: readJsonFile(env, 'REVERIE_USAGE_STATS_PATH', path.join(home, 'call-transcripts', 'usage-stats.json'), USAGE_MAX_AGE_SECONDS),
+        boost: readJsonFile(env, 'REVERIE_BOOST_STATE_PATH', path.join(home, 'call-transcripts', 'boost-state.json'))
+    };
+}
+/**
+ * Awake/dreaming signals from the activation log (and the dream diary when one is configured):
+ * dreaming = a dream.start newer than the last dream.end within the last 30 minutes;
+ * lastActivityAt = the newest event of any kind; recent counts cover the last 15 minutes.
+ */
+export function state(events, now, env = process.env) {
+    const at = now ?? Date.now() / 1000;
+    const file = eventsPath(env);
+    const list = events ?? (file ? tail(file, MAX_LINES) : []); // the whole retained log, so windows are never truncated
+    const ts = (event) => (typeof event.ts === 'number' && Number.isFinite(event.ts) ? event.ts : 0);
+    const newest = (kind) => list.filter((event) => kind === undefined || event.kind === kind).reduce((max, event) => (max === null || ts(event) > max ? ts(event) : max), null);
+    const lastActivity = newest();
+    const dreamStart = newest('dream.start');
+    const dreamEnd = newest('dream.end');
+    const dreaming = dreamStart !== null && at - dreamStart < DREAM_WINDOW_SECONDS && (dreamEnd === null || dreamEnd < dreamStart);
+    const diary = latestDream(env);
+    const lastDreamEvent = [...list].reverse().find((event) => event.kind === 'dream.end') ?? [...list].reverse().find((event) => event.kind === 'dream.start');
+    const recent = list.filter((event) => at - ts(event) < RECENT_WINDOW_SECONDS);
+    return {
+        dreaming,
+        lastActivityAt: lastActivity,
+        lastDreamAt: diary ? diary.at : dreamEnd ?? dreamStart,
+        lastDreamName: diary ? diary.name : typeof lastDreamEvent?.name === 'string' ? lastDreamEvent.name : null,
+        recentReads: recent.filter((event) => event.kind === 'recall').length,
+        recentWrites: recent.filter((event) => event.kind === 'remember' || event.kind === 'connect' || event.kind === 'forget').length,
+        eventsAvailable: file !== null,
+        ...hostStats(),
+        ...presenceStats(env)
+    };
+}
+function round(value, digits) {
+    const factor = 10 ** digits;
+    return Math.round(value * factor) / factor;
+}

package/build/config.js

@@ -0,0 +1,32 @@
+function isSet(value) {
+    return typeof value === 'string' && value.trim() !== '';
+}
+/** Neo4j connection from NEO4J_URI / NEO4J_USERNAME / NEO4J_PASSWORD (+ optional NEO4J_DATABASE); undefined when unset. */
+export function neo4jConfigFromEnv(env = process.env) {
+    const uri = env.NEO4J_URI?.trim();
+    const username = env.NEO4J_USERNAME?.trim();
+    const password = env.NEO4J_PASSWORD; // never trimmed: whitespace can be part of a password
+    if (!uri || !username || !isSet(password)) {
+        return undefined;
+    }
+    return {
+        uri,
+        username,
+        password,
+        database: env.NEO4J_DATABASE?.trim() || undefined // optional for Neo4j Community Edition
+    };
+}
+/** A partial configuration is a mistake worth refusing to start on; returns the message to print, or null. */
+export function neo4jConfigError(env = process.env) {
+    const uri = env.NEO4J_URI?.trim();
+    const username = env.NEO4J_USERNAME?.trim();
+    const password = isSet(env.NEO4J_PASSWORD);
+    if (!(uri || username || password) || (uri && username && password)) {
+        return null;
+    }
+    if (!password)
+        return 'Error: NEO4J_PASSWORD environment variable is required';
+    if (!uri)
+        return 'Error: NEO4J_URI environment variable is required';
+    return 'Error: NEO4J_USERNAME environment variable is required';
+}