mr-memory 3.9.1 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/auto-consolidate.js +292 -0
- package/auto-consolidate.ts +279 -0
- package/import-transport.js +457 -0
- package/import-transport.ts +511 -0
- package/index.js +81 -0
- package/index.ts +82 -0
- package/memoryrouter-api.js +51 -0
- package/memoryrouter-api.ts +59 -0
- package/openclaw.plugin.json +10 -0
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -40,6 +40,19 @@ openclaw mr upload --workspace <dir> # Custom workspace directory
|
|
|
40
40
|
openclaw mr upload --brain <dir> # Custom state dir (sessions from another agent)
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
## Historical Import (agent-driven)
|
|
44
|
+
|
|
45
|
+
Import existing OpenClaw conversation history with a strict division of labor: the server owns the extraction prompt (versioned, fetched at import time), your agent owns discovery/parsing of the local session files, and the CLI owns transport. The agent never uploads; the CLI never parses history.
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
openclaw mr import --instructions # print the server's extraction prompt for your agent
|
|
49
|
+
openclaw mr import --file extract.jsonl # validate, preview, price quote, approve, upload
|
|
50
|
+
openclaw mr import --file extract.jsonl --yes # non-interactive approval
|
|
51
|
+
openclaw mr import --dismiss # permanently silence the import nudge
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Guarantees: the extract is validated locally (four fields per record, `openclaw:` prefixed ids, epoch-ms timestamps); the CLI shows record count, date range, estimated tokens, and the server price quote and requires explicit approval before any write; the import id is content-derived and every batch carries an Idempotency-Key, so re-running after any interruption replays committed batches for free and resumes. Duplicates are never re-billed. The memory key never appears in the prompt or the extract file. When a vault is new, injected context includes a one-line import suggestion until an import completes or you `--dismiss`.
|
|
55
|
+
|
|
43
56
|
## How It Works
|
|
44
57
|
|
|
45
58
|
When enabled, relay hooks retrieve memory before agent/model input and capture the direct user-to-assistant conversation at `agent_end`. Inference and provider API keys remain inside OpenClaw; the plugin does not route provider calls through MemoryRouter and is not a provider-key proxy.
|
|
@@ -56,6 +69,22 @@ Tune it with `keepWarmMs` (milliseconds, default `180000`; `0` disables warmup p
|
|
|
56
69
|
{ "plugins": { "entries": { "mr-memory": { "config": { "keepWarmMs": 180000 } } } } }
|
|
57
70
|
```
|
|
58
71
|
|
|
72
|
+
### Automatic consolidation (default on)
|
|
73
|
+
|
|
74
|
+
MemoryRouter's Reflection Hierarchy consolidates raw memories into higher-tier reflections. The server tracks consolidation debt per vault; when a retrieval response reports debt above the server's threshold, this plugin consolidates automatically:
|
|
75
|
+
|
|
76
|
+
- It starts a dedicated isolated agent turn through the OpenClaw gateway (a background subagent session, never your main conversation) whose only instruction is to call the plugin's `memory_reflect` and `memory_commit` tools. Your agent's model writes the reflections, so the inference is your own model spend; MemoryRouter never calls a provider.
|
|
77
|
+
- At most 3 batches per trigger, with a fresh debt check between batches; the run stops when debt drops below the threshold. The threshold always comes from the server response.
|
|
78
|
+
- A machine-wide lockfile (`~/.memoryrouter/consolidate.lock`, stale after 20 minutes) prevents double-firing across sessions and across other MemoryRouter connectors on the same machine, and a 30 minute cooldown follows each completed run.
|
|
79
|
+
- The consolidation session is excluded from memory capture/injection and its transcript is deleted afterward. Failures never surface in your conversation; everything is logged to `~/.memoryrouter/consolidate.log`.
|
|
80
|
+
- MemoryRouter bills the consolidation checkout/commit tokens at the normal 1x raw rate.
|
|
81
|
+
|
|
82
|
+
Opt out with `"autoConsolidate": false` in the plugin config (restart the gateway to apply). The in-context consolidation hint and the `memory_reflect`/`memory_commit` agent tools keep working; only the automatic background trigger is disabled:
|
|
83
|
+
|
|
84
|
+
```json
|
|
85
|
+
{ "plugins": { "entries": { "mr-memory": { "config": { "autoConsolidate": false } } } } }
|
|
86
|
+
```
|
|
87
|
+
|
|
59
88
|
## Config
|
|
60
89
|
|
|
61
90
|
After `openclaw mr <key>`, config is stored at:
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Automatic threshold-activated consolidation for the OpenClaw plugin.
|
|
3
|
+
*
|
|
4
|
+
* The injected suggestion nudge depends on main-session model discipline,
|
|
5
|
+
* which is a proven failure mode: the model can ignore it forever. This
|
|
6
|
+
* module instead drives consolidation deterministically from plugin code
|
|
7
|
+
* running inside the gateway, using the plugin runtime's subagent capability
|
|
8
|
+
* (api.runtime.subagent.run), which executes a dedicated isolated agent turn.
|
|
9
|
+
* That isolated turn is OpenClaw's equivalent of the headless model runs the
|
|
10
|
+
* claude-code and codex connectors use (claude -p / codex exec): the host's
|
|
11
|
+
* own model does the reflecting in a session whose only instruction is to
|
|
12
|
+
* consolidate, the user pays their own inference, and MemoryRouter never
|
|
13
|
+
* calls a provider.
|
|
14
|
+
*
|
|
15
|
+
* Reliability rails (shared with the other MemoryRouter connectors on this
|
|
16
|
+
* machine via the same files):
|
|
17
|
+
* - Lockfile ~/.memoryrouter/consolidate.lock (pid + timestamp, stale after
|
|
18
|
+
* 20 minutes) so concurrent sessions/connectors never double-fire.
|
|
19
|
+
* - Bounded: at most 3 batches per trigger, re-checking debt via the API
|
|
20
|
+
* between batches and stopping once the server reports no suggestion.
|
|
21
|
+
* - Cooldown: no re-trigger within 30 minutes of the last completed run,
|
|
22
|
+
* nor within 10 minutes of the last attempt.
|
|
23
|
+
* - The threshold always comes from the server's reflection debt block;
|
|
24
|
+
* nothing is hardcoded client-side.
|
|
25
|
+
* - Failures are silent to the user's session and logged to
|
|
26
|
+
* ~/.memoryrouter/consolidate.log.
|
|
27
|
+
*/
|
|
28
|
+
import fs from "node:fs";
|
|
29
|
+
import os from "node:os";
|
|
30
|
+
import path from "node:path";
|
|
31
|
+
import { randomUUID } from "node:crypto";
|
|
32
|
+
import { dateSearch } from "./memoryrouter-api.js";
|
|
33
|
+
export const AUTO_SESSION_MARKER = "mrconsolidate";
|
|
34
|
+
export const LOCK_STALE_MS = 20 * 60 * 1000;
|
|
35
|
+
export const COOLDOWN_MS = 30 * 60 * 1000;
|
|
36
|
+
export const ATTEMPT_COOLDOWN_MS = 10 * 60 * 1000;
|
|
37
|
+
export const MAX_BATCHES_PER_TRIGGER = 3;
|
|
38
|
+
const RUN_TIMEOUT_MS = 12 * 60 * 1000; // inside the server's 15 minute lease
|
|
39
|
+
export function autoDir(env = process.env) {
|
|
40
|
+
return env.MEMORYROUTER_AUTO_DIR || path.join(os.homedir(), ".memoryrouter");
|
|
41
|
+
}
|
|
42
|
+
function lockPath(env) { return path.join(autoDir(env), "consolidate.lock"); }
|
|
43
|
+
function statePath(env) { return path.join(autoDir(env), "consolidate-state.json"); }
|
|
44
|
+
function logPath(env) { return path.join(autoDir(env), "consolidate.log"); }
|
|
45
|
+
function ensureDir(dir) {
|
|
46
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
47
|
+
}
|
|
48
|
+
export function appendAutoLog(line, env = process.env) {
|
|
49
|
+
try {
|
|
50
|
+
ensureDir(autoDir(env));
|
|
51
|
+
const file = logPath(env);
|
|
52
|
+
try {
|
|
53
|
+
if (fs.statSync(file).size > 1024 * 1024)
|
|
54
|
+
fs.renameSync(file, `${file}.1`);
|
|
55
|
+
}
|
|
56
|
+
catch { /* new file */ }
|
|
57
|
+
fs.appendFileSync(file, `[${new Date().toISOString()}] ${line}\n`, { mode: 0o600 });
|
|
58
|
+
}
|
|
59
|
+
catch { /* logging never throws */ }
|
|
60
|
+
}
|
|
61
|
+
export function readAutoState(env = process.env) {
|
|
62
|
+
try {
|
|
63
|
+
return JSON.parse(fs.readFileSync(statePath(env), "utf8")) || {};
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return {};
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
export function writeAutoState(patch, env = process.env) {
|
|
70
|
+
try {
|
|
71
|
+
ensureDir(autoDir(env));
|
|
72
|
+
const next = { ...readAutoState(env), ...patch };
|
|
73
|
+
fs.writeFileSync(statePath(env), JSON.stringify(next, null, 2) + "\n", { mode: 0o600 });
|
|
74
|
+
}
|
|
75
|
+
catch { /* state is best-effort */ }
|
|
76
|
+
}
|
|
77
|
+
export function lockFresh(env = process.env, now = Date.now()) {
|
|
78
|
+
try {
|
|
79
|
+
const info = JSON.parse(fs.readFileSync(lockPath(env), "utf8"));
|
|
80
|
+
return now - Number(info.at) <= LOCK_STALE_MS;
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
if (err?.code === "ENOENT")
|
|
84
|
+
return false;
|
|
85
|
+
try {
|
|
86
|
+
return now - fs.statSync(lockPath(env)).mtimeMs <= LOCK_STALE_MS;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
return false;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export function acquireLock(env = process.env, now = Date.now()) {
|
|
94
|
+
ensureDir(autoDir(env));
|
|
95
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
96
|
+
try {
|
|
97
|
+
const fd = fs.openSync(lockPath(env), "wx", 0o600);
|
|
98
|
+
fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, at: now }) + "\n");
|
|
99
|
+
fs.closeSync(fd);
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
if (err?.code !== "EEXIST")
|
|
104
|
+
return false;
|
|
105
|
+
if (lockFresh(env, now))
|
|
106
|
+
return false;
|
|
107
|
+
try {
|
|
108
|
+
fs.unlinkSync(lockPath(env));
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
export function releaseLock(env = process.env) {
|
|
118
|
+
try {
|
|
119
|
+
fs.unlinkSync(lockPath(env));
|
|
120
|
+
}
|
|
121
|
+
catch { /* already gone */ }
|
|
122
|
+
}
|
|
123
|
+
export function cooldownActive(env = process.env, now = Date.now()) {
|
|
124
|
+
const state = readAutoState(env);
|
|
125
|
+
const completed = Number(state.lastCompletedAt);
|
|
126
|
+
if (Number.isFinite(completed) && now - completed < COOLDOWN_MS)
|
|
127
|
+
return true;
|
|
128
|
+
const attempted = Number(state.lastAttemptAt);
|
|
129
|
+
if (Number.isFinite(attempted) && now - attempted < ATTEMPT_COOLDOWN_MS)
|
|
130
|
+
return true;
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
/** Tier choice comes entirely from the server's debt block. */
|
|
134
|
+
export function pickTier(debt) {
|
|
135
|
+
const threshold = Number(debt?.threshold_importance);
|
|
136
|
+
const rawEquiv = Number(debt?.unconsolidated?.raw?.importance_equiv);
|
|
137
|
+
const tier2 = Number(debt?.unconsolidated?.tier2?.importance);
|
|
138
|
+
if (Number.isFinite(threshold) && threshold > 0) {
|
|
139
|
+
if (Number.isFinite(rawEquiv) && rawEquiv >= threshold)
|
|
140
|
+
return 1;
|
|
141
|
+
if (Number.isFinite(tier2) && tier2 >= threshold)
|
|
142
|
+
return 2;
|
|
143
|
+
}
|
|
144
|
+
return /tier 2/i.test(String(debt?.suggestion || "")) ? 2 : 1;
|
|
145
|
+
}
|
|
146
|
+
/** True for sessions created by this module; hooks skip them entirely. */
|
|
147
|
+
export function isAutoConsolidationSession(sessionKey) {
|
|
148
|
+
return typeof sessionKey === "string" && sessionKey.includes(AUTO_SESSION_MARKER);
|
|
149
|
+
}
|
|
150
|
+
async function probeDebt(endpoint, key, embeddings) {
|
|
151
|
+
// Sanctioned zero-billing debt read: an empty-window date-search returns
|
|
152
|
+
// the reflection debt block with zero results.
|
|
153
|
+
const from = new Date(Date.now() - 1000).toISOString();
|
|
154
|
+
const res = await dateSearch(endpoint, key, { from, max_tokens: 1000, ...(embeddings && { embeddings }) });
|
|
155
|
+
if (!res.ok)
|
|
156
|
+
return null;
|
|
157
|
+
const data = await res.json().catch(() => null);
|
|
158
|
+
return data?.reflection ?? null;
|
|
159
|
+
}
|
|
160
|
+
// The isolated turn's message is connector transport mechanics (which tools
|
|
161
|
+
// to call, in what order). Every reflection semantic comes from the server
|
|
162
|
+
// contract that the memory_reflect tool returns verbatim.
|
|
163
|
+
function consolidationMessage(tier) {
|
|
164
|
+
return [
|
|
165
|
+
"MemoryRouter automatic consolidation run. Complete exactly one consolidation batch:",
|
|
166
|
+
`1. Call the memory_reflect tool with tier: ${tier}.`,
|
|
167
|
+
"2. If it returns batch_id null or an error, reply \"nothing to consolidate\" and stop.",
|
|
168
|
+
"3. Otherwise follow the instructions it returns to write reflection entries, then call memory_commit with that batch_id and your entries.",
|
|
169
|
+
"4. Reply \"done\" once the commit succeeds.",
|
|
170
|
+
"Do not call any other tools. Do not check out more than one batch.",
|
|
171
|
+
].join("\n");
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* The full bounded run: lock -> up to 3 (probe debt -> isolated agent turn)
|
|
175
|
+
* cycles -> record completion -> release. Never throws.
|
|
176
|
+
*/
|
|
177
|
+
export async function runAutoConsolidate(options) {
|
|
178
|
+
const env = options.env || process.env;
|
|
179
|
+
const now = options.now || (() => Date.now());
|
|
180
|
+
const log = options.log || ((line) => appendAutoLog(line, env));
|
|
181
|
+
const bound = Math.min(options.maxBatches ?? MAX_BATCHES_PER_TRIGGER, MAX_BATCHES_PER_TRIGGER);
|
|
182
|
+
if (cooldownActive(env, now())) {
|
|
183
|
+
log("auto: cooldown active; skipping");
|
|
184
|
+
return { committed: 0, reason: "cooldown" };
|
|
185
|
+
}
|
|
186
|
+
if (!acquireLock(env, now())) {
|
|
187
|
+
log("auto: another consolidation run holds the lock; skipping");
|
|
188
|
+
return { committed: 0, reason: "locked" };
|
|
189
|
+
}
|
|
190
|
+
writeAutoState({ lastAttemptAt: now() }, env);
|
|
191
|
+
let committed = 0;
|
|
192
|
+
try {
|
|
193
|
+
for (let batch = 0; batch < bound; batch++) {
|
|
194
|
+
let debt = null;
|
|
195
|
+
try {
|
|
196
|
+
debt = await probeDebt(options.endpoint, options.key, options.embeddings);
|
|
197
|
+
}
|
|
198
|
+
catch (err) {
|
|
199
|
+
log(`auto: debt probe failed: ${err?.message}`);
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
if (!debt || !debt.suggestion) {
|
|
203
|
+
log(`auto: debt below threshold after ${committed} batch(es); stopping`);
|
|
204
|
+
break;
|
|
205
|
+
}
|
|
206
|
+
const tier = pickTier(debt);
|
|
207
|
+
const sessionKey = `agent:${options.agentId}:subagent:${AUTO_SESSION_MARKER}-${randomUUID()}`;
|
|
208
|
+
log(`auto: batch ${batch + 1}/${bound} starting (tier ${tier}; ${debt.suggestion})`);
|
|
209
|
+
let status = "error";
|
|
210
|
+
try {
|
|
211
|
+
const { runId } = await options.runtime.run({
|
|
212
|
+
sessionKey,
|
|
213
|
+
message: consolidationMessage(tier),
|
|
214
|
+
deliver: false,
|
|
215
|
+
});
|
|
216
|
+
const waited = await options.runtime.waitForRun({ runId, timeoutMs: RUN_TIMEOUT_MS });
|
|
217
|
+
status = waited.status;
|
|
218
|
+
if (waited.status !== "ok")
|
|
219
|
+
log(`auto: isolated run ${waited.status}${waited.error ? `: ${waited.error}` : ""}`);
|
|
220
|
+
}
|
|
221
|
+
catch (err) {
|
|
222
|
+
log(`auto: isolated run failed: ${err?.message}`);
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
finally {
|
|
226
|
+
try {
|
|
227
|
+
await options.runtime.deleteSession({ sessionKey, deleteTranscript: true });
|
|
228
|
+
}
|
|
229
|
+
catch { /* cleanup best-effort */ }
|
|
230
|
+
}
|
|
231
|
+
if (status !== "ok")
|
|
232
|
+
break;
|
|
233
|
+
// Verify a commit actually happened: the server-side debt must move.
|
|
234
|
+
let after = null;
|
|
235
|
+
try {
|
|
236
|
+
after = await probeDebt(options.endpoint, options.key, options.embeddings);
|
|
237
|
+
}
|
|
238
|
+
catch { /* treated as no movement */ }
|
|
239
|
+
const before = JSON.stringify(debt.unconsolidated ?? {});
|
|
240
|
+
const current = JSON.stringify(after?.unconsolidated ?? {});
|
|
241
|
+
if (after && before !== current) {
|
|
242
|
+
committed++;
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
log("auto: run finished but debt did not move; stopping (no commit detected)");
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (committed > 0) {
|
|
250
|
+
writeAutoState({ lastCompletedAt: now() }, env);
|
|
251
|
+
log(`auto: run complete, ${committed} batch(es) committed`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
finally {
|
|
255
|
+
releaseLock(env);
|
|
256
|
+
}
|
|
257
|
+
return { committed };
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* Trigger check called from the hook path with the reflection debt block the
|
|
261
|
+
* prepare response already carried. Fire-and-forget; never blocks the hook,
|
|
262
|
+
* never throws.
|
|
263
|
+
*/
|
|
264
|
+
export function maybeTriggerAutoConsolidate(reflection, options) {
|
|
265
|
+
const env = options.env || process.env;
|
|
266
|
+
try {
|
|
267
|
+
if (env.MEMORYROUTER_DISABLE === "1")
|
|
268
|
+
return false;
|
|
269
|
+
if (options.autoConsolidate === false)
|
|
270
|
+
return false;
|
|
271
|
+
if (!reflection || !reflection.suggestion)
|
|
272
|
+
return false;
|
|
273
|
+
if (isAutoConsolidationSession(options.sessionKey))
|
|
274
|
+
return false;
|
|
275
|
+
const now = options.now ? options.now() : Date.now();
|
|
276
|
+
if (cooldownActive(env, now))
|
|
277
|
+
return false;
|
|
278
|
+
if (lockFresh(env, now))
|
|
279
|
+
return false;
|
|
280
|
+
if (!options.runtime || typeof options.runtime.run !== "function")
|
|
281
|
+
return false;
|
|
282
|
+
appendAutoLog(`trigger: starting auto consolidation (suggestion: ${reflection.suggestion})`, env);
|
|
283
|
+
void runAutoConsolidate(options).catch((err) => {
|
|
284
|
+
appendAutoLog(`auto: unexpected error: ${err?.message}`, env);
|
|
285
|
+
});
|
|
286
|
+
return true;
|
|
287
|
+
}
|
|
288
|
+
catch (err) {
|
|
289
|
+
appendAutoLog(`trigger error: ${err?.message}`, env);
|
|
290
|
+
return false;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Automatic threshold-activated consolidation for the OpenClaw plugin.
|
|
3
|
+
*
|
|
4
|
+
* The injected suggestion nudge depends on main-session model discipline,
|
|
5
|
+
* which is a proven failure mode: the model can ignore it forever. This
|
|
6
|
+
* module instead drives consolidation deterministically from plugin code
|
|
7
|
+
* running inside the gateway, using the plugin runtime's subagent capability
|
|
8
|
+
* (api.runtime.subagent.run), which executes a dedicated isolated agent turn.
|
|
9
|
+
* That isolated turn is OpenClaw's equivalent of the headless model runs the
|
|
10
|
+
* claude-code and codex connectors use (claude -p / codex exec): the host's
|
|
11
|
+
* own model does the reflecting in a session whose only instruction is to
|
|
12
|
+
* consolidate, the user pays their own inference, and MemoryRouter never
|
|
13
|
+
* calls a provider.
|
|
14
|
+
*
|
|
15
|
+
* Reliability rails (shared with the other MemoryRouter connectors on this
|
|
16
|
+
* machine via the same files):
|
|
17
|
+
* - Lockfile ~/.memoryrouter/consolidate.lock (pid + timestamp, stale after
|
|
18
|
+
* 20 minutes) so concurrent sessions/connectors never double-fire.
|
|
19
|
+
* - Bounded: at most 3 batches per trigger, re-checking debt via the API
|
|
20
|
+
* between batches and stopping once the server reports no suggestion.
|
|
21
|
+
* - Cooldown: no re-trigger within 30 minutes of the last completed run,
|
|
22
|
+
* nor within 10 minutes of the last attempt.
|
|
23
|
+
* - The threshold always comes from the server's reflection debt block;
|
|
24
|
+
* nothing is hardcoded client-side.
|
|
25
|
+
* - Failures are silent to the user's session and logged to
|
|
26
|
+
* ~/.memoryrouter/consolidate.log.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import fs from "node:fs";
|
|
30
|
+
import os from "node:os";
|
|
31
|
+
import path from "node:path";
|
|
32
|
+
import { randomUUID } from "node:crypto";
|
|
33
|
+
import { dateSearch } from "./memoryrouter-api.js";
|
|
34
|
+
|
|
35
|
+
export const AUTO_SESSION_MARKER = "mrconsolidate";
|
|
36
|
+
export const LOCK_STALE_MS = 20 * 60 * 1000;
|
|
37
|
+
export const COOLDOWN_MS = 30 * 60 * 1000;
|
|
38
|
+
export const ATTEMPT_COOLDOWN_MS = 10 * 60 * 1000;
|
|
39
|
+
export const MAX_BATCHES_PER_TRIGGER = 3;
|
|
40
|
+
const RUN_TIMEOUT_MS = 12 * 60 * 1000; // inside the server's 15 minute lease
|
|
41
|
+
|
|
42
|
+
export type ReflectionDebtBlock = {
|
|
43
|
+
suggestion?: string | null;
|
|
44
|
+
threshold_importance?: number;
|
|
45
|
+
unconsolidated?: {
|
|
46
|
+
raw?: { units?: number; est_tokens?: number; importance_equiv?: number };
|
|
47
|
+
tier2?: { count?: number; importance?: number; est_tokens?: number };
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
type SubagentRuntime = {
|
|
52
|
+
run: (params: { sessionKey: string; message: string; deliver?: boolean; lane?: string }) => Promise<{ runId: string }>;
|
|
53
|
+
waitForRun: (params: { runId: string; timeoutMs?: number }) => Promise<{ status: string; error?: string }>;
|
|
54
|
+
deleteSession: (params: { sessionKey: string; deleteTranscript?: boolean }) => Promise<void>;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export function autoDir(env: NodeJS.ProcessEnv = process.env): string {
|
|
58
|
+
return env.MEMORYROUTER_AUTO_DIR || path.join(os.homedir(), ".memoryrouter");
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function lockPath(env?: NodeJS.ProcessEnv): string { return path.join(autoDir(env), "consolidate.lock"); }
|
|
62
|
+
function statePath(env?: NodeJS.ProcessEnv): string { return path.join(autoDir(env), "consolidate-state.json"); }
|
|
63
|
+
function logPath(env?: NodeJS.ProcessEnv): string { return path.join(autoDir(env), "consolidate.log"); }
|
|
64
|
+
|
|
65
|
+
function ensureDir(dir: string): void {
|
|
66
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function appendAutoLog(line: string, env: NodeJS.ProcessEnv = process.env): void {
|
|
70
|
+
try {
|
|
71
|
+
ensureDir(autoDir(env));
|
|
72
|
+
const file = logPath(env);
|
|
73
|
+
try {
|
|
74
|
+
if (fs.statSync(file).size > 1024 * 1024) fs.renameSync(file, `${file}.1`);
|
|
75
|
+
} catch { /* new file */ }
|
|
76
|
+
fs.appendFileSync(file, `[${new Date().toISOString()}] ${line}\n`, { mode: 0o600 });
|
|
77
|
+
} catch { /* logging never throws */ }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function readAutoState(env: NodeJS.ProcessEnv = process.env): Record<string, unknown> {
|
|
81
|
+
try { return JSON.parse(fs.readFileSync(statePath(env), "utf8")) || {}; } catch { return {}; }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function writeAutoState(patch: Record<string, unknown>, env: NodeJS.ProcessEnv = process.env): void {
|
|
85
|
+
try {
|
|
86
|
+
ensureDir(autoDir(env));
|
|
87
|
+
const next = { ...readAutoState(env), ...patch };
|
|
88
|
+
fs.writeFileSync(statePath(env), JSON.stringify(next, null, 2) + "\n", { mode: 0o600 });
|
|
89
|
+
} catch { /* state is best-effort */ }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function lockFresh(env: NodeJS.ProcessEnv = process.env, now: number = Date.now()): boolean {
|
|
93
|
+
try {
|
|
94
|
+
const info = JSON.parse(fs.readFileSync(lockPath(env), "utf8"));
|
|
95
|
+
return now - Number(info.at) <= LOCK_STALE_MS;
|
|
96
|
+
} catch (err: any) {
|
|
97
|
+
if (err?.code === "ENOENT") return false;
|
|
98
|
+
try { return now - fs.statSync(lockPath(env)).mtimeMs <= LOCK_STALE_MS; } catch { return false; }
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function acquireLock(env: NodeJS.ProcessEnv = process.env, now: number = Date.now()): boolean {
|
|
103
|
+
ensureDir(autoDir(env));
|
|
104
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
105
|
+
try {
|
|
106
|
+
const fd = fs.openSync(lockPath(env), "wx", 0o600);
|
|
107
|
+
fs.writeFileSync(fd, JSON.stringify({ pid: process.pid, at: now }) + "\n");
|
|
108
|
+
fs.closeSync(fd);
|
|
109
|
+
return true;
|
|
110
|
+
} catch (err: any) {
|
|
111
|
+
if (err?.code !== "EEXIST") return false;
|
|
112
|
+
if (lockFresh(env, now)) return false;
|
|
113
|
+
try { fs.unlinkSync(lockPath(env)); } catch { return false; }
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function releaseLock(env: NodeJS.ProcessEnv = process.env): void {
|
|
120
|
+
try { fs.unlinkSync(lockPath(env)); } catch { /* already gone */ }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function cooldownActive(env: NodeJS.ProcessEnv = process.env, now: number = Date.now()): boolean {
|
|
124
|
+
const state = readAutoState(env);
|
|
125
|
+
const completed = Number(state.lastCompletedAt);
|
|
126
|
+
if (Number.isFinite(completed) && now - completed < COOLDOWN_MS) return true;
|
|
127
|
+
const attempted = Number(state.lastAttemptAt);
|
|
128
|
+
if (Number.isFinite(attempted) && now - attempted < ATTEMPT_COOLDOWN_MS) return true;
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Tier choice comes entirely from the server's debt block. */
|
|
133
|
+
export function pickTier(debt: ReflectionDebtBlock | null | undefined): 1 | 2 {
|
|
134
|
+
const threshold = Number(debt?.threshold_importance);
|
|
135
|
+
const rawEquiv = Number(debt?.unconsolidated?.raw?.importance_equiv);
|
|
136
|
+
const tier2 = Number(debt?.unconsolidated?.tier2?.importance);
|
|
137
|
+
if (Number.isFinite(threshold) && threshold > 0) {
|
|
138
|
+
if (Number.isFinite(rawEquiv) && rawEquiv >= threshold) return 1;
|
|
139
|
+
if (Number.isFinite(tier2) && tier2 >= threshold) return 2;
|
|
140
|
+
}
|
|
141
|
+
return /tier 2/i.test(String(debt?.suggestion || "")) ? 2 : 1;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** True for sessions created by this module; hooks skip them entirely. */
|
|
145
|
+
export function isAutoConsolidationSession(sessionKey: string | undefined): boolean {
|
|
146
|
+
return typeof sessionKey === "string" && sessionKey.includes(AUTO_SESSION_MARKER);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function probeDebt(endpoint: string, key: string, embeddings?: string): Promise<ReflectionDebtBlock | null> {
|
|
150
|
+
// Sanctioned zero-billing debt read: an empty-window date-search returns
|
|
151
|
+
// the reflection debt block with zero results.
|
|
152
|
+
const from = new Date(Date.now() - 1000).toISOString();
|
|
153
|
+
const res = await dateSearch(endpoint, key, { from, max_tokens: 1000, ...(embeddings && { embeddings }) });
|
|
154
|
+
if (!res.ok) return null;
|
|
155
|
+
const data = await res.json().catch(() => null) as { reflection?: ReflectionDebtBlock } | null;
|
|
156
|
+
return data?.reflection ?? null;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// The isolated turn's message is connector transport mechanics (which tools
|
|
160
|
+
// to call, in what order). Every reflection semantic comes from the server
|
|
161
|
+
// contract that the memory_reflect tool returns verbatim.
|
|
162
|
+
function consolidationMessage(tier: 1 | 2): string {
|
|
163
|
+
return [
|
|
164
|
+
"MemoryRouter automatic consolidation run. Complete exactly one consolidation batch:",
|
|
165
|
+
`1. Call the memory_reflect tool with tier: ${tier}.`,
|
|
166
|
+
"2. If it returns batch_id null or an error, reply \"nothing to consolidate\" and stop.",
|
|
167
|
+
"3. Otherwise follow the instructions it returns to write reflection entries, then call memory_commit with that batch_id and your entries.",
|
|
168
|
+
"4. Reply \"done\" once the commit succeeds.",
|
|
169
|
+
"Do not call any other tools. Do not check out more than one batch.",
|
|
170
|
+
].join("\n");
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export type AutoConsolidateOptions = {
|
|
174
|
+
runtime: SubagentRuntime;
|
|
175
|
+
agentId: string;
|
|
176
|
+
endpoint: string;
|
|
177
|
+
key: string;
|
|
178
|
+
embeddings?: string;
|
|
179
|
+
env?: NodeJS.ProcessEnv;
|
|
180
|
+
now?: () => number;
|
|
181
|
+
log?: (line: string) => void;
|
|
182
|
+
maxBatches?: number;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* The full bounded run: lock -> up to 3 (probe debt -> isolated agent turn)
|
|
187
|
+
* cycles -> record completion -> release. Never throws.
|
|
188
|
+
*/
|
|
189
|
+
export async function runAutoConsolidate(options: AutoConsolidateOptions): Promise<{ committed: number; reason?: string }> {
|
|
190
|
+
const env = options.env || process.env;
|
|
191
|
+
const now = options.now || (() => Date.now());
|
|
192
|
+
const log = options.log || ((line: string) => appendAutoLog(line, env));
|
|
193
|
+
const bound = Math.min(options.maxBatches ?? MAX_BATCHES_PER_TRIGGER, MAX_BATCHES_PER_TRIGGER);
|
|
194
|
+
|
|
195
|
+
if (cooldownActive(env, now())) { log("auto: cooldown active; skipping"); return { committed: 0, reason: "cooldown" }; }
|
|
196
|
+
if (!acquireLock(env, now())) { log("auto: another consolidation run holds the lock; skipping"); return { committed: 0, reason: "locked" }; }
|
|
197
|
+
|
|
198
|
+
writeAutoState({ lastAttemptAt: now() }, env);
|
|
199
|
+
let committed = 0;
|
|
200
|
+
try {
|
|
201
|
+
for (let batch = 0; batch < bound; batch++) {
|
|
202
|
+
let debt: ReflectionDebtBlock | null = null;
|
|
203
|
+
try { debt = await probeDebt(options.endpoint, options.key, options.embeddings); }
|
|
204
|
+
catch (err: any) { log(`auto: debt probe failed: ${err?.message}`); break; }
|
|
205
|
+
if (!debt || !debt.suggestion) {
|
|
206
|
+
log(`auto: debt below threshold after ${committed} batch(es); stopping`);
|
|
207
|
+
break;
|
|
208
|
+
}
|
|
209
|
+
const tier = pickTier(debt);
|
|
210
|
+
const sessionKey = `agent:${options.agentId}:subagent:${AUTO_SESSION_MARKER}-${randomUUID()}`;
|
|
211
|
+
log(`auto: batch ${batch + 1}/${bound} starting (tier ${tier}; ${debt.suggestion})`);
|
|
212
|
+
let status = "error";
|
|
213
|
+
try {
|
|
214
|
+
const { runId } = await options.runtime.run({
|
|
215
|
+
sessionKey,
|
|
216
|
+
message: consolidationMessage(tier),
|
|
217
|
+
deliver: false,
|
|
218
|
+
});
|
|
219
|
+
const waited = await options.runtime.waitForRun({ runId, timeoutMs: RUN_TIMEOUT_MS });
|
|
220
|
+
status = waited.status;
|
|
221
|
+
if (waited.status !== "ok") log(`auto: isolated run ${waited.status}${waited.error ? `: ${waited.error}` : ""}`);
|
|
222
|
+
} catch (err: any) {
|
|
223
|
+
log(`auto: isolated run failed: ${err?.message}`);
|
|
224
|
+
break;
|
|
225
|
+
} finally {
|
|
226
|
+
try { await options.runtime.deleteSession({ sessionKey, deleteTranscript: true }); } catch { /* cleanup best-effort */ }
|
|
227
|
+
}
|
|
228
|
+
if (status !== "ok") break;
|
|
229
|
+
// Verify a commit actually happened: the server-side debt must move.
|
|
230
|
+
let after: ReflectionDebtBlock | null = null;
|
|
231
|
+
try { after = await probeDebt(options.endpoint, options.key, options.embeddings); } catch { /* treated as no movement */ }
|
|
232
|
+
const before = JSON.stringify(debt.unconsolidated ?? {});
|
|
233
|
+
const current = JSON.stringify(after?.unconsolidated ?? {});
|
|
234
|
+
if (after && before !== current) {
|
|
235
|
+
committed++;
|
|
236
|
+
} else {
|
|
237
|
+
log("auto: run finished but debt did not move; stopping (no commit detected)");
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
if (committed > 0) {
|
|
242
|
+
writeAutoState({ lastCompletedAt: now() }, env);
|
|
243
|
+
log(`auto: run complete, ${committed} batch(es) committed`);
|
|
244
|
+
}
|
|
245
|
+
} finally {
|
|
246
|
+
releaseLock(env);
|
|
247
|
+
}
|
|
248
|
+
return { committed };
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Trigger check called from the hook path with the reflection debt block the
|
|
253
|
+
* prepare response already carried. Fire-and-forget; never blocks the hook,
|
|
254
|
+
* never throws.
|
|
255
|
+
*/
|
|
256
|
+
export function maybeTriggerAutoConsolidate(
|
|
257
|
+
reflection: ReflectionDebtBlock | null | undefined,
|
|
258
|
+
options: Omit<AutoConsolidateOptions, "now"> & { autoConsolidate?: boolean; sessionKey?: string; now?: () => number },
|
|
259
|
+
): boolean {
|
|
260
|
+
const env = options.env || process.env;
|
|
261
|
+
try {
|
|
262
|
+
if (env.MEMORYROUTER_DISABLE === "1") return false;
|
|
263
|
+
if (options.autoConsolidate === false) return false;
|
|
264
|
+
if (!reflection || !reflection.suggestion) return false;
|
|
265
|
+
if (isAutoConsolidationSession(options.sessionKey)) return false;
|
|
266
|
+
const now = options.now ? options.now() : Date.now();
|
|
267
|
+
if (cooldownActive(env, now)) return false;
|
|
268
|
+
if (lockFresh(env, now)) return false;
|
|
269
|
+
if (!options.runtime || typeof options.runtime.run !== "function") return false;
|
|
270
|
+
appendAutoLog(`trigger: starting auto consolidation (suggestion: ${reflection.suggestion})`, env);
|
|
271
|
+
void runAutoConsolidate(options).catch((err: any) => {
|
|
272
|
+
appendAutoLog(`auto: unexpected error: ${err?.message}`, env);
|
|
273
|
+
});
|
|
274
|
+
return true;
|
|
275
|
+
} catch (err: any) {
|
|
276
|
+
appendAutoLog(`trigger error: ${err?.message}`, env);
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
279
|
+
}
|