opencode-self-improvement 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -6
- package/package.json +1 -1
- package/src/blocklist.ts +26 -11
- package/src/capture.ts +9 -2
- package/src/gates.ts +29 -0
- package/src/index.ts +11 -3
- package/src/inject.ts +9 -5
- package/src/lock.ts +202 -0
- package/src/logfmt.ts +23 -1
- package/src/memory.ts +161 -118
- package/src/portable.ts +9 -1
- package/src/sanitize.ts +9 -0
- package/src/schema.ts +18 -8
- package/src/undo.ts +61 -37
- package/src/viewer/mutate.ts +19 -9
- package/src/viewer/server.ts +6 -6
package/README.md
CHANGED
|
@@ -41,6 +41,12 @@ LLMs waste a lot of tokens reading and re-reading your code and asking you for c
|
|
|
41
41
|
superseded memories are rejected. Blocked writes are logged with a reason
|
|
42
42
|
(`memory_evidence`), and nothing is ever deleted — updates supersede the old row and
|
|
43
43
|
`memory_undo` rolls back.
|
|
44
|
+
- **Safe across concurrent sessions** — the daily store file is shared by every
|
|
45
|
+
opencode session on the machine plus the viewer, so the whole-file rewrites
|
|
46
|
+
(update, forget, undo, viewer delete) hold a per-file lock that is honored
|
|
47
|
+
across processes, and hand the new file over with an atomic rename. Another
|
|
48
|
+
session's write is never overwritten, and a reader never sees a half-written
|
|
49
|
+
store.
|
|
44
50
|
- Automatically looks for insights when you complete the verification gates (tests, linting, etc)
|
|
45
51
|
- Always defaults to memories first before sending a prompt to the LLM, which saves tokens!
|
|
46
52
|
|
|
@@ -182,7 +188,7 @@ Add a `plugins` entry to your opencode config —
|
|
|
182
188
|
```json
|
|
183
189
|
{
|
|
184
190
|
"$schema": "https://opencode.ai/config.json",
|
|
185
|
-
"plugins": ["opencode-self-improvement
|
|
191
|
+
"plugins": ["opencode-self-improvement"]
|
|
186
192
|
}
|
|
187
193
|
```
|
|
188
194
|
|
|
@@ -190,13 +196,15 @@ Opencode installs the package (and its dependencies) automatically with Bun at
|
|
|
190
196
|
startup. Quit and restart opencode; the plugin loads and `/repo-init` installs
|
|
191
197
|
itself. Nothing else to do.
|
|
192
198
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
199
|
+
An unpinned entry resolves `@latest` exactly once and is never re-resolved
|
|
200
|
+
afterwards, so updates are deliberate: `opencode plugin update
|
|
201
|
+
opencode-self-improvement`. To install a specific release, append
|
|
202
|
+
`@<version>` — see [npm](https://www.npmjs.com/package/opencode-self-improvement)
|
|
203
|
+
for what's current — and bump it when you want to move. On Opencode v2 you
|
|
204
|
+
can also manage it with the CLI:
|
|
197
205
|
|
|
198
206
|
```sh
|
|
199
|
-
opencode plugin add opencode-self-improvement
|
|
207
|
+
opencode plugin add opencode-self-improvement
|
|
200
208
|
opencode plugin update opencode-self-improvement
|
|
201
209
|
```
|
|
202
210
|
|
|
@@ -398,6 +406,7 @@ Run `bun memory-view` in your terminal and visit http://127.0.0.1:8787 in your
|
|
|
398
406
|
| `memory_remember` says blocked | A gate rejected the write (stale code reference, a cited `path/to/file.ts` that doesn't exist in the repo, overlap with a superseded memory, or an ambiguous claim). The reason is logged — check `memory_evidence`, then rephrase against current code and retry. |
|
|
399
407
|
| `invalid arguments to memory_*` | The tool call didn't match the tool's input schema — nothing was done, and the message names the offending fields. Schema validation lives at the tool boundary; gate blocks look different (`blocked at GATE (…)`). |
|
|
400
408
|
| STORE shows `N malformed` | Rows in a daily file have a `ts=` that is not a valid ISO instant (typically a hand-edit gone wrong). Every reader skips them; fix the `ts=` value or restore the file with `memory_undo`. |
|
|
409
|
+
| `memory store is busy`, or a stray `*.logfmt.lock` | Another session or the viewer has held that store file for over 5s. Ordinary writes proceed anyway; only `memory_undo` and the viewer's restore refuse, because a forced rollback under a live writer would drop rows nobody agreed to drop. Retry. A lock left behind by a killed process is taken over after 10s; `*.lock` and `*.tmp` files in the store are write bookkeeping, safe to delete while nothing is writing. |
|
|
401
410
|
| `/repo-init` missing from the command list | The self-install couldn't write to `~/.config/opencode/commands/` (e.g. read-only config dir). Create it, restart, or just ask the model to "init memory for this repo" — it calls the same tool. |
|
|
402
411
|
| Init says `Graph: unavailable` | `codebase-memory-mcp` couldn't start (not installed, or its once-only download is blocked). Install pinned `0.11.0` via the one-liner above, or point `OPENCODE_CBM_BIN` at the binary. Scripts are still remembered without it. |
|
|
403
412
|
| Stale plugin after an update | Bump the pinned version in the `plugin` entry (or `opencode plugin update opencode-self-improvement` on v2) and restart opencode. |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-self-improvement",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"description": "Durable memory for opencode: remember facts, decisions, and commands across sessions, plus a one-command per-repo setup (/repo-init).",
|
|
5
5
|
"main": "index.ts",
|
|
6
6
|
"exports": {
|
package/src/blocklist.ts
CHANGED
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
// memory scan must never pick it up.
|
|
6
6
|
import { join } from "node:path";
|
|
7
7
|
import { memoryRoot } from "./config.ts";
|
|
8
|
-
import { appendLog, formatLine, parseLine, readLines } from "./logfmt.ts";
|
|
8
|
+
import { appendLog, formatLine, parseLine, readLines, writeAtomic } from "./logfmt.ts";
|
|
9
|
+
import { withStoreLock, LOCK_WRITE } from "./lock.ts";
|
|
9
10
|
|
|
10
11
|
const BLOCKLIST_FILENAME = "blocklist.logfmt";
|
|
11
12
|
|
|
@@ -40,17 +41,31 @@ export async function blockedTs(): Promise<Set<string>> {
|
|
|
40
41
|
|
|
41
42
|
export async function addBlock(ts: string, reason: string): Promise<string> {
|
|
42
43
|
if (!reason.trim()) throw new Error("reason is required to block a memory");
|
|
43
|
-
|
|
44
|
+
const file = blocklistFile();
|
|
45
|
+
await withStoreLock(
|
|
46
|
+
file,
|
|
47
|
+
() => appendLog(file, formatLine({ ts, reason, blocked_at: new Date().toISOString() })),
|
|
48
|
+
LOCK_WRITE,
|
|
49
|
+
);
|
|
44
50
|
return ts;
|
|
45
51
|
}
|
|
46
52
|
|
|
53
|
+
// Unblocking filters the file down, so it republishes it whole: both paths hold
|
|
54
|
+
// the blocklist lock or the slower one loses the other's entries.
|
|
47
55
|
export async function removeBlock(ts: string): Promise<boolean> {
|
|
48
|
-
const file =
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
const file = blocklistFile();
|
|
57
|
+
return withStoreLock(
|
|
58
|
+
file,
|
|
59
|
+
async () => {
|
|
60
|
+
const store = Bun.file(file);
|
|
61
|
+
if (!(await store.exists())) return false;
|
|
62
|
+
const kept = (await store.text()).split("\n").filter((l) => {
|
|
63
|
+
const row = parseLine(l);
|
|
64
|
+
return !(row && row.ts === ts);
|
|
65
|
+
});
|
|
66
|
+
await writeAtomic(file, kept.join("\n") + (kept.length ? "\n" : ""));
|
|
67
|
+
return true;
|
|
68
|
+
},
|
|
69
|
+
LOCK_WRITE,
|
|
70
|
+
);
|
|
71
|
+
}
|
package/src/capture.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// All parsing functions are TOTAL: any malformed input yields [] / "" / null,
|
|
8
8
|
// never a throw.
|
|
9
9
|
import { STATUSES, TYPES } from "./types.ts";
|
|
10
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
10
11
|
|
|
11
12
|
export interface CaptureOptions {
|
|
12
13
|
model: string; // "provider/id"
|
|
@@ -97,10 +98,16 @@ export function parseCandidates(raw: string, max: number): CaptureCandidate[] {
|
|
|
97
98
|
if (out.length >= max) break;
|
|
98
99
|
if (!item || typeof item !== "object" || Array.isArray(item)) continue;
|
|
99
100
|
const o = item as Record<string, unknown>;
|
|
100
|
-
const
|
|
101
|
-
const
|
|
101
|
+
const rawType = typeof o.type === "string" ? o.type : "";
|
|
102
|
+
const rawScope = typeof o.scope === "string" ? o.scope : "";
|
|
103
|
+
const type = rawType.trim();
|
|
104
|
+
const scope = rawScope.trim();
|
|
102
105
|
const content = typeof o.content === "string" ? o.content.trim() : "";
|
|
103
106
|
if (!type || !scope || !content) continue;
|
|
107
|
+
if (
|
|
108
|
+
flattenMemoryText(rawType) !== rawType ||
|
|
109
|
+
flattenMemoryText(rawScope) !== rawScope
|
|
110
|
+
) continue;
|
|
104
111
|
if (!VALID_TYPES.has(type)) continue;
|
|
105
112
|
if (content.length < 20 || content.length > 600) continue;
|
|
106
113
|
const status = o.status === "open" || o.status === "settled" ? o.status : undefined;
|
package/src/gates.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
type Memory,
|
|
15
15
|
} from "./memory.ts";
|
|
16
16
|
import { chainId, contentHash, record, recordWriteBlocked } from "./evidence.ts";
|
|
17
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
17
18
|
import {
|
|
18
19
|
displayCall,
|
|
19
20
|
extractCallForms,
|
|
@@ -222,6 +223,34 @@ export async function runWriteGate(
|
|
|
222
223
|
};
|
|
223
224
|
}
|
|
224
225
|
|
|
226
|
+
const badMetadata: string[] = ([
|
|
227
|
+
["type", candidate.type],
|
|
228
|
+
["scope", candidate.scope],
|
|
229
|
+
["issue", candidate.issue],
|
|
230
|
+
] as const)
|
|
231
|
+
.filter(
|
|
232
|
+
([, value]) =>
|
|
233
|
+
value !== undefined &&
|
|
234
|
+
(typeof value !== "string" || flattenMemoryText(value) !== value),
|
|
235
|
+
)
|
|
236
|
+
.map(([field]) => field);
|
|
237
|
+
if (
|
|
238
|
+
candidate.tags !== undefined &&
|
|
239
|
+
(!Array.isArray(candidate.tags) ||
|
|
240
|
+
candidate.tags.some((tag) => typeof tag !== "string" || flattenMemoryText(tag) !== tag))
|
|
241
|
+
) {
|
|
242
|
+
badMetadata.push("tags");
|
|
243
|
+
}
|
|
244
|
+
if (badMetadata.length) {
|
|
245
|
+
await recordWriteBlocked(4, "invalid_metadata", [`fields=${badMetadata.join(",")}`], chain);
|
|
246
|
+
return {
|
|
247
|
+
action: "blocked",
|
|
248
|
+
reason: "invalid_metadata",
|
|
249
|
+
chain,
|
|
250
|
+
hint: `${badMetadata.join(", ")} must not contain control characters.`,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
225
254
|
const ch = contentHash(candidate.content);
|
|
226
255
|
await record("gate_start", {
|
|
227
256
|
chain,
|
package/src/index.ts
CHANGED
|
@@ -65,6 +65,7 @@ import { getCurrent, setCurrent } from "./state.ts";
|
|
|
65
65
|
import { resolveRepo, setRepoScope, rememberSessionDir, sessionDir } from "./repokey.ts";
|
|
66
66
|
import { readLines } from "./logfmt.ts";
|
|
67
67
|
import { buildCapturePrompt, flattenTranscript, parseCandidates, parseModelRef } from "./capture.ts";
|
|
68
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
68
69
|
|
|
69
70
|
// Tool/input boundaries are zod schemas (src/schema.ts): the JSON Schema the
|
|
70
71
|
// host sees is GENERATED from them, and raw args are re-parsed at execution.
|
|
@@ -387,7 +388,10 @@ const impl = {
|
|
|
387
388
|
types.set(m.type, (types.get(m.type) ?? 0) + 1);
|
|
388
389
|
}
|
|
389
390
|
const fmt = (map: Map<string, number>) =>
|
|
390
|
-
[...map.entries()]
|
|
391
|
+
[...map.entries()]
|
|
392
|
+
.sort((a, b) => b[1] - a[1])
|
|
393
|
+
.map(([k, c]) => ` ${flattenMemoryText(k)}: ${c}`)
|
|
394
|
+
.join("\n");
|
|
391
395
|
return { content: `Total: ${mems.length}\n\nScopes:\n${fmt(scopes)}\n\nTypes:\n${fmt(types)}` };
|
|
392
396
|
},
|
|
393
397
|
}));
|
|
@@ -415,7 +419,8 @@ const impl = {
|
|
|
415
419
|
const kept = res.mode === "merge" && (res.concurrentAdded.length || res.concurrentChanged.length)
|
|
416
420
|
? ` Kept ${res.concurrentAdded.length} added / ${res.concurrentChanged.length} changed row(s) written after the snapshot.`
|
|
417
421
|
: "";
|
|
418
|
-
|
|
422
|
+
const safety = res.safety ? ` Pre-restore file snapshotted -> ${res.safety}.` : "";
|
|
423
|
+
return { content: `Restored ${basename(res.target)} from ${res.mode} snapshot -> ${res.target}.${kept}${safety}` };
|
|
419
424
|
}
|
|
420
425
|
const snaps = await listSnapshots();
|
|
421
426
|
if (!snaps.length) return { content: "No snapshots" };
|
|
@@ -522,7 +527,10 @@ const impl = {
|
|
|
522
527
|
return { content: `No active open threads older than ${days} days.` };
|
|
523
528
|
}
|
|
524
529
|
const listing = stale
|
|
525
|
-
.map(
|
|
530
|
+
.map(
|
|
531
|
+
(m) =>
|
|
532
|
+
` ${flattenMemoryText(m.ts)} ${flattenMemoryText(m.type)}/${flattenMemoryText(m.scope)}: ${flattenMemoryText(m.content).slice(0, 100)}`,
|
|
533
|
+
)
|
|
526
534
|
.join("\n");
|
|
527
535
|
if (dryRun) {
|
|
528
536
|
await record(
|
package/src/inject.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { record } from "./evidence.ts";
|
|
|
10
10
|
import { dateStr } from "./config.ts";
|
|
11
11
|
import { bracketList } from "./logfmt.ts";
|
|
12
12
|
import { recentConvergenceSummary } from "./triggers.ts";
|
|
13
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
13
14
|
|
|
14
15
|
export interface TransformOutput {
|
|
15
16
|
system: string[];
|
|
@@ -53,9 +54,12 @@ export function formatMemory(m: Memory): string {
|
|
|
53
54
|
// One memory is ONE line in every injected block: the store round-trips real
|
|
54
55
|
// newlines, so a planted "\n## Fake header" would forge its own section and
|
|
55
56
|
// escape the framing (and throw capToBudget's line accounting off). Line
|
|
56
|
-
// breaks collapse to spaces; ordinary spacing survives.
|
|
57
|
-
const
|
|
58
|
-
|
|
57
|
+
// breaks and other controls collapse to spaces; ordinary spacing survives.
|
|
58
|
+
const date = flattenMemoryText(dateOf(m.ts));
|
|
59
|
+
const type = flattenMemoryText(m.type);
|
|
60
|
+
const scope = flattenMemoryText(m.scope);
|
|
61
|
+
const content = flattenMemoryText(m.content);
|
|
62
|
+
return `- [${date}]${tag}${src} ${type}/${scope}: ${content}`;
|
|
59
63
|
}
|
|
60
64
|
|
|
61
65
|
function argText(args: unknown): string {
|
|
@@ -141,7 +145,7 @@ export async function systemDigest(
|
|
|
141
145
|
// Repo-scoped: model-facing reads see this repo + global rows.
|
|
142
146
|
const mems = visibleHere((await loadAll()).filter(isActive));
|
|
143
147
|
if (!mems.length) return;
|
|
144
|
-
const latest = mems.map((m) => m.ts).sort().pop() ?? "?";
|
|
148
|
+
const latest = flattenMemoryText(mems.map((m) => m.ts).sort().pop() ?? "?");
|
|
145
149
|
output.system.push(
|
|
146
150
|
"\n## Persistent memory (opencode-self-improvement)\n" +
|
|
147
151
|
`${mems.length} durable memories; latest: ${latest}. ` +
|
|
@@ -177,7 +181,7 @@ export async function systemDigest(
|
|
|
177
181
|
"frontier_injected",
|
|
178
182
|
{
|
|
179
183
|
session: sid,
|
|
180
|
-
scopes: bracketList([...new Set(open.map((m) => m.scope))]),
|
|
184
|
+
scopes: bracketList([...new Set(open.map((m) => flattenMemoryText(m.scope)))]),
|
|
181
185
|
shown: open.length,
|
|
182
186
|
open_total: openMems.length,
|
|
183
187
|
ts_since: dateStr(),
|
package/src/lock.ts
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
// Serialization for the store's write paths.
|
|
2
|
+
//
|
|
3
|
+
// Daily store files are shared by every opencode session on the machine plus
|
|
4
|
+
// the viewer, which is a separate process. Appending is atomic on its own
|
|
5
|
+
// (O_APPEND, see logfmt.ts), but supersede, prune, viewer delete and snapshot
|
|
6
|
+
// restore each REWRITE a whole file: a writer that read before someone else
|
|
7
|
+
// wrote publishes a stale copy and silently drops the other's rows. Measured
|
|
8
|
+
// before this module existed: 40 concurrent appends against 40 supersedes left
|
|
9
|
+
// 40 rows on disk; four processes writing one store kept 20 of 120.
|
|
10
|
+
//
|
|
11
|
+
// One lock per store file, held across the whole read-modify-write. Appends take
|
|
12
|
+
// the same lock so they cannot land inside an RMW window. Two layers, sharing
|
|
13
|
+
// one timeout budget:
|
|
14
|
+
// - a FIFO hand-off between async tasks in THIS process, so contending
|
|
15
|
+
// writers take their turn instead of spinning;
|
|
16
|
+
// - a lock FILE created with O_EXCL, which is atomic across processes (and
|
|
17
|
+
// exclusive against our own tasks too, since the check is in the kernel).
|
|
18
|
+
// A killed holder is recovered by age. A writer that waits past the budget
|
|
19
|
+
// gives up waiting — see onTimeout.
|
|
20
|
+
//
|
|
21
|
+
// Readers take NO lock: publishers hand over the new file with a rename
|
|
22
|
+
// (writeAtomic in logfmt.ts), so a reader always sees a complete store, never
|
|
23
|
+
// the middle of a rewrite.
|
|
24
|
+
import { mkdir, open, stat, unlink } from "node:fs/promises";
|
|
25
|
+
import { basename, dirname } from "node:path";
|
|
26
|
+
import { record } from "./evidence.ts";
|
|
27
|
+
|
|
28
|
+
const LOCK_SUFFIX = ".lock";
|
|
29
|
+
// A lock older than this belongs to a process that died holding it.
|
|
30
|
+
const STALE_MS = 10_000;
|
|
31
|
+
const DEFAULT_TIMEOUT_MS = 5_000;
|
|
32
|
+
const MIN_BACKOFF_MS = 1;
|
|
33
|
+
const MAX_BACKOFF_MS = 25;
|
|
34
|
+
|
|
35
|
+
export function lockFileFor(file: string): string {
|
|
36
|
+
return file + LOCK_SUFFIX;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Waiting past the timeout, or taking over a dead holder's lock. */
|
|
40
|
+
export type LockEvent = "steal" | "timeout";
|
|
41
|
+
|
|
42
|
+
export interface LockOptions {
|
|
43
|
+
timeoutMs?: number;
|
|
44
|
+
/**
|
|
45
|
+
* What waiting past `timeoutMs` means. "fail" refuses the mutation (for
|
|
46
|
+
* callers where proceeding unlocked could destroy more than it saves, e.g.
|
|
47
|
+
* snapshot restore). "proceed" runs it anyway: strictly no worse than the
|
|
48
|
+
* pre-lock behaviour, and it never costs a user their write.
|
|
49
|
+
*/
|
|
50
|
+
onTimeout?: "proceed" | "fail";
|
|
51
|
+
onEvent?: (event: LockEvent, waitedMs: number) => void;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// How each store caller treats contention. An ordinary write PROCEEDS past the
|
|
55
|
+
// timeout rather than lose the user's memory, and the wait is audited; restore
|
|
56
|
+
// fails closed, because "force" drops rows and must not guess under a racing
|
|
57
|
+
// writer.
|
|
58
|
+
export const LOCK_WRITE: LockOptions = {
|
|
59
|
+
onEvent: (event, waitedMs) => {
|
|
60
|
+
void record("store_lock", { event, waited_ms: waitedMs, op: "write" });
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export const LOCK_RESTORE: LockOptions = {
|
|
65
|
+
onTimeout: "fail",
|
|
66
|
+
onEvent: (event, waitedMs) => {
|
|
67
|
+
void record("store_lock", { event, waited_ms: waitedMs, op: "restore" });
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
72
|
+
|
|
73
|
+
// Resolves when the task stored as this key has finished, handing the turn to
|
|
74
|
+
// whoever queued after it.
|
|
75
|
+
const turns = new Map<string, Promise<void>>();
|
|
76
|
+
|
|
77
|
+
async function timed(prev: Promise<void>, ms: number): Promise<boolean> {
|
|
78
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
79
|
+
const gained = await Promise.race([
|
|
80
|
+
prev.then(() => true),
|
|
81
|
+
new Promise<boolean>((res) => (timer = setTimeout(() => res(false), ms))),
|
|
82
|
+
]);
|
|
83
|
+
clearTimeout(timer);
|
|
84
|
+
return gained;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function stealIfStale(lockPath: string, now: number): Promise<boolean> {
|
|
88
|
+
let age = -1;
|
|
89
|
+
try {
|
|
90
|
+
age = now - (await stat(lockPath)).mtimeMs;
|
|
91
|
+
} catch {
|
|
92
|
+
return true; // vanished between EEXIST and stat: retry the create
|
|
93
|
+
}
|
|
94
|
+
if (age < STALE_MS) return false;
|
|
95
|
+
try {
|
|
96
|
+
await unlink(lockPath);
|
|
97
|
+
} catch {
|
|
98
|
+
return false; // another stealer won; its create decides
|
|
99
|
+
}
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Cross-process half: hold the lock file while `fn` runs. */
|
|
104
|
+
async function holdFileLock<T>(
|
|
105
|
+
lockPath: string,
|
|
106
|
+
file: string,
|
|
107
|
+
fn: () => Promise<T>,
|
|
108
|
+
opts: LockOptions,
|
|
109
|
+
budgetMs: number,
|
|
110
|
+
): Promise<T> {
|
|
111
|
+
await mkdir(dirname(lockPath), { recursive: true });
|
|
112
|
+
const token = `${process.pid}-${Math.random().toString(36).slice(2)}`;
|
|
113
|
+
const started = Date.now();
|
|
114
|
+
let backoff = MIN_BACKOFF_MS;
|
|
115
|
+
for (;;) {
|
|
116
|
+
try {
|
|
117
|
+
const handle = await open(lockPath, "wx");
|
|
118
|
+
try {
|
|
119
|
+
await handle.write(token);
|
|
120
|
+
} finally {
|
|
121
|
+
await handle.close();
|
|
122
|
+
}
|
|
123
|
+
break;
|
|
124
|
+
} catch (err) {
|
|
125
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
126
|
+
if (code !== "EEXIST") throw err;
|
|
127
|
+
if (await stealIfStale(lockPath, Date.now())) {
|
|
128
|
+
opts.onEvent?.("steal", Date.now() - started);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (Date.now() - started >= budgetMs) {
|
|
132
|
+
opts.onEvent?.("timeout", Date.now() - started);
|
|
133
|
+
if (opts.onTimeout === "fail") {
|
|
134
|
+
throw new Error(
|
|
135
|
+
`memory store is busy: ${basename(file)} has been locked for ${Date.now() - started}ms — retry`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
return fn();
|
|
139
|
+
}
|
|
140
|
+
await sleep(backoff);
|
|
141
|
+
backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
try {
|
|
145
|
+
return await fn();
|
|
146
|
+
} finally {
|
|
147
|
+
// Never release a lock that is no longer ours: if we ran long and someone
|
|
148
|
+
// stole it, unlinking would clobber the new holder.
|
|
149
|
+
try {
|
|
150
|
+
if ((await Bun.file(lockPath).text()) === token) await unlink(lockPath);
|
|
151
|
+
} catch {
|
|
152
|
+
// already gone
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Run `fn` as the only writer of `file`, in this process and every other one.
|
|
159
|
+
* Sections run flat: calling this again for the same file from inside `fn`
|
|
160
|
+
* waits for its own lock.
|
|
161
|
+
*/
|
|
162
|
+
export async function withStoreLock<T>(
|
|
163
|
+
file: string,
|
|
164
|
+
fn: () => Promise<T>,
|
|
165
|
+
opts: LockOptions = {},
|
|
166
|
+
): Promise<T> {
|
|
167
|
+
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
168
|
+
const started = Date.now();
|
|
169
|
+
const lockPath = lockFileFor(file);
|
|
170
|
+
|
|
171
|
+
let releaseTurn!: () => void;
|
|
172
|
+
const myTurnEnded = new Promise<void>((res) => (releaseTurn = res));
|
|
173
|
+
const queuedBehind = turns.get(file);
|
|
174
|
+
turns.set(file, myTurnEnded);
|
|
175
|
+
|
|
176
|
+
const finish = () => {
|
|
177
|
+
releaseTurn();
|
|
178
|
+
if (turns.get(file) === myTurnEnded) turns.delete(file);
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
if (queuedBehind) {
|
|
182
|
+
const gained = await timed(queuedBehind, timeoutMs);
|
|
183
|
+
if (!gained) {
|
|
184
|
+
// Give up our place in line rather than park everyone behind us on a
|
|
185
|
+
// writer that is not going to finish.
|
|
186
|
+
finish();
|
|
187
|
+
opts.onEvent?.("timeout", Date.now() - started);
|
|
188
|
+
if (opts.onTimeout === "fail") {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`memory store is busy: ${basename(file)} has been locked for ${Date.now() - started}ms — retry`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
return fn();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
try {
|
|
198
|
+
return await holdFileLock(lockPath, file, fn, opts, timeoutMs - (Date.now() - started));
|
|
199
|
+
} finally {
|
|
200
|
+
finish();
|
|
201
|
+
}
|
|
202
|
+
}
|
package/src/logfmt.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Minimal logfmt reader/writer shared by the durable store and the evidence
|
|
2
2
|
// log. Field order is preserved on write so chains are diffable.
|
|
3
3
|
import { dirname } from "node:path";
|
|
4
|
-
import { appendFile, mkdir } from "node:fs/promises";
|
|
4
|
+
import { appendFile, mkdir, rename, unlink } from "node:fs/promises";
|
|
5
5
|
|
|
6
6
|
export type Fields = Record<string, unknown>;
|
|
7
7
|
|
|
@@ -88,9 +88,31 @@ export async function appendLog(file: string, line: string): Promise<void> {
|
|
|
88
88
|
// covering interleaved async tasks in the service AND a second process
|
|
89
89
|
// (the viewer) appending the same day file. Do NOT "simplify" back to
|
|
90
90
|
// Bun.write with an options object: on Bun 1.4.3 that silently TRUNCATES.
|
|
91
|
+
// Appending is safe against other appenders only. Callers writing a file that
|
|
92
|
+
// is ALSO rewritten whole (a daily memory file, the blocklist) hold the store
|
|
93
|
+
// lock around this — see lock.ts.
|
|
91
94
|
await appendFile(file, next);
|
|
92
95
|
}
|
|
93
96
|
|
|
97
|
+
// Publish whole-file content so no reader can observe a partial store. Recall,
|
|
98
|
+
// injection and the viewer read these files without taking the lock, so the
|
|
99
|
+
// handover is a same-directory rename (atomic on POSIX): a reader sees either
|
|
100
|
+
// the previous file or the new one, never a truncated middle. Callers hold the
|
|
101
|
+
// store lock for the whole read-modify-write.
|
|
102
|
+
export async function writeAtomic(file: string, content: string): Promise<void> {
|
|
103
|
+
await ensureDir(dirname(file));
|
|
104
|
+
// One writer per file (the store lock), so the temp name is stable per
|
|
105
|
+
// process and a crash leaves one leftover instead of one per attempt.
|
|
106
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
107
|
+
try {
|
|
108
|
+
await Bun.write(tmp, content);
|
|
109
|
+
await rename(tmp, file);
|
|
110
|
+
} catch (err) {
|
|
111
|
+
await unlink(tmp).catch(() => undefined);
|
|
112
|
+
throw err;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
94
116
|
export async function readLines(file: string): Promise<string[]> {
|
|
95
117
|
const f = Bun.file(file);
|
|
96
118
|
if (!(await f.exists())) return [];
|
package/src/memory.ts
CHANGED
|
@@ -3,12 +3,15 @@
|
|
|
3
3
|
// vendor/opencode-memory/LICENSE) and extends it with the fields the write-gate
|
|
4
4
|
// needs: supersession markers, snapshots and audited deletion.
|
|
5
5
|
import { existsSync } from "node:fs";
|
|
6
|
+
import { open } from "node:fs/promises";
|
|
6
7
|
import { join, basename } from "node:path";
|
|
7
|
-
import { memoryRoot, deletionsFile, snapshotDir } from "./config.ts";
|
|
8
|
+
import { dateStr, memoryRoot, deletionsFile, snapshotDir } from "./config.ts";
|
|
8
9
|
import { repoScopeFor } from "./repokey.ts";
|
|
9
10
|
import { getCurrent } from "./state.ts";
|
|
10
|
-
import { appendLog, ensureDir, formatLine, parseLine, readLines } from "./logfmt.ts";
|
|
11
|
+
import { appendLog, ensureDir, formatLine, parseLine, readLines, writeAtomic, type Fields } from "./logfmt.ts";
|
|
12
|
+
import { withStoreLock, LOCK_WRITE } from "./lock.ts";
|
|
11
13
|
import { blockedTs } from "./blocklist.ts";
|
|
14
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
12
15
|
|
|
13
16
|
export type MemoryStatus = "open" | "settled";
|
|
14
17
|
|
|
@@ -143,16 +146,15 @@ export interface AppendMemoryInput {
|
|
|
143
146
|
repoPath?: string;
|
|
144
147
|
}
|
|
145
148
|
|
|
146
|
-
//
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
|
|
149
|
+
// ts values already in `file`. An appended row's ts is "now", so the only rows
|
|
150
|
+
// it can collide with are ones this same file received in the same millisecond;
|
|
151
|
+
// scanning just it keeps the locked section to one read of one file. (A row
|
|
152
|
+
// hand-placed with today's ts in another day's file is outside this promise.)
|
|
153
|
+
async function existingTs(file: string): Promise<Set<string>> {
|
|
150
154
|
const set = new Set<string>();
|
|
151
|
-
for (const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
if (f && typeof f.ts === "string") set.add(f.ts);
|
|
155
|
-
}
|
|
155
|
+
for (const line of await readLines(file)) {
|
|
156
|
+
const f = parseLine(line);
|
|
157
|
+
if (f && typeof f.ts === "string") set.add(f.ts);
|
|
156
158
|
}
|
|
157
159
|
return set;
|
|
158
160
|
}
|
|
@@ -166,17 +168,23 @@ export async function appendMemory(input: AppendMemoryInput): Promise<string> {
|
|
|
166
168
|
if (input.content.length > MAX_CONTENT_CHARS) {
|
|
167
169
|
throw new Error(`memory content exceeds ${MAX_CONTENT_CHARS} chars (${input.content.length})`);
|
|
168
170
|
}
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
171
|
+
for (const [field, value] of [
|
|
172
|
+
["type", input.type],
|
|
173
|
+
["scope", input.scope],
|
|
174
|
+
["issue", input.issue],
|
|
175
|
+
] as const) {
|
|
176
|
+
if (value !== undefined && flattenMemoryText(value) !== value) {
|
|
177
|
+
throw new Error(`memory ${field} must not contain control characters`);
|
|
178
|
+
}
|
|
174
179
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
+
if (input.tags?.some((tag) => flattenMemoryText(tag) !== tag)) {
|
|
181
|
+
throw new Error("memory tags must not contain control characters");
|
|
182
|
+
}
|
|
183
|
+
// Repo stamp at the single write choke point: gated writes, viewer restores
|
|
184
|
+
// and edits all pass through here. Explicit "global" survives; an unscoped
|
|
185
|
+
// process (viewer) stamps nothing = global. Session identity wins over the
|
|
186
|
+
// module scope so a server hosting several workspaces stamps the writing
|
|
187
|
+
// session's repo.
|
|
180
188
|
let repo = input.repo;
|
|
181
189
|
let repoPath = input.repoPath;
|
|
182
190
|
if (repo === undefined) {
|
|
@@ -186,22 +194,36 @@ export async function appendMemory(input: AppendMemoryInput): Promise<string> {
|
|
|
186
194
|
repoPath = cur.path;
|
|
187
195
|
}
|
|
188
196
|
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
|
|
197
|
+
// Day file chosen before the lock (it IS the lock key). A ts bumped across
|
|
198
|
+
// midnight therefore lands under the previous day's name; readers glob every
|
|
199
|
+
// file and decay compares ts values, so the grouping is cosmetic.
|
|
200
|
+
const file = join(memoryRoot(), `${dateStr()}.logfmt`);
|
|
201
|
+
// Locked end to end: seeding ts from a scan another process can race yields
|
|
202
|
+
// two rows with the same identity, and an append that lands inside someone
|
|
203
|
+
// else's whole-file rewrite is simply not there afterwards.
|
|
204
|
+
return withStoreLock(file, async () => {
|
|
205
|
+
const taken = await existingTs(file);
|
|
206
|
+
let ts = new Date().toISOString();
|
|
207
|
+
let guard = 0;
|
|
208
|
+
while (taken.has(ts) && guard++ < 1000) {
|
|
209
|
+
ts = new Date(new Date(ts).getTime() + 1).toISOString();
|
|
210
|
+
}
|
|
211
|
+
await appendLog(
|
|
212
|
+
file,
|
|
213
|
+
formatLine({
|
|
214
|
+
ts,
|
|
215
|
+
type: input.type,
|
|
216
|
+
scope: input.scope,
|
|
217
|
+
content: input.content,
|
|
218
|
+
issue: input.issue,
|
|
219
|
+
tags: input.tags?.length ? input.tags.join(",") : undefined,
|
|
220
|
+
status: input.status,
|
|
221
|
+
repo,
|
|
222
|
+
repoPath,
|
|
223
|
+
}),
|
|
224
|
+
);
|
|
225
|
+
return ts;
|
|
226
|
+
}, LOCK_WRITE);
|
|
205
227
|
}
|
|
206
228
|
|
|
207
229
|
export interface SupersedeFields {
|
|
@@ -213,44 +235,52 @@ export interface SupersedeFields {
|
|
|
213
235
|
entity?: string;
|
|
214
236
|
}
|
|
215
237
|
|
|
216
|
-
// Copy `file` into snapshots/ before a mutation.
|
|
217
|
-
// `<stamp>-<original>` because /undo strips the stamp to find its target, so a
|
|
218
|
-
// same-millisecond collision would clobber an earlier, still-undoable snapshot
|
|
219
|
-
// (supersede + clear in one merge_review can land in the same millisecond).
|
|
238
|
+
// Copy `file` into snapshots/ before a mutation.
|
|
220
239
|
async function snapshotBefore(file: string): Promise<void> {
|
|
221
|
-
await
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
240
|
+
await snapshotFiles([file]);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// Read-modify-write ONE row of a store file: read, rewrite the line, publish
|
|
244
|
+
// atomically, all under the store lock, so an append or another rewrite by a
|
|
245
|
+
// concurrent session cannot be overwritten by our stale copy. Resolves to null
|
|
246
|
+
// when the row is not in this file, otherwise the mutation's outcome.
|
|
247
|
+
async function stampRow(
|
|
248
|
+
file: string,
|
|
249
|
+
ts: string,
|
|
250
|
+
mutate: (fields: Fields) => string | null,
|
|
251
|
+
): Promise<boolean | null> {
|
|
252
|
+
return withStoreLock(
|
|
253
|
+
file,
|
|
254
|
+
async () => {
|
|
255
|
+
const lines = await readLines(file);
|
|
256
|
+
const idx = lines.findIndex((l) => {
|
|
257
|
+
const f = parseLine(l);
|
|
258
|
+
return f && f.ts === ts;
|
|
259
|
+
});
|
|
260
|
+
if (idx === -1) return null;
|
|
261
|
+
const parsed = parseLine(lines[idx]);
|
|
262
|
+
if (!parsed) return false;
|
|
263
|
+
const next = mutate(parsed);
|
|
264
|
+
if (next === null) return false;
|
|
265
|
+
lines[idx] = next;
|
|
266
|
+
await snapshotBefore(file);
|
|
267
|
+
await writeAtomic(file, lines.join("\n") + "\n");
|
|
268
|
+
return true;
|
|
269
|
+
},
|
|
270
|
+
LOCK_WRITE,
|
|
271
|
+
);
|
|
226
272
|
}
|
|
227
273
|
|
|
228
274
|
// Mark an existing memory as superseded IN PLACE (entry retained, recoverable).
|
|
229
275
|
// Idempotent: a second call is a no-op, so no double `superseded_by`.
|
|
230
276
|
export async function markSuperseded(ts: string, fields: SupersedeFields): Promise<boolean> {
|
|
231
277
|
for (const file of await memoryFiles()) {
|
|
232
|
-
|
|
233
|
-
const idx = lines.findIndex((l) => {
|
|
234
|
-
const f = parseLine(l);
|
|
235
|
-
return f && f.ts === ts;
|
|
236
|
-
});
|
|
237
|
-
if (idx === -1) continue;
|
|
238
|
-
const parsed = parseLine(lines[idx]);
|
|
239
|
-
if (parsed && typeof parsed.superseded_by === "string") return false;
|
|
240
|
-
// Rewrite the whole line rather than appending to it, so an old-format line
|
|
278
|
+
// The whole line is rewritten rather than extended, so an old-format line
|
|
241
279
|
// comes out in the current format with its values unchanged.
|
|
242
|
-
|
|
243
|
-
...parsed,
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
symbol: fields.symbol,
|
|
247
|
-
from: fields.from,
|
|
248
|
-
to: fields.to,
|
|
249
|
-
entity: fields.entity,
|
|
250
|
-
});
|
|
251
|
-
await snapshotBefore(file);
|
|
252
|
-
await Bun.write(file, lines.join("\n") + "\n");
|
|
253
|
-
return true;
|
|
280
|
+
const outcome = await stampRow(file, ts, (parsed) =>
|
|
281
|
+
typeof parsed.superseded_by === "string" ? null : formatLine({ ...parsed, ...fields }),
|
|
282
|
+
);
|
|
283
|
+
if (outcome !== null) return outcome;
|
|
254
284
|
}
|
|
255
285
|
return false;
|
|
256
286
|
}
|
|
@@ -260,25 +290,18 @@ export async function markSuperseded(ts: string, fields: SupersedeFields): Promi
|
|
|
260
290
|
// even when an earlier row — now the loser — had superseded it first.
|
|
261
291
|
export async function clearSuperseded(ts: string): Promise<boolean> {
|
|
262
292
|
for (const file of await memoryFiles()) {
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
293
|
+
const outcome = await stampRow(file, ts, (parsed) => {
|
|
294
|
+
if (typeof parsed.superseded_by !== "string") return null;
|
|
295
|
+
// These five keys are written only by markSuperseded (drift, canonical,
|
|
296
|
+
// update and merge_review), never by appendMemory, so they are all part of
|
|
297
|
+
// the marker and go with it. Field order round-trips because parseLine
|
|
298
|
+
// inserts keys in source order.
|
|
299
|
+
for (const key of ["superseded_by", "reason", "symbol", "from", "to", "entity"]) {
|
|
300
|
+
delete parsed[key];
|
|
301
|
+
}
|
|
302
|
+
return formatLine(parsed);
|
|
267
303
|
});
|
|
268
|
-
if (
|
|
269
|
-
const parsed = parseLine(lines[idx]);
|
|
270
|
-
if (!parsed || typeof parsed.superseded_by !== "string") return false;
|
|
271
|
-
// These five keys are written only by markSuperseded (drift, canonical,
|
|
272
|
-
// update and merge_review), never by appendMemory, so they are all part of
|
|
273
|
-
// the marker and go with it. Field order round-trips because parseLine
|
|
274
|
-
// inserts keys in source order.
|
|
275
|
-
for (const key of ["superseded_by", "reason", "symbol", "from", "to", "entity"]) {
|
|
276
|
-
delete parsed[key];
|
|
277
|
-
}
|
|
278
|
-
lines[idx] = formatLine(parsed);
|
|
279
|
-
await snapshotBefore(file);
|
|
280
|
-
await Bun.write(file, lines.join("\n") + "\n");
|
|
281
|
-
return true;
|
|
304
|
+
if (outcome !== null) return outcome;
|
|
282
305
|
}
|
|
283
306
|
return false;
|
|
284
307
|
}
|
|
@@ -334,24 +357,35 @@ export interface DeletedMemory {
|
|
|
334
357
|
reason: string;
|
|
335
358
|
}
|
|
336
359
|
|
|
360
|
+
// Claim a snapshot name with O_EXCL so "never overwrite a snapshot" holds
|
|
361
|
+
// across processes, not just inside one: two sessions mutating the same file in
|
|
362
|
+
// one millisecond both bump instead of one clobbering the other's still-undoable
|
|
363
|
+
// copy. Names are `<epoch-ms>-<original>` because /undo strips the leading stamp
|
|
364
|
+
// to find its target, so nothing may go in front of it.
|
|
365
|
+
async function claimSnapshot(file: string): Promise<string> {
|
|
366
|
+
await ensureDir(snapshotDir());
|
|
367
|
+
let stamp = Date.now();
|
|
368
|
+
for (;;) {
|
|
369
|
+
const snap = join(snapshotDir(), `${stamp}-${basename(file)}`);
|
|
370
|
+
try {
|
|
371
|
+
const handle = await open(snap, "wx");
|
|
372
|
+
await handle.close();
|
|
373
|
+
return snap;
|
|
374
|
+
} catch (err) {
|
|
375
|
+
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
|
376
|
+
stamp++;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
337
381
|
// Snapshot-before-mutate: copy each affected file into snapshots/ so /undo can
|
|
338
382
|
// restore it. Returns the snapshot paths keyed by original file.
|
|
339
383
|
export async function snapshotFiles(files: string[]): Promise<string[]> {
|
|
340
|
-
await ensureDir(snapshotDir());
|
|
341
|
-
let stamp = Date.now();
|
|
342
384
|
const snaps: string[] = [];
|
|
343
385
|
for (const file of files) {
|
|
344
386
|
const f = Bun.file(file);
|
|
345
387
|
if (!(await f.exists())) continue;
|
|
346
|
-
|
|
347
|
-
// mutations happen in the same millisecond, and a clobbered snapshot once
|
|
348
|
-
// let a restore read post-edit content. Bump until free; never overwrite
|
|
349
|
-
// a snapshot.
|
|
350
|
-
let snap = join(snapshotDir(), `${stamp}-${basename(file)}`);
|
|
351
|
-
while (await Bun.file(snap).exists()) {
|
|
352
|
-
stamp++;
|
|
353
|
-
snap = join(snapshotDir(), `${stamp}-${basename(file)}`);
|
|
354
|
-
}
|
|
388
|
+
const snap = await claimSnapshot(file);
|
|
355
389
|
await Bun.write(snap, await f.text());
|
|
356
390
|
snaps.push(snap);
|
|
357
391
|
}
|
|
@@ -379,30 +413,39 @@ export interface PruneResult {
|
|
|
379
413
|
}
|
|
380
414
|
|
|
381
415
|
// Remove matching memories (used by memory_forget), snapshotting first and
|
|
382
|
-
// logging every deletion with its reason.
|
|
416
|
+
// logging every deletion with its reason. Each file is filtered and republished
|
|
417
|
+
// under the store lock, so a row another session appended while we were reading
|
|
418
|
+
// is not quietly filtered out along with the targets.
|
|
383
419
|
export async function prune(scope: string, type: string, reason: string): Promise<PruneResult> {
|
|
384
|
-
const files = await memoryFiles();
|
|
385
|
-
const snapshot = await snapshotFiles(files);
|
|
386
420
|
const deleted: DeletedMemory[] = [];
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
421
|
+
const snapshot: string[] = [];
|
|
422
|
+
for (const file of await memoryFiles()) {
|
|
423
|
+
await withStoreLock(
|
|
424
|
+
file,
|
|
425
|
+
async () => {
|
|
426
|
+
const lines = await readLines(file);
|
|
427
|
+
const kept: string[] = [];
|
|
428
|
+
let changed = false;
|
|
429
|
+
for (const line of lines) {
|
|
430
|
+
const parsed = parseLine(line);
|
|
431
|
+
const m = parsed ? toMemory(parsed, file, 0) : null;
|
|
432
|
+
// Repo floor: forgetting scope X here must never delete
|
|
433
|
+
// another repo's row for scope X — same basename collisions are real
|
|
434
|
+
// (a fork next to its upstream).
|
|
435
|
+
if (m && m.scope === scope && m.type === type && isVisibleHere(m)) {
|
|
436
|
+
deleted.push({ memory: m, reason });
|
|
437
|
+
await logDeletion(m, reason);
|
|
438
|
+
changed = true;
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
kept.push(line);
|
|
442
|
+
}
|
|
443
|
+
if (!changed) return;
|
|
444
|
+
snapshot.push(...(await snapshotFiles([file])));
|
|
445
|
+
await writeAtomic(file, kept.join("\n") + (kept.length ? "\n" : ""));
|
|
446
|
+
},
|
|
447
|
+
LOCK_WRITE,
|
|
448
|
+
);
|
|
406
449
|
}
|
|
407
450
|
return { deleted, snapshot };
|
|
408
451
|
}
|
package/src/portable.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { memoryRoot } from "./config.ts";
|
|
|
8
8
|
import { isActive, isVisibleHere, loadAll, type Memory } from "./memory.ts";
|
|
9
9
|
import { TYPES } from "./types.ts";
|
|
10
10
|
import type { WriteCandidate } from "./gates.ts";
|
|
11
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
11
12
|
|
|
12
13
|
// --- export -------------------------------------------------------------------
|
|
13
14
|
|
|
@@ -101,7 +102,8 @@ function exportHeader(line: string): { version: number } | null {
|
|
|
101
102
|
|
|
102
103
|
// A line is malformed (skipped, never gated) unless it is a JSON object with a
|
|
103
104
|
// known type, a non-empty scope, and non-empty content. Optional fields are
|
|
104
|
-
// sanitized rather than fatal
|
|
105
|
+
// sanitized rather than fatal where noted. Control characters in structured
|
|
106
|
+
// metadata make the row malformed; content alone may remain multiline.
|
|
105
107
|
function parseRow(line: string): WriteCandidate | null {
|
|
106
108
|
let row: unknown;
|
|
107
109
|
try {
|
|
@@ -113,7 +115,13 @@ function parseRow(line: string): WriteCandidate | null {
|
|
|
113
115
|
const r = row as Record<string, unknown>;
|
|
114
116
|
if (typeof r.type !== "string" || !(TYPES as readonly string[]).includes(r.type)) return null;
|
|
115
117
|
if (typeof r.scope !== "string" || !r.scope.trim()) return null;
|
|
118
|
+
if (flattenMemoryText(r.scope) !== r.scope) return null;
|
|
116
119
|
if (typeof r.content !== "string" || !r.content.trim()) return null;
|
|
120
|
+
if (
|
|
121
|
+
Array.isArray(r.tags) &&
|
|
122
|
+
r.tags.some((t) => typeof t === "string" && flattenMemoryText(t) !== t)
|
|
123
|
+
) return null;
|
|
124
|
+
if (typeof r.issue === "string" && flattenMemoryText(r.issue) !== r.issue) return null;
|
|
117
125
|
const c: WriteCandidate = { type: r.type, scope: r.scope, content: r.content };
|
|
118
126
|
if (r.status === "open" || r.status === "settled") c.status = r.status;
|
|
119
127
|
if (Array.isArray(r.tags) && r.tags.every((t) => typeof t === "string")) c.tags = r.tags;
|
package/src/sanitize.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Memory content may legitimately be multiline in the store, but every field
|
|
2
|
+
// rendered into a model-facing line must stay on that line. C0/C1 controls and
|
|
3
|
+
// Unicode line/paragraph separators all collapse through this one helper so
|
|
4
|
+
// renderers and write-boundary validators cannot drift.
|
|
5
|
+
const CONTROL_CHARS = /[\p{Cc}\p{Zl}\p{Zp}]+/gu;
|
|
6
|
+
|
|
7
|
+
export function flattenMemoryText(value: string): string {
|
|
8
|
+
return value.replace(CONTROL_CHARS, " ");
|
|
9
|
+
}
|
package/src/schema.ts
CHANGED
|
@@ -16,20 +16,30 @@
|
|
|
16
16
|
// check in memory.ts (row identity), where the row schema lives.
|
|
17
17
|
import { z, type ZodTypeAny } from "zod";
|
|
18
18
|
import { STATUSES, TYPES } from "./types.ts";
|
|
19
|
+
import { flattenMemoryText } from "./sanitize.ts";
|
|
19
20
|
|
|
20
21
|
type JSONSchema = Record<string, unknown>;
|
|
21
22
|
|
|
22
23
|
/** Row identity anywhere it arrives off a boundary: an ISO instant. */
|
|
23
24
|
export const isoTs = z.iso.datetime();
|
|
24
25
|
|
|
26
|
+
// Structured row fields become labels in model-facing text. Content remains
|
|
27
|
+
// multiline by design and is flattened only when rendered; metadata controls
|
|
28
|
+
// are rejected at every write boundary instead. Lookup-only scopes (recall,
|
|
29
|
+
// forget, export) stay plain strings so a row stored before this check can
|
|
30
|
+
// still be found and forgotten.
|
|
31
|
+
const memoryField = z.string().refine((value) => flattenMemoryText(value) === value, {
|
|
32
|
+
message: "must not contain control characters",
|
|
33
|
+
});
|
|
34
|
+
|
|
25
35
|
// ---- tool argument schemas (JSON Schema generated from these) ------------
|
|
26
36
|
|
|
27
37
|
export const RememberArgs = z.object({
|
|
28
38
|
type: z.enum(TYPES).describe("Type of memory"),
|
|
29
|
-
scope:
|
|
39
|
+
scope: memoryField.describe("Scope/area (e.g., auth, api, ctl)"),
|
|
30
40
|
content: z.string().describe("The memory content"),
|
|
31
|
-
issue:
|
|
32
|
-
tags: z.array(
|
|
41
|
+
issue: memoryField.describe("Related issue").optional(),
|
|
42
|
+
tags: z.array(memoryField).optional(),
|
|
33
43
|
status: z
|
|
34
44
|
.enum(STATUSES)
|
|
35
45
|
.describe(
|
|
@@ -60,7 +70,7 @@ export const RecallArgs = z.object({
|
|
|
60
70
|
});
|
|
61
71
|
|
|
62
72
|
export const UpdateArgs = z.object({
|
|
63
|
-
scope:
|
|
73
|
+
scope: memoryField,
|
|
64
74
|
type: z.enum(TYPES),
|
|
65
75
|
content: z.string(),
|
|
66
76
|
query: z
|
|
@@ -140,10 +150,10 @@ export const DecayArgs = z.object({
|
|
|
140
150
|
export const EditBody = z.object({
|
|
141
151
|
ts: isoTs,
|
|
142
152
|
type: z.enum(TYPES),
|
|
143
|
-
scope:
|
|
153
|
+
scope: memoryField.min(1),
|
|
144
154
|
content: z.string().min(1),
|
|
145
|
-
issue:
|
|
146
|
-
tags: z.array(
|
|
155
|
+
issue: memoryField.optional(),
|
|
156
|
+
tags: z.array(memoryField).optional(),
|
|
147
157
|
// min NOT enforced here: the mutate layer owns the audit-reason contract
|
|
148
158
|
// (and its tests pin its message). Shape is the boundary's job only.
|
|
149
159
|
reason: z.string(),
|
|
@@ -156,7 +166,7 @@ export const UnblockBody = z.object({ ts: isoTs });
|
|
|
156
166
|
// POSTs append rows with empty type/scope/content.
|
|
157
167
|
export const RestoreEntryBody = z.object({
|
|
158
168
|
type: z.enum(TYPES),
|
|
159
|
-
scope:
|
|
169
|
+
scope: memoryField.min(1),
|
|
160
170
|
content: z.string().min(1),
|
|
161
171
|
});
|
|
162
172
|
|
package/src/undo.ts
CHANGED
|
@@ -3,15 +3,18 @@
|
|
|
3
3
|
//
|
|
4
4
|
// Daily logfmt files are SHARED across concurrent sessions, so a naive
|
|
5
5
|
// whole-file write can silently clobber rows another session appended after
|
|
6
|
-
// the snapshot. Restore is therefore
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
6
|
+
// the snapshot. Restore is therefore conflict-aware: the snapshot cannot tell
|
|
7
|
+
// "rows I added since the snapshot, which undo should remove" apart from
|
|
8
|
+
// "rows another session added, which must survive", so that call belongs to
|
|
9
|
+
// the caller. The default mode CHECK refuses and lists the divergent rows;
|
|
10
|
+
// MERGE replays them on top of the snapshot; FORCE is the old exact rollback,
|
|
11
|
+
// and additionally snapshots the file it replaces, so the state it drops stays
|
|
12
|
+
// undoable (result.safety).
|
|
12
13
|
import { basename, join, resolve, sep } from "node:path";
|
|
13
14
|
import { snapshotDir } from "./config.ts";
|
|
14
|
-
import { ensureDir } from "./logfmt.ts";
|
|
15
|
+
import { ensureDir, writeAtomic } from "./logfmt.ts";
|
|
16
|
+
import { withStoreLock, LOCK_RESTORE } from "./lock.ts";
|
|
17
|
+
import { snapshotFiles } from "./memory.ts";
|
|
15
18
|
|
|
16
19
|
export interface SnapshotInfo {
|
|
17
20
|
file: string;
|
|
@@ -45,6 +48,9 @@ export interface RestoreResult {
|
|
|
45
48
|
target: string;
|
|
46
49
|
restored: boolean;
|
|
47
50
|
mode: RestoreMode;
|
|
51
|
+
// Snapshot of the file as it stood the moment the restore held the lock —
|
|
52
|
+
// present on a force restore, which is the mode that drops rows.
|
|
53
|
+
safety?: string;
|
|
48
54
|
// ts of rows present in the CURRENT file but absent from the snapshot
|
|
49
55
|
// (added after it — or in-place edits, listed under their original ts).
|
|
50
56
|
concurrentAdded: string[];
|
|
@@ -87,37 +93,55 @@ export async function restoreSnapshot(
|
|
|
87
93
|
throw new Error(`snapshot target escapes the memory root: ${original}`);
|
|
88
94
|
}
|
|
89
95
|
const snapLines = rows(await f.text());
|
|
90
|
-
const cur = Bun.file(target);
|
|
91
|
-
const curLines = (await cur.exists()) ? rows(await cur.text()) : [];
|
|
92
96
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
}
|
|
97
|
+
// Comparing and publishing happen under the same lock: a divergence check on
|
|
98
|
+
// a stale read would wave through a row that landed after it. A restore waits
|
|
99
|
+
// for the store rather than proceeding unlocked, because "force" deliberately
|
|
100
|
+
// drops rows — doing that on top of a racing writer would drop rows nobody
|
|
101
|
+
// agreed to drop.
|
|
102
|
+
return withStoreLock(
|
|
103
|
+
target,
|
|
104
|
+
async () => {
|
|
105
|
+
const cur = Bun.file(target);
|
|
106
|
+
const curLines = (await cur.exists()) ? rows(await cur.text()) : [];
|
|
104
107
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
+
const snapByKey = new Map(snapLines.map((l) => [rowKey(l), l]));
|
|
109
|
+
const concurrentAdded: string[] = [];
|
|
110
|
+
const concurrentChanged: string[] = [];
|
|
111
|
+
const curByKey = new Map<string, string>();
|
|
112
|
+
for (const l of curLines) {
|
|
113
|
+
const k = rowKey(l);
|
|
114
|
+
curByKey.set(k, l);
|
|
115
|
+
const s = snapByKey.get(k);
|
|
116
|
+
if (s === undefined) concurrentAdded.push(k);
|
|
117
|
+
else if (s !== l) concurrentChanged.push(k);
|
|
118
|
+
}
|
|
108
119
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
120
|
+
if (mode === "check" && (concurrentAdded.length || concurrentChanged.length)) {
|
|
121
|
+
return { target, restored: false, mode, concurrentAdded, concurrentChanged };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// FORCE is the mode that drops rows, so the file it replaces is
|
|
125
|
+
// snapshotted HERE — inside the lock, which is what makes rows a
|
|
126
|
+
// concurrent session appended moments earlier part of the copy. A caller
|
|
127
|
+
// that snapshots before this call races the very write it means to save.
|
|
128
|
+
const safety = mode === "force" ? (await snapshotFiles([target]))[0] : undefined;
|
|
129
|
+
|
|
130
|
+
let out = snapLines;
|
|
131
|
+
if (mode === "merge") {
|
|
132
|
+
// Keep the CURRENT version of rows the snapshot also has (in-place edits —
|
|
133
|
+
// e.g. a supersede stamp — made after the snapshot win), then append the
|
|
134
|
+
// rows the snapshot never saw. Merge is a deliberate superset: rows the
|
|
135
|
+
// snapshotter wrote after the snapshot are NOT dropped here; choose FORCE
|
|
136
|
+
// for that.
|
|
137
|
+
out = snapLines.map((l) => curByKey.get(rowKey(l)) ?? l).concat(
|
|
138
|
+
curLines.filter((l) => !snapByKey.has(rowKey(l))),
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
await ensureDir(memoryRootDir);
|
|
142
|
+
await writeAtomic(target, out.length ? out.join("\n") + "\n" : "");
|
|
143
|
+
return { target, restored: true, mode, safety, concurrentAdded, concurrentChanged };
|
|
144
|
+
},
|
|
145
|
+
LOCK_RESTORE,
|
|
146
|
+
);
|
|
123
147
|
}
|
package/src/viewer/mutate.ts
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
import { deletionsFile } from "../config.ts";
|
|
10
10
|
import { addBlock, removeBlock } from "../blocklist.ts";
|
|
11
11
|
import { record, chainId } from "../evidence.ts";
|
|
12
|
-
import { appendLog, formatLine, parseLine, readLines } from "../logfmt.ts";
|
|
12
|
+
import { appendLog, formatLine, parseLine, readLines, writeAtomic } from "../logfmt.ts";
|
|
13
|
+
import { withStoreLock, LOCK_WRITE } from "../lock.ts";
|
|
13
14
|
import {
|
|
14
15
|
appendMemory,
|
|
15
16
|
loadAll,
|
|
@@ -99,14 +100,23 @@ export async function deleteMemory(input: DeleteInput): Promise<MutationResult>
|
|
|
99
100
|
const chain = chainId();
|
|
100
101
|
const target = await findMemory(input.ts);
|
|
101
102
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
// Snapshot, filter and republish happen as one locked step, so a row another
|
|
104
|
+
// session appended while we read cannot be filtered out with the target.
|
|
105
|
+
let snapshot: string[] = [];
|
|
106
|
+
await withStoreLock(
|
|
107
|
+
target.file,
|
|
108
|
+
async () => {
|
|
109
|
+
const lines = await readLines(target.file);
|
|
110
|
+
const kept = lines.filter((l) => {
|
|
111
|
+
const f = parseLine(l);
|
|
112
|
+
return !(f && f.ts === input.ts);
|
|
113
|
+
});
|
|
114
|
+
if (kept.length === lines.length) throw new Error(`memory line not found: ${input.ts}`);
|
|
115
|
+
snapshot = await snapshotFiles([target.file]);
|
|
116
|
+
await writeAtomic(target.file, kept.join("\n") + (kept.length ? "\n" : ""));
|
|
117
|
+
},
|
|
118
|
+
LOCK_WRITE,
|
|
119
|
+
);
|
|
110
120
|
|
|
111
121
|
await audit("deleted", target, input.ts, undefined, input.reason);
|
|
112
122
|
await record("prune", {
|
package/src/viewer/server.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { BlockBody, DeleteBody, EditBody, RestoreEntryBody, UnblockBody } from "
|
|
|
11
11
|
import { deletionsFile, memoryRoot, snapshotDir } from "../config.ts";
|
|
12
12
|
import { blockedEntries } from "../blocklist.ts";
|
|
13
13
|
import { listSnapshots, restoreSnapshot } from "../undo.ts";
|
|
14
|
-
import { appendMemory, loadAll, memoryFiles
|
|
14
|
+
import { appendMemory, loadAll, memoryFiles } from "../memory.ts";
|
|
15
15
|
import { parseLine, readLines } from "../logfmt.ts";
|
|
16
16
|
import { deleteMemory, editMemory, addBlock, removeBlock, type MutationResult } from "./mutate.ts";
|
|
17
17
|
|
|
@@ -269,14 +269,14 @@ async function restoreRoute(body: Record<string, unknown>): Promise<Response> {
|
|
|
269
269
|
if (!/^\d+-\d{4}-\d{2}-\d{2}\.logfmt$/.test(name)) {
|
|
270
270
|
return error("invalid snapshot name", 400);
|
|
271
271
|
}
|
|
272
|
-
|
|
273
|
-
//
|
|
274
|
-
//
|
|
275
|
-
//
|
|
276
|
-
await snapshotFiles([join(memoryRoot(), original)]);
|
|
272
|
+
// Explicit human admin action: exact rollback. The pre-restore snapshot the
|
|
273
|
+
// dropped rows can be recovered from is taken by restoreSnapshot itself,
|
|
274
|
+
// under the store lock — snapshotting here instead would miss anything
|
|
275
|
+
// another session appended between this call and the rollback.
|
|
277
276
|
const res = await restoreSnapshot(join(snapshotDir(), name), memoryRoot(), "force");
|
|
278
277
|
return json({
|
|
279
278
|
restored: res.target,
|
|
279
|
+
safety: res.safety ?? null,
|
|
280
280
|
rolledBack: res.concurrentAdded.length + res.concurrentChanged.length,
|
|
281
281
|
});
|
|
282
282
|
}
|