opencode-episodic-memory 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/package.json +8 -5
- package/plugin/episodic-memory.ts +5 -4
- package/src/cli.ts +5 -4
- package/src/indexer.ts +7 -2
- package/src/parser.ts +10 -5
- package/src/reader.test.ts +205 -0
- package/src/reader.ts +111 -67
package/README.md
CHANGED
|
@@ -35,6 +35,17 @@ bun install # first embed downloads the model (~100 MB, cached afterwards)
|
|
|
35
35
|
}
|
|
36
36
|
```
|
|
37
37
|
|
|
38
|
+
Or from npm — pin the version. OpenCode caches npm plugins and never
|
|
39
|
+
re-resolves a bare name / `@latest`
|
|
40
|
+
([anomalyco/opencode#25293](https://github.com/anomalyco/opencode/issues/25293)),
|
|
41
|
+
so to update later you bump the pin:
|
|
42
|
+
|
|
43
|
+
```jsonc
|
|
44
|
+
{
|
|
45
|
+
"plugin": ["opencode-episodic-memory@0.1.1"]
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
38
49
|
Copy the skill so the agent knows when to search:
|
|
39
50
|
|
|
40
51
|
```bash
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-episodic-memory",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Semantic search over past OpenCode conversations. Port of obra/episodic-memory to OpenCode primitives.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "robertn702",
|
|
8
8
|
"repository": {
|
|
9
9
|
"type": "git",
|
|
10
|
-
"url": "https://github.com/robertn702/opencode-episodic-memory"
|
|
10
|
+
"url": "git+https://github.com/robertn702/opencode-episodic-memory.git"
|
|
11
11
|
},
|
|
12
12
|
"keywords": [
|
|
13
13
|
"opencode",
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
"embeddings",
|
|
18
18
|
"transformersjs"
|
|
19
19
|
],
|
|
20
|
+
"main": "./plugin/episodic-memory.ts",
|
|
20
21
|
"exports": {
|
|
21
22
|
".": "./plugin/episodic-memory.ts",
|
|
22
23
|
"./cli": "./src/cli.ts"
|
|
@@ -27,17 +28,19 @@
|
|
|
27
28
|
"skills"
|
|
28
29
|
],
|
|
29
30
|
"bin": {
|
|
30
|
-
"opencode-episodic": "
|
|
31
|
+
"opencode-episodic": "src/cli.ts"
|
|
31
32
|
},
|
|
32
33
|
"scripts": {
|
|
33
34
|
"spike": "bun run spikes/spike.ts",
|
|
34
35
|
"test": "bun test",
|
|
35
36
|
"typecheck": "tsc --noEmit",
|
|
36
|
-
"
|
|
37
|
+
"verify:entrypoint": "bun run spikes/verify-opencode-entrypoint.ts",
|
|
38
|
+
"prepublishOnly": "bun run typecheck && bun test && bun run verify:entrypoint"
|
|
37
39
|
},
|
|
38
40
|
"dependencies": {
|
|
39
41
|
"@huggingface/transformers": "^4.2.0",
|
|
40
|
-
"@opencode-ai/plugin": "^1.18.4"
|
|
42
|
+
"@opencode-ai/plugin": "^1.18.4",
|
|
43
|
+
"zod": "^4.4.3"
|
|
41
44
|
},
|
|
42
45
|
"devDependencies": {
|
|
43
46
|
"@types/bun": "latest",
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// - Native tools: episodic_search, episodic_read
|
|
3
3
|
// - Incremental reindex on session.idle (fire-and-forget, debounced)
|
|
4
4
|
import { type Plugin, tool } from "@opencode-ai/plugin";
|
|
5
|
-
import { openSource, getSession, getTranscript } from "../src/reader";
|
|
5
|
+
import { openSource, getSession, getTranscript, transcriptHasMarker } from "../src/reader";
|
|
6
6
|
import { openIndex, search, textSearch } from "../src/store";
|
|
7
7
|
import { syncSession, syncAll, pruneOrphans } from "../src/indexer";
|
|
8
8
|
import { embedQuery } from "../src/embed";
|
|
9
|
-
import { hasExcludeMarker } from "../src/parser";
|
|
10
9
|
|
|
11
10
|
// Discriminated result so callers handle the parse error explicitly (no cast to
|
|
12
11
|
// strip the error arm off a union). `ms` is undefined when no date was given.
|
|
@@ -124,10 +123,12 @@ export const EpisodicMemory: Plugin = async ({ client }) => {
|
|
|
124
123
|
const source = openSource();
|
|
125
124
|
const s = getSession(source, args.session_id);
|
|
126
125
|
if (s) {
|
|
127
|
-
|
|
128
|
-
|
|
126
|
+
// Authoritative gate: raw part blobs (a marker in an
|
|
127
|
+
// unparseable blob would be invisible to the parsed-text scan).
|
|
128
|
+
if (transcriptHasMarker(source, args.session_id)) {
|
|
129
129
|
return "Session is marked private (exclusion marker present); transcript withheld.";
|
|
130
130
|
}
|
|
131
|
+
const transcript = getTranscript(source, args.session_id);
|
|
131
132
|
const lines: string[] = [`# ${s.title}`, `${fmtDate(s.time_created)} — ${s.directory} — ${s.id}`, ""];
|
|
132
133
|
for (const m of transcript) {
|
|
133
134
|
const text = m.parts
|
package/src/cli.ts
CHANGED
|
@@ -10,11 +10,10 @@
|
|
|
10
10
|
// stats Index statistics
|
|
11
11
|
// doctor Diagnose setup
|
|
12
12
|
import { existsSync } from "node:fs";
|
|
13
|
-
import { openSource, sourceDbPath, getSession, getTranscript } from "./reader";
|
|
13
|
+
import { openSource, sourceDbPath, getSession, getTranscript, transcriptHasMarker } from "./reader";
|
|
14
14
|
import { openIndex, indexDbPath, search, textSearch, stats, type SearchHit } from "./store";
|
|
15
15
|
import { syncAll } from "./indexer";
|
|
16
16
|
import { embed, embedQuery } from "./embed";
|
|
17
|
-
import { hasExcludeMarker } from "./parser";
|
|
18
17
|
|
|
19
18
|
const [, , command, ...rest] = process.argv;
|
|
20
19
|
|
|
@@ -126,11 +125,13 @@ async function main() {
|
|
|
126
125
|
const source = openSource();
|
|
127
126
|
const s = getSession(source, id);
|
|
128
127
|
if (!s) { console.error("session not found:", id); process.exit(1); }
|
|
129
|
-
|
|
130
|
-
|
|
128
|
+
// Authoritative gate: raw part blobs (a marker in an unparseable blob
|
|
129
|
+
// would be invisible to the parsed-text scan).
|
|
130
|
+
if (transcriptHasMarker(source, id)) {
|
|
131
131
|
console.error("session is marked private (exclusion marker present); transcript withheld");
|
|
132
132
|
process.exit(1);
|
|
133
133
|
}
|
|
134
|
+
const transcript = getTranscript(source, id);
|
|
134
135
|
console.log(`# ${s.title}\n${fmtDate(s.time_created)} — ${s.directory} — ${s.id}\n`);
|
|
135
136
|
for (const m of transcript) {
|
|
136
137
|
const text = m.parts.filter((p) => p.type === "text" && p.text).map((p) => p.text).join("\n");
|
package/src/indexer.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Incremental, idempotent indexer. Watermark = session.time_updated; a session
|
|
2
2
|
// is re-embedded only when the source changed since we last indexed it.
|
|
3
3
|
import type { Database } from "bun:sqlite";
|
|
4
|
-
import { getTranscript, listSessions, type SourceSession } from "./reader";
|
|
4
|
+
import { getTranscript, listSessions, transcriptHasMarker, type SourceSession } from "./reader";
|
|
5
5
|
import { parseTranscript, exchangeText } from "./parser";
|
|
6
6
|
import { embed } from "./embed";
|
|
7
7
|
import { getIndexedSession, replaceSessionChunks } from "./store";
|
|
@@ -24,7 +24,12 @@ export async function syncSession(
|
|
|
24
24
|
const prior = getIndexedSession(index, s.id);
|
|
25
25
|
if (!force && prior && prior.source_time_updated >= s.time_updated) return "fresh";
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
// Authoritative opt-out gate: raw part blobs. The parsed-text scan inside
|
|
28
|
+
// parseTranscript would miss a marker in an unparseable blob.
|
|
29
|
+
const excludedRaw = transcriptHasMarker(source, s.id);
|
|
30
|
+
const { exchanges, excluded } = excludedRaw
|
|
31
|
+
? { exchanges: [], excluded: true }
|
|
32
|
+
: parseTranscript(getTranscript(source, s.id));
|
|
28
33
|
const meta = {
|
|
29
34
|
id: s.id, project_id: s.project_id, parent_id: s.parent_id,
|
|
30
35
|
title: s.title, directory: s.directory,
|
package/src/parser.ts
CHANGED
|
@@ -1,13 +1,18 @@
|
|
|
1
1
|
// Turn a raw transcript into condensed exchanges suitable for embedding.
|
|
2
2
|
// Keeps user text, assistant text, and tool *names* (not tool output, which is
|
|
3
3
|
// bulky and low-signal). Skips reasoning blobs and step markers.
|
|
4
|
-
import type
|
|
4
|
+
import { EXCLUDE_MARKER, type SourceMessage, type SourcePart } from "./reader";
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
// Defined in reader.ts (single source of truth); re-exported for existing
|
|
7
|
+
// consumers of this module.
|
|
8
|
+
export { EXCLUDE_MARKER };
|
|
7
9
|
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
10
|
+
// Fast-path check over PARSED part text. Cheaper than the raw scan, but can
|
|
11
|
+
// miss the marker when a part blob fails to parse and degrades to
|
|
12
|
+
// text: undefined — the AUTHORITATIVE check is transcriptHasMarker() in
|
|
13
|
+
// reader.ts, which substring-matches the raw `data` column. Callers that gate
|
|
14
|
+
// privacy-sensitive paths should use the raw check; this remains useful for
|
|
15
|
+
// parseTranscript's in-memory flow and tests.
|
|
11
16
|
export function hasExcludeMarker(messages: SourceMessage[]): boolean {
|
|
12
17
|
for (const m of messages) {
|
|
13
18
|
for (const p of m.parts) {
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { describe, test, expect } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { listSessions, getSession, getTranscript, transcriptHasMarker, EXCLUDE_MARKER } from "./reader";
|
|
4
|
+
|
|
5
|
+
// A minimal opencode.db mirroring only the columns reader.ts SELECTs. Writable
|
|
6
|
+
// here so we can seed rows; the reader functions take a Database and never write.
|
|
7
|
+
function makeSource(): Database {
|
|
8
|
+
const db = new Database(":memory:");
|
|
9
|
+
db.run(`CREATE TABLE session (
|
|
10
|
+
id TEXT, project_id TEXT, parent_id TEXT, title TEXT, directory TEXT,
|
|
11
|
+
time_created INTEGER, time_updated INTEGER, time_archived INTEGER
|
|
12
|
+
)`);
|
|
13
|
+
db.run(`CREATE TABLE message (
|
|
14
|
+
id TEXT, session_id TEXT, time_created INTEGER, data TEXT
|
|
15
|
+
)`);
|
|
16
|
+
db.run(`CREATE TABLE part (
|
|
17
|
+
id TEXT, message_id TEXT, session_id TEXT, time_created INTEGER, data TEXT
|
|
18
|
+
)`);
|
|
19
|
+
return db;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function addSession(db: Database, s: {
|
|
23
|
+
id: string; parent_id?: string | null; title?: string;
|
|
24
|
+
time_created?: number; time_updated?: number; time_archived?: number | null;
|
|
25
|
+
}): void {
|
|
26
|
+
db.run(
|
|
27
|
+
`INSERT INTO session (id, project_id, parent_id, title, directory, time_created, time_updated, time_archived)
|
|
28
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
29
|
+
[s.id, "proj", s.parent_id ?? null, s.title ?? "Title", "/dir",
|
|
30
|
+
s.time_created ?? 1000, s.time_updated ?? 1000, s.time_archived ?? null]
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
function addMessage(db: Database, id: string, sessionId: string, time: number, data: string): void {
|
|
34
|
+
db.run("INSERT INTO message (id, session_id, time_created, data) VALUES (?, ?, ?, ?)",
|
|
35
|
+
[id, sessionId, time, data]);
|
|
36
|
+
}
|
|
37
|
+
function addPart(db: Database, id: string, messageId: string, sessionId: string, time: number, data: string): void {
|
|
38
|
+
db.run("INSERT INTO part (id, message_id, session_id, time_created, data) VALUES (?, ?, ?, ?, ?)",
|
|
39
|
+
[id, messageId, sessionId, time, data]);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
describe("listSessions / getSession (structural rows)", () => {
|
|
43
|
+
test("lists active sessions ordered by time_created, excludes archived", () => {
|
|
44
|
+
const db = makeSource();
|
|
45
|
+
addSession(db, { id: "ses_b", time_created: 2000 });
|
|
46
|
+
addSession(db, { id: "ses_a", time_created: 1000, parent_id: "ses_b" });
|
|
47
|
+
addSession(db, { id: "ses_arch", time_created: 1500, time_archived: 9999 });
|
|
48
|
+
const sessions = listSessions(db);
|
|
49
|
+
expect(sessions.map((s) => s.id)).toEqual(["ses_a", "ses_b"]);
|
|
50
|
+
expect(sessions[0].parent_id).toBe("ses_b");
|
|
51
|
+
expect(sessions[1].parent_id).toBeNull();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("getSession returns a row, or null for an unknown id", () => {
|
|
55
|
+
const db = makeSource();
|
|
56
|
+
addSession(db, { id: "ses_a", title: "Hello" });
|
|
57
|
+
expect(getSession(db, "ses_a")?.title).toBe("Hello");
|
|
58
|
+
expect(getSession(db, "nope")).toBeNull();
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("throws (does not silently mis-read) when a structural column drifts", () => {
|
|
62
|
+
const db = makeSource();
|
|
63
|
+
// time_created NULL violates z.number() — simulates OpenCode schema drift.
|
|
64
|
+
db.run(
|
|
65
|
+
`INSERT INTO session (id, project_id, parent_id, title, directory, time_created, time_updated, time_archived)
|
|
66
|
+
VALUES ('ses_x', 'p', NULL, 't', '/d', NULL, 1000, NULL)`
|
|
67
|
+
);
|
|
68
|
+
expect(() => listSessions(db)).toThrow();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("getSession throws on a drifted session row for an existing id", () => {
|
|
72
|
+
const db = makeSource();
|
|
73
|
+
// title NULL violates z.string() — simulates OpenCode schema drift.
|
|
74
|
+
db.run(
|
|
75
|
+
`INSERT INTO session (id, project_id, parent_id, title, directory, time_created, time_updated, time_archived)
|
|
76
|
+
VALUES ('ses_y', 'p', NULL, NULL, '/d', 1000, 1000, NULL)`
|
|
77
|
+
);
|
|
78
|
+
expect(() => getSession(db, "ses_y")).toThrow();
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe("getTranscript (JSON blob degradation)", () => {
|
|
83
|
+
test("parses roles and part fields; degrades malformed blobs per-row", () => {
|
|
84
|
+
const db = makeSource();
|
|
85
|
+
addSession(db, { id: "ses_a" });
|
|
86
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
87
|
+
addMessage(db, "m2", "ses_a", 2, `{"role":"assistant"}`);
|
|
88
|
+
addMessage(db, "m3", "ses_a", 3, `{not valid json`); // role -> "unknown"
|
|
89
|
+
addMessage(db, "m4", "ses_a", 4, `{"noRole":true}`); // role -> "unknown"
|
|
90
|
+
|
|
91
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{"type":"text","text":"hello"}`);
|
|
92
|
+
addPart(db, "p2", "m1", "ses_a", 2, `{"type":"tool","tool":"edit"}`);
|
|
93
|
+
addPart(db, "p3", "m2", "ses_a", 3, `{oops not json`); // -> {type:"unknown"}
|
|
94
|
+
addPart(db, "p4", "m2", "ses_a", 4, `{"type":123,"text":"keep"}`); // type->unknown, text kept
|
|
95
|
+
addPart(db, "p5", "m4", "ses_a", 5, `42`); // non-object -> {type:"unknown"}
|
|
96
|
+
|
|
97
|
+
const t = getTranscript(db, "ses_a");
|
|
98
|
+
expect(t.map((m) => m.role)).toEqual(["user", "assistant", "unknown", "unknown"]);
|
|
99
|
+
|
|
100
|
+
expect(t[0].parts).toEqual([
|
|
101
|
+
{ type: "text", text: "hello" },
|
|
102
|
+
{ type: "tool", tool: "edit" },
|
|
103
|
+
]);
|
|
104
|
+
expect(t[1].parts).toEqual([
|
|
105
|
+
{ type: "unknown" },
|
|
106
|
+
{ type: "unknown", text: "keep" },
|
|
107
|
+
]);
|
|
108
|
+
expect(t[3].parts).toEqual([{ type: "unknown" }]);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test("per-field catch: bad text/tool fields are dropped, type is preserved", () => {
|
|
112
|
+
const db = makeSource();
|
|
113
|
+
addSession(db, { id: "ses_a" });
|
|
114
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
115
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{"type":"text","text":123}`); // bad text dropped
|
|
116
|
+
addPart(db, "p2", "m1", "ses_a", 2, `{"type":"tool","tool":123}`); // bad tool dropped
|
|
117
|
+
|
|
118
|
+
const t = getTranscript(db, "ses_a");
|
|
119
|
+
expect(t[0].parts).toEqual([{ type: "text" }, { type: "tool" }]);
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test("throws when a part row's data column is non-string (structural drift)", () => {
|
|
123
|
+
const db = makeSource();
|
|
124
|
+
addSession(db, { id: "ses_a" });
|
|
125
|
+
// Valid message first so the message-row parse passes and the throw comes
|
|
126
|
+
// from the part row below.
|
|
127
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
128
|
+
db.run("INSERT INTO part (id, message_id, session_id, time_created, data) VALUES ('p1', 'm1', 'ses_a', 1, NULL)");
|
|
129
|
+
expect(() => getTranscript(db, "ses_a")).toThrow();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("throws when a message row's data column is non-string (structural drift)", () => {
|
|
133
|
+
const db = makeSource();
|
|
134
|
+
addSession(db, { id: "ses_a" });
|
|
135
|
+
// data NULL violates the row schema's z.string(); structural, so it throws.
|
|
136
|
+
db.run("INSERT INTO message (id, session_id, time_created, data) VALUES ('m1', 'ses_a', 1, NULL)");
|
|
137
|
+
expect(() => getTranscript(db, "ses_a")).toThrow();
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
describe("transcriptHasMarker (raw blob scan)", () => {
|
|
142
|
+
test("detects the marker in a well-formed text part", () => {
|
|
143
|
+
const db = makeSource();
|
|
144
|
+
addSession(db, { id: "ses_a" });
|
|
145
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
146
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{"type":"text","text":"note: ${EXCLUDE_MARKER}"}`);
|
|
147
|
+
expect(transcriptHasMarker(db, "ses_a")).toBe(true);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
// Regression for issue #10: the parsed-text scan degrades this blob to
|
|
151
|
+
// text: undefined, so the marker is invisible to hasExcludeMarker — but the
|
|
152
|
+
// raw scan must still see it. The privacy kill-switch must not depend on
|
|
153
|
+
// blob parseability.
|
|
154
|
+
test("detects the marker inside a malformed/unparseable part blob", () => {
|
|
155
|
+
const db = makeSource();
|
|
156
|
+
addSession(db, { id: "ses_a" });
|
|
157
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
158
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{oops not json ${EXCLUDE_MARKER}`);
|
|
159
|
+
|
|
160
|
+
// Sanity: the parsed view really does lose the marker text.
|
|
161
|
+
const t = getTranscript(db, "ses_a");
|
|
162
|
+
expect(t[0].parts).toEqual([{ type: "unknown" }]);
|
|
163
|
+
|
|
164
|
+
expect(transcriptHasMarker(db, "ses_a")).toBe(true);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("detects the marker in a blob whose fields all fail validation", () => {
|
|
168
|
+
const db = makeSource();
|
|
169
|
+
addSession(db, { id: "ses_a" });
|
|
170
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
171
|
+
// Valid JSON, but type is non-string → degrades to {type:"unknown"}.
|
|
172
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{"type":123,"note":"${EXCLUDE_MARKER}"}`);
|
|
173
|
+
|
|
174
|
+
const t = getTranscript(db, "ses_a");
|
|
175
|
+
expect(t[0].parts).toEqual([{ type: "unknown" }]);
|
|
176
|
+
expect(transcriptHasMarker(db, "ses_a")).toBe(true);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
test("returns false when no part contains the marker", () => {
|
|
180
|
+
const db = makeSource();
|
|
181
|
+
addSession(db, { id: "ses_a" });
|
|
182
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
183
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{"type":"text","text":"hello"}`);
|
|
184
|
+
expect(transcriptHasMarker(db, "ses_a")).toBe(false);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
test("is scoped to the requested session", () => {
|
|
188
|
+
const db = makeSource();
|
|
189
|
+
addSession(db, { id: "ses_a" });
|
|
190
|
+
addSession(db, { id: "ses_b" });
|
|
191
|
+
addMessage(db, "m1", "ses_b", 1, `{"role":"user"}`);
|
|
192
|
+
addPart(db, "p1", "m1", "ses_b", 1, `{"type":"text","text":"${EXCLUDE_MARKER}"}`);
|
|
193
|
+
expect(transcriptHasMarker(db, "ses_a")).toBe(false);
|
|
194
|
+
expect(transcriptHasMarker(db, "ses_b")).toBe(true);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
test("does not match case variants or partial markers (exact substring)", () => {
|
|
198
|
+
const db = makeSource();
|
|
199
|
+
addSession(db, { id: "ses_a" });
|
|
200
|
+
addMessage(db, "m1", "ses_a", 1, `{"role":"user"}`);
|
|
201
|
+
addPart(db, "p1", "m1", "ses_a", 1, `{"type":"text","text":"do not index this chat"}`);
|
|
202
|
+
addPart(db, "p2", "m1", "ses_a", 2, `{"type":"text","text":"DO NOT INDEX THIS"}`);
|
|
203
|
+
expect(transcriptHasMarker(db, "ses_a")).toBe(false);
|
|
204
|
+
});
|
|
205
|
+
});
|
package/src/reader.ts
CHANGED
|
@@ -3,24 +3,67 @@
|
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
|
+
import { z } from "zod";
|
|
6
7
|
|
|
7
8
|
export const DEFAULT_SOURCE_DB = join(homedir(), ".local/share/opencode/opencode.db");
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
10
|
+
// Opt-out marker. Matched as a BARE SUBSTRING anywhere in any message part —
|
|
11
|
+
// broader than upstream's full instruction-tag match, so it also fires on
|
|
12
|
+
// conversations that merely quote the phrase. Re-exported by parser.ts.
|
|
13
|
+
export const EXCLUDE_MARKER = "DO NOT INDEX THIS CHAT";
|
|
14
|
+
|
|
15
|
+
// --- Validation strategy ----------------------------------------------------
|
|
16
|
+
// Two surfaces, two failure modes (see AGENTS.md):
|
|
17
|
+
// 1. Structural rows we SELECT from opencode.db (columns: id, time_created,
|
|
18
|
+
// data, ...). These are a uniform contract; if a column's type/nullability
|
|
19
|
+
// drifts it drifts for every row, so we THROW (`.parse`) to surface
|
|
20
|
+
// OpenCode schema changes loudly instead of silently mis-reading them.
|
|
21
|
+
// 2. The JSON blob inside each `data` column (message role, part contents).
|
|
22
|
+
// This format evolves and carries many part shapes we don't model, so we
|
|
23
|
+
// DEGRADE per-row to "unknown"/undefined (`.catch`): one corrupt or
|
|
24
|
+
// unfamiliar blob can never abort a whole transcript read, and the parser
|
|
25
|
+
// already filters unknown types/roles downstream.
|
|
26
|
+
// No `as` assertions: schemas narrow via `.parse()`.
|
|
27
|
+
|
|
28
|
+
// --- Structural row schemas (throw on drift) --------------------------------
|
|
29
|
+
const SessionRowSchema = z.object({
|
|
30
|
+
id: z.string(),
|
|
31
|
+
project_id: z.string(),
|
|
32
|
+
parent_id: z.string().nullable(),
|
|
33
|
+
title: z.string(),
|
|
34
|
+
directory: z.string(),
|
|
35
|
+
time_created: z.number(),
|
|
36
|
+
time_updated: z.number(),
|
|
37
|
+
});
|
|
38
|
+
export type SourceSession = z.infer<typeof SessionRowSchema>;
|
|
39
|
+
|
|
40
|
+
const MessageRowSchema = z.object({
|
|
41
|
+
id: z.string(),
|
|
42
|
+
time_created: z.number(),
|
|
43
|
+
data: z.string(),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const PartRowSchema = z.object({
|
|
47
|
+
message_id: z.string(),
|
|
48
|
+
data: z.string(),
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
// Aggregate row for the raw marker scan (structural: throw on drift).
|
|
52
|
+
const MarkerCountSchema = z.object({ n: z.number() });
|
|
53
|
+
|
|
54
|
+
// --- JSON blob schemas (degrade to "unknown" on mismatch) -------------------
|
|
55
|
+
const PartDataSchema = z
|
|
56
|
+
.object({
|
|
57
|
+
type: z.string().catch("unknown"),
|
|
58
|
+
text: z.string().optional().catch(undefined),
|
|
59
|
+
tool: z.string().optional().catch(undefined),
|
|
60
|
+
})
|
|
61
|
+
.catch({ type: "unknown" });
|
|
62
|
+
export type SourcePart = z.infer<typeof PartDataSchema>;
|
|
63
|
+
|
|
64
|
+
const MessageDataSchema = z
|
|
65
|
+
.object({ role: z.string().catch("unknown") })
|
|
66
|
+
.catch({ role: "unknown" });
|
|
24
67
|
|
|
25
68
|
export interface SourceMessage {
|
|
26
69
|
id: string;
|
|
@@ -37,75 +80,76 @@ export function sourceDbPath(): string {
|
|
|
37
80
|
return process.env.EPISODIC_SOURCE_DB ?? DEFAULT_SOURCE_DB;
|
|
38
81
|
}
|
|
39
82
|
|
|
83
|
+
// JSON.parse throws on malformed input; return undefined so the blob schema's
|
|
84
|
+
// `.catch` fallback applies (one bad blob can't abort a transcript read).
|
|
85
|
+
function safeJsonParse(data: string): unknown {
|
|
86
|
+
try {
|
|
87
|
+
return JSON.parse(data);
|
|
88
|
+
} catch {
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
40
93
|
export function listSessions(db: Database): SourceSession[] {
|
|
41
|
-
|
|
42
|
-
.prepare
|
|
94
|
+
const rows = db
|
|
95
|
+
.prepare(
|
|
43
96
|
`SELECT id, project_id, parent_id, title, directory, time_created, time_updated
|
|
44
97
|
FROM session WHERE time_archived IS NULL ORDER BY time_created`
|
|
45
98
|
)
|
|
46
99
|
.all();
|
|
100
|
+
return SessionRowSchema.array().parse(rows);
|
|
47
101
|
}
|
|
48
102
|
|
|
49
103
|
export function getSession(db: Database, sessionId: string): SourceSession | null {
|
|
50
|
-
|
|
104
|
+
const row = db
|
|
105
|
+
.prepare(
|
|
106
|
+
`SELECT id, project_id, parent_id, title, directory, time_created, time_updated
|
|
107
|
+
FROM session WHERE id = ?`
|
|
108
|
+
)
|
|
109
|
+
.get(sessionId);
|
|
110
|
+
return row === null || row === undefined ? null : SessionRowSchema.parse(row);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// AUTHORITATIVE exclusion check: bare-substring match over the RAW `data`
|
|
114
|
+
// column of the session's part rows, with no JSON parsing. The parsed-text
|
|
115
|
+
// scan (parser.ts hasExcludeMarker) can miss the marker when a part blob fails
|
|
116
|
+
// to parse and degrades to text: undefined — the privacy kill-switch must not
|
|
117
|
+
// depend on blob parseability. `instr` is an exact, case-sensitive substring
|
|
118
|
+
// match (unlike LIKE, which is case-insensitive and has wildcard chars).
|
|
119
|
+
export function transcriptHasMarker(db: Database, sessionId: string): boolean {
|
|
120
|
+
const row = MarkerCountSchema.parse(
|
|
51
121
|
db
|
|
52
|
-
.prepare
|
|
53
|
-
`SELECT
|
|
54
|
-
|
|
122
|
+
.prepare(
|
|
123
|
+
`SELECT COUNT(*) AS n FROM part
|
|
124
|
+
WHERE session_id = ? AND instr(data, ?) > 0`
|
|
55
125
|
)
|
|
56
|
-
.get(sessionId)
|
|
126
|
+
.get(sessionId, EXCLUDE_MARKER)
|
|
57
127
|
);
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
// JSON.parse returns `any`; these guards validate shape at runtime so no type
|
|
61
|
-
// assertion is needed. Malformed rows degrade gracefully (unknown type/role) —
|
|
62
|
-
// including a corrupt `data` blob whose JSON.parse throws, so one bad row can't
|
|
63
|
-
// abort the whole transcript read.
|
|
64
|
-
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
65
|
-
return typeof v === "object" && v !== null;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function safeParse(data: string): unknown {
|
|
69
|
-
try {
|
|
70
|
-
return JSON.parse(data);
|
|
71
|
-
} catch {
|
|
72
|
-
return undefined;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
function parsePart(data: string): SourcePart {
|
|
77
|
-
const raw = safeParse(data);
|
|
78
|
-
if (!isRecord(raw)) return { type: "unknown" };
|
|
79
|
-
return {
|
|
80
|
-
type: typeof raw.type === "string" ? raw.type : "unknown",
|
|
81
|
-
text: typeof raw.text === "string" ? raw.text : undefined,
|
|
82
|
-
tool: typeof raw.tool === "string" ? raw.tool : undefined,
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
function parseRole(data: string): string {
|
|
87
|
-
const raw = safeParse(data);
|
|
88
|
-
return isRecord(raw) && typeof raw.role === "string" ? raw.role : "unknown";
|
|
128
|
+
return row.n > 0;
|
|
89
129
|
}
|
|
90
130
|
|
|
91
131
|
export function getTranscript(db: Database, sessionId: string): SourceMessage[] {
|
|
92
|
-
const messages =
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
132
|
+
const messages = MessageRowSchema.array().parse(
|
|
133
|
+
db
|
|
134
|
+
.prepare(
|
|
135
|
+
`SELECT id, time_created, data FROM message
|
|
136
|
+
WHERE session_id = ? ORDER BY time_created, id`
|
|
137
|
+
)
|
|
138
|
+
.all(sessionId)
|
|
139
|
+
);
|
|
98
140
|
|
|
99
|
-
const parts =
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
141
|
+
const parts = PartRowSchema.array().parse(
|
|
142
|
+
db
|
|
143
|
+
.prepare(
|
|
144
|
+
`SELECT message_id, data FROM part
|
|
145
|
+
WHERE session_id = ? ORDER BY time_created, id`
|
|
146
|
+
)
|
|
147
|
+
.all(sessionId)
|
|
148
|
+
);
|
|
105
149
|
|
|
106
150
|
const partsByMsg = new Map<string, SourcePart[]>();
|
|
107
151
|
for (const p of parts) {
|
|
108
|
-
const d =
|
|
152
|
+
const d = PartDataSchema.parse(safeJsonParse(p.data));
|
|
109
153
|
let list = partsByMsg.get(p.message_id);
|
|
110
154
|
if (!list) partsByMsg.set(p.message_id, (list = []));
|
|
111
155
|
list.push(d);
|
|
@@ -113,7 +157,7 @@ export function getTranscript(db: Database, sessionId: string): SourceMessage[]
|
|
|
113
157
|
|
|
114
158
|
return messages.map((m) => ({
|
|
115
159
|
id: m.id,
|
|
116
|
-
role:
|
|
160
|
+
role: MessageDataSchema.parse(safeJsonParse(m.data)).role,
|
|
117
161
|
timeCreated: m.time_created,
|
|
118
162
|
parts: partsByMsg.get(m.id) ?? [],
|
|
119
163
|
}));
|