opencode-episodic-memory 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +102 -0
- package/package.json +50 -0
- package/plugin/episodic-memory.ts +161 -0
- package/skills/remembering-conversations/SKILL.md +53 -0
- package/src/cli.ts +187 -0
- package/src/embed.ts +62 -0
- package/src/indexer.ts +93 -0
- package/src/parser.test.ts +64 -0
- package/src/parser.ts +78 -0
- package/src/reader.ts +120 -0
- package/src/store.test.ts +72 -0
- package/src/store.ts +193 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 robertn702
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# opencode-episodic-memory
|
|
2
|
+
|
|
3
|
+
Semantic search over your past [OpenCode](https://opencode.ai) conversations.
|
|
4
|
+
Remember past discussions, decisions, and patterns across sessions.
|
|
5
|
+
|
|
6
|
+
Inspired by [obra/episodic-memory](https://github.com/obra/episodic-memory),
|
|
7
|
+
rebuilt for OpenCode primitives — native plugin tools instead of an MCP server,
|
|
8
|
+
plugin events instead of hooks, and OpenCode's own session database as the
|
|
9
|
+
source.
|
|
10
|
+
|
|
11
|
+
## How it works
|
|
12
|
+
|
|
13
|
+
1. **Read** — sessions/messages/parts from OpenCode's `~/.local/share/opencode/opencode.db` (read-only)
|
|
14
|
+
2. **Parse** — condensed exchanges (user text, assistant text, tool names; no reasoning blobs or tool output)
|
|
15
|
+
3. **Embed** — local, offline embeddings via Transformers.js (`Snowflake/snowflake-arctic-embed-m-v1.5` q8, 768 dims; retrieval prefix on search queries). Chosen by empirical eval on a real corpus — see [docs/embedding-model-eval.md](docs/embedding-model-eval.md)
|
|
16
|
+
4. **Index** — plain SQLite at `~/.local/share/opencode-episodic-memory/index.db`; brute-force cosine over Float32 blobs
|
|
17
|
+
5. **Recall** — native plugin tools `episodic_search` / `episodic_read`, plus a `remembering-conversations` skill that teaches the agent when to search
|
|
18
|
+
6. **Stay fresh** — the plugin re-indexes each session on the `session.idle` event
|
|
19
|
+
|
|
20
|
+
Design note: `bun:sqlite` cannot load dynamic extensions, so sqlite-vec is not
|
|
21
|
+
usable inside OpenCode plugins. Brute-force cosine is single-digit milliseconds
|
|
22
|
+
at this scale (thousands of chunks) and has zero native-dependency risk. The
|
|
23
|
+
store layer is the single swap point if a real ANN index is ever needed.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
bun install # first embed downloads the model (~100 MB, cached afterwards)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
```jsonc
|
|
32
|
+
// ~/.config/opencode/opencode.json
|
|
33
|
+
{
|
|
34
|
+
"plugin": ["/path/to/opencode-episodic-memory/plugin/episodic-memory.ts"]
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Copy the skill so the agent knows when to search:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
cp -r skills/remembering-conversations ~/.config/opencode/skills/
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Then backfill existing history and restart OpenCode:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
bun run src/cli.ts sync
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## CLI
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
bun run src/cli.ts sync [--force] # index new/changed sessions
|
|
54
|
+
bun run src/cli.ts search "query" # semantic search
|
|
55
|
+
bun run src/cli.ts search q --text "exact" # require substring
|
|
56
|
+
bun run src/cli.ts search q --after 2026-07-01 --limit 5
|
|
57
|
+
bun run src/cli.ts read <session-id> # full transcript (live store)
|
|
58
|
+
bun run src/cli.ts read <id> --indexed # indexed excerpts (survives deletion)
|
|
59
|
+
bun run src/cli.ts stats # index statistics
|
|
60
|
+
bun run src/cli.ts doctor # diagnose setup
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`--after`/`--before` take `YYYY-MM-DD` (midnight UTC). `--after D` is inclusive
|
|
64
|
+
of day D; `--before D` is exclusive of day D (i.e. up to the start of that day).
|
|
65
|
+
|
|
66
|
+
## Agent tools
|
|
67
|
+
|
|
68
|
+
- **`episodic_search`** — `query` (+ optional `text`, `mode: vector|text`, `after`, `before`, `limit`). Returns dated excerpts with session IDs and similarity scores.
|
|
69
|
+
- **`episodic_read`** — `session_id` (+ optional `indexed`). Full transcript from the live store, falling back to indexed excerpts.
|
|
70
|
+
|
|
71
|
+
## Excluding conversations
|
|
72
|
+
|
|
73
|
+
Any conversation containing this marker is archived nowhere and indexed nowhere:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
DO NOT INDEX THIS CHAT
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Note: the marker is matched as a bare substring anywhere in any message part, so
|
|
80
|
+
this also excludes conversations that merely *quote* the phrase (such as
|
|
81
|
+
discussions about this tool itself). This is broader than upstream's full
|
|
82
|
+
instruction-tag match — the intent is the same, but our matching is literal.
|
|
83
|
+
|
|
84
|
+
## Configuration (env vars)
|
|
85
|
+
|
|
86
|
+
| Variable | Default | Purpose |
|
|
87
|
+
|---|---|---|
|
|
88
|
+
| `EPISODIC_SOURCE_DB` | `~/.local/share/opencode/opencode.db` | OpenCode session store |
|
|
89
|
+
| `EPISODIC_INDEX_DB` | `~/.local/share/opencode-episodic-memory/index.db` | Index location |
|
|
90
|
+
| `EPISODIC_EMBED_MODEL` | `Snowflake/snowflake-arctic-embed-m-v1.5` | Transformers.js embedding model |
|
|
91
|
+
|
|
92
|
+
## Not yet implemented (deliberate)
|
|
93
|
+
|
|
94
|
+
- LLM-generated per-session summaries embedded instead of raw exchange text
|
|
95
|
+
(upstream does this; deferred until search quality data says it's needed —
|
|
96
|
+
would use OpenCode provider auth via `client.session.prompt`)
|
|
97
|
+
- Multi-concept AND search, MCP server wrapper for non-OpenCode clients
|
|
98
|
+
- ANN index (see design note above)
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
MIT
|
package/package.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "opencode-episodic-memory",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Semantic search over past OpenCode conversations. Port of obra/episodic-memory to OpenCode primitives.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "robertn702",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/robertn702/opencode-episodic-memory"
|
|
11
|
+
},
|
|
12
|
+
"keywords": [
|
|
13
|
+
"opencode",
|
|
14
|
+
"plugin",
|
|
15
|
+
"episodic-memory",
|
|
16
|
+
"semantic-search",
|
|
17
|
+
"embeddings",
|
|
18
|
+
"transformersjs"
|
|
19
|
+
],
|
|
20
|
+
"exports": {
|
|
21
|
+
".": "./plugin/episodic-memory.ts",
|
|
22
|
+
"./cli": "./src/cli.ts"
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
"plugin",
|
|
26
|
+
"src",
|
|
27
|
+
"skills"
|
|
28
|
+
],
|
|
29
|
+
"bin": {
|
|
30
|
+
"opencode-episodic": "./src/cli.ts"
|
|
31
|
+
},
|
|
32
|
+
"scripts": {
|
|
33
|
+
"spike": "bun run spikes/spike.ts",
|
|
34
|
+
"test": "bun test",
|
|
35
|
+
"typecheck": "tsc --noEmit",
|
|
36
|
+
"prepublishOnly": "bun run typecheck && bun test"
|
|
37
|
+
},
|
|
38
|
+
"dependencies": {
|
|
39
|
+
"@huggingface/transformers": "^4.2.0",
|
|
40
|
+
"@opencode-ai/plugin": "^1.18.4"
|
|
41
|
+
},
|
|
42
|
+
"devDependencies": {
|
|
43
|
+
"@types/bun": "latest",
|
|
44
|
+
"typescript": "^7.0.2"
|
|
45
|
+
},
|
|
46
|
+
"trustedDependencies": [
|
|
47
|
+
"onnxruntime-node",
|
|
48
|
+
"protobufjs"
|
|
49
|
+
]
|
|
50
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// OpenCode plugin: episodic memory over past conversations.
|
|
2
|
+
// - Native tools: episodic_search, episodic_read
|
|
3
|
+
// - Incremental reindex on session.idle (fire-and-forget, debounced)
|
|
4
|
+
import { type Plugin, tool } from "@opencode-ai/plugin";
|
|
5
|
+
import { openSource, getSession, getTranscript } from "../src/reader";
|
|
6
|
+
import { openIndex, search, textSearch } from "../src/store";
|
|
7
|
+
import { syncSession, syncAll, pruneOrphans } from "../src/indexer";
|
|
8
|
+
import { embedQuery } from "../src/embed";
|
|
9
|
+
import { hasExcludeMarker } from "../src/parser";
|
|
10
|
+
|
|
11
|
+
// Discriminated result so callers handle the parse error explicitly (no cast to
|
|
12
|
+
// strip the error arm off a union). `ms` is undefined when no date was given.
|
|
13
|
+
type ParsedDate = { ok: true; ms?: number } | { ok: false; error: string };
|
|
14
|
+
const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
|
15
|
+
// Require strict YYYY-MM-DD, then round-trip to reject impossible calendar dates
|
|
16
|
+
// (`new Date("2024-02-31")` silently normalizes to March 2 rather than failing).
|
|
17
|
+
const parseDateArg = (s?: string): ParsedDate => {
|
|
18
|
+
if (!s) return { ok: true };
|
|
19
|
+
const ms = new Date(s).getTime();
|
|
20
|
+
if (!DATE_RE.test(s) || Number.isNaN(ms) || new Date(ms).toISOString().slice(0, 10) !== s) {
|
|
21
|
+
return { ok: false, error: `Invalid date "${s}" (expected YYYY-MM-DD).` };
|
|
22
|
+
}
|
|
23
|
+
return { ok: true, ms };
|
|
24
|
+
};
|
|
25
|
+
const fmtDate = (ms: number) => new Date(ms).toISOString().slice(0, 10);
|
|
26
|
+
|
|
27
|
+
export const EpisodicMemory: Plugin = async ({ client }) => {
|
|
28
|
+
const log = (level: "info" | "warn" | "error", message: string) =>
|
|
29
|
+
client.app
|
|
30
|
+
.log({ body: { service: "episodic-memory", level, message } })
|
|
31
|
+
.catch(() => {});
|
|
32
|
+
|
|
33
|
+
// Debounce concurrent reindex runs for the same session.
|
|
34
|
+
const inflight = new Map<string, Promise<void>>();
|
|
35
|
+
function reindex(sessionId?: string) {
|
|
36
|
+
const key = sessionId ?? "__all__";
|
|
37
|
+
if (inflight.has(key)) return inflight.get(key)!;
|
|
38
|
+
const p = (async () => {
|
|
39
|
+
try {
|
|
40
|
+
const source = openSource();
|
|
41
|
+
const index = openIndex();
|
|
42
|
+
if (sessionId) {
|
|
43
|
+
const s = getSession(source, sessionId);
|
|
44
|
+
if (s) await syncSession(source, index, s);
|
|
45
|
+
// Cheap (two small SELECTs + rare DELETEs), so prune on every idle:
|
|
46
|
+
// the syncAll path below effectively never fires (session.idle always
|
|
47
|
+
// carries a sessionID), and without this, deleted conversations would
|
|
48
|
+
// linger in the index — searchable and readable — for plugin-only users.
|
|
49
|
+
pruneOrphans(source, index);
|
|
50
|
+
} else {
|
|
51
|
+
await syncAll(source, index); // syncAll prunes source-deleted orphans
|
|
52
|
+
}
|
|
53
|
+
await log("info", `reindexed ${key}`);
|
|
54
|
+
} catch (e) {
|
|
55
|
+
await log("warn", `reindex failed for ${key}: ${e}`);
|
|
56
|
+
} finally {
|
|
57
|
+
inflight.delete(key);
|
|
58
|
+
}
|
|
59
|
+
})();
|
|
60
|
+
inflight.set(key, p);
|
|
61
|
+
return p;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return {
|
|
65
|
+
event: async ({ event }) => {
|
|
66
|
+
if (event.type === "session.idle") {
|
|
67
|
+
// event is narrowed to EventSessionIdle here; properties.sessionID is typed.
|
|
68
|
+
reindex(event.properties.sessionID); // fire-and-forget; never block the session
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
|
|
72
|
+
tool: {
|
|
73
|
+
episodic_search: tool({
|
|
74
|
+
description:
|
|
75
|
+
"Semantic search over your PAST OpenCode conversations. Use when the user references prior work, past decisions, or previous sessions (e.g. 'how did we handle X', 'the conversation about Y', 'what did we decide about Z'). Returns dated excerpts with session IDs; follow up with episodic_read for the full conversation.",
|
|
76
|
+
args: {
|
|
77
|
+
query: tool.schema.string().describe("Natural-language description of what you're looking for"),
|
|
78
|
+
text: tool.schema.string().optional().describe("Exact substring to require in results (ANDed with semantic ranking)"),
|
|
79
|
+
mode: tool.schema.enum(["vector", "text"]).optional().describe("'vector' (default) semantic search; 'text' exact substring only"),
|
|
80
|
+
after: tool.schema.string().optional().describe("Only conversations after YYYY-MM-DD"),
|
|
81
|
+
before: tool.schema.string().optional().describe("Only conversations before YYYY-MM-DD"),
|
|
82
|
+
limit: tool.schema.number().optional().describe("Max results, 1-50 (default 10)"),
|
|
83
|
+
},
|
|
84
|
+
async execute(args) {
|
|
85
|
+
const index = openIndex();
|
|
86
|
+
const after = parseDateArg(args.after);
|
|
87
|
+
if (!after.ok) return after.error;
|
|
88
|
+
const before = parseDateArg(args.before);
|
|
89
|
+
if (!before.ok) return before.error;
|
|
90
|
+
const opts = {
|
|
91
|
+
limit: Math.min(Math.max(args.limit ?? 10, 1), 50),
|
|
92
|
+
after: after.ms,
|
|
93
|
+
before: before.ms,
|
|
94
|
+
text: args.text,
|
|
95
|
+
};
|
|
96
|
+
const hits =
|
|
97
|
+
args.mode === "text"
|
|
98
|
+
? textSearch(index, args.query, opts)
|
|
99
|
+
: search(index, (await embedQuery(args.query))[0], opts);
|
|
100
|
+
if (hits.length === 0) {
|
|
101
|
+
const chunkCount = index.prepare<{ n: number }, []>("SELECT COUNT(*) n FROM chunks").get()?.n ?? 0;
|
|
102
|
+
if (chunkCount === 0) return "No matching past conversations found. The index is empty — run `bun run src/cli.ts sync` to index conversations.";
|
|
103
|
+
return "No matching past conversations found.";
|
|
104
|
+
}
|
|
105
|
+
return hits
|
|
106
|
+
.map((h) => {
|
|
107
|
+
const snippet = h.text.replace(/\s+/g, " ").slice(0, 400);
|
|
108
|
+
return `## ${fmtDate(h.time_created)} — ${h.title}\nsession: ${h.session_id} score: ${h.score.toFixed(3)}\n${h.directory}\n> ${snippet}`;
|
|
109
|
+
})
|
|
110
|
+
.join("\n\n");
|
|
111
|
+
},
|
|
112
|
+
}),
|
|
113
|
+
|
|
114
|
+
episodic_read: tool({
|
|
115
|
+
description:
|
|
116
|
+
"Read the full transcript of a past OpenCode conversation, given a session ID (from episodic_search results). Reconstructs from the live session store; falls back to indexed excerpts if the session was deleted.",
|
|
117
|
+
args: {
|
|
118
|
+
session_id: tool.schema.string().describe("Session ID, e.g. ses_..."),
|
|
119
|
+
indexed: tool.schema.boolean().optional().describe("Force reading from the index instead of the live session store"),
|
|
120
|
+
},
|
|
121
|
+
async execute(args) {
|
|
122
|
+
if (!args.indexed) {
|
|
123
|
+
try {
|
|
124
|
+
const source = openSource();
|
|
125
|
+
const s = getSession(source, args.session_id);
|
|
126
|
+
if (s) {
|
|
127
|
+
const transcript = getTranscript(source, args.session_id);
|
|
128
|
+
if (hasExcludeMarker(transcript)) {
|
|
129
|
+
return "Session is marked private (exclusion marker present); transcript withheld.";
|
|
130
|
+
}
|
|
131
|
+
const lines: string[] = [`# ${s.title}`, `${fmtDate(s.time_created)} — ${s.directory} — ${s.id}`, ""];
|
|
132
|
+
for (const m of transcript) {
|
|
133
|
+
const text = m.parts
|
|
134
|
+
.filter((p) => p.type === "text" && p.text)
|
|
135
|
+
.map((p) => p.text)
|
|
136
|
+
.join("\n");
|
|
137
|
+
const tools = m.parts.filter((p) => p.type === "tool" && p.tool).map((p) => p.tool);
|
|
138
|
+
if (!text && tools.length === 0) continue;
|
|
139
|
+
lines.push(`## ${m.role}`);
|
|
140
|
+
if (text) lines.push(text);
|
|
141
|
+
if (tools.length) lines.push(`*(tools: ${tools.join(", ")})*`, "");
|
|
142
|
+
}
|
|
143
|
+
return lines.join("\n").slice(0, 50000);
|
|
144
|
+
}
|
|
145
|
+
} catch {
|
|
146
|
+
// fall through to indexed copy
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const index = openIndex();
|
|
150
|
+
const rows = index
|
|
151
|
+
.prepare<{ text: string }, [string]>("SELECT text FROM chunks WHERE session_id = ? ORDER BY seq")
|
|
152
|
+
.all(args.session_id);
|
|
153
|
+
if (rows.length === 0) return `No conversation found for session ${args.session_id}.`;
|
|
154
|
+
return `(indexed excerpts — live session unavailable)\n\n${rows.map((r) => r.text).join("\n\n---\n\n")}`.slice(0, 50000);
|
|
155
|
+
},
|
|
156
|
+
}),
|
|
157
|
+
},
|
|
158
|
+
};
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
export default EpisodicMemory;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: remembering-conversations
|
|
3
|
+
description: Recall past OpenCode conversations when the user references previous work, past decisions, or earlier sessions — "how did we handle X", "the conversation about Y", "what did we decide", "we tried this before", "last week we...". Use the episodic_search and episodic_read tools to search semantically and read full transcripts.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Remembering Conversations
|
|
7
|
+
|
|
8
|
+
You have episodic memory: every past OpenCode session is indexed and searchable
|
|
9
|
+
via two native tools.
|
|
10
|
+
|
|
11
|
+
## When to search
|
|
12
|
+
|
|
13
|
+
Search proactively when:
|
|
14
|
+
|
|
15
|
+
- The user references prior work: "like we did with X", "the conversation about Y",
|
|
16
|
+
"what did we decide about Z", "we tried that before"
|
|
17
|
+
- You're about to propose an approach the user may have already evaluated or rejected
|
|
18
|
+
- A bug or error message feels familiar ("didn't we see this before?")
|
|
19
|
+
- The user asks about their own history: "when did we set up X", "which repo was Y in"
|
|
20
|
+
|
|
21
|
+
Do NOT search for questions answerable from the current codebase or the current
|
|
22
|
+
conversation — the index is for cross-session recall, not code search.
|
|
23
|
+
|
|
24
|
+
## How
|
|
25
|
+
|
|
26
|
+
1. `episodic_search` with a natural-language query describing the *topic and intent*,
|
|
27
|
+
not exact keywords ("migrating from Claude Code to OpenCode", not "claude opencode").
|
|
28
|
+
- Narrow with `after`/`before` dates or an exact `text` substring when you know one
|
|
29
|
+
(an error string, a flag name, a file path).
|
|
30
|
+
- `mode: "text"` for exact-phrase lookup only.
|
|
31
|
+
2. Skim the returned excerpts (date, session title, score). Similarity scores are
|
|
32
|
+
NOT calibrated probabilities: ≥ ~0.55 is a strong match, 0.4–0.55 is likely
|
|
33
|
+
relevant, < ~0.35 is weak or merely adjacent — judge by the snippet, not the
|
|
34
|
+
number, and say when the corpus doesn't really contain the topic.
|
|
35
|
+
3. `episodic_read` with the session ID for the full transcript when an excerpt
|
|
36
|
+
isn't enough.
|
|
37
|
+
|
|
38
|
+
## Answering
|
|
39
|
+
|
|
40
|
+
- Cite what you found with its date and session title ("on 2026-07-19, in
|
|
41
|
+
'Fix login User-Agent to get past the bot wall', we concluded...").
|
|
42
|
+
- Distinguish "we decided X" from "we tried X and abandoned it" — the transcript
|
|
43
|
+
usually records the verdict; report it accurately.
|
|
44
|
+
- If search returns nothing relevant, say "I don't have a past conversation about
|
|
45
|
+
that" rather than confabulating.
|
|
46
|
+
|
|
47
|
+
## Limits
|
|
48
|
+
|
|
49
|
+
- Only OpenCode sessions are indexed (anything before the OpenCode switch is not,
|
|
50
|
+
unless it lives in `opencode.db`).
|
|
51
|
+
- Conversations containing the marker `DO NOT INDEX THIS CHAT` are excluded — that
|
|
52
|
+
includes conversations *about* this tool itself that quote the marker.
|
|
53
|
+
- Excerpts embed user/assistant text and tool names, not tool output.
|
package/src/cli.ts
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
// opencode-episodic <command> [options]
|
|
3
|
+
// sync [--force] Index new/changed sessions from opencode.db
|
|
4
|
+
// search <query> [options] Semantic search over indexed conversations
|
|
5
|
+
// --text "phrase" Exact substring match instead of vector
|
|
6
|
+
// --after YYYY-MM-DD Only conversations after this date
|
|
7
|
+
// --before YYYY-MM-DD Only conversations before this date
|
|
8
|
+
// --limit N Max results (default 10)
|
|
9
|
+
// read <session-id> [--indexed] Print a readable transcript (live DB, or --indexed for index copy)
|
|
10
|
+
// stats Index statistics
|
|
11
|
+
// doctor Diagnose setup
|
|
12
|
+
import { existsSync } from "node:fs";
|
|
13
|
+
import { openSource, sourceDbPath, getSession, getTranscript } from "./reader";
|
|
14
|
+
import { openIndex, indexDbPath, search, textSearch, stats, type SearchHit } from "./store";
|
|
15
|
+
import { syncAll } from "./indexer";
|
|
16
|
+
import { embed, embedQuery } from "./embed";
|
|
17
|
+
import { hasExcludeMarker } from "./parser";
|
|
18
|
+
|
|
19
|
+
const [, , command, ...rest] = process.argv;
|
|
20
|
+
|
|
21
|
+
function flag(name: string): string | null {
|
|
22
|
+
const i = rest.indexOf(`--${name}`);
|
|
23
|
+
return i >= 0 ? rest[i + 1] ?? null : null;
|
|
24
|
+
}
|
|
25
|
+
function hasFlag(name: string): boolean {
|
|
26
|
+
return rest.includes(`--${name}`);
|
|
27
|
+
}
|
|
28
|
+
const VALUE_FLAGS = new Set(["--text", "--after", "--before", "--limit"]);
|
|
29
|
+
function positional(): string[] {
|
|
30
|
+
const out: string[] = [];
|
|
31
|
+
for (let i = 0; i < rest.length; i++) {
|
|
32
|
+
const t = rest[i];
|
|
33
|
+
if (t.startsWith("--")) {
|
|
34
|
+
if (VALUE_FLAGS.has(t)) {
|
|
35
|
+
const next = rest[i + 1];
|
|
36
|
+
if (next === undefined || next.startsWith("--")) {
|
|
37
|
+
console.error(`error: ${t} requires a value`);
|
|
38
|
+
process.exit(1);
|
|
39
|
+
}
|
|
40
|
+
i++; // only value flags consume the next token
|
|
41
|
+
}
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
out.push(t);
|
|
45
|
+
}
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
|
49
|
+
// Require strict YYYY-MM-DD, then round-trip to reject impossible calendar dates
|
|
50
|
+
// (`new Date("2024-02-31")` silently normalizes to March 2 rather than failing).
|
|
51
|
+
const dateMs = (s: string | null): number | undefined => {
|
|
52
|
+
if (!s) return undefined;
|
|
53
|
+
const ms = new Date(s).getTime();
|
|
54
|
+
if (!DATE_RE.test(s) || Number.isNaN(ms) || new Date(ms).toISOString().slice(0, 10) !== s) {
|
|
55
|
+
console.error(`error: invalid date "${s}" (expected YYYY-MM-DD)`);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
return ms;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
function fmtDate(ms: number): string {
|
|
62
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function printHits(hits: SearchHit[]): void {
|
|
66
|
+
if (hits.length === 0) { console.log("No results."); return; }
|
|
67
|
+
for (const h of hits) {
|
|
68
|
+
const snippet = h.text.replace(/\s+/g, " ").slice(0, 220);
|
|
69
|
+
console.log(`## ${fmtDate(h.time_created)} — ${h.title}`);
|
|
70
|
+
console.log(`session: ${h.session_id} score: ${h.score.toFixed(3)}`);
|
|
71
|
+
console.log(`${h.directory}`);
|
|
72
|
+
console.log(`> ${snippet}\n`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function main() {
|
|
77
|
+
switch (command) {
|
|
78
|
+
case "sync": {
|
|
79
|
+
const source = openSource();
|
|
80
|
+
const index = openIndex();
|
|
81
|
+
const r = await syncAll(source, index, {
|
|
82
|
+
force: hasFlag("force"),
|
|
83
|
+
onProgress: (done, total, title) =>
|
|
84
|
+
process.stderr.write(`\r[${done}/${total}] ${title.slice(0, 60)} `),
|
|
85
|
+
});
|
|
86
|
+
process.stderr.write("\n");
|
|
87
|
+
console.log(
|
|
88
|
+
`scanned=${r.scanned} indexed=${r.indexed} fresh=${r.skippedFresh} excluded=${r.excluded} empty=${r.empty} pruned=${r.pruned}`
|
|
89
|
+
);
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
case "search": {
|
|
94
|
+
const query = positional().join(" ");
|
|
95
|
+
if (!query) { console.error("usage: opencode-episodic search <query> [--text p] [--after d] [--before d] [--limit n]"); process.exit(1); }
|
|
96
|
+
const index = openIndex();
|
|
97
|
+
const opts = {
|
|
98
|
+
limit: Number(flag("limit") ?? 10),
|
|
99
|
+
after: dateMs(flag("after")),
|
|
100
|
+
before: dateMs(flag("before")),
|
|
101
|
+
};
|
|
102
|
+
const textFlag = flag("text");
|
|
103
|
+
const hits = textFlag
|
|
104
|
+
? textSearch(index, textFlag, opts)
|
|
105
|
+
: search(index, (await embedQuery(query))[0], opts);
|
|
106
|
+
if (hits.length === 0 && stats(index).chunks === 0) {
|
|
107
|
+
console.log("No results. The index is empty — run: bun run src/cli.ts sync");
|
|
108
|
+
} else {
|
|
109
|
+
printHits(hits);
|
|
110
|
+
}
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
case "read": {
|
|
115
|
+
const id = positional()[0];
|
|
116
|
+
if (!id) { console.error("usage: opencode-episodic read <session-id> [--indexed]"); process.exit(1); }
|
|
117
|
+
if (hasFlag("indexed")) {
|
|
118
|
+
const index = openIndex();
|
|
119
|
+
const rows = index
|
|
120
|
+
.prepare<{ seq: number; text: string }, [string]>("SELECT seq, text FROM chunks WHERE session_id = ? ORDER BY seq")
|
|
121
|
+
.all(id);
|
|
122
|
+
if (rows.length === 0) { console.error("no indexed content for", id); process.exit(1); }
|
|
123
|
+
for (const r of rows) console.log(r.text, "\n---");
|
|
124
|
+
break;
|
|
125
|
+
}
|
|
126
|
+
const source = openSource();
|
|
127
|
+
const s = getSession(source, id);
|
|
128
|
+
if (!s) { console.error("session not found:", id); process.exit(1); }
|
|
129
|
+
const transcript = getTranscript(source, id);
|
|
130
|
+
if (hasExcludeMarker(transcript)) {
|
|
131
|
+
console.error("session is marked private (exclusion marker present); transcript withheld");
|
|
132
|
+
process.exit(1);
|
|
133
|
+
}
|
|
134
|
+
console.log(`# ${s.title}\n${fmtDate(s.time_created)} — ${s.directory} — ${s.id}\n`);
|
|
135
|
+
for (const m of transcript) {
|
|
136
|
+
const text = m.parts.filter((p) => p.type === "text" && p.text).map((p) => p.text).join("\n");
|
|
137
|
+
const tools = m.parts.filter((p) => p.type === "tool" && p.tool).map((p) => p.tool);
|
|
138
|
+
if (!text && tools.length === 0) continue;
|
|
139
|
+
console.log(`## ${m.role}`);
|
|
140
|
+
if (text) console.log(text);
|
|
141
|
+
if (tools.length) console.log(`*(tools: ${tools.join(", ")})*`);
|
|
142
|
+
console.log();
|
|
143
|
+
}
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
case "stats": {
|
|
148
|
+
const index = openIndex();
|
|
149
|
+
const s = stats(index);
|
|
150
|
+
console.log(`sessions: ${s.sessions} (${s.excluded} excluded/empty), chunks: ${s.chunks}`);
|
|
151
|
+
if (s.oldest) console.log(`range: ${fmtDate(Number(s.oldest))} → ${fmtDate(Number(s.newest))}`);
|
|
152
|
+
console.log("\nTop directories:");
|
|
153
|
+
for (const row of s.byDirectory) {
|
|
154
|
+
console.log(` ${row.n}\t${row.directory}`);
|
|
155
|
+
}
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
case "doctor": {
|
|
160
|
+
let ok = true;
|
|
161
|
+
const src = sourceDbPath();
|
|
162
|
+
if (existsSync(src)) console.log(`✓ source DB: ${src}`);
|
|
163
|
+
else { console.error(`✗ source DB missing: ${src}`); ok = false; }
|
|
164
|
+
try {
|
|
165
|
+
const source = openSource();
|
|
166
|
+
const n = source.prepare<{ n: number }, []>("SELECT COUNT(*) n FROM session").get()?.n ?? 0;
|
|
167
|
+
console.log(`✓ source readable: ${n} sessions`);
|
|
168
|
+
} catch (e) { console.error(`✗ source unreadable: ${e}`); ok = false; }
|
|
169
|
+
try {
|
|
170
|
+
const idx = openIndex();
|
|
171
|
+
console.log(`✓ index writable: ${indexDbPath()}`);
|
|
172
|
+
idx.close();
|
|
173
|
+
} catch (e) { console.error(`✗ index not writable: ${e}`); ok = false; }
|
|
174
|
+
try {
|
|
175
|
+
const v = await embed(["doctor check"]);
|
|
176
|
+
console.log(`✓ embedder: ${v[0].length} dims`);
|
|
177
|
+
} catch (e) { console.error(`✗ embedder failed: ${e}`); ok = false; }
|
|
178
|
+
process.exit(ok ? 0 : 1);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
default:
|
|
182
|
+
console.log("commands: sync | search | read | stats | doctor");
|
|
183
|
+
process.exit(command ? 1 : 0);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
await main();
|
package/src/embed.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Local, offline embeddings via Transformers.js. CLS-pooled + L2-normalized,
|
|
2
|
+
// so cosine similarity is a plain dot product.
|
|
3
|
+
//
|
|
4
|
+
// Model: Snowflake/snowflake-arctic-embed-m-v1.5 (q8) — 768 dims, Apache-2.0,
|
|
5
|
+
// official ONNX export in the model repo. Chosen over Xenova/bge-small-en-v1.5
|
|
6
|
+
// by empirical eval on our real corpus (2026-07-22, see
|
|
7
|
+
// docs/embedding-model-eval.md): equal top-1, better top-3, and far better
|
|
8
|
+
// score separation (negatives max ~0.33 vs bge's ~0.66), so minScore
|
|
9
|
+
// thresholding is meaningful. Asymmetric retriever: queries get a task
|
|
10
|
+
// prefix, documents go through unmodified.
|
|
11
|
+
import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
|
|
12
|
+
|
|
13
|
+
export const DEFAULT_MODEL = "Snowflake/snowflake-arctic-embed-m-v1.5";
|
|
14
|
+
|
|
15
|
+
// BGE/Snowflake convention (identical prompt for both): prefix QUERIES only.
|
|
16
|
+
// Idempotent via embedQuery().
|
|
17
|
+
export const QUERY_PREFIX = "Represent this sentence for searching relevant passages: ";
|
|
18
|
+
|
|
19
|
+
// Upstream measured retrieval quality peaks at 2000 chars; longer inputs
|
|
20
|
+
// degrade embeddings (and this model's window is 512 tokens anyway).
|
|
21
|
+
export const MAX_CHARS = 2000;
|
|
22
|
+
|
|
23
|
+
let cached: Promise<FeatureExtractionPipeline> | null = null;
|
|
24
|
+
|
|
25
|
+
export function getEmbedder(): Promise<FeatureExtractionPipeline> {
|
|
26
|
+
if (!cached) {
|
|
27
|
+
// transformers.js declares pipeline<"feature-extraction"> as its task-metadata
|
|
28
|
+
// record, not the FeatureExtractionPipeline instance it returns at runtime, so
|
|
29
|
+
// this cast restores the documented return type (matches HuggingFace's examples).
|
|
30
|
+
cached = pipeline("feature-extraction", process.env.EPISODIC_EMBED_MODEL ?? DEFAULT_MODEL, {
|
|
31
|
+
dtype: "q8",
|
|
32
|
+
}) as Promise<FeatureExtractionPipeline>;
|
|
33
|
+
// A rejected promise (e.g. failed model download) would poison the cache
|
|
34
|
+
// for the lifetime of the process; reset so the next call retries.
|
|
35
|
+
cached.catch(() => { if (cached) cached = null; });
|
|
36
|
+
}
|
|
37
|
+
return cached;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function embedRaw(texts: string[]): Promise<Float32Array[]> {
|
|
41
|
+
if (texts.length === 0) return [];
|
|
42
|
+
const e = await getEmbedder();
|
|
43
|
+
const out = await e(texts.map((t) => t.slice(0, MAX_CHARS)), { pooling: "cls", normalize: true });
|
|
44
|
+
const dims: number = out.dims[out.dims.length - 1];
|
|
45
|
+
// out.data is DataArray (a union incl. bigint typed arrays); a feature-extraction
|
|
46
|
+
// tensor with normalize:true is a Float32Array at runtime, so this cast is safe.
|
|
47
|
+
const flat = new Float32Array(out.data as Float32Array);
|
|
48
|
+
const vectors: Float32Array[] = [];
|
|
49
|
+
for (let i = 0; i < texts.length; i++) {
|
|
50
|
+
vectors.push(flat.subarray(i * dims, (i + 1) * dims));
|
|
51
|
+
}
|
|
52
|
+
return vectors;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Embed documents (conversation chunks). No prefix. */
|
|
56
|
+
export const embed = embedRaw;
|
|
57
|
+
|
|
58
|
+
/** Embed a search query. Prepends the retrieval prefix. */
|
|
59
|
+
export function embedQuery(query: string): Promise<Float32Array[]> {
|
|
60
|
+
const q = query.startsWith(QUERY_PREFIX) ? query : QUERY_PREFIX + query;
|
|
61
|
+
return embedRaw([q]);
|
|
62
|
+
}
|
package/src/indexer.ts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
// Incremental, idempotent indexer. Watermark = session.time_updated; a session
|
|
2
|
+
// is re-embedded only when the source changed since we last indexed it.
|
|
3
|
+
import type { Database } from "bun:sqlite";
|
|
4
|
+
import { getTranscript, listSessions, type SourceSession } from "./reader";
|
|
5
|
+
import { parseTranscript, exchangeText } from "./parser";
|
|
6
|
+
import { embed } from "./embed";
|
|
7
|
+
import { getIndexedSession, replaceSessionChunks } from "./store";
|
|
8
|
+
|
|
9
|
+
export interface SyncResult {
|
|
10
|
+
scanned: number;
|
|
11
|
+
indexed: number;
|
|
12
|
+
skippedFresh: number;
|
|
13
|
+
excluded: number;
|
|
14
|
+
empty: number;
|
|
15
|
+
pruned: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function syncSession(
|
|
19
|
+
source: Database,
|
|
20
|
+
index: Database,
|
|
21
|
+
s: SourceSession,
|
|
22
|
+
force = false
|
|
23
|
+
): Promise<"indexed" | "fresh" | "excluded" | "empty"> {
|
|
24
|
+
const prior = getIndexedSession(index, s.id);
|
|
25
|
+
if (!force && prior && prior.source_time_updated >= s.time_updated) return "fresh";
|
|
26
|
+
|
|
27
|
+
const { exchanges, excluded } = parseTranscript(getTranscript(source, s.id));
|
|
28
|
+
const meta = {
|
|
29
|
+
id: s.id, project_id: s.project_id, parent_id: s.parent_id,
|
|
30
|
+
title: s.title, directory: s.directory,
|
|
31
|
+
time_created: s.time_created, source_time_updated: s.time_updated,
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
if (excluded) {
|
|
35
|
+
replaceSessionChunks(index, meta, [], "excluded");
|
|
36
|
+
return "excluded";
|
|
37
|
+
}
|
|
38
|
+
if (exchanges.length === 0) {
|
|
39
|
+
replaceSessionChunks(index, meta, [], "empty");
|
|
40
|
+
return "empty";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const date = new Date(s.time_created).toISOString().slice(0, 10);
|
|
44
|
+
const texts = exchanges.map((e) => exchangeText(s.title, date, e));
|
|
45
|
+
const vectors = await embed(texts);
|
|
46
|
+
replaceSessionChunks(
|
|
47
|
+
index,
|
|
48
|
+
meta,
|
|
49
|
+
exchanges.map((e, i) => ({ seq: i, time_created: e.time, text: texts[i], embedding: vectors[i] }))
|
|
50
|
+
);
|
|
51
|
+
return "indexed";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function syncAll(
|
|
55
|
+
source: Database,
|
|
56
|
+
index: Database,
|
|
57
|
+
opts: { force?: boolean; onProgress?: (done: number, total: number, title: string) => void } = {}
|
|
58
|
+
): Promise<SyncResult> {
|
|
59
|
+
const sessions = listSessions(source);
|
|
60
|
+
const result: SyncResult = { scanned: sessions.length, indexed: 0, skippedFresh: 0, excluded: 0, empty: 0, pruned: 0 };
|
|
61
|
+
for (let i = 0; i < sessions.length; i++) {
|
|
62
|
+
const s = sessions[i];
|
|
63
|
+
const r = await syncSession(source, index, s, opts.force);
|
|
64
|
+
if (r === "indexed") result.indexed++;
|
|
65
|
+
else if (r === "fresh") result.skippedFresh++;
|
|
66
|
+
else if (r === "excluded") result.excluded++;
|
|
67
|
+
else result.empty++;
|
|
68
|
+
opts.onProgress?.(i + 1, sessions.length, s.title);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Prune index rows whose session no longer exists in the source DB;
|
|
72
|
+
// otherwise their stale (possibly wrong-dims) chunks linger forever.
|
|
73
|
+
result.pruned = pruneOrphans(source, index, sessions);
|
|
74
|
+
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Delete index rows (sessions + chunks) whose session has been removed from the
|
|
79
|
+
// source DB. Extracted so the plugin's full-reindex path can call it without
|
|
80
|
+
// re-running the whole sync. Pass already-fetched sessions to avoid a redundant
|
|
81
|
+
// query in syncAll; omitted, it re-reads the source.
|
|
82
|
+
export function pruneOrphans(source: Database, index: Database, knownSource?: SourceSession[]): number {
|
|
83
|
+
const sourceIds = new Set((knownSource ?? listSessions(source)).map((s) => s.id));
|
|
84
|
+
const indexedIds = index.prepare<{ id: string }, []>("SELECT id FROM sessions").all();
|
|
85
|
+
let pruned = 0;
|
|
86
|
+
for (const { id } of indexedIds) {
|
|
87
|
+
if (sourceIds.has(id)) continue;
|
|
88
|
+
index.run("DELETE FROM chunks WHERE session_id = ?", [id]);
|
|
89
|
+
index.run("DELETE FROM sessions WHERE id = ?", [id]);
|
|
90
|
+
pruned++;
|
|
91
|
+
}
|
|
92
|
+
return pruned;
|
|
93
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { describe, test, expect } from "bun:test";
|
|
2
|
+
import { parseTranscript, exchangeText, EXCLUDE_MARKER } from "./parser";
|
|
3
|
+
import type { SourceMessage } from "./reader";
|
|
4
|
+
|
|
5
|
+
const msg = (role: string, timeCreated: number, parts: SourceMessage["parts"]): SourceMessage =>
|
|
6
|
+
({ id: `${role}-${timeCreated}`, role, timeCreated, parts });
|
|
7
|
+
|
|
8
|
+
describe("parseTranscript", () => {
|
|
9
|
+
test("builds exchanges from user/assistant pairs with tool names", () => {
|
|
10
|
+
const { exchanges, excluded } = parseTranscript([
|
|
11
|
+
msg("assistant", 1, [{ type: "text", text: "dropped: no user context" }]),
|
|
12
|
+
msg("user", 2, [{ type: "text", text: "how do I fix the redirect?" }]),
|
|
13
|
+
msg("assistant", 3, [
|
|
14
|
+
{ type: "reasoning", text: "thinking..." },
|
|
15
|
+
{ type: "text", text: "Change the callback URL." },
|
|
16
|
+
{ type: "tool", tool: "edit" },
|
|
17
|
+
{ type: "tool", tool: "bash" },
|
|
18
|
+
]),
|
|
19
|
+
msg("user", 4, [{ type: "text", text: "thanks" }]),
|
|
20
|
+
msg("assistant", 5, [{ type: "text", text: "anytime" }]),
|
|
21
|
+
]);
|
|
22
|
+
expect(excluded).toBe(false);
|
|
23
|
+
expect(exchanges).toHaveLength(2);
|
|
24
|
+
expect(exchanges[0].user).toBe("how do I fix the redirect?");
|
|
25
|
+
expect(exchanges[0].assistant).toBe("Change the callback URL.");
|
|
26
|
+
expect(exchanges[0].tools).toEqual(["edit", "bash"]);
|
|
27
|
+
expect(exchanges[1].tools).toEqual([]);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("user turns without text are skipped", () => {
|
|
31
|
+
const { exchanges } = parseTranscript([
|
|
32
|
+
msg("user", 1, [{ type: "tool", tool: "read" }]), // pure tool-result turn
|
|
33
|
+
msg("user", 2, [{ type: "text", text: "real question" }]),
|
|
34
|
+
]);
|
|
35
|
+
expect(exchanges).toHaveLength(1);
|
|
36
|
+
expect(exchanges[0].user).toBe("real question");
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("exclusion marker anywhere opts out the whole transcript", () => {
|
|
40
|
+
const { exchanges, excluded } = parseTranscript([
|
|
41
|
+
msg("user", 1, [{ type: "text", text: "hi" }]),
|
|
42
|
+
msg("assistant", 2, [{ type: "text", text: `note: ${EXCLUDE_MARKER}` }]),
|
|
43
|
+
]);
|
|
44
|
+
expect(excluded).toBe(true);
|
|
45
|
+
expect(exchanges).toHaveLength(0);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
describe("exchangeText", () => {
|
|
50
|
+
test("includes date, title, participants, deduped tools", () => {
|
|
51
|
+
const text = exchangeText("My session", "2026-07-22", {
|
|
52
|
+
user: "q", assistant: "a", tools: ["bash", "bash", "edit"], time: 0,
|
|
53
|
+
});
|
|
54
|
+
expect(text).toStartWith("2026-07-22 — My session\nUser: q\nAssistant: a");
|
|
55
|
+
expect(text).toContain("Tools used: bash, edit");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("caps at 4000 chars", () => {
|
|
59
|
+
const text = exchangeText("t", "2026-07-22", {
|
|
60
|
+
user: "x".repeat(10000), assistant: "", tools: [], time: 0,
|
|
61
|
+
});
|
|
62
|
+
expect(text.length).toBe(4000);
|
|
63
|
+
});
|
|
64
|
+
});
|
package/src/parser.ts
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// Turn a raw transcript into condensed exchanges suitable for embedding.
|
|
2
|
+
// Keeps user text, assistant text, and tool *names* (not tool output, which is
|
|
3
|
+
// bulky and low-signal). Skips reasoning blobs and step markers.
|
|
4
|
+
import type { SourceMessage, SourcePart } from "./reader";
|
|
5
|
+
|
|
6
|
+
export const EXCLUDE_MARKER = "DO NOT INDEX THIS CHAT";
|
|
7
|
+
|
|
8
|
+
// True if any text part in the conversation contains the opt-out marker.
|
|
9
|
+
// Used both at index time (skip embedding) and at read time (refuse to return
|
|
10
|
+
// the transcript), so a markered chat is never surfaced verbatim.
|
|
11
|
+
export function hasExcludeMarker(messages: SourceMessage[]): boolean {
|
|
12
|
+
for (const m of messages) {
|
|
13
|
+
for (const p of m.parts) {
|
|
14
|
+
if (p.text?.includes(EXCLUDE_MARKER)) return true;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface Exchange {
|
|
21
|
+
user: string;
|
|
22
|
+
assistant: string;
|
|
23
|
+
tools: string[];
|
|
24
|
+
time: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const SKIP_PART_TYPES = new Set(["reasoning", "step-start", "step-finish", "file", "patch", "snapshot"]);
|
|
28
|
+
|
|
29
|
+
function textOf(parts: SourcePart[]): string {
|
|
30
|
+
return parts
|
|
31
|
+
.filter((p) => p.type === "text" && p.text)
|
|
32
|
+
.map((p) => p.text!.trim())
|
|
33
|
+
.filter(Boolean)
|
|
34
|
+
.join("\n");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function toolNames(parts: SourcePart[]): string[] {
|
|
38
|
+
return parts
|
|
39
|
+
.filter((p) => p.type === "tool" && p.tool && !SKIP_PART_TYPES.has(p.type))
|
|
40
|
+
.map((p) => p.tool!);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function parseTranscript(messages: SourceMessage[]): {
|
|
44
|
+
exchanges: Exchange[];
|
|
45
|
+
excluded: boolean;
|
|
46
|
+
} {
|
|
47
|
+
// Honor the opt-out marker anywhere in the conversation.
|
|
48
|
+
if (hasExcludeMarker(messages)) return { exchanges: [], excluded: true };
|
|
49
|
+
|
|
50
|
+
const exchanges: Exchange[] = [];
|
|
51
|
+
let current: Exchange | null = null;
|
|
52
|
+
|
|
53
|
+
for (const m of messages) {
|
|
54
|
+
if (m.role === "user") {
|
|
55
|
+
const text = textOf(m.parts);
|
|
56
|
+
if (!text) continue; // e.g. pure tool-result turns
|
|
57
|
+
current = { user: text, assistant: "", tools: [], time: m.timeCreated };
|
|
58
|
+
exchanges.push(current);
|
|
59
|
+
} else if (m.role === "assistant" && current) {
|
|
60
|
+
const text = textOf(m.parts);
|
|
61
|
+
if (text) current.assistant = current.assistant ? `${current.assistant}\n${text}` : text;
|
|
62
|
+
current.tools.push(...toolNames(m.parts));
|
|
63
|
+
}
|
|
64
|
+
// assistant messages before the first user message are dropped (no context)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return { exchanges: exchanges.filter((e) => e.user || e.assistant), excluded: false };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Text stored per chunk (also displayed by episodic_read). Capped at 4000 chars
|
|
71
|
+
// to keep storage sane; the embedding step (embed.ts) further truncates to 2000
|
|
72
|
+
// chars where retrieval quality peaks. The head of an exchange carries the
|
|
73
|
+
// most signal.
|
|
74
|
+
export function exchangeText(sessionTitle: string, date: string, e: Exchange): string {
|
|
75
|
+
const tools = e.tools.length ? `\nTools used: ${[...new Set(e.tools)].join(", ")}` : "";
|
|
76
|
+
const body = `User: ${e.user}\nAssistant: ${e.assistant}${tools}`;
|
|
77
|
+
return `${date} — ${sessionTitle}\n${body}`.slice(0, 4000);
|
|
78
|
+
}
|
package/src/reader.ts
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// Read-only access to OpenCode's session store (opencode.db).
|
|
2
|
+
// Schema (verified 2026-07-22): session / message / part tables, JSON blobs in `data`.
|
|
3
|
+
import { Database } from "bun:sqlite";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_SOURCE_DB = join(homedir(), ".local/share/opencode/opencode.db");
|
|
8
|
+
|
|
9
|
+
export interface SourceSession {
|
|
10
|
+
id: string;
|
|
11
|
+
project_id: string;
|
|
12
|
+
parent_id: string | null;
|
|
13
|
+
title: string;
|
|
14
|
+
directory: string;
|
|
15
|
+
time_created: number;
|
|
16
|
+
time_updated: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface SourcePart {
|
|
20
|
+
type: string;
|
|
21
|
+
text?: string;
|
|
22
|
+
tool?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface SourceMessage {
|
|
26
|
+
id: string;
|
|
27
|
+
role: string;
|
|
28
|
+
timeCreated: number;
|
|
29
|
+
parts: SourcePart[];
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function openSource(path: string = sourceDbPath()): Database {
|
|
33
|
+
return new Database(path, { readonly: true });
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function sourceDbPath(): string {
|
|
37
|
+
return process.env.EPISODIC_SOURCE_DB ?? DEFAULT_SOURCE_DB;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function listSessions(db: Database): SourceSession[] {
|
|
41
|
+
return db
|
|
42
|
+
.prepare<SourceSession, []>(
|
|
43
|
+
`SELECT id, project_id, parent_id, title, directory, time_created, time_updated
|
|
44
|
+
FROM session WHERE time_archived IS NULL ORDER BY time_created`
|
|
45
|
+
)
|
|
46
|
+
.all();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function getSession(db: Database, sessionId: string): SourceSession | null {
|
|
50
|
+
return (
|
|
51
|
+
db
|
|
52
|
+
.prepare<SourceSession, [string]>(
|
|
53
|
+
`SELECT id, project_id, parent_id, title, directory, time_created, time_updated
|
|
54
|
+
FROM session WHERE id = ?`
|
|
55
|
+
)
|
|
56
|
+
.get(sessionId) ?? null
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// JSON.parse returns `any`; these guards validate shape at runtime so no type
|
|
61
|
+
// assertion is needed. Malformed rows degrade gracefully (unknown type/role) —
|
|
62
|
+
// including a corrupt `data` blob whose JSON.parse throws, so one bad row can't
|
|
63
|
+
// abort the whole transcript read.
|
|
64
|
+
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
65
|
+
return typeof v === "object" && v !== null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function safeParse(data: string): unknown {
|
|
69
|
+
try {
|
|
70
|
+
return JSON.parse(data);
|
|
71
|
+
} catch {
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function parsePart(data: string): SourcePart {
|
|
77
|
+
const raw = safeParse(data);
|
|
78
|
+
if (!isRecord(raw)) return { type: "unknown" };
|
|
79
|
+
return {
|
|
80
|
+
type: typeof raw.type === "string" ? raw.type : "unknown",
|
|
81
|
+
text: typeof raw.text === "string" ? raw.text : undefined,
|
|
82
|
+
tool: typeof raw.tool === "string" ? raw.tool : undefined,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function parseRole(data: string): string {
|
|
87
|
+
const raw = safeParse(data);
|
|
88
|
+
return isRecord(raw) && typeof raw.role === "string" ? raw.role : "unknown";
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function getTranscript(db: Database, sessionId: string): SourceMessage[] {
|
|
92
|
+
const messages = db
|
|
93
|
+
.prepare<{ id: string; time_created: number; data: string }, [string]>(
|
|
94
|
+
`SELECT id, time_created, data FROM message
|
|
95
|
+
WHERE session_id = ? ORDER BY time_created, id`
|
|
96
|
+
)
|
|
97
|
+
.all(sessionId);
|
|
98
|
+
|
|
99
|
+
const parts = db
|
|
100
|
+
.prepare<{ message_id: string; data: string }, [string]>(
|
|
101
|
+
`SELECT message_id, data FROM part
|
|
102
|
+
WHERE session_id = ? ORDER BY time_created, id`
|
|
103
|
+
)
|
|
104
|
+
.all(sessionId);
|
|
105
|
+
|
|
106
|
+
const partsByMsg = new Map<string, SourcePart[]>();
|
|
107
|
+
for (const p of parts) {
|
|
108
|
+
const d = parsePart(p.data);
|
|
109
|
+
let list = partsByMsg.get(p.message_id);
|
|
110
|
+
if (!list) partsByMsg.set(p.message_id, (list = []));
|
|
111
|
+
list.push(d);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return messages.map((m) => ({
|
|
115
|
+
id: m.id,
|
|
116
|
+
role: parseRole(m.data),
|
|
117
|
+
timeCreated: m.time_created,
|
|
118
|
+
parts: partsByMsg.get(m.id) ?? [],
|
|
119
|
+
}));
|
|
120
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { describe, test, expect, afterAll } from "bun:test";
|
|
2
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { openIndex, replaceSessionChunks, search, textSearch, getIndexedSession } from "./store";
|
|
6
|
+
|
|
7
|
+
const dir = mkdtempSync(join(tmpdir(), "episodic-store-test-"));
|
|
8
|
+
const db = openIndex(join(dir, "index.db"));
|
|
9
|
+
afterAll(() => rmSync(dir, { recursive: true, force: true }));
|
|
10
|
+
|
|
11
|
+
const meta = {
|
|
12
|
+
id: "ses_test", project_id: "p", parent_id: null,
|
|
13
|
+
title: "Test session", directory: "/tmp",
|
|
14
|
+
time_created: 1000, source_time_updated: 1000,
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
describe("store", () => {
|
|
18
|
+
test("replaceSessionChunks + search round-trip ranks by cosine", () => {
|
|
19
|
+
replaceSessionChunks(db, meta, [
|
|
20
|
+
{ seq: 0, time_created: 1000, text: "alpha chunk", embedding: new Float32Array([1, 0]) },
|
|
21
|
+
{ seq: 1, time_created: 1001, text: "beta chunk", embedding: new Float32Array([0, 1]) },
|
|
22
|
+
]);
|
|
23
|
+
const hits = search(db, new Float32Array([1, 0]));
|
|
24
|
+
expect(hits).toHaveLength(2);
|
|
25
|
+
expect(hits[0].text).toBe("alpha chunk");
|
|
26
|
+
expect(hits[0].score).toBeCloseTo(1);
|
|
27
|
+
expect(hits[1].score).toBeCloseTo(0);
|
|
28
|
+
expect(getIndexedSession(db, "ses_test")?.title).toBe("Test session");
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("search skips embeddings with mismatched dims instead of crashing", () => {
|
|
32
|
+
// 4-byte blob while the query is 2 dims (8 bytes) — must be skipped.
|
|
33
|
+
db.run("INSERT INTO chunks (session_id, seq, time_created, text, embedding) VALUES (?, ?, ?, ?, ?)",
|
|
34
|
+
["ses_test", 99, 1002, "stale wrong-dims chunk", new Float32Array([0.5])]);
|
|
35
|
+
const hits = search(db, new Float32Array([1, 0]));
|
|
36
|
+
expect(hits.map((h) => h.text)).not.toContain("stale wrong-dims chunk");
|
|
37
|
+
expect(hits).toHaveLength(2);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test("re-embedding a session replaces its chunks", () => {
|
|
41
|
+
replaceSessionChunks(db, meta, [
|
|
42
|
+
{ seq: 0, time_created: 1000, text: "only chunk now", embedding: new Float32Array([1, 0]) },
|
|
43
|
+
]);
|
|
44
|
+
const hits = search(db, new Float32Array([1, 0]));
|
|
45
|
+
expect(hits.map((h) => h.text)).toEqual(["only chunk now"]);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("textSearch does exact substring matching", () => {
|
|
49
|
+
expect(textSearch(db, "only chunk")).toHaveLength(1);
|
|
50
|
+
expect(textSearch(db, "no such phrase")).toHaveLength(0);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("LIKE wildcards in user input are escaped (treated literally)", () => {
|
|
54
|
+
replaceSessionChunks(db, meta, [
|
|
55
|
+
{ seq: 0, time_created: 1000, text: "progress at 50% done", embedding: new Float32Array([1, 0]) },
|
|
56
|
+
{ seq: 1, time_created: 1001, text: "snake_case name here", embedding: new Float32Array([0, 1]) },
|
|
57
|
+
{ seq: 2, time_created: 1002, text: "path \\tmp", embedding: new Float32Array([1, 0]) },
|
|
58
|
+
]);
|
|
59
|
+
// % must match literally, not as a wildcard
|
|
60
|
+
expect(textSearch(db, "50%").map((h) => h.text)).toEqual(["progress at 50% done"]);
|
|
61
|
+
// _ must match literally, not as a single-char wildcard
|
|
62
|
+
expect(textSearch(db, "snake_case").map((h) => h.text)).toEqual(["snake_case name here"]);
|
|
63
|
+
// a bare % should NOT match everything (would if unescaped)
|
|
64
|
+
expect(textSearch(db, "%")).toHaveLength(1);
|
|
65
|
+
expect(textSearch(db, "_")).toHaveLength(1);
|
|
66
|
+
// a literal backslash must match only the row containing one — escapeLike
|
|
67
|
+
// escapes the escape char itself, so this would break if that were missed
|
|
68
|
+
expect(textSearch(db, "\\").map((h) => h.text)).toEqual(["path \\tmp"]);
|
|
69
|
+
// search() text filter should also escape
|
|
70
|
+
expect(search(db, new Float32Array([1, 0]), { text: "50%" })).toHaveLength(1);
|
|
71
|
+
});
|
|
72
|
+
});
|
package/src/store.ts
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Index database: plain SQLite (bun:sqlite). Embeddings stored as Float32
|
|
2
|
+
// blobs; similarity is brute-force cosine in JS. At our scale (tens of
|
|
3
|
+
// thousands of chunks) this is single-digit milliseconds per query and has
|
|
4
|
+
// zero native-extension risk. (sqlite-vec was rejected in Phase 0: bun:sqlite
|
|
5
|
+
// cannot load dynamic extensions. Swap in a vec0 backend here if scale ever
|
|
6
|
+
// demands it.)
|
|
7
|
+
import { Database } from "bun:sqlite";
|
|
8
|
+
import { mkdirSync } from "node:fs";
|
|
9
|
+
import { homedir } from "node:os";
|
|
10
|
+
import { dirname, join } from "node:path";
|
|
11
|
+
|
|
12
|
+
export const DEFAULT_INDEX_DB = join(homedir(), ".local/share/opencode-episodic-memory/index.db");
|
|
13
|
+
|
|
14
|
+
export function indexDbPath(): string {
|
|
15
|
+
return process.env.EPISODIC_INDEX_DB ?? DEFAULT_INDEX_DB;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function openIndex(path: string = indexDbPath()): Database {
|
|
19
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
20
|
+
const db = new Database(path);
|
|
21
|
+
db.run("PRAGMA journal_mode = WAL");
|
|
22
|
+
db.run(`CREATE TABLE IF NOT EXISTS sessions (
|
|
23
|
+
id TEXT PRIMARY KEY,
|
|
24
|
+
project_id TEXT NOT NULL,
|
|
25
|
+
parent_id TEXT,
|
|
26
|
+
title TEXT NOT NULL,
|
|
27
|
+
directory TEXT NOT NULL,
|
|
28
|
+
time_created INTEGER NOT NULL,
|
|
29
|
+
source_time_updated INTEGER NOT NULL,
|
|
30
|
+
indexed_at INTEGER NOT NULL,
|
|
31
|
+
status TEXT NOT NULL DEFAULT 'indexed' -- 'indexed' | 'excluded' | 'empty'
|
|
32
|
+
)`);
|
|
33
|
+
db.run(`CREATE TABLE IF NOT EXISTS chunks (
|
|
34
|
+
session_id TEXT NOT NULL,
|
|
35
|
+
seq INTEGER NOT NULL,
|
|
36
|
+
time_created INTEGER NOT NULL,
|
|
37
|
+
text TEXT NOT NULL,
|
|
38
|
+
embedding BLOB NOT NULL,
|
|
39
|
+
PRIMARY KEY (session_id, seq)
|
|
40
|
+
)`);
|
|
41
|
+
db.run("CREATE INDEX IF NOT EXISTS chunks_time_idx ON chunks(time_created)");
|
|
42
|
+
return db;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface IndexedSession {
|
|
46
|
+
id: string;
|
|
47
|
+
project_id: string;
|
|
48
|
+
parent_id: string | null;
|
|
49
|
+
title: string;
|
|
50
|
+
directory: string;
|
|
51
|
+
time_created: number;
|
|
52
|
+
source_time_updated: number;
|
|
53
|
+
indexed_at: number;
|
|
54
|
+
status: string;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function getIndexedSession(db: Database, id: string): IndexedSession | null {
|
|
58
|
+
return db.prepare<IndexedSession, [string]>("SELECT * FROM sessions WHERE id = ?").get(id) ?? null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function replaceSessionChunks(
|
|
62
|
+
db: Database,
|
|
63
|
+
s: { id: string; project_id: string; parent_id: string | null; title: string; directory: string; time_created: number; source_time_updated: number },
|
|
64
|
+
chunks: { seq: number; time_created: number; text: string; embedding: Float32Array }[],
|
|
65
|
+
status: string = "indexed"
|
|
66
|
+
): void {
|
|
67
|
+
db.transaction(() => {
|
|
68
|
+
db.run(
|
|
69
|
+
`INSERT INTO sessions (id, project_id, parent_id, title, directory, time_created, source_time_updated, indexed_at, status)
|
|
70
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
71
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
72
|
+
title=excluded.title, directory=excluded.directory,
|
|
73
|
+
source_time_updated=excluded.source_time_updated,
|
|
74
|
+
indexed_at=excluded.indexed_at, status=excluded.status`,
|
|
75
|
+
[s.id, s.project_id, s.parent_id, s.title, s.directory, s.time_created, s.source_time_updated, Date.now(), status]
|
|
76
|
+
);
|
|
77
|
+
db.run("DELETE FROM chunks WHERE session_id = ?", [s.id]);
|
|
78
|
+
const ins = db.prepare(
|
|
79
|
+
"INSERT INTO chunks (session_id, seq, time_created, text, embedding) VALUES (?, ?, ?, ?, ?)"
|
|
80
|
+
);
|
|
81
|
+
for (const c of chunks) ins.run(s.id, c.seq, c.time_created, c.text, c.embedding);
|
|
82
|
+
})();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface SearchHit {
|
|
86
|
+
session_id: string;
|
|
87
|
+
seq: number;
|
|
88
|
+
time_created: number;
|
|
89
|
+
text: string;
|
|
90
|
+
score: number;
|
|
91
|
+
title: string;
|
|
92
|
+
directory: string;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Escape LIKE wildcards so user input can't broaden a substring filter.
|
|
96
|
+
// Backslash escapes itself; used with the `ESCAPE '\'` clause below.
|
|
97
|
+
function escapeLike(s: string): string {
|
|
98
|
+
return s.replace(/[\\%_]/g, (c) => "\\" + c);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface SearchOptions {
|
|
102
|
+
limit?: number;
|
|
103
|
+
after?: number; // ms epoch
|
|
104
|
+
before?: number; // ms epoch
|
|
105
|
+
text?: string; // exact substring filter (ANDed with vector ranking)
|
|
106
|
+
minScore?: number;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function search(db: Database, queryVec: Float32Array, opts: SearchOptions = {}): SearchHit[] {
|
|
110
|
+
const limit = opts.limit ?? 10;
|
|
111
|
+
const clauses: string[] = [];
|
|
112
|
+
const params: (string | number)[] = [];
|
|
113
|
+
if (opts.after) { clauses.push("c.time_created >= ?"); params.push(opts.after); }
|
|
114
|
+
if (opts.before) { clauses.push("c.time_created < ?"); params.push(opts.before); }
|
|
115
|
+
if (opts.text) { clauses.push("c.text LIKE ? ESCAPE '\\'"); params.push(`%${escapeLike(opts.text)}%`); }
|
|
116
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
117
|
+
|
|
118
|
+
const rows = db
|
|
119
|
+
.prepare<{
|
|
120
|
+
session_id: string; seq: number; time_created: number; text: string;
|
|
121
|
+
embedding: Uint8Array; title: string; directory: string;
|
|
122
|
+
}, (string | number)[]>(
|
|
123
|
+
`SELECT c.session_id, c.seq, c.time_created, c.text, c.embedding, s.title, s.directory
|
|
124
|
+
FROM chunks c JOIN sessions s ON s.id = c.session_id ${where}`
|
|
125
|
+
)
|
|
126
|
+
.all(...params);
|
|
127
|
+
|
|
128
|
+
const dims = queryVec.length;
|
|
129
|
+
const minScore = opts.minScore ?? 0;
|
|
130
|
+
return rows
|
|
131
|
+
// Skip vectors from a different embedding model (e.g. mid-migration or
|
|
132
|
+
// orphaned rows) — a dims mismatch would corrupt the dot product or throw.
|
|
133
|
+
.filter((r) => r.embedding.byteLength === dims * 4)
|
|
134
|
+
.map((r) => {
|
|
135
|
+
const v = new Float32Array(r.embedding.buffer, r.embedding.byteOffset, dims);
|
|
136
|
+
let dot = 0;
|
|
137
|
+
for (let i = 0; i < dims; i++) dot += queryVec[i] * v[i];
|
|
138
|
+
return {
|
|
139
|
+
session_id: r.session_id, seq: r.seq, time_created: r.time_created,
|
|
140
|
+
text: r.text, score: dot, title: r.title, directory: r.directory,
|
|
141
|
+
};
|
|
142
|
+
})
|
|
143
|
+
.filter((h) => h.score >= minScore)
|
|
144
|
+
.sort((a, b) => b.score - a.score)
|
|
145
|
+
.slice(0, limit);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function textSearch(db: Database, query: string, opts: SearchOptions = {}): SearchHit[] {
|
|
149
|
+
const limit = opts.limit ?? 10;
|
|
150
|
+
const clauses = ["c.text LIKE ? ESCAPE '\\'"];
|
|
151
|
+
const params: (string | number)[] = [`%${escapeLike(query)}%`];
|
|
152
|
+
if (opts.after) { clauses.push("c.time_created >= ?"); params.push(opts.after); }
|
|
153
|
+
if (opts.before) { clauses.push("c.time_created < ?"); params.push(opts.before); }
|
|
154
|
+
const rows = db
|
|
155
|
+
.prepare<Omit<SearchHit, "score">, (string | number)[]>(
|
|
156
|
+
`SELECT c.session_id, c.seq, c.time_created, c.text, s.title, s.directory
|
|
157
|
+
FROM chunks c JOIN sessions s ON s.id = c.session_id
|
|
158
|
+
WHERE ${clauses.join(" AND ")} ORDER BY c.time_created DESC LIMIT ?`
|
|
159
|
+
)
|
|
160
|
+
.all(...params, limit);
|
|
161
|
+
return rows.map((r) => ({ ...r, score: 1 }));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export interface IndexStats {
|
|
165
|
+
sessions: number;
|
|
166
|
+
excluded: number;
|
|
167
|
+
chunks: number;
|
|
168
|
+
oldest: number | null;
|
|
169
|
+
newest: number | null;
|
|
170
|
+
byDirectory: { directory: string; n: number }[];
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function stats(db: Database): IndexStats {
|
|
174
|
+
// COUNT/MIN/MAX always return exactly one row; guard anyway so the row type
|
|
175
|
+
// stays non-null without a cast.
|
|
176
|
+
function one<T>(sql: string): T {
|
|
177
|
+
const row = db.prepare<T, []>(sql).get();
|
|
178
|
+
if (!row) throw new Error(`stats query returned no row: ${sql}`);
|
|
179
|
+
return row;
|
|
180
|
+
}
|
|
181
|
+
return {
|
|
182
|
+
sessions: one<{ n: number }>("SELECT COUNT(*) n FROM sessions").n,
|
|
183
|
+
excluded: one<{ n: number }>("SELECT COUNT(*) n FROM sessions WHERE status != 'indexed'").n,
|
|
184
|
+
chunks: one<{ n: number }>("SELECT COUNT(*) n FROM chunks").n,
|
|
185
|
+
oldest: one<{ t: number | null }>("SELECT MIN(time_created) t FROM chunks").t,
|
|
186
|
+
newest: one<{ t: number | null }>("SELECT MAX(time_created) t FROM chunks").t,
|
|
187
|
+
byDirectory: db
|
|
188
|
+
.prepare<{ directory: string; n: number }, []>(
|
|
189
|
+
"SELECT directory, COUNT(*) n FROM sessions WHERE status = 'indexed' GROUP BY directory ORDER BY n DESC LIMIT 10"
|
|
190
|
+
)
|
|
191
|
+
.all(),
|
|
192
|
+
};
|
|
193
|
+
}
|