@nogataka/claw-memory 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,16 @@
1
+ {
2
+ "name": "claw-memory-marketplace",
3
+ "owner": { "name": "nogataka" },
4
+ "metadata": {
5
+ "description": "claw-memory: local semantic memory + raw-log search for Claude Code",
6
+ "version": "0.1.0"
7
+ },
8
+ "plugins": [
9
+ {
10
+ "name": "claw-memory",
11
+ "source": "./",
12
+ "description": "Local semantic memory: auto-distill, recall injection, and raw Claude Code + Codex transcript search.",
13
+ "version": "0.1.0"
14
+ }
15
+ ]
16
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "name": "claw-memory",
3
+ "version": "0.1.0",
4
+ "description": "Local semantic memory for Claude Code: auto-distills sessions, injects relevant past context, and searches raw Claude Code + Codex transcripts. sqlite-vec + local embeddings, no daemon, no Python.",
5
+ "author": { "name": "nogataka" },
6
+ "repository": "https://github.com/nogataka/claw-memory",
7
+ "license": "MIT",
8
+ "keywords": ["memory", "mcp", "context", "search", "codex"]
9
+ }
package/.mcp.json ADDED
@@ -0,0 +1,8 @@
1
+ {
2
+ "mcpServers": {
3
+ "claw-memory": {
4
+ "command": "${CLAUDE_PLUGIN_ROOT}/hooks/claw-hook.sh",
5
+ "args": ["mcp"]
6
+ }
7
+ }
8
+ }
package/README.md ADDED
@@ -0,0 +1,168 @@
1
+ # claw-memory
2
+
3
+ Independent, **in-process** semantic memory for any MCP-capable agent (Claude Code, Codex, …).
4
+ No daemon, no Python, no external vector DB.
5
+
6
+ - **Storage**: `better-sqlite3` + `sqlite-vec` (vectors live *inside* the SQLite file)
7
+ - **Embeddings**: local `Xenova/multilingual-e5-small` (384-dim, multilingual, offline)
8
+ - **Search**: semantic (cosine KNN, per-project metadata filter) + FTS5 keyword fallback
9
+ - **Write**: LLM distillation of session transcripts → summary + preferences + chunks
10
+ - **Viewer**: optional, on-demand lightweight web UI (no framework, no build)
11
+
12
+ Data lives at `~/.claw-memory/memory.db` (override with `CLAW_MEMORY_DIR`). Completely
13
+ separate from any other app.
14
+
15
+ ## Install
16
+
17
+ Published as `@nogataka/claw-memory`. Install once globally so hooks/MCP resolve fast
18
+ (otherwise they fall back to `npx`):
19
+
20
+ ```bash
21
+ npm install -g @nogataka/claw-memory
22
+ ```
23
+
24
+ First run downloads the e5 model (~100 MB, cached in `~/.cache`).
25
+
26
+ ### Claude Code (plugin)
27
+
28
+ ```
29
+ /plugin marketplace add nogataka/claw-memory
30
+ /plugin install claw-memory
31
+ ```
32
+
33
+ Restart Claude Code. This auto-registers the MCP server and the hooks
34
+ (SessionStart/UserPromptSubmit → recall injection, Stop → auto-distill).
35
+ No manual setup. (Not using the plugin? Run `claw-memory install --claude-code`
36
+ to merge the MCP server + hooks into `~/.claude/settings.json`.)
37
+
38
+ ### Codex (installer)
39
+
40
+ Codex has no third-party plugin marketplace, so register via the installer:
41
+
42
+ ```bash
43
+ claw-memory install --codex # adds [mcp_servers.claw-memory] to ~/.codex/config.toml
44
+ # + memory-recall skill + AGENTS.md recall instruction
45
+ ```
46
+
47
+ Restart Codex. Recall is available via the `memory_recall` MCP tool (the AGENTS.md
48
+ note tells the agent to call it at session start). **Auto-distill is manual on Codex**
49
+ (no notify hook); run periodically:
50
+
51
+ ```bash
52
+ claw-memory distill-codex --recent # distill recent Codex sessions (watermark-deduped)
53
+ claw-memory distill-codex --all # backfill everything
54
+ ```
55
+
56
+ Remove with `claw-memory uninstall --codex` (or `--claude-code`).
57
+
58
+ ### From source
59
+
60
+ ```bash
61
+ cd claw-memory
62
+ npm install # builds native better-sqlite3 / sqlite-vec
63
+ npm run build # tsc -> dist/
64
+ ```
65
+
66
+ ## MCP tools
67
+
68
+ | Tool | Purpose |
69
+ |------|---------|
70
+ | `memory_recall(query, cwd?, topK?)` | Ready-to-read context block: preferences + summaries + similar past conversations |
71
+ | `memory_search(query, cwd?, limit?)` | Token-efficient hit index (id + title + date) |
72
+ | `memory_get(ids)` | Full text for given chunk ids |
73
+ | `memory_remember(text, cwd?, sessionId?)` | Store a free-text note |
74
+ | `memory_distill(cwd, sessionId? \| transcriptPath?)` | Summarize a session into memory (needs LLM creds) |
75
+ | `memory_get_preferences(cwd?)` | List stored preferences |
76
+ | `memory_search_logs(query, sources?, projectPath?, startDate?, endDate?, limit?, offset?)` | Full-text search over RAW Claude Code + Codex transcripts (second memory source; finds undistilled history) |
77
+ | `memory_forget(ids)` | Soft-delete chunks (hidden from search/recall/viewer) |
78
+
79
+ `memory_distill` uses an LLM (see *LLM backend* below). `memory_search_logs` reads
80
+ `~/.claude/projects` and `~/.codex/sessions` directly. All other tools are fully local.
81
+
82
+ ## LLM backend (distill only)
83
+
84
+ distill needs a single tool-less completion, selectable via `CLAW_MEMORY_LLM_BACKEND`:
85
+
86
+ | Backend | Auth | Notes |
87
+ |---------|------|-------|
88
+ | `agent-sdk` (default) | Claude Code CLI login (Claude Pro/Max/Team/Enterprise) | zero-config, no API key |
89
+ | `codex-sdk` | Codex CLI login (ChatGPT/Codex plan) | `@openai/codex-sdk`, no API key; requires Codex CLI. Model via `CLAW_MEMORY_CODEX_MODEL` |
90
+ | `anthropic` | `ANTHROPIC_API_KEY` (`ANTHROPIC_BASE_URL` optional) | plain Messages API |
91
+ | `openai-compatible` | `CLAW_MEMORY_OPENAI_API_KEY` + `CLAW_MEMORY_OPENAI_BASE_URL` | Gemini / OpenRouter / LM Studio (set `CLAW_MEMORY_MODEL`) |
92
+
93
+ ```bash
94
+ export CLAW_MEMORY_LLM_BACKEND=codex-sdk # use the Codex subscription for distill
95
+ ```
96
+
97
+ Both SDK backends reuse a subscription login (no API key). Tier routing (cheap models for
98
+ simple work): `CLAW_MEMORY_TIER_SMART` / `_SUMMARY` / `_SIMPLE`, else `AGENT_SDK_MODEL` /
99
+ `CLAW_MEMORY_MODEL` (agent-sdk/anthropic default `claude-sonnet-4-5`; codex-sdk uses the Codex default).
100
+
101
+ ## Automatic capture & recall (Claude Code hooks)
102
+
103
+ No daemon — hooks spawn the CLI per event. Register in `~/.claude/settings.json`:
104
+
105
+ ```json
106
+ {
107
+ "hooks": {
108
+ "SessionStart": [{ "hooks": [{ "type": "command", "command": "/Volumes/Data/dev/claw-memory/hooks/claw-hook.sh recall" }] }],
109
+ "UserPromptSubmit": [{ "hooks": [{ "type": "command", "command": "/Volumes/Data/dev/claw-memory/hooks/claw-hook.sh recall" }] }],
110
+ "Stop": [{ "hooks": [{ "type": "command", "command": "/Volumes/Data/dev/claw-memory/hooks/claw-hook.sh distill" }] }]
111
+ }
112
+ }
113
+ ```
114
+
115
+ - **recall** (SessionStart / UserPromptSubmit): prints the memory block to stdout → injected into context. Preferences + recent summaries, plus semantically similar past conversations when the prompt is present.
116
+ - **distill** (Stop): detached, fire-and-forget. Incremental via a watermark (skips sessions with no new content). Honors `CLAW_MEMORY_EXCLUDED_PROJECTS` and `<private>…</private>`.
117
+
118
+ ## Register with an agent
119
+
120
+ ### Claude Code — `.mcp.json` (project root) or `~/.claude.json`
121
+
122
+ ```json
123
+ {
124
+ "mcpServers": {
125
+ "memory": {
126
+ "command": "node",
127
+ "args": ["/Volumes/Data/dev/claw-memory/dist/cli.js", "mcp"]
128
+ }
129
+ }
130
+ }
131
+ ```
132
+
133
+ ### Codex — `~/.codex/config.toml`
134
+
135
+ ```toml
136
+ [mcp_servers.memory]
137
+ command = "node"
138
+ args = ["/Volumes/Data/dev/claw-memory/dist/cli.js", "mcp"]
139
+ ```
140
+
141
+ Then add to your `CLAUDE.md` / `AGENTS.md`:
142
+ > 会話の冒頭で `memory_recall` を呼び、過去の文脈と好みを取得すること。
143
+
144
+ ## Memory viewer
145
+
146
+ ```bash
147
+ npm run ui # http://localhost:4319 , opens browser
148
+ # or
149
+ node dist/cli.js ui --port 4319 --open
150
+ ```
151
+
152
+ Read-only. Browse projects, session summaries, conversation chunks and preferences.
153
+ Start it only when you want to inspect; nothing runs in the background otherwise.
154
+
155
+ ## CLI
156
+
157
+ ```bash
158
+ node dist/cli.js mcp # stdio MCP server
159
+ node dist/cli.js ui [--port N] [--open] # viewer
160
+ node dist/cli.js distill --cwd <dir> --session <id> # distill a CC session
161
+ node dist/cli.js remember --cwd <dir> "a note" # store a note
162
+ ```
163
+
164
+ ## Notes
165
+
166
+ - `better-sqlite3` / `sqlite-vec` are native; rebuild on Node ABI changes (`npm rebuild`).
167
+ - The MCP server is long-lived per agent session, so the embedding model loads once.
168
+ - Viewer + MCP can run simultaneously (SQLite WAL handles concurrent read/write).
package/dist/cli.js ADDED
@@ -0,0 +1,223 @@
1
+ #!/usr/bin/env node
2
+ // src/cli.ts
3
+ //
4
+ // claw-memory CLI. Subcommands:
5
+ // mcp start the stdio MCP server (what agents spawn)
6
+ // ui [--port N] [--open] start the on-demand memory viewer
7
+ // distill --cwd P --session ID [--path FILE]
8
+ // remember --cwd P "text"
9
+ function getFlag(args, name) {
10
+ const i = args.indexOf(`--${name}`);
11
+ return i >= 0 ? args[i + 1] : undefined;
12
+ }
13
+ function hasFlag(args, name) {
14
+ return args.includes(`--${name}`);
15
+ }
16
+ async function main() {
17
+ const [cmd, ...rest] = process.argv.slice(2);
18
+ switch (cmd) {
19
+ case "mcp": {
20
+ const { runMcpServer } = await import("./mcp/server.js");
21
+ await runMcpServer();
22
+ return; // stays alive on stdio
23
+ }
24
+ case "ui": {
25
+ const { runUiServer } = await import("./ui/server.js");
26
+ const port = Number(getFlag(rest, "port") ?? process.env.CLAW_MEMORY_UI_PORT ?? 4319);
27
+ runUiServer(port, hasFlag(rest, "open"));
28
+ return;
29
+ }
30
+ case "distill": {
31
+ const { getOrCreateProjectByPath } = await import("./core/projects.js");
32
+ const { distill } = await import("./core/distill.js");
33
+ const { resolveSessionJsonl } = await import("./core/transcript.js");
34
+ const cwd = getFlag(rest, "cwd") ?? process.cwd();
35
+ const session = getFlag(rest, "session") ?? "";
36
+ const path = getFlag(rest, "path") ?? (session ? resolveSessionJsonl(cwd, session) : undefined);
37
+ if (!path)
38
+ throw new Error("distill requires --session or --path");
39
+ const project = getOrCreateProjectByPath(cwd);
40
+ if (hasFlag(rest, "if-stale")) {
41
+ // Incremental mode (used by hooks): skip if the transcript hasn't grown
42
+ // since last distill; stamp the watermark on success.
43
+ const { statSync } = await import("node:fs");
44
+ const { shouldDistill, markDistilled } = await import("./core/watermark.js");
45
+ let mtimeMs = 0;
46
+ try {
47
+ mtimeMs = statSync(path).mtimeMs;
48
+ }
49
+ catch {
50
+ console.log('{"skipped":"no transcript"}');
51
+ process.exit(0);
52
+ }
53
+ if (!shouldDistill(path, mtimeMs)) {
54
+ console.log('{"skipped":"up-to-date"}');
55
+ process.exit(0);
56
+ }
57
+ const res = await distill({ projectId: project.id, sessionId: session || path, transcriptPath: path });
58
+ markDistilled(path, mtimeMs);
59
+ console.log(JSON.stringify(res, null, 2));
60
+ process.exit(0);
61
+ }
62
+ const res = await distill({ projectId: project.id, sessionId: session || path, transcriptPath: path });
63
+ console.log(JSON.stringify(res, null, 2));
64
+ process.exit(0);
65
+ }
66
+ case "hook": {
67
+ const { readStdinJson, runDistillHook, runRecallHook } = await import("./core/hooks.js");
68
+ const mode = rest[0]; // "distill" | "recall"
69
+ const input = await readStdinJson();
70
+ if (mode === "distill") {
71
+ runDistillHook(input);
72
+ process.exit(0);
73
+ }
74
+ if (mode === "recall") {
75
+ await runRecallHook(input);
76
+ process.exit(0);
77
+ }
78
+ throw new Error("hook requires a mode: distill | recall");
79
+ }
80
+ case "inject-recall": {
81
+ // Convenience alias for `hook recall` (SessionStart/UserPromptSubmit).
82
+ const { readStdinJson, runRecallHook } = await import("./core/hooks.js");
83
+ await runRecallHook(await readStdinJson());
84
+ process.exit(0);
85
+ }
86
+ case "install": {
87
+ const target = hasFlag(rest, "claude-code") ? "claude" : "codex";
88
+ if (target === "codex") {
89
+ const { installCodex } = await import("./core/installer/codex.js");
90
+ console.log("claw-memory installed for Codex:\n " + installCodex().join("\n "));
91
+ console.log("\n注意: Codex を再起動してください。自動 distill は手動 (`claw-memory distill-codex --recent`)。");
92
+ }
93
+ else {
94
+ const { installClaude } = await import("./core/installer/claude.js");
95
+ console.log("claw-memory installed for Claude Code (manual):\n " + installClaude().join("\n "));
96
+ console.log("\n注意: Claude Code を再起動してください。プラグイン利用時はこの手動設定は不要です。");
97
+ }
98
+ process.exit(0);
99
+ }
100
+ case "uninstall": {
101
+ const target = hasFlag(rest, "claude-code") ? "claude" : "codex";
102
+ if (target === "codex") {
103
+ const { uninstallCodex } = await import("./core/installer/codex.js");
104
+ console.log("removed:\n " + uninstallCodex().join("\n "));
105
+ }
106
+ else {
107
+ const { uninstallClaude } = await import("./core/installer/claude.js");
108
+ console.log("removed:\n " + uninstallClaude().join("\n "));
109
+ }
110
+ process.exit(0);
111
+ }
112
+ case "distill-codex": {
113
+ const { listCodexSessionFiles, codexSessionId } = await import("./core/logsearch/recent.js");
114
+ const { parseCodexSession } = await import("./core/logsearch/parse.js");
115
+ const { distill } = await import("./core/distill.js");
116
+ const { getOrCreateProjectByPath } = await import("./core/projects.js");
117
+ const { shouldDistill, markDistilled } = await import("./core/watermark.js");
118
+ const { isExcludedPath } = await import("./core/excludes.js");
119
+ const { statSync, readFileSync } = await import("node:fs");
120
+ const all = hasFlag(rest, "all");
121
+ const limit = Number(getFlag(rest, "limit") ?? (all ? 100000 : 20));
122
+ const files = (await listCodexSessionFiles()).slice(0, limit);
123
+ let distilled = 0, skipped = 0, failed = 0;
124
+ for (const f of files) {
125
+ let mtimeMs = 0;
126
+ try {
127
+ mtimeMs = statSync(f.path).mtimeMs;
128
+ }
129
+ catch {
130
+ continue;
131
+ }
132
+ if (!shouldDistill(f.path, mtimeMs)) {
133
+ skipped++;
134
+ continue;
135
+ }
136
+ let cwd = process.cwd();
137
+ try {
138
+ const c = parseCodexSession(readFileSync(f.path, "utf-8")).cwd;
139
+ if (c)
140
+ cwd = c;
141
+ }
142
+ catch { /* keep default */ }
143
+ if (isExcludedPath(cwd)) {
144
+ markDistilled(f.path, mtimeMs);
145
+ skipped++;
146
+ continue;
147
+ }
148
+ const project = getOrCreateProjectByPath(cwd);
149
+ try {
150
+ const res = await distill({ projectId: project.id, sessionId: codexSessionId(f.path), transcriptPath: f.path });
151
+ markDistilled(f.path, mtimeMs);
152
+ if (res.skipped)
153
+ skipped++;
154
+ else
155
+ distilled++;
156
+ }
157
+ catch (e) {
158
+ failed++;
159
+ console.error(`distill failed ${f.path}: ${String(e)}`);
160
+ }
161
+ }
162
+ console.log(JSON.stringify({ scanned: files.length, distilled, skipped, failed }, null, 2));
163
+ process.exit(0);
164
+ }
165
+ case "remember": {
166
+ const { getOrCreateProjectByPath } = await import("./core/projects.js");
167
+ const { rememberText } = await import("./core/distill.js");
168
+ const cwd = getFlag(rest, "cwd") ?? process.cwd();
169
+ const text = rest.filter((a) => !a.startsWith("--") && a !== cwd).join(" ").trim();
170
+ if (!text)
171
+ throw new Error("remember requires text");
172
+ const project = getOrCreateProjectByPath(cwd);
173
+ const id = await rememberText({ projectId: project.id, sessionId: "manual", text });
174
+ console.log(`saved id=${id}`);
175
+ process.exit(0);
176
+ }
177
+ case "search-logs": {
178
+ const { searchLogs } = await import("./core/logsearch/search.js");
179
+ // Positional query = tokens that are neither a --flag nor a flag's value.
180
+ const positionals = [];
181
+ for (let i = 0; i < rest.length; i++) {
182
+ if (rest[i].startsWith("--")) {
183
+ i++; // skip this flag's value
184
+ continue;
185
+ }
186
+ positionals.push(rest[i]);
187
+ }
188
+ const query = positionals.join(" ").trim();
189
+ if (!query)
190
+ throw new Error("search-logs requires a query");
191
+ const sourcesArg = getFlag(rest, "source");
192
+ const out = await searchLogs({
193
+ query,
194
+ sources: sourcesArg ? sourcesArg.split(",") : undefined,
195
+ projectPath: getFlag(rest, "project"),
196
+ startDate: getFlag(rest, "start"),
197
+ endDate: getFlag(rest, "end"),
198
+ limit: Number(getFlag(rest, "limit") ?? 20),
199
+ offset: Number(getFlag(rest, "offset") ?? 0),
200
+ });
201
+ console.log(JSON.stringify(out, null, 2));
202
+ process.exit(0);
203
+ }
204
+ default:
205
+ console.error("Usage: claw-memory <command>\n" +
206
+ " mcp start stdio MCP server\n" +
207
+ " ui [--port N] [--open] start memory viewer\n" +
208
+ " distill --cwd P --session ID [--path FILE] [--if-stale]\n" +
209
+ " distill-codex [--recent] [--limit N] [--all] distill recent Codex sessions\n" +
210
+ " remember --cwd P \"text\"\n" +
211
+ " search-logs \"query\" [--source claude-code,codex] [--project P] [--start ISO] [--end ISO] [--limit N] [--offset N]\n" +
212
+ " hook <distill|recall> run a Claude Code lifecycle hook (reads JSON on stdin)\n" +
213
+ " inject-recall alias for `hook recall`\n" +
214
+ " install [--codex|--claude-code] register MCP + hooks (default: codex)\n" +
215
+ " uninstall [--codex|--claude-code] remove claw-memory config");
216
+ process.exit(cmd ? 1 : 0);
217
+ }
218
+ }
219
+ main().catch((err) => {
220
+ console.error(err);
221
+ process.exit(1);
222
+ });
223
+ export {};
@@ -0,0 +1,14 @@
1
+ // src/core/config.ts
2
+ import { sqlite } from "./db.js";
3
+ export function getConfig(key) {
4
+ const row = sqlite
5
+ .prepare("SELECT value FROM app_config WHERE key = ?")
6
+ .get(key);
7
+ return row?.value;
8
+ }
9
+ export function setConfig(key, value) {
10
+ sqlite
11
+ .prepare(`INSERT INTO app_config(key, value, updated_at) VALUES (?, ?, datetime('now'))
12
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`)
13
+ .run(key, value);
14
+ }
@@ -0,0 +1,92 @@
1
+ // src/core/db.ts
2
+ //
3
+ // Storage foundation: better-sqlite3 + sqlite-vec (in-process vector search) +
4
+ // FTS5 keyword fallback. No ORM, no daemon, no Python.
5
+ import Database from "better-sqlite3";
6
+ import * as sqliteVec from "sqlite-vec";
7
+ import { dbPath } from "./paths.js";
8
+ export const sqlite = new Database(dbPath);
9
+ sqlite.pragma("journal_mode = WAL");
10
+ sqlite.pragma("busy_timeout = 5000");
11
+ sqlite.pragma("foreign_keys = ON");
12
+ // Load the sqlite-vec extension (provides the vec0 virtual table).
13
+ sqliteVec.load(sqlite);
14
+ sqlite.exec(`
15
+ CREATE TABLE IF NOT EXISTS projects (
16
+ id TEXT PRIMARY KEY,
17
+ name TEXT NOT NULL,
18
+ path TEXT NOT NULL UNIQUE,
19
+ created_at TEXT NOT NULL,
20
+ last_accessed_at TEXT NOT NULL
21
+ );
22
+ CREATE TABLE IF NOT EXISTS session_summaries (
23
+ id TEXT PRIMARY KEY,
24
+ project_id TEXT NOT NULL,
25
+ session_id TEXT NOT NULL,
26
+ summary TEXT NOT NULL,
27
+ created_at TEXT NOT NULL DEFAULT (datetime('now'))
28
+ );
29
+ CREATE TABLE IF NOT EXISTS user_preferences (
30
+ id TEXT PRIMARY KEY,
31
+ project_id TEXT,
32
+ key TEXT NOT NULL,
33
+ value TEXT NOT NULL,
34
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
35
+ );
36
+ CREATE TABLE IF NOT EXISTS app_config (
37
+ key TEXT PRIMARY KEY,
38
+ value TEXT NOT NULL,
39
+ updated_at TEXT NOT NULL DEFAULT (datetime('now'))
40
+ );
41
+ CREATE TABLE IF NOT EXISTS conversation_chunks (
42
+ id TEXT PRIMARY KEY,
43
+ vec_rowid INTEGER NOT NULL,
44
+ project_id TEXT NOT NULL,
45
+ session_id TEXT NOT NULL,
46
+ user_text TEXT NOT NULL,
47
+ assistant_text TEXT NOT NULL,
48
+ created_at TEXT NOT NULL
49
+ );
50
+ CREATE TABLE IF NOT EXISTS distill_watermarks (
51
+ path TEXT PRIMARY KEY,
52
+ mtime_ms INTEGER NOT NULL,
53
+ distilled_at TEXT NOT NULL
54
+ );
55
+ `);
56
+ // --- Additive, migration-less schema evolution -----------------------------
57
+ // New columns are added with try/catch so existing ~/.claw-memory/memory.db
58
+ // files keep working (SQLite ALTER TABLE ADD COLUMN can't be IF NOT EXISTS).
59
+ for (const col of [
60
+ "obs_type TEXT",
61
+ "concepts TEXT",
62
+ "files_read TEXT",
63
+ "files_modified TEXT",
64
+ "deleted_at TEXT",
65
+ ]) {
66
+ try {
67
+ sqlite.exec(`ALTER TABLE conversation_chunks ADD COLUMN ${col}`);
68
+ }
69
+ catch {
70
+ // column already exists
71
+ }
72
+ }
73
+ // vec0 virtual table — embedding + project_id metadata column so KNN can filter
74
+ // by project inside the MATCH query (no post-filter loss).
75
+ sqlite.exec(`
76
+ CREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(
77
+ embedding float[384] distance_metric=cosine,
78
+ project_id text
79
+ );
80
+ `);
81
+ // FTS5 keyword index over chunk text (claude-mem-style fallback when the query
82
+ // is keyword-ish and semantic similarity is weak). Content-less; chunk_id links
83
+ // back to conversation_chunks.id. Uses the trigram tokenizer: it matches
84
+ // case-insensitive substrings (>=3 chars), which is essential for Japanese text
85
+ // that has no word spaces and for Latin words glued to CJK (e.g. "TypeScriptの").
86
+ sqlite.exec(`
87
+ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
88
+ chunk_id UNINDEXED,
89
+ text,
90
+ tokenize = 'trigram'
91
+ );
92
+ `);
@@ -0,0 +1,146 @@
1
+ // src/core/distill.ts
2
+ //
3
+ // Write path: distill a conversation into a 1-2 sentence summary + user
4
+ // preferences + embedded conversation chunks. Mirrors agent-claw's summarize
5
+ // pipeline, minus the HTTP layer.
6
+ import { complete } from "./llm.js";
7
+ import { embedPassage } from "./embeddings.js";
8
+ import { addSessionSummary, setPreference } from "./memory.js";
9
+ import { saveChunks, deleteChunksBySession, chunkExists, } from "./vector-memory.js";
10
+ import { loadTranscript } from "./transcript.js";
11
+ import { stripPrivate } from "./private.js";
12
+ import { log } from "./logger.js";
13
+ const MIN_MESSAGES = 2;
14
+ const MIN_TEXT_LENGTH = 100;
15
+ const MAX_CHARS_PER_MESSAGE = 500;
16
+ const OBS_TYPES = ["discovery", "bugfix", "feature", "decision", "change", "other"];
17
+ const PROMPT = (transcript) => `以下の会話を分析して JSON で回答してください。
18
+
19
+ 1. summary: 会話の構造化要約。会話と同じ言語で、次の節を含む簡潔なMarkdown:
20
+ "### 依頼" / "### 調査・判明" / "### 完了" / "### 次の一手"(該当が無い節は省略可)。
21
+ 2. obs_type: この会話の主目的を1つ選ぶ: ${OBS_TYPES.join(" | ")}
22
+ 3. concepts: 主要トピック・技術キーワードの配列(3〜8個程度)。
23
+ 4. files_read: 会話中で読んだ/参照したファイルパスの配列(無ければ空配列)。
24
+ 5. files_modified: 会話中で作成/編集したファイルパスの配列(無ければ空配列)。
25
+ 6. preferences: ユーザーの設定や好みが読み取れた場合のみ。
26
+ key は必ず次のいずれか(該当しなければ出さない):
27
+ language | response_style | detail_level | code_style | framework | tone | tools
28
+ 変更がなければ空配列。
29
+
30
+ JSON のみ回答:
31
+ {"summary": "...", "obs_type": "...", "concepts": ["..."], "files_read": ["..."], "files_modified": ["..."], "preferences": [{"key": "...", "value": "..."}]}
32
+
33
+ <conversation>
34
+ ${transcript}
35
+ </conversation>`;
36
+ export async function distill(input) {
37
+ const loaded = input.messages ??
38
+ (input.transcriptPath ? loadTranscript(input.transcriptPath) : []);
39
+ // Drop <private>…</private> spans before anything is persisted or sent to the LLM.
40
+ const messages = loaded
41
+ .map((m) => ({ ...m, text: stripPrivate(m.text) }))
42
+ .filter((m) => m.text.trim());
43
+ if (messages.length < MIN_MESSAGES) {
44
+ return { skipped: true, reason: "too few messages" };
45
+ }
46
+ const transcript = messages
47
+ .map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.text.slice(0, MAX_CHARS_PER_MESSAGE)}`)
48
+ .join("\n");
49
+ if (transcript.length < MIN_TEXT_LENGTH) {
50
+ return { skipped: true, reason: "too short" };
51
+ }
52
+ // --- LLM: summary + preferences (single tool-less turn) ---
53
+ const responseText = await complete({
54
+ prompt: PROMPT(transcript),
55
+ tier: "summary",
56
+ });
57
+ const jsonMatch = responseText.match(/\{[\s\S]*\}/);
58
+ if (!jsonMatch)
59
+ throw new Error("Failed to parse distill JSON response");
60
+ const result = JSON.parse(jsonMatch[0]);
61
+ if (result.summary) {
62
+ addSessionSummary(input.projectId, input.sessionId, result.summary);
63
+ }
64
+ if (Array.isArray(result.preferences)) {
65
+ for (const pref of result.preferences) {
66
+ if (pref.key && pref.value) {
67
+ setPreference(input.projectId, pref.key, pref.value);
68
+ }
69
+ }
70
+ }
71
+ // Session-level structured metadata applied to every chunk of this session.
72
+ const obsType = result.obs_type && OBS_TYPES.includes(result.obs_type)
73
+ ? result.obs_type
74
+ : null;
75
+ const concepts = (result.concepts ?? []).map(String).filter(Boolean);
76
+ const filesRead = (result.files_read ?? []).map(String).filter(Boolean);
77
+ const filesModified = (result.files_modified ?? []).map(String).filter(Boolean);
78
+ // --- Vector memory: re-chunk this session (idempotent) ---
79
+ let chunkCount = 0;
80
+ try {
81
+ deleteChunksBySession(input.sessionId);
82
+ const pairs = [];
83
+ for (let i = 0; i < messages.length; i++) {
84
+ const m = messages[i];
85
+ if (m.role === "user" && m.text.trim()) {
86
+ const next = messages[i + 1];
87
+ pairs.push({
88
+ userText: m.text.slice(0, 500),
89
+ assistantText: next?.role === "assistant" ? next.text.slice(0, 500) : "",
90
+ });
91
+ }
92
+ }
93
+ if (pairs.length > 0) {
94
+ const toSave = [];
95
+ for (const p of pairs) {
96
+ // Cross-session dedup: skip pairs already stored verbatim.
97
+ if (chunkExists(input.projectId, p.userText, p.assistantText))
98
+ continue;
99
+ const embedding = await embedPassage(`User: ${p.userText}\nAssistant: ${p.assistantText}`);
100
+ toSave.push({
101
+ projectId: input.projectId,
102
+ sessionId: input.sessionId,
103
+ userText: p.userText,
104
+ assistantText: p.assistantText,
105
+ embedding,
106
+ obsType,
107
+ concepts,
108
+ filesRead,
109
+ filesModified,
110
+ });
111
+ }
112
+ if (toSave.length > 0)
113
+ saveChunks(toSave);
114
+ chunkCount = toSave.length;
115
+ }
116
+ }
117
+ catch (err) {
118
+ console.error("[claw-memory] chunk embed failed:", err);
119
+ }
120
+ log("distill", {
121
+ projectId: input.projectId,
122
+ sessionId: input.sessionId,
123
+ obsType,
124
+ chunks: chunkCount,
125
+ preferences: result.preferences?.length ?? 0,
126
+ });
127
+ return {
128
+ summary: result.summary,
129
+ preferencesCount: result.preferences?.length ?? 0,
130
+ chunks: chunkCount,
131
+ };
132
+ }
133
+ /** Store a single free-text note as an embedded chunk (memory_remember). */
134
+ export async function rememberText(args) {
135
+ const embedding = await embedPassage(args.text);
136
+ const [id] = saveChunks([
137
+ {
138
+ projectId: args.projectId,
139
+ sessionId: args.sessionId,
140
+ userText: args.text.slice(0, 500),
141
+ assistantText: "",
142
+ embedding,
143
+ },
144
+ ]);
145
+ return id;
146
+ }