@nogataka/claw-memory 0.1.0

package/dist/core/logsearch/search.js

@@ -0,0 +1,233 @@
+// src/core/logsearch/search.ts
+//
+// cc-search port: full-text substring search across the RAW agent transcripts of
+// Claude Code and Codex — a second memory source, independent of claw-memory's
+// distilled DB. Surfaces conversations that were never distilled. Read-only,
+// dependency-free (node stdlib only).
+import { readdir, readFile, stat } from "node:fs/promises";
+import { resolve, join, basename } from "node:path";
+import { createInterface } from "node:readline";
+import { createReadStream } from "node:fs";
+import { claudeProjectsRoot, codexSessionsRoot, MAX_LOG_FILE_SIZE, UUID_JSONL_RE, } from "./paths.js";
+import { parseClaudeCodeLine, parseCodexSession } from "./parse.js";
+const CONTEXT = 100;
+export async function searchLogs(opts) {
+    const query = opts.query.trim();
+    if (!query)
+        return { results: [], total: 0 };
+    const sources = opts.sources ?? ["claude-code", "codex"];
+    const limit = opts.limit ?? 20;
+    const offset = opts.offset ?? 0;
+    const all = [];
+    if (sources.includes("claude-code")) {
+        all.push(...(await searchClaudeCode(query, opts)));
+    }
+    if (sources.includes("codex")) {
+        all.push(...(await searchCodex(query, opts)));
+    }
+    all.sort((a, b) => tsValue(b.timestamp) - tsValue(a.timestamp));
+    return { results: all.slice(offset, offset + limit), total: all.length };
+}
+function tsValue(ts) {
+    if (!ts)
+        return 0;
+    const n = new Date(ts).getTime();
+    return Number.isNaN(n) ? 0 : n;
+}
+function inDateRange(ts, start, end) {
+    if (!start && !end)
+        return true;
+    if (!ts)
+        return false;
+    const t = tsValue(ts);
+    if (start && t < new Date(start).getTime())
+        return false;
+    if (end && t > new Date(end).getTime() + 86_400_000)
+        return false; // end-of-day
+    return true;
+}
+function matchAll(text, query) {
+    const out = [];
+    const hay = text.toLowerCase();
+    const needle = query.toLowerCase();
+    let from = 0;
+    while (true) {
+        const i = hay.indexOf(needle, from);
+        if (i === -1)
+            break;
+        out.push({ index: i, length: query.length });
+        from = i + query.length;
+    }
+    return out;
+}
+function buildResult(base, text, index, length) {
+    return {
+        ...base,
+        matchedText: text.slice(index, index + length),
+        contextBefore: text.slice(Math.max(0, index - CONTEXT), index),
+        contextAfter: text.slice(index + length, index + length + CONTEXT),
+    };
+}
+// --- Claude Code ------------------------------------------------------------
+async function readClaudeCwd(filePath) {
+    try {
+        const rl = createInterface({
+            input: createReadStream(filePath, { encoding: "utf-8" }),
+        });
+        for await (const line of rl) {
+            try {
+                const data = JSON.parse(line);
+                if (typeof data.cwd === "string") {
+                    rl.close();
+                    return data.cwd;
+                }
+            }
+            catch {
+                // skip
+            }
+        }
+    }
+    catch {
+        // unreadable
+    }
+    return null;
+}
+async function resolveClaudeProjectPath(dir) {
+    // sessions-index.json → first jsonl cwd → lossy dir-name conversion.
+    try {
+        const idx = JSON.parse(await readFile(join(dir, "sessions-index.json"), "utf-8"));
+        if (idx.originalPath)
+            return idx.originalPath;
+        if (idx.entries?.[0]?.projectPath)
+            return idx.entries[0].projectPath;
+    }
+    catch {
+        // none
+    }
+    try {
+        const files = await readdir(dir);
+        const f = files.find((x) => UUID_JSONL_RE.test(x));
+        if (f) {
+            const cwd = await readClaudeCwd(join(dir, f));
+            if (cwd)
+                return cwd;
+        }
+    }
+    catch {
+        // none
+    }
+    return basename(dir).replace(/-/g, "/");
+}
+async function searchClaudeCode(query, opts) {
+    const results = [];
+    let dirs;
+    try {
+        dirs = await readdir(claudeProjectsRoot);
+    }
+    catch {
+        return results;
+    }
+    for (const dirName of dirs) {
+        const dir = resolve(claudeProjectsRoot, dirName);
+        const ds = await stat(dir).catch(() => null);
+        if (!ds?.isDirectory())
+            continue;
+        const projectPath = await resolveClaudeProjectPath(dir);
+        if (opts.projectPath && !projectPath.includes(opts.projectPath))
+            continue;
+        const files = (await readdir(dir).catch(() => [])).filter((f) => UUID_JSONL_RE.test(f));
+        for (const file of files) {
+            const p = resolve(dir, file);
+            const fs = await stat(p).catch(() => null);
+            if (!fs || fs.size > MAX_LOG_FILE_SIZE)
+                continue;
+            const sessionId = file.replace(".jsonl", "");
+            let content;
+            try {
+                content = await readFile(p, "utf-8");
+            }
+            catch {
+                continue;
+            }
+            for (const line of content.split("\n")) {
+                if (!line.trim())
+                    continue;
+                const msg = parseClaudeCodeLine(line);
+                if (!msg)
+                    continue;
+                if (!inDateRange(msg.timestamp, opts.startDate, opts.endDate))
+                    continue;
+                for (const m of matchAll(msg.text, query)) {
+                    results.push(buildResult({
+                        source: "claude-code",
+                        projectPath,
+                        sessionId,
+                        timestamp: msg.timestamp,
+                        role: msg.role,
+                    }, msg.text, m.index, m.length));
+                }
+            }
+        }
+    }
+    return results;
+}
+// --- Codex ------------------------------------------------------------------
+async function listCodexFiles(root) {
+    const out = [];
+    const stack = [root];
+    while (stack.length) {
+        const cur = stack.pop();
+        let entries;
+        try {
+            entries = await readdir(cur, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const e of entries) {
+            const full = join(cur, e.name);
+            if (e.isDirectory())
+                stack.push(full);
+            else if (e.isFile() && e.name.endsWith(".jsonl"))
+                out.push(full);
+        }
+    }
+    return out;
+}
+async function searchCodex(query, opts) {
+    const results = [];
+    const files = await listCodexFiles(codexSessionsRoot);
+    for (const p of files) {
+        const fs = await stat(p).catch(() => null);
+        if (!fs || fs.size > MAX_LOG_FILE_SIZE)
+            continue;
+        let content;
+        try {
+            content = await readFile(p, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        const { cwd, messages } = parseCodexSession(content);
+        const projectPath = cwd ?? basename(p);
+        if (opts.projectPath && !projectPath.includes(opts.projectPath))
+            continue;
+        // sessionUuid is embedded in the rollout filename: rollout-<ts>-<uuid>.jsonl
+        const m = basename(p).match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i);
+        const sessionId = m ? m[1] : basename(p).replace(".jsonl", "");
+        for (const msg of messages) {
+            if (!inDateRange(msg.timestamp, opts.startDate, opts.endDate))
+                continue;
+            for (const hit of matchAll(msg.text, query)) {
+                results.push(buildResult({
+                    source: "codex",
+                    projectPath,
+                    sessionId,
+                    timestamp: msg.timestamp,
+                    role: msg.role,
+                }, msg.text, hit.index, hit.length));
+            }
+        }
+    }
+    return results;
+}

package/dist/core/memory.js

@@ -0,0 +1,80 @@
+// src/core/memory.ts
+import { randomUUID } from "node:crypto";
+import { sqlite } from "./db.js";
+/**
+ * Canonical preference keys. The distill prompt is instructed to choose from
+ * this set, and setPreference() normalizes incoming keys toward it so synonyms
+ * (e.g. "preferred_language" / "言語") don't create duplicate rows.
+ */
+export const CANONICAL_PREFERENCE_KEYS = [
+    "language",
+    "response_style",
+    "detail_level",
+    "code_style",
+    "framework",
+    "tone",
+    "tools",
+];
+const KEY_ALIASES = {
+    preferred_language: "language",
+    lang: "language",
+    言語: "language",
+    言語設定: "language",
+    response_format: "response_style",
+    reply_style: "response_style",
+    回答スタイル: "response_style",
+    応答スタイル: "response_style",
+    verbosity: "detail_level",
+    detail: "detail_level",
+    詳細度: "detail_level",
+    coding_style: "code_style",
+    コードスタイル: "code_style",
+    frameworks: "framework",
+    フレームワーク: "framework",
+    口調: "tone",
+    tooling: "tools",
+    ツール: "tools",
+};
+/** Map known aliases to a canonical key; otherwise lower/snake-case (kept, not dropped). */
+export function normalizePreferenceKey(key) {
+    const trimmed = key.trim();
+    const lower = trimmed.toLowerCase();
+    if (KEY_ALIASES[trimmed])
+        return KEY_ALIASES[trimmed];
+    if (KEY_ALIASES[lower])
+        return KEY_ALIASES[lower];
+    if (CANONICAL_PREFERENCE_KEYS.includes(lower)) {
+        return lower;
+    }
+    return lower.replace(/[\s-]+/g, "_");
+}
+export function addSessionSummary(projectId, sessionId, summary) {
+    // Upsert by session: delete existing then insert.
+    sqlite
+        .prepare("DELETE FROM session_summaries WHERE session_id = ?")
+        .run(sessionId);
+    sqlite
+        .prepare("INSERT INTO session_summaries(id, project_id, session_id, summary, created_at) VALUES (?, ?, ?, ?, ?)")
+        .run(randomUUID(), projectId, sessionId, summary, new Date().toISOString());
+}
+export function getRecentSummaries(projectId, limit = 5) {
+    return sqlite
+        .prepare("SELECT * FROM session_summaries WHERE project_id = ? ORDER BY created_at DESC LIMIT ?")
+        .all(projectId, limit);
+}
+export function getPreferences(projectId) {
+    return sqlite
+        .prepare("SELECT * FROM user_preferences WHERE project_id = ?")
+        .all(projectId);
+}
+export function setPreference(projectId, key, value) {
+    key = normalizePreferenceKey(key);
+    if (projectId) {
+        sqlite
+            .prepare("DELETE FROM user_preferences WHERE project_id = ? AND key = ?")
+            .run(projectId, key);
+    }
+    sqlite
+        .prepare("INSERT INTO user_preferences(id, project_id, key, value, updated_at) VALUES (?, ?, ?, ?, ?)")
+        .run(randomUUID(), projectId, key, value, new Date().toISOString());
+}

package/dist/core/paths.js

@@ -0,0 +1,13 @@
+// src/core/paths.ts
+import path from "node:path";
+import os from "node:os";
+import fs from "node:fs";
+/**
+ * Root data directory for claw-memory. Independent of agent-claw's data/chat.db.
+ * Override with CLAW_MEMORY_DIR.
+ */
+export const dataDir = process.env.CLAW_MEMORY_DIR || path.join(os.homedir(), ".claw-memory");
+if (!fs.existsSync(dataDir)) {
+    fs.mkdirSync(dataDir, { recursive: true });
+}
+export const dbPath = path.join(dataDir, "memory.db");

package/dist/core/private.js

@@ -0,0 +1,8 @@
+// src/core/private.ts
+//
+// Remove <private>…</private> spans from text before it is persisted or sent to
+// an LLM. Case-insensitive, multiline, non-greedy.
+const PRIVATE_RE = /<private>[\s\S]*?<\/private>/gi;
+export function stripPrivate(text) {
+    return text.replace(PRIVATE_RE, "").trim();
+}

package/dist/core/projects.js

@@ -0,0 +1,46 @@
+// src/core/projects.ts
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { sqlite } from "./db.js";
+export function getProjectByPath(p) {
+    return sqlite
+        .prepare("SELECT * FROM projects WHERE path = ?")
+        .get(p);
+}
+export function getProject(id) {
+    return sqlite
+        .prepare("SELECT * FROM projects WHERE id = ?")
+        .get(id);
+}
+export function listProjects() {
+    return sqlite
+        .prepare("SELECT * FROM projects ORDER BY last_accessed_at DESC")
+        .all();
+}
+/**
+ * Resolve a working directory to a stable project record, creating it on first
+ * sight. Paths are normalized with path.resolve so symlink/trailing-slash
+ * variants don't fork into separate projects.
+ */
+export function getOrCreateProjectByPath(cwd) {
+    const normalized = path.resolve(cwd);
+    const existing = getProjectByPath(normalized);
+    const now = new Date().toISOString();
+    if (existing) {
+        sqlite
+            .prepare("UPDATE projects SET last_accessed_at = ? WHERE id = ?")
+            .run(now, existing.id);
+        return existing;
+    }
+    const row = {
+        id: randomUUID(),
+        name: path.basename(normalized) || normalized,
+        path: normalized,
+        created_at: now,
+        last_accessed_at: now,
+    };
+    sqlite
+        .prepare("INSERT INTO projects(id, name, path, created_at, last_accessed_at) VALUES (?, ?, ?, ?, ?)")
+        .run(row.id, row.name, row.path, row.created_at, row.last_accessed_at);
+    return row;
+}

package/dist/core/providers.js

@@ -0,0 +1,17 @@
+// src/core/providers.ts
+//
+// SDK environment helper for the Claude Agent SDK backend. Model selection and
+// multi-backend routing now live in llm.ts (see complete()/modelForTier()).
+// Default: Claude with the CLI's stored credentials. Override model with
+// AGENT_SDK_MODEL; for non-default endpoints set ANTHROPIC_BASE_URL /
+// ANTHROPIC_AUTH_TOKEN / ANTHROPIC_API_KEY in the environment.
+/** Inherit process.env minus Claude Code's own injected vars (avoid recursion). */
+export function buildFullSdkEnv() {
+    const env = {};
+    for (const [k, v] of Object.entries(process.env)) {
+        if (v !== undefined && !k.startsWith("CLAUDE_CODE_") && k !== "CLAUDECODE") {
+            env[k] = v;
+        }
+    }
+    return env;
+}

package/dist/core/recall.js

@@ -0,0 +1,59 @@
+// src/core/recall.ts
+//
+// Build the formatted memory block injected into an agent's context. Mirrors
+// agent-claw's buildSystemPrompt memory section: preferences (always-apply) +
+// summaries & similar conversations (reference-only).
+import { embedQuery } from "./embeddings.js";
+import { searchSimilar } from "./vector-memory.js";
+import { getPreferences, getRecentSummaries } from "./memory.js";
+const MEMORY_MAX_DISTANCE = Number(process.env.MEMORY_SIMILARITY_MAX_DISTANCE ?? 0.4);
+export async function buildMemoryBlock(projectId, query, topK = 5) {
+    const prefs = getPreferences(projectId);
+    const summaries = getRecentSummaries(projectId, 5);
+    let similar = [];
+    if (query.trim()) {
+        try {
+            const emb = await embedQuery(query);
+            similar = searchSimilar(emb, projectId, topK, MEMORY_MAX_DISTANCE);
+        }
+        catch (err) {
+            console.error("[claw-memory] semantic search failed:", err);
+        }
+    }
+    let text = "";
+    if (prefs.length > 0) {
+        text += '<user-preferences instruction="always-apply">\n';
+        text += "以下のユーザー設定は常に従ってください。\n";
+        for (const p of prefs)
+            text += `- ${p.key}: ${p.value}\n`;
+        text += "</user-preferences>\n";
+    }
+    if (summaries.length > 0 || similar.length > 0) {
+        text += '\n<memory-context instruction="reference-only">\n';
+        text +=
+            "以下は過去の会話から得られた参考情報です。\n" +
+                "背景知識として参照する程度に留め、自発的に言及しないでください。\n" +
+                "ユーザーが明示的に過去の話題に触れた場合にのみ活用してください。\n";
+        if (summaries.length > 0) {
+            text += "\n<previous-session-summaries>\n";
+            for (const s of summaries)
+                text += `- ${s.summary}\n`;
+            text += "</previous-session-summaries>\n";
+        }
+        if (similar.length > 0) {
+            text += "\n<relevant-past-conversations>\n";
+            for (const c of similar) {
+                const date = c.createdAt.split("T")[0];
+                text += `- [${date}] User: ${c.userText}\n  Assistant: ${c.assistantText}\n`;
+            }
+            text += "</relevant-past-conversations>\n";
+        }
+        text += "</memory-context>\n";
+    }
+    return {
+        fullText: text.trim(),
+        preferences: prefs.map((p) => ({ key: p.key, value: p.value })),
+        summaries: summaries.map((s) => s.summary),
+        similar,
+    };
+}

package/dist/core/search.js

@@ -0,0 +1,45 @@
+// src/core/search.ts
+//
+// Token-efficient layered search (claude-mem-style). search() returns a light
+// index of ids+titles; callers fetch full bodies with getChunksByIds() only for
+// the entries they care about.
+import { embedQuery } from "./embeddings.js";
+import { searchSimilar, searchKeyword, } from "./vector-memory.js";
+const MAX_DISTANCE = Number(process.env.MEMORY_SIMILARITY_MAX_DISTANCE ?? 0.6);
+/**
+ * Hybrid search: semantic (vector) first, augmented with FTS5 keyword hits so
+ * exact-token queries that embeddings miss still surface. De-duplicated by id.
+ * Optional metadata filters (type / concept / file / date) narrow both passes.
+ */
+export async function searchIndex(projectId, query, limit = 8, filter) {
+    const hits = new Map();
+    if (query.trim()) {
+        try {
+            const emb = await embedQuery(query);
+            for (const c of searchSimilar(emb, projectId, limit, MAX_DISTANCE, filter)) {
+                hits.set(c.id, toHit(c, "semantic", c.distance));
+            }
+        }
+        catch (err) {
+            console.error("[claw-memory] semantic search failed:", err);
+        }
+        for (const c of searchKeyword(query, projectId, limit, filter)) {
+            if (!hits.has(c.id))
+                hits.set(c.id, toHit(c, "keyword"));
+        }
+    }
+    return [...hits.values()]
+        .sort((a, b) => (a.distance ?? 1) - (b.distance ?? 1))
+        .slice(0, limit);
+}
+function toHit(c, source, distance) {
+    const title = c.userText.replace(/\s+/g, " ").slice(0, 80);
+    return {
+        id: c.id,
+        title,
+        date: c.createdAt.split("T")[0],
+        source,
+        obsType: c.obsType,
+        distance,
+    };
+}

package/dist/core/transcript.js

@@ -0,0 +1,79 @@
+// src/core/transcript.ts
+//
+// Load a conversation transcript. Supports an explicit .jsonl path or a Claude
+// Code session id resolved under ~/.claude/projects/<encoded-cwd>/<id>.jsonl.
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { parseCodexSession } from "./logsearch/parse.js";
+const CLAUDE_PROJECTS_DIR = path.join(os.homedir(), ".claude", "projects");
+export function encodeCwdToDir(cwd) {
+    return path.resolve(cwd).replace(/\//g, "-");
+}
+export function resolveSessionJsonl(cwd, sessionId) {
+    return path.join(CLAUDE_PROJECTS_DIR, encodeCwdToDir(cwd), `${sessionId}.jsonl`);
+}
+/**
+ * Parse a transcript into ordered user/assistant text messages. Auto-detects
+ * the format: Codex rollout logs (first line is `session_meta`, or path under
+ * ~/.codex) are delegated to the Codex parser; otherwise Claude Code JSONL.
+ */
+export function loadTranscript(filePath) {
+    if (!fs.existsSync(filePath))
+        return [];
+    const raw = fs.readFileSync(filePath, "utf-8");
+    if (isCodexTranscript(filePath, raw)) {
+        const { messages } = parseCodexSession(raw);
+        return messages.map((m) => ({ role: m.role, text: m.text }));
+    }
+    const out = [];
+    for (const line of raw.split("\n")) {
+        if (!line.trim())
+            continue;
+        let entry;
+        try {
+            entry = JSON.parse(line);
+        }
+        catch {
+            continue;
+        }
+        if (entry.parent_tool_use_id)
+            continue;
+        const role = entry?.message?.role;
+        if (entry.type !== "user" && entry.type !== "assistant")
+            continue;
+        if (role !== "user" && role !== "assistant")
+            continue;
+        const content = entry.message.content;
+        let text = "";
+        if (typeof content === "string") {
+            text = content;
+        }
+        else if (Array.isArray(content)) {
+            // Skip tool_result-only user turns.
+            if (content.some((b) => b?.type === "tool_result"))
+                continue;
+            text = content
+                .filter((b) => b?.type === "text")
+                .map((b) => b.text)
+                .join("");
+        }
+        if (text.trim())
+            out.push({ role, text });
+    }
+    return out;
+}
+/** Heuristic: a Codex rollout log lives under ~/.codex or starts with session_meta. */
+function isCodexTranscript(filePath, raw) {
+    if (filePath.includes(`${path.sep}.codex${path.sep}`))
+        return true;
+    const firstLine = raw.slice(0, raw.indexOf("\n") + 1 || undefined).trim();
+    if (!firstLine)
+        return false;
+    try {
+        return JSON.parse(firstLine).type === "session_meta";
+    }
+    catch {
+        return false;
+    }
+}