@nogataka/claw-memory 0.1.3 → 0.2.0

package/dist/core/lessons.js

@@ -0,0 +1,335 @@
+// src/core/lessons.ts
+//
+// Lesson store: reusable, abstracted knowledge distilled from sessions. Mirrors
+// vector-memory.ts (vec0 embedding + readable metadata row + FTS5 keyword row),
+// but adds lifecycle (candidate → approved/rejected/archived/superseded), an
+// event audit trail, and inter-lesson links (duplicate / conflict / supersede).
+//
+// JSON-array fields (applies_when / avoid_when / concepts / files /
+// source_chunk_ids) are stored as TEXT, parsed on read — same convention as
+// conversation_chunks.concepts.
+import { randomUUID } from "node:crypto";
+import { sqlite } from "./db.js";
+function jsonArr(v) {
+    return v && v.length ? JSON.stringify(v) : null;
+}
+function parseArr(v) {
+    if (typeof v !== "string" || !v)
+        return [];
+    try {
+        const a = JSON.parse(v);
+        return Array.isArray(a) ? a.map(String) : [];
+    }
+    catch {
+        return [];
+    }
+}
+const LESSON_COLS = `l.id, l.project_id, l.repo_id, l.session_id, l.title, l.lesson,
+  l.applies_when, l.avoid_when, l.evidence, l.scope, l.obs_type, l.concepts, l.files,
+  l.confidence, l.source_chunk_ids, l.status, l.superseded_by, l.valid_until,
+  l.last_used_at, l.created_at, l.updated_at`;
+function mapRow(r) {
+    return {
+        id: r.id,
+        projectId: r.project_id,
+        repoId: r.repo_id,
+        sessionId: r.session_id,
+        title: r.title,
+        lesson: r.lesson,
+        appliesWhen: parseArr(r.applies_when),
+        avoidWhen: parseArr(r.avoid_when),
+        evidence: r.evidence,
+        scope: r.scope,
+        obsType: r.obs_type,
+        concepts: parseArr(r.concepts),
+        files: parseArr(r.files),
+        confidence: r.confidence,
+        sourceChunkIds: parseArr(r.source_chunk_ids),
+        status: r.status,
+        supersededBy: r.superseded_by,
+        validUntil: r.valid_until,
+        lastUsedAt: r.last_used_at,
+        createdAt: r.created_at,
+        updatedAt: r.updated_at,
+        distance: r.distance ?? 0,
+    };
+}
+/** Record a lifecycle event (status change, decay, scope edit, …). */
+export function recordEvent(lessonId, eventType, opts = {}) {
+    sqlite
+        .prepare(`INSERT INTO lesson_events(id, lesson_id, event_type, old_status, new_status, note, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?)`)
+        .run(randomUUID(), lessonId, eventType, opts.oldStatus ?? null, opts.newStatus ?? null, opts.note ?? null, new Date().toISOString());
+}
+/** Save one lesson: vec0 embedding + metadata row + FTS keyword row, atomically. */
+export function saveLesson(input) {
+    const insertVec = sqlite.prepare("INSERT INTO lesson_vec(embedding, project_id, scope) VALUES (?, ?, ?)");
+    const insertMeta = sqlite.prepare(`INSERT INTO lessons(
+        id, vec_rowid, project_id, repo_id, session_id, title, lesson,
+        applies_when, avoid_when, evidence, scope, obs_type, concepts, files,
+        confidence, source_chunk_ids, status, created_at, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+    const insertFts = sqlite.prepare("INSERT INTO lessons_fts(lesson_id, text) VALUES (?, ?)");
+    const id = randomUUID();
+    const now = new Date().toISOString();
+    const scope = input.scope ?? "repo";
+    const status = input.status ?? "candidate";
+    const confidence = input.confidence ?? 0.5;
+    const tx = sqlite.transaction(() => {
+        const res = insertVec.run(Buffer.from(input.embedding.buffer), input.projectId ?? null, scope);
+        const vecRowid = res.lastInsertRowid;
+        insertMeta.run(id, vecRowid, input.projectId ?? null, input.repoId ?? null, input.sessionId ?? null, input.title, input.lesson, jsonArr(input.appliesWhen), jsonArr(input.avoidWhen), input.evidence ?? null, scope, input.obsType ?? null, jsonArr(input.concepts), jsonArr(input.files), confidence, jsonArr(input.sourceChunkIds), status, now, now);
+        insertFts.run(id, `${input.title}\n${input.lesson}`);
+        recordEvent(id, "created", { newStatus: status });
+    });
+    tx();
+    return id;
+}
+export function getLesson(id) {
+    const row = sqlite
+        .prepare(`SELECT ${LESSON_COLS} FROM lessons l WHERE l.id = ?`)
+        .get(id);
+    return row ? mapRow(row) : null;
+}
+export function getLessonsByIds(ids) {
+    if (ids.length === 0)
+        return [];
+    const placeholders = ids.map(() => "?").join(",");
+    const rows = sqlite
+        .prepare(`SELECT ${LESSON_COLS} FROM lessons l WHERE l.id IN (${placeholders})`)
+        .all(...ids);
+    return rows.map(mapRow);
+}
+/** Build a WHERE fragment from a LessonFilter. */
+function lessonWhere(filter) {
+    const clauses = [];
+    const params = [];
+    if (filter?.status) {
+        clauses.push("l.status = ?");
+        params.push(filter.status);
+    }
+    if (filter?.scope) {
+        clauses.push("l.scope = ?");
+        params.push(filter.scope);
+    }
+    if (filter?.projectId) {
+        clauses.push("l.project_id = ?");
+        params.push(filter.projectId);
+    }
+    if (filter?.concept) {
+        clauses.push("l.concepts LIKE ?");
+        params.push(`%${filter.concept}%`);
+    }
+    if (filter?.file) {
+        clauses.push("l.files LIKE ?");
+        params.push(`%${filter.file}%`);
+    }
+    return { sql: clauses.length ? clauses.join(" AND ") : "1=1", params };
+}
+export function listLessons(filter, limit = 200) {
+    const w = lessonWhere(filter);
+    const rows = sqlite
+        .prepare(`SELECT ${LESSON_COLS} FROM lessons l
+       WHERE ${w.sql}
+       ORDER BY l.updated_at DESC LIMIT ?`)
+        .all(...w.params, limit);
+    return rows.map(mapRow);
+}
+export function getLessonCount(filter) {
+    const w = lessonWhere(filter);
+    const row = sqlite
+        .prepare(`SELECT COUNT(*) c FROM lessons l WHERE ${w.sql}`)
+        .get(...w.params);
+    return row.c;
+}
+/** Lessons involved in a conflicts_with link (for Conflict Review). */
+export function listConflicts(projectId, limit = 200) {
+    const proj = projectId ? "AND l.project_id = ?" : "";
+    const rows = sqlite
+        .prepare(`SELECT DISTINCT ${LESSON_COLS} FROM lessons l
+       INNER JOIN lesson_links k
+         ON (k.lesson_id = l.id OR k.linked_lesson_id = l.id)
+       WHERE k.relation = 'conflicts_with' AND l.status != 'rejected' ${proj}
+       ORDER BY l.updated_at DESC LIMIT ?`)
+        .all(...(projectId ? [projectId] : []), limit);
+    return rows.map(mapRow);
+}
+export function getConflictCount(projectId) {
+    const proj = projectId ? "AND l.project_id = ?" : "";
+    const row = sqlite
+        .prepare(`SELECT COUNT(DISTINCT l.id) c FROM lessons l
+       INNER JOIN lesson_links k
+         ON (k.lesson_id = l.id OR k.linked_lesson_id = l.id)
+       WHERE k.relation = 'conflicts_with' AND l.status != 'rejected' ${proj}`)
+        .get(...(projectId ? [projectId] : []));
+    return row.c;
+}
+/** KNN semantic search over lessons, filtered by project/scope inside MATCH. */
+export function searchSimilarLessons(queryEmbedding, opts = {}) {
+    const topK = opts.topK ?? 8;
+    // vec0 MATCH metadata filters: project must match OR be a cross-project scope.
+    const stmt = sqlite.prepare(`
+    SELECT ${LESSON_COLS}, v.distance
+    FROM (
+      SELECT rowid, distance FROM lesson_vec
+      WHERE embedding MATCH ? AND k = ?
+    ) v
+    INNER JOIN lessons l ON l.vec_rowid = v.rowid
+    WHERE ${opts.status ? "l.status = ?" : "1=1"}
+      ${opts.maxDistance != null ? "AND v.distance <= ?" : ""}
+    ORDER BY v.distance
+  `);
+    const params = [
+        Buffer.from(queryEmbedding.buffer),
+        topK,
+    ];
+    if (opts.status)
+        params.push(opts.status);
+    if (opts.maxDistance != null)
+        params.push(opts.maxDistance);
+    const rows = stmt.all(...params);
+    return rows.map(mapRow);
+}
+/** FTS5 keyword search over lesson title + body. */
+export function searchKeywordLessons(query, opts = {}) {
+    const match = toFtsQuery(query);
+    if (!match)
+        return [];
+    const rows = sqlite
+        .prepare(`SELECT ${LESSON_COLS}
+       FROM lessons_fts f
+       INNER JOIN lessons l ON l.id = f.lesson_id
+       WHERE f.text MATCH ? ${opts.status ? "AND l.status = ?" : ""}
+       LIMIT ?`)
+        .all(match, ...(opts.status ? [opts.status] : []), opts.topK ?? 8);
+    return rows.map(mapRow);
+}
+/** Transition a lesson's status, stamping updated_at and recording the event. */
+export function setStatus(id, newStatus, note) {
+    const current = sqlite
+        .prepare("SELECT status FROM lessons WHERE id = ?")
+        .get(id);
+    if (!current)
+        return false;
+    const now = new Date().toISOString();
+    sqlite
+        .prepare("UPDATE lessons SET status = ?, updated_at = ? WHERE id = ?")
+        .run(newStatus, now, id);
+    recordEvent(id, "status_change", {
+        oldStatus: current.status,
+        newStatus,
+        note,
+    });
+    return true;
+}
+/** Mark `oldId` superseded by `newId` and link them. */
+export function supersede(oldId, newId) {
+    const current = sqlite
+        .prepare("SELECT status FROM lessons WHERE id = ?")
+        .get(oldId);
+    if (!current)
+        return false;
+    const now = new Date().toISOString();
+    const tx = sqlite.transaction(() => {
+        sqlite
+            .prepare("UPDATE lessons SET status = 'superseded', superseded_by = ?, updated_at = ? WHERE id = ?")
+            .run(newId, now, oldId);
+        recordEvent(oldId, "superseded", {
+            oldStatus: current.status,
+            newStatus: "superseded",
+            note: `superseded_by=${newId}`,
+        });
+        linkLessons(newId, oldId, "supersedes");
+    });
+    tx();
+    return true;
+}
+/** Update one or more editable fields (scope / confidence / valid_until). */
+export function updateLesson(id, fields) {
+    const sets = [];
+    const params = [];
+    if (fields.scope !== undefined) {
+        sets.push("scope = ?");
+        params.push(fields.scope);
+    }
+    if (fields.confidence !== undefined) {
+        sets.push("confidence = ?");
+        params.push(fields.confidence);
+    }
+    if (fields.validUntil !== undefined) {
+        sets.push("valid_until = ?");
+        params.push(fields.validUntil);
+    }
+    if (sets.length === 0)
+        return false;
+    sets.push("updated_at = ?");
+    params.push(new Date().toISOString());
+    params.push(id);
+    const res = sqlite
+        .prepare(`UPDATE lessons SET ${sets.join(", ")} WHERE id = ?`)
+        .run(...params);
+    if (res.changes > 0)
+        recordEvent(id, "edited", { note: sets.join(",") });
+    return res.changes > 0;
+}
+/** Bump last_used_at when a lesson is surfaced by search/inject. */
+export function markUsed(ids) {
+    if (ids.length === 0)
+        return;
+    const now = new Date().toISOString();
+    const stmt = sqlite.prepare("UPDATE lessons SET last_used_at = ? WHERE id = ?");
+    const tx = sqlite.transaction(() => {
+        for (const id of ids)
+            stmt.run(now, id);
+    });
+    tx();
+}
+export function linkLessons(lessonId, linkedLessonId, relation) {
+    sqlite
+        .prepare(`INSERT INTO lesson_links(id, lesson_id, linked_lesson_id, relation, created_at)
+       VALUES (?, ?, ?, ?, ?)`)
+        .run(randomUUID(), lessonId, linkedLessonId, relation, new Date().toISOString());
+}
+export function getEvents(lessonId) {
+    const rows = sqlite
+        .prepare(`SELECT id, event_type, old_status, new_status, note, created_at
+       FROM lesson_events WHERE lesson_id = ? ORDER BY created_at ASC`)
+        .all(lessonId);
+    return rows.map((r) => ({
+        id: r.id,
+        eventType: r.event_type,
+        oldStatus: r.old_status,
+        newStatus: r.new_status,
+        note: r.note,
+        createdAt: r.created_at,
+    }));
+}
+/** Links where this lesson is either side (incoming + outgoing). */
+export function getLinks(lessonId) {
+    const rows = sqlite
+        .prepare(`SELECT id, lesson_id, linked_lesson_id, relation, created_at
+       FROM lesson_links WHERE lesson_id = ? OR linked_lesson_id = ?
+       ORDER BY created_at ASC`)
+        .all(lessonId, lessonId);
+    return rows.map((r) => ({
+        id: r.id,
+        lessonId: r.lesson_id,
+        linkedLessonId: r.linked_lesson_id,
+        relation: r.relation,
+        createdAt: r.created_at,
+    }));
+}
+/**
+ * Turn a free-text query into a safe FTS5 OR-query of quoted terms (trigram
+ * tokenizer needs >=3-char terms). Mirrors vector-memory.toFtsQuery.
+ */
+function toFtsQuery(query) {
+    const terms = query
+        .replace(/["()*:^]/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length >= 3)
+        .map((t) => `"${t.replace(/"/g, "")}"`);
+    if (terms.length === 0)
+        return null;
+    return terms.join(" OR ");
+}

package/dist/core/logsearch/paths.js

@@ -6,6 +6,28 @@ import os from "node:os";
 import path from "node:path";
 export const claudeProjectsRoot = path.join(os.homedir(), ".claude", "projects");
 export const codexSessionsRoot = path.join(os.homedir(), ".codex", "sessions");
+/**
+ * Where to look for ChatGPT web "Export data" bundles (`conversations.json`).
+ * Unlike Claude Code / Codex, ChatGPT web conversations are NOT stored locally —
+ * the user downloads the official export and drops the file(s) here. Override
+ * `CLAW_MEMORY_CHATGPT_EXPORT` with either a single file or a directory.
+ */
+export const chatgptExportRoot = process.env.CLAW_MEMORY_CHATGPT_EXPORT ||
+    path.join(os.homedir(), ".claw-memory", "chatgpt");
+/**
+ * Stable synthetic "project" that distilled ChatGPT conversations are grouped
+ * under. Deliberately independent of where the export file lives, so moving the
+ * export doesn't fork a new project and the viewer always shows one tidy
+ * "chatgpt" project. ChatGPT conversations have no repo/cwd of their own.
+ */
+export const chatgptProjectPath = path.join(os.homedir(), ".claw-memory", "chatgpt");
 /** Max transcript size to scan; larger files are skipped (cc-search parity). */
 export const MAX_LOG_FILE_SIZE = 10 * 1024 * 1024;
+/**
+ * ChatGPT exports are a single JSON file holding the whole account history, so
+ * they can be much larger than a per-session transcript. JSON.parse reads the
+ * whole file into memory; this cap (default 200 MB) bounds that. Override with
+ * `CLAW_MEMORY_CHATGPT_MAX_BYTES`.
+ */
+export const MAX_CHATGPT_FILE_SIZE = Number(process.env.CLAW_MEMORY_CHATGPT_MAX_BYTES ?? 200 * 1024 * 1024);
 export const UUID_JSONL_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.jsonl$/i;

package/dist/core/logsearch/recent.js

@@ -2,9 +2,10 @@
 //
 // Enumerate Codex rollout transcripts newest-first, for batch distillation
 // (claw-memory distill-codex). Read-only over ~/.codex/sessions.
-import { readdir, stat } from "node:fs/promises";
+import { readdir, stat, readFile } from "node:fs/promises";
 import { join } from "node:path";
-import { codexSessionsRoot } from "./paths.js";
+import { codexSessionsRoot, chatgptExportRoot, MAX_CHATGPT_FILE_SIZE } from "./paths.js";
+import { parseChatgptExport } from "./parse.js";
 /** All Codex `*.jsonl` rollout files under ~/.codex/sessions, newest first. */
 export async function listCodexSessionFiles() {
     const out = [];
@@ -38,3 +39,46 @@ export function codexSessionId(filePath) {
     const m = filePath.match(/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i);
     return m ? m[1] : filePath;
 }
+/** List the ChatGPT export `*.json` files (export root is a file or directory). */
+async function listChatgptExportFiles() {
+    const s = await stat(chatgptExportRoot).catch(() => null);
+    if (!s)
+        return [];
+    if (s.isFile())
+        return [chatgptExportRoot];
+    if (!s.isDirectory())
+        return [];
+    const entries = await readdir(chatgptExportRoot).catch(() => []);
+    return entries
+        .filter((f) => f.toLowerCase().endsWith(".json"))
+        .map((f) => join(chatgptExportRoot, f));
+}
+/**
+ * Load all ChatGPT conversations from the export bundle(s), newest write first.
+ * Used by `distill-chatgpt` to feed each conversation into the distill pipeline.
+ */
+export async function loadChatgptConversations() {
+    const files = await listChatgptExportFiles();
+    const out = [];
+    for (const p of files) {
+        const s = await stat(p).catch(() => null);
+        if (!s || s.size > MAX_CHATGPT_FILE_SIZE)
+            continue;
+        let content;
+        try {
+            content = await readFile(p, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        out.push(...parseChatgptExport(content));
+    }
+    out.sort((a, b) => tsValue(b.updateTime) - tsValue(a.updateTime));
+    return out;
+}
+function tsValue(ts) {
+    if (!ts)
+        return 0;
+    const n = new Date(ts).getTime();
+    return Number.isNaN(n) ? 0 : n;
+}

package/dist/core/logsearch/search.js

@@ -8,14 +8,14 @@ import { readdir, readFile, stat } from "node:fs/promises";
 import { resolve, join, basename } from "node:path";
 import { createInterface } from "node:readline";
 import { createReadStream } from "node:fs";
-import { claudeProjectsRoot, codexSessionsRoot, MAX_LOG_FILE_SIZE, UUID_JSONL_RE, } from "./paths.js";
-import { parseClaudeCodeLine, parseCodexSession } from "./parse.js";
+import { claudeProjectsRoot, codexSessionsRoot, chatgptExportRoot, MAX_LOG_FILE_SIZE, MAX_CHATGPT_FILE_SIZE, UUID_JSONL_RE, } from "./paths.js";
+import { parseClaudeCodeLine, parseCodexSession, parseChatgptExport, } from "./parse.js";
 const CONTEXT = 100;
 export async function searchLogs(opts) {
     const query = opts.query.trim();
     if (!query)
         return { results: [], total: 0 };
-    const sources = opts.sources ?? ["claude-code", "codex"];
+    const sources = opts.sources ?? ["claude-code", "codex", "chatgpt-web"];
     const limit = opts.limit ?? 20;
     const offset = opts.offset ?? 0;
     const all = [];
@@ -25,6 +25,9 @@ export async function searchLogs(opts) {
     if (sources.includes("codex")) {
         all.push(...(await searchCodex(query, opts)));
     }
+    if (sources.includes("chatgpt-web")) {
+        all.push(...(await searchChatgpt(query, opts)));
+    }
     all.sort((a, b) => tsValue(b.timestamp) - tsValue(a.timestamp));
     return { results: all.slice(offset, offset + limit), total: all.length };
 }
@@ -231,3 +234,54 @@ async function searchCodex(query, opts) {
     }
     return results;
 }
+// --- ChatGPT web (official data export) -------------------------------------
+/** Resolve the export root to a list of `*.json` files (file or directory). */
+async function listChatgptFiles(root) {
+    const s = await stat(root).catch(() => null);
+    if (!s)
+        return [];
+    if (s.isFile())
+        return [root];
+    if (!s.isDirectory())
+        return [];
+    const entries = await readdir(root).catch(() => []);
+    return entries
+        .filter((f) => f.toLowerCase().endsWith(".json"))
+        .map((f) => join(root, f));
+}
+async function searchChatgpt(query, opts) {
+    const results = [];
+    const files = await listChatgptFiles(chatgptExportRoot);
+    for (const p of files) {
+        const fs = await stat(p).catch(() => null);
+        if (!fs || fs.size > MAX_CHATGPT_FILE_SIZE)
+            continue;
+        let content;
+        try {
+            content = await readFile(p, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        for (const conv of parseChatgptExport(content)) {
+            // ChatGPT has no cwd/project; the conversation title stands in for it.
+            const projectPath = conv.title;
+            if (opts.projectPath && !projectPath.includes(opts.projectPath))
+                continue;
+            for (const msg of conv.messages) {
+                if (!inDateRange(msg.timestamp, opts.startDate, opts.endDate))
+                    continue;
+                for (const hit of matchAll(msg.text, query)) {
+                    results.push(buildResult({
+                        source: "chatgpt-web",
+                        projectPath,
+                        sessionId: conv.conversationId,
+                        timestamp: msg.timestamp,
+                        role: msg.role,
+                    }, msg.text, hit.index, hit.length));
+                }
+            }
+        }
+    }
+    return results;
+}

package/dist/core/recall.js

@@ -6,7 +6,11 @@
 import { embedQuery } from "./embeddings.js";
 import { searchSimilar } from "./vector-memory.js";
 import { getPreferences, getRecentSummaries } from "./memory.js";
+import { searchLessons, formatLessonBlock, } from "./lesson-search.js";
 const MEMORY_MAX_DISTANCE = Number(process.env.MEMORY_SIMILARITY_MAX_DISTANCE ?? 0.4);
+// Approved lessons injected into the recall block. Kept small to avoid context
+// bloat; 0 disables lesson injection entirely.
+const RECALL_LESSON_LIMIT = Number(process.env.LESSON_RECALL_LIMIT ?? 3);
 export async function buildMemoryBlock(projectId, query, topK = 5) {
     const prefs = getPreferences(projectId);
     const summaries = getRecentSummaries(projectId, 5);
@@ -20,6 +24,16 @@ export async function buildMemoryBlock(projectId, query, topK = 5) {
             console.error("[claw-memory] semantic search failed:", err);
         }
     }
+    // Approved, reusable lessons relevant to this request (best-effort).
+    let lessons = [];
+    if (query.trim() && RECALL_LESSON_LIMIT > 0) {
+        try {
+            lessons = await searchLessons(query, { projectId }, { limit: RECALL_LESSON_LIMIT });
+        }
+        catch (err) {
+            console.error("[claw-memory] lesson recall failed:", err);
+        }
+    }
     let text = "";
     if (prefs.length > 0) {
         text += '<user-preferences instruction="always-apply">\n';
@@ -50,10 +64,14 @@ export async function buildMemoryBlock(projectId, query, topK = 5) {
         }
         text += "</memory-context>\n";
     }
+    const lessonBlock = formatLessonBlock(lessons);
+    if (lessonBlock)
+        text += `\n${lessonBlock}\n`;
     return {
         fullText: text.trim(),
         preferences: prefs.map((p) => ({ key: p.key, value: p.value })),
         summaries: summaries.map((s) => s.summary),
         similar,
+        lessons,
     };
 }