@nogataka/claw-memory 0.1.2 → 0.2.0

package/dist/core/lesson-extract.js

@@ -0,0 +1,233 @@
+// src/core/lesson-extract.ts
+//
+// Turn distilled session signal into candidate lessons. Two entry points:
+//
+//  - saveCandidates(): persist already-extracted LessonCandidate[] (the default
+//    path — candidates ride along in distill()'s existing summary JSON, so no
+//    extra LLM call).
+//  - extractDedicated(): a separate, higher-quality extraction pass over the
+//    transcript using the section-12 prompt + the "smart" tier. Opt-in via
+//    CLAW_MEMORY_LESSON_DEDICATED=1.
+//
+// All lessons are saved as status='candidate'; only human/rule approval promotes
+// them to 'approved' (the status that lesson_search / recall actually surface).
+import { execFileSync } from "node:child_process";
+import { complete } from "./llm.js";
+import { embedPassage } from "./embeddings.js";
+import { saveLesson, linkLessons, searchSimilarLessons, } from "./lessons.js";
+import { getProject } from "./projects.js";
+import { log } from "./logger.js";
+// Distance thresholds (cosine, 0..2) for relation classification.
+const DUP_DISTANCE = Number(process.env.LESSON_DUP_DISTANCE ?? 0.08);
+const RELATED_DISTANCE = Number(process.env.LESSON_RELATED_DISTANCE ?? 0.28);
+const VALID_SCOPES = [
+    "global",
+    "project",
+    "repo",
+    "file",
+    "task",
+    "user_preference",
+    "team",
+];
+export function normalizeScope(scope) {
+    const s = (scope ?? "").trim().toLowerCase();
+    return VALID_SCOPES.includes(s) ? s : "repo";
+}
+export function clampConfidence(c) {
+    if (typeof c !== "number" || Number.isNaN(c))
+        return 0.5;
+    return Math.max(0, Math.min(1, c));
+}
+/**
+ * Resolve a stable repo id for a project: the git top-level path, falling back
+ * to the project's own path. Best-effort — git failures are non-fatal. Cached
+ * per project path so repeated distills don't re-spawn git.
+ */
+const repoCache = new Map();
+export function resolveRepoId(projectId) {
+    if (!projectId)
+        return null;
+    const project = getProject(projectId);
+    if (!project)
+        return null;
+    const cached = repoCache.get(project.path);
+    if (cached !== undefined)
+        return cached;
+    let repo = project.path;
+    try {
+        repo = execFileSync("git", ["-C", project.path, "rev-parse", "--show-toplevel"], {
+            encoding: "utf-8",
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim() || project.path;
+    }
+    catch {
+        // not a git repo — repo id is the project path itself
+    }
+    repoCache.set(project.path, repo);
+    return repo;
+}
+/** Persist candidate lessons (embed + saveLesson as 'candidate'). */
+export async function saveCandidates(input) {
+    const repoId = resolveRepoId(input.projectId);
+    const ids = [];
+    for (const c of input.candidates) {
+        const title = (c.title ?? "").trim();
+        const lesson = (c.lesson ?? "").trim();
+        if (!title || !lesson)
+            continue;
+        const embedding = await embedPassage(`${title}\n${lesson}`);
+        const id = saveLesson({
+            projectId: input.projectId ?? null,
+            repoId,
+            sessionId: input.sessionId ?? null,
+            title: title.slice(0, 200),
+            lesson: lesson.slice(0, 2000),
+            appliesWhen: (c.applies_when ?? []).map(String).filter(Boolean),
+            avoidWhen: (c.avoid_when ?? []).map(String).filter(Boolean),
+            evidence: c.evidence ? String(c.evidence).slice(0, 1000) : null,
+            scope: normalizeScope(c.scope),
+            concepts: (c.concepts ?? []).map(String).filter(Boolean),
+            files: (c.files ?? []).map(String).filter(Boolean),
+            confidence: clampConfidence(c.confidence),
+            sourceChunkIds: input.sourceChunkIds ?? [],
+            status: "candidate",
+            embedding,
+        });
+        // Detect duplicates / related / conflicts vs existing lessons (best-effort).
+        try {
+            await detectRelations({ lessonId: id, embedding, candidate: c });
+        }
+        catch (err) {
+            console.error("[claw-memory] lesson relation detection failed:", err);
+        }
+        ids.push(id);
+    }
+    log("lesson-extract", {
+        projectId: input.projectId,
+        sessionId: input.sessionId,
+        saved: ids.length,
+    });
+    return ids;
+}
+function shareAny(a, b) {
+    if (!a.length || !b.length)
+        return false;
+    const set = new Set(a.map((s) => s.toLowerCase()));
+    return b.some((s) => set.has(s.toLowerCase()));
+}
+const CONFLICT_PROMPT = (a, b) => `次の2つのレッスンの関係を判定してください。
+
+Lesson A: ${a}
+Lesson B: ${b}
+
+関係を1つ選び JSON のみで回答:
+{"relation": "duplicate" | "conflict" | "related" | "none"}
+- duplicate: 実質同じ内容
+- conflict: 互いに矛盾する主張
+- related: 関連するが別の知識
+- none: 無関係`;
+function conflictLlmEnabled() {
+    return process.env.CLAW_MEMORY_LESSON_CONFLICT_LLM === "1";
+}
+/**
+ * Compare a freshly-saved lesson against existing lessons and record
+ * lesson_links (duplicate / related_to / conflicts_with). Embedding distance
+ * gives duplicate/related deterministically; an opt-in LLM pass (env
+ * CLAW_MEMORY_LESSON_CONFLICT_LLM=1) adds semantic conflict detection. A
+ * detected conflict leaves the new lesson as 'candidate' for Conflict Review.
+ */
+export async function detectRelations(opts) {
+    const result = { duplicates: 0, related: 0, conflicts: 0 };
+    const neighbors = searchSimilarLessons(opts.embedding, {
+        topK: 6,
+        maxDistance: RELATED_DISTANCE,
+    }).filter((n) => n.id !== opts.lessonId &&
+        n.status !== "rejected" &&
+        n.status !== "superseded");
+    if (neighbors.length === 0)
+        return result;
+    const newConcepts = (opts.candidate.concepts ?? []).map(String);
+    const newFiles = (opts.candidate.files ?? []).map(String);
+    const link = (target, relation) => linkLessons(opts.lessonId, target, relation);
+    // Deterministic: nearest neighbor classification by distance + overlap.
+    let llmDone = false;
+    for (const n of neighbors) {
+        if (n.distance <= DUP_DISTANCE) {
+            link(n.id, "duplicate");
+            result.duplicates++;
+            continue;
+        }
+        const overlaps = shareAny(newConcepts, n.concepts) || shareAny(newFiles, n.files);
+        // Opt-in LLM judgment on the closest overlapping neighbor only (1 call max).
+        if (conflictLlmEnabled() && !llmDone && overlaps) {
+            llmDone = true;
+            try {
+                const text = await complete({
+                    prompt: CONFLICT_PROMPT(`${opts.candidate.title}: ${opts.candidate.lesson}`, `${n.title}: ${n.lesson}`),
+                    tier: "simple",
+                });
+                const m = text.match(/"relation"\s*:\s*"(\w+)"/);
+                const rel = m?.[1];
+                if (rel === "conflict") {
+                    link(n.id, "conflicts_with");
+                    result.conflicts++;
+                    continue;
+                }
+                if (rel === "duplicate") {
+                    link(n.id, "duplicate");
+                    result.duplicates++;
+                    continue;
+                }
+            }
+            catch {
+                // LLM unavailable — fall through to deterministic related link
+            }
+        }
+        if (overlaps) {
+            link(n.id, "related_to");
+            result.related++;
+        }
+    }
+    return result;
+}
+const DEDICATED_PROMPT = (transcript) => `あなたはAIコーディングセッションから、将来再利用可能なレッスンを抽出する役割です。
+
+抽出するのは、次回以降の作業で役立つ可能性が高い教訓だけです。
+
+抽出しないもの:
+- 一時的な会話
+- 検証されていない推測
+- 秘密情報（APIキー・トークン・個人情報）
+- 生ログの丸写し
+- 再利用性の低い一回限りのコマンド
+- 既存レッスンと重複する内容
+
+各レッスンは、実行可能で、具体的で、再利用可能である必要があります。
+該当が無ければ空配列を返してください。
+
+JSON のみ回答:
+{"lessons": [{"title": "...", "lesson": "...", "applies_when": ["..."], "avoid_when": ["..."], "scope": "global|project|repo|file|task|user_preference", "confidence": 0.0, "evidence": "...", "concepts": ["..."], "files": ["..."]}]}
+
+<conversation>
+${transcript}
+</conversation>`;
+/** Dedicated extraction pass (opt-in). Returns parsed candidates. */
+export async function extractDedicated(transcript) {
+    const responseText = await complete({
+        prompt: DEDICATED_PROMPT(transcript),
+        tier: "smart",
+    });
+    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+    if (!jsonMatch)
+        return [];
+    try {
+        const parsed = JSON.parse(jsonMatch[0]);
+        return Array.isArray(parsed.lessons) ? parsed.lessons : [];
+    }
+    catch {
+        return [];
+    }
+}
+export function dedicatedEnabled() {
+    return process.env.CLAW_MEMORY_LESSON_DEDICATED === "1";
+}

package/dist/core/lesson-quality.js

@@ -0,0 +1,79 @@
+// src/core/lesson-quality.ts
+//
+// Lesson hygiene over time (design doc section 19 + section 7's quality phase):
+//  - confidence decay: lessons not surfaced in a while lose confidence, so
+//    stale advice naturally sinks in ranking. Long-lived general knowledge
+//    (global / user_preference) decays toward a higher floor.
+//  - expiry: lessons past their valid_until are auto-archived.
+//  - stale detection: report lessons unused for N days (no mutation).
+//
+// Intended to run periodically (CLI `claw-memory lessons decay`), not on the
+// hot path.
+import { sqlite } from "./db.js";
+import { listLessons, setStatus, recordEvent } from "./lessons.js";
+const DECAY_FACTOR = Number(process.env.LESSON_DECAY_FACTOR ?? 0.9);
+const STALE_DAYS = Number(process.env.LESSON_STALE_DAYS ?? 30);
+const FLOOR_DEFAULT = Number(process.env.LESSON_CONFIDENCE_FLOOR ?? 0.2);
+// General, broadly-true knowledge shouldn't decay into irrelevance.
+const FLOOR_GENERAL = Number(process.env.LESSON_CONFIDENCE_FLOOR_GENERAL ?? 0.5);
+function ageDays(ref, now) {
+    const t = new Date(ref).getTime();
+    if (!Number.isFinite(t))
+        return 0;
+    return (now - t) / 86_400_000;
+}
+function floorFor(scope) {
+    return scope === "global" || scope === "user_preference"
+        ? FLOOR_GENERAL
+        : FLOOR_DEFAULT;
+}
+/**
+ * Decay confidence of stale candidate/approved lessons and auto-archive expired
+ * ones. Returns counts. Idempotent enough to run on a schedule.
+ */
+export function decayConfidence(opts = {}) {
+    const factor = opts.factor ?? DECAY_FACTOR;
+    const staleDays = opts.staleDays ?? STALE_DAYS;
+    const now = Date.now();
+    const result = { scanned: 0, decayed: 0, archivedExpired: 0 };
+    const updateConfidence = sqlite.prepare("UPDATE lessons SET confidence = ?, updated_at = ? WHERE id = ?");
+    for (const status of ["candidate", "approved"]) {
+        for (const l of listLessons({ status }, 100_000)) {
+            result.scanned++;
+            // Expiry: valid_until in the past → archive.
+            if (l.validUntil && new Date(l.validUntil).getTime() < now) {
+                if (!opts.dryRun)
+                    setStatus(l.id, "archived", "valid_until passed");
+                result.archivedExpired++;
+                continue;
+            }
+            const age = ageDays(l.lastUsedAt ?? l.createdAt, now);
+            if (age < staleDays)
+                continue;
+            const floor = floorFor(l.scope);
+            const next = Math.max(floor, Math.round(l.confidence * factor * 10000) / 10000);
+            if (next < l.confidence - 1e-9) {
+                if (!opts.dryRun) {
+                    updateConfidence.run(next, new Date().toISOString(), l.id);
+                    recordEvent(l.id, "decay", {
+                        note: `confidence ${l.confidence.toFixed(4)}→${next.toFixed(4)} (age ${Math.round(age)}d)`,
+                    });
+                }
+                result.decayed++;
+            }
+        }
+    }
+    return result;
+}
+/** Lessons unused for >= `days` (no mutation) — candidates for review/removal. */
+export function listStale(days = STALE_DAYS) {
+    const now = Date.now();
+    const out = [];
+    for (const status of ["candidate", "approved"]) {
+        for (const l of listLessons({ status }, 100_000)) {
+            if (ageDays(l.lastUsedAt ?? l.createdAt, now) >= days)
+                out.push(l);
+        }
+    }
+    return out;
+}

package/dist/core/lesson-search.js

@@ -0,0 +1,145 @@
+// src/core/lesson-search.ts
+//
+// Lesson retrieval. Unlike conversation search (pure vector distance), lessons
+// are ranked by a weighted blend (design doc section 20): semantic similarity,
+// scope fit, confidence, recency, repo match and file match. Only 'approved'
+// lessons are surfaced by default — candidates stay out of agent context until
+// a human/rule promotes them.
+import { embedQuery } from "./embeddings.js";
+import { searchSimilarLessons, searchKeywordLessons, markUsed, } from "./lessons.js";
+import { resolveRepoId } from "./lesson-extract.js";
+const MAX_DISTANCE = Number(process.env.LESSON_SIMILARITY_MAX_DISTANCE ?? 0.6);
+const RECENCY_HALF_DAYS = Number(process.env.LESSON_RECENCY_DAYS ?? 180);
+// Section-20 weights (sum = 1.0).
+const W = {
+    semantic: 0.4,
+    scope: 0.2,
+    confidence: 0.15,
+    recency: 0.1,
+    repo: 0.1,
+    file: 0.05,
+};
+function fileOverlap(a, b) {
+    if (!b || b.length === 0 || a.length === 0)
+        return false;
+    return a.some((f) => b.some((g) => f.includes(g) || g.includes(f)));
+}
+/** How well a lesson's scope applies to the current context (0..1). */
+function scopeMatch(lesson, ctx) {
+    switch (lesson.scope) {
+        case "global":
+        case "user_preference":
+            return 0.7;
+        case "team":
+            return 0.6;
+        case "project":
+            return ctx.projectId && lesson.projectId === ctx.projectId ? 1 : 0.2;
+        case "repo":
+            return ctx.repoId && lesson.repoId === ctx.repoId ? 1 : 0.2;
+        case "file":
+            return fileOverlap(lesson.files, ctx.files) ? 1 : 0.2;
+        case "task":
+            return 0.3;
+        default:
+            return 0.3;
+    }
+}
+function recencyScore(lesson) {
+    const ref = lesson.lastUsedAt ?? lesson.createdAt;
+    const ageMs = Date.now() - new Date(ref).getTime();
+    const ageDays = ageMs / 86_400_000;
+    if (!Number.isFinite(ageDays) || ageDays < 0)
+        return 1;
+    return Math.max(0, 1 - ageDays / RECENCY_HALF_DAYS);
+}
+/** Cosine distance (0..2) → similarity (0..1). Keyword-only hits get a baseline. */
+function semanticScore(distance) {
+    if (distance == null)
+        return 0.5;
+    return Math.max(0, Math.min(1, 1 - distance));
+}
+function computeScore(lesson, distance, ctx) {
+    const repoMatch = ctx.repoId && lesson.repoId === ctx.repoId ? 1 : 0;
+    const fMatch = fileOverlap(lesson.files, ctx.files) ? 1 : 0;
+    return (W.semantic * semanticScore(distance) +
+        W.scope * scopeMatch(lesson, ctx) +
+        W.confidence * lesson.confidence +
+        W.recency * recencyScore(lesson) +
+        W.repo * repoMatch +
+        W.file * fMatch);
+}
+/**
+ * Hybrid lesson search: semantic (vector) + FTS keyword, de-duplicated by id,
+ * re-ranked by the section-20 blend. Returns highest-scoring first.
+ */
+export async function searchLessons(query, ctx = {}, opts = {}) {
+    const limit = opts.limit ?? 5;
+    const status = opts.status ?? "approved";
+    const repoId = ctx.repoId ?? resolveRepoId(ctx.projectId);
+    const fullCtx = { ...ctx, repoId };
+    const byId = new Map();
+    if (query.trim()) {
+        try {
+            const emb = await embedQuery(query);
+            for (const l of searchSimilarLessons(emb, {
+                topK: limit * 3,
+                maxDistance: MAX_DISTANCE,
+                status,
+            })) {
+                byId.set(l.id, { lesson: l, distance: l.distance });
+            }
+        }
+        catch (err) {
+            console.error("[claw-memory] lesson semantic search failed:", err);
+        }
+        for (const l of searchKeywordLessons(query, { topK: limit * 3, status })) {
+            if (!byId.has(l.id))
+                byId.set(l.id, { lesson: l });
+        }
+    }
+    const ranked = [...byId.values()]
+        .map(({ lesson, distance }) => ({
+        ...lesson,
+        distance: distance ?? 1,
+        score: computeScore(lesson, distance, fullCtx),
+    }))
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    if (opts.touch !== false && ranked.length > 0) {
+        markUsed(ranked.map((l) => l.id));
+    }
+    return ranked;
+}
+/**
+ * Format lessons as an agent-ready context block (design doc section 21).
+ * Lessons are presented as HINTS, not absolute facts — the agent must verify
+ * against current repository state before applying.
+ */
+export function formatLessonBlock(lessons) {
+    if (lessons.length === 0)
+        return "";
+    let text = "<relevant-lessons>\n";
+    text +=
+        "以下は、過去の類似セッションから抽出されたレッスンです。\n" +
+            "絶対的な事実ではなく、作業時のヒントとして扱ってください。\n" +
+            "適用前に現在のリポジトリ状態を確認してください。\n\n";
+    lessons.forEach((l, i) => {
+        text += `${i + 1}. ${l.lesson}\n`;
+        if (l.appliesWhen.length > 0) {
+            text += `   Applies when: ${l.appliesWhen.join("; ")}\n`;
+        }
+        if (l.avoidWhen.length > 0) {
+            text += `   Avoid when: ${l.avoidWhen.join("; ")}\n`;
+        }
+        text += `   Scope: ${l.scope} | Confidence: ${l.confidence.toFixed(2)}\n`;
+        if (l.sessionId)
+            text += `   Source: ${l.sessionId}\n`;
+    });
+    text += "</relevant-lessons>";
+    return text;
+}
+/** Convenience: search + format in one call (lesson_inject). */
+export async function injectLessons(query, ctx = {}, opts = {}) {
+    const lessons = await searchLessons(query, ctx, opts);
+    return formatLessonBlock(lessons);
+}

package/dist/core/lesson-share.js

@@ -0,0 +1,80 @@
+// src/core/lesson-share.ts
+//
+// Team sharing (design doc section 8 / Phase 8). claw-memory is local-first, so
+// there is no daemon and no automatic external sync. Sharing is explicit and
+// file-based: export a portable JSON bundle, import it elsewhere. Embeddings are
+// NOT exported — they are recomputed on import (the local e5 model is
+// deterministic), keeping bundles small and engine-version-independent.
+//
+// Imported lessons default to status 'candidate' so shared knowledge is
+// reviewed before it enters another developer's recall — the "public/team
+// 共有時の追加レビュー" requirement.
+import { embedPassage } from "./embeddings.js";
+import { listLessons, saveLesson, } from "./lessons.js";
+export const EXPORT_VERSION = 1;
+function toExported(l) {
+    return {
+        title: l.title,
+        lesson: l.lesson,
+        applies_when: l.appliesWhen,
+        avoid_when: l.avoidWhen,
+        evidence: l.evidence,
+        scope: l.scope,
+        obs_type: l.obsType,
+        concepts: l.concepts,
+        files: l.files,
+        confidence: l.confidence,
+        status: l.status,
+        created_at: l.createdAt,
+    };
+}
+/** Build a portable bundle. Defaults to approved lessons. */
+export function exportLessons(opts = {}) {
+    const status = opts.status ?? "approved";
+    const rows = listLessons({ projectId: opts.projectId ?? undefined, status }, 100_000);
+    return {
+        version: EXPORT_VERSION,
+        exported_at: new Date().toISOString(),
+        count: rows.length,
+        lessons: rows.map(toExported),
+    };
+}
+/** Import a bundle: re-embed each lesson and store it (default 'candidate'). */
+export async function importLessons(bundle, opts = {}) {
+    if (!bundle || !Array.isArray(bundle.lessons)) {
+        throw new Error("invalid lesson bundle: missing lessons[]");
+    }
+    if (bundle.version !== EXPORT_VERSION) {
+        throw new Error(`unsupported bundle version ${bundle.version} (expected ${EXPORT_VERSION})`);
+    }
+    const status = opts.status ?? "candidate";
+    let imported = 0;
+    let skipped = 0;
+    for (const it of bundle.lessons) {
+        const title = (it.title ?? "").trim();
+        const lesson = (it.lesson ?? "").trim();
+        if (!title || !lesson) {
+            skipped++;
+            continue;
+        }
+        const embedding = await embedPassage(`${title}\n${lesson}`);
+        saveLesson({
+            projectId: opts.projectId ?? null,
+            sessionId: opts.sessionId ?? "import",
+            title: title.slice(0, 200),
+            lesson: lesson.slice(0, 2000),
+            appliesWhen: (it.applies_when ?? []).map(String),
+            avoidWhen: (it.avoid_when ?? []).map(String),
+            evidence: it.evidence ?? null,
+            scope: it.scope ?? "repo",
+            obsType: it.obs_type ?? null,
+            concepts: (it.concepts ?? []).map(String),
+            files: (it.files ?? []).map(String),
+            confidence: typeof it.confidence === "number" ? it.confidence : 0.5,
+            status,
+            embedding,
+        });
+        imported++;
+    }
+    return { imported, skipped };
+}