@nogataka/claw-memory 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +5 -3
- package/.claude-plugin/plugin.json +11 -3
- package/README.ja.md +55 -11
- package/README.md +60 -11
- package/dist/cli.js +215 -1
- package/dist/core/db.js +132 -9
- package/dist/core/distill.js +97 -5
- package/dist/core/hooks.js +8 -7
- package/dist/core/installer/codex.js +19 -5
- package/dist/core/lesson-extract.js +233 -0
- package/dist/core/lesson-quality.js +79 -0
- package/dist/core/lesson-search.js +145 -0
- package/dist/core/lesson-share.js +80 -0
- package/dist/core/lessons.js +332 -0
- package/dist/core/logsearch/parse.js +0 -0
- package/dist/core/logsearch/paths.js +22 -0
- package/dist/core/logsearch/recent.js +46 -2
- package/dist/core/logsearch/search.js +57 -3
- package/dist/core/recall.js +61 -0
- package/dist/core/vector-memory.js +27 -10
- package/dist/mcp/server.js +177 -5
- package/dist/ui/page.js +125 -4
- package/dist/ui/server.js +65 -1
- package/hooks/claw-hook.sh +33 -9
- package/hooks/hooks.codex.json +0 -11
- package/hooks/hooks.json +0 -11
- package/package.json +4 -6
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// src/core/lesson-quality.ts
|
|
2
|
+
//
|
|
3
|
+
// Lesson hygiene over time (design doc section 19 + section 7's quality phase):
|
|
4
|
+
// - confidence decay: lessons not surfaced in a while lose confidence, so
|
|
5
|
+
// stale advice naturally sinks in ranking. Long-lived general knowledge
|
|
6
|
+
// (global / user_preference) decays toward a higher floor.
|
|
7
|
+
// - expiry: lessons past their valid_until are auto-archived.
|
|
8
|
+
// - stale detection: report lessons unused for N days (no mutation).
|
|
9
|
+
//
|
|
10
|
+
// Intended to run periodically (CLI `claw-memory lessons decay`), not on the
|
|
11
|
+
// hot path.
|
|
12
|
+
import { sqlite } from "./db.js";
|
|
13
|
+
import { listLessons, setStatus, recordEvent } from "./lessons.js";
|
|
14
|
+
const DECAY_FACTOR = Number(process.env.LESSON_DECAY_FACTOR ?? 0.9);
|
|
15
|
+
const STALE_DAYS = Number(process.env.LESSON_STALE_DAYS ?? 30);
|
|
16
|
+
const FLOOR_DEFAULT = Number(process.env.LESSON_CONFIDENCE_FLOOR ?? 0.2);
|
|
17
|
+
// General, broadly-true knowledge shouldn't decay into irrelevance.
|
|
18
|
+
const FLOOR_GENERAL = Number(process.env.LESSON_CONFIDENCE_FLOOR_GENERAL ?? 0.5);
|
|
19
|
+
function ageDays(ref, now) {
|
|
20
|
+
const t = new Date(ref).getTime();
|
|
21
|
+
if (!Number.isFinite(t))
|
|
22
|
+
return 0;
|
|
23
|
+
return (now - t) / 86_400_000;
|
|
24
|
+
}
|
|
25
|
+
function floorFor(scope) {
|
|
26
|
+
return scope === "global" || scope === "user_preference"
|
|
27
|
+
? FLOOR_GENERAL
|
|
28
|
+
: FLOOR_DEFAULT;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Decay confidence of stale candidate/approved lessons and auto-archive expired
|
|
32
|
+
* ones. Returns counts. Idempotent enough to run on a schedule.
|
|
33
|
+
*/
|
|
34
|
+
export function decayConfidence(opts = {}) {
|
|
35
|
+
const factor = opts.factor ?? DECAY_FACTOR;
|
|
36
|
+
const staleDays = opts.staleDays ?? STALE_DAYS;
|
|
37
|
+
const now = Date.now();
|
|
38
|
+
const result = { scanned: 0, decayed: 0, archivedExpired: 0 };
|
|
39
|
+
const updateConfidence = sqlite.prepare("UPDATE lessons SET confidence = ?, updated_at = ? WHERE id = ?");
|
|
40
|
+
for (const status of ["candidate", "approved"]) {
|
|
41
|
+
for (const l of listLessons({ status }, 100_000)) {
|
|
42
|
+
result.scanned++;
|
|
43
|
+
// Expiry: valid_until in the past → archive.
|
|
44
|
+
if (l.validUntil && new Date(l.validUntil).getTime() < now) {
|
|
45
|
+
if (!opts.dryRun)
|
|
46
|
+
setStatus(l.id, "archived", "valid_until passed");
|
|
47
|
+
result.archivedExpired++;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const age = ageDays(l.lastUsedAt ?? l.createdAt, now);
|
|
51
|
+
if (age < staleDays)
|
|
52
|
+
continue;
|
|
53
|
+
const floor = floorFor(l.scope);
|
|
54
|
+
const next = Math.max(floor, Math.round(l.confidence * factor * 10000) / 10000);
|
|
55
|
+
if (next < l.confidence - 1e-9) {
|
|
56
|
+
if (!opts.dryRun) {
|
|
57
|
+
updateConfidence.run(next, new Date().toISOString(), l.id);
|
|
58
|
+
recordEvent(l.id, "decay", {
|
|
59
|
+
note: `confidence ${l.confidence.toFixed(4)}→${next.toFixed(4)} (age ${Math.round(age)}d)`,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
result.decayed++;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
/** Lessons unused for >= `days` (no mutation) — candidates for review/removal. */
|
|
69
|
+
export function listStale(days = STALE_DAYS) {
|
|
70
|
+
const now = Date.now();
|
|
71
|
+
const out = [];
|
|
72
|
+
for (const status of ["candidate", "approved"]) {
|
|
73
|
+
for (const l of listLessons({ status }, 100_000)) {
|
|
74
|
+
if (ageDays(l.lastUsedAt ?? l.createdAt, now) >= days)
|
|
75
|
+
out.push(l);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// src/core/lesson-search.ts
|
|
2
|
+
//
|
|
3
|
+
// Lesson retrieval. Unlike conversation search (pure vector distance), lessons
|
|
4
|
+
// are ranked by a weighted blend (design doc section 20): semantic similarity,
|
|
5
|
+
// scope fit, confidence, recency, repo match and file match. Only 'approved'
|
|
6
|
+
// lessons are surfaced by default — candidates stay out of agent context until
|
|
7
|
+
// a human/rule promotes them.
|
|
8
|
+
import { embedQuery } from "./embeddings.js";
|
|
9
|
+
import { searchSimilarLessons, searchKeywordLessons, markUsed, } from "./lessons.js";
|
|
10
|
+
import { resolveRepoId } from "./lesson-extract.js";
|
|
11
|
+
const MAX_DISTANCE = Number(process.env.LESSON_SIMILARITY_MAX_DISTANCE ?? 0.6);
|
|
12
|
+
const RECENCY_HALF_DAYS = Number(process.env.LESSON_RECENCY_DAYS ?? 180);
|
|
13
|
+
// Section-20 weights (sum = 1.0).
|
|
14
|
+
const W = {
|
|
15
|
+
semantic: 0.4,
|
|
16
|
+
scope: 0.2,
|
|
17
|
+
confidence: 0.15,
|
|
18
|
+
recency: 0.1,
|
|
19
|
+
repo: 0.1,
|
|
20
|
+
file: 0.05,
|
|
21
|
+
};
|
|
22
|
+
function fileOverlap(a, b) {
|
|
23
|
+
if (!b || b.length === 0 || a.length === 0)
|
|
24
|
+
return false;
|
|
25
|
+
return a.some((f) => b.some((g) => f.includes(g) || g.includes(f)));
|
|
26
|
+
}
|
|
27
|
+
/** How well a lesson's scope applies to the current context (0..1). */
|
|
28
|
+
function scopeMatch(lesson, ctx) {
|
|
29
|
+
switch (lesson.scope) {
|
|
30
|
+
case "global":
|
|
31
|
+
case "user_preference":
|
|
32
|
+
return 0.7;
|
|
33
|
+
case "team":
|
|
34
|
+
return 0.6;
|
|
35
|
+
case "project":
|
|
36
|
+
return ctx.projectId && lesson.projectId === ctx.projectId ? 1 : 0.2;
|
|
37
|
+
case "repo":
|
|
38
|
+
return ctx.repoId && lesson.repoId === ctx.repoId ? 1 : 0.2;
|
|
39
|
+
case "file":
|
|
40
|
+
return fileOverlap(lesson.files, ctx.files) ? 1 : 0.2;
|
|
41
|
+
case "task":
|
|
42
|
+
return 0.3;
|
|
43
|
+
default:
|
|
44
|
+
return 0.3;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
function recencyScore(lesson) {
|
|
48
|
+
const ref = lesson.lastUsedAt ?? lesson.createdAt;
|
|
49
|
+
const ageMs = Date.now() - new Date(ref).getTime();
|
|
50
|
+
const ageDays = ageMs / 86_400_000;
|
|
51
|
+
if (!Number.isFinite(ageDays) || ageDays < 0)
|
|
52
|
+
return 1;
|
|
53
|
+
return Math.max(0, 1 - ageDays / RECENCY_HALF_DAYS);
|
|
54
|
+
}
|
|
55
|
+
/** Cosine distance (0..2) → similarity (0..1). Keyword-only hits get a baseline. */
|
|
56
|
+
function semanticScore(distance) {
|
|
57
|
+
if (distance == null)
|
|
58
|
+
return 0.5;
|
|
59
|
+
return Math.max(0, Math.min(1, 1 - distance));
|
|
60
|
+
}
|
|
61
|
+
function computeScore(lesson, distance, ctx) {
|
|
62
|
+
const repoMatch = ctx.repoId && lesson.repoId === ctx.repoId ? 1 : 0;
|
|
63
|
+
const fMatch = fileOverlap(lesson.files, ctx.files) ? 1 : 0;
|
|
64
|
+
return (W.semantic * semanticScore(distance) +
|
|
65
|
+
W.scope * scopeMatch(lesson, ctx) +
|
|
66
|
+
W.confidence * lesson.confidence +
|
|
67
|
+
W.recency * recencyScore(lesson) +
|
|
68
|
+
W.repo * repoMatch +
|
|
69
|
+
W.file * fMatch);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Hybrid lesson search: semantic (vector) + FTS keyword, de-duplicated by id,
|
|
73
|
+
* re-ranked by the section-20 blend. Returns highest-scoring first.
|
|
74
|
+
*/
|
|
75
|
+
export async function searchLessons(query, ctx = {}, opts = {}) {
|
|
76
|
+
const limit = opts.limit ?? 5;
|
|
77
|
+
const status = opts.status ?? "approved";
|
|
78
|
+
const repoId = ctx.repoId ?? resolveRepoId(ctx.projectId);
|
|
79
|
+
const fullCtx = { ...ctx, repoId };
|
|
80
|
+
const byId = new Map();
|
|
81
|
+
if (query.trim()) {
|
|
82
|
+
try {
|
|
83
|
+
const emb = await embedQuery(query);
|
|
84
|
+
for (const l of searchSimilarLessons(emb, {
|
|
85
|
+
topK: limit * 3,
|
|
86
|
+
maxDistance: MAX_DISTANCE,
|
|
87
|
+
status,
|
|
88
|
+
})) {
|
|
89
|
+
byId.set(l.id, { lesson: l, distance: l.distance });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
console.error("[claw-memory] lesson semantic search failed:", err);
|
|
94
|
+
}
|
|
95
|
+
for (const l of searchKeywordLessons(query, { topK: limit * 3, status })) {
|
|
96
|
+
if (!byId.has(l.id))
|
|
97
|
+
byId.set(l.id, { lesson: l });
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const ranked = [...byId.values()]
|
|
101
|
+
.map(({ lesson, distance }) => ({
|
|
102
|
+
...lesson,
|
|
103
|
+
distance: distance ?? 1,
|
|
104
|
+
score: computeScore(lesson, distance, fullCtx),
|
|
105
|
+
}))
|
|
106
|
+
.sort((a, b) => b.score - a.score)
|
|
107
|
+
.slice(0, limit);
|
|
108
|
+
if (opts.touch !== false && ranked.length > 0) {
|
|
109
|
+
markUsed(ranked.map((l) => l.id));
|
|
110
|
+
}
|
|
111
|
+
return ranked;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Format lessons as an agent-ready context block (design doc section 21).
|
|
115
|
+
* Lessons are presented as HINTS, not absolute facts — the agent must verify
|
|
116
|
+
* against current repository state before applying.
|
|
117
|
+
*/
|
|
118
|
+
export function formatLessonBlock(lessons) {
|
|
119
|
+
if (lessons.length === 0)
|
|
120
|
+
return "";
|
|
121
|
+
let text = "<relevant-lessons>\n";
|
|
122
|
+
text +=
|
|
123
|
+
"以下は、過去の類似セッションから抽出されたレッスンです。\n" +
|
|
124
|
+
"絶対的な事実ではなく、作業時のヒントとして扱ってください。\n" +
|
|
125
|
+
"適用前に現在のリポジトリ状態を確認してください。\n\n";
|
|
126
|
+
lessons.forEach((l, i) => {
|
|
127
|
+
text += `${i + 1}. ${l.lesson}\n`;
|
|
128
|
+
if (l.appliesWhen.length > 0) {
|
|
129
|
+
text += ` Applies when: ${l.appliesWhen.join("; ")}\n`;
|
|
130
|
+
}
|
|
131
|
+
if (l.avoidWhen.length > 0) {
|
|
132
|
+
text += ` Avoid when: ${l.avoidWhen.join("; ")}\n`;
|
|
133
|
+
}
|
|
134
|
+
text += ` Scope: ${l.scope} | Confidence: ${l.confidence.toFixed(2)}\n`;
|
|
135
|
+
if (l.sessionId)
|
|
136
|
+
text += ` Source: ${l.sessionId}\n`;
|
|
137
|
+
});
|
|
138
|
+
text += "</relevant-lessons>";
|
|
139
|
+
return text;
|
|
140
|
+
}
|
|
141
|
+
/** Convenience: search + format in one call (lesson_inject). */
|
|
142
|
+
export async function injectLessons(query, ctx = {}, opts = {}) {
|
|
143
|
+
const lessons = await searchLessons(query, ctx, opts);
|
|
144
|
+
return formatLessonBlock(lessons);
|
|
145
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// src/core/lesson-share.ts
|
|
2
|
+
//
|
|
3
|
+
// Team sharing (design doc section 8 / Phase 8). claw-memory is local-first, so
|
|
4
|
+
// there is no daemon and no automatic external sync. Sharing is explicit and
|
|
5
|
+
// file-based: export a portable JSON bundle, import it elsewhere. Embeddings are
|
|
6
|
+
// NOT exported — they are recomputed on import (the local e5 model is
|
|
7
|
+
// deterministic), keeping bundles small and engine-version-independent.
|
|
8
|
+
//
|
|
9
|
+
// Imported lessons default to status 'candidate' so shared knowledge is
|
|
10
|
+
// reviewed before it enters another developer's recall — the "public/team
|
|
11
|
+
// 共有時の追加レビュー" requirement.
|
|
12
|
+
import { embedPassage } from "./embeddings.js";
|
|
13
|
+
import { listLessons, saveLesson, } from "./lessons.js";
|
|
14
|
+
export const EXPORT_VERSION = 1;
|
|
15
|
+
function toExported(l) {
|
|
16
|
+
return {
|
|
17
|
+
title: l.title,
|
|
18
|
+
lesson: l.lesson,
|
|
19
|
+
applies_when: l.appliesWhen,
|
|
20
|
+
avoid_when: l.avoidWhen,
|
|
21
|
+
evidence: l.evidence,
|
|
22
|
+
scope: l.scope,
|
|
23
|
+
obs_type: l.obsType,
|
|
24
|
+
concepts: l.concepts,
|
|
25
|
+
files: l.files,
|
|
26
|
+
confidence: l.confidence,
|
|
27
|
+
status: l.status,
|
|
28
|
+
created_at: l.createdAt,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** Build a portable bundle. Defaults to approved lessons. */
|
|
32
|
+
export function exportLessons(opts = {}) {
|
|
33
|
+
const status = opts.status ?? "approved";
|
|
34
|
+
const rows = listLessons({ projectId: opts.projectId ?? undefined, status }, 100_000);
|
|
35
|
+
return {
|
|
36
|
+
version: EXPORT_VERSION,
|
|
37
|
+
exported_at: new Date().toISOString(),
|
|
38
|
+
count: rows.length,
|
|
39
|
+
lessons: rows.map(toExported),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/** Import a bundle: re-embed each lesson and store it (default 'candidate'). */
|
|
43
|
+
export async function importLessons(bundle, opts = {}) {
|
|
44
|
+
if (!bundle || !Array.isArray(bundle.lessons)) {
|
|
45
|
+
throw new Error("invalid lesson bundle: missing lessons[]");
|
|
46
|
+
}
|
|
47
|
+
if (bundle.version !== EXPORT_VERSION) {
|
|
48
|
+
throw new Error(`unsupported bundle version ${bundle.version} (expected ${EXPORT_VERSION})`);
|
|
49
|
+
}
|
|
50
|
+
const status = opts.status ?? "candidate";
|
|
51
|
+
let imported = 0;
|
|
52
|
+
let skipped = 0;
|
|
53
|
+
for (const it of bundle.lessons) {
|
|
54
|
+
const title = (it.title ?? "").trim();
|
|
55
|
+
const lesson = (it.lesson ?? "").trim();
|
|
56
|
+
if (!title || !lesson) {
|
|
57
|
+
skipped++;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
const embedding = await embedPassage(`${title}\n${lesson}`);
|
|
61
|
+
saveLesson({
|
|
62
|
+
projectId: opts.projectId ?? null,
|
|
63
|
+
sessionId: opts.sessionId ?? "import",
|
|
64
|
+
title: title.slice(0, 200),
|
|
65
|
+
lesson: lesson.slice(0, 2000),
|
|
66
|
+
appliesWhen: (it.applies_when ?? []).map(String),
|
|
67
|
+
avoidWhen: (it.avoid_when ?? []).map(String),
|
|
68
|
+
evidence: it.evidence ?? null,
|
|
69
|
+
scope: it.scope ?? "repo",
|
|
70
|
+
obsType: it.obs_type ?? null,
|
|
71
|
+
concepts: (it.concepts ?? []).map(String),
|
|
72
|
+
files: (it.files ?? []).map(String),
|
|
73
|
+
confidence: typeof it.confidence === "number" ? it.confidence : 0.5,
|
|
74
|
+
status,
|
|
75
|
+
embedding,
|
|
76
|
+
});
|
|
77
|
+
imported++;
|
|
78
|
+
}
|
|
79
|
+
return { imported, skipped };
|
|
80
|
+
}
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
// src/core/lessons.ts
|
|
2
|
+
//
|
|
3
|
+
// Lesson store: reusable, abstracted knowledge distilled from sessions. Mirrors
|
|
4
|
+
// vector-memory.ts (vec0 embedding + readable metadata row + FTS5 keyword row),
|
|
5
|
+
// but adds lifecycle (candidate → approved/rejected/archived/superseded), an
|
|
6
|
+
// event audit trail, and inter-lesson links (duplicate / conflict / supersede).
|
|
7
|
+
//
|
|
8
|
+
// JSON-array fields (applies_when / avoid_when / concepts / files /
|
|
9
|
+
// source_chunk_ids) are stored as TEXT, parsed on read — same convention as
|
|
10
|
+
// conversation_chunks.concepts.
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
12
|
+
import { sqlite, transaction } from "./db.js";
|
|
13
|
+
function jsonArr(v) {
|
|
14
|
+
return v && v.length ? JSON.stringify(v) : null;
|
|
15
|
+
}
|
|
16
|
+
function parseArr(v) {
|
|
17
|
+
if (typeof v !== "string" || !v)
|
|
18
|
+
return [];
|
|
19
|
+
try {
|
|
20
|
+
const a = JSON.parse(v);
|
|
21
|
+
return Array.isArray(a) ? a.map(String) : [];
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const LESSON_COLS = `l.id, l.project_id, l.repo_id, l.session_id, l.title, l.lesson,
|
|
28
|
+
l.applies_when, l.avoid_when, l.evidence, l.scope, l.obs_type, l.concepts, l.files,
|
|
29
|
+
l.confidence, l.source_chunk_ids, l.status, l.superseded_by, l.valid_until,
|
|
30
|
+
l.last_used_at, l.created_at, l.updated_at`;
|
|
31
|
+
function mapRow(r) {
|
|
32
|
+
return {
|
|
33
|
+
id: r.id,
|
|
34
|
+
projectId: r.project_id,
|
|
35
|
+
repoId: r.repo_id,
|
|
36
|
+
sessionId: r.session_id,
|
|
37
|
+
title: r.title,
|
|
38
|
+
lesson: r.lesson,
|
|
39
|
+
appliesWhen: parseArr(r.applies_when),
|
|
40
|
+
avoidWhen: parseArr(r.avoid_when),
|
|
41
|
+
evidence: r.evidence,
|
|
42
|
+
scope: r.scope,
|
|
43
|
+
obsType: r.obs_type,
|
|
44
|
+
concepts: parseArr(r.concepts),
|
|
45
|
+
files: parseArr(r.files),
|
|
46
|
+
confidence: r.confidence,
|
|
47
|
+
sourceChunkIds: parseArr(r.source_chunk_ids),
|
|
48
|
+
status: r.status,
|
|
49
|
+
supersededBy: r.superseded_by,
|
|
50
|
+
validUntil: r.valid_until,
|
|
51
|
+
lastUsedAt: r.last_used_at,
|
|
52
|
+
createdAt: r.created_at,
|
|
53
|
+
updatedAt: r.updated_at,
|
|
54
|
+
distance: r.distance ?? 0,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** Record a lifecycle event (status change, decay, scope edit, …). */
|
|
58
|
+
export function recordEvent(lessonId, eventType, opts = {}) {
|
|
59
|
+
sqlite
|
|
60
|
+
.prepare(`INSERT INTO lesson_events(id, lesson_id, event_type, old_status, new_status, note, created_at)
|
|
61
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
|
62
|
+
.run(randomUUID(), lessonId, eventType, opts.oldStatus ?? null, opts.newStatus ?? null, opts.note ?? null, new Date().toISOString());
|
|
63
|
+
}
|
|
64
|
+
/** Save one lesson: vec0 embedding + metadata row + FTS keyword row, atomically. */
|
|
65
|
+
export function saveLesson(input) {
|
|
66
|
+
const insertVec = sqlite.prepare("INSERT INTO lesson_vec(embedding, project_id, scope) VALUES (?, ?, ?)");
|
|
67
|
+
const insertMeta = sqlite.prepare(`INSERT INTO lessons(
|
|
68
|
+
id, vec_rowid, project_id, repo_id, session_id, title, lesson,
|
|
69
|
+
applies_when, avoid_when, evidence, scope, obs_type, concepts, files,
|
|
70
|
+
confidence, source_chunk_ids, status, created_at, updated_at)
|
|
71
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
72
|
+
const insertFts = sqlite.prepare("INSERT INTO lessons_fts(lesson_id, text) VALUES (?, ?)");
|
|
73
|
+
const id = randomUUID();
|
|
74
|
+
const now = new Date().toISOString();
|
|
75
|
+
const scope = input.scope ?? "repo";
|
|
76
|
+
const status = input.status ?? "candidate";
|
|
77
|
+
const confidence = input.confidence ?? 0.5;
|
|
78
|
+
transaction(() => {
|
|
79
|
+
const res = insertVec.run(Buffer.from(input.embedding.buffer), input.projectId ?? null, scope);
|
|
80
|
+
const vecRowid = res.lastInsertRowid;
|
|
81
|
+
insertMeta.run(id, vecRowid, input.projectId ?? null, input.repoId ?? null, input.sessionId ?? null, input.title, input.lesson, jsonArr(input.appliesWhen), jsonArr(input.avoidWhen), input.evidence ?? null, scope, input.obsType ?? null, jsonArr(input.concepts), jsonArr(input.files), confidence, jsonArr(input.sourceChunkIds), status, now, now);
|
|
82
|
+
insertFts.run(id, `${input.title}\n${input.lesson}`);
|
|
83
|
+
recordEvent(id, "created", { newStatus: status });
|
|
84
|
+
});
|
|
85
|
+
return id;
|
|
86
|
+
}
|
|
87
|
+
export function getLesson(id) {
|
|
88
|
+
const row = sqlite
|
|
89
|
+
.prepare(`SELECT ${LESSON_COLS} FROM lessons l WHERE l.id = ?`)
|
|
90
|
+
.get(id);
|
|
91
|
+
return row ? mapRow(row) : null;
|
|
92
|
+
}
|
|
93
|
+
export function getLessonsByIds(ids) {
|
|
94
|
+
if (ids.length === 0)
|
|
95
|
+
return [];
|
|
96
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
97
|
+
const rows = sqlite
|
|
98
|
+
.prepare(`SELECT ${LESSON_COLS} FROM lessons l WHERE l.id IN (${placeholders})`)
|
|
99
|
+
.all(...ids);
|
|
100
|
+
return rows.map(mapRow);
|
|
101
|
+
}
|
|
102
|
+
/** Build a WHERE fragment from a LessonFilter. */
|
|
103
|
+
function lessonWhere(filter) {
|
|
104
|
+
const clauses = [];
|
|
105
|
+
const params = [];
|
|
106
|
+
if (filter?.status) {
|
|
107
|
+
clauses.push("l.status = ?");
|
|
108
|
+
params.push(filter.status);
|
|
109
|
+
}
|
|
110
|
+
if (filter?.scope) {
|
|
111
|
+
clauses.push("l.scope = ?");
|
|
112
|
+
params.push(filter.scope);
|
|
113
|
+
}
|
|
114
|
+
if (filter?.projectId) {
|
|
115
|
+
clauses.push("l.project_id = ?");
|
|
116
|
+
params.push(filter.projectId);
|
|
117
|
+
}
|
|
118
|
+
if (filter?.concept) {
|
|
119
|
+
clauses.push("l.concepts LIKE ?");
|
|
120
|
+
params.push(`%${filter.concept}%`);
|
|
121
|
+
}
|
|
122
|
+
if (filter?.file) {
|
|
123
|
+
clauses.push("l.files LIKE ?");
|
|
124
|
+
params.push(`%${filter.file}%`);
|
|
125
|
+
}
|
|
126
|
+
return { sql: clauses.length ? clauses.join(" AND ") : "1=1", params };
|
|
127
|
+
}
|
|
128
|
+
export function listLessons(filter, limit = 200) {
|
|
129
|
+
const w = lessonWhere(filter);
|
|
130
|
+
const rows = sqlite
|
|
131
|
+
.prepare(`SELECT ${LESSON_COLS} FROM lessons l
|
|
132
|
+
WHERE ${w.sql}
|
|
133
|
+
ORDER BY l.updated_at DESC LIMIT ?`)
|
|
134
|
+
.all(...w.params, limit);
|
|
135
|
+
return rows.map(mapRow);
|
|
136
|
+
}
|
|
137
|
+
export function getLessonCount(filter) {
|
|
138
|
+
const w = lessonWhere(filter);
|
|
139
|
+
const row = sqlite
|
|
140
|
+
.prepare(`SELECT COUNT(*) c FROM lessons l WHERE ${w.sql}`)
|
|
141
|
+
.get(...w.params);
|
|
142
|
+
return row.c;
|
|
143
|
+
}
|
|
144
|
+
/** Lessons involved in a conflicts_with link (for Conflict Review). */
|
|
145
|
+
export function listConflicts(projectId, limit = 200) {
|
|
146
|
+
const proj = projectId ? "AND l.project_id = ?" : "";
|
|
147
|
+
const rows = sqlite
|
|
148
|
+
.prepare(`SELECT DISTINCT ${LESSON_COLS} FROM lessons l
|
|
149
|
+
INNER JOIN lesson_links k
|
|
150
|
+
ON (k.lesson_id = l.id OR k.linked_lesson_id = l.id)
|
|
151
|
+
WHERE k.relation = 'conflicts_with' AND l.status != 'rejected' ${proj}
|
|
152
|
+
ORDER BY l.updated_at DESC LIMIT ?`)
|
|
153
|
+
.all(...(projectId ? [projectId] : []), limit);
|
|
154
|
+
return rows.map(mapRow);
|
|
155
|
+
}
|
|
156
|
+
export function getConflictCount(projectId) {
|
|
157
|
+
const proj = projectId ? "AND l.project_id = ?" : "";
|
|
158
|
+
const row = sqlite
|
|
159
|
+
.prepare(`SELECT COUNT(DISTINCT l.id) c FROM lessons l
|
|
160
|
+
INNER JOIN lesson_links k
|
|
161
|
+
ON (k.lesson_id = l.id OR k.linked_lesson_id = l.id)
|
|
162
|
+
WHERE k.relation = 'conflicts_with' AND l.status != 'rejected' ${proj}`)
|
|
163
|
+
.get(...(projectId ? [projectId] : []));
|
|
164
|
+
return row.c;
|
|
165
|
+
}
|
|
166
|
+
/** KNN semantic search over lessons, filtered by project/scope inside MATCH. */
|
|
167
|
+
export function searchSimilarLessons(queryEmbedding, opts = {}) {
|
|
168
|
+
const topK = opts.topK ?? 8;
|
|
169
|
+
// vec0 MATCH metadata filters: project must match OR be a cross-project scope.
|
|
170
|
+
const stmt = sqlite.prepare(`
|
|
171
|
+
SELECT ${LESSON_COLS}, v.distance
|
|
172
|
+
FROM (
|
|
173
|
+
SELECT rowid, distance FROM lesson_vec
|
|
174
|
+
WHERE embedding MATCH ? AND k = ?
|
|
175
|
+
) v
|
|
176
|
+
INNER JOIN lessons l ON l.vec_rowid = v.rowid
|
|
177
|
+
WHERE ${opts.status ? "l.status = ?" : "1=1"}
|
|
178
|
+
${opts.maxDistance != null ? "AND v.distance <= ?" : ""}
|
|
179
|
+
ORDER BY v.distance
|
|
180
|
+
`);
|
|
181
|
+
const params = [
|
|
182
|
+
Buffer.from(queryEmbedding.buffer),
|
|
183
|
+
topK,
|
|
184
|
+
];
|
|
185
|
+
if (opts.status)
|
|
186
|
+
params.push(opts.status);
|
|
187
|
+
if (opts.maxDistance != null)
|
|
188
|
+
params.push(opts.maxDistance);
|
|
189
|
+
const rows = stmt.all(...params);
|
|
190
|
+
return rows.map(mapRow);
|
|
191
|
+
}
|
|
192
|
+
/** FTS5 keyword search over lesson title + body. */
|
|
193
|
+
export function searchKeywordLessons(query, opts = {}) {
|
|
194
|
+
const match = toFtsQuery(query);
|
|
195
|
+
if (!match)
|
|
196
|
+
return [];
|
|
197
|
+
const rows = sqlite
|
|
198
|
+
.prepare(`SELECT ${LESSON_COLS}
|
|
199
|
+
FROM lessons_fts f
|
|
200
|
+
INNER JOIN lessons l ON l.id = f.lesson_id
|
|
201
|
+
WHERE f.text MATCH ? ${opts.status ? "AND l.status = ?" : ""}
|
|
202
|
+
LIMIT ?`)
|
|
203
|
+
.all(match, ...(opts.status ? [opts.status] : []), opts.topK ?? 8);
|
|
204
|
+
return rows.map(mapRow);
|
|
205
|
+
}
|
|
206
|
+
/** Transition a lesson's status, stamping updated_at and recording the event. */
|
|
207
|
+
export function setStatus(id, newStatus, note) {
|
|
208
|
+
const current = sqlite
|
|
209
|
+
.prepare("SELECT status FROM lessons WHERE id = ?")
|
|
210
|
+
.get(id);
|
|
211
|
+
if (!current)
|
|
212
|
+
return false;
|
|
213
|
+
const now = new Date().toISOString();
|
|
214
|
+
sqlite
|
|
215
|
+
.prepare("UPDATE lessons SET status = ?, updated_at = ? WHERE id = ?")
|
|
216
|
+
.run(newStatus, now, id);
|
|
217
|
+
recordEvent(id, "status_change", {
|
|
218
|
+
oldStatus: current.status,
|
|
219
|
+
newStatus,
|
|
220
|
+
note,
|
|
221
|
+
});
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
/** Mark `oldId` superseded by `newId` and link them. */
|
|
225
|
+
export function supersede(oldId, newId) {
|
|
226
|
+
const current = sqlite
|
|
227
|
+
.prepare("SELECT status FROM lessons WHERE id = ?")
|
|
228
|
+
.get(oldId);
|
|
229
|
+
if (!current)
|
|
230
|
+
return false;
|
|
231
|
+
const now = new Date().toISOString();
|
|
232
|
+
transaction(() => {
|
|
233
|
+
sqlite
|
|
234
|
+
.prepare("UPDATE lessons SET status = 'superseded', superseded_by = ?, updated_at = ? WHERE id = ?")
|
|
235
|
+
.run(newId, now, oldId);
|
|
236
|
+
recordEvent(oldId, "superseded", {
|
|
237
|
+
oldStatus: current.status,
|
|
238
|
+
newStatus: "superseded",
|
|
239
|
+
note: `superseded_by=${newId}`,
|
|
240
|
+
});
|
|
241
|
+
linkLessons(newId, oldId, "supersedes");
|
|
242
|
+
});
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
/** Update one or more editable fields (scope / confidence / valid_until). */
|
|
246
|
+
export function updateLesson(id, fields) {
|
|
247
|
+
const sets = [];
|
|
248
|
+
const params = [];
|
|
249
|
+
if (fields.scope !== undefined) {
|
|
250
|
+
sets.push("scope = ?");
|
|
251
|
+
params.push(fields.scope);
|
|
252
|
+
}
|
|
253
|
+
if (fields.confidence !== undefined) {
|
|
254
|
+
sets.push("confidence = ?");
|
|
255
|
+
params.push(fields.confidence);
|
|
256
|
+
}
|
|
257
|
+
if (fields.validUntil !== undefined) {
|
|
258
|
+
sets.push("valid_until = ?");
|
|
259
|
+
params.push(fields.validUntil);
|
|
260
|
+
}
|
|
261
|
+
if (sets.length === 0)
|
|
262
|
+
return false;
|
|
263
|
+
sets.push("updated_at = ?");
|
|
264
|
+
params.push(new Date().toISOString());
|
|
265
|
+
params.push(id);
|
|
266
|
+
const res = sqlite
|
|
267
|
+
.prepare(`UPDATE lessons SET ${sets.join(", ")} WHERE id = ?`)
|
|
268
|
+
.run(...params);
|
|
269
|
+
if (res.changes > 0)
|
|
270
|
+
recordEvent(id, "edited", { note: sets.join(",") });
|
|
271
|
+
return res.changes > 0;
|
|
272
|
+
}
|
|
273
|
+
/** Bump last_used_at when a lesson is surfaced by search/inject. */
|
|
274
|
+
export function markUsed(ids) {
|
|
275
|
+
if (ids.length === 0)
|
|
276
|
+
return;
|
|
277
|
+
const now = new Date().toISOString();
|
|
278
|
+
const stmt = sqlite.prepare("UPDATE lessons SET last_used_at = ? WHERE id = ?");
|
|
279
|
+
transaction(() => {
|
|
280
|
+
for (const id of ids)
|
|
281
|
+
stmt.run(now, id);
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
export function linkLessons(lessonId, linkedLessonId, relation) {
|
|
285
|
+
sqlite
|
|
286
|
+
.prepare(`INSERT INTO lesson_links(id, lesson_id, linked_lesson_id, relation, created_at)
|
|
287
|
+
VALUES (?, ?, ?, ?, ?)`)
|
|
288
|
+
.run(randomUUID(), lessonId, linkedLessonId, relation, new Date().toISOString());
|
|
289
|
+
}
|
|
290
|
+
export function getEvents(lessonId) {
|
|
291
|
+
const rows = sqlite
|
|
292
|
+
.prepare(`SELECT id, event_type, old_status, new_status, note, created_at
|
|
293
|
+
FROM lesson_events WHERE lesson_id = ? ORDER BY created_at ASC`)
|
|
294
|
+
.all(lessonId);
|
|
295
|
+
return rows.map((r) => ({
|
|
296
|
+
id: r.id,
|
|
297
|
+
eventType: r.event_type,
|
|
298
|
+
oldStatus: r.old_status,
|
|
299
|
+
newStatus: r.new_status,
|
|
300
|
+
note: r.note,
|
|
301
|
+
createdAt: r.created_at,
|
|
302
|
+
}));
|
|
303
|
+
}
|
|
304
|
+
/** Links where this lesson is either side (incoming + outgoing). */
|
|
305
|
+
export function getLinks(lessonId) {
|
|
306
|
+
const rows = sqlite
|
|
307
|
+
.prepare(`SELECT id, lesson_id, linked_lesson_id, relation, created_at
|
|
308
|
+
FROM lesson_links WHERE lesson_id = ? OR linked_lesson_id = ?
|
|
309
|
+
ORDER BY created_at ASC`)
|
|
310
|
+
.all(lessonId, lessonId);
|
|
311
|
+
return rows.map((r) => ({
|
|
312
|
+
id: r.id,
|
|
313
|
+
lessonId: r.lesson_id,
|
|
314
|
+
linkedLessonId: r.linked_lesson_id,
|
|
315
|
+
relation: r.relation,
|
|
316
|
+
createdAt: r.created_at,
|
|
317
|
+
}));
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Turn a free-text query into a safe FTS5 OR-query of quoted terms (trigram
|
|
321
|
+
* tokenizer needs >=3-char terms). Mirrors vector-memory.toFtsQuery.
|
|
322
|
+
*/
|
|
323
|
+
function toFtsQuery(query) {
|
|
324
|
+
const terms = query
|
|
325
|
+
.replace(/["()*:^]/g, " ")
|
|
326
|
+
.split(/\s+/)
|
|
327
|
+
.filter((t) => t.length >= 3)
|
|
328
|
+
.map((t) => `"${t.replace(/"/g, "")}"`);
|
|
329
|
+
if (terms.length === 0)
|
|
330
|
+
return null;
|
|
331
|
+
return terms.join(" OR ");
|
|
332
|
+
}
|
|
Binary file
|
|
@@ -6,6 +6,28 @@ import os from "node:os";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
export const claudeProjectsRoot = path.join(os.homedir(), ".claude", "projects");
|
|
8
8
|
export const codexSessionsRoot = path.join(os.homedir(), ".codex", "sessions");
|
|
9
|
+
/**
|
|
10
|
+
* Where to look for ChatGPT web "Export data" bundles (`conversations.json`).
|
|
11
|
+
* Unlike Claude Code / Codex, ChatGPT web conversations are NOT stored locally —
|
|
12
|
+
* the user downloads the official export and drops the file(s) here. Override
|
|
13
|
+
* `CLAW_MEMORY_CHATGPT_EXPORT` with either a single file or a directory.
|
|
14
|
+
*/
|
|
15
|
+
export const chatgptExportRoot = process.env.CLAW_MEMORY_CHATGPT_EXPORT ||
|
|
16
|
+
path.join(os.homedir(), ".claw-memory", "chatgpt");
|
|
17
|
+
/**
|
|
18
|
+
* Stable synthetic "project" that distilled ChatGPT conversations are grouped
|
|
19
|
+
* under. Deliberately independent of where the export file lives, so moving the
|
|
20
|
+
* export doesn't fork a new project and the viewer always shows one tidy
|
|
21
|
+
* "chatgpt" project. ChatGPT conversations have no repo/cwd of their own.
|
|
22
|
+
*/
|
|
23
|
+
export const chatgptProjectPath = path.join(os.homedir(), ".claw-memory", "chatgpt");
|
|
9
24
|
/** Max transcript size to scan; larger files are skipped (cc-search parity). */
|
|
10
25
|
export const MAX_LOG_FILE_SIZE = 10 * 1024 * 1024;
|
|
26
|
+
/**
|
|
27
|
+
* ChatGPT exports are a single JSON file holding the whole account history, so
|
|
28
|
+
* they can be much larger than a per-session transcript. JSON.parse reads the
|
|
29
|
+
* whole file into memory; this cap (default 200 MB) bounds that. Override with
|
|
30
|
+
* `CLAW_MEMORY_CHATGPT_MAX_BYTES`.
|
|
31
|
+
*/
|
|
32
|
+
export const MAX_CHATGPT_FILE_SIZE = Number(process.env.CLAW_MEMORY_CHATGPT_MAX_BYTES ?? 200 * 1024 * 1024);
|
|
11
33
|
export const UUID_JSONL_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.jsonl$/i;
|