engrm 0.4.0 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -5
- package/dist/cli.js +288 -28
- package/dist/hooks/codex-stop.js +62 -0
- package/dist/hooks/elicitation-result.js +1690 -1637
- package/dist/hooks/post-tool-use.js +326 -231
- package/dist/hooks/pre-compact.js +410 -78
- package/dist/hooks/sentinel.js +150 -103
- package/dist/hooks/session-start.js +2311 -1983
- package/dist/hooks/stop.js +302 -147
- package/dist/server.js +634 -118
- package/package.json +6 -5
- package/bin/build.mjs +0 -97
- package/bin/engrm.mjs +0 -13
|
@@ -2,1750 +2,1821 @@
|
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
4
4
|
|
|
5
|
-
// src/
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
5
|
+
// src/tools/save.ts
|
|
6
|
+
import { relative, isAbsolute } from "node:path";
|
|
7
|
+
|
|
8
|
+
// src/capture/scrubber.ts
|
|
9
|
+
var DEFAULT_PATTERNS = [
|
|
10
|
+
{
|
|
11
|
+
source: "sk-[a-zA-Z0-9]{20,}",
|
|
12
|
+
flags: "g",
|
|
13
|
+
replacement: "[REDACTED_API_KEY]",
|
|
14
|
+
description: "OpenAI API keys",
|
|
15
|
+
category: "api_key",
|
|
16
|
+
severity: "critical"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
|
|
20
|
+
flags: "g",
|
|
21
|
+
replacement: "[REDACTED_BEARER]",
|
|
22
|
+
description: "Bearer auth tokens",
|
|
23
|
+
category: "token",
|
|
24
|
+
severity: "medium"
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
source: "password[=:]\\s*\\S+",
|
|
28
|
+
flags: "gi",
|
|
29
|
+
replacement: "password=[REDACTED]",
|
|
30
|
+
description: "Passwords in config",
|
|
31
|
+
category: "password",
|
|
32
|
+
severity: "high"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
source: "postgresql://[^\\s]+",
|
|
36
|
+
flags: "g",
|
|
37
|
+
replacement: "[REDACTED_DB_URL]",
|
|
38
|
+
description: "PostgreSQL connection strings",
|
|
39
|
+
category: "db_url",
|
|
40
|
+
severity: "high"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
source: "mongodb://[^\\s]+",
|
|
44
|
+
flags: "g",
|
|
45
|
+
replacement: "[REDACTED_DB_URL]",
|
|
46
|
+
description: "MongoDB connection strings",
|
|
47
|
+
category: "db_url",
|
|
48
|
+
severity: "high"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
source: "mysql://[^\\s]+",
|
|
52
|
+
flags: "g",
|
|
53
|
+
replacement: "[REDACTED_DB_URL]",
|
|
54
|
+
description: "MySQL connection strings",
|
|
55
|
+
category: "db_url",
|
|
56
|
+
severity: "high"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
source: "AKIA[A-Z0-9]{16}",
|
|
60
|
+
flags: "g",
|
|
61
|
+
replacement: "[REDACTED_AWS_KEY]",
|
|
62
|
+
description: "AWS access keys",
|
|
63
|
+
category: "api_key",
|
|
64
|
+
severity: "critical"
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
source: "ghp_[a-zA-Z0-9]{36}",
|
|
68
|
+
flags: "g",
|
|
69
|
+
replacement: "[REDACTED_GH_TOKEN]",
|
|
70
|
+
description: "GitHub personal access tokens",
|
|
71
|
+
category: "token",
|
|
72
|
+
severity: "high"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
source: "gho_[a-zA-Z0-9]{36}",
|
|
76
|
+
flags: "g",
|
|
77
|
+
replacement: "[REDACTED_GH_TOKEN]",
|
|
78
|
+
description: "GitHub OAuth tokens",
|
|
79
|
+
category: "token",
|
|
80
|
+
severity: "high"
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
source: "github_pat_[a-zA-Z0-9_]{22,}",
|
|
84
|
+
flags: "g",
|
|
85
|
+
replacement: "[REDACTED_GH_TOKEN]",
|
|
86
|
+
description: "GitHub fine-grained PATs",
|
|
87
|
+
category: "token",
|
|
88
|
+
severity: "high"
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
source: "cvk_[a-f0-9]{64}",
|
|
92
|
+
flags: "g",
|
|
93
|
+
replacement: "[REDACTED_CANDENGO_KEY]",
|
|
94
|
+
description: "Candengo API keys",
|
|
95
|
+
category: "api_key",
|
|
96
|
+
severity: "critical"
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
source: "xox[bpras]-[a-zA-Z0-9\\-]+",
|
|
100
|
+
flags: "g",
|
|
101
|
+
replacement: "[REDACTED_SLACK_TOKEN]",
|
|
102
|
+
description: "Slack tokens",
|
|
103
|
+
category: "token",
|
|
104
|
+
severity: "high"
|
|
105
|
+
}
|
|
106
|
+
];
|
|
107
|
+
function compileCustomPatterns(patterns) {
|
|
108
|
+
const compiled = [];
|
|
109
|
+
for (const pattern of patterns) {
|
|
110
|
+
try {
|
|
111
|
+
new RegExp(pattern);
|
|
112
|
+
compiled.push({
|
|
113
|
+
source: pattern,
|
|
114
|
+
flags: "g",
|
|
115
|
+
replacement: "[REDACTED_CUSTOM]",
|
|
116
|
+
description: `Custom pattern: ${pattern}`,
|
|
117
|
+
category: "custom",
|
|
118
|
+
severity: "medium"
|
|
119
|
+
});
|
|
120
|
+
} catch {}
|
|
121
|
+
}
|
|
122
|
+
return compiled;
|
|
15
123
|
}
|
|
16
|
-
function
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
if (!entries)
|
|
22
|
-
continue;
|
|
23
|
-
for (const entry of entries) {
|
|
24
|
-
if (!entry.internal && entry.mac && entry.mac !== "00:00:00:00:00:00") {
|
|
25
|
-
mac = entry.mac;
|
|
26
|
-
break;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
if (mac)
|
|
30
|
-
break;
|
|
124
|
+
function scrubSecrets(text, customPatterns = []) {
|
|
125
|
+
let result = text;
|
|
126
|
+
const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
|
|
127
|
+
for (const pattern of allPatterns) {
|
|
128
|
+
result = result.replace(new RegExp(pattern.source, pattern.flags), pattern.replacement);
|
|
31
129
|
}
|
|
32
|
-
|
|
33
|
-
const suffix = createHash("sha256").update(material).digest("hex").slice(0, 8);
|
|
34
|
-
return `${host}-${suffix}`;
|
|
130
|
+
return result;
|
|
35
131
|
}
|
|
36
|
-
function
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
user_email: "",
|
|
44
|
-
device_id: generateDeviceId(),
|
|
45
|
-
teams: [],
|
|
46
|
-
sync: {
|
|
47
|
-
enabled: true,
|
|
48
|
-
interval_seconds: 30,
|
|
49
|
-
batch_size: 50
|
|
50
|
-
},
|
|
51
|
-
search: {
|
|
52
|
-
default_limit: 10,
|
|
53
|
-
local_boost: 1.2,
|
|
54
|
-
scope: "all"
|
|
55
|
-
},
|
|
56
|
-
scrubbing: {
|
|
57
|
-
enabled: true,
|
|
58
|
-
custom_patterns: [],
|
|
59
|
-
default_sensitivity: "shared"
|
|
60
|
-
},
|
|
61
|
-
sentinel: {
|
|
62
|
-
enabled: false,
|
|
63
|
-
mode: "advisory",
|
|
64
|
-
provider: "openai",
|
|
65
|
-
model: "gpt-4o-mini",
|
|
66
|
-
api_key: "",
|
|
67
|
-
base_url: "",
|
|
68
|
-
skip_patterns: [],
|
|
69
|
-
daily_limit: 100,
|
|
70
|
-
tier: "free"
|
|
71
|
-
},
|
|
72
|
-
observer: {
|
|
73
|
-
enabled: true,
|
|
74
|
-
mode: "per_event",
|
|
75
|
-
model: "sonnet"
|
|
76
|
-
},
|
|
77
|
-
transcript_analysis: {
|
|
78
|
-
enabled: false
|
|
79
|
-
}
|
|
80
|
-
};
|
|
132
|
+
function containsSecrets(text, customPatterns = []) {
|
|
133
|
+
const allPatterns = [...DEFAULT_PATTERNS, ...compileCustomPatterns(customPatterns)];
|
|
134
|
+
for (const pattern of allPatterns) {
|
|
135
|
+
if (new RegExp(pattern.source, pattern.flags).test(text))
|
|
136
|
+
return true;
|
|
137
|
+
}
|
|
138
|
+
return false;
|
|
81
139
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
140
|
+
|
|
141
|
+
// src/capture/quality.ts
|
|
142
|
+
var QUALITY_THRESHOLD = 0.1;
|
|
143
|
+
function scoreQuality(input) {
|
|
144
|
+
let score = 0;
|
|
145
|
+
switch (input.type) {
|
|
146
|
+
case "bugfix":
|
|
147
|
+
score += 0.3;
|
|
148
|
+
break;
|
|
149
|
+
case "decision":
|
|
150
|
+
score += 0.3;
|
|
151
|
+
break;
|
|
152
|
+
case "discovery":
|
|
153
|
+
score += 0.2;
|
|
154
|
+
break;
|
|
155
|
+
case "pattern":
|
|
156
|
+
score += 0.2;
|
|
157
|
+
break;
|
|
158
|
+
case "feature":
|
|
159
|
+
score += 0.15;
|
|
160
|
+
break;
|
|
161
|
+
case "refactor":
|
|
162
|
+
score += 0.15;
|
|
163
|
+
break;
|
|
164
|
+
case "change":
|
|
165
|
+
score += 0.05;
|
|
166
|
+
break;
|
|
167
|
+
case "digest":
|
|
168
|
+
score += 0.3;
|
|
169
|
+
break;
|
|
170
|
+
case "standard":
|
|
171
|
+
score += 0.25;
|
|
172
|
+
break;
|
|
173
|
+
case "message":
|
|
174
|
+
score += 0.1;
|
|
175
|
+
break;
|
|
85
176
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
try {
|
|
89
|
-
parsed = JSON.parse(raw);
|
|
90
|
-
} catch {
|
|
91
|
-
throw new Error(`Invalid JSON in ${SETTINGS_PATH}`);
|
|
177
|
+
if (input.narrative && input.narrative.length > 50) {
|
|
178
|
+
score += 0.15;
|
|
92
179
|
}
|
|
93
|
-
if (
|
|
94
|
-
|
|
180
|
+
if (input.facts) {
|
|
181
|
+
try {
|
|
182
|
+
const factsArray = JSON.parse(input.facts);
|
|
183
|
+
if (factsArray.length >= 2)
|
|
184
|
+
score += 0.15;
|
|
185
|
+
else if (factsArray.length === 1)
|
|
186
|
+
score += 0.05;
|
|
187
|
+
} catch {
|
|
188
|
+
if (input.facts.length > 20)
|
|
189
|
+
score += 0.05;
|
|
190
|
+
}
|
|
95
191
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
user_email: asString(config["user_email"], defaults.user_email),
|
|
105
|
-
device_id: asString(config["device_id"], defaults.device_id),
|
|
106
|
-
teams: asTeams(config["teams"], defaults.teams),
|
|
107
|
-
sync: {
|
|
108
|
-
enabled: asBool(config["sync"]?.["enabled"], defaults.sync.enabled),
|
|
109
|
-
interval_seconds: asNumber(config["sync"]?.["interval_seconds"], defaults.sync.interval_seconds),
|
|
110
|
-
batch_size: asNumber(config["sync"]?.["batch_size"], defaults.sync.batch_size)
|
|
111
|
-
},
|
|
112
|
-
search: {
|
|
113
|
-
default_limit: asNumber(config["search"]?.["default_limit"], defaults.search.default_limit),
|
|
114
|
-
local_boost: asNumber(config["search"]?.["local_boost"], defaults.search.local_boost),
|
|
115
|
-
scope: asScope(config["search"]?.["scope"], defaults.search.scope)
|
|
116
|
-
},
|
|
117
|
-
scrubbing: {
|
|
118
|
-
enabled: asBool(config["scrubbing"]?.["enabled"], defaults.scrubbing.enabled),
|
|
119
|
-
custom_patterns: asStringArray(config["scrubbing"]?.["custom_patterns"], defaults.scrubbing.custom_patterns),
|
|
120
|
-
default_sensitivity: asSensitivity(config["scrubbing"]?.["default_sensitivity"], defaults.scrubbing.default_sensitivity)
|
|
121
|
-
},
|
|
122
|
-
sentinel: {
|
|
123
|
-
enabled: asBool(config["sentinel"]?.["enabled"], defaults.sentinel.enabled),
|
|
124
|
-
mode: asSentinelMode(config["sentinel"]?.["mode"], defaults.sentinel.mode),
|
|
125
|
-
provider: asLlmProvider(config["sentinel"]?.["provider"], defaults.sentinel.provider),
|
|
126
|
-
model: asString(config["sentinel"]?.["model"], defaults.sentinel.model),
|
|
127
|
-
api_key: asString(config["sentinel"]?.["api_key"], defaults.sentinel.api_key),
|
|
128
|
-
base_url: asString(config["sentinel"]?.["base_url"], defaults.sentinel.base_url),
|
|
129
|
-
skip_patterns: asStringArray(config["sentinel"]?.["skip_patterns"], defaults.sentinel.skip_patterns),
|
|
130
|
-
daily_limit: asNumber(config["sentinel"]?.["daily_limit"], defaults.sentinel.daily_limit),
|
|
131
|
-
tier: asTier(config["sentinel"]?.["tier"], defaults.sentinel.tier)
|
|
132
|
-
},
|
|
133
|
-
observer: {
|
|
134
|
-
enabled: asBool(config["observer"]?.["enabled"], defaults.observer.enabled),
|
|
135
|
-
mode: asObserverMode(config["observer"]?.["mode"], defaults.observer.mode),
|
|
136
|
-
model: asString(config["observer"]?.["model"], defaults.observer.model)
|
|
137
|
-
},
|
|
138
|
-
transcript_analysis: {
|
|
139
|
-
enabled: asBool(config["transcript_analysis"]?.["enabled"], defaults.transcript_analysis.enabled)
|
|
192
|
+
if (input.concepts) {
|
|
193
|
+
try {
|
|
194
|
+
const conceptsArray = JSON.parse(input.concepts);
|
|
195
|
+
if (conceptsArray.length >= 1)
|
|
196
|
+
score += 0.1;
|
|
197
|
+
} catch {
|
|
198
|
+
if (input.concepts.length > 10)
|
|
199
|
+
score += 0.05;
|
|
140
200
|
}
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
function saveConfig(config) {
|
|
144
|
-
if (!existsSync(CONFIG_DIR)) {
|
|
145
|
-
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
146
201
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
function asNumber(value, fallback) {
|
|
157
|
-
return typeof value === "number" && !Number.isNaN(value) ? value : fallback;
|
|
202
|
+
const modifiedCount = input.filesModified?.length ?? 0;
|
|
203
|
+
if (modifiedCount >= 3)
|
|
204
|
+
score += 0.2;
|
|
205
|
+
else if (modifiedCount >= 1)
|
|
206
|
+
score += 0.1;
|
|
207
|
+
if (input.isDuplicate) {
|
|
208
|
+
score -= 0.3;
|
|
209
|
+
}
|
|
210
|
+
return Math.max(0, Math.min(1, score));
|
|
158
211
|
}
|
|
159
|
-
function
|
|
160
|
-
return
|
|
212
|
+
function meetsQualityThreshold(input) {
|
|
213
|
+
return scoreQuality(input) >= QUALITY_THRESHOLD;
|
|
161
214
|
}
|
|
162
|
-
|
|
163
|
-
|
|
215
|
+
|
|
216
|
+
// src/capture/dedup.ts
|
|
217
|
+
function tokenise(text) {
|
|
218
|
+
const cleaned = text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").trim();
|
|
219
|
+
const tokens = cleaned.split(/\s+/).filter((t) => t.length > 0);
|
|
220
|
+
return new Set(tokens);
|
|
164
221
|
}
|
|
165
|
-
function
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
222
|
+
function jaccardSimilarity(a, b) {
|
|
223
|
+
const tokensA = tokenise(a);
|
|
224
|
+
const tokensB = tokenise(b);
|
|
225
|
+
if (tokensA.size === 0 && tokensB.size === 0)
|
|
226
|
+
return 1;
|
|
227
|
+
if (tokensA.size === 0 || tokensB.size === 0)
|
|
228
|
+
return 0;
|
|
229
|
+
let intersectionSize = 0;
|
|
230
|
+
for (const token of tokensA) {
|
|
231
|
+
if (tokensB.has(token))
|
|
232
|
+
intersectionSize++;
|
|
233
|
+
}
|
|
234
|
+
const unionSize = tokensA.size + tokensB.size - intersectionSize;
|
|
235
|
+
if (unionSize === 0)
|
|
236
|
+
return 0;
|
|
237
|
+
return intersectionSize / unionSize;
|
|
169
238
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
239
|
+
var DEDUP_THRESHOLD = 0.8;
|
|
240
|
+
function findDuplicate(newTitle, candidates) {
|
|
241
|
+
let bestMatch = null;
|
|
242
|
+
let bestScore = 0;
|
|
243
|
+
for (const candidate of candidates) {
|
|
244
|
+
const similarity = jaccardSimilarity(newTitle, candidate.title);
|
|
245
|
+
if (similarity > DEDUP_THRESHOLD && similarity > bestScore) {
|
|
246
|
+
bestScore = similarity;
|
|
247
|
+
bestMatch = candidate;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return bestMatch;
|
|
174
251
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
252
|
+
|
|
253
|
+
// src/storage/projects.ts
|
|
254
|
+
import { execSync } from "node:child_process";
|
|
255
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
256
|
+
import { basename, join } from "node:path";
|
|
257
|
+
function normaliseGitRemoteUrl(remoteUrl) {
|
|
258
|
+
let url = remoteUrl.trim();
|
|
259
|
+
url = url.replace(/^(?:https?|ssh|git):\/\//, "");
|
|
260
|
+
url = url.replace(/^[^@]+@/, "");
|
|
261
|
+
url = url.replace(/^([^/:]+):(?!\d)/, "$1/");
|
|
262
|
+
url = url.replace(/\.git$/, "");
|
|
263
|
+
url = url.replace(/\/+$/, "");
|
|
264
|
+
const slashIndex = url.indexOf("/");
|
|
265
|
+
if (slashIndex !== -1) {
|
|
266
|
+
const host = url.substring(0, slashIndex).toLowerCase();
|
|
267
|
+
const path = url.substring(slashIndex);
|
|
268
|
+
url = host + path;
|
|
269
|
+
} else {
|
|
270
|
+
url = url.toLowerCase();
|
|
271
|
+
}
|
|
272
|
+
return url;
|
|
179
273
|
}
|
|
180
|
-
function
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
return fallback;
|
|
274
|
+
function projectNameFromCanonicalId(canonicalId) {
|
|
275
|
+
const parts = canonicalId.split("/");
|
|
276
|
+
return parts[parts.length - 1] ?? canonicalId;
|
|
184
277
|
}
|
|
185
|
-
function
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
278
|
+
function getGitRemoteUrl(directory) {
|
|
279
|
+
try {
|
|
280
|
+
const url = execSync("git remote get-url origin", {
|
|
281
|
+
cwd: directory,
|
|
282
|
+
encoding: "utf-8",
|
|
283
|
+
timeout: 5000,
|
|
284
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
285
|
+
}).trim();
|
|
286
|
+
return url || null;
|
|
287
|
+
} catch {
|
|
288
|
+
try {
|
|
289
|
+
const remotes = execSync("git remote", {
|
|
290
|
+
cwd: directory,
|
|
291
|
+
encoding: "utf-8",
|
|
292
|
+
timeout: 5000,
|
|
293
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
294
|
+
}).trim().split(`
|
|
295
|
+
`).filter(Boolean);
|
|
296
|
+
if (remotes.length === 0)
|
|
297
|
+
return null;
|
|
298
|
+
const url = execSync(`git remote get-url ${remotes[0]}`, {
|
|
299
|
+
cwd: directory,
|
|
300
|
+
encoding: "utf-8",
|
|
301
|
+
timeout: 5000,
|
|
302
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
303
|
+
}).trim();
|
|
304
|
+
return url || null;
|
|
305
|
+
} catch {
|
|
306
|
+
return null;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
189
309
|
}
|
|
190
|
-
function
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
310
|
+
function readProjectConfigFile(directory) {
|
|
311
|
+
const configPath = join(directory, ".engrm.json");
|
|
312
|
+
if (!existsSync(configPath))
|
|
313
|
+
return null;
|
|
314
|
+
try {
|
|
315
|
+
const raw = readFileSync(configPath, "utf-8");
|
|
316
|
+
const parsed = JSON.parse(raw);
|
|
317
|
+
if (typeof parsed["project_id"] !== "string" || !parsed["project_id"]) {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
project_id: parsed["project_id"],
|
|
322
|
+
name: typeof parsed["name"] === "string" ? parsed["name"] : undefined
|
|
323
|
+
};
|
|
324
|
+
} catch {
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
194
327
|
}
|
|
195
|
-
function
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
328
|
+
function detectProject(directory) {
|
|
329
|
+
const remoteUrl = getGitRemoteUrl(directory);
|
|
330
|
+
if (remoteUrl) {
|
|
331
|
+
const canonicalId = normaliseGitRemoteUrl(remoteUrl);
|
|
332
|
+
return {
|
|
333
|
+
canonical_id: canonicalId,
|
|
334
|
+
name: projectNameFromCanonicalId(canonicalId),
|
|
335
|
+
remote_url: remoteUrl,
|
|
336
|
+
local_path: directory
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
const configFile = readProjectConfigFile(directory);
|
|
340
|
+
if (configFile) {
|
|
341
|
+
return {
|
|
342
|
+
canonical_id: configFile.project_id,
|
|
343
|
+
name: configFile.name ?? projectNameFromCanonicalId(configFile.project_id),
|
|
344
|
+
remote_url: null,
|
|
345
|
+
local_path: directory
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const dirName = basename(directory);
|
|
349
|
+
return {
|
|
350
|
+
canonical_id: `local/${dirName}`,
|
|
351
|
+
name: dirName,
|
|
352
|
+
remote_url: null,
|
|
353
|
+
local_path: directory
|
|
354
|
+
};
|
|
199
355
|
}
|
|
200
356
|
|
|
201
|
-
// src/
|
|
202
|
-
var
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
210
|
-
canonical_id TEXT UNIQUE NOT NULL,
|
|
211
|
-
name TEXT NOT NULL,
|
|
212
|
-
local_path TEXT,
|
|
213
|
-
remote_url TEXT,
|
|
214
|
-
first_seen_epoch INTEGER NOT NULL,
|
|
215
|
-
last_active_epoch INTEGER NOT NULL
|
|
216
|
-
);
|
|
217
|
-
|
|
218
|
-
-- Core observations table
|
|
219
|
-
CREATE TABLE observations (
|
|
220
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
221
|
-
session_id TEXT,
|
|
222
|
-
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
223
|
-
type TEXT NOT NULL CHECK (type IN (
|
|
224
|
-
'bugfix', 'discovery', 'decision', 'pattern',
|
|
225
|
-
'change', 'feature', 'refactor', 'digest'
|
|
226
|
-
)),
|
|
227
|
-
title TEXT NOT NULL,
|
|
228
|
-
narrative TEXT,
|
|
229
|
-
facts TEXT,
|
|
230
|
-
concepts TEXT,
|
|
231
|
-
files_read TEXT,
|
|
232
|
-
files_modified TEXT,
|
|
233
|
-
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
234
|
-
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN (
|
|
235
|
-
'active', 'aging', 'archived', 'purged', 'pinned'
|
|
236
|
-
)),
|
|
237
|
-
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN (
|
|
238
|
-
'shared', 'personal', 'secret'
|
|
239
|
-
)),
|
|
240
|
-
user_id TEXT NOT NULL,
|
|
241
|
-
device_id TEXT NOT NULL,
|
|
242
|
-
agent TEXT DEFAULT 'claude-code',
|
|
243
|
-
created_at TEXT NOT NULL,
|
|
244
|
-
created_at_epoch INTEGER NOT NULL,
|
|
245
|
-
archived_at_epoch INTEGER,
|
|
246
|
-
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL
|
|
247
|
-
);
|
|
248
|
-
|
|
249
|
-
-- Session tracking
|
|
250
|
-
CREATE TABLE sessions (
|
|
251
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
252
|
-
session_id TEXT UNIQUE NOT NULL,
|
|
253
|
-
project_id INTEGER REFERENCES projects(id),
|
|
254
|
-
user_id TEXT NOT NULL,
|
|
255
|
-
device_id TEXT NOT NULL,
|
|
256
|
-
agent TEXT DEFAULT 'claude-code',
|
|
257
|
-
status TEXT DEFAULT 'active' CHECK (status IN ('active', 'completed')),
|
|
258
|
-
observation_count INTEGER DEFAULT 0,
|
|
259
|
-
started_at_epoch INTEGER,
|
|
260
|
-
completed_at_epoch INTEGER
|
|
261
|
-
);
|
|
262
|
-
|
|
263
|
-
-- Session summaries (generated on Stop hook)
|
|
264
|
-
CREATE TABLE session_summaries (
|
|
265
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
266
|
-
session_id TEXT UNIQUE NOT NULL,
|
|
267
|
-
project_id INTEGER REFERENCES projects(id),
|
|
268
|
-
user_id TEXT NOT NULL,
|
|
269
|
-
request TEXT,
|
|
270
|
-
investigated TEXT,
|
|
271
|
-
learned TEXT,
|
|
272
|
-
completed TEXT,
|
|
273
|
-
next_steps TEXT,
|
|
274
|
-
created_at_epoch INTEGER
|
|
275
|
-
);
|
|
276
|
-
|
|
277
|
-
-- Sync outbox (offline-first queue)
|
|
278
|
-
CREATE TABLE sync_outbox (
|
|
279
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
280
|
-
record_type TEXT NOT NULL CHECK (record_type IN ('observation', 'summary')),
|
|
281
|
-
record_id INTEGER NOT NULL,
|
|
282
|
-
status TEXT DEFAULT 'pending' CHECK (status IN (
|
|
283
|
-
'pending', 'syncing', 'synced', 'failed'
|
|
284
|
-
)),
|
|
285
|
-
retry_count INTEGER DEFAULT 0,
|
|
286
|
-
max_retries INTEGER DEFAULT 10,
|
|
287
|
-
last_error TEXT,
|
|
288
|
-
created_at_epoch INTEGER NOT NULL,
|
|
289
|
-
synced_at_epoch INTEGER,
|
|
290
|
-
next_retry_epoch INTEGER
|
|
291
|
-
);
|
|
292
|
-
|
|
293
|
-
-- Sync high-water mark and lifecycle job tracking
|
|
294
|
-
CREATE TABLE sync_state (
|
|
295
|
-
key TEXT PRIMARY KEY,
|
|
296
|
-
value TEXT NOT NULL
|
|
297
|
-
);
|
|
298
|
-
|
|
299
|
-
-- FTS5 for local offline search (external content mode)
|
|
300
|
-
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
301
|
-
title, narrative, facts, concepts,
|
|
302
|
-
content=observations,
|
|
303
|
-
content_rowid=id
|
|
304
|
-
);
|
|
305
|
-
|
|
306
|
-
-- Indexes: observations
|
|
307
|
-
CREATE INDEX idx_observations_project ON observations(project_id);
|
|
308
|
-
CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
|
|
309
|
-
CREATE INDEX idx_observations_type ON observations(type);
|
|
310
|
-
CREATE INDEX idx_observations_created ON observations(created_at_epoch);
|
|
311
|
-
CREATE INDEX idx_observations_session ON observations(session_id);
|
|
312
|
-
CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
|
|
313
|
-
CREATE INDEX idx_observations_quality ON observations(quality);
|
|
314
|
-
CREATE INDEX idx_observations_user ON observations(user_id);
|
|
315
|
-
|
|
316
|
-
-- Indexes: sessions
|
|
317
|
-
CREATE INDEX idx_sessions_project ON sessions(project_id);
|
|
318
|
-
|
|
319
|
-
-- Indexes: sync outbox
|
|
320
|
-
CREATE INDEX idx_outbox_status ON sync_outbox(status, next_retry_epoch);
|
|
321
|
-
CREATE INDEX idx_outbox_record ON sync_outbox(record_type, record_id);
|
|
322
|
-
`
|
|
323
|
-
},
|
|
324
|
-
{
|
|
325
|
-
version: 2,
|
|
326
|
-
description: "Add superseded_by for knowledge supersession",
|
|
327
|
-
sql: `
|
|
328
|
-
ALTER TABLE observations ADD COLUMN superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL;
|
|
329
|
-
CREATE INDEX idx_observations_superseded ON observations(superseded_by);
|
|
330
|
-
`
|
|
331
|
-
},
|
|
332
|
-
{
|
|
333
|
-
version: 3,
|
|
334
|
-
description: "Add remote_source_id for pull deduplication",
|
|
335
|
-
sql: `
|
|
336
|
-
ALTER TABLE observations ADD COLUMN remote_source_id TEXT;
|
|
337
|
-
CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
338
|
-
`
|
|
339
|
-
},
|
|
340
|
-
{
|
|
341
|
-
version: 4,
|
|
342
|
-
description: "Add sqlite-vec for local semantic search",
|
|
343
|
-
sql: `
|
|
344
|
-
CREATE VIRTUAL TABLE vec_observations USING vec0(
|
|
345
|
-
observation_id INTEGER PRIMARY KEY,
|
|
346
|
-
embedding float[384]
|
|
347
|
-
);
|
|
348
|
-
`,
|
|
349
|
-
condition: (db) => isVecExtensionLoaded(db)
|
|
350
|
-
},
|
|
351
|
-
{
|
|
352
|
-
version: 5,
|
|
353
|
-
description: "Session metrics and security findings",
|
|
354
|
-
sql: `
|
|
355
|
-
ALTER TABLE sessions ADD COLUMN files_touched_count INTEGER DEFAULT 0;
|
|
356
|
-
ALTER TABLE sessions ADD COLUMN searches_performed INTEGER DEFAULT 0;
|
|
357
|
-
ALTER TABLE sessions ADD COLUMN tool_calls_count INTEGER DEFAULT 0;
|
|
358
|
-
|
|
359
|
-
CREATE TABLE security_findings (
|
|
360
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
361
|
-
session_id TEXT,
|
|
362
|
-
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
363
|
-
finding_type TEXT NOT NULL,
|
|
364
|
-
severity TEXT NOT NULL DEFAULT 'medium' CHECK (severity IN ('critical', 'high', 'medium', 'low')),
|
|
365
|
-
pattern_name TEXT NOT NULL,
|
|
366
|
-
file_path TEXT,
|
|
367
|
-
snippet TEXT,
|
|
368
|
-
tool_name TEXT,
|
|
369
|
-
user_id TEXT NOT NULL,
|
|
370
|
-
device_id TEXT NOT NULL,
|
|
371
|
-
created_at_epoch INTEGER NOT NULL
|
|
372
|
-
);
|
|
373
|
-
|
|
374
|
-
CREATE INDEX idx_security_findings_session ON security_findings(session_id);
|
|
375
|
-
CREATE INDEX idx_security_findings_project ON security_findings(project_id, created_at_epoch);
|
|
376
|
-
CREATE INDEX idx_security_findings_severity ON security_findings(severity);
|
|
377
|
-
`
|
|
378
|
-
},
|
|
379
|
-
{
|
|
380
|
-
version: 6,
|
|
381
|
-
description: "Add risk_score, expand observation types to include standard",
|
|
382
|
-
sql: `
|
|
383
|
-
ALTER TABLE sessions ADD COLUMN risk_score INTEGER;
|
|
384
|
-
|
|
385
|
-
-- Recreate observations table with expanded type CHECK to include 'standard'
|
|
386
|
-
-- SQLite doesn't support ALTER CHECK, so we recreate the table
|
|
387
|
-
CREATE TABLE observations_new (
|
|
388
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
389
|
-
session_id TEXT,
|
|
390
|
-
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
391
|
-
type TEXT NOT NULL CHECK (type IN (
|
|
392
|
-
'bugfix', 'discovery', 'decision', 'pattern',
|
|
393
|
-
'change', 'feature', 'refactor', 'digest', 'standard'
|
|
394
|
-
)),
|
|
395
|
-
title TEXT NOT NULL,
|
|
396
|
-
narrative TEXT,
|
|
397
|
-
facts TEXT,
|
|
398
|
-
concepts TEXT,
|
|
399
|
-
files_read TEXT,
|
|
400
|
-
files_modified TEXT,
|
|
401
|
-
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
402
|
-
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN (
|
|
403
|
-
'active', 'aging', 'archived', 'purged', 'pinned'
|
|
404
|
-
)),
|
|
405
|
-
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN (
|
|
406
|
-
'shared', 'personal', 'secret'
|
|
407
|
-
)),
|
|
408
|
-
user_id TEXT NOT NULL,
|
|
409
|
-
device_id TEXT NOT NULL,
|
|
410
|
-
agent TEXT DEFAULT 'claude-code',
|
|
411
|
-
created_at TEXT NOT NULL,
|
|
412
|
-
created_at_epoch INTEGER NOT NULL,
|
|
413
|
-
archived_at_epoch INTEGER,
|
|
414
|
-
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
415
|
-
superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
416
|
-
remote_source_id TEXT
|
|
417
|
-
);
|
|
418
|
-
|
|
419
|
-
INSERT INTO observations_new SELECT * FROM observations;
|
|
420
|
-
|
|
421
|
-
DROP TABLE observations;
|
|
422
|
-
ALTER TABLE observations_new RENAME TO observations;
|
|
423
|
-
|
|
424
|
-
-- Recreate indexes
|
|
425
|
-
CREATE INDEX idx_observations_project ON observations(project_id);
|
|
426
|
-
CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
|
|
427
|
-
CREATE INDEX idx_observations_type ON observations(type);
|
|
428
|
-
CREATE INDEX idx_observations_created ON observations(created_at_epoch);
|
|
429
|
-
CREATE INDEX idx_observations_session ON observations(session_id);
|
|
430
|
-
CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
|
|
431
|
-
CREATE INDEX idx_observations_quality ON observations(quality);
|
|
432
|
-
CREATE INDEX idx_observations_user ON observations(user_id);
|
|
433
|
-
CREATE INDEX idx_observations_superseded ON observations(superseded_by);
|
|
434
|
-
CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
435
|
-
|
|
436
|
-
-- Recreate FTS5 (external content mode — must rebuild after table recreation)
|
|
437
|
-
DROP TABLE IF EXISTS observations_fts;
|
|
438
|
-
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
439
|
-
title, narrative, facts, concepts,
|
|
440
|
-
content=observations,
|
|
441
|
-
content_rowid=id
|
|
442
|
-
);
|
|
443
|
-
-- Rebuild FTS index
|
|
444
|
-
INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
|
|
445
|
-
`
|
|
446
|
-
},
|
|
447
|
-
{
|
|
448
|
-
version: 7,
|
|
449
|
-
description: "Add packs_installed table for help pack tracking",
|
|
450
|
-
sql: `
|
|
451
|
-
CREATE TABLE IF NOT EXISTS packs_installed (
|
|
452
|
-
name TEXT PRIMARY KEY,
|
|
453
|
-
installed_at INTEGER NOT NULL,
|
|
454
|
-
observation_count INTEGER DEFAULT 0
|
|
455
|
-
);
|
|
456
|
-
`
|
|
457
|
-
},
|
|
458
|
-
{
|
|
459
|
-
version: 8,
|
|
460
|
-
description: "Add message type to observations CHECK constraint",
|
|
461
|
-
sql: `
|
|
462
|
-
CREATE TABLE IF NOT EXISTS observations_v8 (
|
|
463
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
464
|
-
session_id TEXT,
|
|
465
|
-
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
466
|
-
type TEXT NOT NULL CHECK (type IN (
|
|
467
|
-
'bugfix', 'discovery', 'decision', 'pattern',
|
|
468
|
-
'change', 'feature', 'refactor', 'digest', 'standard', 'message'
|
|
469
|
-
)),
|
|
470
|
-
title TEXT NOT NULL,
|
|
471
|
-
narrative TEXT,
|
|
472
|
-
facts TEXT,
|
|
473
|
-
concepts TEXT,
|
|
474
|
-
files_read TEXT,
|
|
475
|
-
files_modified TEXT,
|
|
476
|
-
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
477
|
-
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN (
|
|
478
|
-
'active', 'aging', 'archived', 'purged', 'pinned'
|
|
479
|
-
)),
|
|
480
|
-
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN (
|
|
481
|
-
'shared', 'personal', 'secret'
|
|
482
|
-
)),
|
|
483
|
-
user_id TEXT NOT NULL,
|
|
484
|
-
device_id TEXT NOT NULL,
|
|
485
|
-
agent TEXT DEFAULT 'claude-code',
|
|
486
|
-
created_at TEXT NOT NULL,
|
|
487
|
-
created_at_epoch INTEGER NOT NULL,
|
|
488
|
-
archived_at_epoch INTEGER,
|
|
489
|
-
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
490
|
-
superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
491
|
-
remote_source_id TEXT
|
|
492
|
-
);
|
|
493
|
-
INSERT INTO observations_v8 SELECT * FROM observations;
|
|
494
|
-
DROP TABLE observations;
|
|
495
|
-
ALTER TABLE observations_v8 RENAME TO observations;
|
|
496
|
-
CREATE INDEX idx_observations_project ON observations(project_id);
|
|
497
|
-
CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
|
|
498
|
-
CREATE INDEX idx_observations_type ON observations(type);
|
|
499
|
-
CREATE INDEX idx_observations_created ON observations(created_at_epoch);
|
|
500
|
-
CREATE INDEX idx_observations_session ON observations(session_id);
|
|
501
|
-
CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
|
|
502
|
-
CREATE INDEX idx_observations_quality ON observations(quality);
|
|
503
|
-
CREATE INDEX idx_observations_user ON observations(user_id);
|
|
504
|
-
CREATE INDEX idx_observations_superseded ON observations(superseded_by);
|
|
505
|
-
CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
506
|
-
DROP TABLE IF EXISTS observations_fts;
|
|
507
|
-
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
508
|
-
title, narrative, facts, concepts,
|
|
509
|
-
content=observations,
|
|
510
|
-
content_rowid=id
|
|
511
|
-
);
|
|
512
|
-
INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
|
|
513
|
-
`
|
|
514
|
-
}
|
|
515
|
-
];
|
|
516
|
-
function isVecExtensionLoaded(db) {
|
|
517
|
-
try {
|
|
518
|
-
db.exec("SELECT vec_version()");
|
|
519
|
-
return true;
|
|
520
|
-
} catch {
|
|
521
|
-
return false;
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
function runMigrations(db) {
|
|
525
|
-
const currentVersion = db.query("PRAGMA user_version").get();
|
|
526
|
-
let version = currentVersion.user_version;
|
|
527
|
-
for (const migration of MIGRATIONS) {
|
|
528
|
-
if (migration.version <= version)
|
|
529
|
-
continue;
|
|
530
|
-
if (migration.condition && !migration.condition(db)) {
|
|
531
|
-
continue;
|
|
532
|
-
}
|
|
533
|
-
db.exec("BEGIN TRANSACTION");
|
|
534
|
-
try {
|
|
535
|
-
db.exec(migration.sql);
|
|
536
|
-
db.exec(`PRAGMA user_version = ${migration.version}`);
|
|
537
|
-
db.exec("COMMIT");
|
|
538
|
-
version = migration.version;
|
|
539
|
-
} catch (error) {
|
|
540
|
-
db.exec("ROLLBACK");
|
|
541
|
-
throw new Error(`Migration ${migration.version} (${migration.description}) failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
|
-
function ensureObservationTypes(db) {
|
|
357
|
+
// src/embeddings/embedder.ts
|
|
358
|
+
var _available = null;
|
|
359
|
+
var _pipeline = null;
|
|
360
|
+
var MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
361
|
+
async function embedText(text) {
|
|
362
|
+
const pipe = await getPipeline();
|
|
363
|
+
if (!pipe)
|
|
364
|
+
return null;
|
|
546
365
|
try {
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
} catch {
|
|
550
|
-
|
|
551
|
-
try {
|
|
552
|
-
db.exec(`
|
|
553
|
-
CREATE TABLE observations_repair (
|
|
554
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT,
|
|
555
|
-
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
556
|
-
type TEXT NOT NULL CHECK (type IN (
|
|
557
|
-
'bugfix','discovery','decision','pattern','change','feature',
|
|
558
|
-
'refactor','digest','standard','message')),
|
|
559
|
-
title TEXT NOT NULL, narrative TEXT, facts TEXT, concepts TEXT,
|
|
560
|
-
files_read TEXT, files_modified TEXT,
|
|
561
|
-
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
562
|
-
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN ('active','aging','archived','purged','pinned')),
|
|
563
|
-
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN ('shared','personal','secret')),
|
|
564
|
-
user_id TEXT NOT NULL, device_id TEXT NOT NULL, agent TEXT DEFAULT 'claude-code',
|
|
565
|
-
created_at TEXT NOT NULL, created_at_epoch INTEGER NOT NULL,
|
|
566
|
-
archived_at_epoch INTEGER,
|
|
567
|
-
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
568
|
-
superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
569
|
-
remote_source_id TEXT
|
|
570
|
-
);
|
|
571
|
-
INSERT INTO observations_repair SELECT * FROM observations;
|
|
572
|
-
DROP TABLE observations;
|
|
573
|
-
ALTER TABLE observations_repair RENAME TO observations;
|
|
574
|
-
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id);
|
|
575
|
-
CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
|
|
576
|
-
CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at_epoch);
|
|
577
|
-
CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
|
|
578
|
-
CREATE INDEX IF NOT EXISTS idx_observations_lifecycle ON observations(lifecycle);
|
|
579
|
-
CREATE INDEX IF NOT EXISTS idx_observations_quality ON observations(quality);
|
|
580
|
-
CREATE INDEX IF NOT EXISTS idx_observations_user ON observations(user_id);
|
|
581
|
-
CREATE INDEX IF NOT EXISTS idx_observations_superseded ON observations(superseded_by);
|
|
582
|
-
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
583
|
-
DROP TABLE IF EXISTS observations_fts;
|
|
584
|
-
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
585
|
-
title, narrative, facts, concepts, content=observations, content_rowid=id
|
|
586
|
-
);
|
|
587
|
-
INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
|
|
588
|
-
`);
|
|
589
|
-
db.exec("COMMIT");
|
|
590
|
-
} catch (err) {
|
|
591
|
-
db.exec("ROLLBACK");
|
|
592
|
-
}
|
|
593
|
-
}
|
|
594
|
-
}
|
|
595
|
-
var LATEST_SCHEMA_VERSION = MIGRATIONS.filter((m) => !m.condition).reduce((max, m) => Math.max(max, m.version), 0);
|
|
596
|
-
|
|
597
|
-
// src/storage/sqlite.ts
|
|
598
|
-
var IS_BUN = typeof globalThis.Bun !== "undefined";
|
|
599
|
-
function openDatabase(dbPath) {
|
|
600
|
-
if (IS_BUN) {
|
|
601
|
-
return openBunDatabase(dbPath);
|
|
602
|
-
}
|
|
603
|
-
return openNodeDatabase(dbPath);
|
|
604
|
-
}
|
|
605
|
-
function openBunDatabase(dbPath) {
|
|
606
|
-
const { Database } = __require("bun:sqlite");
|
|
607
|
-
if (process.platform === "darwin") {
|
|
608
|
-
const { existsSync: existsSync2 } = __require("node:fs");
|
|
609
|
-
const paths = [
|
|
610
|
-
"/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib",
|
|
611
|
-
"/usr/local/opt/sqlite3/lib/libsqlite3.dylib"
|
|
612
|
-
];
|
|
613
|
-
for (const p of paths) {
|
|
614
|
-
if (existsSync2(p)) {
|
|
615
|
-
try {
|
|
616
|
-
Database.setCustomSQLite(p);
|
|
617
|
-
} catch {}
|
|
618
|
-
break;
|
|
619
|
-
}
|
|
620
|
-
}
|
|
366
|
+
const output = await pipe(text, { pooling: "mean", normalize: true });
|
|
367
|
+
return new Float32Array(output.data);
|
|
368
|
+
} catch {
|
|
369
|
+
return null;
|
|
621
370
|
}
|
|
622
|
-
const db = new Database(dbPath);
|
|
623
|
-
return db;
|
|
624
|
-
}
|
|
625
|
-
function openNodeDatabase(dbPath) {
|
|
626
|
-
const BetterSqlite3 = __require("better-sqlite3");
|
|
627
|
-
const raw = new BetterSqlite3(dbPath);
|
|
628
|
-
return {
|
|
629
|
-
query(sql) {
|
|
630
|
-
const stmt = raw.prepare(sql);
|
|
631
|
-
return {
|
|
632
|
-
get(...params) {
|
|
633
|
-
return stmt.get(...params);
|
|
634
|
-
},
|
|
635
|
-
all(...params) {
|
|
636
|
-
return stmt.all(...params);
|
|
637
|
-
},
|
|
638
|
-
run(...params) {
|
|
639
|
-
return stmt.run(...params);
|
|
640
|
-
}
|
|
641
|
-
};
|
|
642
|
-
},
|
|
643
|
-
exec(sql) {
|
|
644
|
-
raw.exec(sql);
|
|
645
|
-
},
|
|
646
|
-
close() {
|
|
647
|
-
raw.close();
|
|
648
|
-
}
|
|
649
|
-
};
|
|
650
371
|
}
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
this.db = openDatabase(dbPath);
|
|
657
|
-
this.db.exec("PRAGMA journal_mode = WAL");
|
|
658
|
-
this.db.exec("PRAGMA foreign_keys = ON");
|
|
659
|
-
this.vecAvailable = this.loadVecExtension();
|
|
660
|
-
runMigrations(this.db);
|
|
661
|
-
ensureObservationTypes(this.db);
|
|
662
|
-
}
|
|
663
|
-
loadVecExtension() {
|
|
372
|
+
function composeEmbeddingText(obs) {
|
|
373
|
+
const parts = [obs.title];
|
|
374
|
+
if (obs.narrative)
|
|
375
|
+
parts.push(obs.narrative);
|
|
376
|
+
if (obs.facts) {
|
|
664
377
|
try {
|
|
665
|
-
const
|
|
666
|
-
|
|
667
|
-
|
|
378
|
+
const facts = JSON.parse(obs.facts);
|
|
379
|
+
if (Array.isArray(facts) && facts.length > 0) {
|
|
380
|
+
parts.push(facts.map((f) => `- ${f}`).join(`
|
|
381
|
+
`));
|
|
382
|
+
}
|
|
668
383
|
} catch {
|
|
669
|
-
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
close() {
|
|
673
|
-
this.db.close();
|
|
674
|
-
}
|
|
675
|
-
upsertProject(project) {
|
|
676
|
-
const now = Math.floor(Date.now() / 1000);
|
|
677
|
-
const existing = this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(project.canonical_id);
|
|
678
|
-
if (existing) {
|
|
679
|
-
this.db.query(`UPDATE projects SET
|
|
680
|
-
local_path = COALESCE(?, local_path),
|
|
681
|
-
remote_url = COALESCE(?, remote_url),
|
|
682
|
-
last_active_epoch = ?
|
|
683
|
-
WHERE id = ?`).run(project.local_path ?? null, project.remote_url ?? null, now, existing.id);
|
|
684
|
-
return {
|
|
685
|
-
...existing,
|
|
686
|
-
local_path: project.local_path ?? existing.local_path,
|
|
687
|
-
remote_url: project.remote_url ?? existing.remote_url,
|
|
688
|
-
last_active_epoch: now
|
|
689
|
-
};
|
|
690
|
-
}
|
|
691
|
-
const result = this.db.query(`INSERT INTO projects (canonical_id, name, local_path, remote_url, first_seen_epoch, last_active_epoch)
|
|
692
|
-
VALUES (?, ?, ?, ?, ?, ?)`).run(project.canonical_id, project.name, project.local_path ?? null, project.remote_url ?? null, now, now);
|
|
693
|
-
return this.db.query("SELECT * FROM projects WHERE id = ?").get(Number(result.lastInsertRowid));
|
|
694
|
-
}
|
|
695
|
-
getProjectByCanonicalId(canonicalId) {
|
|
696
|
-
return this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(canonicalId) ?? null;
|
|
697
|
-
}
|
|
698
|
-
getProjectById(id) {
|
|
699
|
-
return this.db.query("SELECT * FROM projects WHERE id = ?").get(id) ?? null;
|
|
700
|
-
}
|
|
701
|
-
insertObservation(obs) {
|
|
702
|
-
const now = Math.floor(Date.now() / 1000);
|
|
703
|
-
const createdAt = new Date().toISOString();
|
|
704
|
-
const result = this.db.query(`INSERT INTO observations (
|
|
705
|
-
session_id, project_id, type, title, narrative, facts, concepts,
|
|
706
|
-
files_read, files_modified, quality, lifecycle, sensitivity,
|
|
707
|
-
user_id, device_id, agent, created_at, created_at_epoch
|
|
708
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(obs.session_id ?? null, obs.project_id, obs.type, obs.title, obs.narrative ?? null, obs.facts ?? null, obs.concepts ?? null, obs.files_read ?? null, obs.files_modified ?? null, obs.quality, obs.lifecycle ?? "active", obs.sensitivity ?? "shared", obs.user_id, obs.device_id, obs.agent ?? "claude-code", createdAt, now);
|
|
709
|
-
const id = Number(result.lastInsertRowid);
|
|
710
|
-
const row = this.getObservationById(id);
|
|
711
|
-
this.ftsInsert(row);
|
|
712
|
-
if (obs.session_id) {
|
|
713
|
-
this.db.query("UPDATE sessions SET observation_count = observation_count + 1 WHERE session_id = ?").run(obs.session_id);
|
|
714
|
-
}
|
|
715
|
-
return row;
|
|
716
|
-
}
|
|
717
|
-
getObservationById(id) {
|
|
718
|
-
return this.db.query("SELECT * FROM observations WHERE id = ?").get(id) ?? null;
|
|
719
|
-
}
|
|
720
|
-
getObservationsByIds(ids) {
|
|
721
|
-
if (ids.length === 0)
|
|
722
|
-
return [];
|
|
723
|
-
const placeholders = ids.map(() => "?").join(",");
|
|
724
|
-
return this.db.query(`SELECT * FROM observations WHERE id IN (${placeholders}) ORDER BY created_at_epoch DESC`).all(...ids);
|
|
725
|
-
}
|
|
726
|
-
getRecentObservations(projectId, sincEpoch, limit = 50) {
|
|
727
|
-
return this.db.query(`SELECT * FROM observations
|
|
728
|
-
WHERE project_id = ? AND created_at_epoch > ?
|
|
729
|
-
ORDER BY created_at_epoch DESC
|
|
730
|
-
LIMIT ?`).all(projectId, sincEpoch, limit);
|
|
731
|
-
}
|
|
732
|
-
searchFts(query, projectId, lifecycles = ["active", "aging", "pinned"], limit = 20) {
|
|
733
|
-
const lifecyclePlaceholders = lifecycles.map(() => "?").join(",");
|
|
734
|
-
if (projectId !== null) {
|
|
735
|
-
return this.db.query(`SELECT o.id, observations_fts.rank
|
|
736
|
-
FROM observations_fts
|
|
737
|
-
JOIN observations o ON o.id = observations_fts.rowid
|
|
738
|
-
WHERE observations_fts MATCH ?
|
|
739
|
-
AND o.project_id = ?
|
|
740
|
-
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
741
|
-
ORDER BY observations_fts.rank
|
|
742
|
-
LIMIT ?`).all(query, projectId, ...lifecycles, limit);
|
|
384
|
+
parts.push(obs.facts);
|
|
743
385
|
}
|
|
744
|
-
return this.db.query(`SELECT o.id, observations_fts.rank
|
|
745
|
-
FROM observations_fts
|
|
746
|
-
JOIN observations o ON o.id = observations_fts.rowid
|
|
747
|
-
WHERE observations_fts MATCH ?
|
|
748
|
-
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
749
|
-
ORDER BY observations_fts.rank
|
|
750
|
-
LIMIT ?`).all(query, ...lifecycles, limit);
|
|
751
|
-
}
|
|
752
|
-
getTimeline(anchorId, projectId, depthBefore = 3, depthAfter = 3) {
|
|
753
|
-
const anchor = this.getObservationById(anchorId);
|
|
754
|
-
if (!anchor)
|
|
755
|
-
return [];
|
|
756
|
-
const projectFilter = projectId !== null ? "AND project_id = ?" : "";
|
|
757
|
-
const projectParams = projectId !== null ? [projectId] : [];
|
|
758
|
-
const before = this.db.query(`SELECT * FROM observations
|
|
759
|
-
WHERE created_at_epoch < ? ${projectFilter}
|
|
760
|
-
AND lifecycle IN ('active', 'aging', 'pinned')
|
|
761
|
-
ORDER BY created_at_epoch DESC
|
|
762
|
-
LIMIT ?`).all(anchor.created_at_epoch, ...projectParams, depthBefore);
|
|
763
|
-
const after = this.db.query(`SELECT * FROM observations
|
|
764
|
-
WHERE created_at_epoch > ? ${projectFilter}
|
|
765
|
-
AND lifecycle IN ('active', 'aging', 'pinned')
|
|
766
|
-
ORDER BY created_at_epoch ASC
|
|
767
|
-
LIMIT ?`).all(anchor.created_at_epoch, ...projectParams, depthAfter);
|
|
768
|
-
return [...before.reverse(), anchor, ...after];
|
|
769
386
|
}
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
this.db.query("UPDATE observations SET lifecycle = 'pinned' WHERE id = ?").run(id);
|
|
778
|
-
} else {
|
|
779
|
-
if (obs.lifecycle !== "pinned")
|
|
780
|
-
return false;
|
|
781
|
-
this.db.query("UPDATE observations SET lifecycle = 'active' WHERE id = ?").run(id);
|
|
782
|
-
}
|
|
783
|
-
return true;
|
|
387
|
+
if (obs.concepts) {
|
|
388
|
+
try {
|
|
389
|
+
const concepts = JSON.parse(obs.concepts);
|
|
390
|
+
if (Array.isArray(concepts) && concepts.length > 0) {
|
|
391
|
+
parts.push(concepts.join(", "));
|
|
392
|
+
}
|
|
393
|
+
} catch {}
|
|
784
394
|
}
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
395
|
+
return parts.join(`
|
|
396
|
+
|
|
397
|
+
`);
|
|
398
|
+
}
|
|
399
|
+
async function getPipeline() {
|
|
400
|
+
if (_pipeline)
|
|
401
|
+
return _pipeline;
|
|
402
|
+
if (_available === false)
|
|
403
|
+
return null;
|
|
404
|
+
try {
|
|
405
|
+
const { pipeline } = await import("@xenova/transformers");
|
|
406
|
+
_pipeline = await pipeline("feature-extraction", MODEL_NAME);
|
|
407
|
+
_available = true;
|
|
408
|
+
return _pipeline;
|
|
409
|
+
} catch (err) {
|
|
410
|
+
_available = false;
|
|
411
|
+
console.error(`[engrm] Local embedding model unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
412
|
+
return null;
|
|
797
413
|
}
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
const visited = new Set;
|
|
806
|
-
for (let depth = 0;depth < 10; depth++) {
|
|
807
|
-
const target2 = this.getObservationById(targetId);
|
|
808
|
-
if (!target2)
|
|
809
|
-
return false;
|
|
810
|
-
if (target2.superseded_by === null)
|
|
811
|
-
break;
|
|
812
|
-
if (target2.superseded_by === newId)
|
|
813
|
-
return true;
|
|
814
|
-
visited.add(targetId);
|
|
815
|
-
targetId = target2.superseded_by;
|
|
816
|
-
if (visited.has(targetId))
|
|
817
|
-
return false;
|
|
818
|
-
}
|
|
819
|
-
const target = this.getObservationById(targetId);
|
|
820
|
-
if (!target)
|
|
821
|
-
return false;
|
|
822
|
-
if (target.superseded_by !== null)
|
|
823
|
-
return false;
|
|
824
|
-
if (targetId === newId)
|
|
825
|
-
return false;
|
|
826
|
-
const now = Math.floor(Date.now() / 1000);
|
|
827
|
-
this.db.query(`UPDATE observations
|
|
828
|
-
SET superseded_by = ?, lifecycle = 'archived', archived_at_epoch = ?
|
|
829
|
-
WHERE id = ?`).run(newId, now, targetId);
|
|
830
|
-
this.ftsDelete(target);
|
|
831
|
-
this.vecDelete(targetId);
|
|
832
|
-
return true;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// src/capture/recurrence.ts
|
|
417
|
+
var DISTANCE_THRESHOLD = 0.15;
|
|
418
|
+
async function detectRecurrence(db, config, observation) {
|
|
419
|
+
if (observation.type !== "bugfix") {
|
|
420
|
+
return { patternCreated: false };
|
|
833
421
|
}
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
return obs !== null && obs.superseded_by !== null;
|
|
422
|
+
if (!db.vecAvailable) {
|
|
423
|
+
return { patternCreated: false };
|
|
837
424
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
const now = Math.floor(Date.now() / 1000);
|
|
843
|
-
this.db.query(`INSERT INTO sessions (session_id, project_id, user_id, device_id, agent, started_at_epoch)
|
|
844
|
-
VALUES (?, ?, ?, ?, ?, ?)`).run(sessionId, projectId, userId, deviceId, agent, now);
|
|
845
|
-
return this.db.query("SELECT * FROM sessions WHERE session_id = ?").get(sessionId);
|
|
425
|
+
const text = composeEmbeddingText(observation);
|
|
426
|
+
const embedding = await embedText(text);
|
|
427
|
+
if (!embedding) {
|
|
428
|
+
return { patternCreated: false };
|
|
846
429
|
}
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
430
|
+
const vecResults = db.searchVec(embedding, null, ["active", "aging", "pinned"], 10);
|
|
431
|
+
for (const match of vecResults) {
|
|
432
|
+
if (match.observation_id === observation.id)
|
|
433
|
+
continue;
|
|
434
|
+
if (match.distance > DISTANCE_THRESHOLD)
|
|
435
|
+
continue;
|
|
436
|
+
const matched = db.getObservationById(match.observation_id);
|
|
437
|
+
if (!matched)
|
|
438
|
+
continue;
|
|
439
|
+
if (matched.type !== "bugfix")
|
|
440
|
+
continue;
|
|
441
|
+
if (matched.session_id === observation.session_id)
|
|
442
|
+
continue;
|
|
443
|
+
if (await patternAlreadyExists(db, observation, matched))
|
|
444
|
+
continue;
|
|
445
|
+
let matchedProjectName;
|
|
446
|
+
if (matched.project_id !== observation.project_id) {
|
|
447
|
+
const proj = db.getProjectById(matched.project_id);
|
|
448
|
+
if (proj)
|
|
449
|
+
matchedProjectName = proj.name;
|
|
450
|
+
}
|
|
451
|
+
const similarity = 1 - match.distance;
|
|
452
|
+
const result = await saveObservation(db, config, {
|
|
453
|
+
type: "pattern",
|
|
454
|
+
title: `Recurring bugfix: ${observation.title}`,
|
|
455
|
+
narrative: `This bug pattern has appeared in multiple sessions. Original: "${matched.title}" (session ${matched.session_id?.slice(0, 8) ?? "unknown"}). Latest: "${observation.title}". Similarity: ${(similarity * 100).toFixed(0)}%. Consider addressing the root cause.`,
|
|
456
|
+
facts: [
|
|
457
|
+
`First seen: ${matched.created_at.split("T")[0]}`,
|
|
458
|
+
`Recurred: ${observation.created_at.split("T")[0]}`,
|
|
459
|
+
`Similarity: ${(similarity * 100).toFixed(0)}%`
|
|
460
|
+
],
|
|
461
|
+
concepts: mergeConceptsFromBoth(observation, matched),
|
|
462
|
+
cwd: process.cwd(),
|
|
463
|
+
session_id: observation.session_id ?? undefined
|
|
464
|
+
});
|
|
465
|
+
if (result.success && result.observation_id) {
|
|
466
|
+
return {
|
|
467
|
+
patternCreated: true,
|
|
468
|
+
patternId: result.observation_id,
|
|
469
|
+
matchedObservationId: matched.id,
|
|
470
|
+
matchedProjectName,
|
|
471
|
+
matchedTitle: matched.title,
|
|
472
|
+
similarity
|
|
473
|
+
};
|
|
474
|
+
}
|
|
850
475
|
}
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
476
|
+
return { patternCreated: false };
|
|
477
|
+
}
|
|
478
|
+
async function patternAlreadyExists(db, obs1, obs2) {
|
|
479
|
+
const recentPatterns = db.db.query(`SELECT * FROM observations
|
|
480
|
+
WHERE type = 'pattern' AND lifecycle IN ('active', 'aging', 'pinned')
|
|
481
|
+
AND title LIKE ?
|
|
482
|
+
ORDER BY created_at_epoch DESC LIMIT 5`).all(`%${obs1.title.slice(0, 30)}%`);
|
|
483
|
+
for (const p of recentPatterns) {
|
|
484
|
+
if (p.narrative?.includes(obs2.title.slice(0, 30)))
|
|
485
|
+
return true;
|
|
855
486
|
}
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
487
|
+
return false;
|
|
488
|
+
}
|
|
489
|
+
function mergeConceptsFromBoth(obs1, obs2) {
|
|
490
|
+
const concepts = new Set;
|
|
491
|
+
for (const obs of [obs1, obs2]) {
|
|
492
|
+
if (obs.concepts) {
|
|
493
|
+
try {
|
|
494
|
+
const parsed = JSON.parse(obs.concepts);
|
|
495
|
+
if (Array.isArray(parsed)) {
|
|
496
|
+
for (const c of parsed) {
|
|
497
|
+
if (typeof c === "string")
|
|
498
|
+
concepts.add(c);
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
} catch {}
|
|
502
|
+
}
|
|
859
503
|
}
|
|
860
|
-
|
|
861
|
-
|
|
504
|
+
return [...concepts];
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// src/capture/conflict.ts
|
|
508
|
+
var SIMILARITY_THRESHOLD = 0.25;
|
|
509
|
+
async function detectDecisionConflict(db, observation) {
|
|
510
|
+
if (observation.type !== "decision") {
|
|
511
|
+
return { hasConflict: false };
|
|
862
512
|
}
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
VALUES (?, ?, ?, ?, ?)`).run(obs.id, obs.title, obs.narrative, obs.facts, obs.concepts);
|
|
513
|
+
if (!observation.narrative || observation.narrative.trim().length < 20) {
|
|
514
|
+
return { hasConflict: false };
|
|
866
515
|
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
VALUES ('delete', ?, ?, ?, ?, ?)`).run(obs.id, obs.title, obs.narrative, obs.facts, obs.concepts);
|
|
516
|
+
if (db.vecAvailable) {
|
|
517
|
+
return detectViaVec(db, observation);
|
|
870
518
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
519
|
+
return detectViaFts(db, observation);
|
|
520
|
+
}
|
|
521
|
+
async function detectViaVec(db, observation) {
|
|
522
|
+
const text = composeEmbeddingText(observation);
|
|
523
|
+
const embedding = await embedText(text);
|
|
524
|
+
if (!embedding)
|
|
525
|
+
return { hasConflict: false };
|
|
526
|
+
const results = db.searchVec(embedding, observation.project_id, ["active", "aging", "pinned"], 10);
|
|
527
|
+
for (const match of results) {
|
|
528
|
+
if (match.observation_id === observation.id)
|
|
529
|
+
continue;
|
|
530
|
+
if (match.distance > SIMILARITY_THRESHOLD)
|
|
531
|
+
continue;
|
|
532
|
+
const existing = db.getObservationById(match.observation_id);
|
|
533
|
+
if (!existing)
|
|
534
|
+
continue;
|
|
535
|
+
if (existing.type !== "decision")
|
|
536
|
+
continue;
|
|
537
|
+
if (!existing.narrative)
|
|
538
|
+
continue;
|
|
539
|
+
const conflict = narrativesConflict(observation.narrative, existing.narrative);
|
|
540
|
+
if (conflict) {
|
|
541
|
+
return {
|
|
542
|
+
hasConflict: true,
|
|
543
|
+
conflictingId: existing.id,
|
|
544
|
+
conflictingTitle: existing.title,
|
|
545
|
+
reason: conflict
|
|
546
|
+
};
|
|
547
|
+
}
|
|
875
548
|
}
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
549
|
+
return { hasConflict: false };
|
|
550
|
+
}
|
|
551
|
+
async function detectViaFts(db, observation) {
|
|
552
|
+
const keywords = observation.title.split(/\s+/).filter((w) => w.length > 3).slice(0, 5).join(" ");
|
|
553
|
+
if (!keywords)
|
|
554
|
+
return { hasConflict: false };
|
|
555
|
+
const ftsResults = db.searchFts(keywords, observation.project_id, ["active", "aging", "pinned"], 10);
|
|
556
|
+
for (const match of ftsResults) {
|
|
557
|
+
if (match.id === observation.id)
|
|
558
|
+
continue;
|
|
559
|
+
const existing = db.getObservationById(match.id);
|
|
560
|
+
if (!existing)
|
|
561
|
+
continue;
|
|
562
|
+
if (existing.type !== "decision")
|
|
563
|
+
continue;
|
|
564
|
+
if (!existing.narrative)
|
|
565
|
+
continue;
|
|
566
|
+
const conflict = narrativesConflict(observation.narrative, existing.narrative);
|
|
567
|
+
if (conflict) {
|
|
568
|
+
return {
|
|
569
|
+
hasConflict: true,
|
|
570
|
+
conflictingId: existing.id,
|
|
571
|
+
conflictingTitle: existing.title,
|
|
572
|
+
reason: conflict
|
|
573
|
+
};
|
|
574
|
+
}
|
|
880
575
|
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
576
|
+
return { hasConflict: false };
|
|
577
|
+
}
|
|
578
|
+
function narrativesConflict(narrative1, narrative2) {
|
|
579
|
+
const n1 = narrative1.toLowerCase();
|
|
580
|
+
const n2 = narrative2.toLowerCase();
|
|
581
|
+
const opposingPairs = [
|
|
582
|
+
[["should use", "decided to use", "chose", "prefer", "went with"], ["should not", "decided against", "avoid", "rejected", "don't use"]],
|
|
583
|
+
[["enable", "turn on", "activate", "add"], ["disable", "turn off", "deactivate", "remove"]],
|
|
584
|
+
[["increase", "more", "higher", "scale up"], ["decrease", "less", "lower", "scale down"]],
|
|
585
|
+
[["keep", "maintain", "preserve"], ["replace", "migrate", "switch from", "deprecate"]]
|
|
586
|
+
];
|
|
587
|
+
for (const [positive, negative] of opposingPairs) {
|
|
588
|
+
const n1HasPositive = positive.some((w) => n1.includes(w));
|
|
589
|
+
const n1HasNegative = negative.some((w) => n1.includes(w));
|
|
590
|
+
const n2HasPositive = positive.some((w) => n2.includes(w));
|
|
591
|
+
const n2HasNegative = negative.some((w) => n2.includes(w));
|
|
592
|
+
if (n1HasPositive && n2HasNegative || n1HasNegative && n2HasPositive) {
|
|
593
|
+
return "Narratives suggest opposing conclusions on a similar topic";
|
|
895
594
|
}
|
|
896
|
-
return this.db.query(`SELECT v.observation_id, v.distance
|
|
897
|
-
FROM vec_observations v
|
|
898
|
-
JOIN observations o ON o.id = v.observation_id
|
|
899
|
-
WHERE v.embedding MATCH ?
|
|
900
|
-
AND k = ?
|
|
901
|
-
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
902
|
-
AND o.superseded_by IS NULL`).all(embeddingBlob, limit, ...lifecycles);
|
|
903
595
|
}
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
596
|
+
return null;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// src/tools/save.ts
|
|
600
|
+
var VALID_TYPES = [
|
|
601
|
+
"bugfix",
|
|
602
|
+
"discovery",
|
|
603
|
+
"decision",
|
|
604
|
+
"pattern",
|
|
605
|
+
"change",
|
|
606
|
+
"feature",
|
|
607
|
+
"refactor",
|
|
608
|
+
"digest",
|
|
609
|
+
"standard",
|
|
610
|
+
"message"
|
|
611
|
+
];
|
|
612
|
+
async function saveObservation(db, config, input) {
|
|
613
|
+
if (!VALID_TYPES.includes(input.type)) {
|
|
614
|
+
return {
|
|
615
|
+
success: false,
|
|
616
|
+
reason: `Invalid type '${input.type}'. Must be one of: ${VALID_TYPES.join(", ")}`
|
|
617
|
+
};
|
|
914
618
|
}
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
return [];
|
|
918
|
-
return this.db.query(`SELECT o.* FROM observations o
|
|
919
|
-
WHERE o.lifecycle IN ('active', 'aging', 'pinned')
|
|
920
|
-
AND o.superseded_by IS NULL
|
|
921
|
-
AND NOT EXISTS (
|
|
922
|
-
SELECT 1 FROM vec_observations v WHERE v.observation_id = o.id
|
|
923
|
-
)
|
|
924
|
-
ORDER BY o.created_at_epoch DESC
|
|
925
|
-
LIMIT ?`).all(limit);
|
|
619
|
+
if (!input.title || input.title.trim().length === 0) {
|
|
620
|
+
return { success: false, reason: "Title is required" };
|
|
926
621
|
}
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
622
|
+
const cwd = input.cwd ?? process.cwd();
|
|
623
|
+
const detected = detectProject(cwd);
|
|
624
|
+
const project = db.upsertProject({
|
|
625
|
+
canonical_id: detected.canonical_id,
|
|
626
|
+
name: detected.name,
|
|
627
|
+
local_path: detected.local_path,
|
|
628
|
+
remote_url: detected.remote_url
|
|
629
|
+
});
|
|
630
|
+
const customPatterns = config.scrubbing.enabled ? config.scrubbing.custom_patterns : [];
|
|
631
|
+
const title = config.scrubbing.enabled ? scrubSecrets(input.title, customPatterns) : input.title;
|
|
632
|
+
const narrative = input.narrative ? config.scrubbing.enabled ? scrubSecrets(input.narrative, customPatterns) : input.narrative : null;
|
|
633
|
+
const factsJson = input.facts ? config.scrubbing.enabled ? scrubSecrets(JSON.stringify(input.facts), customPatterns) : JSON.stringify(input.facts) : null;
|
|
634
|
+
const conceptsJson = input.concepts ? JSON.stringify(input.concepts) : null;
|
|
635
|
+
const filesRead = input.files_read ? input.files_read.map((f) => toRelativePath(f, cwd)) : null;
|
|
636
|
+
const filesModified = input.files_modified ? input.files_modified.map((f) => toRelativePath(f, cwd)) : null;
|
|
637
|
+
const filesReadJson = filesRead ? JSON.stringify(filesRead) : null;
|
|
638
|
+
const filesModifiedJson = filesModified ? JSON.stringify(filesModified) : null;
|
|
639
|
+
let sensitivity = input.sensitivity ?? config.scrubbing.default_sensitivity;
|
|
640
|
+
if (config.scrubbing.enabled && containsSecrets([input.title, input.narrative, JSON.stringify(input.facts)].filter(Boolean).join(" "), customPatterns)) {
|
|
641
|
+
if (sensitivity === "shared") {
|
|
642
|
+
sensitivity = "personal";
|
|
643
|
+
}
|
|
933
644
|
}
|
|
934
|
-
|
|
935
|
-
|
|
645
|
+
const oneDayAgo = Math.floor(Date.now() / 1000) - 86400;
|
|
646
|
+
const recentObs = db.getRecentObservations(project.id, oneDayAgo);
|
|
647
|
+
const candidates = recentObs.map((o) => ({
|
|
648
|
+
id: o.id,
|
|
649
|
+
title: o.title
|
|
650
|
+
}));
|
|
651
|
+
const duplicate = findDuplicate(title, candidates);
|
|
652
|
+
const qualityInput = {
|
|
653
|
+
type: input.type,
|
|
654
|
+
title,
|
|
655
|
+
narrative,
|
|
656
|
+
facts: factsJson,
|
|
657
|
+
concepts: conceptsJson,
|
|
658
|
+
filesRead,
|
|
659
|
+
filesModified,
|
|
660
|
+
isDuplicate: duplicate !== null
|
|
661
|
+
};
|
|
662
|
+
const qualityScore = scoreQuality(qualityInput);
|
|
663
|
+
if (!meetsQualityThreshold(qualityInput)) {
|
|
664
|
+
return {
|
|
665
|
+
success: false,
|
|
666
|
+
quality_score: qualityScore,
|
|
667
|
+
reason: `Quality score ${qualityScore.toFixed(2)} below threshold`
|
|
668
|
+
};
|
|
936
669
|
}
|
|
937
|
-
|
|
938
|
-
return
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
670
|
+
if (duplicate) {
|
|
671
|
+
return {
|
|
672
|
+
success: true,
|
|
673
|
+
merged_into: duplicate.id,
|
|
674
|
+
quality_score: qualityScore,
|
|
675
|
+
reason: `Merged into existing observation #${duplicate.id}`
|
|
676
|
+
};
|
|
942
677
|
}
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
678
|
+
const obs = db.insertObservation({
|
|
679
|
+
session_id: input.session_id ?? null,
|
|
680
|
+
project_id: project.id,
|
|
681
|
+
type: input.type,
|
|
682
|
+
title,
|
|
683
|
+
narrative,
|
|
684
|
+
facts: factsJson,
|
|
685
|
+
concepts: conceptsJson,
|
|
686
|
+
files_read: filesReadJson,
|
|
687
|
+
files_modified: filesModifiedJson,
|
|
688
|
+
quality: qualityScore,
|
|
689
|
+
lifecycle: "active",
|
|
690
|
+
sensitivity,
|
|
691
|
+
user_id: config.user_id,
|
|
692
|
+
device_id: config.device_id,
|
|
693
|
+
agent: input.agent ?? "claude-code"
|
|
694
|
+
});
|
|
695
|
+
db.addToOutbox("observation", obs.id);
|
|
696
|
+
if (db.vecAvailable) {
|
|
697
|
+
try {
|
|
698
|
+
const text = composeEmbeddingText(obs);
|
|
699
|
+
const embedding = await embedText(text);
|
|
700
|
+
if (embedding) {
|
|
701
|
+
db.vecInsert(obs.id, embedding);
|
|
702
|
+
}
|
|
703
|
+
} catch {}
|
|
962
704
|
}
|
|
963
|
-
|
|
964
|
-
|
|
705
|
+
let recallHint;
|
|
706
|
+
if (input.type === "bugfix") {
|
|
707
|
+
try {
|
|
708
|
+
const recurrence = await detectRecurrence(db, config, obs);
|
|
709
|
+
if (recurrence.patternCreated && recurrence.matchedTitle) {
|
|
710
|
+
const projectLabel = recurrence.matchedProjectName ? ` in ${recurrence.matchedProjectName}` : "";
|
|
711
|
+
recallHint = `You solved a similar issue${projectLabel}: "${recurrence.matchedTitle}"`;
|
|
712
|
+
}
|
|
713
|
+
} catch {}
|
|
965
714
|
}
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
715
|
+
let conflictWarning;
|
|
716
|
+
if (input.type === "decision") {
|
|
717
|
+
try {
|
|
718
|
+
const conflict = await detectDecisionConflict(db, obs);
|
|
719
|
+
if (conflict.hasConflict && conflict.conflictingTitle) {
|
|
720
|
+
conflictWarning = `Potential conflict with existing decision: "${conflict.conflictingTitle}" — ${conflict.reason}`;
|
|
721
|
+
}
|
|
722
|
+
} catch {}
|
|
972
723
|
}
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
724
|
+
return {
|
|
725
|
+
success: true,
|
|
726
|
+
observation_id: obs.id,
|
|
727
|
+
quality_score: qualityScore,
|
|
728
|
+
recall_hint: recallHint,
|
|
729
|
+
conflict_warning: conflictWarning
|
|
730
|
+
};
|
|
731
|
+
}
|
|
732
|
+
function toRelativePath(filePath, projectRoot) {
|
|
733
|
+
if (!isAbsolute(filePath))
|
|
734
|
+
return filePath;
|
|
735
|
+
const rel = relative(projectRoot, filePath);
|
|
736
|
+
if (rel.startsWith(".."))
|
|
737
|
+
return filePath;
|
|
738
|
+
return rel;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// src/config.ts
|
|
742
|
+
import { existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "node:fs";
|
|
743
|
+
import { homedir, hostname, networkInterfaces } from "node:os";
|
|
744
|
+
import { join as join2 } from "node:path";
|
|
745
|
+
import { createHash } from "node:crypto";
|
|
746
|
+
var CONFIG_DIR = join2(homedir(), ".engrm");
|
|
747
|
+
var SETTINGS_PATH = join2(CONFIG_DIR, "settings.json");
|
|
748
|
+
var DB_PATH = join2(CONFIG_DIR, "engrm.db");
|
|
749
|
+
function getDbPath() {
|
|
750
|
+
return DB_PATH;
|
|
751
|
+
}
|
|
752
|
+
function generateDeviceId() {
|
|
753
|
+
const host = hostname().toLowerCase().replace(/[^a-z0-9-]/g, "");
|
|
754
|
+
let mac = "";
|
|
755
|
+
const ifaces = networkInterfaces();
|
|
756
|
+
for (const entries of Object.values(ifaces)) {
|
|
757
|
+
if (!entries)
|
|
758
|
+
continue;
|
|
759
|
+
for (const entry of entries) {
|
|
760
|
+
if (!entry.internal && entry.mac && entry.mac !== "00:00:00:00:00:00") {
|
|
761
|
+
mac = entry.mac;
|
|
762
|
+
break;
|
|
763
|
+
}
|
|
980
764
|
}
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
ORDER BY created_at_epoch DESC
|
|
984
|
-
LIMIT ?`).all(projectId, limit);
|
|
765
|
+
if (mac)
|
|
766
|
+
break;
|
|
985
767
|
}
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
768
|
+
const material = `${host}:${mac || "no-mac"}`;
|
|
769
|
+
const suffix = createHash("sha256").update(material).digest("hex").slice(0, 8);
|
|
770
|
+
return `${host}-${suffix}`;
|
|
771
|
+
}
|
|
772
|
+
function createDefaultConfig() {
|
|
773
|
+
return {
|
|
774
|
+
candengo_url: "",
|
|
775
|
+
candengo_api_key: "",
|
|
776
|
+
site_id: "",
|
|
777
|
+
namespace: "",
|
|
778
|
+
user_id: "",
|
|
779
|
+
user_email: "",
|
|
780
|
+
device_id: generateDeviceId(),
|
|
781
|
+
teams: [],
|
|
782
|
+
sync: {
|
|
783
|
+
enabled: true,
|
|
784
|
+
interval_seconds: 30,
|
|
785
|
+
batch_size: 50
|
|
786
|
+
},
|
|
787
|
+
search: {
|
|
788
|
+
default_limit: 10,
|
|
789
|
+
local_boost: 1.2,
|
|
790
|
+
scope: "all"
|
|
791
|
+
},
|
|
792
|
+
scrubbing: {
|
|
793
|
+
enabled: true,
|
|
794
|
+
custom_patterns: [],
|
|
795
|
+
default_sensitivity: "shared"
|
|
796
|
+
},
|
|
797
|
+
sentinel: {
|
|
798
|
+
enabled: false,
|
|
799
|
+
mode: "advisory",
|
|
800
|
+
provider: "openai",
|
|
801
|
+
model: "gpt-4o-mini",
|
|
802
|
+
api_key: "",
|
|
803
|
+
base_url: "",
|
|
804
|
+
skip_patterns: [],
|
|
805
|
+
daily_limit: 100,
|
|
806
|
+
tier: "free"
|
|
807
|
+
},
|
|
808
|
+
observer: {
|
|
809
|
+
enabled: true,
|
|
810
|
+
mode: "per_event",
|
|
811
|
+
model: "sonnet"
|
|
812
|
+
},
|
|
813
|
+
transcript_analysis: {
|
|
814
|
+
enabled: false
|
|
998
815
|
}
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
function loadConfig() {
|
|
819
|
+
if (!existsSync2(SETTINGS_PATH)) {
|
|
820
|
+
throw new Error(`Config not found at ${SETTINGS_PATH}. Run 'engrm init --manual' to configure.`);
|
|
1003
821
|
}
|
|
1004
|
-
|
|
1005
|
-
|
|
822
|
+
const raw = readFileSync2(SETTINGS_PATH, "utf-8");
|
|
823
|
+
let parsed;
|
|
824
|
+
try {
|
|
825
|
+
parsed = JSON.parse(raw);
|
|
826
|
+
} catch {
|
|
827
|
+
throw new Error(`Invalid JSON in ${SETTINGS_PATH}`);
|
|
1006
828
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
const rows = this.db.query("SELECT name FROM packs_installed").all();
|
|
1010
|
-
return rows.map((r) => r.name);
|
|
1011
|
-
} catch {
|
|
1012
|
-
return [];
|
|
1013
|
-
}
|
|
829
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
830
|
+
throw new Error(`Config at ${SETTINGS_PATH} is not a JSON object`);
|
|
1014
831
|
}
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
832
|
+
const config = parsed;
|
|
833
|
+
const defaults = createDefaultConfig();
|
|
834
|
+
return {
|
|
835
|
+
candengo_url: asString(config["candengo_url"], defaults.candengo_url),
|
|
836
|
+
candengo_api_key: asString(config["candengo_api_key"], defaults.candengo_api_key),
|
|
837
|
+
site_id: asString(config["site_id"], defaults.site_id),
|
|
838
|
+
namespace: asString(config["namespace"], defaults.namespace),
|
|
839
|
+
user_id: asString(config["user_id"], defaults.user_id),
|
|
840
|
+
user_email: asString(config["user_email"], defaults.user_email),
|
|
841
|
+
device_id: asString(config["device_id"], defaults.device_id),
|
|
842
|
+
teams: asTeams(config["teams"], defaults.teams),
|
|
843
|
+
sync: {
|
|
844
|
+
enabled: asBool(config["sync"]?.["enabled"], defaults.sync.enabled),
|
|
845
|
+
interval_seconds: asNumber(config["sync"]?.["interval_seconds"], defaults.sync.interval_seconds),
|
|
846
|
+
batch_size: asNumber(config["sync"]?.["batch_size"], defaults.sync.batch_size)
|
|
847
|
+
},
|
|
848
|
+
search: {
|
|
849
|
+
default_limit: asNumber(config["search"]?.["default_limit"], defaults.search.default_limit),
|
|
850
|
+
local_boost: asNumber(config["search"]?.["local_boost"], defaults.search.local_boost),
|
|
851
|
+
scope: asScope(config["search"]?.["scope"], defaults.search.scope)
|
|
852
|
+
},
|
|
853
|
+
scrubbing: {
|
|
854
|
+
enabled: asBool(config["scrubbing"]?.["enabled"], defaults.scrubbing.enabled),
|
|
855
|
+
custom_patterns: asStringArray(config["scrubbing"]?.["custom_patterns"], defaults.scrubbing.custom_patterns),
|
|
856
|
+
default_sensitivity: asSensitivity(config["scrubbing"]?.["default_sensitivity"], defaults.scrubbing.default_sensitivity)
|
|
857
|
+
},
|
|
858
|
+
sentinel: {
|
|
859
|
+
enabled: asBool(config["sentinel"]?.["enabled"], defaults.sentinel.enabled),
|
|
860
|
+
mode: asSentinelMode(config["sentinel"]?.["mode"], defaults.sentinel.mode),
|
|
861
|
+
provider: asLlmProvider(config["sentinel"]?.["provider"], defaults.sentinel.provider),
|
|
862
|
+
model: asString(config["sentinel"]?.["model"], defaults.sentinel.model),
|
|
863
|
+
api_key: asString(config["sentinel"]?.["api_key"], defaults.sentinel.api_key),
|
|
864
|
+
base_url: asString(config["sentinel"]?.["base_url"], defaults.sentinel.base_url),
|
|
865
|
+
skip_patterns: asStringArray(config["sentinel"]?.["skip_patterns"], defaults.sentinel.skip_patterns),
|
|
866
|
+
daily_limit: asNumber(config["sentinel"]?.["daily_limit"], defaults.sentinel.daily_limit),
|
|
867
|
+
tier: asTier(config["sentinel"]?.["tier"], defaults.sentinel.tier)
|
|
868
|
+
},
|
|
869
|
+
observer: {
|
|
870
|
+
enabled: asBool(config["observer"]?.["enabled"], defaults.observer.enabled),
|
|
871
|
+
mode: asObserverMode(config["observer"]?.["mode"], defaults.observer.mode),
|
|
872
|
+
model: asString(config["observer"]?.["model"], defaults.observer.model)
|
|
873
|
+
},
|
|
874
|
+
transcript_analysis: {
|
|
875
|
+
enabled: asBool(config["transcript_analysis"]?.["enabled"], defaults.transcript_analysis.enabled)
|
|
876
|
+
}
|
|
877
|
+
};
|
|
878
|
+
}
|
|
879
|
+
function saveConfig(config) {
|
|
880
|
+
if (!existsSync2(CONFIG_DIR)) {
|
|
881
|
+
mkdirSync(CONFIG_DIR, { recursive: true });
|
|
1018
882
|
}
|
|
883
|
+
writeFileSync(SETTINGS_PATH, JSON.stringify(config, null, 2) + `
|
|
884
|
+
`, "utf-8");
|
|
885
|
+
}
|
|
886
|
+
function configExists() {
|
|
887
|
+
return existsSync2(SETTINGS_PATH);
|
|
888
|
+
}
|
|
889
|
+
function asString(value, fallback) {
|
|
890
|
+
return typeof value === "string" ? value : fallback;
|
|
891
|
+
}
|
|
892
|
+
function asNumber(value, fallback) {
|
|
893
|
+
return typeof value === "number" && !Number.isNaN(value) ? value : fallback;
|
|
894
|
+
}
|
|
895
|
+
function asBool(value, fallback) {
|
|
896
|
+
return typeof value === "boolean" ? value : fallback;
|
|
897
|
+
}
|
|
898
|
+
function asStringArray(value, fallback) {
|
|
899
|
+
return Array.isArray(value) && value.every((v) => typeof v === "string") ? value : fallback;
|
|
900
|
+
}
|
|
901
|
+
function asScope(value, fallback) {
|
|
902
|
+
if (value === "personal" || value === "team" || value === "all")
|
|
903
|
+
return value;
|
|
904
|
+
return fallback;
|
|
905
|
+
}
|
|
906
|
+
function asSensitivity(value, fallback) {
|
|
907
|
+
if (value === "shared" || value === "personal" || value === "secret")
|
|
908
|
+
return value;
|
|
909
|
+
return fallback;
|
|
910
|
+
}
|
|
911
|
+
function asSentinelMode(value, fallback) {
|
|
912
|
+
if (value === "advisory" || value === "blocking")
|
|
913
|
+
return value;
|
|
914
|
+
return fallback;
|
|
915
|
+
}
|
|
916
|
+
function asLlmProvider(value, fallback) {
|
|
917
|
+
if (value === "openai" || value === "anthropic" || value === "ollama" || value === "custom")
|
|
918
|
+
return value;
|
|
919
|
+
return fallback;
|
|
920
|
+
}
|
|
921
|
+
function asTier(value, fallback) {
|
|
922
|
+
if (value === "free" || value === "vibe" || value === "solo" || value === "pro" || value === "team" || value === "enterprise")
|
|
923
|
+
return value;
|
|
924
|
+
return fallback;
|
|
925
|
+
}
|
|
926
|
+
function asObserverMode(value, fallback) {
|
|
927
|
+
if (value === "per_event" || value === "per_session")
|
|
928
|
+
return value;
|
|
929
|
+
return fallback;
|
|
930
|
+
}
|
|
931
|
+
function asTeams(value, fallback) {
|
|
932
|
+
if (!Array.isArray(value))
|
|
933
|
+
return fallback;
|
|
934
|
+
return value.filter((t) => typeof t === "object" && t !== null && typeof t.id === "string" && typeof t.name === "string" && typeof t.namespace === "string");
|
|
1019
935
|
}
|
|
1020
936
|
|
|
1021
|
-
// src/
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
// src/capture/scrubber.ts
|
|
1025
|
-
var DEFAULT_PATTERNS = [
|
|
1026
|
-
{
|
|
1027
|
-
source: "sk-[a-zA-Z0-9]{20,}",
|
|
1028
|
-
flags: "g",
|
|
1029
|
-
replacement: "[REDACTED_API_KEY]",
|
|
1030
|
-
description: "OpenAI API keys",
|
|
1031
|
-
category: "api_key",
|
|
1032
|
-
severity: "critical"
|
|
1033
|
-
},
|
|
1034
|
-
{
|
|
1035
|
-
source: "Bearer [a-zA-Z0-9\\-._~+/]+=*",
|
|
1036
|
-
flags: "g",
|
|
1037
|
-
replacement: "[REDACTED_BEARER]",
|
|
1038
|
-
description: "Bearer auth tokens",
|
|
1039
|
-
category: "token",
|
|
1040
|
-
severity: "medium"
|
|
1041
|
-
},
|
|
1042
|
-
{
|
|
1043
|
-
source: "password[=:]\\s*\\S+",
|
|
1044
|
-
flags: "gi",
|
|
1045
|
-
replacement: "password=[REDACTED]",
|
|
1046
|
-
description: "Passwords in config",
|
|
1047
|
-
category: "password",
|
|
1048
|
-
severity: "high"
|
|
1049
|
-
},
|
|
1050
|
-
{
|
|
1051
|
-
source: "postgresql://[^\\s]+",
|
|
1052
|
-
flags: "g",
|
|
1053
|
-
replacement: "[REDACTED_DB_URL]",
|
|
1054
|
-
description: "PostgreSQL connection strings",
|
|
1055
|
-
category: "db_url",
|
|
1056
|
-
severity: "high"
|
|
1057
|
-
},
|
|
937
|
+
// src/storage/migrations.ts
|
|
938
|
+
var MIGRATIONS = [
|
|
1058
939
|
{
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
940
|
+
version: 1,
|
|
941
|
+
description: "Initial schema: projects, observations, sessions, sync, FTS5",
|
|
942
|
+
sql: `
|
|
943
|
+
-- Projects (canonical identity across machines)
|
|
944
|
+
CREATE TABLE projects (
|
|
945
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
946
|
+
canonical_id TEXT UNIQUE NOT NULL,
|
|
947
|
+
name TEXT NOT NULL,
|
|
948
|
+
local_path TEXT,
|
|
949
|
+
remote_url TEXT,
|
|
950
|
+
first_seen_epoch INTEGER NOT NULL,
|
|
951
|
+
last_active_epoch INTEGER NOT NULL
|
|
952
|
+
);
|
|
953
|
+
|
|
954
|
+
-- Core observations table
|
|
955
|
+
CREATE TABLE observations (
|
|
956
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
957
|
+
session_id TEXT,
|
|
958
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
959
|
+
type TEXT NOT NULL CHECK (type IN (
|
|
960
|
+
'bugfix', 'discovery', 'decision', 'pattern',
|
|
961
|
+
'change', 'feature', 'refactor', 'digest'
|
|
962
|
+
)),
|
|
963
|
+
title TEXT NOT NULL,
|
|
964
|
+
narrative TEXT,
|
|
965
|
+
facts TEXT,
|
|
966
|
+
concepts TEXT,
|
|
967
|
+
files_read TEXT,
|
|
968
|
+
files_modified TEXT,
|
|
969
|
+
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
970
|
+
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN (
|
|
971
|
+
'active', 'aging', 'archived', 'purged', 'pinned'
|
|
972
|
+
)),
|
|
973
|
+
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN (
|
|
974
|
+
'shared', 'personal', 'secret'
|
|
975
|
+
)),
|
|
976
|
+
user_id TEXT NOT NULL,
|
|
977
|
+
device_id TEXT NOT NULL,
|
|
978
|
+
agent TEXT DEFAULT 'claude-code',
|
|
979
|
+
created_at TEXT NOT NULL,
|
|
980
|
+
created_at_epoch INTEGER NOT NULL,
|
|
981
|
+
archived_at_epoch INTEGER,
|
|
982
|
+
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL
|
|
983
|
+
);
|
|
984
|
+
|
|
985
|
+
-- Session tracking
|
|
986
|
+
CREATE TABLE sessions (
|
|
987
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
988
|
+
session_id TEXT UNIQUE NOT NULL,
|
|
989
|
+
project_id INTEGER REFERENCES projects(id),
|
|
990
|
+
user_id TEXT NOT NULL,
|
|
991
|
+
device_id TEXT NOT NULL,
|
|
992
|
+
agent TEXT DEFAULT 'claude-code',
|
|
993
|
+
status TEXT DEFAULT 'active' CHECK (status IN ('active', 'completed')),
|
|
994
|
+
observation_count INTEGER DEFAULT 0,
|
|
995
|
+
started_at_epoch INTEGER,
|
|
996
|
+
completed_at_epoch INTEGER
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
-- Session summaries (generated on Stop hook)
|
|
1000
|
+
CREATE TABLE session_summaries (
|
|
1001
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1002
|
+
session_id TEXT UNIQUE NOT NULL,
|
|
1003
|
+
project_id INTEGER REFERENCES projects(id),
|
|
1004
|
+
user_id TEXT NOT NULL,
|
|
1005
|
+
request TEXT,
|
|
1006
|
+
investigated TEXT,
|
|
1007
|
+
learned TEXT,
|
|
1008
|
+
completed TEXT,
|
|
1009
|
+
next_steps TEXT,
|
|
1010
|
+
created_at_epoch INTEGER
|
|
1011
|
+
);
|
|
1012
|
+
|
|
1013
|
+
-- Sync outbox (offline-first queue)
|
|
1014
|
+
CREATE TABLE sync_outbox (
|
|
1015
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1016
|
+
record_type TEXT NOT NULL CHECK (record_type IN ('observation', 'summary')),
|
|
1017
|
+
record_id INTEGER NOT NULL,
|
|
1018
|
+
status TEXT DEFAULT 'pending' CHECK (status IN (
|
|
1019
|
+
'pending', 'syncing', 'synced', 'failed'
|
|
1020
|
+
)),
|
|
1021
|
+
retry_count INTEGER DEFAULT 0,
|
|
1022
|
+
max_retries INTEGER DEFAULT 10,
|
|
1023
|
+
last_error TEXT,
|
|
1024
|
+
created_at_epoch INTEGER NOT NULL,
|
|
1025
|
+
synced_at_epoch INTEGER,
|
|
1026
|
+
next_retry_epoch INTEGER
|
|
1027
|
+
);
|
|
1028
|
+
|
|
1029
|
+
-- Sync high-water mark and lifecycle job tracking
|
|
1030
|
+
CREATE TABLE sync_state (
|
|
1031
|
+
key TEXT PRIMARY KEY,
|
|
1032
|
+
value TEXT NOT NULL
|
|
1033
|
+
);
|
|
1034
|
+
|
|
1035
|
+
-- FTS5 for local offline search (external content mode)
|
|
1036
|
+
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
1037
|
+
title, narrative, facts, concepts,
|
|
1038
|
+
content=observations,
|
|
1039
|
+
content_rowid=id
|
|
1040
|
+
);
|
|
1041
|
+
|
|
1042
|
+
-- Indexes: observations
|
|
1043
|
+
CREATE INDEX idx_observations_project ON observations(project_id);
|
|
1044
|
+
CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
|
|
1045
|
+
CREATE INDEX idx_observations_type ON observations(type);
|
|
1046
|
+
CREATE INDEX idx_observations_created ON observations(created_at_epoch);
|
|
1047
|
+
CREATE INDEX idx_observations_session ON observations(session_id);
|
|
1048
|
+
CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
|
|
1049
|
+
CREATE INDEX idx_observations_quality ON observations(quality);
|
|
1050
|
+
CREATE INDEX idx_observations_user ON observations(user_id);
|
|
1051
|
+
|
|
1052
|
+
-- Indexes: sessions
|
|
1053
|
+
CREATE INDEX idx_sessions_project ON sessions(project_id);
|
|
1054
|
+
|
|
1055
|
+
-- Indexes: sync outbox
|
|
1056
|
+
CREATE INDEX idx_outbox_status ON sync_outbox(status, next_retry_epoch);
|
|
1057
|
+
CREATE INDEX idx_outbox_record ON sync_outbox(record_type, record_id);
|
|
1058
|
+
`
|
|
1065
1059
|
},
|
|
1066
1060
|
{
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1061
|
+
version: 2,
|
|
1062
|
+
description: "Add superseded_by for knowledge supersession",
|
|
1063
|
+
sql: `
|
|
1064
|
+
ALTER TABLE observations ADD COLUMN superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL;
|
|
1065
|
+
CREATE INDEX idx_observations_superseded ON observations(superseded_by);
|
|
1066
|
+
`
|
|
1073
1067
|
},
|
|
1074
1068
|
{
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1069
|
+
version: 3,
|
|
1070
|
+
description: "Add remote_source_id for pull deduplication",
|
|
1071
|
+
sql: `
|
|
1072
|
+
ALTER TABLE observations ADD COLUMN remote_source_id TEXT;
|
|
1073
|
+
CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
1074
|
+
`
|
|
1081
1075
|
},
|
|
1082
1076
|
{
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1077
|
+
version: 4,
|
|
1078
|
+
description: "Add sqlite-vec for local semantic search",
|
|
1079
|
+
sql: `
|
|
1080
|
+
CREATE VIRTUAL TABLE vec_observations USING vec0(
|
|
1081
|
+
observation_id INTEGER PRIMARY KEY,
|
|
1082
|
+
embedding float[384]
|
|
1083
|
+
);
|
|
1084
|
+
`,
|
|
1085
|
+
condition: (db) => isVecExtensionLoaded(db)
|
|
1089
1086
|
},
|
|
1090
1087
|
{
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1088
|
+
version: 5,
|
|
1089
|
+
description: "Session metrics and security findings",
|
|
1090
|
+
sql: `
|
|
1091
|
+
ALTER TABLE sessions ADD COLUMN files_touched_count INTEGER DEFAULT 0;
|
|
1092
|
+
ALTER TABLE sessions ADD COLUMN searches_performed INTEGER DEFAULT 0;
|
|
1093
|
+
ALTER TABLE sessions ADD COLUMN tool_calls_count INTEGER DEFAULT 0;
|
|
1094
|
+
|
|
1095
|
+
CREATE TABLE security_findings (
|
|
1096
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1097
|
+
session_id TEXT,
|
|
1098
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
1099
|
+
finding_type TEXT NOT NULL,
|
|
1100
|
+
severity TEXT NOT NULL DEFAULT 'medium' CHECK (severity IN ('critical', 'high', 'medium', 'low')),
|
|
1101
|
+
pattern_name TEXT NOT NULL,
|
|
1102
|
+
file_path TEXT,
|
|
1103
|
+
snippet TEXT,
|
|
1104
|
+
tool_name TEXT,
|
|
1105
|
+
user_id TEXT NOT NULL,
|
|
1106
|
+
device_id TEXT NOT NULL,
|
|
1107
|
+
created_at_epoch INTEGER NOT NULL
|
|
1108
|
+
);
|
|
1109
|
+
|
|
1110
|
+
CREATE INDEX idx_security_findings_session ON security_findings(session_id);
|
|
1111
|
+
CREATE INDEX idx_security_findings_project ON security_findings(project_id, created_at_epoch);
|
|
1112
|
+
CREATE INDEX idx_security_findings_severity ON security_findings(severity);
|
|
1113
|
+
`
|
|
1097
1114
|
},
|
|
1098
1115
|
{
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1116
|
+
version: 6,
|
|
1117
|
+
description: "Add risk_score, expand observation types to include standard",
|
|
1118
|
+
sql: `
|
|
1119
|
+
ALTER TABLE sessions ADD COLUMN risk_score INTEGER;
|
|
1120
|
+
|
|
1121
|
+
-- Recreate observations table with expanded type CHECK to include 'standard'
|
|
1122
|
+
-- SQLite doesn't support ALTER CHECK, so we recreate the table
|
|
1123
|
+
CREATE TABLE observations_new (
|
|
1124
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1125
|
+
session_id TEXT,
|
|
1126
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
1127
|
+
type TEXT NOT NULL CHECK (type IN (
|
|
1128
|
+
'bugfix', 'discovery', 'decision', 'pattern',
|
|
1129
|
+
'change', 'feature', 'refactor', 'digest', 'standard'
|
|
1130
|
+
)),
|
|
1131
|
+
title TEXT NOT NULL,
|
|
1132
|
+
narrative TEXT,
|
|
1133
|
+
facts TEXT,
|
|
1134
|
+
concepts TEXT,
|
|
1135
|
+
files_read TEXT,
|
|
1136
|
+
files_modified TEXT,
|
|
1137
|
+
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
1138
|
+
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN (
|
|
1139
|
+
'active', 'aging', 'archived', 'purged', 'pinned'
|
|
1140
|
+
)),
|
|
1141
|
+
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN (
|
|
1142
|
+
'shared', 'personal', 'secret'
|
|
1143
|
+
)),
|
|
1144
|
+
user_id TEXT NOT NULL,
|
|
1145
|
+
device_id TEXT NOT NULL,
|
|
1146
|
+
agent TEXT DEFAULT 'claude-code',
|
|
1147
|
+
created_at TEXT NOT NULL,
|
|
1148
|
+
created_at_epoch INTEGER NOT NULL,
|
|
1149
|
+
archived_at_epoch INTEGER,
|
|
1150
|
+
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
1151
|
+
superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
1152
|
+
remote_source_id TEXT
|
|
1153
|
+
);
|
|
1154
|
+
|
|
1155
|
+
INSERT INTO observations_new SELECT * FROM observations;
|
|
1156
|
+
|
|
1157
|
+
DROP TABLE observations;
|
|
1158
|
+
ALTER TABLE observations_new RENAME TO observations;
|
|
1159
|
+
|
|
1160
|
+
-- Recreate indexes
|
|
1161
|
+
CREATE INDEX idx_observations_project ON observations(project_id);
|
|
1162
|
+
CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
|
|
1163
|
+
CREATE INDEX idx_observations_type ON observations(type);
|
|
1164
|
+
CREATE INDEX idx_observations_created ON observations(created_at_epoch);
|
|
1165
|
+
CREATE INDEX idx_observations_session ON observations(session_id);
|
|
1166
|
+
CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
|
|
1167
|
+
CREATE INDEX idx_observations_quality ON observations(quality);
|
|
1168
|
+
CREATE INDEX idx_observations_user ON observations(user_id);
|
|
1169
|
+
CREATE INDEX idx_observations_superseded ON observations(superseded_by);
|
|
1170
|
+
CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
1171
|
+
|
|
1172
|
+
-- Recreate FTS5 (external content mode — must rebuild after table recreation)
|
|
1173
|
+
DROP TABLE IF EXISTS observations_fts;
|
|
1174
|
+
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
1175
|
+
title, narrative, facts, concepts,
|
|
1176
|
+
content=observations,
|
|
1177
|
+
content_rowid=id
|
|
1178
|
+
);
|
|
1179
|
+
-- Rebuild FTS index
|
|
1180
|
+
INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
|
|
1181
|
+
`
|
|
1105
1182
|
},
|
|
1106
1183
|
{
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1184
|
+
version: 7,
|
|
1185
|
+
description: "Add packs_installed table for help pack tracking",
|
|
1186
|
+
sql: `
|
|
1187
|
+
CREATE TABLE IF NOT EXISTS packs_installed (
|
|
1188
|
+
name TEXT PRIMARY KEY,
|
|
1189
|
+
installed_at INTEGER NOT NULL,
|
|
1190
|
+
observation_count INTEGER DEFAULT 0
|
|
1191
|
+
);
|
|
1192
|
+
`
|
|
1113
1193
|
},
|
|
1114
1194
|
{
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1195
|
+
version: 8,
|
|
1196
|
+
description: "Add message type to observations CHECK constraint",
|
|
1197
|
+
sql: `
|
|
1198
|
+
CREATE TABLE IF NOT EXISTS observations_v8 (
|
|
1199
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1200
|
+
session_id TEXT,
|
|
1201
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
1202
|
+
type TEXT NOT NULL CHECK (type IN (
|
|
1203
|
+
'bugfix', 'discovery', 'decision', 'pattern',
|
|
1204
|
+
'change', 'feature', 'refactor', 'digest', 'standard', 'message'
|
|
1205
|
+
)),
|
|
1206
|
+
title TEXT NOT NULL,
|
|
1207
|
+
narrative TEXT,
|
|
1208
|
+
facts TEXT,
|
|
1209
|
+
concepts TEXT,
|
|
1210
|
+
files_read TEXT,
|
|
1211
|
+
files_modified TEXT,
|
|
1212
|
+
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
1213
|
+
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN (
|
|
1214
|
+
'active', 'aging', 'archived', 'purged', 'pinned'
|
|
1215
|
+
)),
|
|
1216
|
+
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN (
|
|
1217
|
+
'shared', 'personal', 'secret'
|
|
1218
|
+
)),
|
|
1219
|
+
user_id TEXT NOT NULL,
|
|
1220
|
+
device_id TEXT NOT NULL,
|
|
1221
|
+
agent TEXT DEFAULT 'claude-code',
|
|
1222
|
+
created_at TEXT NOT NULL,
|
|
1223
|
+
created_at_epoch INTEGER NOT NULL,
|
|
1224
|
+
archived_at_epoch INTEGER,
|
|
1225
|
+
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
1226
|
+
superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
1227
|
+
remote_source_id TEXT
|
|
1228
|
+
);
|
|
1229
|
+
INSERT INTO observations_v8 SELECT * FROM observations;
|
|
1230
|
+
DROP TABLE observations;
|
|
1231
|
+
ALTER TABLE observations_v8 RENAME TO observations;
|
|
1232
|
+
CREATE INDEX idx_observations_project ON observations(project_id);
|
|
1233
|
+
CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
|
|
1234
|
+
CREATE INDEX idx_observations_type ON observations(type);
|
|
1235
|
+
CREATE INDEX idx_observations_created ON observations(created_at_epoch);
|
|
1236
|
+
CREATE INDEX idx_observations_session ON observations(session_id);
|
|
1237
|
+
CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
|
|
1238
|
+
CREATE INDEX idx_observations_quality ON observations(quality);
|
|
1239
|
+
CREATE INDEX idx_observations_user ON observations(user_id);
|
|
1240
|
+
CREATE INDEX idx_observations_superseded ON observations(superseded_by);
|
|
1241
|
+
CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
1242
|
+
DROP TABLE IF EXISTS observations_fts;
|
|
1243
|
+
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
1244
|
+
title, narrative, facts, concepts,
|
|
1245
|
+
content=observations,
|
|
1246
|
+
content_rowid=id
|
|
1247
|
+
);
|
|
1248
|
+
INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
|
|
1249
|
+
`
|
|
1250
|
+
}
|
|
1251
|
+
];
|
|
1252
|
+
function isVecExtensionLoaded(db) {
|
|
1253
|
+
try {
|
|
1254
|
+
db.exec("SELECT vec_version()");
|
|
1255
|
+
return true;
|
|
1256
|
+
} catch {
|
|
1257
|
+
return false;
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
function runMigrations(db) {
|
|
1261
|
+
const currentVersion = db.query("PRAGMA user_version").get();
|
|
1262
|
+
let version = currentVersion.user_version;
|
|
1263
|
+
for (const migration of MIGRATIONS) {
|
|
1264
|
+
if (migration.version <= version)
|
|
1265
|
+
continue;
|
|
1266
|
+
if (migration.condition && !migration.condition(db)) {
|
|
1267
|
+
continue;
|
|
1268
|
+
}
|
|
1269
|
+
db.exec("BEGIN TRANSACTION");
|
|
1270
|
+
try {
|
|
1271
|
+
db.exec(migration.sql);
|
|
1272
|
+
db.exec(`PRAGMA user_version = ${migration.version}`);
|
|
1273
|
+
db.exec("COMMIT");
|
|
1274
|
+
version = migration.version;
|
|
1275
|
+
} catch (error) {
|
|
1276
|
+
db.exec("ROLLBACK");
|
|
1277
|
+
throw new Error(`Migration ${migration.version} (${migration.description}) failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1278
|
+
}
|
|
1121
1279
|
}
|
|
1122
|
-
|
|
1123
|
-
function
|
|
1124
|
-
|
|
1125
|
-
|
|
1280
|
+
}
|
|
1281
|
+
function ensureObservationTypes(db) {
|
|
1282
|
+
try {
|
|
1283
|
+
db.exec("INSERT INTO observations (session_id, project_id, type, title, user_id, device_id, agent, created_at, created_at_epoch) " + "VALUES ('_typecheck', 1, 'message', '_test', '_test', '_test', '_test', '2000-01-01', 0)");
|
|
1284
|
+
db.exec("DELETE FROM observations WHERE session_id = '_typecheck'");
|
|
1285
|
+
} catch {
|
|
1286
|
+
db.exec("BEGIN TRANSACTION");
|
|
1126
1287
|
try {
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1288
|
+
db.exec(`
|
|
1289
|
+
CREATE TABLE observations_repair (
|
|
1290
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT,
|
|
1291
|
+
project_id INTEGER NOT NULL REFERENCES projects(id),
|
|
1292
|
+
type TEXT NOT NULL CHECK (type IN (
|
|
1293
|
+
'bugfix','discovery','decision','pattern','change','feature',
|
|
1294
|
+
'refactor','digest','standard','message')),
|
|
1295
|
+
title TEXT NOT NULL, narrative TEXT, facts TEXT, concepts TEXT,
|
|
1296
|
+
files_read TEXT, files_modified TEXT,
|
|
1297
|
+
quality REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
|
|
1298
|
+
lifecycle TEXT DEFAULT 'active' CHECK (lifecycle IN ('active','aging','archived','purged','pinned')),
|
|
1299
|
+
sensitivity TEXT DEFAULT 'shared' CHECK (sensitivity IN ('shared','personal','secret')),
|
|
1300
|
+
user_id TEXT NOT NULL, device_id TEXT NOT NULL, agent TEXT DEFAULT 'claude-code',
|
|
1301
|
+
created_at TEXT NOT NULL, created_at_epoch INTEGER NOT NULL,
|
|
1302
|
+
archived_at_epoch INTEGER,
|
|
1303
|
+
compacted_into INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
1304
|
+
superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL,
|
|
1305
|
+
remote_source_id TEXT
|
|
1306
|
+
);
|
|
1307
|
+
INSERT INTO observations_repair SELECT * FROM observations;
|
|
1308
|
+
DROP TABLE observations;
|
|
1309
|
+
ALTER TABLE observations_repair RENAME TO observations;
|
|
1310
|
+
CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id);
|
|
1311
|
+
CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
|
|
1312
|
+
CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at_epoch);
|
|
1313
|
+
CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
|
|
1314
|
+
CREATE INDEX IF NOT EXISTS idx_observations_lifecycle ON observations(lifecycle);
|
|
1315
|
+
CREATE INDEX IF NOT EXISTS idx_observations_quality ON observations(quality);
|
|
1316
|
+
CREATE INDEX IF NOT EXISTS idx_observations_user ON observations(user_id);
|
|
1317
|
+
CREATE INDEX IF NOT EXISTS idx_observations_superseded ON observations(superseded_by);
|
|
1318
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
|
|
1319
|
+
DROP TABLE IF EXISTS observations_fts;
|
|
1320
|
+
CREATE VIRTUAL TABLE observations_fts USING fts5(
|
|
1321
|
+
title, narrative, facts, concepts, content=observations, content_rowid=id
|
|
1322
|
+
);
|
|
1323
|
+
INSERT INTO observations_fts(observations_fts) VALUES('rebuild');
|
|
1324
|
+
`);
|
|
1325
|
+
db.exec("COMMIT");
|
|
1326
|
+
} catch (err) {
|
|
1327
|
+
db.exec("ROLLBACK");
|
|
1328
|
+
}
|
|
1137
1329
|
}
|
|
1138
|
-
return compiled;
|
|
1139
1330
|
}
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1331
|
+
var LATEST_SCHEMA_VERSION = MIGRATIONS.filter((m) => !m.condition).reduce((max, m) => Math.max(max, m.version), 0);
|
|
1332
|
+
|
|
1333
|
+
// src/storage/sqlite.ts
|
|
1334
|
+
var IS_BUN = typeof globalThis.Bun !== "undefined";
|
|
1335
|
+
function openDatabase(dbPath) {
|
|
1336
|
+
if (IS_BUN) {
|
|
1337
|
+
return openBunDatabase(dbPath);
|
|
1145
1338
|
}
|
|
1146
|
-
return
|
|
1339
|
+
return openNodeDatabase(dbPath);
|
|
1147
1340
|
}
|
|
1148
|
-
function
|
|
1149
|
-
const
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1341
|
+
function openBunDatabase(dbPath) {
|
|
1342
|
+
const { Database } = __require("bun:sqlite");
|
|
1343
|
+
if (process.platform === "darwin") {
|
|
1344
|
+
const { existsSync: existsSync3 } = __require("node:fs");
|
|
1345
|
+
const paths = [
|
|
1346
|
+
"/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib",
|
|
1347
|
+
"/usr/local/opt/sqlite3/lib/libsqlite3.dylib"
|
|
1348
|
+
];
|
|
1349
|
+
for (const p of paths) {
|
|
1350
|
+
if (existsSync3(p)) {
|
|
1351
|
+
try {
|
|
1352
|
+
Database.setCustomSQLite(p);
|
|
1353
|
+
} catch {}
|
|
1354
|
+
break;
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1153
1357
|
}
|
|
1154
|
-
|
|
1358
|
+
const db = new Database(dbPath);
|
|
1359
|
+
return db;
|
|
1360
|
+
}
|
|
1361
|
+
function openNodeDatabase(dbPath) {
|
|
1362
|
+
const BetterSqlite3 = __require("better-sqlite3");
|
|
1363
|
+
const raw = new BetterSqlite3(dbPath);
|
|
1364
|
+
return {
|
|
1365
|
+
query(sql) {
|
|
1366
|
+
const stmt = raw.prepare(sql);
|
|
1367
|
+
return {
|
|
1368
|
+
get(...params) {
|
|
1369
|
+
return stmt.get(...params);
|
|
1370
|
+
},
|
|
1371
|
+
all(...params) {
|
|
1372
|
+
return stmt.all(...params);
|
|
1373
|
+
},
|
|
1374
|
+
run(...params) {
|
|
1375
|
+
return stmt.run(...params);
|
|
1376
|
+
}
|
|
1377
|
+
};
|
|
1378
|
+
},
|
|
1379
|
+
exec(sql) {
|
|
1380
|
+
raw.exec(sql);
|
|
1381
|
+
},
|
|
1382
|
+
close() {
|
|
1383
|
+
raw.close();
|
|
1384
|
+
}
|
|
1385
|
+
};
|
|
1155
1386
|
}
|
|
1156
1387
|
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
break;
|
|
1168
|
-
case "discovery":
|
|
1169
|
-
score += 0.2;
|
|
1170
|
-
break;
|
|
1171
|
-
case "pattern":
|
|
1172
|
-
score += 0.2;
|
|
1173
|
-
break;
|
|
1174
|
-
case "feature":
|
|
1175
|
-
score += 0.15;
|
|
1176
|
-
break;
|
|
1177
|
-
case "refactor":
|
|
1178
|
-
score += 0.15;
|
|
1179
|
-
break;
|
|
1180
|
-
case "change":
|
|
1181
|
-
score += 0.05;
|
|
1182
|
-
break;
|
|
1183
|
-
case "digest":
|
|
1184
|
-
score += 0.3;
|
|
1185
|
-
break;
|
|
1186
|
-
}
|
|
1187
|
-
if (input.narrative && input.narrative.length > 50) {
|
|
1188
|
-
score += 0.15;
|
|
1388
|
+
class MemDatabase {
|
|
1389
|
+
db;
|
|
1390
|
+
vecAvailable;
|
|
1391
|
+
constructor(dbPath) {
|
|
1392
|
+
this.db = openDatabase(dbPath);
|
|
1393
|
+
this.db.exec("PRAGMA journal_mode = WAL");
|
|
1394
|
+
this.db.exec("PRAGMA foreign_keys = ON");
|
|
1395
|
+
this.vecAvailable = this.loadVecExtension();
|
|
1396
|
+
runMigrations(this.db);
|
|
1397
|
+
ensureObservationTypes(this.db);
|
|
1189
1398
|
}
|
|
1190
|
-
|
|
1399
|
+
loadVecExtension() {
|
|
1191
1400
|
try {
|
|
1192
|
-
const
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
else if (factsArray.length === 1)
|
|
1196
|
-
score += 0.05;
|
|
1401
|
+
const sqliteVec = __require("sqlite-vec");
|
|
1402
|
+
sqliteVec.load(this.db);
|
|
1403
|
+
return true;
|
|
1197
1404
|
} catch {
|
|
1198
|
-
|
|
1199
|
-
score += 0.05;
|
|
1405
|
+
return false;
|
|
1200
1406
|
}
|
|
1201
1407
|
}
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1408
|
+
close() {
|
|
1409
|
+
this.db.close();
|
|
1410
|
+
}
|
|
1411
|
+
upsertProject(project) {
|
|
1412
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1413
|
+
const existing = this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(project.canonical_id);
|
|
1414
|
+
if (existing) {
|
|
1415
|
+
this.db.query(`UPDATE projects SET
|
|
1416
|
+
local_path = COALESCE(?, local_path),
|
|
1417
|
+
remote_url = COALESCE(?, remote_url),
|
|
1418
|
+
last_active_epoch = ?
|
|
1419
|
+
WHERE id = ?`).run(project.local_path ?? null, project.remote_url ?? null, now, existing.id);
|
|
1420
|
+
return {
|
|
1421
|
+
...existing,
|
|
1422
|
+
local_path: project.local_path ?? existing.local_path,
|
|
1423
|
+
remote_url: project.remote_url ?? existing.remote_url,
|
|
1424
|
+
last_active_epoch: now
|
|
1425
|
+
};
|
|
1210
1426
|
}
|
|
1427
|
+
const result = this.db.query(`INSERT INTO projects (canonical_id, name, local_path, remote_url, first_seen_epoch, last_active_epoch)
|
|
1428
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(project.canonical_id, project.name, project.local_path ?? null, project.remote_url ?? null, now, now);
|
|
1429
|
+
return this.db.query("SELECT * FROM projects WHERE id = ?").get(Number(result.lastInsertRowid));
|
|
1211
1430
|
}
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
score += 0.2;
|
|
1215
|
-
else if (modifiedCount >= 1)
|
|
1216
|
-
score += 0.1;
|
|
1217
|
-
if (input.isDuplicate) {
|
|
1218
|
-
score -= 0.3;
|
|
1431
|
+
getProjectByCanonicalId(canonicalId) {
|
|
1432
|
+
return this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(canonicalId) ?? null;
|
|
1219
1433
|
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
return 0;
|
|
1239
|
-
let intersectionSize = 0;
|
|
1240
|
-
for (const token of tokensA) {
|
|
1241
|
-
if (tokensB.has(token))
|
|
1242
|
-
intersectionSize++;
|
|
1434
|
+
getProjectById(id) {
|
|
1435
|
+
return this.db.query("SELECT * FROM projects WHERE id = ?").get(id) ?? null;
|
|
1436
|
+
}
|
|
1437
|
+
insertObservation(obs) {
|
|
1438
|
+
const now = obs.created_at_epoch ?? Math.floor(Date.now() / 1000);
|
|
1439
|
+
const createdAt = obs.created_at ?? new Date(now * 1000).toISOString();
|
|
1440
|
+
const result = this.db.query(`INSERT INTO observations (
|
|
1441
|
+
session_id, project_id, type, title, narrative, facts, concepts,
|
|
1442
|
+
files_read, files_modified, quality, lifecycle, sensitivity,
|
|
1443
|
+
user_id, device_id, agent, created_at, created_at_epoch
|
|
1444
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(obs.session_id ?? null, obs.project_id, obs.type, obs.title, obs.narrative ?? null, obs.facts ?? null, obs.concepts ?? null, obs.files_read ?? null, obs.files_modified ?? null, obs.quality, obs.lifecycle ?? "active", obs.sensitivity ?? "shared", obs.user_id, obs.device_id, obs.agent ?? "claude-code", createdAt, now);
|
|
1445
|
+
const id = Number(result.lastInsertRowid);
|
|
1446
|
+
const row = this.getObservationById(id);
|
|
1447
|
+
this.ftsInsert(row);
|
|
1448
|
+
if (obs.session_id) {
|
|
1449
|
+
this.db.query("UPDATE sessions SET observation_count = observation_count + 1 WHERE session_id = ?").run(obs.session_id);
|
|
1450
|
+
}
|
|
1451
|
+
return row;
|
|
1243
1452
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1453
|
+
getObservationById(id) {
|
|
1454
|
+
return this.db.query("SELECT * FROM observations WHERE id = ?").get(id) ?? null;
|
|
1455
|
+
}
|
|
1456
|
+
getObservationsByIds(ids, userId) {
|
|
1457
|
+
if (ids.length === 0)
|
|
1458
|
+
return [];
|
|
1459
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
1460
|
+
const visibilityClause = userId ? " AND (sensitivity != 'personal' OR user_id = ?)" : "";
|
|
1461
|
+
return this.db.query(`SELECT * FROM observations
|
|
1462
|
+
WHERE id IN (${placeholders})${visibilityClause}
|
|
1463
|
+
ORDER BY created_at_epoch DESC`).all(...ids, ...userId ? [userId] : []);
|
|
1464
|
+
}
|
|
1465
|
+
getRecentObservations(projectId, sincEpoch, limit = 50) {
|
|
1466
|
+
return this.db.query(`SELECT * FROM observations
|
|
1467
|
+
WHERE project_id = ? AND created_at_epoch > ?
|
|
1468
|
+
ORDER BY created_at_epoch DESC
|
|
1469
|
+
LIMIT ?`).all(projectId, sincEpoch, limit);
|
|
1470
|
+
}
|
|
1471
|
+
searchFts(query, projectId, lifecycles = ["active", "aging", "pinned"], limit = 20, userId) {
|
|
1472
|
+
const lifecyclePlaceholders = lifecycles.map(() => "?").join(",");
|
|
1473
|
+
const visibilityClause = userId ? " AND (o.sensitivity != 'personal' OR o.user_id = ?)" : "";
|
|
1474
|
+
if (projectId !== null) {
|
|
1475
|
+
return this.db.query(`SELECT o.id, observations_fts.rank
|
|
1476
|
+
FROM observations_fts
|
|
1477
|
+
JOIN observations o ON o.id = observations_fts.rowid
|
|
1478
|
+
WHERE observations_fts MATCH ?
|
|
1479
|
+
AND o.project_id = ?
|
|
1480
|
+
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
1481
|
+
${visibilityClause}
|
|
1482
|
+
ORDER BY observations_fts.rank
|
|
1483
|
+
LIMIT ?`).all(query, projectId, ...lifecycles, ...userId ? [userId] : [], limit);
|
|
1258
1484
|
}
|
|
1485
|
+
return this.db.query(`SELECT o.id, observations_fts.rank
|
|
1486
|
+
FROM observations_fts
|
|
1487
|
+
JOIN observations o ON o.id = observations_fts.rowid
|
|
1488
|
+
WHERE observations_fts MATCH ?
|
|
1489
|
+
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
1490
|
+
${visibilityClause}
|
|
1491
|
+
ORDER BY observations_fts.rank
|
|
1492
|
+
LIMIT ?`).all(query, ...lifecycles, ...userId ? [userId] : [], limit);
|
|
1259
1493
|
}
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1494
|
+
getTimeline(anchorId, projectId, depthBefore = 3, depthAfter = 3, userId) {
|
|
1495
|
+
const visibilityClause = userId ? " AND (sensitivity != 'personal' OR user_id = ?)" : "";
|
|
1496
|
+
const anchor = this.db.query(`SELECT * FROM observations WHERE id = ?${visibilityClause}`).get(anchorId, ...userId ? [userId] : []) ?? null;
|
|
1497
|
+
if (!anchor)
|
|
1498
|
+
return [];
|
|
1499
|
+
const projectFilter = projectId !== null ? "AND project_id = ?" : "";
|
|
1500
|
+
const projectParams = projectId !== null ? [projectId] : [];
|
|
1501
|
+
const visibilityParams = userId ? [userId] : [];
|
|
1502
|
+
const before = this.db.query(`SELECT * FROM observations
|
|
1503
|
+
WHERE created_at_epoch < ? ${projectFilter}
|
|
1504
|
+
AND lifecycle IN ('active', 'aging', 'pinned')
|
|
1505
|
+
${visibilityClause}
|
|
1506
|
+
ORDER BY created_at_epoch DESC
|
|
1507
|
+
LIMIT ?`).all(anchor.created_at_epoch, ...projectParams, ...visibilityParams, depthBefore);
|
|
1508
|
+
const after = this.db.query(`SELECT * FROM observations
|
|
1509
|
+
WHERE created_at_epoch > ? ${projectFilter}
|
|
1510
|
+
AND lifecycle IN ('active', 'aging', 'pinned')
|
|
1511
|
+
${visibilityClause}
|
|
1512
|
+
ORDER BY created_at_epoch ASC
|
|
1513
|
+
LIMIT ?`).all(anchor.created_at_epoch, ...projectParams, ...visibilityParams, depthAfter);
|
|
1514
|
+
return [...before.reverse(), anchor, ...after];
|
|
1281
1515
|
}
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1295
|
-
}).trim();
|
|
1296
|
-
return url || null;
|
|
1297
|
-
} catch {
|
|
1298
|
-
try {
|
|
1299
|
-
const remotes = execSync("git remote", {
|
|
1300
|
-
cwd: directory,
|
|
1301
|
-
encoding: "utf-8",
|
|
1302
|
-
timeout: 5000,
|
|
1303
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1304
|
-
}).trim().split(`
|
|
1305
|
-
`).filter(Boolean);
|
|
1306
|
-
if (remotes.length === 0)
|
|
1307
|
-
return null;
|
|
1308
|
-
const url = execSync(`git remote get-url ${remotes[0]}`, {
|
|
1309
|
-
cwd: directory,
|
|
1310
|
-
encoding: "utf-8",
|
|
1311
|
-
timeout: 5000,
|
|
1312
|
-
stdio: ["pipe", "pipe", "pipe"]
|
|
1313
|
-
}).trim();
|
|
1314
|
-
return url || null;
|
|
1315
|
-
} catch {
|
|
1316
|
-
return null;
|
|
1516
|
+
pinObservation(id, pinned) {
|
|
1517
|
+
const obs = this.getObservationById(id);
|
|
1518
|
+
if (!obs)
|
|
1519
|
+
return false;
|
|
1520
|
+
if (pinned) {
|
|
1521
|
+
if (obs.lifecycle !== "active" && obs.lifecycle !== "aging")
|
|
1522
|
+
return false;
|
|
1523
|
+
this.db.query("UPDATE observations SET lifecycle = 'pinned' WHERE id = ?").run(id);
|
|
1524
|
+
} else {
|
|
1525
|
+
if (obs.lifecycle !== "pinned")
|
|
1526
|
+
return false;
|
|
1527
|
+
this.db.query("UPDATE observations SET lifecycle = 'active' WHERE id = ?").run(id);
|
|
1317
1528
|
}
|
|
1529
|
+
return true;
|
|
1318
1530
|
}
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
const parsed = JSON.parse(raw);
|
|
1327
|
-
if (typeof parsed["project_id"] !== "string" || !parsed["project_id"]) {
|
|
1328
|
-
return null;
|
|
1531
|
+
getActiveObservationCount(userId) {
|
|
1532
|
+
if (userId) {
|
|
1533
|
+
const result2 = this.db.query(`SELECT COUNT(*) as count FROM observations
|
|
1534
|
+
WHERE lifecycle IN ('active', 'aging')
|
|
1535
|
+
AND sensitivity != 'secret'
|
|
1536
|
+
AND user_id = ?`).get(userId);
|
|
1537
|
+
return result2?.count ?? 0;
|
|
1329
1538
|
}
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
} catch {
|
|
1335
|
-
return null;
|
|
1539
|
+
const result = this.db.query(`SELECT COUNT(*) as count FROM observations
|
|
1540
|
+
WHERE lifecycle IN ('active', 'aging')
|
|
1541
|
+
AND sensitivity != 'secret'`).get();
|
|
1542
|
+
return result?.count ?? 0;
|
|
1336
1543
|
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1544
|
+
supersedeObservation(oldId, newId) {
|
|
1545
|
+
if (oldId === newId)
|
|
1546
|
+
return false;
|
|
1547
|
+
const replacement = this.getObservationById(newId);
|
|
1548
|
+
if (!replacement)
|
|
1549
|
+
return false;
|
|
1550
|
+
let targetId = oldId;
|
|
1551
|
+
const visited = new Set;
|
|
1552
|
+
for (let depth = 0;depth < 10; depth++) {
|
|
1553
|
+
const target2 = this.getObservationById(targetId);
|
|
1554
|
+
if (!target2)
|
|
1555
|
+
return false;
|
|
1556
|
+
if (target2.superseded_by === null)
|
|
1557
|
+
break;
|
|
1558
|
+
if (target2.superseded_by === newId)
|
|
1559
|
+
return true;
|
|
1560
|
+
visited.add(targetId);
|
|
1561
|
+
targetId = target2.superseded_by;
|
|
1562
|
+
if (visited.has(targetId))
|
|
1563
|
+
return false;
|
|
1564
|
+
}
|
|
1565
|
+
const target = this.getObservationById(targetId);
|
|
1566
|
+
if (!target)
|
|
1567
|
+
return false;
|
|
1568
|
+
if (target.superseded_by !== null)
|
|
1569
|
+
return false;
|
|
1570
|
+
if (targetId === newId)
|
|
1571
|
+
return false;
|
|
1572
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1573
|
+
this.db.query(`UPDATE observations
|
|
1574
|
+
SET superseded_by = ?, lifecycle = 'archived', archived_at_epoch = ?
|
|
1575
|
+
WHERE id = ?`).run(newId, now, targetId);
|
|
1576
|
+
this.ftsDelete(target);
|
|
1577
|
+
this.vecDelete(targetId);
|
|
1578
|
+
return true;
|
|
1348
1579
|
}
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
return
|
|
1352
|
-
canonical_id: configFile.project_id,
|
|
1353
|
-
name: configFile.name ?? projectNameFromCanonicalId(configFile.project_id),
|
|
1354
|
-
remote_url: null,
|
|
1355
|
-
local_path: directory
|
|
1356
|
-
};
|
|
1580
|
+
isSuperseded(id) {
|
|
1581
|
+
const obs = this.getObservationById(id);
|
|
1582
|
+
return obs !== null && obs.superseded_by !== null;
|
|
1357
1583
|
}
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
// src/embeddings/embedder.ts
|
|
1368
|
-
var _available = null;
|
|
1369
|
-
var _pipeline = null;
|
|
1370
|
-
var MODEL_NAME = "Xenova/all-MiniLM-L6-v2";
|
|
1371
|
-
async function embedText(text) {
|
|
1372
|
-
const pipe = await getPipeline();
|
|
1373
|
-
if (!pipe)
|
|
1374
|
-
return null;
|
|
1375
|
-
try {
|
|
1376
|
-
const output = await pipe(text, { pooling: "mean", normalize: true });
|
|
1377
|
-
return new Float32Array(output.data);
|
|
1378
|
-
} catch {
|
|
1379
|
-
return null;
|
|
1584
|
+
upsertSession(sessionId, projectId, userId, deviceId, agent = "claude-code") {
|
|
1585
|
+
const existing = this.db.query("SELECT * FROM sessions WHERE session_id = ?").get(sessionId);
|
|
1586
|
+
if (existing)
|
|
1587
|
+
return existing;
|
|
1588
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1589
|
+
this.db.query(`INSERT INTO sessions (session_id, project_id, user_id, device_id, agent, started_at_epoch)
|
|
1590
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(sessionId, projectId, userId, deviceId, agent, now);
|
|
1591
|
+
return this.db.query("SELECT * FROM sessions WHERE session_id = ?").get(sessionId);
|
|
1380
1592
|
}
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
if (obs.narrative)
|
|
1385
|
-
parts.push(obs.narrative);
|
|
1386
|
-
if (obs.facts) {
|
|
1387
|
-
try {
|
|
1388
|
-
const facts = JSON.parse(obs.facts);
|
|
1389
|
-
if (Array.isArray(facts) && facts.length > 0) {
|
|
1390
|
-
parts.push(facts.map((f) => `- ${f}`).join(`
|
|
1391
|
-
`));
|
|
1392
|
-
}
|
|
1393
|
-
} catch {
|
|
1394
|
-
parts.push(obs.facts);
|
|
1395
|
-
}
|
|
1593
|
+
completeSession(sessionId) {
|
|
1594
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1595
|
+
this.db.query("UPDATE sessions SET status = 'completed', completed_at_epoch = ? WHERE session_id = ?").run(now, sessionId);
|
|
1396
1596
|
}
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
parts.push(concepts.join(", "));
|
|
1402
|
-
}
|
|
1403
|
-
} catch {}
|
|
1597
|
+
addToOutbox(recordType, recordId) {
|
|
1598
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1599
|
+
this.db.query(`INSERT INTO sync_outbox (record_type, record_id, created_at_epoch)
|
|
1600
|
+
VALUES (?, ?, ?)`).run(recordType, recordId, now);
|
|
1404
1601
|
}
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
}
|
|
1409
|
-
async function getPipeline() {
|
|
1410
|
-
if (_pipeline)
|
|
1411
|
-
return _pipeline;
|
|
1412
|
-
if (_available === false)
|
|
1413
|
-
return null;
|
|
1414
|
-
try {
|
|
1415
|
-
const { pipeline } = await import("@xenova/transformers");
|
|
1416
|
-
_pipeline = await pipeline("feature-extraction", MODEL_NAME);
|
|
1417
|
-
_available = true;
|
|
1418
|
-
return _pipeline;
|
|
1419
|
-
} catch (err) {
|
|
1420
|
-
_available = false;
|
|
1421
|
-
console.error(`[engrm] Local embedding model unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
1422
|
-
return null;
|
|
1602
|
+
getSyncState(key) {
|
|
1603
|
+
const row = this.db.query("SELECT value FROM sync_state WHERE key = ?").get(key);
|
|
1604
|
+
return row?.value ?? null;
|
|
1423
1605
|
}
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
// src/capture/recurrence.ts
|
|
1427
|
-
var DISTANCE_THRESHOLD = 0.15;
|
|
1428
|
-
async function detectRecurrence(db, config, observation) {
|
|
1429
|
-
if (observation.type !== "bugfix") {
|
|
1430
|
-
return { patternCreated: false };
|
|
1606
|
+
setSyncState(key, value) {
|
|
1607
|
+
this.db.query("INSERT INTO sync_state (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?").run(key, value, value);
|
|
1431
1608
|
}
|
|
1432
|
-
|
|
1433
|
-
|
|
1609
|
+
ftsInsert(obs) {
|
|
1610
|
+
this.db.query(`INSERT INTO observations_fts (rowid, title, narrative, facts, concepts)
|
|
1611
|
+
VALUES (?, ?, ?, ?, ?)`).run(obs.id, obs.title, obs.narrative, obs.facts, obs.concepts);
|
|
1434
1612
|
}
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
return { patternCreated: false };
|
|
1613
|
+
ftsDelete(obs) {
|
|
1614
|
+
this.db.query(`INSERT INTO observations_fts (observations_fts, rowid, title, narrative, facts, concepts)
|
|
1615
|
+
VALUES ('delete', ?, ?, ?, ?, ?)`).run(obs.id, obs.title, obs.narrative, obs.facts, obs.concepts);
|
|
1439
1616
|
}
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
if (match.distance > DISTANCE_THRESHOLD)
|
|
1445
|
-
continue;
|
|
1446
|
-
const matched = db.getObservationById(match.observation_id);
|
|
1447
|
-
if (!matched)
|
|
1448
|
-
continue;
|
|
1449
|
-
if (matched.type !== "bugfix")
|
|
1450
|
-
continue;
|
|
1451
|
-
if (matched.session_id === observation.session_id)
|
|
1452
|
-
continue;
|
|
1453
|
-
if (await patternAlreadyExists(db, observation, matched))
|
|
1454
|
-
continue;
|
|
1455
|
-
let matchedProjectName;
|
|
1456
|
-
if (matched.project_id !== observation.project_id) {
|
|
1457
|
-
const proj = db.getProjectById(matched.project_id);
|
|
1458
|
-
if (proj)
|
|
1459
|
-
matchedProjectName = proj.name;
|
|
1460
|
-
}
|
|
1461
|
-
const similarity = 1 - match.distance;
|
|
1462
|
-
const result = await saveObservation(db, config, {
|
|
1463
|
-
type: "pattern",
|
|
1464
|
-
title: `Recurring bugfix: ${observation.title}`,
|
|
1465
|
-
narrative: `This bug pattern has appeared in multiple sessions. Original: "${matched.title}" (session ${matched.session_id?.slice(0, 8) ?? "unknown"}). Latest: "${observation.title}". Similarity: ${(similarity * 100).toFixed(0)}%. Consider addressing the root cause.`,
|
|
1466
|
-
facts: [
|
|
1467
|
-
`First seen: ${matched.created_at.split("T")[0]}`,
|
|
1468
|
-
`Recurred: ${observation.created_at.split("T")[0]}`,
|
|
1469
|
-
`Similarity: ${(similarity * 100).toFixed(0)}%`
|
|
1470
|
-
],
|
|
1471
|
-
concepts: mergeConceptsFromBoth(observation, matched),
|
|
1472
|
-
cwd: process.cwd(),
|
|
1473
|
-
session_id: observation.session_id ?? undefined
|
|
1474
|
-
});
|
|
1475
|
-
if (result.success && result.observation_id) {
|
|
1476
|
-
return {
|
|
1477
|
-
patternCreated: true,
|
|
1478
|
-
patternId: result.observation_id,
|
|
1479
|
-
matchedObservationId: matched.id,
|
|
1480
|
-
matchedProjectName,
|
|
1481
|
-
matchedTitle: matched.title,
|
|
1482
|
-
similarity
|
|
1483
|
-
};
|
|
1484
|
-
}
|
|
1617
|
+
vecInsert(observationId, embedding) {
|
|
1618
|
+
if (!this.vecAvailable)
|
|
1619
|
+
return;
|
|
1620
|
+
this.db.query("INSERT OR REPLACE INTO vec_observations (observation_id, embedding) VALUES (?, ?)").run(observationId, new Uint8Array(embedding.buffer));
|
|
1485
1621
|
}
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
WHERE type = 'pattern' AND lifecycle IN ('active', 'aging', 'pinned')
|
|
1491
|
-
AND title LIKE ?
|
|
1492
|
-
ORDER BY created_at_epoch DESC LIMIT 5`).all(`%${obs1.title.slice(0, 30)}%`);
|
|
1493
|
-
for (const p of recentPatterns) {
|
|
1494
|
-
if (p.narrative?.includes(obs2.title.slice(0, 30)))
|
|
1495
|
-
return true;
|
|
1622
|
+
vecDelete(observationId) {
|
|
1623
|
+
if (!this.vecAvailable)
|
|
1624
|
+
return;
|
|
1625
|
+
this.db.query("DELETE FROM vec_observations WHERE observation_id = ?").run(observationId);
|
|
1496
1626
|
}
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1627
|
+
searchVec(queryEmbedding, projectId, lifecycles = ["active", "aging", "pinned"], limit = 20, userId) {
|
|
1628
|
+
if (!this.vecAvailable)
|
|
1629
|
+
return [];
|
|
1630
|
+
const lifecyclePlaceholders = lifecycles.map(() => "?").join(",");
|
|
1631
|
+
const embeddingBlob = new Uint8Array(queryEmbedding.buffer);
|
|
1632
|
+
const visibilityClause = userId ? " AND (o.sensitivity != 'personal' OR o.user_id = ?)" : "";
|
|
1633
|
+
if (projectId !== null) {
|
|
1634
|
+
return this.db.query(`SELECT v.observation_id, v.distance
|
|
1635
|
+
FROM vec_observations v
|
|
1636
|
+
JOIN observations o ON o.id = v.observation_id
|
|
1637
|
+
WHERE v.embedding MATCH ?
|
|
1638
|
+
AND k = ?
|
|
1639
|
+
AND o.project_id = ?
|
|
1640
|
+
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
1641
|
+
AND o.superseded_by IS NULL` + visibilityClause).all(embeddingBlob, limit, projectId, ...lifecycles, ...userId ? [userId] : []);
|
|
1512
1642
|
}
|
|
1643
|
+
return this.db.query(`SELECT v.observation_id, v.distance
|
|
1644
|
+
FROM vec_observations v
|
|
1645
|
+
JOIN observations o ON o.id = v.observation_id
|
|
1646
|
+
WHERE v.embedding MATCH ?
|
|
1647
|
+
AND k = ?
|
|
1648
|
+
AND o.lifecycle IN (${lifecyclePlaceholders})
|
|
1649
|
+
AND o.superseded_by IS NULL` + visibilityClause).all(embeddingBlob, limit, ...lifecycles, ...userId ? [userId] : []);
|
|
1513
1650
|
}
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1651
|
+
getUnembeddedCount() {
|
|
1652
|
+
if (!this.vecAvailable)
|
|
1653
|
+
return 0;
|
|
1654
|
+
const result = this.db.query(`SELECT COUNT(*) as count FROM observations o
|
|
1655
|
+
WHERE o.lifecycle IN ('active', 'aging', 'pinned')
|
|
1656
|
+
AND o.superseded_by IS NULL
|
|
1657
|
+
AND NOT EXISTS (
|
|
1658
|
+
SELECT 1 FROM vec_observations v WHERE v.observation_id = o.id
|
|
1659
|
+
)`).get();
|
|
1660
|
+
return result?.count ?? 0;
|
|
1522
1661
|
}
|
|
1523
|
-
|
|
1524
|
-
|
|
1662
|
+
getUnembeddedObservations(limit = 100) {
|
|
1663
|
+
if (!this.vecAvailable)
|
|
1664
|
+
return [];
|
|
1665
|
+
return this.db.query(`SELECT o.* FROM observations o
|
|
1666
|
+
WHERE o.lifecycle IN ('active', 'aging', 'pinned')
|
|
1667
|
+
AND o.superseded_by IS NULL
|
|
1668
|
+
AND NOT EXISTS (
|
|
1669
|
+
SELECT 1 FROM vec_observations v WHERE v.observation_id = o.id
|
|
1670
|
+
)
|
|
1671
|
+
ORDER BY o.created_at_epoch DESC
|
|
1672
|
+
LIMIT ?`).all(limit);
|
|
1525
1673
|
}
|
|
1526
|
-
|
|
1527
|
-
|
|
1674
|
+
insertSessionSummary(summary) {
|
|
1675
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1676
|
+
const result = this.db.query(`INSERT INTO session_summaries (session_id, project_id, user_id, request, investigated, learned, completed, next_steps, created_at_epoch)
|
|
1677
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(summary.session_id, summary.project_id, summary.user_id, summary.request, summary.investigated, summary.learned, summary.completed, summary.next_steps, now);
|
|
1678
|
+
const id = Number(result.lastInsertRowid);
|
|
1679
|
+
return this.db.query("SELECT * FROM session_summaries WHERE id = ?").get(id);
|
|
1528
1680
|
}
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
async function detectViaVec(db, observation) {
|
|
1532
|
-
const text = composeEmbeddingText(observation);
|
|
1533
|
-
const embedding = await embedText(text);
|
|
1534
|
-
if (!embedding)
|
|
1535
|
-
return { hasConflict: false };
|
|
1536
|
-
const results = db.searchVec(embedding, observation.project_id, ["active", "aging", "pinned"], 10);
|
|
1537
|
-
for (const match of results) {
|
|
1538
|
-
if (match.observation_id === observation.id)
|
|
1539
|
-
continue;
|
|
1540
|
-
if (match.distance > SIMILARITY_THRESHOLD)
|
|
1541
|
-
continue;
|
|
1542
|
-
const existing = db.getObservationById(match.observation_id);
|
|
1543
|
-
if (!existing)
|
|
1544
|
-
continue;
|
|
1545
|
-
if (existing.type !== "decision")
|
|
1546
|
-
continue;
|
|
1547
|
-
if (!existing.narrative)
|
|
1548
|
-
continue;
|
|
1549
|
-
const conflict = narrativesConflict(observation.narrative, existing.narrative);
|
|
1550
|
-
if (conflict) {
|
|
1551
|
-
return {
|
|
1552
|
-
hasConflict: true,
|
|
1553
|
-
conflictingId: existing.id,
|
|
1554
|
-
conflictingTitle: existing.title,
|
|
1555
|
-
reason: conflict
|
|
1556
|
-
};
|
|
1557
|
-
}
|
|
1681
|
+
getSessionSummary(sessionId) {
|
|
1682
|
+
return this.db.query("SELECT * FROM session_summaries WHERE session_id = ?").get(sessionId) ?? null;
|
|
1558
1683
|
}
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
continue;
|
|
1572
|
-
if (existing.type !== "decision")
|
|
1573
|
-
continue;
|
|
1574
|
-
if (!existing.narrative)
|
|
1575
|
-
continue;
|
|
1576
|
-
const conflict = narrativesConflict(observation.narrative, existing.narrative);
|
|
1577
|
-
if (conflict) {
|
|
1578
|
-
return {
|
|
1579
|
-
hasConflict: true,
|
|
1580
|
-
conflictingId: existing.id,
|
|
1581
|
-
conflictingTitle: existing.title,
|
|
1582
|
-
reason: conflict
|
|
1583
|
-
};
|
|
1684
|
+
getRecentSummaries(projectId, limit = 5) {
|
|
1685
|
+
return this.db.query(`SELECT * FROM session_summaries
|
|
1686
|
+
WHERE project_id = ?
|
|
1687
|
+
ORDER BY created_at_epoch DESC, id DESC
|
|
1688
|
+
LIMIT ?`).all(projectId, limit);
|
|
1689
|
+
}
|
|
1690
|
+
incrementSessionMetrics(sessionId, increments) {
|
|
1691
|
+
const sets = [];
|
|
1692
|
+
const params = [];
|
|
1693
|
+
if (increments.files) {
|
|
1694
|
+
sets.push("files_touched_count = files_touched_count + ?");
|
|
1695
|
+
params.push(increments.files);
|
|
1584
1696
|
}
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
function narrativesConflict(narrative1, narrative2) {
|
|
1589
|
-
const n1 = narrative1.toLowerCase();
|
|
1590
|
-
const n2 = narrative2.toLowerCase();
|
|
1591
|
-
const opposingPairs = [
|
|
1592
|
-
[["should use", "decided to use", "chose", "prefer", "went with"], ["should not", "decided against", "avoid", "rejected", "don't use"]],
|
|
1593
|
-
[["enable", "turn on", "activate", "add"], ["disable", "turn off", "deactivate", "remove"]],
|
|
1594
|
-
[["increase", "more", "higher", "scale up"], ["decrease", "less", "lower", "scale down"]],
|
|
1595
|
-
[["keep", "maintain", "preserve"], ["replace", "migrate", "switch from", "deprecate"]]
|
|
1596
|
-
];
|
|
1597
|
-
for (const [positive, negative] of opposingPairs) {
|
|
1598
|
-
const n1HasPositive = positive.some((w) => n1.includes(w));
|
|
1599
|
-
const n1HasNegative = negative.some((w) => n1.includes(w));
|
|
1600
|
-
const n2HasPositive = positive.some((w) => n2.includes(w));
|
|
1601
|
-
const n2HasNegative = negative.some((w) => n2.includes(w));
|
|
1602
|
-
if (n1HasPositive && n2HasNegative || n1HasNegative && n2HasPositive) {
|
|
1603
|
-
return "Narratives suggest opposing conclusions on a similar topic";
|
|
1697
|
+
if (increments.searches) {
|
|
1698
|
+
sets.push("searches_performed = searches_performed + ?");
|
|
1699
|
+
params.push(increments.searches);
|
|
1604
1700
|
}
|
|
1701
|
+
if (increments.toolCalls) {
|
|
1702
|
+
sets.push("tool_calls_count = tool_calls_count + ?");
|
|
1703
|
+
params.push(increments.toolCalls);
|
|
1704
|
+
}
|
|
1705
|
+
if (sets.length === 0)
|
|
1706
|
+
return;
|
|
1707
|
+
params.push(sessionId);
|
|
1708
|
+
this.db.query(`UPDATE sessions SET ${sets.join(", ")} WHERE session_id = ?`).run(...params);
|
|
1605
1709
|
}
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
// src/tools/save.ts
|
|
1610
|
-
var VALID_TYPES = [
|
|
1611
|
-
"bugfix",
|
|
1612
|
-
"discovery",
|
|
1613
|
-
"decision",
|
|
1614
|
-
"pattern",
|
|
1615
|
-
"change",
|
|
1616
|
-
"feature",
|
|
1617
|
-
"refactor",
|
|
1618
|
-
"digest",
|
|
1619
|
-
"standard",
|
|
1620
|
-
"message"
|
|
1621
|
-
];
|
|
1622
|
-
async function saveObservation(db, config, input) {
|
|
1623
|
-
if (!VALID_TYPES.includes(input.type)) {
|
|
1624
|
-
return {
|
|
1625
|
-
success: false,
|
|
1626
|
-
reason: `Invalid type '${input.type}'. Must be one of: ${VALID_TYPES.join(", ")}`
|
|
1627
|
-
};
|
|
1710
|
+
getSessionMetrics(sessionId) {
|
|
1711
|
+
return this.db.query("SELECT * FROM sessions WHERE session_id = ?").get(sessionId) ?? null;
|
|
1628
1712
|
}
|
|
1629
|
-
|
|
1630
|
-
|
|
1713
|
+
insertSecurityFinding(finding) {
|
|
1714
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1715
|
+
const result = this.db.query(`INSERT INTO security_findings (session_id, project_id, finding_type, severity, pattern_name, file_path, snippet, tool_name, user_id, device_id, created_at_epoch)
|
|
1716
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(finding.session_id ?? null, finding.project_id, finding.finding_type, finding.severity, finding.pattern_name, finding.file_path ?? null, finding.snippet ?? null, finding.tool_name ?? null, finding.user_id, finding.device_id, now);
|
|
1717
|
+
const id = Number(result.lastInsertRowid);
|
|
1718
|
+
return this.db.query("SELECT * FROM security_findings WHERE id = ?").get(id);
|
|
1631
1719
|
}
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
});
|
|
1640
|
-
const customPatterns = config.scrubbing.enabled ? config.scrubbing.custom_patterns : [];
|
|
1641
|
-
const title = config.scrubbing.enabled ? scrubSecrets(input.title, customPatterns) : input.title;
|
|
1642
|
-
const narrative = input.narrative ? config.scrubbing.enabled ? scrubSecrets(input.narrative, customPatterns) : input.narrative : null;
|
|
1643
|
-
const factsJson = input.facts ? config.scrubbing.enabled ? scrubSecrets(JSON.stringify(input.facts), customPatterns) : JSON.stringify(input.facts) : null;
|
|
1644
|
-
const conceptsJson = input.concepts ? JSON.stringify(input.concepts) : null;
|
|
1645
|
-
const filesRead = input.files_read ? input.files_read.map((f) => toRelativePath(f, cwd)) : null;
|
|
1646
|
-
const filesModified = input.files_modified ? input.files_modified.map((f) => toRelativePath(f, cwd)) : null;
|
|
1647
|
-
const filesReadJson = filesRead ? JSON.stringify(filesRead) : null;
|
|
1648
|
-
const filesModifiedJson = filesModified ? JSON.stringify(filesModified) : null;
|
|
1649
|
-
let sensitivity = input.sensitivity ?? config.scrubbing.default_sensitivity;
|
|
1650
|
-
if (config.scrubbing.enabled && containsSecrets([input.title, input.narrative, JSON.stringify(input.facts)].filter(Boolean).join(" "), customPatterns)) {
|
|
1651
|
-
if (sensitivity === "shared") {
|
|
1652
|
-
sensitivity = "personal";
|
|
1720
|
+
getSecurityFindings(projectId, options = {}) {
|
|
1721
|
+
const limit = options.limit ?? 50;
|
|
1722
|
+
if (options.severity) {
|
|
1723
|
+
return this.db.query(`SELECT * FROM security_findings
|
|
1724
|
+
WHERE project_id = ? AND severity = ?
|
|
1725
|
+
ORDER BY created_at_epoch DESC
|
|
1726
|
+
LIMIT ?`).all(projectId, options.severity, limit);
|
|
1653
1727
|
}
|
|
1728
|
+
return this.db.query(`SELECT * FROM security_findings
|
|
1729
|
+
WHERE project_id = ?
|
|
1730
|
+
ORDER BY created_at_epoch DESC
|
|
1731
|
+
LIMIT ?`).all(projectId, limit);
|
|
1654
1732
|
}
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
title,
|
|
1665
|
-
narrative,
|
|
1666
|
-
facts: factsJson,
|
|
1667
|
-
concepts: conceptsJson,
|
|
1668
|
-
filesRead,
|
|
1669
|
-
filesModified,
|
|
1670
|
-
isDuplicate: duplicate !== null
|
|
1671
|
-
};
|
|
1672
|
-
const qualityScore = scoreQuality(qualityInput);
|
|
1673
|
-
if (!meetsQualityThreshold(qualityInput)) {
|
|
1674
|
-
return {
|
|
1675
|
-
success: false,
|
|
1676
|
-
quality_score: qualityScore,
|
|
1677
|
-
reason: `Quality score ${qualityScore.toFixed(2)} below threshold`
|
|
1733
|
+
getSecurityFindingsCount(projectId) {
|
|
1734
|
+
const rows = this.db.query(`SELECT severity, COUNT(*) as count FROM security_findings
|
|
1735
|
+
WHERE project_id = ?
|
|
1736
|
+
GROUP BY severity`).all(projectId);
|
|
1737
|
+
const counts = {
|
|
1738
|
+
critical: 0,
|
|
1739
|
+
high: 0,
|
|
1740
|
+
medium: 0,
|
|
1741
|
+
low: 0
|
|
1678
1742
|
};
|
|
1743
|
+
for (const row of rows) {
|
|
1744
|
+
counts[row.severity] = row.count;
|
|
1745
|
+
}
|
|
1746
|
+
return counts;
|
|
1679
1747
|
}
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
success: true,
|
|
1683
|
-
merged_into: duplicate.id,
|
|
1684
|
-
quality_score: qualityScore,
|
|
1685
|
-
reason: `Merged into existing observation #${duplicate.id}`
|
|
1686
|
-
};
|
|
1748
|
+
setSessionRiskScore(sessionId, score) {
|
|
1749
|
+
this.db.query("UPDATE sessions SET risk_score = ? WHERE session_id = ?").run(score, sessionId);
|
|
1687
1750
|
}
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
project_id: project.id,
|
|
1691
|
-
type: input.type,
|
|
1692
|
-
title,
|
|
1693
|
-
narrative,
|
|
1694
|
-
facts: factsJson,
|
|
1695
|
-
concepts: conceptsJson,
|
|
1696
|
-
files_read: filesReadJson,
|
|
1697
|
-
files_modified: filesModifiedJson,
|
|
1698
|
-
quality: qualityScore,
|
|
1699
|
-
lifecycle: "active",
|
|
1700
|
-
sensitivity,
|
|
1701
|
-
user_id: config.user_id,
|
|
1702
|
-
device_id: config.device_id,
|
|
1703
|
-
agent: input.agent ?? "claude-code"
|
|
1704
|
-
});
|
|
1705
|
-
db.addToOutbox("observation", obs.id);
|
|
1706
|
-
if (db.vecAvailable) {
|
|
1707
|
-
try {
|
|
1708
|
-
const text = composeEmbeddingText(obs);
|
|
1709
|
-
const embedding = await embedText(text);
|
|
1710
|
-
if (embedding) {
|
|
1711
|
-
db.vecInsert(obs.id, embedding);
|
|
1712
|
-
}
|
|
1713
|
-
} catch {}
|
|
1751
|
+
getObservationsBySession(sessionId) {
|
|
1752
|
+
return this.db.query(`SELECT * FROM observations WHERE session_id = ? ORDER BY created_at_epoch ASC`).all(sessionId);
|
|
1714
1753
|
}
|
|
1715
|
-
|
|
1716
|
-
if (input.type === "bugfix") {
|
|
1754
|
+
getInstalledPacks() {
|
|
1717
1755
|
try {
|
|
1718
|
-
const
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
} catch {}
|
|
1756
|
+
const rows = this.db.query("SELECT name FROM packs_installed").all();
|
|
1757
|
+
return rows.map((r) => r.name);
|
|
1758
|
+
} catch {
|
|
1759
|
+
return [];
|
|
1760
|
+
}
|
|
1724
1761
|
}
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
const conflict = await detectDecisionConflict(db, obs);
|
|
1729
|
-
if (conflict.hasConflict && conflict.conflictingTitle) {
|
|
1730
|
-
conflictWarning = `Potential conflict with existing decision: "${conflict.conflictingTitle}" — ${conflict.reason}`;
|
|
1731
|
-
}
|
|
1732
|
-
} catch {}
|
|
1762
|
+
markPackInstalled(name, observationCount) {
|
|
1763
|
+
const now = Math.floor(Date.now() / 1000);
|
|
1764
|
+
this.db.query("INSERT OR REPLACE INTO packs_installed (name, installed_at, observation_count) VALUES (?, ?, ?)").run(name, now, observationCount);
|
|
1733
1765
|
}
|
|
1734
|
-
return {
|
|
1735
|
-
success: true,
|
|
1736
|
-
observation_id: obs.id,
|
|
1737
|
-
quality_score: qualityScore,
|
|
1738
|
-
recall_hint: recallHint,
|
|
1739
|
-
conflict_warning: conflictWarning
|
|
1740
|
-
};
|
|
1741
1766
|
}
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1767
|
+
|
|
1768
|
+
// src/hooks/common.ts
|
|
1769
|
+
var c = {
|
|
1770
|
+
dim: "\x1B[2m",
|
|
1771
|
+
yellow: "\x1B[33m",
|
|
1772
|
+
reset: "\x1B[0m"
|
|
1773
|
+
};
|
|
1774
|
+
async function readStdin() {
|
|
1775
|
+
const chunks = [];
|
|
1776
|
+
for await (const chunk of process.stdin) {
|
|
1777
|
+
chunks.push(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString());
|
|
1778
|
+
}
|
|
1779
|
+
return chunks.join("");
|
|
1780
|
+
}
|
|
1781
|
+
async function parseStdinJson() {
|
|
1782
|
+
const raw = await readStdin();
|
|
1783
|
+
if (!raw.trim())
|
|
1784
|
+
return null;
|
|
1785
|
+
try {
|
|
1786
|
+
return JSON.parse(raw);
|
|
1787
|
+
} catch {
|
|
1788
|
+
return null;
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
function bootstrapHook(hookName) {
|
|
1792
|
+
if (!configExists()) {
|
|
1793
|
+
warnUser(hookName, "Engrm not configured. Run: npx engrm init");
|
|
1794
|
+
return null;
|
|
1795
|
+
}
|
|
1796
|
+
let config;
|
|
1797
|
+
try {
|
|
1798
|
+
config = loadConfig();
|
|
1799
|
+
} catch (err) {
|
|
1800
|
+
warnUser(hookName, `Config error: ${err instanceof Error ? err.message : String(err)}`);
|
|
1801
|
+
return null;
|
|
1802
|
+
}
|
|
1803
|
+
let db;
|
|
1804
|
+
try {
|
|
1805
|
+
db = new MemDatabase(getDbPath());
|
|
1806
|
+
} catch (err) {
|
|
1807
|
+
warnUser(hookName, `Database error: ${err instanceof Error ? err.message : String(err)}`);
|
|
1808
|
+
return null;
|
|
1809
|
+
}
|
|
1810
|
+
return { config, db };
|
|
1811
|
+
}
|
|
1812
|
+
function warnUser(hookName, message) {
|
|
1813
|
+
console.error(`${c.yellow}engrm ${hookName}:${c.reset} ${c.dim}${message}${c.reset}`);
|
|
1814
|
+
}
|
|
1815
|
+
function runHook(hookName, fn) {
|
|
1816
|
+
fn().catch((err) => {
|
|
1817
|
+
warnUser(hookName, `Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
|
|
1818
|
+
process.exit(0);
|
|
1819
|
+
});
|
|
1749
1820
|
}
|
|
1750
1821
|
|
|
1751
1822
|
// src/capture/scanner.ts
|
|
@@ -1819,33 +1890,17 @@ var SENSITIVE_FIELD_PATTERNS = [
|
|
|
1819
1890
|
/bearer/i
|
|
1820
1891
|
];
|
|
1821
1892
|
async function main() {
|
|
1822
|
-
const
|
|
1823
|
-
|
|
1824
|
-
chunks.push(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString());
|
|
1825
|
-
}
|
|
1826
|
-
const raw = chunks.join("");
|
|
1827
|
-
if (!raw.trim())
|
|
1893
|
+
const event = await parseStdinJson();
|
|
1894
|
+
if (!event)
|
|
1828
1895
|
process.exit(0);
|
|
1829
|
-
let event;
|
|
1830
|
-
try {
|
|
1831
|
-
event = JSON.parse(raw);
|
|
1832
|
-
} catch {
|
|
1833
|
-
process.exit(0);
|
|
1834
|
-
}
|
|
1835
1896
|
if (event.action !== "accept" || !event.content)
|
|
1836
1897
|
process.exit(0);
|
|
1837
1898
|
if (event.mcp_server_name === "engrm")
|
|
1838
1899
|
process.exit(0);
|
|
1839
|
-
|
|
1900
|
+
const boot = bootstrapHook("elicitation");
|
|
1901
|
+
if (!boot)
|
|
1840
1902
|
process.exit(0);
|
|
1841
|
-
|
|
1842
|
-
let db;
|
|
1843
|
-
try {
|
|
1844
|
-
config = loadConfig();
|
|
1845
|
-
db = new MemDatabase(getDbPath());
|
|
1846
|
-
} catch {
|
|
1847
|
-
process.exit(0);
|
|
1848
|
-
}
|
|
1903
|
+
const { config, db } = boot;
|
|
1849
1904
|
try {
|
|
1850
1905
|
const contentStr = JSON.stringify(event.content);
|
|
1851
1906
|
const findings = scanForSecrets(contentStr, config.scrubbing.custom_patterns);
|
|
@@ -1918,6 +1973,4 @@ function summarizeValue(value) {
|
|
|
1918
1973
|
}
|
|
1919
1974
|
return JSON.stringify(value).slice(0, 80);
|
|
1920
1975
|
}
|
|
1921
|
-
main
|
|
1922
|
-
process.exit(0);
|
|
1923
|
-
});
|
|
1976
|
+
runHook("elicitation", main);
|