codetrap 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +159 -51
- package/docs/installation.md +113 -29
- package/package.json +4 -3
- package/plugins/codetrap-agent/.codex-plugin/plugin.json +1 -2
- package/plugins/codetrap-agent/hooks/post-flight-capture.example.md +19 -17
- package/plugins/codetrap-agent/hooks.json +2 -2
- package/{skills → plugins/codetrap-agent/skills}/codetrap-add/SKILL.md +10 -4
- package/plugins/codetrap-agent/skills/codetrap-capture/SKILL.md +14 -3
- package/plugins/codetrap-agent/skills/codetrap-capture-external/SKILL.md +52 -9
- package/plugins/codetrap-agent/skills/codetrap-check/SKILL.md +74 -6
- package/{skills → plugins/codetrap-agent/skills}/codetrap-search/SKILL.md +6 -5
- package/plugins/codetrap-agent/templates/AGENTS.codetrap.md +31 -5
- package/scripts/search-policy-sweep.ts +131 -0
- package/src/commands/workflow.ts +186 -68
- package/src/db/connection.ts +6 -6
- package/src/db/embedding-queries.ts +230 -48
- package/src/db/queries.ts +0 -25
- package/src/db/repository.ts +32 -21
- package/src/db/schema.ts +80 -0
- package/src/index.ts +32 -7
- package/src/lib/command-requests.ts +134 -1
- package/src/lib/config.ts +57 -7
- package/src/lib/constants.ts +1 -1
- package/src/lib/doctor.ts +96 -6
- package/src/lib/embed-output.ts +26 -0
- package/src/lib/embedder.ts +118 -3
- package/src/lib/embedding-health.ts +3 -1
- package/src/lib/embedding-job.ts +3 -0
- package/src/lib/embedding-management.ts +65 -0
- package/src/lib/embedding-runtime.ts +177 -0
- package/src/lib/output-json.ts +0 -2
- package/src/lib/scope-context.ts +17 -11
- package/src/lib/scope-migration.ts +2 -1
- package/src/lib/scope.ts +4 -6
- package/src/lib/search-eval.ts +136 -23
- package/src/lib/search-policy-sweep.ts +563 -0
- package/src/lib/search-policy.ts +0 -4
- package/src/lib/search-service.ts +14 -15
- package/src/lib/session-candidate-document.ts +175 -0
- package/src/lib/session-candidate-scope.ts +6 -0
- package/src/lib/session-capture.ts +298 -32
- package/src/lib/session-codec.ts +1 -8
- package/src/lib/session-operations.ts +111 -51
- package/src/lib/session-review.ts +327 -0
- package/src/lib/session-store.ts +177 -55
- package/src/lib/store.ts +79 -11
- package/src/lib/string-list.ts +3 -0
- package/src/lib/text-lines.ts +7 -0
- package/src/lib/trap-search-document.ts +2 -1
- package/src/lib/value-types.ts +3 -0
- package/src/web/client-review.ts +171 -0
- package/src/web/client-script.ts +1543 -0
- package/src/web/client-shell.ts +414 -0
- package/src/web/client-text.ts +447 -0
- package/src/web/project-registry.ts +3 -5
- package/src/web/server.ts +184 -111
- package/src/web/static.ts +581 -484
- package/skills/codetrap-capture-external/SKILL.md +0 -62
- package/skills/codetrap-check/SKILL.md +0 -69
- package/src/lib/embedding-index.ts +0 -53
|
@@ -5,15 +5,17 @@ import {
|
|
|
5
5
|
decodeEmbedding,
|
|
6
6
|
encodeEmbedding,
|
|
7
7
|
type EmbeddingConfig,
|
|
8
|
+
embeddingProfileId,
|
|
8
9
|
type FreshEmbedding,
|
|
9
10
|
type StoredEmbedding,
|
|
10
11
|
} from "../lib/embedder";
|
|
11
12
|
import type { EmbeddingStateCounts } from "../lib/embedding-health";
|
|
12
|
-
import {
|
|
13
|
-
import { countTraps
|
|
13
|
+
import { passageHashForTrap } from "../lib/trap-search-document";
|
|
14
|
+
import { countTraps } from "./queries";
|
|
14
15
|
|
|
15
16
|
type TrapEmbeddingRow = {
|
|
16
17
|
trap_id: number;
|
|
18
|
+
profile_id: string;
|
|
17
19
|
provider: string;
|
|
18
20
|
model: string;
|
|
19
21
|
dimensions: number;
|
|
@@ -23,33 +25,98 @@ type TrapEmbeddingRow = {
|
|
|
23
25
|
updated_at: string;
|
|
24
26
|
};
|
|
25
27
|
|
|
26
|
-
export
|
|
28
|
+
export type EmbeddingProfileSummary = {
|
|
29
|
+
id: string;
|
|
30
|
+
provider: string;
|
|
31
|
+
model: string;
|
|
32
|
+
dimensions: number;
|
|
33
|
+
passage_version: number;
|
|
34
|
+
embedding_count: number;
|
|
35
|
+
updated_at: string | null;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
type EmbeddingProfileRow = Omit<EmbeddingProfileSummary, "embedding_count"> & {
|
|
39
|
+
embedding_count: number;
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
type TrapEmbeddingStateRow = Trap & {
|
|
43
|
+
embedding_passage_hash: string | null;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
type TrapAnyEmbeddingStateRow = Trap & {
|
|
47
|
+
has_embedding: number;
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export function getEmbedding(
|
|
51
|
+
db: Database,
|
|
52
|
+
trapId: number,
|
|
53
|
+
config?: EmbeddingConfig
|
|
54
|
+
): StoredEmbedding | null {
|
|
55
|
+
const profileClause = config ? "AND e.profile_id = ?" : "";
|
|
56
|
+
const params: SQLQueryBindings[] = [trapId];
|
|
57
|
+
if (config) params.push(embeddingProfileId(config));
|
|
58
|
+
|
|
27
59
|
const row = db
|
|
28
|
-
.query(
|
|
29
|
-
|
|
60
|
+
.query(`
|
|
61
|
+
SELECT
|
|
62
|
+
e.trap_id,
|
|
63
|
+
e.profile_id,
|
|
64
|
+
p.provider,
|
|
65
|
+
p.model,
|
|
66
|
+
p.dimensions,
|
|
67
|
+
p.passage_version,
|
|
68
|
+
e.passage_hash,
|
|
69
|
+
e.embedding,
|
|
70
|
+
e.updated_at
|
|
71
|
+
FROM trap_embeddings e
|
|
72
|
+
JOIN embedding_profiles p ON p.id = e.profile_id
|
|
73
|
+
WHERE e.trap_id = ? ${profileClause}
|
|
74
|
+
ORDER BY e.updated_at DESC
|
|
75
|
+
LIMIT 1
|
|
76
|
+
`)
|
|
77
|
+
.get(...params) as TrapEmbeddingRow | null;
|
|
30
78
|
return row ? rowToStoredEmbedding(row) : null;
|
|
31
79
|
}
|
|
32
80
|
|
|
33
81
|
export function upsertEmbedding(db: Database, record: StoredEmbedding): void {
|
|
82
|
+
const profileId = record.profile_id || embeddingProfileId({
|
|
83
|
+
provider: record.provider,
|
|
84
|
+
model: record.model,
|
|
85
|
+
dimensions: record.dimensions,
|
|
86
|
+
passageVersion: record.passage_version,
|
|
87
|
+
});
|
|
88
|
+
|
|
34
89
|
db.prepare(`
|
|
35
|
-
INSERT INTO
|
|
36
|
-
|
|
90
|
+
INSERT INTO embedding_profiles (
|
|
91
|
+
id, provider, model, dimensions, passage_version, created_at, updated_at
|
|
37
92
|
)
|
|
38
|
-
VALUES (?, ?, ?, ?, ?,
|
|
39
|
-
ON CONFLICT(
|
|
93
|
+
VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))
|
|
94
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
40
95
|
provider = excluded.provider,
|
|
41
96
|
model = excluded.model,
|
|
42
97
|
dimensions = excluded.dimensions,
|
|
43
98
|
passage_version = excluded.passage_version,
|
|
44
|
-
passage_hash = excluded.passage_hash,
|
|
45
|
-
embedding = excluded.embedding,
|
|
46
99
|
updated_at = datetime('now')
|
|
47
100
|
`).run(
|
|
48
|
-
|
|
101
|
+
profileId,
|
|
49
102
|
record.provider,
|
|
50
103
|
record.model,
|
|
51
104
|
record.dimensions,
|
|
52
|
-
record.passage_version
|
|
105
|
+
record.passage_version
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
db.prepare(`
|
|
109
|
+
INSERT INTO trap_embeddings (
|
|
110
|
+
trap_id, profile_id, passage_hash, embedding, updated_at
|
|
111
|
+
)
|
|
112
|
+
VALUES (?, ?, ?, ?, datetime('now'))
|
|
113
|
+
ON CONFLICT(trap_id, profile_id) DO UPDATE SET
|
|
114
|
+
passage_hash = excluded.passage_hash,
|
|
115
|
+
embedding = excluded.embedding,
|
|
116
|
+
updated_at = datetime('now')
|
|
117
|
+
`).run(
|
|
118
|
+
record.trap_id,
|
|
119
|
+
profileId,
|
|
53
120
|
record.passage_hash,
|
|
54
121
|
encodeEmbedding(record.embedding)
|
|
55
122
|
);
|
|
@@ -65,16 +132,10 @@ export function getAllFreshEmbeddings(
|
|
|
65
132
|
opts: { category?: string; scope?: string; status?: TrapStatus | "all" } = {}
|
|
66
133
|
): FreshEmbedding[] {
|
|
67
134
|
const conditions = [
|
|
68
|
-
"e.
|
|
69
|
-
"e.model = ?",
|
|
70
|
-
"e.dimensions = ?",
|
|
71
|
-
"e.passage_version = ?",
|
|
135
|
+
"e.profile_id = ?",
|
|
72
136
|
];
|
|
73
137
|
const params: SQLQueryBindings[] = [
|
|
74
|
-
config
|
|
75
|
-
config.model,
|
|
76
|
-
config.dimensions,
|
|
77
|
-
config.passageVersion,
|
|
138
|
+
embeddingProfileId(config),
|
|
78
139
|
];
|
|
79
140
|
|
|
80
141
|
if (opts.category) {
|
|
@@ -116,22 +177,11 @@ export function getTrapsNeedingEmbeddings(
|
|
|
116
177
|
config: EmbeddingConfig,
|
|
117
178
|
opts: { scope?: string; category?: string; status?: TrapStatus | "all"; force?: boolean; limit?: number } = {}
|
|
118
179
|
): Trap[] {
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
});
|
|
125
|
-
const needed: Trap[] = [];
|
|
126
|
-
|
|
127
|
-
for (const trap of traps) {
|
|
128
|
-
const embedding = getEmbedding(db, trap.id);
|
|
129
|
-
if (opts.force || !embeddingIsFresh(trap, embedding, config)) {
|
|
130
|
-
needed.push(trap);
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
return needed.slice(0, opts.limit ?? needed.length);
|
|
180
|
+
const rows = trapEmbeddingStateRows(db, config, opts);
|
|
181
|
+
const needed = opts.force
|
|
182
|
+
? rows
|
|
183
|
+
: rows.filter((row) => row.embedding_passage_hash !== passageHashForTrap(row));
|
|
184
|
+
return needed.map(rowToTrap).slice(0, opts.limit ?? needed.length);
|
|
135
185
|
}
|
|
136
186
|
|
|
137
187
|
export function countEmbeddableTraps(
|
|
@@ -146,24 +196,20 @@ export function getEmbeddingStateCounts(
|
|
|
146
196
|
config: EmbeddingConfig | null,
|
|
147
197
|
opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
|
|
148
198
|
): EmbeddingStateCounts {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
status: opts.status,
|
|
153
|
-
limit: 100000,
|
|
154
|
-
});
|
|
199
|
+
if (!config) return getAnyEmbeddingStateCounts(db, opts);
|
|
200
|
+
|
|
201
|
+
const rows = trapEmbeddingStateRows(db, config, opts);
|
|
155
202
|
const counts: EmbeddingStateCounts = {
|
|
156
|
-
total:
|
|
203
|
+
total: rows.length,
|
|
157
204
|
fresh: 0,
|
|
158
205
|
stale: 0,
|
|
159
206
|
missing: 0,
|
|
160
207
|
};
|
|
161
208
|
|
|
162
|
-
for (const
|
|
163
|
-
|
|
164
|
-
if (!embedding) {
|
|
209
|
+
for (const row of rows) {
|
|
210
|
+
if (!row.embedding_passage_hash) {
|
|
165
211
|
counts.missing++;
|
|
166
|
-
} else if (
|
|
212
|
+
} else if (row.embedding_passage_hash === passageHashForTrap(row)) {
|
|
167
213
|
counts.fresh++;
|
|
168
214
|
} else {
|
|
169
215
|
counts.stale++;
|
|
@@ -173,9 +219,56 @@ export function getEmbeddingStateCounts(
|
|
|
173
219
|
return counts;
|
|
174
220
|
}
|
|
175
221
|
|
|
222
|
+
export function listEmbeddingProfiles(
|
|
223
|
+
db: Database,
|
|
224
|
+
opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
|
|
225
|
+
): EmbeddingProfileSummary[] {
|
|
226
|
+
const conditions: string[] = [];
|
|
227
|
+
const params: SQLQueryBindings[] = [];
|
|
228
|
+
|
|
229
|
+
if (opts.category) {
|
|
230
|
+
conditions.push("t.category = ?");
|
|
231
|
+
params.push(opts.category);
|
|
232
|
+
}
|
|
233
|
+
if (opts.scope) {
|
|
234
|
+
conditions.push("t.scope = ?");
|
|
235
|
+
params.push(opts.scope);
|
|
236
|
+
}
|
|
237
|
+
if (opts.status !== "all") {
|
|
238
|
+
conditions.push("t.status = ?");
|
|
239
|
+
params.push(opts.status ?? DEFAULT_TRAP_STATUS);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
243
|
+
const rows = db
|
|
244
|
+
.query(`
|
|
245
|
+
SELECT
|
|
246
|
+
p.id,
|
|
247
|
+
p.provider,
|
|
248
|
+
p.model,
|
|
249
|
+
p.dimensions,
|
|
250
|
+
p.passage_version,
|
|
251
|
+
COUNT(e.trap_id) AS embedding_count,
|
|
252
|
+
MAX(e.updated_at) AS updated_at
|
|
253
|
+
FROM embedding_profiles p
|
|
254
|
+
JOIN trap_embeddings e ON e.profile_id = p.id
|
|
255
|
+
JOIN traps t ON t.id = e.trap_id
|
|
256
|
+
${where}
|
|
257
|
+
GROUP BY p.id, p.provider, p.model, p.dimensions, p.passage_version
|
|
258
|
+
ORDER BY updated_at DESC, p.provider ASC, p.model ASC
|
|
259
|
+
`)
|
|
260
|
+
.all(...params) as EmbeddingProfileRow[];
|
|
261
|
+
|
|
262
|
+
return rows.map((row) => ({
|
|
263
|
+
...row,
|
|
264
|
+
embedding_count: Number(row.embedding_count),
|
|
265
|
+
}));
|
|
266
|
+
}
|
|
267
|
+
|
|
176
268
|
function rowToStoredEmbedding(row: TrapEmbeddingRow): StoredEmbedding {
|
|
177
269
|
return {
|
|
178
270
|
trap_id: row.trap_id,
|
|
271
|
+
profile_id: row.profile_id,
|
|
179
272
|
provider: row.provider,
|
|
180
273
|
model: row.model,
|
|
181
274
|
dimensions: row.dimensions,
|
|
@@ -185,3 +278,92 @@ function rowToStoredEmbedding(row: TrapEmbeddingRow): StoredEmbedding {
|
|
|
185
278
|
updated_at: row.updated_at,
|
|
186
279
|
};
|
|
187
280
|
}
|
|
281
|
+
|
|
282
|
+
function getAnyEmbeddingStateCounts(
|
|
283
|
+
db: Database,
|
|
284
|
+
opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
|
|
285
|
+
): EmbeddingStateCounts {
|
|
286
|
+
const rows = trapAnyEmbeddingStateRows(db, opts);
|
|
287
|
+
const stale = rows.filter((row) => row.has_embedding > 0).length;
|
|
288
|
+
return {
|
|
289
|
+
total: rows.length,
|
|
290
|
+
fresh: 0,
|
|
291
|
+
stale,
|
|
292
|
+
missing: rows.length - stale,
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function trapEmbeddingStateRows(
|
|
297
|
+
db: Database,
|
|
298
|
+
config: EmbeddingConfig,
|
|
299
|
+
opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
|
|
300
|
+
): TrapEmbeddingStateRow[] {
|
|
301
|
+
const conditions: string[] = [];
|
|
302
|
+
const filterParams = trapFilterParams(conditions, opts, "t");
|
|
303
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
304
|
+
return db
|
|
305
|
+
.query(`
|
|
306
|
+
SELECT
|
|
307
|
+
t.*,
|
|
308
|
+
e.passage_hash AS embedding_passage_hash
|
|
309
|
+
FROM traps t
|
|
310
|
+
LEFT JOIN trap_embeddings e
|
|
311
|
+
ON e.trap_id = t.id
|
|
312
|
+
AND e.profile_id = ?
|
|
313
|
+
${where}
|
|
314
|
+
ORDER BY t.updated_at DESC
|
|
315
|
+
LIMIT 100000
|
|
316
|
+
`)
|
|
317
|
+
.all(embeddingProfileId(config), ...filterParams) as TrapEmbeddingStateRow[];
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function trapAnyEmbeddingStateRows(
|
|
321
|
+
db: Database,
|
|
322
|
+
opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
|
|
323
|
+
): TrapAnyEmbeddingStateRow[] {
|
|
324
|
+
const conditions: string[] = [];
|
|
325
|
+
const params = trapFilterParams(conditions, opts, "t");
|
|
326
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
327
|
+
return db
|
|
328
|
+
.query(`
|
|
329
|
+
SELECT
|
|
330
|
+
t.*,
|
|
331
|
+
EXISTS (
|
|
332
|
+
SELECT 1
|
|
333
|
+
FROM trap_embeddings e
|
|
334
|
+
WHERE e.trap_id = t.id
|
|
335
|
+
) AS has_embedding
|
|
336
|
+
FROM traps t
|
|
337
|
+
${where}
|
|
338
|
+
ORDER BY t.updated_at DESC
|
|
339
|
+
LIMIT 100000
|
|
340
|
+
`)
|
|
341
|
+
.all(...params) as TrapAnyEmbeddingStateRow[];
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
function trapFilterParams(
|
|
345
|
+
conditions: string[],
|
|
346
|
+
opts: { scope?: string; category?: string; status?: TrapStatus | "all" },
|
|
347
|
+
alias: string
|
|
348
|
+
): SQLQueryBindings[] {
|
|
349
|
+
const params: SQLQueryBindings[] = [];
|
|
350
|
+
const prefix = `${alias}.`;
|
|
351
|
+
if (opts.category) {
|
|
352
|
+
conditions.push(`${prefix}category = ?`);
|
|
353
|
+
params.push(opts.category);
|
|
354
|
+
}
|
|
355
|
+
if (opts.scope) {
|
|
356
|
+
conditions.push(`${prefix}scope = ?`);
|
|
357
|
+
params.push(opts.scope);
|
|
358
|
+
}
|
|
359
|
+
if (opts.status !== "all") {
|
|
360
|
+
conditions.push(`${prefix}status = ?`);
|
|
361
|
+
params.push(opts.status ?? DEFAULT_TRAP_STATUS);
|
|
362
|
+
}
|
|
363
|
+
return params;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function rowToTrap(row: TrapEmbeddingStateRow): Trap {
|
|
367
|
+
const { embedding_passage_hash: _embeddingPassageHash, ...trap } = row;
|
|
368
|
+
return trap as Trap;
|
|
369
|
+
}
|
package/src/db/queries.ts
CHANGED
|
@@ -179,10 +179,6 @@ export function listTrapEvidence(db: Database, trapId: number): TrapEvidence[] {
|
|
|
179
179
|
.all(trapId) as TrapEvidence[];
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
-
export function archiveTrap(db: Database, id: number): boolean {
|
|
183
|
-
return markTrapArchived(db, id);
|
|
184
|
-
}
|
|
185
|
-
|
|
186
182
|
export function markTrapArchived(db: Database, id: number): boolean {
|
|
187
183
|
const result = db
|
|
188
184
|
.prepare(
|
|
@@ -198,27 +194,6 @@ export function markTrapArchived(db: Database, id: number): boolean {
|
|
|
198
194
|
return result.changes > 0;
|
|
199
195
|
}
|
|
200
196
|
|
|
201
|
-
export function supersedeTrap(
|
|
202
|
-
db: Database,
|
|
203
|
-
id: number,
|
|
204
|
-
supersededById: number,
|
|
205
|
-
stateKey?: string
|
|
206
|
-
): boolean {
|
|
207
|
-
if (id === supersededById) return false;
|
|
208
|
-
|
|
209
|
-
const oldTrap = getTrap(db, id);
|
|
210
|
-
const newTrap = getTrap(db, supersededById);
|
|
211
|
-
if (!oldTrap || !newTrap) return false;
|
|
212
|
-
|
|
213
|
-
const key = stateKey ?? oldTrap.state_key ?? newTrap.state_key ?? `trap:${id}`;
|
|
214
|
-
const tx = db.transaction(() => {
|
|
215
|
-
markTrapSuperseded(db, id, key);
|
|
216
|
-
markTrapSuperseding(db, supersededById, id, key);
|
|
217
|
-
});
|
|
218
|
-
tx();
|
|
219
|
-
return true;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
197
|
export function markTrapSuperseded(db: Database, id: number, stateKey: string): boolean {
|
|
223
198
|
const result = db.prepare(supersedeTrapSql).run(stateKey, id);
|
|
224
199
|
return result.changes > 0;
|
package/src/db/repository.ts
CHANGED
|
@@ -11,32 +11,39 @@ import type {
|
|
|
11
11
|
import { SearchService, type SearchOptions } from "../lib/search-service";
|
|
12
12
|
import {
|
|
13
13
|
type EmbeddingConfig,
|
|
14
|
-
type EmbeddingProvider,
|
|
15
14
|
type StoredEmbedding,
|
|
16
15
|
} from "../lib/embedder";
|
|
16
|
+
import {
|
|
17
|
+
embeddingRuntimeFrom,
|
|
18
|
+
type EmbeddingRuntime,
|
|
19
|
+
type EmbeddingRuntimeInput,
|
|
20
|
+
} from "../lib/embedding-runtime";
|
|
17
21
|
import { runEmbeddingJob } from "../lib/embedding-job";
|
|
18
22
|
import { passageFieldsChanged } from "../lib/trap-search-document";
|
|
23
|
+
import * as embeddingQueries from "./embedding-queries";
|
|
19
24
|
import * as queries from "./queries";
|
|
20
25
|
import type { TrapStatus } from "../lib/constants";
|
|
21
26
|
import { TrapSearchPolicy } from "../lib/search-policy";
|
|
22
|
-
import {
|
|
27
|
+
import type { RankingConfig } from "../lib/search-policy";
|
|
23
28
|
import { archiveTrapLifecycle, supersedeTrapLifecycle } from "../lib/trap-lifecycle";
|
|
29
|
+
import type { EmbeddingProfileSummary } from "./embedding-queries";
|
|
24
30
|
|
|
25
31
|
export type TrapStats = ReturnType<typeof queries.getStats>;
|
|
26
|
-
export type EmbeddingStateCounts = ReturnType<
|
|
32
|
+
export type EmbeddingStateCounts = ReturnType<typeof embeddingQueries.getEmbeddingStateCounts>;
|
|
27
33
|
export type TrapRecordInsert = queries.TrapRecordInsert;
|
|
28
34
|
|
|
29
35
|
export class TrapRepository {
|
|
30
36
|
private readonly searchService: SearchService;
|
|
31
37
|
private readonly searchPolicy = new TrapSearchPolicy();
|
|
32
|
-
private readonly
|
|
38
|
+
private readonly embeddings: EmbeddingRuntime;
|
|
33
39
|
|
|
34
40
|
constructor(
|
|
35
41
|
private readonly db: Database,
|
|
36
|
-
|
|
42
|
+
embeddings?: EmbeddingRuntimeInput,
|
|
43
|
+
ranking?: RankingConfig
|
|
37
44
|
) {
|
|
38
|
-
this.
|
|
39
|
-
this.
|
|
45
|
+
this.embeddings = embeddingRuntimeFrom(embeddings);
|
|
46
|
+
this.searchService = new SearchService(db, this.embeddings, ranking);
|
|
40
47
|
}
|
|
41
48
|
|
|
42
49
|
add(input: TrapInput): number {
|
|
@@ -79,7 +86,7 @@ export class TrapRepository {
|
|
|
79
86
|
update(id: number, input: TrapUpdate): boolean {
|
|
80
87
|
const success = queries.updateTrap(this.db, id, input);
|
|
81
88
|
if (success && passageFieldsChanged(input)) {
|
|
82
|
-
this.
|
|
89
|
+
embeddingQueries.deleteEmbedding(this.db, id);
|
|
83
90
|
}
|
|
84
91
|
return success;
|
|
85
92
|
}
|
|
@@ -114,7 +121,11 @@ export class TrapRepository {
|
|
|
114
121
|
}
|
|
115
122
|
|
|
116
123
|
embeddingStats(config: EmbeddingConfig | null, opts: { scope?: string; status?: TrapStatus | "all" } = {}): EmbeddingStateCounts {
|
|
117
|
-
return this.
|
|
124
|
+
return embeddingQueries.getEmbeddingStateCounts(this.db, config, opts);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
embeddingProfiles(opts: { scope?: string; status?: TrapStatus | "all" } = {}): EmbeddingProfileSummary[] {
|
|
128
|
+
return embeddingQueries.listEmbeddingProfiles(this.db, opts);
|
|
118
129
|
}
|
|
119
130
|
|
|
120
131
|
exportAll(): TrapExportRecord[] {
|
|
@@ -145,23 +156,23 @@ export class TrapRepository {
|
|
|
145
156
|
return this.db.transaction(callback)();
|
|
146
157
|
}
|
|
147
158
|
|
|
148
|
-
getEmbedding(trapId: number): StoredEmbedding | null {
|
|
149
|
-
return this.
|
|
159
|
+
getEmbedding(trapId: number, config?: EmbeddingConfig): StoredEmbedding | null {
|
|
160
|
+
return embeddingQueries.getEmbedding(this.db, trapId, config);
|
|
150
161
|
}
|
|
151
162
|
|
|
152
163
|
upsertEmbedding(record: StoredEmbedding): void {
|
|
153
|
-
this.
|
|
164
|
+
embeddingQueries.upsertEmbedding(this.db, record);
|
|
154
165
|
}
|
|
155
166
|
|
|
156
167
|
deleteEmbedding(trapId: number): void {
|
|
157
|
-
this.
|
|
168
|
+
embeddingQueries.deleteEmbedding(this.db, trapId);
|
|
158
169
|
}
|
|
159
170
|
|
|
160
171
|
getTrapsNeedingEmbeddings(
|
|
161
172
|
config: EmbeddingConfig,
|
|
162
173
|
opts: { scope?: string; category?: string; status?: TrapStatus | "all"; force?: boolean; limit?: number } = {}
|
|
163
174
|
): Trap[] {
|
|
164
|
-
return this.
|
|
175
|
+
return embeddingQueries.getTrapsNeedingEmbeddings(this.db, config, opts);
|
|
165
176
|
}
|
|
166
177
|
|
|
167
178
|
async ensureEmbeddings(opts: { scope?: string; category?: string; limit?: number; force?: boolean; batchSize?: number } = {}): Promise<{
|
|
@@ -169,18 +180,18 @@ export class TrapRepository {
|
|
|
169
180
|
skipped: number;
|
|
170
181
|
batches: number;
|
|
171
182
|
}> {
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
183
|
+
const provider = this.embeddings.requireProvider(
|
|
184
|
+
"Embedding provider is unavailable. Configure an embedding provider to generate embeddings."
|
|
185
|
+
);
|
|
175
186
|
|
|
176
187
|
return runEmbeddingJob(
|
|
177
188
|
{
|
|
178
|
-
countEmbeddable: (countOpts) => this.
|
|
189
|
+
countEmbeddable: (countOpts) => embeddingQueries.countEmbeddableTraps(this.db, countOpts),
|
|
179
190
|
trapsNeedingEmbeddings: (config, jobOpts) =>
|
|
180
|
-
this.
|
|
181
|
-
saveEmbedding: (record) => this.
|
|
191
|
+
embeddingQueries.getTrapsNeedingEmbeddings(this.db, config, jobOpts),
|
|
192
|
+
saveEmbedding: (record) => embeddingQueries.upsertEmbedding(this.db, record),
|
|
182
193
|
},
|
|
183
|
-
|
|
194
|
+
provider,
|
|
184
195
|
opts
|
|
185
196
|
);
|
|
186
197
|
}
|
package/src/db/schema.ts
CHANGED
|
@@ -175,6 +175,86 @@ function applyMigrations(db: Database, from: number): void {
|
|
|
175
175
|
|
|
176
176
|
db.prepare("UPDATE schema_version SET version = ?").run(5);
|
|
177
177
|
}
|
|
178
|
+
|
|
179
|
+
if (from < 6) {
|
|
180
|
+
migrateEmbeddingProfiles(db);
|
|
181
|
+
db.prepare("UPDATE schema_version SET version = ?").run(6);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function migrateEmbeddingProfiles(db: Database): void {
|
|
186
|
+
db.exec(`
|
|
187
|
+
CREATE TABLE IF NOT EXISTS embedding_profiles (
|
|
188
|
+
id TEXT PRIMARY KEY,
|
|
189
|
+
provider TEXT NOT NULL,
|
|
190
|
+
model TEXT NOT NULL,
|
|
191
|
+
dimensions INTEGER NOT NULL,
|
|
192
|
+
passage_version INTEGER NOT NULL,
|
|
193
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
194
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
195
|
+
);
|
|
196
|
+
`);
|
|
197
|
+
|
|
198
|
+
const hasTrapEmbeddings = tableExists(db, "trap_embeddings");
|
|
199
|
+
if (!hasTrapEmbeddings || columnExists(db, "trap_embeddings", "profile_id")) {
|
|
200
|
+
createProfileAwareEmbeddingsTable(db);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
db.exec("ALTER TABLE trap_embeddings RENAME TO trap_embeddings_legacy_v5");
|
|
205
|
+
createProfileAwareEmbeddingsTable(db);
|
|
206
|
+
|
|
207
|
+
db.exec(`
|
|
208
|
+
INSERT OR IGNORE INTO embedding_profiles (
|
|
209
|
+
id, provider, model, dimensions, passage_version, created_at, updated_at
|
|
210
|
+
)
|
|
211
|
+
SELECT
|
|
212
|
+
provider || ':' || model || ':' || dimensions || ':p' || passage_version,
|
|
213
|
+
provider,
|
|
214
|
+
model,
|
|
215
|
+
dimensions,
|
|
216
|
+
passage_version,
|
|
217
|
+
MIN(updated_at),
|
|
218
|
+
MAX(updated_at)
|
|
219
|
+
FROM trap_embeddings_legacy_v5
|
|
220
|
+
GROUP BY provider, model, dimensions, passage_version;
|
|
221
|
+
|
|
222
|
+
INSERT OR REPLACE INTO trap_embeddings (
|
|
223
|
+
trap_id, profile_id, passage_hash, embedding, updated_at
|
|
224
|
+
)
|
|
225
|
+
SELECT
|
|
226
|
+
trap_id,
|
|
227
|
+
provider || ':' || model || ':' || dimensions || ':p' || passage_version,
|
|
228
|
+
passage_hash,
|
|
229
|
+
embedding,
|
|
230
|
+
updated_at
|
|
231
|
+
FROM trap_embeddings_legacy_v5;
|
|
232
|
+
|
|
233
|
+
DROP TABLE trap_embeddings_legacy_v5;
|
|
234
|
+
`);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function createProfileAwareEmbeddingsTable(db: Database): void {
|
|
238
|
+
db.exec(`
|
|
239
|
+
CREATE TABLE IF NOT EXISTS trap_embeddings (
|
|
240
|
+
trap_id INTEGER NOT NULL REFERENCES traps(id) ON DELETE CASCADE,
|
|
241
|
+
profile_id TEXT NOT NULL REFERENCES embedding_profiles(id) ON DELETE CASCADE,
|
|
242
|
+
passage_hash TEXT NOT NULL,
|
|
243
|
+
embedding BLOB NOT NULL,
|
|
244
|
+
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
245
|
+
PRIMARY KEY (trap_id, profile_id)
|
|
246
|
+
);
|
|
247
|
+
|
|
248
|
+
CREATE INDEX IF NOT EXISTS idx_trap_embeddings_profile
|
|
249
|
+
ON trap_embeddings(profile_id);
|
|
250
|
+
`);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function tableExists(db: Database, table: string): boolean {
|
|
254
|
+
const row = db
|
|
255
|
+
.query("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
|
|
256
|
+
.get(table) as { name: string } | null;
|
|
257
|
+
return row !== null;
|
|
178
258
|
}
|
|
179
259
|
|
|
180
260
|
function columnExists(db: Database, table: string, column: string): boolean {
|
package/src/index.ts
CHANGED
|
@@ -15,8 +15,12 @@ if (args.length === 0) {
|
|
|
15
15
|
} else if (args[0] === "serve") {
|
|
16
16
|
import("./mcp/server").then((m) => m.start());
|
|
17
17
|
} else if (args[0] === "web") {
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
if (args.slice(1).some(isHelpArg)) {
|
|
19
|
+
showWebHelp();
|
|
20
|
+
} else {
|
|
21
|
+
const { startWebServerFromArgs } = await import("./web/server");
|
|
22
|
+
await startWebServerFromArgs(args.slice(1));
|
|
23
|
+
}
|
|
20
24
|
} else if (args[0] === "init") {
|
|
21
25
|
const cwd = process.cwd();
|
|
22
26
|
if (findProjectRoot(cwd)) {
|
|
@@ -31,6 +35,10 @@ if (args.length === 0) {
|
|
|
31
35
|
await run(args, store);
|
|
32
36
|
}
|
|
33
37
|
|
|
38
|
+
function isHelpArg(arg: string | undefined): boolean {
|
|
39
|
+
return arg === "--help" || arg === "-h" || arg === "help";
|
|
40
|
+
}
|
|
41
|
+
|
|
34
42
|
function showHelp(): void {
|
|
35
43
|
console.log("codetrap — capture coding pitfalls so AI doesn't repeat mistakes");
|
|
36
44
|
console.log("");
|
|
@@ -45,9 +53,10 @@ function showHelp(): void {
|
|
|
45
53
|
console.log(" add_trap_evidence Attach evidence to a trap");
|
|
46
54
|
console.log(" archive_trap Archive a trap");
|
|
47
55
|
console.log(" supersede_trap Mark one trap as superseded by another");
|
|
48
|
-
console.log(" embed Generate embeddings for semantic search");
|
|
56
|
+
console.log(" embed Generate embeddings for semantic search (Ollama or Jina)");
|
|
57
|
+
console.log(" embeddings Manage embedding profiles, provider config, and reindexing");
|
|
49
58
|
console.log(" session Record implementation notes and capture candidate traps");
|
|
50
|
-
console.log(" web Start local
|
|
59
|
+
console.log(" web Start local review and trap library console");
|
|
51
60
|
console.log(" export Export traps as JSON");
|
|
52
61
|
console.log(" import <file.json> Import traps from JSON");
|
|
53
62
|
console.log(" stats Show statistics");
|
|
@@ -68,12 +77,28 @@ function showHelp(): void {
|
|
|
68
77
|
console.log(" --no-rerank Disable query-aware search reranking");
|
|
69
78
|
console.log(" --ranking-signals Include search ranking diagnostics in JSON cards");
|
|
70
79
|
console.log(" --batch-size <n> Embedding generation batch size");
|
|
71
|
-
console.log(" --project <path> Project path for web
|
|
72
|
-
console.log(" --host <host> Host for web
|
|
73
|
-
console.log(" --port <n> Port for web
|
|
80
|
+
console.log(" --project <path> Project path for web console");
|
|
81
|
+
console.log(" --host <host> Host for web console (default 127.0.0.1)");
|
|
82
|
+
console.log(" --port <n> Port for web console (default 4737)");
|
|
74
83
|
console.log(" --json JSON output for search/show/list/stats/doctor; JSON input for add/edit");
|
|
75
84
|
console.log(" --output-json JSON output for add/edit when --json is used as input");
|
|
76
85
|
console.log(" --from-project-path <path> Source project path for scope repair/migration");
|
|
77
86
|
console.log(" --to-project-path <path> Destination project path for scope repair/migration");
|
|
78
87
|
console.log(" --dry-run|--apply Preview or apply scope repair/migration");
|
|
79
88
|
}
|
|
89
|
+
|
|
90
|
+
function showWebHelp(): void {
|
|
91
|
+
console.log("codetrap web — start the local review and trap library console");
|
|
92
|
+
console.log("");
|
|
93
|
+
console.log("Usage:");
|
|
94
|
+
console.log(" codetrap web [--project <path>] [--host <host>] [--port <n>]");
|
|
95
|
+
console.log("");
|
|
96
|
+
console.log("Options:");
|
|
97
|
+
console.log(" --project <path> Project path to open in the web console");
|
|
98
|
+
console.log(" --host <host> Host to bind (default 127.0.0.1)");
|
|
99
|
+
console.log(" --port <n> Port to try first (default 4737; next free port is used if busy)");
|
|
100
|
+
console.log("");
|
|
101
|
+
console.log("Examples:");
|
|
102
|
+
console.log(" codetrap web");
|
|
103
|
+
console.log(" codetrap web --project /path/to/project --port 4789");
|
|
104
|
+
}
|