claude-db 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/capture/flush.d.ts +22 -0
- package/dist/capture/flush.js +45 -10
- package/dist/capture/identity.d.ts +6 -0
- package/dist/capture/identity.js +14 -0
- package/dist/capture/index.d.ts +5 -3
- package/dist/capture/index.js +4 -3
- package/dist/capture/manual.d.ts +21 -0
- package/dist/capture/manual.js +40 -0
- package/dist/capture/transcript.d.ts +12 -0
- package/dist/capture/transcript.js +69 -1
- package/dist/capture/turn-extractor.js +40 -8
- package/dist/cli/index.js +376 -30
- package/dist/cli/install.d.ts +15 -1
- package/dist/cli/install.js +127 -8
- package/dist/config/schema.d.ts +1 -0
- package/dist/config/schema.js +9 -1
- package/dist/context.js +24 -2
- package/dist/embed/builtin.js +3 -1
- package/dist/hooks/relevance.js +11 -4
- package/dist/hooks/session-end.js +2 -1
- package/dist/hooks/session-start.js +22 -3
- package/dist/mcp/server.js +56 -6
- package/dist/search/service.js +4 -1
- package/dist/store/adapter.d.ts +22 -6
- package/dist/store/adapter.js +10 -1
- package/dist/store/mongo/driver.d.ts +1 -0
- package/dist/store/mongo/index.d.ts +3 -2
- package/dist/store/mongo/index.js +57 -7
- package/dist/store/postgres/index.d.ts +22 -8
- package/dist/store/postgres/index.js +165 -21
- package/dist/store/sqlite/index.d.ts +3 -2
- package/dist/store/sqlite/index.js +112 -9
- package/dist/store/sqlite/schema.sql +4 -3
- package/dist/types.d.ts +28 -0
- package/dist/util/project.d.ts +9 -10
- package/dist/util/project.js +52 -13
- package/dist/util/vector.d.ts +11 -1
- package/dist/util/vector.js +14 -3
- package/package.json +2 -2
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { importMongo } from './driver.js';
|
|
2
|
+
import { isWholeScope } from '../adapter.js';
|
|
2
3
|
import { cosine } from '../../util/vector.js';
|
|
3
4
|
import { partitionIds } from '../../util/shortid.js';
|
|
4
5
|
/**
|
|
@@ -94,12 +95,47 @@ export class MongoStore {
|
|
|
94
95
|
.toArray();
|
|
95
96
|
return docs.map(toObservation);
|
|
96
97
|
}
|
|
97
|
-
async
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
98
|
+
async remove(filter) {
|
|
99
|
+
if (filter.ids?.length === 0)
|
|
100
|
+
return 0;
|
|
101
|
+
const query = {};
|
|
102
|
+
if (filter.project)
|
|
103
|
+
query['project'] = filter.project;
|
|
104
|
+
if (filter.kind)
|
|
105
|
+
query['kind'] = filter.kind;
|
|
106
|
+
if (filter.before !== undefined)
|
|
107
|
+
query['createdAt'] = { $lt: filter.before };
|
|
108
|
+
if (filter.ids) {
|
|
109
|
+
const { exact, prefixes } = partitionIds(filter.ids);
|
|
110
|
+
const alternatives = [];
|
|
111
|
+
if (exact.length > 0)
|
|
112
|
+
alternatives.push({ _id: { $in: exact } });
|
|
113
|
+
for (const prefix of prefixes) {
|
|
114
|
+
alternatives.push({ _id: { $regex: `^${escapeRegex(prefix)}` } });
|
|
115
|
+
}
|
|
116
|
+
query['$or'] = alternatives;
|
|
117
|
+
}
|
|
118
|
+
const count = await this.observations.countDocuments(query);
|
|
119
|
+
await this.observations.deleteMany(query);
|
|
120
|
+
if (isWholeScope(filter)) {
|
|
121
|
+
await this.sessions.deleteMany(filter.project ? { project: filter.project } : {});
|
|
122
|
+
}
|
|
123
|
+
return count;
|
|
124
|
+
}
|
|
125
|
+
async list(filter) {
|
|
126
|
+
const query = {};
|
|
127
|
+
if (filter.project)
|
|
128
|
+
query['project'] = filter.project;
|
|
129
|
+
if (filter.kind)
|
|
130
|
+
query['kind'] = filter.kind;
|
|
131
|
+
if (filter.after !== undefined)
|
|
132
|
+
query['createdAt'] = { $gt: filter.after };
|
|
133
|
+
const docs = await this.observations
|
|
134
|
+
.find(query)
|
|
135
|
+
.sort({ createdAt: 1 })
|
|
136
|
+
.limit(filter.limit ?? 1000)
|
|
137
|
+
.toArray();
|
|
138
|
+
return docs.map(toObservation);
|
|
103
139
|
}
|
|
104
140
|
async listProjects() {
|
|
105
141
|
const rows = await this.observations
|
|
@@ -156,7 +192,13 @@ export class MongoStore {
|
|
|
156
192
|
// Portable path: score candidates in process. Bounded by the scope filter,
|
|
157
193
|
// which in practice narrows to a single project.
|
|
158
194
|
const docs = await this.observations
|
|
159
|
-
.find({
|
|
195
|
+
.find({
|
|
196
|
+
embedding: { $exists: true },
|
|
197
|
+
...(query.embedder
|
|
198
|
+
? { $or: [{ embedder: query.embedder }, { embedder: { $exists: false } }] }
|
|
199
|
+
: {}),
|
|
200
|
+
...this.scopeFilter(query),
|
|
201
|
+
}, { projection: { kind: 1, title: 1, project: 1, createdAt: 1, embedding: 1 } })
|
|
160
202
|
.sort({ createdAt: -1 })
|
|
161
203
|
.limit(query.maxScanCandidates ?? 25000)
|
|
162
204
|
.toArray();
|
|
@@ -245,6 +287,10 @@ function toDoc(obs) {
|
|
|
245
287
|
};
|
|
246
288
|
if (obs.embedding)
|
|
247
289
|
doc.embedding = obs.embedding;
|
|
290
|
+
if (obs.embedder)
|
|
291
|
+
doc.embedder = obs.embedder;
|
|
292
|
+
if (obs.author)
|
|
293
|
+
doc.author = obs.author;
|
|
248
294
|
return doc;
|
|
249
295
|
}
|
|
250
296
|
function toObservation(doc) {
|
|
@@ -261,6 +307,10 @@ function toObservation(doc) {
|
|
|
261
307
|
};
|
|
262
308
|
if (doc.embedding)
|
|
263
309
|
obs.embedding = doc.embedding;
|
|
310
|
+
if (doc.embedder)
|
|
311
|
+
obs.embedder = doc.embedder;
|
|
312
|
+
if (doc.author)
|
|
313
|
+
obs.author = doc.author;
|
|
264
314
|
return obs;
|
|
265
315
|
}
|
|
266
316
|
function toIndexEntry(doc, score) {
|
|
@@ -1,18 +1,31 @@
|
|
|
1
1
|
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
2
|
-
import type { Observation, ObservationIndexEntry, SearchQuery, Session, TimelineQuery } from '../../types.js';
|
|
3
|
-
/**
|
|
4
|
-
* Postgres backend. Uses tsvector for keyword search, and pgvector for
|
|
5
|
-
* embeddings when the extension is available. If `CREATE EXTENSION vector`
|
|
6
|
-
* is not permitted (common on locked-down managed instances) the store still
|
|
7
|
-
* works, minus semantic recall, which the search service handles gracefully.
|
|
8
|
-
*/
|
|
2
|
+
import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, SearchQuery, Session, TimelineQuery } from '../../types.js';
|
|
9
3
|
export declare class PostgresStore implements MemoryStore {
|
|
10
4
|
private readonly pool;
|
|
11
5
|
readonly kind = "postgres";
|
|
12
6
|
private vectorEnabled;
|
|
7
|
+
private vectorDims;
|
|
8
|
+
private warnedDims;
|
|
13
9
|
private constructor();
|
|
14
10
|
static create(uri: string): Promise<PostgresStore>;
|
|
15
11
|
init(): Promise<void>;
|
|
12
|
+
/**
|
|
13
|
+
* CREATE TABLE IF NOT EXISTS leaves an existing generated column alone, so a
|
|
14
|
+
* database created before tags were indexed keeps the old expression. Rebuild
|
|
15
|
+
* it once; dropping the column takes its index with it.
|
|
16
|
+
*/
|
|
17
|
+
private ensureTagsIndexed;
|
|
18
|
+
private readVectorDims;
|
|
19
|
+
/**
|
|
20
|
+
* Creates the embedding column sized to the vectors actually being stored,
|
|
21
|
+
* since pgvector needs a fixed width but the width belongs to the embedder:
|
|
22
|
+
* 256 for the builtin, 384 for MiniLM. Hardcoding 384 meant every insert on
|
|
23
|
+
* a default install failed on a dimension mismatch and captured nothing.
|
|
24
|
+
* Returns false when this width cannot be stored, and the caller keeps the
|
|
25
|
+
* observation without a vector rather than losing it. `dims` is bounded
|
|
26
|
+
* above because DDL cannot be parameterised.
|
|
27
|
+
*/
|
|
28
|
+
private ensureVectorColumn;
|
|
16
29
|
close(): Promise<void>;
|
|
17
30
|
ping(): Promise<boolean>;
|
|
18
31
|
upsertSession(session: Session): Promise<void>;
|
|
@@ -20,7 +33,8 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
20
33
|
recentSessions(project: string, limit: number): Promise<Session[]>;
|
|
21
34
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
22
35
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
23
|
-
|
|
36
|
+
remove(filter: RemoveFilter): Promise<number>;
|
|
37
|
+
list(filter: ListFilter): Promise<Observation[]>;
|
|
24
38
|
listProjects(): Promise<ProjectSummary[]>;
|
|
25
39
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
26
40
|
searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isWholeScope } from '../adapter.js';
|
|
1
2
|
import { partitionIds } from '../../util/shortid.js';
|
|
2
3
|
/**
|
|
3
4
|
* Postgres backend. Uses tsvector for keyword search, and pgvector for
|
|
@@ -5,10 +6,22 @@ import { partitionIds } from '../../util/shortid.js';
|
|
|
5
6
|
* is not permitted (common on locked-down managed instances) the store still
|
|
6
7
|
* works, minus semantic recall, which the search service handles gracefully.
|
|
7
8
|
*/
|
|
9
|
+
/**
|
|
10
|
+
* Title outranks tags, tags outrank body — the same ordering FTS5 gets from its
|
|
11
|
+
* column weights. Tags were missing here entirely, so an observation tagged
|
|
12
|
+
* with the repository it touched was findable on SQLite and Mongo but not on
|
|
13
|
+
* Postgres.
|
|
14
|
+
*/
|
|
15
|
+
const TSV_EXPRESSION = `
|
|
16
|
+
setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
|
|
17
|
+
setweight(to_tsvector('english', coalesce(tags::text, '')), 'B') ||
|
|
18
|
+
setweight(to_tsvector('english', coalesce(body, '')), 'C')`;
|
|
8
19
|
export class PostgresStore {
|
|
9
20
|
pool;
|
|
10
21
|
kind = 'postgres';
|
|
11
22
|
vectorEnabled = false;
|
|
23
|
+
vectorDims = null;
|
|
24
|
+
warnedDims = false;
|
|
12
25
|
constructor(pool) {
|
|
13
26
|
this.pool = pool;
|
|
14
27
|
}
|
|
@@ -40,20 +53,72 @@ export class PostgresStore {
|
|
|
40
53
|
files JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
41
54
|
tags JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
42
55
|
created_at BIGINT NOT NULL,
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
) STORED
|
|
56
|
+
embedder TEXT,
|
|
57
|
+
author TEXT,
|
|
58
|
+
tsv TSVECTOR GENERATED ALWAYS AS (${TSV_EXPRESSION}) STORED
|
|
47
59
|
);
|
|
48
60
|
CREATE INDEX IF NOT EXISTS idx_obs_tsv ON observations USING GIN(tsv);
|
|
49
61
|
CREATE INDEX IF NOT EXISTS idx_obs_project_time
|
|
50
62
|
ON observations(project, created_at DESC);
|
|
63
|
+
|
|
64
|
+
ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedder TEXT;
|
|
65
|
+
ALTER TABLE observations ADD COLUMN IF NOT EXISTS author TEXT;
|
|
51
66
|
`);
|
|
52
|
-
|
|
53
|
-
|
|
67
|
+
await this.ensureTagsIndexed();
|
|
68
|
+
if (this.vectorEnabled)
|
|
69
|
+
this.vectorDims = await this.readVectorDims();
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* CREATE TABLE IF NOT EXISTS leaves an existing generated column alone, so a
|
|
73
|
+
* database created before tags were indexed keeps the old expression. Rebuild
|
|
74
|
+
* it once; dropping the column takes its index with it.
|
|
75
|
+
*/
|
|
76
|
+
async ensureTagsIndexed() {
|
|
77
|
+
const res = await this.pool.query(`SELECT pg_get_expr(d.adbin, d.adrelid) AS expr
|
|
78
|
+
FROM pg_attrdef d
|
|
79
|
+
JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
|
80
|
+
WHERE d.adrelid = 'observations'::regclass AND a.attname = 'tsv'`);
|
|
81
|
+
const expr = String(res.rows[0]?.['expr'] ?? '');
|
|
82
|
+
if (expr.length === 0 || expr.includes('tags'))
|
|
83
|
+
return;
|
|
84
|
+
await this.pool.query('ALTER TABLE observations DROP COLUMN tsv');
|
|
85
|
+
await this.pool.query(`ALTER TABLE observations
|
|
86
|
+
ADD COLUMN tsv TSVECTOR GENERATED ALWAYS AS (${TSV_EXPRESSION}) STORED`);
|
|
87
|
+
await this.pool.query('CREATE INDEX IF NOT EXISTS idx_obs_tsv ON observations USING GIN(tsv)');
|
|
88
|
+
}
|
|
89
|
+
async readVectorDims() {
|
|
90
|
+
const res = await this.pool.query(`SELECT atttypmod AS dims FROM pg_attribute
|
|
91
|
+
WHERE attrelid = 'observations'::regclass
|
|
92
|
+
AND attname = 'embedding' AND NOT attisdropped`);
|
|
93
|
+
const dims = Number(res.rows[0]?.['dims'] ?? -1);
|
|
94
|
+
return dims > 0 ? dims : null;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Creates the embedding column sized to the vectors actually being stored,
|
|
98
|
+
* since pgvector needs a fixed width but the width belongs to the embedder:
|
|
99
|
+
* 256 for the builtin, 384 for MiniLM. Hardcoding 384 meant every insert on
|
|
100
|
+
* a default install failed on a dimension mismatch and captured nothing.
|
|
101
|
+
* Returns false when this width cannot be stored, and the caller keeps the
|
|
102
|
+
* observation without a vector rather than losing it. `dims` is bounded
|
|
103
|
+
* above because DDL cannot be parameterised.
|
|
104
|
+
*/
|
|
105
|
+
async ensureVectorColumn(dims) {
|
|
106
|
+
if (!this.vectorEnabled)
|
|
107
|
+
return false;
|
|
108
|
+
if (!Number.isInteger(dims) || dims < 1 || dims > 16000)
|
|
109
|
+
return false;
|
|
110
|
+
if (this.vectorDims === null) {
|
|
111
|
+
await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
|
|
54
112
|
await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
|
|
55
113
|
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
114
|
+
this.vectorDims = dims;
|
|
115
|
+
}
|
|
116
|
+
if (this.vectorDims !== dims && !this.warnedDims) {
|
|
117
|
+
this.warnedDims = true;
|
|
118
|
+
process.stderr.write(`[claude-db] embeddings are ${dims}d but this database stores ` +
|
|
119
|
+
`${this.vectorDims}d; storing text only. Keyword search is unaffected.\n`);
|
|
56
120
|
}
|
|
121
|
+
return this.vectorDims === dims;
|
|
57
122
|
}
|
|
58
123
|
async close() {
|
|
59
124
|
await this.pool.end();
|
|
@@ -66,6 +131,7 @@ export class PostgresStore {
|
|
|
66
131
|
await this.pool.query(`INSERT INTO sessions (id, project, started_at, ended_at, summary)
|
|
67
132
|
VALUES ($1, $2, $3, $4, $5)
|
|
68
133
|
ON CONFLICT (id) DO UPDATE SET
|
|
134
|
+
project = EXCLUDED.project,
|
|
69
135
|
ended_at = COALESCE(EXCLUDED.ended_at, sessions.ended_at),
|
|
70
136
|
summary = COALESCE(EXCLUDED.summary, sessions.summary)`, [
|
|
71
137
|
session.id,
|
|
@@ -89,22 +155,32 @@ export class PostgresStore {
|
|
|
89
155
|
async insertObservations(observations) {
|
|
90
156
|
if (observations.length === 0)
|
|
91
157
|
return;
|
|
158
|
+
const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
|
|
159
|
+
const vectors = width !== undefined && (await this.ensureVectorColumn(width));
|
|
92
160
|
const client = await this.pool.connect();
|
|
93
161
|
try {
|
|
94
162
|
await client.query('BEGIN');
|
|
95
163
|
for (const obs of observations) {
|
|
96
|
-
const columns =
|
|
97
|
-
|
|
164
|
+
const columns = [
|
|
165
|
+
'id', 'session_id', 'project', 'kind', 'title', 'body',
|
|
166
|
+
'files', 'tags', 'created_at', 'embedder', 'author',
|
|
167
|
+
];
|
|
98
168
|
const values = [
|
|
99
169
|
obs.id, obs.sessionId, obs.project, obs.kind, obs.title, obs.body,
|
|
100
170
|
JSON.stringify(obs.files), JSON.stringify(obs.tags), obs.createdAt,
|
|
171
|
+
obs.embedder ?? null, obs.author ?? null,
|
|
101
172
|
];
|
|
102
|
-
if (
|
|
173
|
+
if (vectors && obs.embedding?.length === width) {
|
|
174
|
+
columns.push('embedding');
|
|
103
175
|
values.push(`[${obs.embedding.join(',')}]`);
|
|
104
176
|
}
|
|
105
177
|
const holes = values.map((_, i) => `$${i + 1}`).join(', ');
|
|
106
|
-
|
|
107
|
-
|
|
178
|
+
const updates = columns
|
|
179
|
+
.slice(1)
|
|
180
|
+
.map((column) => `${column} = EXCLUDED.${column}`)
|
|
181
|
+
.join(', ');
|
|
182
|
+
await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
|
|
183
|
+
ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
|
|
108
184
|
}
|
|
109
185
|
await client.query('COMMIT');
|
|
110
186
|
}
|
|
@@ -134,15 +210,45 @@ export class PostgresStore {
|
|
|
134
210
|
const res = await this.pool.query(`SELECT * FROM observations WHERE ${clauses.join(' OR ')}`, values);
|
|
135
211
|
return res.rows.map(toObservation);
|
|
136
212
|
}
|
|
137
|
-
async
|
|
138
|
-
|
|
139
|
-
|
|
213
|
+
async remove(filter) {
|
|
214
|
+
if (filter.ids?.length === 0)
|
|
215
|
+
return 0;
|
|
216
|
+
const conditions = [];
|
|
217
|
+
const values = [];
|
|
218
|
+
if (filter.ids) {
|
|
219
|
+
const { exact, prefixes } = partitionIds(filter.ids);
|
|
220
|
+
const alternatives = [];
|
|
221
|
+
if (exact.length > 0) {
|
|
222
|
+
values.push(exact);
|
|
223
|
+
alternatives.push(`id = ANY($${values.length}::text[])`);
|
|
224
|
+
}
|
|
225
|
+
for (const prefix of prefixes) {
|
|
226
|
+
values.push(`${prefix.replace(/[%_]/g, '')}%`);
|
|
227
|
+
alternatives.push(`id LIKE $${values.length}`);
|
|
228
|
+
}
|
|
229
|
+
conditions.push(`(${alternatives.join(' OR ')})`);
|
|
230
|
+
}
|
|
231
|
+
if (filter.project) {
|
|
232
|
+
values.push(filter.project);
|
|
233
|
+
conditions.push(`project = $${values.length}`);
|
|
234
|
+
}
|
|
235
|
+
if (filter.kind) {
|
|
236
|
+
values.push(filter.kind);
|
|
237
|
+
conditions.push(`kind = $${values.length}`);
|
|
238
|
+
}
|
|
239
|
+
if (filter.before !== undefined) {
|
|
240
|
+
values.push(filter.before);
|
|
241
|
+
conditions.push(`created_at < $${values.length}`);
|
|
242
|
+
}
|
|
243
|
+
const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
|
|
140
244
|
const client = await this.pool.connect();
|
|
141
245
|
try {
|
|
142
246
|
await client.query('BEGIN');
|
|
143
247
|
const counted = await client.query(`SELECT COUNT(*)::int AS n FROM observations ${where}`, values);
|
|
144
248
|
await client.query(`DELETE FROM observations ${where}`, values);
|
|
145
|
-
|
|
249
|
+
if (isWholeScope(filter)) {
|
|
250
|
+
await client.query(`DELETE FROM sessions ${filter.project ? 'WHERE project = $1' : ''}`, filter.project ? [filter.project] : []);
|
|
251
|
+
}
|
|
146
252
|
await client.query('COMMIT');
|
|
147
253
|
return Number(counted.rows[0]?.['n'] ?? 0);
|
|
148
254
|
}
|
|
@@ -154,6 +260,27 @@ export class PostgresStore {
|
|
|
154
260
|
client.release();
|
|
155
261
|
}
|
|
156
262
|
}
|
|
263
|
+
async list(filter) {
|
|
264
|
+
const conditions = [];
|
|
265
|
+
const values = [];
|
|
266
|
+
if (filter.project) {
|
|
267
|
+
values.push(filter.project);
|
|
268
|
+
conditions.push(`project = $${values.length}`);
|
|
269
|
+
}
|
|
270
|
+
if (filter.kind) {
|
|
271
|
+
values.push(filter.kind);
|
|
272
|
+
conditions.push(`kind = $${values.length}`);
|
|
273
|
+
}
|
|
274
|
+
if (filter.after !== undefined) {
|
|
275
|
+
values.push(filter.after);
|
|
276
|
+
conditions.push(`created_at > $${values.length}`);
|
|
277
|
+
}
|
|
278
|
+
values.push(filter.limit ?? 1000);
|
|
279
|
+
const res = await this.pool.query(`SELECT * FROM observations
|
|
280
|
+
${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
|
|
281
|
+
ORDER BY created_at ASC LIMIT $${values.length}`, values);
|
|
282
|
+
return res.rows.map(toObservation);
|
|
283
|
+
}
|
|
157
284
|
async listProjects() {
|
|
158
285
|
const res = await this.pool.query(`SELECT project, COUNT(*)::int AS n, MAX(created_at) AS last
|
|
159
286
|
FROM observations GROUP BY project ORDER BY last DESC`);
|
|
@@ -164,24 +291,36 @@ export class PostgresStore {
|
|
|
164
291
|
}));
|
|
165
292
|
}
|
|
166
293
|
async searchKeyword(query) {
|
|
167
|
-
const conditions = ['tsv @@
|
|
294
|
+
const conditions = ['tsv @@ q.tsq'];
|
|
168
295
|
const values = [query.text];
|
|
169
296
|
this.appendScope(query, conditions, values);
|
|
170
297
|
values.push(query.limit);
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
298
|
+
// plainto_tsquery ANDs every term, so a natural-language prompt only
|
|
299
|
+
// matches a row containing all of it — which for a real prompt is nothing
|
|
300
|
+
// at all, and injection would silently never fire. Flipping the operator on
|
|
301
|
+
// its output keeps Postgres's own parsing, stemming and stopword removal
|
|
302
|
+
// while giving the forgiving recall the other two adapters have; ts_rank
|
|
303
|
+
// then sorts out what actually mattered.
|
|
304
|
+
const res = await this.pool.query(`WITH q AS (
|
|
305
|
+
SELECT replace(plainto_tsquery('english', $1)::text, '&', '|')::tsquery AS tsq
|
|
306
|
+
)
|
|
307
|
+
SELECT id, kind, title, project, created_at, ts_rank(tsv, q.tsq) AS score
|
|
308
|
+
FROM observations, q
|
|
174
309
|
WHERE ${conditions.join(' AND ')}
|
|
175
310
|
ORDER BY score DESC
|
|
176
311
|
LIMIT $${values.length}`, values);
|
|
177
312
|
return res.rows.map(toIndexEntry);
|
|
178
313
|
}
|
|
179
314
|
async searchVector(vector, query) {
|
|
180
|
-
if (!this.vectorEnabled)
|
|
315
|
+
if (!this.vectorEnabled || this.vectorDims !== vector.length)
|
|
181
316
|
return [];
|
|
182
317
|
const conditions = ['embedding IS NOT NULL'];
|
|
183
318
|
const values = [`[${vector.join(',')}]`];
|
|
184
319
|
this.appendScope(query, conditions, values);
|
|
320
|
+
if (query.embedder) {
|
|
321
|
+
values.push(query.embedder);
|
|
322
|
+
conditions.push(`(embedder = $${values.length} OR embedder IS NULL)`);
|
|
323
|
+
}
|
|
185
324
|
values.push(query.limit);
|
|
186
325
|
const res = await this.pool.query(`SELECT id, kind, title, project, created_at,
|
|
187
326
|
1 - (embedding <=> $1::vector) AS score
|
|
@@ -265,7 +404,7 @@ function toSession(row) {
|
|
|
265
404
|
return session;
|
|
266
405
|
}
|
|
267
406
|
function toObservation(row) {
|
|
268
|
-
|
|
407
|
+
const obs = {
|
|
269
408
|
id: row['id'],
|
|
270
409
|
sessionId: row['session_id'],
|
|
271
410
|
project: row['project'],
|
|
@@ -276,6 +415,11 @@ function toObservation(row) {
|
|
|
276
415
|
tags: row['tags'] ?? [],
|
|
277
416
|
createdAt: Number(row['created_at']),
|
|
278
417
|
};
|
|
418
|
+
if (row['embedder'] != null)
|
|
419
|
+
obs.embedder = row['embedder'];
|
|
420
|
+
if (row['author'] != null)
|
|
421
|
+
obs.author = row['author'];
|
|
422
|
+
return obs;
|
|
279
423
|
}
|
|
280
424
|
function toIndexEntry(row) {
|
|
281
425
|
return {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
2
|
-
import type { Observation, ObservationIndexEntry, SearchQuery, Session, TimelineQuery } from '../../types.js';
|
|
2
|
+
import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, SearchQuery, Session, TimelineQuery } from '../../types.js';
|
|
3
3
|
/**
|
|
4
4
|
* Zero-config default backend, built on Node's own `node:sqlite`.
|
|
5
5
|
*
|
|
@@ -22,7 +22,8 @@ export declare class SqliteStore implements MemoryStore {
|
|
|
22
22
|
recentSessions(project: string, limit: number): Promise<Session[]>;
|
|
23
23
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
24
24
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
25
|
-
|
|
25
|
+
remove(filter: RemoveFilter): Promise<number>;
|
|
26
|
+
list(filter: ListFilter): Promise<Observation[]>;
|
|
26
27
|
listProjects(): Promise<ProjectSummary[]>;
|
|
27
28
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
28
29
|
searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -2,6 +2,7 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
2
2
|
import { mkdirSync, readFileSync } from 'node:fs';
|
|
3
3
|
import { dirname, resolve } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { isWholeScope } from '../adapter.js';
|
|
5
6
|
import { cosine, packVector, unpackVector } from '../../util/vector.js';
|
|
6
7
|
import { scopeToken } from '../../util/scope.js';
|
|
7
8
|
import { partitionIds } from '../../util/shortid.js';
|
|
@@ -28,6 +29,27 @@ export class SqliteStore {
|
|
|
28
29
|
}
|
|
29
30
|
async init() {
|
|
30
31
|
this.db.exec(readFileSync(resolve(HERE, 'schema.sql'), 'utf8'));
|
|
32
|
+
const row = this.db.prepare('PRAGMA user_version').get();
|
|
33
|
+
const version = Number(row?.['user_version'] ?? 0);
|
|
34
|
+
// Databases written before recursive_triggers was set carry orphaned FTS
|
|
35
|
+
// rows for every re-ingested observation. One rebuild clears them.
|
|
36
|
+
if (version < 1) {
|
|
37
|
+
this.db.exec(`INSERT INTO observations_fts(observations_fts) VALUES('rebuild');`);
|
|
38
|
+
}
|
|
39
|
+
// CREATE TABLE IF NOT EXISTS leaves an existing table alone, so columns
|
|
40
|
+
// added after the fact have to be applied here.
|
|
41
|
+
if (version < 2) {
|
|
42
|
+
for (const column of ['embedder TEXT', 'author TEXT']) {
|
|
43
|
+
try {
|
|
44
|
+
this.db.exec(`ALTER TABLE observations ADD COLUMN ${column}`);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
// Already present, which is the normal case on a fresh database.
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (version < 2)
|
|
52
|
+
this.db.exec('PRAGMA user_version = 2');
|
|
31
53
|
}
|
|
32
54
|
async close() {
|
|
33
55
|
this.db.close();
|
|
@@ -40,6 +62,7 @@ export class SqliteStore {
|
|
|
40
62
|
.prepare(`INSERT INTO sessions (id, project, started_at, ended_at, summary)
|
|
41
63
|
VALUES (?, ?, ?, ?, ?)
|
|
42
64
|
ON CONFLICT(id) DO UPDATE SET
|
|
65
|
+
project = excluded.project,
|
|
43
66
|
ended_at = COALESCE(excluded.ended_at, sessions.ended_at),
|
|
44
67
|
summary = COALESCE(excluded.summary, sessions.summary)`)
|
|
45
68
|
.run(session.id, session.project, session.startedAt, session.endedAt ?? null, session.summary ?? null);
|
|
@@ -60,12 +83,13 @@ export class SqliteStore {
|
|
|
60
83
|
if (observations.length === 0)
|
|
61
84
|
return;
|
|
62
85
|
const stmt = this.db.prepare(`INSERT OR REPLACE INTO observations
|
|
63
|
-
(id, session_id, project, scope, kind, title, body, files, tags,
|
|
64
|
-
|
|
86
|
+
(id, session_id, project, scope, kind, title, body, files, tags,
|
|
87
|
+
created_at, embedding, embedder, author)
|
|
88
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
65
89
|
this.db.exec('BEGIN');
|
|
66
90
|
try {
|
|
67
91
|
for (const obs of observations) {
|
|
68
|
-
stmt.run(obs.id, obs.sessionId, obs.project, scopeToken(obs.project), obs.kind, obs.title, obs.body, JSON.stringify(obs.files), JSON.stringify(obs.tags), obs.createdAt, obs.embedding ? packVector(obs.embedding) : null);
|
|
92
|
+
stmt.run(obs.id, obs.sessionId, obs.project, scopeToken(obs.project), obs.kind, obs.title, obs.body, JSON.stringify(obs.files), JSON.stringify(obs.tags), obs.createdAt, obs.embedding ? packVector(obs.embedding) : null, obs.embedder ?? null, obs.author ?? null);
|
|
69
93
|
}
|
|
70
94
|
this.db.exec('COMMIT');
|
|
71
95
|
}
|
|
@@ -95,9 +119,11 @@ export class SqliteStore {
|
|
|
95
119
|
.all(...params);
|
|
96
120
|
return rows.map(toObservation);
|
|
97
121
|
}
|
|
98
|
-
async
|
|
99
|
-
|
|
100
|
-
|
|
122
|
+
async remove(filter) {
|
|
123
|
+
if (filter.ids?.length === 0)
|
|
124
|
+
return 0;
|
|
125
|
+
const { where, params } = removeWhere(filter);
|
|
126
|
+
const wipe = isWholeScope(filter);
|
|
101
127
|
const count = this.db
|
|
102
128
|
.prepare(`SELECT COUNT(*) AS n FROM observations ${where}`)
|
|
103
129
|
.get(...params);
|
|
@@ -106,16 +132,47 @@ export class SqliteStore {
|
|
|
106
132
|
// The FTS mirror is kept in sync by triggers, so deleting rows here is
|
|
107
133
|
// enough; dropping it manually would desynchronise the two.
|
|
108
134
|
this.db.prepare(`DELETE FROM observations ${where}`).run(...params);
|
|
109
|
-
|
|
135
|
+
if (wipe) {
|
|
136
|
+
const scope = filter.project ? 'WHERE project = ?' : '';
|
|
137
|
+
this.db
|
|
138
|
+
.prepare(`DELETE FROM sessions ${scope}`)
|
|
139
|
+
.run(...(filter.project ? [filter.project] : []));
|
|
140
|
+
}
|
|
110
141
|
this.db.exec('COMMIT');
|
|
111
142
|
}
|
|
112
143
|
catch (error) {
|
|
113
144
|
this.db.exec('ROLLBACK');
|
|
114
145
|
throw error;
|
|
115
146
|
}
|
|
116
|
-
|
|
147
|
+
// Only worth the rewrite when a whole scope went; forgetting one row does
|
|
148
|
+
// not justify rebuilding the file.
|
|
149
|
+
if (wipe)
|
|
150
|
+
this.db.exec('VACUUM');
|
|
117
151
|
return count?.n ?? 0;
|
|
118
152
|
}
|
|
153
|
+
async list(filter) {
|
|
154
|
+
const conditions = [];
|
|
155
|
+
const params = [];
|
|
156
|
+
if (filter.project) {
|
|
157
|
+
conditions.push('project = ?');
|
|
158
|
+
params.push(filter.project);
|
|
159
|
+
}
|
|
160
|
+
if (filter.kind) {
|
|
161
|
+
conditions.push('kind = ?');
|
|
162
|
+
params.push(filter.kind);
|
|
163
|
+
}
|
|
164
|
+
if (filter.after !== undefined) {
|
|
165
|
+
conditions.push('created_at > ?');
|
|
166
|
+
params.push(filter.after);
|
|
167
|
+
}
|
|
168
|
+
params.push(filter.limit ?? 1000);
|
|
169
|
+
const rows = this.db
|
|
170
|
+
.prepare(`SELECT * FROM observations
|
|
171
|
+
${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
|
|
172
|
+
ORDER BY created_at ASC LIMIT ?`)
|
|
173
|
+
.all(...params);
|
|
174
|
+
return rows.map(toObservation);
|
|
175
|
+
}
|
|
119
176
|
async listProjects() {
|
|
120
177
|
const rows = this.db
|
|
121
178
|
.prepare(`SELECT project, COUNT(*) AS n, MAX(created_at) AS last
|
|
@@ -169,6 +226,12 @@ export class SqliteStore {
|
|
|
169
226
|
const conditions = ['embedding IS NOT NULL'];
|
|
170
227
|
const params = [];
|
|
171
228
|
appendScope(query, conditions, params, '');
|
|
229
|
+
// NULL means the row predates the column, and the width check in cosine()
|
|
230
|
+
// still guards it. Anything else is a different embedding space.
|
|
231
|
+
if (query.embedder) {
|
|
232
|
+
conditions.push('(embedder = ? OR embedder IS NULL)');
|
|
233
|
+
params.push(query.embedder);
|
|
234
|
+
}
|
|
172
235
|
// Newest-first with a hard cap. The scan is linear in rows returned, so
|
|
173
236
|
// this is what keeps p95 flat as history grows: recent memory stays
|
|
174
237
|
// semantically searchable and older memory remains reachable by keyword.
|
|
@@ -215,6 +278,39 @@ export class SqliteStore {
|
|
|
215
278
|
return rows.map(toIndexEntry);
|
|
216
279
|
}
|
|
217
280
|
}
|
|
281
|
+
function removeWhere(filter) {
|
|
282
|
+
const conditions = [];
|
|
283
|
+
const params = [];
|
|
284
|
+
if (filter.ids) {
|
|
285
|
+
const { exact, prefixes } = partitionIds(filter.ids);
|
|
286
|
+
const alternatives = [];
|
|
287
|
+
if (exact.length > 0) {
|
|
288
|
+
alternatives.push(`id IN (${exact.map(() => '?').join(',')})`);
|
|
289
|
+
params.push(...exact);
|
|
290
|
+
}
|
|
291
|
+
for (const prefix of prefixes) {
|
|
292
|
+
alternatives.push('id GLOB ?');
|
|
293
|
+
params.push(`${prefix}*`);
|
|
294
|
+
}
|
|
295
|
+
conditions.push(`(${alternatives.join(' OR ')})`);
|
|
296
|
+
}
|
|
297
|
+
if (filter.project) {
|
|
298
|
+
conditions.push('project = ?');
|
|
299
|
+
params.push(filter.project);
|
|
300
|
+
}
|
|
301
|
+
if (filter.kind) {
|
|
302
|
+
conditions.push('kind = ?');
|
|
303
|
+
params.push(filter.kind);
|
|
304
|
+
}
|
|
305
|
+
if (filter.before !== undefined) {
|
|
306
|
+
conditions.push('created_at < ?');
|
|
307
|
+
params.push(filter.before);
|
|
308
|
+
}
|
|
309
|
+
return {
|
|
310
|
+
where: conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '',
|
|
311
|
+
params,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
218
314
|
function appendScope(query, conditions, params, prefix) {
|
|
219
315
|
if (query.project) {
|
|
220
316
|
conditions.push(`${prefix}project = ?`);
|
|
@@ -252,9 +348,12 @@ function toFilePath(uri) {
|
|
|
252
348
|
* query entirely rather than issue one that matches everything.
|
|
253
349
|
*/
|
|
254
350
|
function toMatchExpression(text, project) {
|
|
351
|
+
// ponytail: unicode61 keeps a run of Han characters as one token, so CJK
|
|
352
|
+
// matches phrases but not substrings. Vector recall covers the gap; swap in
|
|
353
|
+
// an ICU or bigram tokenizer if that stops being enough.
|
|
255
354
|
const tokens = text
|
|
256
355
|
.toLowerCase()
|
|
257
|
-
.split(/[
|
|
356
|
+
.split(/[^\p{L}\p{N}_]+/u)
|
|
258
357
|
.filter((token) => token.length > 1);
|
|
259
358
|
const scope = project ? `scope:${scopeToken(project)}` : null;
|
|
260
359
|
if (tokens.length === 0)
|
|
@@ -295,6 +394,10 @@ function toObservation(row) {
|
|
|
295
394
|
};
|
|
296
395
|
if (row['embedding'])
|
|
297
396
|
obs.embedding = unpackVector(toBuffer(row['embedding']));
|
|
397
|
+
if (row['embedder'] != null)
|
|
398
|
+
obs.embedder = row['embedder'];
|
|
399
|
+
if (row['author'] != null)
|
|
400
|
+
obs.author = row['author'];
|
|
298
401
|
return obs;
|
|
299
402
|
}
|
|
300
403
|
function toIndexEntry(row) {
|