claude-db 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +17 -2
  2. package/dist/capture/flush.d.ts +22 -0
  3. package/dist/capture/flush.js +45 -10
  4. package/dist/capture/identity.d.ts +6 -0
  5. package/dist/capture/identity.js +14 -0
  6. package/dist/capture/index.d.ts +5 -3
  7. package/dist/capture/index.js +4 -3
  8. package/dist/capture/manual.d.ts +21 -0
  9. package/dist/capture/manual.js +40 -0
  10. package/dist/capture/transcript.d.ts +12 -0
  11. package/dist/capture/transcript.js +69 -1
  12. package/dist/capture/turn-extractor.js +40 -8
  13. package/dist/cli/index.js +376 -30
  14. package/dist/cli/install.d.ts +15 -1
  15. package/dist/cli/install.js +127 -8
  16. package/dist/config/schema.d.ts +1 -0
  17. package/dist/config/schema.js +9 -1
  18. package/dist/context.js +24 -2
  19. package/dist/embed/builtin.js +3 -1
  20. package/dist/hooks/relevance.js +11 -4
  21. package/dist/hooks/session-end.js +2 -1
  22. package/dist/hooks/session-start.js +22 -3
  23. package/dist/mcp/server.js +56 -6
  24. package/dist/search/service.js +4 -1
  25. package/dist/store/adapter.d.ts +22 -6
  26. package/dist/store/adapter.js +10 -1
  27. package/dist/store/mongo/driver.d.ts +1 -0
  28. package/dist/store/mongo/index.d.ts +3 -2
  29. package/dist/store/mongo/index.js +57 -7
  30. package/dist/store/postgres/index.d.ts +22 -8
  31. package/dist/store/postgres/index.js +165 -21
  32. package/dist/store/sqlite/index.d.ts +3 -2
  33. package/dist/store/sqlite/index.js +112 -9
  34. package/dist/store/sqlite/schema.sql +4 -3
  35. package/dist/types.d.ts +28 -0
  36. package/dist/util/project.d.ts +9 -10
  37. package/dist/util/project.js +52 -13
  38. package/dist/util/vector.d.ts +11 -1
  39. package/dist/util/vector.js +14 -3
  40. package/package.json +2 -2
@@ -1,4 +1,5 @@
1
1
  import { importMongo } from './driver.js';
2
+ import { isWholeScope } from '../adapter.js';
2
3
  import { cosine } from '../../util/vector.js';
3
4
  import { partitionIds } from '../../util/shortid.js';
4
5
  /**
@@ -94,12 +95,47 @@ export class MongoStore {
94
95
  .toArray();
95
96
  return docs.map(toObservation);
96
97
  }
97
- async clear(project) {
98
- const filter = project ? { project } : {};
99
- const docs = await this.observations.find(filter, { projection: { _id: 1 } }).toArray();
100
- await this.observations.deleteMany(filter);
101
- await this.sessions.deleteMany(filter);
102
- return docs.length;
98
+ async remove(filter) {
99
+ if (filter.ids?.length === 0)
100
+ return 0;
101
+ const query = {};
102
+ if (filter.project)
103
+ query['project'] = filter.project;
104
+ if (filter.kind)
105
+ query['kind'] = filter.kind;
106
+ if (filter.before !== undefined)
107
+ query['createdAt'] = { $lt: filter.before };
108
+ if (filter.ids) {
109
+ const { exact, prefixes } = partitionIds(filter.ids);
110
+ const alternatives = [];
111
+ if (exact.length > 0)
112
+ alternatives.push({ _id: { $in: exact } });
113
+ for (const prefix of prefixes) {
114
+ alternatives.push({ _id: { $regex: `^${escapeRegex(prefix)}` } });
115
+ }
116
+ query['$or'] = alternatives;
117
+ }
118
+ const count = await this.observations.countDocuments(query);
119
+ await this.observations.deleteMany(query);
120
+ if (isWholeScope(filter)) {
121
+ await this.sessions.deleteMany(filter.project ? { project: filter.project } : {});
122
+ }
123
+ return count;
124
+ }
125
+ async list(filter) {
126
+ const query = {};
127
+ if (filter.project)
128
+ query['project'] = filter.project;
129
+ if (filter.kind)
130
+ query['kind'] = filter.kind;
131
+ if (filter.after !== undefined)
132
+ query['createdAt'] = { $gt: filter.after };
133
+ const docs = await this.observations
134
+ .find(query)
135
+ .sort({ createdAt: 1 })
136
+ .limit(filter.limit ?? 1000)
137
+ .toArray();
138
+ return docs.map(toObservation);
103
139
  }
104
140
  async listProjects() {
105
141
  const rows = await this.observations
@@ -156,7 +192,13 @@ export class MongoStore {
156
192
  // Portable path: score candidates in process. Bounded by the scope filter,
157
193
  // which in practice narrows to a single project.
158
194
  const docs = await this.observations
159
- .find({ embedding: { $exists: true }, ...this.scopeFilter(query) }, { projection: { kind: 1, title: 1, project: 1, createdAt: 1, embedding: 1 } })
195
+ .find({
196
+ embedding: { $exists: true },
197
+ ...(query.embedder
198
+ ? { $or: [{ embedder: query.embedder }, { embedder: { $exists: false } }] }
199
+ : {}),
200
+ ...this.scopeFilter(query),
201
+ }, { projection: { kind: 1, title: 1, project: 1, createdAt: 1, embedding: 1 } })
160
202
  .sort({ createdAt: -1 })
161
203
  .limit(query.maxScanCandidates ?? 25000)
162
204
  .toArray();
@@ -245,6 +287,10 @@ function toDoc(obs) {
245
287
  };
246
288
  if (obs.embedding)
247
289
  doc.embedding = obs.embedding;
290
+ if (obs.embedder)
291
+ doc.embedder = obs.embedder;
292
+ if (obs.author)
293
+ doc.author = obs.author;
248
294
  return doc;
249
295
  }
250
296
  function toObservation(doc) {
@@ -261,6 +307,10 @@ function toObservation(doc) {
261
307
  };
262
308
  if (doc.embedding)
263
309
  obs.embedding = doc.embedding;
310
+ if (doc.embedder)
311
+ obs.embedder = doc.embedder;
312
+ if (doc.author)
313
+ obs.author = doc.author;
264
314
  return obs;
265
315
  }
266
316
  function toIndexEntry(doc, score) {
@@ -1,18 +1,31 @@
1
1
  import type { MemoryStore, ProjectSummary } from '../adapter.js';
2
- import type { Observation, ObservationIndexEntry, SearchQuery, Session, TimelineQuery } from '../../types.js';
3
- /**
4
- * Postgres backend. Uses tsvector for keyword search, and pgvector for
5
- * embeddings when the extension is available. If `CREATE EXTENSION vector`
6
- * is not permitted (common on locked-down managed instances) the store still
7
- * works, minus semantic recall, which the search service handles gracefully.
8
- */
2
+ import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, SearchQuery, Session, TimelineQuery } from '../../types.js';
9
3
  export declare class PostgresStore implements MemoryStore {
10
4
  private readonly pool;
11
5
  readonly kind = "postgres";
12
6
  private vectorEnabled;
7
+ private vectorDims;
8
+ private warnedDims;
13
9
  private constructor();
14
10
  static create(uri: string): Promise<PostgresStore>;
15
11
  init(): Promise<void>;
12
+ /**
13
+ * CREATE TABLE IF NOT EXISTS leaves an existing generated column alone, so a
14
+ * database created before tags were indexed keeps the old expression. Rebuild
15
+ * it once; dropping the column takes its index with it.
16
+ */
17
+ private ensureTagsIndexed;
18
+ private readVectorDims;
19
+ /**
20
+ * Creates the embedding column sized to the vectors actually being stored,
21
+ * since pgvector needs a fixed width but the width belongs to the embedder:
22
+ * 256 for the builtin, 384 for MiniLM. Hardcoding 384 meant every insert on
23
+ * a default install failed on a dimension mismatch and captured nothing.
24
+ * Returns false when this width cannot be stored, and the caller keeps the
25
+ * observation without a vector rather than losing it. `dims` is bounded
26
+ * above because DDL cannot be parameterised.
27
+ */
28
+ private ensureVectorColumn;
16
29
  close(): Promise<void>;
17
30
  ping(): Promise<boolean>;
18
31
  upsertSession(session: Session): Promise<void>;
@@ -20,7 +33,8 @@ export declare class PostgresStore implements MemoryStore {
20
33
  recentSessions(project: string, limit: number): Promise<Session[]>;
21
34
  insertObservations(observations: Observation[]): Promise<void>;
22
35
  getObservations(ids: string[]): Promise<Observation[]>;
23
- clear(project?: string): Promise<number>;
36
+ remove(filter: RemoveFilter): Promise<number>;
37
+ list(filter: ListFilter): Promise<Observation[]>;
24
38
  listProjects(): Promise<ProjectSummary[]>;
25
39
  searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
26
40
  searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
@@ -1,3 +1,4 @@
1
+ import { isWholeScope } from '../adapter.js';
1
2
  import { partitionIds } from '../../util/shortid.js';
2
3
  /**
3
4
  * Postgres backend. Uses tsvector for keyword search, and pgvector for
@@ -5,10 +6,22 @@ import { partitionIds } from '../../util/shortid.js';
5
6
  * is not permitted (common on locked-down managed instances) the store still
6
7
  * works, minus semantic recall, which the search service handles gracefully.
7
8
  */
9
+ /**
10
+ * Title outranks tags, tags outrank body — the same ordering FTS5 gets from its
11
+ * column weights. Tags were missing here entirely, so an observation tagged
12
+ * with the repository it touched was findable on SQLite and Mongo but not on
13
+ * Postgres.
14
+ */
15
+ const TSV_EXPRESSION = `
16
+ setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
17
+ setweight(to_tsvector('english', coalesce(tags::text, '')), 'B') ||
18
+ setweight(to_tsvector('english', coalesce(body, '')), 'C')`;
8
19
  export class PostgresStore {
9
20
  pool;
10
21
  kind = 'postgres';
11
22
  vectorEnabled = false;
23
+ vectorDims = null;
24
+ warnedDims = false;
12
25
  constructor(pool) {
13
26
  this.pool = pool;
14
27
  }
@@ -40,20 +53,72 @@ export class PostgresStore {
40
53
  files JSONB NOT NULL DEFAULT '[]'::jsonb,
41
54
  tags JSONB NOT NULL DEFAULT '[]'::jsonb,
42
55
  created_at BIGINT NOT NULL,
43
- tsv TSVECTOR GENERATED ALWAYS AS (
44
- setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
45
- setweight(to_tsvector('english', coalesce(body, '')), 'B')
46
- ) STORED
56
+ embedder TEXT,
57
+ author TEXT,
58
+ tsv TSVECTOR GENERATED ALWAYS AS (${TSV_EXPRESSION}) STORED
47
59
  );
48
60
  CREATE INDEX IF NOT EXISTS idx_obs_tsv ON observations USING GIN(tsv);
49
61
  CREATE INDEX IF NOT EXISTS idx_obs_project_time
50
62
  ON observations(project, created_at DESC);
63
+
64
+ ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedder TEXT;
65
+ ALTER TABLE observations ADD COLUMN IF NOT EXISTS author TEXT;
51
66
  `);
52
- if (this.vectorEnabled) {
53
- await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(384)`);
67
+ await this.ensureTagsIndexed();
68
+ if (this.vectorEnabled)
69
+ this.vectorDims = await this.readVectorDims();
70
+ }
71
+ /**
72
+ * CREATE TABLE IF NOT EXISTS leaves an existing generated column alone, so a
73
+ * database created before tags were indexed keeps the old expression. Rebuild
74
+ * it once; dropping the column takes its index with it.
75
+ */
76
+ async ensureTagsIndexed() {
77
+ const res = await this.pool.query(`SELECT pg_get_expr(d.adbin, d.adrelid) AS expr
78
+ FROM pg_attrdef d
79
+ JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
80
+ WHERE d.adrelid = 'observations'::regclass AND a.attname = 'tsv'`);
81
+ const expr = String(res.rows[0]?.['expr'] ?? '');
82
+ if (expr.length === 0 || expr.includes('tags'))
83
+ return;
84
+ await this.pool.query('ALTER TABLE observations DROP COLUMN tsv');
85
+ await this.pool.query(`ALTER TABLE observations
86
+ ADD COLUMN tsv TSVECTOR GENERATED ALWAYS AS (${TSV_EXPRESSION}) STORED`);
87
+ await this.pool.query('CREATE INDEX IF NOT EXISTS idx_obs_tsv ON observations USING GIN(tsv)');
88
+ }
89
+ async readVectorDims() {
90
+ const res = await this.pool.query(`SELECT atttypmod AS dims FROM pg_attribute
91
+ WHERE attrelid = 'observations'::regclass
92
+ AND attname = 'embedding' AND NOT attisdropped`);
93
+ const dims = Number(res.rows[0]?.['dims'] ?? -1);
94
+ return dims > 0 ? dims : null;
95
+ }
96
+ /**
97
+ * Creates the embedding column sized to the vectors actually being stored,
98
+ * since pgvector needs a fixed width but the width belongs to the embedder:
99
+ * 256 for the builtin, 384 for MiniLM. Hardcoding 384 meant every insert on
100
+ * a default install failed on a dimension mismatch and captured nothing.
101
+ * Returns false when this width cannot be stored, and the caller keeps the
102
+ * observation without a vector rather than losing it. `dims` is bounded
103
+ * above because DDL cannot be parameterised.
104
+ */
105
+ async ensureVectorColumn(dims) {
106
+ if (!this.vectorEnabled)
107
+ return false;
108
+ if (!Number.isInteger(dims) || dims < 1 || dims > 16000)
109
+ return false;
110
+ if (this.vectorDims === null) {
111
+ await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
54
112
  await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
55
113
  ON observations USING hnsw (embedding vector_cosine_ops)`);
114
+ this.vectorDims = dims;
115
+ }
116
+ if (this.vectorDims !== dims && !this.warnedDims) {
117
+ this.warnedDims = true;
118
+ process.stderr.write(`[claude-db] embeddings are ${dims}d but this database stores ` +
119
+ `${this.vectorDims}d; storing text only. Keyword search is unaffected.\n`);
56
120
  }
121
+ return this.vectorDims === dims;
57
122
  }
58
123
  async close() {
59
124
  await this.pool.end();
@@ -66,6 +131,7 @@ export class PostgresStore {
66
131
  await this.pool.query(`INSERT INTO sessions (id, project, started_at, ended_at, summary)
67
132
  VALUES ($1, $2, $3, $4, $5)
68
133
  ON CONFLICT (id) DO UPDATE SET
134
+ project = EXCLUDED.project,
69
135
  ended_at = COALESCE(EXCLUDED.ended_at, sessions.ended_at),
70
136
  summary = COALESCE(EXCLUDED.summary, sessions.summary)`, [
71
137
  session.id,
@@ -89,22 +155,32 @@ export class PostgresStore {
89
155
  async insertObservations(observations) {
90
156
  if (observations.length === 0)
91
157
  return;
158
+ const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
159
+ const vectors = width !== undefined && (await this.ensureVectorColumn(width));
92
160
  const client = await this.pool.connect();
93
161
  try {
94
162
  await client.query('BEGIN');
95
163
  for (const obs of observations) {
96
- const columns = `id, session_id, project, kind, title, body, files, tags, created_at` +
97
- (this.vectorEnabled && obs.embedding ? ', embedding' : '');
164
+ const columns = [
165
+ 'id', 'session_id', 'project', 'kind', 'title', 'body',
166
+ 'files', 'tags', 'created_at', 'embedder', 'author',
167
+ ];
98
168
  const values = [
99
169
  obs.id, obs.sessionId, obs.project, obs.kind, obs.title, obs.body,
100
170
  JSON.stringify(obs.files), JSON.stringify(obs.tags), obs.createdAt,
171
+ obs.embedder ?? null, obs.author ?? null,
101
172
  ];
102
- if (this.vectorEnabled && obs.embedding) {
173
+ if (vectors && obs.embedding?.length === width) {
174
+ columns.push('embedding');
103
175
  values.push(`[${obs.embedding.join(',')}]`);
104
176
  }
105
177
  const holes = values.map((_, i) => `$${i + 1}`).join(', ');
106
- await client.query(`INSERT INTO observations (${columns}) VALUES (${holes})
107
- ON CONFLICT (id) DO NOTHING`, values);
178
+ const updates = columns
179
+ .slice(1)
180
+ .map((column) => `${column} = EXCLUDED.${column}`)
181
+ .join(', ');
182
+ await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
183
+ ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
108
184
  }
109
185
  await client.query('COMMIT');
110
186
  }
@@ -134,15 +210,45 @@ export class PostgresStore {
134
210
  const res = await this.pool.query(`SELECT * FROM observations WHERE ${clauses.join(' OR ')}`, values);
135
211
  return res.rows.map(toObservation);
136
212
  }
137
- async clear(project) {
138
- const where = project ? 'WHERE project = $1' : '';
139
- const values = project ? [project] : [];
213
+ async remove(filter) {
214
+ if (filter.ids?.length === 0)
215
+ return 0;
216
+ const conditions = [];
217
+ const values = [];
218
+ if (filter.ids) {
219
+ const { exact, prefixes } = partitionIds(filter.ids);
220
+ const alternatives = [];
221
+ if (exact.length > 0) {
222
+ values.push(exact);
223
+ alternatives.push(`id = ANY($${values.length}::text[])`);
224
+ }
225
+ for (const prefix of prefixes) {
226
+ values.push(`${prefix.replace(/[%_]/g, '')}%`);
227
+ alternatives.push(`id LIKE $${values.length}`);
228
+ }
229
+ conditions.push(`(${alternatives.join(' OR ')})`);
230
+ }
231
+ if (filter.project) {
232
+ values.push(filter.project);
233
+ conditions.push(`project = $${values.length}`);
234
+ }
235
+ if (filter.kind) {
236
+ values.push(filter.kind);
237
+ conditions.push(`kind = $${values.length}`);
238
+ }
239
+ if (filter.before !== undefined) {
240
+ values.push(filter.before);
241
+ conditions.push(`created_at < $${values.length}`);
242
+ }
243
+ const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
140
244
  const client = await this.pool.connect();
141
245
  try {
142
246
  await client.query('BEGIN');
143
247
  const counted = await client.query(`SELECT COUNT(*)::int AS n FROM observations ${where}`, values);
144
248
  await client.query(`DELETE FROM observations ${where}`, values);
145
- await client.query(`DELETE FROM sessions ${where}`, values);
249
+ if (isWholeScope(filter)) {
250
+ await client.query(`DELETE FROM sessions ${filter.project ? 'WHERE project = $1' : ''}`, filter.project ? [filter.project] : []);
251
+ }
146
252
  await client.query('COMMIT');
147
253
  return Number(counted.rows[0]?.['n'] ?? 0);
148
254
  }
@@ -154,6 +260,27 @@ export class PostgresStore {
154
260
  client.release();
155
261
  }
156
262
  }
263
+ async list(filter) {
264
+ const conditions = [];
265
+ const values = [];
266
+ if (filter.project) {
267
+ values.push(filter.project);
268
+ conditions.push(`project = $${values.length}`);
269
+ }
270
+ if (filter.kind) {
271
+ values.push(filter.kind);
272
+ conditions.push(`kind = $${values.length}`);
273
+ }
274
+ if (filter.after !== undefined) {
275
+ values.push(filter.after);
276
+ conditions.push(`created_at > $${values.length}`);
277
+ }
278
+ values.push(filter.limit ?? 1000);
279
+ const res = await this.pool.query(`SELECT * FROM observations
280
+ ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
281
+ ORDER BY created_at ASC LIMIT $${values.length}`, values);
282
+ return res.rows.map(toObservation);
283
+ }
157
284
  async listProjects() {
158
285
  const res = await this.pool.query(`SELECT project, COUNT(*)::int AS n, MAX(created_at) AS last
159
286
  FROM observations GROUP BY project ORDER BY last DESC`);
@@ -164,24 +291,36 @@ export class PostgresStore {
164
291
  }));
165
292
  }
166
293
  async searchKeyword(query) {
167
- const conditions = ['tsv @@ plainto_tsquery($1)'];
294
+ const conditions = ['tsv @@ q.tsq'];
168
295
  const values = [query.text];
169
296
  this.appendScope(query, conditions, values);
170
297
  values.push(query.limit);
171
- const res = await this.pool.query(`SELECT id, kind, title, project, created_at,
172
- ts_rank(tsv, plainto_tsquery($1)) AS score
173
- FROM observations
298
+ // plainto_tsquery ANDs every term, so a natural-language prompt only
299
+ // matches a row containing all of it — which for a real prompt is nothing
300
+ // at all, and injection would silently never fire. Flipping the operator on
301
+ // its output keeps Postgres's own parsing, stemming and stopword removal
302
+ // while giving the forgiving recall the other two adapters have; ts_rank
303
+ // then sorts out what actually mattered.
304
+ const res = await this.pool.query(`WITH q AS (
305
+ SELECT replace(plainto_tsquery('english', $1)::text, '&', '|')::tsquery AS tsq
306
+ )
307
+ SELECT id, kind, title, project, created_at, ts_rank(tsv, q.tsq) AS score
308
+ FROM observations, q
174
309
  WHERE ${conditions.join(' AND ')}
175
310
  ORDER BY score DESC
176
311
  LIMIT $${values.length}`, values);
177
312
  return res.rows.map(toIndexEntry);
178
313
  }
179
314
  async searchVector(vector, query) {
180
- if (!this.vectorEnabled)
315
+ if (!this.vectorEnabled || this.vectorDims !== vector.length)
181
316
  return [];
182
317
  const conditions = ['embedding IS NOT NULL'];
183
318
  const values = [`[${vector.join(',')}]`];
184
319
  this.appendScope(query, conditions, values);
320
+ if (query.embedder) {
321
+ values.push(query.embedder);
322
+ conditions.push(`(embedder = $${values.length} OR embedder IS NULL)`);
323
+ }
185
324
  values.push(query.limit);
186
325
  const res = await this.pool.query(`SELECT id, kind, title, project, created_at,
187
326
  1 - (embedding <=> $1::vector) AS score
@@ -265,7 +404,7 @@ function toSession(row) {
265
404
  return session;
266
405
  }
267
406
  function toObservation(row) {
268
- return {
407
+ const obs = {
269
408
  id: row['id'],
270
409
  sessionId: row['session_id'],
271
410
  project: row['project'],
@@ -276,6 +415,11 @@ function toObservation(row) {
276
415
  tags: row['tags'] ?? [],
277
416
  createdAt: Number(row['created_at']),
278
417
  };
418
+ if (row['embedder'] != null)
419
+ obs.embedder = row['embedder'];
420
+ if (row['author'] != null)
421
+ obs.author = row['author'];
422
+ return obs;
279
423
  }
280
424
  function toIndexEntry(row) {
281
425
  return {
@@ -1,5 +1,5 @@
1
1
  import type { MemoryStore, ProjectSummary } from '../adapter.js';
2
- import type { Observation, ObservationIndexEntry, SearchQuery, Session, TimelineQuery } from '../../types.js';
2
+ import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, SearchQuery, Session, TimelineQuery } from '../../types.js';
3
3
  /**
4
4
  * Zero-config default backend, built on Node's own `node:sqlite`.
5
5
  *
@@ -22,7 +22,8 @@ export declare class SqliteStore implements MemoryStore {
22
22
  recentSessions(project: string, limit: number): Promise<Session[]>;
23
23
  insertObservations(observations: Observation[]): Promise<void>;
24
24
  getObservations(ids: string[]): Promise<Observation[]>;
25
- clear(project?: string): Promise<number>;
25
+ remove(filter: RemoveFilter): Promise<number>;
26
+ list(filter: ListFilter): Promise<Observation[]>;
26
27
  listProjects(): Promise<ProjectSummary[]>;
27
28
  searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
28
29
  searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
@@ -2,6 +2,7 @@ import { DatabaseSync } from 'node:sqlite';
2
2
  import { mkdirSync, readFileSync } from 'node:fs';
3
3
  import { dirname, resolve } from 'node:path';
4
4
  import { fileURLToPath } from 'node:url';
5
+ import { isWholeScope } from '../adapter.js';
5
6
  import { cosine, packVector, unpackVector } from '../../util/vector.js';
6
7
  import { scopeToken } from '../../util/scope.js';
7
8
  import { partitionIds } from '../../util/shortid.js';
@@ -28,6 +29,27 @@ export class SqliteStore {
28
29
  }
29
30
  async init() {
30
31
  this.db.exec(readFileSync(resolve(HERE, 'schema.sql'), 'utf8'));
32
+ const row = this.db.prepare('PRAGMA user_version').get();
33
+ const version = Number(row?.['user_version'] ?? 0);
34
+ // Databases written before recursive_triggers was set carry orphaned FTS
35
+ // rows for every re-ingested observation. One rebuild clears them.
36
+ if (version < 1) {
37
+ this.db.exec(`INSERT INTO observations_fts(observations_fts) VALUES('rebuild');`);
38
+ }
39
+ // CREATE TABLE IF NOT EXISTS leaves an existing table alone, so columns
40
+ // added after the fact have to be applied here.
41
+ if (version < 2) {
42
+ for (const column of ['embedder TEXT', 'author TEXT']) {
43
+ try {
44
+ this.db.exec(`ALTER TABLE observations ADD COLUMN ${column}`);
45
+ }
46
+ catch {
47
+ // Already present, which is the normal case on a fresh database.
48
+ }
49
+ }
50
+ }
51
+ if (version < 2)
52
+ this.db.exec('PRAGMA user_version = 2');
31
53
  }
32
54
  async close() {
33
55
  this.db.close();
@@ -40,6 +62,7 @@ export class SqliteStore {
40
62
  .prepare(`INSERT INTO sessions (id, project, started_at, ended_at, summary)
41
63
  VALUES (?, ?, ?, ?, ?)
42
64
  ON CONFLICT(id) DO UPDATE SET
65
+ project = excluded.project,
43
66
  ended_at = COALESCE(excluded.ended_at, sessions.ended_at),
44
67
  summary = COALESCE(excluded.summary, sessions.summary)`)
45
68
  .run(session.id, session.project, session.startedAt, session.endedAt ?? null, session.summary ?? null);
@@ -60,12 +83,13 @@ export class SqliteStore {
60
83
  if (observations.length === 0)
61
84
  return;
62
85
  const stmt = this.db.prepare(`INSERT OR REPLACE INTO observations
63
- (id, session_id, project, scope, kind, title, body, files, tags, created_at, embedding)
64
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
86
+ (id, session_id, project, scope, kind, title, body, files, tags,
87
+ created_at, embedding, embedder, author)
88
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
65
89
  this.db.exec('BEGIN');
66
90
  try {
67
91
  for (const obs of observations) {
68
- stmt.run(obs.id, obs.sessionId, obs.project, scopeToken(obs.project), obs.kind, obs.title, obs.body, JSON.stringify(obs.files), JSON.stringify(obs.tags), obs.createdAt, obs.embedding ? packVector(obs.embedding) : null);
92
+ stmt.run(obs.id, obs.sessionId, obs.project, scopeToken(obs.project), obs.kind, obs.title, obs.body, JSON.stringify(obs.files), JSON.stringify(obs.tags), obs.createdAt, obs.embedding ? packVector(obs.embedding) : null, obs.embedder ?? null, obs.author ?? null);
69
93
  }
70
94
  this.db.exec('COMMIT');
71
95
  }
@@ -95,9 +119,11 @@ export class SqliteStore {
95
119
  .all(...params);
96
120
  return rows.map(toObservation);
97
121
  }
98
- async clear(project) {
99
- const where = project ? 'WHERE project = ?' : '';
100
- const params = project ? [project] : [];
122
+ async remove(filter) {
123
+ if (filter.ids?.length === 0)
124
+ return 0;
125
+ const { where, params } = removeWhere(filter);
126
+ const wipe = isWholeScope(filter);
101
127
  const count = this.db
102
128
  .prepare(`SELECT COUNT(*) AS n FROM observations ${where}`)
103
129
  .get(...params);
@@ -106,16 +132,47 @@ export class SqliteStore {
106
132
  // The FTS mirror is kept in sync by triggers, so deleting rows here is
107
133
  // enough; dropping it manually would desynchronise the two.
108
134
  this.db.prepare(`DELETE FROM observations ${where}`).run(...params);
109
- this.db.prepare(`DELETE FROM sessions ${where}`).run(...params);
135
+ if (wipe) {
136
+ const scope = filter.project ? 'WHERE project = ?' : '';
137
+ this.db
138
+ .prepare(`DELETE FROM sessions ${scope}`)
139
+ .run(...(filter.project ? [filter.project] : []));
140
+ }
110
141
  this.db.exec('COMMIT');
111
142
  }
112
143
  catch (error) {
113
144
  this.db.exec('ROLLBACK');
114
145
  throw error;
115
146
  }
116
- this.db.exec('VACUUM');
147
+ // Only worth the rewrite when a whole scope went; forgetting one row does
148
+ // not justify rebuilding the file.
149
+ if (wipe)
150
+ this.db.exec('VACUUM');
117
151
  return count?.n ?? 0;
118
152
  }
153
+ async list(filter) {
154
+ const conditions = [];
155
+ const params = [];
156
+ if (filter.project) {
157
+ conditions.push('project = ?');
158
+ params.push(filter.project);
159
+ }
160
+ if (filter.kind) {
161
+ conditions.push('kind = ?');
162
+ params.push(filter.kind);
163
+ }
164
+ if (filter.after !== undefined) {
165
+ conditions.push('created_at > ?');
166
+ params.push(filter.after);
167
+ }
168
+ params.push(filter.limit ?? 1000);
169
+ const rows = this.db
170
+ .prepare(`SELECT * FROM observations
171
+ ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
172
+ ORDER BY created_at ASC LIMIT ?`)
173
+ .all(...params);
174
+ return rows.map(toObservation);
175
+ }
119
176
  async listProjects() {
120
177
  const rows = this.db
121
178
  .prepare(`SELECT project, COUNT(*) AS n, MAX(created_at) AS last
@@ -169,6 +226,12 @@ export class SqliteStore {
169
226
  const conditions = ['embedding IS NOT NULL'];
170
227
  const params = [];
171
228
  appendScope(query, conditions, params, '');
229
+ // NULL means the row predates the column, and the width check in cosine()
230
+ // still guards it. Anything else is a different embedding space.
231
+ if (query.embedder) {
232
+ conditions.push('(embedder = ? OR embedder IS NULL)');
233
+ params.push(query.embedder);
234
+ }
172
235
  // Newest-first with a hard cap. The scan is linear in rows returned, so
173
236
  // this is what keeps p95 flat as history grows: recent memory stays
174
237
  // semantically searchable and older memory remains reachable by keyword.
@@ -215,6 +278,39 @@ export class SqliteStore {
215
278
  return rows.map(toIndexEntry);
216
279
  }
217
280
  }
281
+ function removeWhere(filter) {
282
+ const conditions = [];
283
+ const params = [];
284
+ if (filter.ids) {
285
+ const { exact, prefixes } = partitionIds(filter.ids);
286
+ const alternatives = [];
287
+ if (exact.length > 0) {
288
+ alternatives.push(`id IN (${exact.map(() => '?').join(',')})`);
289
+ params.push(...exact);
290
+ }
291
+ for (const prefix of prefixes) {
292
+ alternatives.push('id GLOB ?');
293
+ params.push(`${prefix}*`);
294
+ }
295
+ conditions.push(`(${alternatives.join(' OR ')})`);
296
+ }
297
+ if (filter.project) {
298
+ conditions.push('project = ?');
299
+ params.push(filter.project);
300
+ }
301
+ if (filter.kind) {
302
+ conditions.push('kind = ?');
303
+ params.push(filter.kind);
304
+ }
305
+ if (filter.before !== undefined) {
306
+ conditions.push('created_at < ?');
307
+ params.push(filter.before);
308
+ }
309
+ return {
310
+ where: conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '',
311
+ params,
312
+ };
313
+ }
218
314
  function appendScope(query, conditions, params, prefix) {
219
315
  if (query.project) {
220
316
  conditions.push(`${prefix}project = ?`);
@@ -252,9 +348,12 @@ function toFilePath(uri) {
252
348
  * query entirely rather than issue one that matches everything.
253
349
  */
254
350
  function toMatchExpression(text, project) {
351
+ // ponytail: unicode61 keeps a run of Han characters as one token, so CJK
352
+ // matches phrases but not substrings. Vector recall covers the gap; swap in
353
+ // an ICU or bigram tokenizer if that stops being enough.
255
354
  const tokens = text
256
355
  .toLowerCase()
257
- .split(/[^a-z0-9_]+/i)
356
+ .split(/[^\p{L}\p{N}_]+/u)
258
357
  .filter((token) => token.length > 1);
259
358
  const scope = project ? `scope:${scopeToken(project)}` : null;
260
359
  if (tokens.length === 0)
@@ -295,6 +394,10 @@ function toObservation(row) {
295
394
  };
296
395
  if (row['embedding'])
297
396
  obs.embedding = unpackVector(toBuffer(row['embedding']));
397
+ if (row['embedder'] != null)
398
+ obs.embedder = row['embedder'];
399
+ if (row['author'] != null)
400
+ obs.author = row['author'];
298
401
  return obs;
299
402
  }
300
403
  function toIndexEntry(row) {