claude-db 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/dist/capture/flush.d.ts +20 -1
- package/dist/capture/flush.js +45 -5
- package/dist/capture/git.d.ts +2 -0
- package/dist/capture/git.js +96 -0
- package/dist/capture/index.d.ts +4 -3
- package/dist/capture/index.js +4 -3
- package/dist/capture/manual.d.ts +9 -0
- package/dist/capture/manual.js +0 -0
- package/dist/capture/transcript.d.ts +8 -0
- package/dist/capture/transcript.js +30 -0
- package/dist/capture/turn-extractor.d.ts +9 -1
- package/dist/capture/turn-extractor.js +74 -13
- package/dist/cli/index.js +383 -27
- package/dist/cli/install.d.ts +8 -0
- package/dist/cli/install.js +24 -0
- package/dist/mcp/render.d.ts +23 -0
- package/dist/mcp/render.js +49 -0
- package/dist/mcp/server.d.ts +1 -7
- package/dist/mcp/server.js +37 -36
- package/dist/store/adapter.d.ts +10 -0
- package/dist/store/adapter.js +8 -0
- package/dist/store/mongo/index.d.ts +1 -0
- package/dist/store/mongo/index.js +34 -3
- package/dist/store/postgres/index.d.ts +15 -0
- package/dist/store/postgres/index.js +89 -4
- package/dist/store/sqlite/index.d.ts +1 -0
- package/dist/store/sqlite/index.js +32 -2
- package/dist/types.d.ts +16 -0
- package/dist/util/snippet.d.ts +28 -0
- package/dist/util/snippet.js +52 -0
- package/package.json +2 -1
- package/skills/cdb-scan/SKILL.md +72 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { toShortId } from '../util/shortid.js';
|
|
2
|
+
import { clipBody } from '../util/snippet.js';
|
|
3
|
+
/**
|
|
4
|
+
* How the three layers are shown to the agent.
|
|
5
|
+
*
|
|
6
|
+
* Split out of the server so it can be exercised without opening a stdio
|
|
7
|
+
* transport: this is the format every search result is paid in, and it had no
|
|
8
|
+
* test of any kind while the server module could only be imported by starting
|
|
9
|
+
* a server.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* Layer 1. Every character here is multiplied by the number of results and
|
|
13
|
+
* paid on every search, so the format stays terse: short id, kind, month-day,
|
|
14
|
+
* title, and one indented line of the body around the match.
|
|
15
|
+
*
|
|
16
|
+
* The snippet is the half that was missing. Terseness only pays if the row can
|
|
17
|
+
* be chosen from what is shown, and a title often cannot carry that — fifteen
|
|
18
|
+
* results dated the same day, titled things like "Committed as 30daa92", leave
|
|
19
|
+
* a reader nothing to pick on, so they expand two at random and pay a full body
|
|
20
|
+
* for each guess. Sixty characters here saves several thousand there.
|
|
21
|
+
*/
|
|
22
|
+
export function renderIndex(entries) {
|
|
23
|
+
if (entries.length === 0)
|
|
24
|
+
return 'No matching observations.';
|
|
25
|
+
const rows = entries.map((entry) => {
|
|
26
|
+
const date = new Date(entry.createdAt).toISOString().slice(5, 10);
|
|
27
|
+
const head = `${toShortId(entry.id)} ${entry.kind} ${date} ${entry.title}`;
|
|
28
|
+
// Absent for a row only vector search found: it holds no query terms to
|
|
29
|
+
// centre a fragment on.
|
|
30
|
+
return entry.snippet ? `${head}\n ${entry.snippet}` : head;
|
|
31
|
+
});
|
|
32
|
+
return `${entries.length} result(s):\n${rows.join('\n')}`;
|
|
33
|
+
}
|
|
34
|
+
/** Layer 3. Full detail for one observation, bounded by `chars`. */
|
|
35
|
+
export function renderFull(obs, chars) {
|
|
36
|
+
return [
|
|
37
|
+
`id: ${toShortId(obs.id)}`,
|
|
38
|
+
`kind: ${obs.kind}`,
|
|
39
|
+
`when: ${new Date(obs.createdAt).toISOString()}`,
|
|
40
|
+
obs.author ? `who: ${obs.author}` : null,
|
|
41
|
+
obs.files.length > 0 ? `files: ${obs.files.join(', ')}` : null,
|
|
42
|
+
'',
|
|
43
|
+
obs.title,
|
|
44
|
+
'',
|
|
45
|
+
clipBody(obs.body, chars),
|
|
46
|
+
]
|
|
47
|
+
.filter((line) => line !== null)
|
|
48
|
+
.join('\n');
|
|
49
|
+
}
|
package/dist/mcp/server.d.ts
CHANGED
|
@@ -1,8 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Layer 1 rendering. Every character here is multiplied by the number of
|
|
5
|
-
* results and paid on every search, so the format is deliberately terse:
|
|
6
|
-
* short id, kind, month-day, title. No padding, no separators, no year.
|
|
7
|
-
*/
|
|
8
|
-
export declare function renderIndex(entries: ObservationIndexEntry[]): string;
|
|
2
|
+
export {};
|
package/dist/mcp/server.js
CHANGED
|
@@ -8,6 +8,7 @@ import { z } from 'zod';
|
|
|
8
8
|
import { remember } from '../capture/index.js';
|
|
9
9
|
import { createContext } from '../context.js';
|
|
10
10
|
import { toShortId } from '../util/shortid.js';
|
|
11
|
+
import { renderFull, renderIndex } from './render.js';
|
|
11
12
|
import { resolveProject } from '../util/project.js';
|
|
12
13
|
import { silenceSqliteWarning } from '../util/warnings.js';
|
|
13
14
|
silenceSqliteWarning();
|
|
@@ -36,21 +37,28 @@ function packageVersion() {
|
|
|
36
37
|
const ctx = await createContext();
|
|
37
38
|
const server = new McpServer({ name: 'claude-db', version: packageVersion() });
|
|
38
39
|
server.tool('search', 'Layer 1. Search project memory and get a compact index of matches ' +
|
|
39
|
-
'(id, kind, title,
|
|
40
|
-
'
|
|
40
|
+
'(id, kind, date, title, and a line of the matching body) without full ' +
|
|
41
|
+
'bodies. Start here, and use the snippet to decide which ids are worth ' +
|
|
42
|
+
'passing to get_observations rather than expanding on the title alone.', {
|
|
41
43
|
query: z.string().describe('Natural language description of what you need'),
|
|
42
44
|
project: z
|
|
43
45
|
.string()
|
|
44
46
|
.optional()
|
|
45
47
|
.describe('Absolute project path; defaults to cwd. Pass "*" to search every project'),
|
|
46
48
|
kind: z.enum(KINDS).optional(),
|
|
49
|
+
tag: z
|
|
50
|
+
.string()
|
|
51
|
+
.optional()
|
|
52
|
+
.describe('Limit to one repository or top-level directory, e.g. "backend". Useful ' +
|
|
53
|
+
'when a workspace pools several repos under one project'),
|
|
47
54
|
limit: z.number().int().min(1).max(50).default(10),
|
|
48
|
-
}, async ({ query, project, kind, limit }) => {
|
|
55
|
+
}, async ({ query, project, kind, tag, limit }) => {
|
|
49
56
|
const entries = await ctx.search.search({
|
|
50
57
|
text: query,
|
|
51
58
|
limit,
|
|
52
59
|
...(project === '*' ? {} : { project: resolveProject(project) }),
|
|
53
60
|
...(kind ? { kind } : {}),
|
|
61
|
+
...(tag ? { tag } : {}),
|
|
54
62
|
});
|
|
55
63
|
return { content: [{ type: 'text', text: renderIndex(entries) }] };
|
|
56
64
|
});
|
|
@@ -61,11 +69,19 @@ server.tool('remember', 'Record something the user stated outright that no trans
|
|
|
61
69
|
text: z.string().describe('What to remember, in full. First line becomes the title'),
|
|
62
70
|
kind: z.enum(KINDS).default('preference'),
|
|
63
71
|
project: z.string().optional().describe('Absolute project path; defaults to cwd'),
|
|
64
|
-
|
|
72
|
+
key: z
|
|
73
|
+
.string()
|
|
74
|
+
.optional()
|
|
75
|
+
.describe('Stable name for a note meant to be kept current, e.g. "profile:stack". ' +
|
|
76
|
+
'Remembering the same key again replaces it instead of adding a duplicate'),
|
|
77
|
+
tags: z.array(z.string()).max(5).optional().describe('Extra tags for filtering'),
|
|
78
|
+
}, async ({ text, kind, project, key, tags }) => {
|
|
65
79
|
const observation = await remember(ctx, {
|
|
66
80
|
project: resolveProject(project),
|
|
67
81
|
text,
|
|
68
82
|
kind,
|
|
83
|
+
...(key ? { key } : {}),
|
|
84
|
+
...(tags ? { tags } : {}),
|
|
69
85
|
});
|
|
70
86
|
return {
|
|
71
87
|
content: [
|
|
@@ -94,40 +110,25 @@ server.tool('timeline', 'Layer 2. Show what happened immediately before and afte
|
|
|
94
110
|
});
|
|
95
111
|
server.tool('get_observations', 'Layer 3. Fetch full bodies for specific observation ids, using the short ' +
|
|
96
112
|
'ids returned by search. Always batch every id you need into one call ' +
|
|
97
|
-
'rather than calling repeatedly.', {
|
|
113
|
+
'rather than calling repeatedly.', {
|
|
114
|
+
ids: z.array(z.string()).min(1).max(25),
|
|
115
|
+
chars: z
|
|
116
|
+
.number()
|
|
117
|
+
.int()
|
|
118
|
+
.min(200)
|
|
119
|
+
.max(20000)
|
|
120
|
+
.default(2000)
|
|
121
|
+
.describe('Characters of each body to return. Bodies run to 4000 and this call ' +
|
|
122
|
+
'takes 25 ids, so the default keeps a batch readable; raise it when a ' +
|
|
123
|
+
'truncated answer says there is more'),
|
|
124
|
+
}, async ({ ids, chars }) => {
|
|
98
125
|
const observations = await ctx.search.getObservations(ids);
|
|
99
|
-
return {
|
|
126
|
+
return {
|
|
127
|
+
content: [
|
|
128
|
+
{ type: 'text', text: observations.map((obs) => renderFull(obs, chars)).join('\n\n---\n\n') },
|
|
129
|
+
],
|
|
130
|
+
};
|
|
100
131
|
});
|
|
101
|
-
/**
|
|
102
|
-
* Layer 1 rendering. Every character here is multiplied by the number of
|
|
103
|
-
* results and paid on every search, so the format is deliberately terse:
|
|
104
|
-
* short id, kind, month-day, title. No padding, no separators, no year.
|
|
105
|
-
*/
|
|
106
|
-
export function renderIndex(entries) {
|
|
107
|
-
if (entries.length === 0)
|
|
108
|
-
return 'No matching observations.';
|
|
109
|
-
const rows = entries.map((entry) => {
|
|
110
|
-
const date = new Date(entry.createdAt).toISOString().slice(5, 10);
|
|
111
|
-
return `${toShortId(entry.id)} ${entry.kind} ${date} ${entry.title}`;
|
|
112
|
-
});
|
|
113
|
-
return `${entries.length} result(s):\n${rows.join('\n')}`;
|
|
114
|
-
}
|
|
115
|
-
function renderFull(obs) {
|
|
116
|
-
const date = new Date(obs.createdAt).toISOString();
|
|
117
|
-
return [
|
|
118
|
-
`id: ${toShortId(obs.id)}`,
|
|
119
|
-
`kind: ${obs.kind}`,
|
|
120
|
-
`when: ${date}`,
|
|
121
|
-
obs.author ? `who: ${obs.author}` : null,
|
|
122
|
-
obs.files.length > 0 ? `files: ${obs.files.join(', ')}` : null,
|
|
123
|
-
'',
|
|
124
|
-
obs.title,
|
|
125
|
-
'',
|
|
126
|
-
obs.body,
|
|
127
|
-
]
|
|
128
|
-
.filter((line) => line !== null)
|
|
129
|
-
.join('\n');
|
|
130
|
-
}
|
|
131
132
|
const transport = new StdioServerTransport();
|
|
132
133
|
await server.connect(transport);
|
|
133
134
|
for (const signal of ['SIGINT', 'SIGTERM']) {
|
package/dist/store/adapter.d.ts
CHANGED
|
@@ -42,6 +42,15 @@ export interface MemoryStore {
|
|
|
42
42
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
43
43
|
/** Every project with memory in this database, most recently active first. */
|
|
44
44
|
listProjects(): Promise<ProjectSummary[]>;
|
|
45
|
+
/**
|
|
46
|
+
* Tables or collections in this database that belong to somebody else.
|
|
47
|
+
*
|
|
48
|
+
* Called by `use` before `init()` creates ours, because afterwards there is
|
|
49
|
+
* no way to tell an empty memory database from an application database we
|
|
50
|
+
* just added two tables to. One `use` in anger pointed at a live e-commerce
|
|
51
|
+
* database and nothing said a word.
|
|
52
|
+
*/
|
|
53
|
+
inventory(): Promise<string[]>;
|
|
45
54
|
/** Lexical match. Returns scores on an adapter-defined scale. */
|
|
46
55
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
47
56
|
/** Vector match. Return [] when the backend has no vector support. */
|
|
@@ -56,6 +65,7 @@ export interface MemoryStore {
|
|
|
56
65
|
* SessionStart loses sessions whose work is still there.
|
|
57
66
|
*/
|
|
58
67
|
export declare function isWholeScope(filter: RemoveFilter): boolean;
|
|
68
|
+
export declare function foreignNames(names: string[]): string[];
|
|
59
69
|
export interface StoreFactory {
|
|
60
70
|
/** Lowercase URI schemes this factory claims, e.g. ['mongodb','mongodb+srv']. */
|
|
61
71
|
schemes: string[];
|
package/dist/store/adapter.js
CHANGED
|
@@ -8,3 +8,11 @@
|
|
|
8
8
|
export function isWholeScope(filter) {
|
|
9
9
|
return !filter.ids && !filter.kind && filter.before === undefined;
|
|
10
10
|
}
|
|
11
|
+
/**
|
|
12
|
+
* Ours, on any backend: the two tables, SQLite's FTS shadow tables, and the
|
|
13
|
+
* Postgres schema-version row.
|
|
14
|
+
*/
|
|
15
|
+
const OURS = /^(claude_db_meta|sessions|observations)(_fts(_\w+)?)?$/;
|
|
16
|
+
export function foreignNames(names) {
|
|
17
|
+
return names.filter((name) => name.length > 0 && !OURS.test(name));
|
|
18
|
+
}
|
|
@@ -29,6 +29,7 @@ export declare class MongoStore implements MemoryStore {
|
|
|
29
29
|
remove(filter: RemoveFilter): Promise<number>;
|
|
30
30
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
31
31
|
listProjects(): Promise<ProjectSummary[]>;
|
|
32
|
+
inventory(): Promise<string[]>;
|
|
32
33
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
33
34
|
searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
34
35
|
timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { importMongo } from './driver.js';
|
|
2
|
-
import { isWholeScope } from '../adapter.js';
|
|
2
|
+
import { foreignNames, isWholeScope } from '../adapter.js';
|
|
3
3
|
import { cosine } from '../../util/vector.js';
|
|
4
4
|
import { partitionIds } from '../../util/shortid.js';
|
|
5
|
+
import { toSnippet } from '../../util/snippet.js';
|
|
5
6
|
/**
|
|
6
7
|
* Works against any MongoDB: Atlas, self-hosted, or a local container.
|
|
7
8
|
*
|
|
@@ -150,13 +151,37 @@ export class MongoStore {
|
|
|
150
151
|
lastActive: row.last,
|
|
151
152
|
}));
|
|
152
153
|
}
|
|
154
|
+
async inventory() {
|
|
155
|
+
// Through `command` rather than `listCollections`, so the structural driver
|
|
156
|
+
// types this adapter compiles against stay as small as they are.
|
|
157
|
+
const res = await this.db.command({ listCollections: 1, nameOnly: true });
|
|
158
|
+
const batch = res['cursor']?.firstBatch ?? [];
|
|
159
|
+
return foreignNames(batch.map((doc) => String(doc['name'] ?? '')));
|
|
160
|
+
}
|
|
153
161
|
async searchKeyword(query) {
|
|
154
162
|
const filter = {
|
|
155
163
|
$text: { $search: query.text },
|
|
156
164
|
...this.scopeFilter(query),
|
|
157
165
|
};
|
|
166
|
+
// $text has no fragment operator, so unlike FTS5 and ts_headline this is
|
|
167
|
+
// the head of the body rather than the text around the match. Still enough
|
|
168
|
+
// to tell two same-titled rows apart, and it costs one projected field
|
|
169
|
+
// instead of shipping whole bodies to slice them here.
|
|
170
|
+
//
|
|
171
|
+
// Inclusion throughout: MongoDB rejects a projection that mixes included
|
|
172
|
+
// and excluded fields, so the previous `body: 0, embedding: 0` form cannot
|
|
173
|
+
// carry a computed one.
|
|
158
174
|
const docs = await this.observations
|
|
159
|
-
.find(filter, {
|
|
175
|
+
.find(filter, {
|
|
176
|
+
projection: {
|
|
177
|
+
kind: 1,
|
|
178
|
+
title: 1,
|
|
179
|
+
project: 1,
|
|
180
|
+
createdAt: 1,
|
|
181
|
+
snippet: { $substrCP: ['$body', 0, 240] },
|
|
182
|
+
score: { $meta: 'textScore' },
|
|
183
|
+
},
|
|
184
|
+
})
|
|
160
185
|
.sort({ score: { $meta: 'textScore' } })
|
|
161
186
|
.limit(query.limit)
|
|
162
187
|
.toArray();
|
|
@@ -232,6 +257,8 @@ export class MongoStore {
|
|
|
232
257
|
filter['project'] = query.project;
|
|
233
258
|
if (query.kind)
|
|
234
259
|
filter['kind'] = query.kind;
|
|
260
|
+
if (query.tag)
|
|
261
|
+
filter['tags'] = query.tag;
|
|
235
262
|
if (query.since !== undefined || query.until !== undefined) {
|
|
236
263
|
const range = {};
|
|
237
264
|
if (query.since !== undefined)
|
|
@@ -314,7 +341,7 @@ function toObservation(doc) {
|
|
|
314
341
|
return obs;
|
|
315
342
|
}
|
|
316
343
|
function toIndexEntry(doc, score) {
|
|
317
|
-
|
|
344
|
+
const entry = {
|
|
318
345
|
id: doc._id,
|
|
319
346
|
kind: doc.kind,
|
|
320
347
|
title: doc.title,
|
|
@@ -322,4 +349,8 @@ function toIndexEntry(doc, score) {
|
|
|
322
349
|
createdAt: doc.createdAt,
|
|
323
350
|
score,
|
|
324
351
|
};
|
|
352
|
+
const snippet = toSnippet(doc.snippet);
|
|
353
|
+
if (snippet)
|
|
354
|
+
entry.snippet = snippet;
|
|
355
|
+
return entry;
|
|
325
356
|
}
|
|
@@ -8,7 +8,21 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
8
8
|
private warnedDims;
|
|
9
9
|
private constructor();
|
|
10
10
|
static create(uri: string): Promise<PostgresStore>;
|
|
11
|
+
/**
|
|
12
|
+
* Every hook is its own process, so nothing here is ever pooled or reused:
|
|
13
|
+
* this ran in full on every single prompt. Against a managed Postgres, where
|
|
14
|
+
* one round trip measured 49ms at best and 1045ms at the median, four of them
|
|
15
|
+
* before the first real query is most of why a remote backend feels broken.
|
|
16
|
+
*
|
|
17
|
+
* The version row collapses that to one read on the steady-state path. It
|
|
18
|
+
* carries the vector state too, because otherwise proving the extension is
|
|
19
|
+
* available costs back the round trip just saved. Anything unexpected — no
|
|
20
|
+
* table, an older version, an unreadable row — falls through to the full DDL,
|
|
21
|
+
* so the worst case is exactly the behaviour this replaces.
|
|
22
|
+
*/
|
|
11
23
|
init(): Promise<void>;
|
|
24
|
+
private readMeta;
|
|
25
|
+
private writeMeta;
|
|
12
26
|
/**
|
|
13
27
|
* CREATE TABLE IF NOT EXISTS leaves an existing generated column alone, so a
|
|
14
28
|
* database created before tags were indexed keeps the old expression. Rebuild
|
|
@@ -36,6 +50,7 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
36
50
|
remove(filter: RemoveFilter): Promise<number>;
|
|
37
51
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
38
52
|
listProjects(): Promise<ProjectSummary[]>;
|
|
53
|
+
inventory(): Promise<string[]>;
|
|
39
54
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
40
55
|
searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
41
56
|
timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { isWholeScope } from '../adapter.js';
|
|
1
|
+
import { foreignNames, isWholeScope } from '../adapter.js';
|
|
2
2
|
import { partitionIds } from '../../util/shortid.js';
|
|
3
|
+
import { toSnippet } from '../../util/snippet.js';
|
|
3
4
|
/**
|
|
4
5
|
* Postgres backend. Uses tsvector for keyword search, and pgvector for
|
|
5
6
|
* embeddings when the extension is available. If `CREATE EXTENSION vector`
|
|
@@ -16,6 +17,11 @@ const TSV_EXPRESSION = `
|
|
|
16
17
|
setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
|
|
17
18
|
setweight(to_tsvector('english', coalesce(tags::text, '')), 'B') ||
|
|
18
19
|
setweight(to_tsvector('english', coalesce(body, '')), 'C')`;
|
|
20
|
+
/**
|
|
21
|
+
* Bumped whenever the DDL below changes, which makes every process re-run it
|
|
22
|
+
* once. Only ever compared for equality.
|
|
23
|
+
*/
|
|
24
|
+
const SCHEMA_VERSION = 1;
|
|
19
25
|
export class PostgresStore {
|
|
20
26
|
pool;
|
|
21
27
|
kind = 'postgres';
|
|
@@ -30,9 +36,34 @@ export class PostgresStore {
|
|
|
30
36
|
const pool = new pg.Pool({ connectionString: uri });
|
|
31
37
|
return new PostgresStore(pool);
|
|
32
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Every hook is its own process, so nothing here is ever pooled or reused:
|
|
41
|
+
* this ran in full on every single prompt. Against a managed Postgres, where
|
|
42
|
+
* one round trip measured 49ms at best and 1045ms at the median, four of them
|
|
43
|
+
* before the first real query is most of why a remote backend feels broken.
|
|
44
|
+
*
|
|
45
|
+
* The version row collapses that to one read on the steady-state path. It
|
|
46
|
+
* carries the vector state too, because otherwise proving the extension is
|
|
47
|
+
* available costs back the round trip just saved. Anything unexpected — no
|
|
48
|
+
* table, an older version, an unreadable row — falls through to the full DDL,
|
|
49
|
+
* so the worst case is exactly the behaviour this replaces.
|
|
50
|
+
*/
|
|
33
51
|
async init() {
|
|
52
|
+
const meta = await this.readMeta();
|
|
53
|
+
if (meta?.version === SCHEMA_VERSION) {
|
|
54
|
+
this.vectorEnabled = meta.vectorEnabled;
|
|
55
|
+
this.vectorDims = meta.vectorDims;
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
34
58
|
this.vectorEnabled = await this.tryEnableVector();
|
|
35
59
|
await this.pool.query(`
|
|
60
|
+
CREATE TABLE IF NOT EXISTS claude_db_meta (
|
|
61
|
+
id INT PRIMARY KEY,
|
|
62
|
+
version INT NOT NULL,
|
|
63
|
+
vector_enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
|
64
|
+
vector_dims INT
|
|
65
|
+
);
|
|
66
|
+
|
|
36
67
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
37
68
|
id TEXT PRIMARY KEY,
|
|
38
69
|
project TEXT NOT NULL,
|
|
@@ -60,6 +91,7 @@ export class PostgresStore {
|
|
|
60
91
|
CREATE INDEX IF NOT EXISTS idx_obs_tsv ON observations USING GIN(tsv);
|
|
61
92
|
CREATE INDEX IF NOT EXISTS idx_obs_project_time
|
|
62
93
|
ON observations(project, created_at DESC);
|
|
94
|
+
CREATE INDEX IF NOT EXISTS idx_obs_tags ON observations USING GIN(tags);
|
|
63
95
|
|
|
64
96
|
ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedder TEXT;
|
|
65
97
|
ALTER TABLE observations ADD COLUMN IF NOT EXISTS author TEXT;
|
|
@@ -67,6 +99,33 @@ export class PostgresStore {
|
|
|
67
99
|
await this.ensureTagsIndexed();
|
|
68
100
|
if (this.vectorEnabled)
|
|
69
101
|
this.vectorDims = await this.readVectorDims();
|
|
102
|
+
await this.writeMeta();
|
|
103
|
+
}
|
|
104
|
+
async readMeta() {
|
|
105
|
+
try {
|
|
106
|
+
const res = await this.pool.query('SELECT version, vector_enabled, vector_dims FROM claude_db_meta WHERE id = 1');
|
|
107
|
+
const row = res.rows[0];
|
|
108
|
+
if (!row)
|
|
109
|
+
return null;
|
|
110
|
+
return {
|
|
111
|
+
version: Number(row['version']),
|
|
112
|
+
vectorEnabled: row['vector_enabled'] === true,
|
|
113
|
+
vectorDims: row['vector_dims'] == null ? null : Number(row['vector_dims']),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
// No table yet, or no permission to read it. Either way the caller does
|
|
118
|
+
// the full setup, which is what happened before this existed.
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
async writeMeta() {
|
|
123
|
+
await this.pool.query(`INSERT INTO claude_db_meta (id, version, vector_enabled, vector_dims)
|
|
124
|
+
VALUES (1, $1, $2, $3)
|
|
125
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
126
|
+
version = EXCLUDED.version,
|
|
127
|
+
vector_enabled = EXCLUDED.vector_enabled,
|
|
128
|
+
vector_dims = EXCLUDED.vector_dims`, [SCHEMA_VERSION, this.vectorEnabled, this.vectorDims]);
|
|
70
129
|
}
|
|
71
130
|
/**
|
|
72
131
|
* CREATE TABLE IF NOT EXISTS leaves an existing generated column alone, so a
|
|
@@ -111,7 +170,13 @@ export class PostgresStore {
|
|
|
111
170
|
await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
|
|
112
171
|
await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
|
|
113
172
|
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
114
|
-
|
|
173
|
+
// Read back rather than assumed: another process may have created the
|
|
174
|
+
// column at a different width between this one reading the version row
|
|
175
|
+
// and reaching here, and writing 384d values into a 256d column rolls
|
|
176
|
+
// the whole batch back. One extra round trip, once, and only before the
|
|
177
|
+
// first vector this process stores.
|
|
178
|
+
this.vectorDims = await this.readVectorDims();
|
|
179
|
+
await this.writeMeta();
|
|
115
180
|
}
|
|
116
181
|
if (this.vectorDims !== dims && !this.warnedDims) {
|
|
117
182
|
this.warnedDims = true;
|
|
@@ -290,6 +355,11 @@ export class PostgresStore {
|
|
|
290
355
|
lastActive: Number(row['last']),
|
|
291
356
|
}));
|
|
292
357
|
}
|
|
358
|
+
async inventory() {
|
|
359
|
+
const res = await this.pool.query(`SELECT tablename AS name FROM pg_tables
|
|
360
|
+
WHERE schemaname = ANY(current_schemas(false))`);
|
|
361
|
+
return foreignNames(res.rows.map((row) => String(row['name'] ?? '')));
|
|
362
|
+
}
|
|
293
363
|
async searchKeyword(query) {
|
|
294
364
|
const conditions = ['tsv @@ q.tsq'];
|
|
295
365
|
const values = [query.text];
|
|
@@ -304,7 +374,10 @@ export class PostgresStore {
|
|
|
304
374
|
const res = await this.pool.query(`WITH q AS (
|
|
305
375
|
SELECT replace(plainto_tsquery('english', $1)::text, '&', '|')::tsquery AS tsq
|
|
306
376
|
)
|
|
307
|
-
SELECT id, kind, title, project, created_at,
|
|
377
|
+
SELECT id, kind, title, project, created_at,
|
|
378
|
+
ts_headline('english', body, q.tsq,
|
|
379
|
+
'MaxFragments=1, MaxWords=18, MinWords=5') AS snippet,
|
|
380
|
+
ts_rank(tsv, q.tsq) AS score
|
|
308
381
|
FROM observations, q
|
|
309
382
|
WHERE ${conditions.join(' AND ')}
|
|
310
383
|
ORDER BY score DESC
|
|
@@ -356,6 +429,11 @@ export class PostgresStore {
|
|
|
356
429
|
values.push(query.kind);
|
|
357
430
|
conditions.push(`kind = $${values.length}`);
|
|
358
431
|
}
|
|
432
|
+
if (query.tag) {
|
|
433
|
+
// Containment rather than a text match, so `api` cannot match `api-docs`.
|
|
434
|
+
values.push(JSON.stringify([query.tag]));
|
|
435
|
+
conditions.push(`tags @> $${values.length}::jsonb`);
|
|
436
|
+
}
|
|
359
437
|
if (query.since !== undefined) {
|
|
360
438
|
values.push(query.since);
|
|
361
439
|
conditions.push(`created_at >= $${values.length}`);
|
|
@@ -422,7 +500,7 @@ function toObservation(row) {
|
|
|
422
500
|
return obs;
|
|
423
501
|
}
|
|
424
502
|
function toIndexEntry(row) {
|
|
425
|
-
|
|
503
|
+
const entry = {
|
|
426
504
|
id: row['id'],
|
|
427
505
|
kind: row['kind'],
|
|
428
506
|
title: row['title'],
|
|
@@ -430,4 +508,11 @@ function toIndexEntry(row) {
|
|
|
430
508
|
createdAt: Number(row['created_at']),
|
|
431
509
|
score: Number(row['score'] ?? 0),
|
|
432
510
|
};
|
|
511
|
+
// ts_headline wraps matches in <b> by default. Left at the default rather
|
|
512
|
+
// than configured away, because an empty StartSel is not portable across
|
|
513
|
+
// server versions; toSnippet strips them.
|
|
514
|
+
const snippet = toSnippet(row['snippet']);
|
|
515
|
+
if (snippet)
|
|
516
|
+
entry.snippet = snippet;
|
|
517
|
+
return entry;
|
|
433
518
|
}
|
|
@@ -25,6 +25,7 @@ export declare class SqliteStore implements MemoryStore {
|
|
|
25
25
|
remove(filter: RemoveFilter): Promise<number>;
|
|
26
26
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
27
27
|
listProjects(): Promise<ProjectSummary[]>;
|
|
28
|
+
inventory(): Promise<string[]>;
|
|
28
29
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
29
30
|
searchVector(vector: number[], query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
30
31
|
timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -2,9 +2,10 @@ import { DatabaseSync } from 'node:sqlite';
|
|
|
2
2
|
import { mkdirSync, readFileSync } from 'node:fs';
|
|
3
3
|
import { dirname, resolve } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { isWholeScope } from '../adapter.js';
|
|
5
|
+
import { foreignNames, isWholeScope } from '../adapter.js';
|
|
6
6
|
import { cosine, packVector, unpackVector } from '../../util/vector.js';
|
|
7
7
|
import { scopeToken } from '../../util/scope.js';
|
|
8
|
+
import { toSnippet } from '../../util/snippet.js';
|
|
8
9
|
import { partitionIds } from '../../util/shortid.js';
|
|
9
10
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
10
11
|
/**
|
|
@@ -184,6 +185,13 @@ export class SqliteStore {
|
|
|
184
185
|
lastActive: Number(row['last']),
|
|
185
186
|
}));
|
|
186
187
|
}
|
|
188
|
+
async inventory() {
|
|
189
|
+
const rows = this.db
|
|
190
|
+
.prepare(`SELECT name FROM sqlite_master
|
|
191
|
+
WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%'`)
|
|
192
|
+
.all();
|
|
193
|
+
return foreignNames(rows.map((row) => String(row['name'] ?? '')));
|
|
194
|
+
}
|
|
187
195
|
async searchKeyword(query) {
|
|
188
196
|
// The project constraint goes inside the MATCH expression, not the WHERE
|
|
189
197
|
// clause. Filtering after the match makes FTS5 rank every project's
|
|
@@ -198,6 +206,10 @@ export class SqliteStore {
|
|
|
198
206
|
conditions.push('o.kind = ?');
|
|
199
207
|
params.push(query.kind);
|
|
200
208
|
}
|
|
209
|
+
if (query.tag) {
|
|
210
|
+
conditions.push(TAG_PREDICATE('o.'));
|
|
211
|
+
params.push(query.tag);
|
|
212
|
+
}
|
|
201
213
|
if (query.since !== undefined) {
|
|
202
214
|
conditions.push('o.created_at >= ?');
|
|
203
215
|
params.push(query.since);
|
|
@@ -211,8 +223,13 @@ export class SqliteStore {
|
|
|
211
223
|
// weights (title, body, tags, scope) make a hit in the one-sentence claim
|
|
212
224
|
// outrank the same word buried in a long body; scope carries no weight
|
|
213
225
|
// because it is a filter, not a relevance signal.
|
|
226
|
+
// snippet() over column 1 (body) returns the matched fragment, which is
|
|
227
|
+
// what makes a result choosable without expanding it. Markers are empty
|
|
228
|
+
// because the fragment itself is the signal; single-quoted, since SQLite
|
|
229
|
+
// reads a double-quoted literal as an identifier.
|
|
214
230
|
const rows = this.db
|
|
215
231
|
.prepare(`SELECT o.id, o.kind, o.title, o.project, o.created_at,
|
|
232
|
+
snippet(observations_fts, 1, '', '', '…', 14) AS snippet,
|
|
216
233
|
-bm25(observations_fts, 10.0, 1.0, 5.0, 0.0) AS score
|
|
217
234
|
FROM observations_fts
|
|
218
235
|
JOIN observations o ON o.rowid = observations_fts.rowid
|
|
@@ -311,6 +328,11 @@ function removeWhere(filter) {
|
|
|
311
328
|
params,
|
|
312
329
|
};
|
|
313
330
|
}
|
|
331
|
+
/**
|
|
332
|
+
* Exact membership in the stored JSON array, rather than a LIKE over the raw
|
|
333
|
+
* text, so `api` cannot match `api-docs` and a tag containing `%` is harmless.
|
|
334
|
+
*/
|
|
335
|
+
const TAG_PREDICATE = (prefix) => `EXISTS (SELECT 1 FROM json_each(${prefix}tags) WHERE value = ?)`;
|
|
314
336
|
function appendScope(query, conditions, params, prefix) {
|
|
315
337
|
if (query.project) {
|
|
316
338
|
conditions.push(`${prefix}project = ?`);
|
|
@@ -320,6 +342,10 @@ function appendScope(query, conditions, params, prefix) {
|
|
|
320
342
|
conditions.push(`${prefix}kind = ?`);
|
|
321
343
|
params.push(query.kind);
|
|
322
344
|
}
|
|
345
|
+
if (query.tag) {
|
|
346
|
+
conditions.push(TAG_PREDICATE(prefix));
|
|
347
|
+
params.push(query.tag);
|
|
348
|
+
}
|
|
323
349
|
if (query.since !== undefined) {
|
|
324
350
|
conditions.push(`${prefix}created_at >= ?`);
|
|
325
351
|
params.push(query.since);
|
|
@@ -401,7 +427,7 @@ function toObservation(row) {
|
|
|
401
427
|
return obs;
|
|
402
428
|
}
|
|
403
429
|
function toIndexEntry(row) {
|
|
404
|
-
|
|
430
|
+
const entry = {
|
|
405
431
|
id: row['id'],
|
|
406
432
|
kind: row['kind'],
|
|
407
433
|
title: row['title'],
|
|
@@ -409,4 +435,8 @@ function toIndexEntry(row) {
|
|
|
409
435
|
createdAt: Number(row['created_at']),
|
|
410
436
|
score: Number(row['score'] ?? 0),
|
|
411
437
|
};
|
|
438
|
+
const snippet = toSnippet(row['snippet']);
|
|
439
|
+
if (snippet)
|
|
440
|
+
entry.snippet = snippet;
|
|
441
|
+
return entry;
|
|
412
442
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -40,11 +40,27 @@ export interface ObservationIndexEntry {
|
|
|
40
40
|
project: string;
|
|
41
41
|
createdAt: number;
|
|
42
42
|
score: number;
|
|
43
|
+
/**
|
|
44
|
+
* A line of the body around the match, so a result can be chosen without
|
|
45
|
+
* being expanded first. Produced by the keyword retriever, which is the only
|
|
46
|
+
* one with query terms to centre on; a row found by vector similarity alone
|
|
47
|
+
* has none.
|
|
48
|
+
*/
|
|
49
|
+
snippet?: string;
|
|
43
50
|
}
|
|
44
51
|
export interface SearchQuery {
|
|
45
52
|
text: string;
|
|
46
53
|
project?: string;
|
|
47
54
|
kind?: ObservationKind;
|
|
55
|
+
/**
|
|
56
|
+
* Narrows to observations touching one repository or top-level directory.
|
|
57
|
+
*
|
|
58
|
+
* A workspace that pools several repositories under one project key gets a
|
|
59
|
+
* memory covering all of them, and "the backend only" is a natural thing to
|
|
60
|
+
* ask that could not be expressed: tags were recorded and ranked, but never
|
|
61
|
+
* filtered on.
|
|
62
|
+
*/
|
|
63
|
+
tag?: string;
|
|
48
64
|
since?: number;
|
|
49
65
|
until?: number;
|
|
50
66
|
limit: number;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shaping for the two places a body is shown in part rather than in full.
|
|
3
|
+
*
|
|
4
|
+
* Layer 1 was built on the assumption that a title tells you which row holds
|
|
5
|
+
* the answer. It often does not — "Committed as 30daa92" is a true report of an
|
|
6
|
+
* outcome and says nothing about whether to expand it — so a search returned
|
|
7
|
+
* fifteen rows that all looked alike and the only way to choose was to guess.
|
|
8
|
+
* A snippet costs about sixty characters and saves a whole body per wrong
|
|
9
|
+
* guess, which is the trade the terse index was making in the wrong direction.
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* One line, whitespace collapsed, clipped at a word boundary.
|
|
13
|
+
*
|
|
14
|
+
* Bodies carry newlines, markdown tables and code, none of which survive being
|
|
15
|
+
* shown on one line of an index; and the search engines wrap matches in their
|
|
16
|
+
* own markers, which are noise once the fragment is already the point.
|
|
17
|
+
*/
|
|
18
|
+
export declare function toSnippet(text: unknown, max?: number): string | undefined;
|
|
19
|
+
/**
|
|
20
|
+
* Bounds a full body at layer 3.
|
|
21
|
+
*
|
|
22
|
+
* The cost argument that made the index terse was never applied one layer down,
|
|
23
|
+
* where a single call may ask for twenty-five bodies at once — so the multiplier
|
|
24
|
+
* is far worse there than in the index it was written for. The remainder is
|
|
25
|
+
* counted rather than dropped silently, because a reader who needs the rest has
|
|
26
|
+
* to know it exists.
|
|
27
|
+
*/
|
|
28
|
+
export declare function clipBody(body: string, max: number): string;
|