claude-db 0.12.3 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/dist/capture/cursors.d.ts +8 -0
- package/dist/capture/cursors.js +99 -0
- package/dist/capture/flush-lock.d.ts +11 -0
- package/dist/capture/flush-lock.js +108 -0
- package/dist/capture/flush.d.ts +4 -4
- package/dist/capture/flush.js +27 -60
- package/dist/capture/index.d.ts +3 -3
- package/dist/capture/index.js +3 -3
- package/dist/capture/progress.js +8 -1
- package/dist/capture/reingest.d.ts +1 -0
- package/dist/capture/reingest.js +4 -1
- package/dist/capture/scrub.d.ts +1 -0
- package/dist/capture/scrub.js +4 -1
- package/dist/capture/vectors.js +2 -1
- package/dist/catchup/gather.js +1 -0
- package/dist/cli/commands/daemon.d.ts +1 -0
- package/dist/cli/commands/daemon.js +29 -0
- package/dist/cli/commands/distill.js +2 -1
- package/dist/cli/commands/graph.js +2 -1
- package/dist/cli/commands/maintain.js +3 -1
- package/dist/cli/commands/sync.js +5 -4
- package/dist/cli/commands/transfer.js +4 -3
- package/dist/cli/index.js +4 -0
- package/dist/cli/usage.js +1 -0
- package/dist/config/schema.d.ts +4 -0
- package/dist/config/schema.js +6 -0
- package/dist/context.d.ts +1 -0
- package/dist/context.js +21 -6
- package/dist/daemon/client.d.ts +12 -0
- package/dist/daemon/client.js +88 -0
- package/dist/daemon/paths.d.ts +16 -0
- package/dist/daemon/paths.js +19 -0
- package/dist/daemon/server.d.ts +2 -0
- package/dist/daemon/server.js +225 -0
- package/dist/facts/budget.d.ts +1 -0
- package/dist/facts/budget.js +4 -0
- package/dist/facts/claude-memory.js +7 -2
- package/dist/facts/distill.js +6 -3
- package/dist/facts/handoff.js +6 -1
- package/dist/facts/known.js +3 -1
- package/dist/facts/model.d.ts +1 -1
- package/dist/facts/model.js +18 -7
- package/dist/facts/start.js +2 -2
- package/dist/graph/query/answer.js +1 -1
- package/dist/graph/query/path.js +69 -37
- package/dist/graph/query/refresh.js +5 -1
- package/dist/graph/query/suggest.js +1 -2
- package/dist/graph/scan/cache.d.ts +1 -0
- package/dist/graph/scan/cache.js +1 -1
- package/dist/graph/scan/persist.js +12 -16
- package/dist/hooks/compact.d.ts +2 -1
- package/dist/hooks/compact.js +9 -3
- package/dist/hooks/detached.d.ts +1 -0
- package/dist/hooks/detached.js +6 -0
- package/dist/hooks/graph-presence.d.ts +3 -0
- package/dist/hooks/graph-presence.js +42 -0
- package/dist/hooks/handlers/prefer-usages.d.ts +2 -0
- package/dist/hooks/handlers/prefer-usages.js +113 -0
- package/dist/hooks/handlers/session-end.d.ts +2 -0
- package/dist/hooks/handlers/session-end.js +62 -0
- package/dist/hooks/handlers/session-start.d.ts +2 -0
- package/dist/hooks/handlers/session-start.js +104 -0
- package/dist/hooks/handlers/turn-end.d.ts +2 -0
- package/dist/hooks/handlers/turn-end.js +31 -0
- package/dist/hooks/handlers/user-prompt.d.ts +2 -0
- package/dist/hooks/handlers/user-prompt.js +55 -0
- package/dist/hooks/local.d.ts +2 -0
- package/dist/hooks/local.js +23 -0
- package/dist/hooks/payload.d.ts +9 -4
- package/dist/hooks/payload.js +26 -17
- package/dist/hooks/prefer-usages.js +2 -118
- package/dist/hooks/route.d.ts +2 -0
- package/dist/hooks/route.js +28 -0
- package/dist/hooks/run.d.ts +14 -0
- package/dist/hooks/run.js +12 -0
- package/dist/hooks/save-turn.d.ts +2 -0
- package/dist/hooks/save-turn.js +11 -0
- package/dist/hooks/session-end.js +2 -63
- package/dist/hooks/session-start.js +2 -90
- package/dist/hooks/start-legacy.js +1 -1
- package/dist/hooks/turn-end.js +2 -25
- package/dist/hooks/turn-save.js +10 -15
- package/dist/hooks/user-prompt.js +2 -58
- package/dist/search/service.d.ts +1 -0
- package/dist/search/service.js +16 -6
- package/dist/store/adapter.d.ts +11 -7
- package/dist/store/each.d.ts +2 -4
- package/dist/store/each.js +1 -0
- package/dist/store/mongo/graph.d.ts +2 -1
- package/dist/store/mongo/graph.js +17 -2
- package/dist/store/mongo/index.d.ts +7 -8
- package/dist/store/mongo/index.js +13 -2
- package/dist/store/mongo/observations.d.ts +2 -1
- package/dist/store/mongo/observations.js +38 -4
- package/dist/store/postgres/driver.d.ts +10 -3
- package/dist/store/postgres/driver.js +12 -0
- package/dist/store/postgres/graph.d.ts +5 -6
- package/dist/store/postgres/graph.js +78 -57
- package/dist/store/postgres/index.d.ts +7 -8
- package/dist/store/postgres/index.js +16 -11
- package/dist/store/postgres/insert.js +98 -48
- package/dist/store/postgres/meta.d.ts +1 -0
- package/dist/store/postgres/meta.js +4 -0
- package/dist/store/postgres/observations.d.ts +3 -2
- package/dist/store/postgres/observations.js +40 -6
- package/dist/store/postgres/rows.js +1 -1
- package/dist/store/project-resolver.d.ts +3 -0
- package/dist/store/project-resolver.js +34 -23
- package/dist/store/scoped-store.js +5 -1
- package/dist/store/sqlite/graph.d.ts +5 -6
- package/dist/store/sqlite/graph.js +61 -35
- package/dist/store/sqlite/index.d.ts +7 -8
- package/dist/store/sqlite/index.js +11 -2
- package/dist/store/sqlite/observations.d.ts +2 -1
- package/dist/store/sqlite/observations.js +38 -7
- package/dist/types.d.ts +9 -0
- package/dist/util/job-lock.d.ts +1 -0
- package/dist/util/job-lock.js +50 -10
- package/docs/setup-guide.md +5 -0
- package/package.json +6 -8
|
@@ -25,40 +25,56 @@ async function upsertChunked(client, rows, width, sql, columns) {
|
|
|
25
25
|
await client.query(sql(placeholders(slice.length, width)), values);
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
|
-
|
|
28
|
+
async function writeGraphRows(client, scan) {
|
|
29
|
+
await upsertChunked(client, lastById(scan.symbols, (s) => s.id), 8, (values) => `INSERT INTO symbols (id, project, name, kind, file, line, lang, signature)
|
|
30
|
+
VALUES ${values}
|
|
31
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
32
|
+
name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
|
|
33
|
+
line = EXCLUDED.line, lang = EXCLUDED.lang,
|
|
34
|
+
signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
|
|
35
|
+
await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
|
|
36
|
+
relation, confidence, score, file, line)
|
|
37
|
+
VALUES ${values}
|
|
38
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
39
|
+
dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
|
|
40
|
+
relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
|
|
41
|
+
score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
|
|
42
|
+
e.id,
|
|
43
|
+
e.project,
|
|
44
|
+
e.srcId,
|
|
45
|
+
e.srcName,
|
|
46
|
+
e.dstId,
|
|
47
|
+
e.dstName,
|
|
48
|
+
e.relation,
|
|
49
|
+
e.confidence,
|
|
50
|
+
e.score,
|
|
51
|
+
e.file,
|
|
52
|
+
e.line,
|
|
53
|
+
]);
|
|
54
|
+
await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
|
|
55
|
+
VALUES ${values}
|
|
56
|
+
ON CONFLICT (project, path) DO UPDATE SET
|
|
57
|
+
hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
|
|
58
|
+
}
|
|
59
|
+
async function deleteGraphRows(client, project, files) {
|
|
60
|
+
if (files !== null && files.length === 0)
|
|
61
|
+
return 0;
|
|
62
|
+
const values = files === null ? [project] : [project, files];
|
|
63
|
+
const where = files === null ? 'project = $1' : 'project = $1 AND file = ANY($2::text[])';
|
|
64
|
+
const cacheWhere = files === null ? 'project = $1' : 'project = $1 AND path = ANY($2::text[])';
|
|
65
|
+
const res = await client.query(`WITH gone AS (DELETE FROM symbols WHERE ${where} RETURNING 1),
|
|
66
|
+
edges AS (DELETE FROM symbol_edges WHERE ${where}),
|
|
67
|
+
cached AS (DELETE FROM scanned_files WHERE ${cacheWhere})
|
|
68
|
+
SELECT COUNT(*)::int AS n FROM gone`, values);
|
|
69
|
+
return Number(res.rows[0]?.['n'] ?? 0);
|
|
70
|
+
}
|
|
71
|
+
async function inTransaction(pool, work) {
|
|
29
72
|
const client = await pool.connect();
|
|
30
73
|
try {
|
|
31
74
|
await client.query('BEGIN');
|
|
32
|
-
|
|
33
|
-
VALUES ${values}
|
|
34
|
-
ON CONFLICT (id) DO UPDATE SET
|
|
35
|
-
name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
|
|
36
|
-
line = EXCLUDED.line, lang = EXCLUDED.lang,
|
|
37
|
-
signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
|
|
38
|
-
await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
|
|
39
|
-
relation, confidence, score, file, line)
|
|
40
|
-
VALUES ${values}
|
|
41
|
-
ON CONFLICT (id) DO UPDATE SET
|
|
42
|
-
dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
|
|
43
|
-
relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
|
|
44
|
-
score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
|
|
45
|
-
e.id,
|
|
46
|
-
e.project,
|
|
47
|
-
e.srcId,
|
|
48
|
-
e.srcName,
|
|
49
|
-
e.dstId,
|
|
50
|
-
e.dstName,
|
|
51
|
-
e.relation,
|
|
52
|
-
e.confidence,
|
|
53
|
-
e.score,
|
|
54
|
-
e.file,
|
|
55
|
-
e.line,
|
|
56
|
-
]);
|
|
57
|
-
await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
|
|
58
|
-
VALUES ${values}
|
|
59
|
-
ON CONFLICT (project, path) DO UPDATE SET
|
|
60
|
-
hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
|
|
75
|
+
const result = await work(client);
|
|
61
76
|
await client.query('COMMIT');
|
|
77
|
+
return result;
|
|
62
78
|
}
|
|
63
79
|
catch (error) {
|
|
64
80
|
await client.query('ROLLBACK');
|
|
@@ -68,6 +84,17 @@ export async function upsertGraph(pool, scan) {
|
|
|
68
84
|
client.release();
|
|
69
85
|
}
|
|
70
86
|
}
|
|
87
|
+
export async function upsertGraph(pool, scan) {
|
|
88
|
+
await inTransaction(pool, (client) => writeGraphRows(client, scan));
|
|
89
|
+
}
|
|
90
|
+
export async function replaceGraph(pool, project, files, scan) {
|
|
91
|
+
await inTransaction(pool, async (client) => {
|
|
92
|
+
await deleteGraphRows(client, project, files);
|
|
93
|
+
await writeGraphRows(client, scan);
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
const SYMBOL_COLUMNS = 'id, name, kind, file, line, lang, signature';
|
|
97
|
+
const EDGE_COLUMNS = 'id, src_id, src_name, dst_id, dst_name, relation, confidence, score, file, line';
|
|
71
98
|
export async function findSymbols(pool, filter) {
|
|
72
99
|
const values = [filter.project];
|
|
73
100
|
const conditions = ['project = $1'];
|
|
@@ -80,9 +107,13 @@ export async function findSymbols(pool, filter) {
|
|
|
80
107
|
conditions.push(`file = $${values.length}`);
|
|
81
108
|
}
|
|
82
109
|
values.push(filter.limit ?? 500);
|
|
83
|
-
const res = await pool.query(`SELECT
|
|
110
|
+
const res = await pool.query(`SELECT ${SYMBOL_COLUMNS} FROM symbols WHERE ${conditions.join(' AND ')}
|
|
84
111
|
ORDER BY file, line LIMIT $${values.length}`, values);
|
|
85
|
-
return res.rows.map(toSymbol);
|
|
112
|
+
return res.rows.map((row) => toSymbol({ ...row, project: filter.project }));
|
|
113
|
+
}
|
|
114
|
+
export async function symbolNames(pool, project, limit) {
|
|
115
|
+
const res = await pool.query('SELECT name FROM symbols WHERE project = $1 ORDER BY file, line LIMIT $2', [project, limit]);
|
|
116
|
+
return res.rows.map((row) => row['name']);
|
|
86
117
|
}
|
|
87
118
|
export async function findEdges(pool, filter) {
|
|
88
119
|
const values = [filter.project];
|
|
@@ -96,20 +127,29 @@ export async function findEdges(pool, filter) {
|
|
|
96
127
|
values.push(ids);
|
|
97
128
|
sides.push(`${column} = ANY($${values.length}::text[])`);
|
|
98
129
|
}
|
|
130
|
+
for (const [side, names] of [
|
|
131
|
+
['src', filter.unresolvedSrcNames],
|
|
132
|
+
['dst', filter.unresolvedDstNames],
|
|
133
|
+
]) {
|
|
134
|
+
if (!names || names.length === 0)
|
|
135
|
+
continue;
|
|
136
|
+
values.push(names);
|
|
137
|
+
sides.push(`(${side}_id = '' AND ${side}_name = ANY($${values.length}::text[]))`);
|
|
138
|
+
}
|
|
99
139
|
let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
|
|
100
140
|
if (filter.dstName !== undefined) {
|
|
101
141
|
values.push(filter.dstName, `%.${filter.dstName}`);
|
|
102
142
|
where += ` AND (dst_name = $${values.length - 1} OR dst_name LIKE $${values.length})`;
|
|
103
143
|
}
|
|
104
144
|
values.push(filter.limit ?? 2000);
|
|
105
|
-
const res = await pool.query(`SELECT
|
|
145
|
+
const res = await pool.query(`SELECT ${EDGE_COLUMNS} FROM symbol_edges WHERE project = $1${where}
|
|
106
146
|
ORDER BY file, line, id LIMIT $${values.length}`, values);
|
|
107
|
-
return res.rows.map(toEdge);
|
|
147
|
+
return res.rows.map((row) => toEdge({ ...row, project: filter.project }));
|
|
108
148
|
}
|
|
109
|
-
export async function scannedFiles(pool, project) {
|
|
110
|
-
const res = await pool.query(
|
|
149
|
+
export async function scannedFiles(pool, project, limit) {
|
|
150
|
+
const res = await pool.query(`SELECT path, hash, scanned_at FROM scanned_files WHERE project = $1${limit === undefined ? '' : ' LIMIT $2'}`, limit === undefined ? [project] : [project, limit]);
|
|
111
151
|
return res.rows.map((row) => ({
|
|
112
|
-
project
|
|
152
|
+
project,
|
|
113
153
|
path: row['path'],
|
|
114
154
|
hash: row['hash'],
|
|
115
155
|
scannedAt: Number(row['scanned_at']),
|
|
@@ -117,24 +157,5 @@ export async function scannedFiles(pool, project) {
|
|
|
117
157
|
}
|
|
118
158
|
export async function removeGraph(pool, project, files) {
|
|
119
159
|
const scoped = files !== undefined && files.length > 0;
|
|
120
|
-
|
|
121
|
-
const where = scoped ? 'project = $1 AND file = ANY($2::text[])' : 'project = $1';
|
|
122
|
-
const cacheWhere = scoped ? 'project = $1 AND path = ANY($2::text[])' : 'project = $1';
|
|
123
|
-
const client = await pool.connect();
|
|
124
|
-
try {
|
|
125
|
-
await client.query('BEGIN');
|
|
126
|
-
const counted = await client.query(`SELECT COUNT(*)::int AS n FROM symbols WHERE ${where}`, values);
|
|
127
|
-
await client.query(`DELETE FROM symbols WHERE ${where}`, values);
|
|
128
|
-
await client.query(`DELETE FROM symbol_edges WHERE ${where}`, values);
|
|
129
|
-
await client.query(`DELETE FROM scanned_files WHERE ${cacheWhere}`, values);
|
|
130
|
-
await client.query('COMMIT');
|
|
131
|
-
return Number(counted.rows[0]?.['n'] ?? 0);
|
|
132
|
-
}
|
|
133
|
-
catch (error) {
|
|
134
|
-
await client.query('ROLLBACK');
|
|
135
|
-
throw error;
|
|
136
|
-
}
|
|
137
|
-
finally {
|
|
138
|
-
client.release();
|
|
139
|
-
}
|
|
160
|
+
return deleteGraphRows(pool, project, scoped ? files : null);
|
|
140
161
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
2
|
-
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
1
|
+
import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
2
|
+
import type { GraphRows, MemoryStore, ProjectSummary } from '../adapter.js';
|
|
3
3
|
export declare class PostgresStore implements MemoryStore {
|
|
4
4
|
private readonly pool;
|
|
5
5
|
readonly kind = "postgres";
|
|
@@ -33,6 +33,7 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
33
33
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
34
34
|
remove(filter: RemoveFilter): Promise<number>;
|
|
35
35
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
36
|
+
count(filter: CountFilter): Promise<number>;
|
|
36
37
|
listProjects(): Promise<ProjectSummary[]>;
|
|
37
38
|
inventory(): Promise<string[]>;
|
|
38
39
|
searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -41,13 +42,11 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
41
42
|
markReplaced(ids: string[]): Promise<number>;
|
|
42
43
|
timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
43
44
|
tryEnableVector(): Promise<boolean>;
|
|
44
|
-
upsertGraph(scan:
|
|
45
|
-
|
|
46
|
-
edges: CodeEdge[];
|
|
47
|
-
files: ScannedFile[];
|
|
48
|
-
}): Promise<void>;
|
|
45
|
+
upsertGraph(scan: GraphRows): Promise<void>;
|
|
46
|
+
replaceGraph(project: string, files: string[] | null, scan: GraphRows): Promise<void>;
|
|
49
47
|
findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
48
|
+
symbolNames(project: string, limit: number): Promise<string[]>;
|
|
50
49
|
findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
51
|
-
scannedFiles(project: string): Promise<ScannedFile[]>;
|
|
50
|
+
scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
|
|
52
51
|
removeGraph(project: string, files?: string[]): Promise<number>;
|
|
53
52
|
}
|
|
@@ -6,7 +6,7 @@ import * as observationsOps from './observations.js';
|
|
|
6
6
|
import * as sessionsOps from './sessions.js';
|
|
7
7
|
import * as graphOps from './graph.js';
|
|
8
8
|
import * as projectLinksOps from './project-links.js';
|
|
9
|
-
import {
|
|
9
|
+
import { openPool } from './driver.js';
|
|
10
10
|
import { DDL, SCHEMA_VERSION } from './schema.js';
|
|
11
11
|
export class PostgresStore {
|
|
12
12
|
pool;
|
|
@@ -18,9 +18,7 @@ export class PostgresStore {
|
|
|
18
18
|
this.pool = pool;
|
|
19
19
|
}
|
|
20
20
|
static async create(uri) {
|
|
21
|
-
|
|
22
|
-
const pool = new pg.Pool({ connectionString: uri });
|
|
23
|
-
return new PostgresStore(pool);
|
|
21
|
+
return new PostgresStore(await openPool(uri));
|
|
24
22
|
}
|
|
25
23
|
async init() {
|
|
26
24
|
const meta = await this.readMeta();
|
|
@@ -58,8 +56,7 @@ export class PostgresStore {
|
|
|
58
56
|
await this.pool.query('ALTER TABLE observations DROP COLUMN IF EXISTS embedding');
|
|
59
57
|
await this.pool.query(`ALTER TABLE observations ADD COLUMN embedding vector(${dims})`);
|
|
60
58
|
await this.pool.query('UPDATE observations SET embedder = NULL');
|
|
61
|
-
await this.pool
|
|
62
|
-
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
59
|
+
await metaOps.indexEmbeddings(this.pool);
|
|
63
60
|
this.vectorDims = await this.readVectorDims();
|
|
64
61
|
this.warnedDims = false;
|
|
65
62
|
await this.writeMeta();
|
|
@@ -75,8 +72,7 @@ export class PostgresStore {
|
|
|
75
72
|
return false;
|
|
76
73
|
if (this.vectorDims === null) {
|
|
77
74
|
await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
|
|
78
|
-
await this.pool
|
|
79
|
-
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
75
|
+
await metaOps.indexEmbeddings(this.pool);
|
|
80
76
|
this.vectorDims = await this.readVectorDims();
|
|
81
77
|
await this.writeMeta();
|
|
82
78
|
}
|
|
@@ -125,7 +121,10 @@ export class PostgresStore {
|
|
|
125
121
|
return observationsOps.remove(this.pool, filter);
|
|
126
122
|
}
|
|
127
123
|
async list(filter) {
|
|
128
|
-
return observationsOps.list(this.pool, filter);
|
|
124
|
+
return observationsOps.list(this.pool, filter, this.vectorDims !== null);
|
|
125
|
+
}
|
|
126
|
+
async count(filter) {
|
|
127
|
+
return observationsOps.count(this.pool, filter);
|
|
129
128
|
}
|
|
130
129
|
async listProjects() {
|
|
131
130
|
return observationsOps.listProjects(this.pool);
|
|
@@ -154,14 +153,20 @@ export class PostgresStore {
|
|
|
154
153
|
async upsertGraph(scan) {
|
|
155
154
|
return graphOps.upsertGraph(this.pool, scan);
|
|
156
155
|
}
|
|
156
|
+
async replaceGraph(project, files, scan) {
|
|
157
|
+
return graphOps.replaceGraph(this.pool, project, files, scan);
|
|
158
|
+
}
|
|
157
159
|
async findSymbols(filter) {
|
|
158
160
|
return graphOps.findSymbols(this.pool, filter);
|
|
159
161
|
}
|
|
162
|
+
async symbolNames(project, limit) {
|
|
163
|
+
return graphOps.symbolNames(this.pool, project, limit);
|
|
164
|
+
}
|
|
160
165
|
async findEdges(filter) {
|
|
161
166
|
return graphOps.findEdges(this.pool, filter);
|
|
162
167
|
}
|
|
163
|
-
async scannedFiles(project) {
|
|
164
|
-
return graphOps.scannedFiles(this.pool, project);
|
|
168
|
+
async scannedFiles(project, limit) {
|
|
169
|
+
return graphOps.scannedFiles(this.pool, project, limit);
|
|
165
170
|
}
|
|
166
171
|
async removeGraph(project, files) {
|
|
167
172
|
return graphOps.removeGraph(this.pool, project, files);
|
|
@@ -1,54 +1,91 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
const COLUMNS = [
|
|
2
|
+
'id',
|
|
3
|
+
'session_id',
|
|
4
|
+
'project',
|
|
5
|
+
'kind',
|
|
6
|
+
'title',
|
|
7
|
+
'body',
|
|
8
|
+
'files',
|
|
9
|
+
'tags',
|
|
10
|
+
'created_at',
|
|
11
|
+
'embedder',
|
|
12
|
+
'author',
|
|
13
|
+
'status',
|
|
14
|
+
];
|
|
15
|
+
const ROWS_PER_STATEMENT = 400;
|
|
16
|
+
const KEEP_CLOSED = `status = CASE
|
|
17
|
+
WHEN observations.status = 'done' AND EXCLUDED.status = 'open'
|
|
18
|
+
AND observations.files = EXCLUDED.files THEN observations.status
|
|
19
|
+
ELSE EXCLUDED.status END`;
|
|
20
|
+
function rowValues(obs, shape) {
|
|
21
|
+
const values = [
|
|
22
|
+
obs.id,
|
|
23
|
+
obs.sessionId,
|
|
24
|
+
obs.project,
|
|
25
|
+
obs.kind,
|
|
26
|
+
obs.title,
|
|
27
|
+
obs.body,
|
|
28
|
+
JSON.stringify(obs.files),
|
|
29
|
+
JSON.stringify(obs.tags),
|
|
30
|
+
obs.createdAt,
|
|
31
|
+
shape.vectorStored ? (obs.embedder ?? null) : null,
|
|
32
|
+
obs.author ?? null,
|
|
33
|
+
obs.status ?? 'done',
|
|
34
|
+
];
|
|
35
|
+
if (shape.vectorStored && obs.embedding)
|
|
36
|
+
values.push(`[${obs.embedding.join(',')}]`);
|
|
37
|
+
return values;
|
|
38
|
+
}
|
|
39
|
+
function statement(rows, shape, clearVector) {
|
|
40
|
+
const columns = shape.vectorStored ? [...COLUMNS, 'embedding'] : COLUMNS;
|
|
41
|
+
const values = [];
|
|
42
|
+
const tuples = rows.map((obs) => {
|
|
43
|
+
const start = values.length;
|
|
44
|
+
values.push(...rowValues(obs, shape));
|
|
45
|
+
return `(${columns.map((_, i) => `$${start + i + 1}`).join(', ')})`;
|
|
46
|
+
});
|
|
47
|
+
const updates = columns
|
|
48
|
+
.slice(1)
|
|
49
|
+
.filter((column) => !(shape.vectorKept && column === 'embedder'))
|
|
50
|
+
.map((column) => (column === 'status' ? KEEP_CLOSED : `${column} = EXCLUDED.${column}`))
|
|
51
|
+
.concat(clearVector ? ['embedding = NULL'] : [])
|
|
52
|
+
.join(', ');
|
|
53
|
+
return {
|
|
54
|
+
sql: `INSERT INTO observations (${columns.join(', ')}) VALUES ${tuples.join(', ')}
|
|
55
|
+
ON CONFLICT (id) DO UPDATE SET ${updates}`,
|
|
56
|
+
values,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
function lastPerId(observations) {
|
|
60
|
+
return [...new Map(observations.map((obs) => [obs.id, obs])).values()];
|
|
61
|
+
}
|
|
62
|
+
function insertStatements(observations, width, vectors, hasVectorColumn) {
|
|
63
|
+
const groups = new Map();
|
|
64
|
+
for (const obs of lastPerId(observations)) {
|
|
65
|
+
const shape = {
|
|
66
|
+
vectorStored: vectors && obs.embedding?.length === width,
|
|
67
|
+
vectorKept: !obs.embedding?.length && obs.embedder != null,
|
|
68
|
+
};
|
|
69
|
+
const id = `${shape.vectorStored}:${shape.vectorKept}`;
|
|
70
|
+
const group = groups.get(id) ?? { shape, rows: [] };
|
|
71
|
+
group.rows.push(obs);
|
|
72
|
+
groups.set(id, group);
|
|
73
|
+
}
|
|
74
|
+
const statements = [];
|
|
75
|
+
for (const { shape, rows } of groups.values()) {
|
|
76
|
+
const clearVector = !shape.vectorStored && !shape.vectorKept && (hasVectorColumn || vectors);
|
|
77
|
+
for (let i = 0; i < rows.length; i += ROWS_PER_STATEMENT) {
|
|
78
|
+
statements.push(statement(rows.slice(i, i + ROWS_PER_STATEMENT), shape, clearVector));
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return statements;
|
|
82
|
+
}
|
|
83
|
+
async function inTransaction(pool, statements) {
|
|
6
84
|
const client = await pool.connect();
|
|
7
85
|
try {
|
|
8
86
|
await client.query('BEGIN');
|
|
9
|
-
for (const
|
|
10
|
-
|
|
11
|
-
const columns = [
|
|
12
|
-
'id',
|
|
13
|
-
'session_id',
|
|
14
|
-
'project',
|
|
15
|
-
'kind',
|
|
16
|
-
'title',
|
|
17
|
-
'body',
|
|
18
|
-
'files',
|
|
19
|
-
'tags',
|
|
20
|
-
'created_at',
|
|
21
|
-
'embedder',
|
|
22
|
-
'author',
|
|
23
|
-
'status',
|
|
24
|
-
];
|
|
25
|
-
const values = [
|
|
26
|
-
obs.id,
|
|
27
|
-
obs.sessionId,
|
|
28
|
-
obs.project,
|
|
29
|
-
obs.kind,
|
|
30
|
-
obs.title,
|
|
31
|
-
obs.body,
|
|
32
|
-
JSON.stringify(obs.files),
|
|
33
|
-
JSON.stringify(obs.tags),
|
|
34
|
-
obs.createdAt,
|
|
35
|
-
vectorStored ? (obs.embedder ?? null) : null,
|
|
36
|
-
obs.author ?? null,
|
|
37
|
-
obs.status ?? 'done',
|
|
38
|
-
];
|
|
39
|
-
if (vectorStored && obs.embedding) {
|
|
40
|
-
columns.push('embedding');
|
|
41
|
-
values.push(`[${obs.embedding.join(',')}]`);
|
|
42
|
-
}
|
|
43
|
-
const holes = values.map((_, i) => `$${i + 1}`).join(', ');
|
|
44
|
-
const updates = columns
|
|
45
|
-
.slice(1)
|
|
46
|
-
.map((column) => `${column} = EXCLUDED.${column}`)
|
|
47
|
-
.concat(!vectorStored && (hasVectorColumn || vectors) ? ['embedding = NULL'] : [])
|
|
48
|
-
.join(', ');
|
|
49
|
-
await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
|
|
50
|
-
ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
|
|
51
|
-
}
|
|
87
|
+
for (const { sql, values } of statements)
|
|
88
|
+
await client.query(sql, values);
|
|
52
89
|
await client.query('COMMIT');
|
|
53
90
|
}
|
|
54
91
|
catch (error) {
|
|
@@ -59,3 +96,16 @@ export async function insertObservations(pool, ensureVectorColumn, observations,
|
|
|
59
96
|
client.release();
|
|
60
97
|
}
|
|
61
98
|
}
|
|
99
|
+
export async function insertObservations(pool, ensureVectorColumn, observations, hasVectorColumn = false) {
|
|
100
|
+
if (observations.length === 0)
|
|
101
|
+
return;
|
|
102
|
+
const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
|
|
103
|
+
const vectors = width !== undefined && (await ensureVectorColumn(width));
|
|
104
|
+
const statements = insertStatements(observations, width, vectors, hasVectorColumn);
|
|
105
|
+
const [only] = statements;
|
|
106
|
+
if (statements.length === 1 && only) {
|
|
107
|
+
await pool.query(only.sql, only.values);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
await inTransaction(pool, statements);
|
|
111
|
+
}
|
|
@@ -8,3 +8,4 @@ export declare function ensureTagsIndexed(pool: Pool): Promise<void>;
|
|
|
8
8
|
export declare function readVectorDims(pool: Pool): Promise<number | null>;
|
|
9
9
|
export declare function tryEnableVector(pool: Pool): Promise<boolean>;
|
|
10
10
|
export declare function writeMeta(pool: Pool, vectorEnabled: boolean, vectorDims: number | null): Promise<void>;
|
|
11
|
+
export declare function indexEmbeddings(pool: Pool): Promise<void>;
|
|
@@ -65,3 +65,7 @@ export async function writeMeta(pool, vectorEnabled, vectorDims) {
|
|
|
65
65
|
vector_enabled = EXCLUDED.vector_enabled,
|
|
66
66
|
vector_dims = EXCLUDED.vector_dims`, [SCHEMA_VERSION, vectorEnabled, vectorDims]);
|
|
67
67
|
}
|
|
68
|
+
export async function indexEmbeddings(pool) {
|
|
69
|
+
await pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
|
|
70
|
+
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
71
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import type { Pool } from './driver.js';
|
|
2
|
-
import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, TimelineQuery } from '../../types.js';
|
|
2
|
+
import type { CountFilter, ListFilter, Observation, ObservationIndexEntry, RemoveFilter, TimelineQuery } from '../../types.js';
|
|
3
3
|
import type { ProjectSummary } from '../adapter.js';
|
|
4
4
|
export declare function getObservations(pool: Pool, ids: string[]): Promise<Observation[]>;
|
|
5
5
|
export declare function remove(pool: Pool, filter: RemoveFilter): Promise<number>;
|
|
6
|
-
export declare function list(pool: Pool, filter: ListFilter): Promise<Observation[]>;
|
|
6
|
+
export declare function list(pool: Pool, filter: ListFilter, hasVectorColumn?: boolean): Promise<Observation[]>;
|
|
7
|
+
export declare function count(pool: Pool, filter: CountFilter): Promise<number>;
|
|
7
8
|
export declare function listProjects(pool: Pool): Promise<ProjectSummary[]>;
|
|
8
9
|
export declare function inventory(pool: Pool): Promise<string[]>;
|
|
9
10
|
export declare function timeline(pool: Pool, query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
@@ -3,6 +3,13 @@ import { partitionIds } from '../../util/shortid.js';
|
|
|
3
3
|
import { excludeSessionsClause, projectClause } from './filters.js';
|
|
4
4
|
import { toIndexEntry, toObservation } from './rows.js';
|
|
5
5
|
import { noProjects } from '../project-scope.js';
|
|
6
|
+
const COLUMNS = 'id, session_id, project, kind, title, body, files, tags, created_at, embedder, author, status';
|
|
7
|
+
const WITHOUT_BODY = COLUMNS.replace(' body,', '');
|
|
8
|
+
function listColumns(filter) {
|
|
9
|
+
if (filter.vectors)
|
|
10
|
+
return '*';
|
|
11
|
+
return filter.bodies === false ? WITHOUT_BODY : COLUMNS;
|
|
12
|
+
}
|
|
6
13
|
export async function getObservations(pool, ids) {
|
|
7
14
|
if (ids.length === 0)
|
|
8
15
|
return [];
|
|
@@ -17,7 +24,7 @@ export async function getObservations(pool, ids) {
|
|
|
17
24
|
values.push(`${prefix.replace(/[%_]/g, '')}%`);
|
|
18
25
|
clauses.push(`id LIKE $${values.length}`);
|
|
19
26
|
}
|
|
20
|
-
const res = await pool.query(`SELECT
|
|
27
|
+
const res = await pool.query(`SELECT ${COLUMNS} FROM observations WHERE ${clauses.join(' OR ')}`, values);
|
|
21
28
|
return res.rows.map(toObservation);
|
|
22
29
|
}
|
|
23
30
|
export async function remove(pool, filter) {
|
|
@@ -76,11 +83,8 @@ export async function remove(pool, filter) {
|
|
|
76
83
|
client.release();
|
|
77
84
|
}
|
|
78
85
|
}
|
|
79
|
-
|
|
80
|
-
if (noProjects(filter.project))
|
|
81
|
-
return [];
|
|
86
|
+
function scopeConditions(filter, values) {
|
|
82
87
|
const conditions = [];
|
|
83
|
-
const values = [];
|
|
84
88
|
const project = projectClause(filter.project, values);
|
|
85
89
|
if (project)
|
|
86
90
|
conditions.push(project);
|
|
@@ -99,6 +103,17 @@ export async function list(pool, filter) {
|
|
|
99
103
|
values.push(filter.status);
|
|
100
104
|
conditions.push(`status = $${values.length}`);
|
|
101
105
|
}
|
|
106
|
+
if (filter.tag) {
|
|
107
|
+
values.push(JSON.stringify([filter.tag]));
|
|
108
|
+
conditions.push(`tags @> $${values.length}::jsonb`);
|
|
109
|
+
}
|
|
110
|
+
return conditions;
|
|
111
|
+
}
|
|
112
|
+
export async function list(pool, filter, hasVectorColumn = false) {
|
|
113
|
+
if (noProjects(filter.project))
|
|
114
|
+
return [];
|
|
115
|
+
const values = [];
|
|
116
|
+
const conditions = scopeConditions(filter, values);
|
|
102
117
|
if (filter.after !== undefined) {
|
|
103
118
|
if (filter.afterId === undefined) {
|
|
104
119
|
values.push(filter.after);
|
|
@@ -109,13 +124,32 @@ export async function list(pool, filter) {
|
|
|
109
124
|
conditions.push(`(created_at > $${values.length - 2} OR (created_at = $${values.length - 1} AND id > $${values.length}))`);
|
|
110
125
|
}
|
|
111
126
|
}
|
|
127
|
+
if (filter.vectorStale) {
|
|
128
|
+
const stale = hasVectorColumn
|
|
129
|
+
? ['embedder IS NULL', 'embedding IS NULL']
|
|
130
|
+
: ['embedder IS NULL'];
|
|
131
|
+
if (filter.vectorStale.embedder !== undefined) {
|
|
132
|
+
values.push(filter.vectorStale.embedder);
|
|
133
|
+
stale.push(`embedder <> $${values.length}`);
|
|
134
|
+
}
|
|
135
|
+
conditions.push(`(${stale.join(' OR ')})`);
|
|
136
|
+
}
|
|
112
137
|
values.push(filter.limit ?? 1000);
|
|
113
138
|
const direction = filter.newest ? 'DESC' : 'ASC';
|
|
114
|
-
const res = await pool.query(`SELECT
|
|
139
|
+
const res = await pool.query(`SELECT ${listColumns(filter)} FROM observations
|
|
115
140
|
${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
|
|
116
141
|
ORDER BY created_at ${direction}, id ${direction} LIMIT $${values.length}`, values);
|
|
117
142
|
return res.rows.map(toObservation);
|
|
118
143
|
}
|
|
144
|
+
export async function count(pool, filter) {
|
|
145
|
+
if (noProjects(filter.project))
|
|
146
|
+
return 0;
|
|
147
|
+
const values = [];
|
|
148
|
+
const conditions = scopeConditions(filter, values);
|
|
149
|
+
const res = await pool.query(`SELECT COUNT(*)::int AS n FROM observations
|
|
150
|
+
${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}`, values);
|
|
151
|
+
return Number(res.rows[0]?.['n'] ?? 0);
|
|
152
|
+
}
|
|
119
153
|
export async function listProjects(pool) {
|
|
120
154
|
const res = await pool.query(`SELECT project, COUNT(*)::int AS n, MAX(created_at) AS last
|
|
121
155
|
FROM observations GROUP BY project ORDER BY last DESC`);
|
|
@@ -4,12 +4,15 @@ export interface ScopeResolverOptions {
|
|
|
4
4
|
remote: string;
|
|
5
5
|
isFolder?: (value: string) => boolean;
|
|
6
6
|
keyOf?: (folder: string, remote: string) => string;
|
|
7
|
+
scopeTtlMs?: number;
|
|
8
|
+
now?: () => number;
|
|
7
9
|
}
|
|
8
10
|
export interface ProjectScope {
|
|
9
11
|
key: string;
|
|
10
12
|
names: string[];
|
|
11
13
|
}
|
|
12
14
|
export interface ScopeResolver {
|
|
15
|
+
key(value: string): Promise<string>;
|
|
13
16
|
resolve(value: string): Promise<ProjectScope>;
|
|
14
17
|
}
|
|
15
18
|
export declare function isExistingFolder(value: string): boolean;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { statSync } from 'node:fs';
|
|
2
2
|
import { isAbsolute } from 'node:path';
|
|
3
3
|
import { projectKey } from '../util/project-key.js';
|
|
4
|
-
import { orderScope } from './project-scope.js';
|
|
4
|
+
import { isLinkable, orderScope } from './project-scope.js';
|
|
5
|
+
const SCOPE_TTL_MS = 60_000;
|
|
5
6
|
export function isExistingFolder(value) {
|
|
6
7
|
if (!isAbsolute(value))
|
|
7
8
|
return false;
|
|
@@ -13,37 +14,47 @@ export function isExistingFolder(value) {
|
|
|
13
14
|
}
|
|
14
15
|
}
|
|
15
16
|
export function createScopeResolver(options) {
|
|
16
|
-
const { store, remote, isFolder = isExistingFolder, keyOf = (folder, name) => projectKey(folder, { remote: name }), } = options;
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
const
|
|
17
|
+
const { store, remote, isFolder = isExistingFolder, keyOf = (folder, name) => projectKey(folder, { remote: name }), scopeTtlMs = SCOPE_TTL_MS, now = Date.now, } = options;
|
|
18
|
+
const scopes = new Map();
|
|
19
|
+
function forgetOthers(folder) {
|
|
20
|
+
for (const cached of [...scopes.keys()])
|
|
21
|
+
if (cached !== folder)
|
|
22
|
+
scopes.delete(cached);
|
|
23
|
+
}
|
|
24
|
+
async function linked(folder) {
|
|
20
25
|
try {
|
|
21
|
-
await store.
|
|
26
|
+
return await store.projectScope(folder);
|
|
22
27
|
}
|
|
23
28
|
catch {
|
|
24
|
-
return
|
|
29
|
+
return null;
|
|
25
30
|
}
|
|
26
|
-
return key;
|
|
27
31
|
}
|
|
28
|
-
async function
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
async function lookUp(folder) {
|
|
33
|
+
const key = keyOf(folder, remote);
|
|
34
|
+
let related = await linked(folder);
|
|
35
|
+
if (isLinkable(folder, key) && !related?.includes(key)) {
|
|
36
|
+
const before = related;
|
|
37
|
+
related = await store.linkProject(folder, key).then(() => {
|
|
38
|
+
forgetOthers(folder);
|
|
39
|
+
return linked(folder);
|
|
40
|
+
}, () => before);
|
|
34
41
|
}
|
|
42
|
+
return { key, names: orderScope(folder, [...(related ?? []), key]) };
|
|
43
|
+
}
|
|
44
|
+
function scopeOf(folder) {
|
|
45
|
+
const cached = scopes.get(folder);
|
|
46
|
+
if (cached && now() - cached.at < scopeTtlMs)
|
|
47
|
+
return cached.scope;
|
|
48
|
+
const scope = lookUp(folder);
|
|
49
|
+
scopes.set(folder, { at: now(), scope });
|
|
50
|
+
return scope;
|
|
35
51
|
}
|
|
36
52
|
return {
|
|
53
|
+
async key(value) {
|
|
54
|
+
return isFolder(value) ? (await scopeOf(value)).key : value;
|
|
55
|
+
},
|
|
37
56
|
async resolve(value) {
|
|
38
|
-
|
|
39
|
-
return { key: value, names: [value] };
|
|
40
|
-
let pending = keys.get(value);
|
|
41
|
-
if (!pending) {
|
|
42
|
-
pending = linkedKey(value);
|
|
43
|
-
keys.set(value, pending);
|
|
44
|
-
}
|
|
45
|
-
const key = await pending;
|
|
46
|
-
return { key, names: await namesFor(value, key) };
|
|
57
|
+
return isFolder(value) ? scopeOf(value) : { key: value, names: [value] };
|
|
47
58
|
},
|
|
48
59
|
};
|
|
49
60
|
}
|