claude-db 0.12.4 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/capture/cursors.d.ts +8 -0
- package/dist/capture/cursors.js +99 -0
- package/dist/capture/flush-lock.d.ts +11 -0
- package/dist/capture/flush-lock.js +108 -0
- package/dist/capture/flush.d.ts +4 -4
- package/dist/capture/flush.js +19 -80
- package/dist/capture/index.d.ts +3 -3
- package/dist/capture/index.js +3 -3
- package/dist/capture/reingest.d.ts +1 -0
- package/dist/capture/reingest.js +4 -1
- package/dist/capture/scrub.d.ts +1 -0
- package/dist/capture/scrub.js +4 -1
- package/dist/capture/vectors.js +2 -1
- package/dist/cli/commands/daemon.d.ts +1 -0
- package/dist/cli/commands/daemon.js +29 -0
- package/dist/cli/commands/distill.js +2 -1
- package/dist/cli/commands/graph.js +2 -1
- package/dist/cli/commands/maintain.js +3 -1
- package/dist/cli/index.js +4 -0
- package/dist/cli/usage.js +1 -0
- package/dist/config/schema.d.ts +4 -0
- package/dist/config/schema.js +6 -0
- package/dist/context.d.ts +1 -0
- package/dist/context.js +21 -6
- package/dist/daemon/client.d.ts +12 -0
- package/dist/daemon/client.js +88 -0
- package/dist/daemon/paths.d.ts +16 -0
- package/dist/daemon/paths.js +19 -0
- package/dist/daemon/server.d.ts +2 -0
- package/dist/daemon/server.js +225 -0
- package/dist/facts/budget.d.ts +1 -0
- package/dist/facts/budget.js +4 -0
- package/dist/facts/claude-memory.js +1 -1
- package/dist/facts/distill.js +2 -2
- package/dist/facts/known.js +1 -1
- package/dist/facts/model.d.ts +1 -1
- package/dist/facts/model.js +18 -7
- package/dist/facts/start.js +1 -1
- package/dist/graph/query/refresh.js +5 -1
- package/dist/graph/scan/cache.d.ts +1 -0
- package/dist/graph/scan/cache.js +1 -1
- package/dist/graph/scan/persist.js +12 -16
- package/dist/hooks/compact.d.ts +2 -1
- package/dist/hooks/compact.js +2 -2
- package/dist/hooks/detached.d.ts +1 -0
- package/dist/hooks/detached.js +6 -0
- package/dist/hooks/graph-presence.d.ts +3 -0
- package/dist/hooks/graph-presence.js +42 -0
- package/dist/hooks/handlers/prefer-usages.d.ts +2 -0
- package/dist/hooks/handlers/prefer-usages.js +113 -0
- package/dist/hooks/handlers/session-end.d.ts +2 -0
- package/dist/hooks/handlers/session-end.js +62 -0
- package/dist/hooks/handlers/session-start.d.ts +2 -0
- package/dist/hooks/handlers/session-start.js +104 -0
- package/dist/hooks/handlers/turn-end.d.ts +2 -0
- package/dist/hooks/handlers/turn-end.js +31 -0
- package/dist/hooks/handlers/user-prompt.d.ts +2 -0
- package/dist/hooks/handlers/user-prompt.js +55 -0
- package/dist/hooks/local.d.ts +2 -0
- package/dist/hooks/local.js +23 -0
- package/dist/hooks/payload.d.ts +9 -4
- package/dist/hooks/payload.js +26 -17
- package/dist/hooks/prefer-usages.js +2 -118
- package/dist/hooks/route.d.ts +2 -0
- package/dist/hooks/route.js +28 -0
- package/dist/hooks/run.d.ts +14 -0
- package/dist/hooks/run.js +12 -0
- package/dist/hooks/save-turn.d.ts +2 -0
- package/dist/hooks/save-turn.js +11 -0
- package/dist/hooks/session-end.js +2 -63
- package/dist/hooks/session-start.js +2 -90
- package/dist/hooks/turn-end.js +2 -25
- package/dist/hooks/turn-save.js +10 -15
- package/dist/hooks/user-prompt.js +2 -58
- package/dist/search/service.d.ts +1 -0
- package/dist/search/service.js +16 -6
- package/dist/store/adapter.d.ts +7 -5
- package/dist/store/mongo/index.d.ts +3 -6
- package/dist/store/mongo/index.js +5 -0
- package/dist/store/mongo/observations.js +13 -1
- package/dist/store/postgres/driver.d.ts +10 -3
- package/dist/store/postgres/driver.js +12 -0
- package/dist/store/postgres/graph.d.ts +3 -5
- package/dist/store/postgres/graph.js +56 -50
- package/dist/store/postgres/index.d.ts +3 -6
- package/dist/store/postgres/index.js +5 -4
- package/dist/store/postgres/insert.js +98 -50
- package/dist/store/project-resolver.d.ts +3 -0
- package/dist/store/project-resolver.js +34 -23
- package/dist/store/scoped-store.js +1 -1
- package/dist/store/sqlite/graph.d.ts +3 -5
- package/dist/store/sqlite/graph.js +42 -33
- package/dist/store/sqlite/index.d.ts +3 -6
- package/dist/store/sqlite/index.js +3 -0
- package/dist/store/sqlite/observations.js +6 -2
- package/dist/util/job-lock.d.ts +1 -0
- package/dist/util/job-lock.js +50 -10
- package/docs/setup-guide.md +5 -0
- package/package.json +6 -6
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import type { Pool } from './driver.js';
|
|
2
2
|
import type { CodeEdge, CodeSymbol, EdgeFilter, ScannedFile, SymbolFilter } from '../../types.js';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
files: ScannedFile[];
|
|
7
|
-
}): Promise<void>;
|
|
3
|
+
import type { GraphRows } from '../adapter.js';
|
|
4
|
+
export declare function upsertGraph(pool: Pool, scan: GraphRows): Promise<void>;
|
|
5
|
+
export declare function replaceGraph(pool: Pool, project: string, files: string[] | null, scan: GraphRows): Promise<void>;
|
|
8
6
|
export declare function findSymbols(pool: Pool, filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
9
7
|
export declare function symbolNames(pool: Pool, project: string, limit: number): Promise<string[]>;
|
|
10
8
|
export declare function findEdges(pool: Pool, filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
@@ -25,40 +25,56 @@ async function upsertChunked(client, rows, width, sql, columns) {
|
|
|
25
25
|
await client.query(sql(placeholders(slice.length, width)), values);
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
|
-
|
|
28
|
+
async function writeGraphRows(client, scan) {
|
|
29
|
+
await upsertChunked(client, lastById(scan.symbols, (s) => s.id), 8, (values) => `INSERT INTO symbols (id, project, name, kind, file, line, lang, signature)
|
|
30
|
+
VALUES ${values}
|
|
31
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
32
|
+
name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
|
|
33
|
+
line = EXCLUDED.line, lang = EXCLUDED.lang,
|
|
34
|
+
signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
|
|
35
|
+
await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
|
|
36
|
+
relation, confidence, score, file, line)
|
|
37
|
+
VALUES ${values}
|
|
38
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
39
|
+
dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
|
|
40
|
+
relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
|
|
41
|
+
score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
|
|
42
|
+
e.id,
|
|
43
|
+
e.project,
|
|
44
|
+
e.srcId,
|
|
45
|
+
e.srcName,
|
|
46
|
+
e.dstId,
|
|
47
|
+
e.dstName,
|
|
48
|
+
e.relation,
|
|
49
|
+
e.confidence,
|
|
50
|
+
e.score,
|
|
51
|
+
e.file,
|
|
52
|
+
e.line,
|
|
53
|
+
]);
|
|
54
|
+
await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
|
|
55
|
+
VALUES ${values}
|
|
56
|
+
ON CONFLICT (project, path) DO UPDATE SET
|
|
57
|
+
hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
|
|
58
|
+
}
|
|
59
|
+
async function deleteGraphRows(client, project, files) {
|
|
60
|
+
if (files !== null && files.length === 0)
|
|
61
|
+
return 0;
|
|
62
|
+
const values = files === null ? [project] : [project, files];
|
|
63
|
+
const where = files === null ? 'project = $1' : 'project = $1 AND file = ANY($2::text[])';
|
|
64
|
+
const cacheWhere = files === null ? 'project = $1' : 'project = $1 AND path = ANY($2::text[])';
|
|
65
|
+
const res = await client.query(`WITH gone AS (DELETE FROM symbols WHERE ${where} RETURNING 1),
|
|
66
|
+
edges AS (DELETE FROM symbol_edges WHERE ${where}),
|
|
67
|
+
cached AS (DELETE FROM scanned_files WHERE ${cacheWhere})
|
|
68
|
+
SELECT COUNT(*)::int AS n FROM gone`, values);
|
|
69
|
+
return Number(res.rows[0]?.['n'] ?? 0);
|
|
70
|
+
}
|
|
71
|
+
async function inTransaction(pool, work) {
|
|
29
72
|
const client = await pool.connect();
|
|
30
73
|
try {
|
|
31
74
|
await client.query('BEGIN');
|
|
32
|
-
|
|
33
|
-
VALUES ${values}
|
|
34
|
-
ON CONFLICT (id) DO UPDATE SET
|
|
35
|
-
name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
|
|
36
|
-
line = EXCLUDED.line, lang = EXCLUDED.lang,
|
|
37
|
-
signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
|
|
38
|
-
await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
|
|
39
|
-
relation, confidence, score, file, line)
|
|
40
|
-
VALUES ${values}
|
|
41
|
-
ON CONFLICT (id) DO UPDATE SET
|
|
42
|
-
dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
|
|
43
|
-
relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
|
|
44
|
-
score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
|
|
45
|
-
e.id,
|
|
46
|
-
e.project,
|
|
47
|
-
e.srcId,
|
|
48
|
-
e.srcName,
|
|
49
|
-
e.dstId,
|
|
50
|
-
e.dstName,
|
|
51
|
-
e.relation,
|
|
52
|
-
e.confidence,
|
|
53
|
-
e.score,
|
|
54
|
-
e.file,
|
|
55
|
-
e.line,
|
|
56
|
-
]);
|
|
57
|
-
await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
|
|
58
|
-
VALUES ${values}
|
|
59
|
-
ON CONFLICT (project, path) DO UPDATE SET
|
|
60
|
-
hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
|
|
75
|
+
const result = await work(client);
|
|
61
76
|
await client.query('COMMIT');
|
|
77
|
+
return result;
|
|
62
78
|
}
|
|
63
79
|
catch (error) {
|
|
64
80
|
await client.query('ROLLBACK');
|
|
@@ -68,6 +84,15 @@ export async function upsertGraph(pool, scan) {
|
|
|
68
84
|
client.release();
|
|
69
85
|
}
|
|
70
86
|
}
|
|
87
|
+
export async function upsertGraph(pool, scan) {
|
|
88
|
+
await inTransaction(pool, (client) => writeGraphRows(client, scan));
|
|
89
|
+
}
|
|
90
|
+
export async function replaceGraph(pool, project, files, scan) {
|
|
91
|
+
await inTransaction(pool, async (client) => {
|
|
92
|
+
await deleteGraphRows(client, project, files);
|
|
93
|
+
await writeGraphRows(client, scan);
|
|
94
|
+
});
|
|
95
|
+
}
|
|
71
96
|
const SYMBOL_COLUMNS = 'id, name, kind, file, line, lang, signature';
|
|
72
97
|
const EDGE_COLUMNS = 'id, src_id, src_name, dst_id, dst_name, relation, confidence, score, file, line';
|
|
73
98
|
export async function findSymbols(pool, filter) {
|
|
@@ -132,24 +157,5 @@ export async function scannedFiles(pool, project, limit) {
|
|
|
132
157
|
}
|
|
133
158
|
export async function removeGraph(pool, project, files) {
|
|
134
159
|
const scoped = files !== undefined && files.length > 0;
|
|
135
|
-
|
|
136
|
-
const where = scoped ? 'project = $1 AND file = ANY($2::text[])' : 'project = $1';
|
|
137
|
-
const cacheWhere = scoped ? 'project = $1 AND path = ANY($2::text[])' : 'project = $1';
|
|
138
|
-
const client = await pool.connect();
|
|
139
|
-
try {
|
|
140
|
-
await client.query('BEGIN');
|
|
141
|
-
const counted = await client.query(`SELECT COUNT(*)::int AS n FROM symbols WHERE ${where}`, values);
|
|
142
|
-
await client.query(`DELETE FROM symbols WHERE ${where}`, values);
|
|
143
|
-
await client.query(`DELETE FROM symbol_edges WHERE ${where}`, values);
|
|
144
|
-
await client.query(`DELETE FROM scanned_files WHERE ${cacheWhere}`, values);
|
|
145
|
-
await client.query('COMMIT');
|
|
146
|
-
return Number(counted.rows[0]?.['n'] ?? 0);
|
|
147
|
-
}
|
|
148
|
-
catch (error) {
|
|
149
|
-
await client.query('ROLLBACK');
|
|
150
|
-
throw error;
|
|
151
|
-
}
|
|
152
|
-
finally {
|
|
153
|
-
client.release();
|
|
154
|
-
}
|
|
160
|
+
return deleteGraphRows(pool, project, scoped ? files : null);
|
|
155
161
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
2
|
-
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
2
|
+
import type { GraphRows, MemoryStore, ProjectSummary } from '../adapter.js';
|
|
3
3
|
export declare class PostgresStore implements MemoryStore {
|
|
4
4
|
private readonly pool;
|
|
5
5
|
readonly kind = "postgres";
|
|
@@ -42,11 +42,8 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
42
42
|
markReplaced(ids: string[]): Promise<number>;
|
|
43
43
|
timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
44
44
|
tryEnableVector(): Promise<boolean>;
|
|
45
|
-
upsertGraph(scan:
|
|
46
|
-
|
|
47
|
-
edges: CodeEdge[];
|
|
48
|
-
files: ScannedFile[];
|
|
49
|
-
}): Promise<void>;
|
|
45
|
+
upsertGraph(scan: GraphRows): Promise<void>;
|
|
46
|
+
replaceGraph(project: string, files: string[] | null, scan: GraphRows): Promise<void>;
|
|
50
47
|
findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
51
48
|
symbolNames(project: string, limit: number): Promise<string[]>;
|
|
52
49
|
findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
@@ -6,7 +6,7 @@ import * as observationsOps from './observations.js';
|
|
|
6
6
|
import * as sessionsOps from './sessions.js';
|
|
7
7
|
import * as graphOps from './graph.js';
|
|
8
8
|
import * as projectLinksOps from './project-links.js';
|
|
9
|
-
import {
|
|
9
|
+
import { openPool } from './driver.js';
|
|
10
10
|
import { DDL, SCHEMA_VERSION } from './schema.js';
|
|
11
11
|
export class PostgresStore {
|
|
12
12
|
pool;
|
|
@@ -18,9 +18,7 @@ export class PostgresStore {
|
|
|
18
18
|
this.pool = pool;
|
|
19
19
|
}
|
|
20
20
|
static async create(uri) {
|
|
21
|
-
|
|
22
|
-
const pool = new pg.Pool({ connectionString: uri });
|
|
23
|
-
return new PostgresStore(pool);
|
|
21
|
+
return new PostgresStore(await openPool(uri));
|
|
24
22
|
}
|
|
25
23
|
async init() {
|
|
26
24
|
const meta = await this.readMeta();
|
|
@@ -155,6 +153,9 @@ export class PostgresStore {
|
|
|
155
153
|
async upsertGraph(scan) {
|
|
156
154
|
return graphOps.upsertGraph(this.pool, scan);
|
|
157
155
|
}
|
|
156
|
+
async replaceGraph(project, files, scan) {
|
|
157
|
+
return graphOps.replaceGraph(this.pool, project, files, scan);
|
|
158
|
+
}
|
|
158
159
|
async findSymbols(filter) {
|
|
159
160
|
return graphOps.findSymbols(this.pool, filter);
|
|
160
161
|
}
|
|
@@ -1,56 +1,91 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
const COLUMNS = [
|
|
2
|
+
'id',
|
|
3
|
+
'session_id',
|
|
4
|
+
'project',
|
|
5
|
+
'kind',
|
|
6
|
+
'title',
|
|
7
|
+
'body',
|
|
8
|
+
'files',
|
|
9
|
+
'tags',
|
|
10
|
+
'created_at',
|
|
11
|
+
'embedder',
|
|
12
|
+
'author',
|
|
13
|
+
'status',
|
|
14
|
+
];
|
|
15
|
+
const ROWS_PER_STATEMENT = 400;
|
|
16
|
+
const KEEP_CLOSED = `status = CASE
|
|
17
|
+
WHEN observations.status = 'done' AND EXCLUDED.status = 'open'
|
|
18
|
+
AND observations.files = EXCLUDED.files THEN observations.status
|
|
19
|
+
ELSE EXCLUDED.status END`;
|
|
20
|
+
function rowValues(obs, shape) {
|
|
21
|
+
const values = [
|
|
22
|
+
obs.id,
|
|
23
|
+
obs.sessionId,
|
|
24
|
+
obs.project,
|
|
25
|
+
obs.kind,
|
|
26
|
+
obs.title,
|
|
27
|
+
obs.body,
|
|
28
|
+
JSON.stringify(obs.files),
|
|
29
|
+
JSON.stringify(obs.tags),
|
|
30
|
+
obs.createdAt,
|
|
31
|
+
shape.vectorStored ? (obs.embedder ?? null) : null,
|
|
32
|
+
obs.author ?? null,
|
|
33
|
+
obs.status ?? 'done',
|
|
34
|
+
];
|
|
35
|
+
if (shape.vectorStored && obs.embedding)
|
|
36
|
+
values.push(`[${obs.embedding.join(',')}]`);
|
|
37
|
+
return values;
|
|
38
|
+
}
|
|
39
|
+
function statement(rows, shape, clearVector) {
|
|
40
|
+
const columns = shape.vectorStored ? [...COLUMNS, 'embedding'] : COLUMNS;
|
|
41
|
+
const values = [];
|
|
42
|
+
const tuples = rows.map((obs) => {
|
|
43
|
+
const start = values.length;
|
|
44
|
+
values.push(...rowValues(obs, shape));
|
|
45
|
+
return `(${columns.map((_, i) => `$${start + i + 1}`).join(', ')})`;
|
|
46
|
+
});
|
|
47
|
+
const updates = columns
|
|
48
|
+
.slice(1)
|
|
49
|
+
.filter((column) => !(shape.vectorKept && column === 'embedder'))
|
|
50
|
+
.map((column) => (column === 'status' ? KEEP_CLOSED : `${column} = EXCLUDED.${column}`))
|
|
51
|
+
.concat(clearVector ? ['embedding = NULL'] : [])
|
|
52
|
+
.join(', ');
|
|
53
|
+
return {
|
|
54
|
+
sql: `INSERT INTO observations (${columns.join(', ')}) VALUES ${tuples.join(', ')}
|
|
55
|
+
ON CONFLICT (id) DO UPDATE SET ${updates}`,
|
|
56
|
+
values,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
function lastPerId(observations) {
|
|
60
|
+
return [...new Map(observations.map((obs) => [obs.id, obs])).values()];
|
|
61
|
+
}
|
|
62
|
+
function insertStatements(observations, width, vectors, hasVectorColumn) {
|
|
63
|
+
const groups = new Map();
|
|
64
|
+
for (const obs of lastPerId(observations)) {
|
|
65
|
+
const shape = {
|
|
66
|
+
vectorStored: vectors && obs.embedding?.length === width,
|
|
67
|
+
vectorKept: !obs.embedding?.length && obs.embedder != null,
|
|
68
|
+
};
|
|
69
|
+
const id = `${shape.vectorStored}:${shape.vectorKept}`;
|
|
70
|
+
const group = groups.get(id) ?? { shape, rows: [] };
|
|
71
|
+
group.rows.push(obs);
|
|
72
|
+
groups.set(id, group);
|
|
73
|
+
}
|
|
74
|
+
const statements = [];
|
|
75
|
+
for (const { shape, rows } of groups.values()) {
|
|
76
|
+
const clearVector = !shape.vectorStored && !shape.vectorKept && (hasVectorColumn || vectors);
|
|
77
|
+
for (let i = 0; i < rows.length; i += ROWS_PER_STATEMENT) {
|
|
78
|
+
statements.push(statement(rows.slice(i, i + ROWS_PER_STATEMENT), shape, clearVector));
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return statements;
|
|
82
|
+
}
|
|
83
|
+
async function inTransaction(pool, statements) {
|
|
6
84
|
const client = await pool.connect();
|
|
7
85
|
try {
|
|
8
86
|
await client.query('BEGIN');
|
|
9
|
-
for (const
|
|
10
|
-
|
|
11
|
-
const vectorKept = !obs.embedding?.length && obs.embedder != null;
|
|
12
|
-
const columns = [
|
|
13
|
-
'id',
|
|
14
|
-
'session_id',
|
|
15
|
-
'project',
|
|
16
|
-
'kind',
|
|
17
|
-
'title',
|
|
18
|
-
'body',
|
|
19
|
-
'files',
|
|
20
|
-
'tags',
|
|
21
|
-
'created_at',
|
|
22
|
-
'embedder',
|
|
23
|
-
'author',
|
|
24
|
-
'status',
|
|
25
|
-
];
|
|
26
|
-
const values = [
|
|
27
|
-
obs.id,
|
|
28
|
-
obs.sessionId,
|
|
29
|
-
obs.project,
|
|
30
|
-
obs.kind,
|
|
31
|
-
obs.title,
|
|
32
|
-
obs.body,
|
|
33
|
-
JSON.stringify(obs.files),
|
|
34
|
-
JSON.stringify(obs.tags),
|
|
35
|
-
obs.createdAt,
|
|
36
|
-
vectorStored ? (obs.embedder ?? null) : null,
|
|
37
|
-
obs.author ?? null,
|
|
38
|
-
obs.status ?? 'done',
|
|
39
|
-
];
|
|
40
|
-
if (vectorStored && obs.embedding) {
|
|
41
|
-
columns.push('embedding');
|
|
42
|
-
values.push(`[${obs.embedding.join(',')}]`);
|
|
43
|
-
}
|
|
44
|
-
const holes = values.map((_, i) => `$${i + 1}`).join(', ');
|
|
45
|
-
const updates = columns
|
|
46
|
-
.slice(1)
|
|
47
|
-
.filter((column) => !(vectorKept && column === 'embedder'))
|
|
48
|
-
.map((column) => `${column} = EXCLUDED.${column}`)
|
|
49
|
-
.concat(!vectorStored && !vectorKept && (hasVectorColumn || vectors) ? ['embedding = NULL'] : [])
|
|
50
|
-
.join(', ');
|
|
51
|
-
await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
|
|
52
|
-
ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
|
|
53
|
-
}
|
|
87
|
+
for (const { sql, values } of statements)
|
|
88
|
+
await client.query(sql, values);
|
|
54
89
|
await client.query('COMMIT');
|
|
55
90
|
}
|
|
56
91
|
catch (error) {
|
|
@@ -61,3 +96,16 @@ export async function insertObservations(pool, ensureVectorColumn, observations,
|
|
|
61
96
|
client.release();
|
|
62
97
|
}
|
|
63
98
|
}
|
|
99
|
+
export async function insertObservations(pool, ensureVectorColumn, observations, hasVectorColumn = false) {
|
|
100
|
+
if (observations.length === 0)
|
|
101
|
+
return;
|
|
102
|
+
const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
|
|
103
|
+
const vectors = width !== undefined && (await ensureVectorColumn(width));
|
|
104
|
+
const statements = insertStatements(observations, width, vectors, hasVectorColumn);
|
|
105
|
+
const [only] = statements;
|
|
106
|
+
if (statements.length === 1 && only) {
|
|
107
|
+
await pool.query(only.sql, only.values);
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
await inTransaction(pool, statements);
|
|
111
|
+
}
|
|
@@ -4,12 +4,15 @@ export interface ScopeResolverOptions {
|
|
|
4
4
|
remote: string;
|
|
5
5
|
isFolder?: (value: string) => boolean;
|
|
6
6
|
keyOf?: (folder: string, remote: string) => string;
|
|
7
|
+
scopeTtlMs?: number;
|
|
8
|
+
now?: () => number;
|
|
7
9
|
}
|
|
8
10
|
export interface ProjectScope {
|
|
9
11
|
key: string;
|
|
10
12
|
names: string[];
|
|
11
13
|
}
|
|
12
14
|
export interface ScopeResolver {
|
|
15
|
+
key(value: string): Promise<string>;
|
|
13
16
|
resolve(value: string): Promise<ProjectScope>;
|
|
14
17
|
}
|
|
15
18
|
export declare function isExistingFolder(value: string): boolean;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { statSync } from 'node:fs';
|
|
2
2
|
import { isAbsolute } from 'node:path';
|
|
3
3
|
import { projectKey } from '../util/project-key.js';
|
|
4
|
-
import { orderScope } from './project-scope.js';
|
|
4
|
+
import { isLinkable, orderScope } from './project-scope.js';
|
|
5
|
+
const SCOPE_TTL_MS = 60_000;
|
|
5
6
|
export function isExistingFolder(value) {
|
|
6
7
|
if (!isAbsolute(value))
|
|
7
8
|
return false;
|
|
@@ -13,37 +14,47 @@ export function isExistingFolder(value) {
|
|
|
13
14
|
}
|
|
14
15
|
}
|
|
15
16
|
export function createScopeResolver(options) {
|
|
16
|
-
const { store, remote, isFolder = isExistingFolder, keyOf = (folder, name) => projectKey(folder, { remote: name }), } = options;
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
const
|
|
17
|
+
const { store, remote, isFolder = isExistingFolder, keyOf = (folder, name) => projectKey(folder, { remote: name }), scopeTtlMs = SCOPE_TTL_MS, now = Date.now, } = options;
|
|
18
|
+
const scopes = new Map();
|
|
19
|
+
function forgetOthers(folder) {
|
|
20
|
+
for (const cached of [...scopes.keys()])
|
|
21
|
+
if (cached !== folder)
|
|
22
|
+
scopes.delete(cached);
|
|
23
|
+
}
|
|
24
|
+
async function linked(folder) {
|
|
20
25
|
try {
|
|
21
|
-
await store.
|
|
26
|
+
return await store.projectScope(folder);
|
|
22
27
|
}
|
|
23
28
|
catch {
|
|
24
|
-
return
|
|
29
|
+
return null;
|
|
25
30
|
}
|
|
26
|
-
return key;
|
|
27
31
|
}
|
|
28
|
-
async function
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
async function lookUp(folder) {
|
|
33
|
+
const key = keyOf(folder, remote);
|
|
34
|
+
let related = await linked(folder);
|
|
35
|
+
if (isLinkable(folder, key) && !related?.includes(key)) {
|
|
36
|
+
const before = related;
|
|
37
|
+
related = await store.linkProject(folder, key).then(() => {
|
|
38
|
+
forgetOthers(folder);
|
|
39
|
+
return linked(folder);
|
|
40
|
+
}, () => before);
|
|
34
41
|
}
|
|
42
|
+
return { key, names: orderScope(folder, [...(related ?? []), key]) };
|
|
43
|
+
}
|
|
44
|
+
function scopeOf(folder) {
|
|
45
|
+
const cached = scopes.get(folder);
|
|
46
|
+
if (cached && now() - cached.at < scopeTtlMs)
|
|
47
|
+
return cached.scope;
|
|
48
|
+
const scope = lookUp(folder);
|
|
49
|
+
scopes.set(folder, { at: now(), scope });
|
|
50
|
+
return scope;
|
|
35
51
|
}
|
|
36
52
|
return {
|
|
53
|
+
async key(value) {
|
|
54
|
+
return isFolder(value) ? (await scopeOf(value)).key : value;
|
|
55
|
+
},
|
|
37
56
|
async resolve(value) {
|
|
38
|
-
|
|
39
|
-
return { key: value, names: [value] };
|
|
40
|
-
let pending = keys.get(value);
|
|
41
|
-
if (!pending) {
|
|
42
|
-
pending = linkedKey(value);
|
|
43
|
-
keys.set(value, pending);
|
|
44
|
-
}
|
|
45
|
-
const key = await pending;
|
|
46
|
-
return { key, names: await namesFor(value, key) };
|
|
57
|
+
return isFolder(value) ? scopeOf(value) : { key: value, names: [value] };
|
|
47
58
|
},
|
|
48
59
|
};
|
|
49
60
|
}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import type { DatabaseSync } from 'node:sqlite';
|
|
2
2
|
import type { CodeEdge, CodeSymbol, EdgeFilter, ScannedFile, SymbolFilter } from '../../types.js';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
files: ScannedFile[];
|
|
7
|
-
}): Promise<void>;
|
|
3
|
+
import type { GraphRows } from '../adapter.js';
|
|
4
|
+
export declare function upsertGraph(db: DatabaseSync, scan: GraphRows): Promise<void>;
|
|
5
|
+
export declare function replaceGraph(db: DatabaseSync, project: string, files: string[] | null, scan: GraphRows): Promise<void>;
|
|
8
6
|
export declare function findSymbols(db: DatabaseSync, filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
9
7
|
export declare function symbolNames(db: DatabaseSync, project: string, limit: number): Promise<string[]>;
|
|
10
8
|
export declare function findEdges(db: DatabaseSync, filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { toEdge, toSymbol } from './rows.js';
|
|
2
|
-
|
|
2
|
+
function writeGraphRows(db, scan) {
|
|
3
3
|
const symbol = db.prepare(`INSERT OR REPLACE INTO symbols
|
|
4
4
|
(id, project, name, kind, file, line, lang, signature)
|
|
5
5
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
@@ -9,24 +9,54 @@ export async function upsertGraph(db, scan) {
|
|
|
9
9
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
10
10
|
const file = db.prepare(`INSERT OR REPLACE INTO scanned_files (project, path, hash, scanned_at)
|
|
11
11
|
VALUES (?, ?, ?, ?)`);
|
|
12
|
+
for (const s of scan.symbols) {
|
|
13
|
+
symbol.run(s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature);
|
|
14
|
+
}
|
|
15
|
+
for (const e of scan.edges) {
|
|
16
|
+
edge.run(e.id, e.project, e.srcId, e.srcName, e.dstId, e.dstName, e.relation, e.confidence, e.score, e.file, e.line);
|
|
17
|
+
}
|
|
18
|
+
for (const f of scan.files)
|
|
19
|
+
file.run(f.project, f.path, f.hash, f.scannedAt);
|
|
20
|
+
}
|
|
21
|
+
const DELETE_BATCH = 500;
|
|
22
|
+
function deleteGraphRows(db, project, files) {
|
|
23
|
+
const batches = files === null ? [null] : [];
|
|
24
|
+
for (let start = 0; files !== null && start < files.length; start += DELETE_BATCH) {
|
|
25
|
+
batches.push(files.slice(start, start + DELETE_BATCH));
|
|
26
|
+
}
|
|
27
|
+
let removed = 0;
|
|
28
|
+
for (const batch of batches) {
|
|
29
|
+
const holes = batch?.map(() => '?').join(',');
|
|
30
|
+
const where = batch ? `project = ? AND file IN (${holes})` : 'project = ?';
|
|
31
|
+
const cacheWhere = batch ? `project = ? AND path IN (${holes})` : 'project = ?';
|
|
32
|
+
const params = (batch ? [project, ...batch] : [project]);
|
|
33
|
+
removed += Number(db.prepare(`DELETE FROM symbols WHERE ${where}`).run(...params).changes);
|
|
34
|
+
db.prepare(`DELETE FROM symbol_edges WHERE ${where}`).run(...params);
|
|
35
|
+
db.prepare(`DELETE FROM scanned_files WHERE ${cacheWhere}`).run(...params);
|
|
36
|
+
}
|
|
37
|
+
return removed;
|
|
38
|
+
}
|
|
39
|
+
function inTransaction(db, work) {
|
|
12
40
|
db.exec('BEGIN');
|
|
13
41
|
try {
|
|
14
|
-
|
|
15
|
-
symbol.run(s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature);
|
|
16
|
-
}
|
|
17
|
-
for (const e of scan.edges) {
|
|
18
|
-
edge.run(e.id, e.project, e.srcId, e.srcName, e.dstId, e.dstName, e.relation, e.confidence, e.score, e.file, e.line);
|
|
19
|
-
}
|
|
20
|
-
for (const f of scan.files) {
|
|
21
|
-
file.run(f.project, f.path, f.hash, f.scannedAt);
|
|
22
|
-
}
|
|
42
|
+
const result = work();
|
|
23
43
|
db.exec('COMMIT');
|
|
44
|
+
return result;
|
|
24
45
|
}
|
|
25
46
|
catch (error) {
|
|
26
47
|
db.exec('ROLLBACK');
|
|
27
48
|
throw error;
|
|
28
49
|
}
|
|
29
50
|
}
|
|
51
|
+
export async function upsertGraph(db, scan) {
|
|
52
|
+
inTransaction(db, () => writeGraphRows(db, scan));
|
|
53
|
+
}
|
|
54
|
+
export async function replaceGraph(db, project, files, scan) {
|
|
55
|
+
inTransaction(db, () => {
|
|
56
|
+
deleteGraphRows(db, project, files);
|
|
57
|
+
writeGraphRows(db, scan);
|
|
58
|
+
});
|
|
59
|
+
}
|
|
30
60
|
export async function findSymbols(db, filter) {
|
|
31
61
|
const conditions = ['project = ?'];
|
|
32
62
|
const params = [filter.project];
|
|
@@ -95,27 +125,6 @@ export async function scannedFiles(db, project, limit) {
|
|
|
95
125
|
}));
|
|
96
126
|
}
|
|
97
127
|
export async function removeGraph(db, project, files) {
|
|
98
|
-
const scoped = files && files.length > 0;
|
|
99
|
-
|
|
100
|
-
? `project = ? AND file IN (${files.map(() => '?').join(',')})`
|
|
101
|
-
: 'project = ?';
|
|
102
|
-
const params = scoped ? [project, ...files] : [project];
|
|
103
|
-
const count = db
|
|
104
|
-
.prepare(`SELECT COUNT(*) AS n FROM symbols WHERE ${where}`)
|
|
105
|
-
.get(...params);
|
|
106
|
-
db.exec('BEGIN');
|
|
107
|
-
try {
|
|
108
|
-
db.prepare(`DELETE FROM symbols WHERE ${where}`).run(...params);
|
|
109
|
-
db.prepare(`DELETE FROM symbol_edges WHERE ${where}`).run(...params);
|
|
110
|
-
const cacheWhere = scoped
|
|
111
|
-
? `project = ? AND path IN (${files.map(() => '?').join(',')})`
|
|
112
|
-
: 'project = ?';
|
|
113
|
-
db.prepare(`DELETE FROM scanned_files WHERE ${cacheWhere}`).run(...params);
|
|
114
|
-
db.exec('COMMIT');
|
|
115
|
-
}
|
|
116
|
-
catch (error) {
|
|
117
|
-
db.exec('ROLLBACK');
|
|
118
|
-
throw error;
|
|
119
|
-
}
|
|
120
|
-
return count?.n ?? 0;
|
|
128
|
+
const scoped = files !== undefined && files.length > 0;
|
|
129
|
+
return inTransaction(db, () => deleteGraphRows(db, project, scoped ? files : null));
|
|
121
130
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
|
|
2
|
-
import type { MemoryStore, ProjectSummary } from '../adapter.js';
|
|
2
|
+
import type { GraphRows, MemoryStore, ProjectSummary } from '../adapter.js';
|
|
3
3
|
export declare const HERE: string;
|
|
4
4
|
export declare class SqliteStore implements MemoryStore {
|
|
5
5
|
private readonly db;
|
|
@@ -28,11 +28,8 @@ export declare class SqliteStore implements MemoryStore {
|
|
|
28
28
|
closeObservations(ids: string[]): Promise<number>;
|
|
29
29
|
markReplaced(ids: string[]): Promise<number>;
|
|
30
30
|
timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
|
|
31
|
-
upsertGraph(scan:
|
|
32
|
-
|
|
33
|
-
edges: CodeEdge[];
|
|
34
|
-
files: ScannedFile[];
|
|
35
|
-
}): Promise<void>;
|
|
31
|
+
upsertGraph(scan: GraphRows): Promise<void>;
|
|
32
|
+
replaceGraph(project: string, files: string[] | null, scan: GraphRows): Promise<void>;
|
|
36
33
|
findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
|
|
37
34
|
symbolNames(project: string, limit: number): Promise<string[]>;
|
|
38
35
|
findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
|
|
@@ -127,6 +127,9 @@ export class SqliteStore {
|
|
|
127
127
|
async upsertGraph(scan) {
|
|
128
128
|
return graphOps.upsertGraph(this.db, scan);
|
|
129
129
|
}
|
|
130
|
+
async replaceGraph(project, files, scan) {
|
|
131
|
+
return graphOps.replaceGraph(this.db, project, files, scan);
|
|
132
|
+
}
|
|
130
133
|
async findSymbols(filter) {
|
|
131
134
|
return graphOps.findSymbols(this.db, filter);
|
|
132
135
|
}
|
|
@@ -11,11 +11,15 @@ export async function insertObservations(db, observations) {
|
|
|
11
11
|
const stmt = db.prepare(`INSERT OR REPLACE INTO observations
|
|
12
12
|
(id, session_id, project, scope, kind, title, body, files, tags,
|
|
13
13
|
created_at, embedding, embedder, author, status)
|
|
14
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
14
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
15
|
+
COALESCE((SELECT status FROM observations
|
|
16
|
+
WHERE id = ? AND status = 'done' AND ? = 'open' AND files = ?), ?))`);
|
|
15
17
|
db.exec('BEGIN');
|
|
16
18
|
try {
|
|
17
19
|
for (const obs of observations) {
|
|
18
|
-
|
|
20
|
+
const files = JSON.stringify(obs.files);
|
|
21
|
+
const status = obs.status ?? 'done';
|
|
22
|
+
stmt.run(obs.id, obs.sessionId, obs.project, scopeToken(obs.project), obs.kind, obs.title, obs.body, files, JSON.stringify(obs.tags), obs.createdAt, obs.embedding ? packVector(obs.embedding) : null, obs.embedder ?? null, obs.author ?? null, obs.id, status, files, status);
|
|
19
23
|
}
|
|
20
24
|
db.exec('COMMIT');
|
|
21
25
|
}
|
package/dist/util/job-lock.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export declare function jobMark(job: string, key: string): number;
|
|
2
2
|
export declare function claimJob(job: string, key: string, now?: number): boolean;
|
|
3
|
+
export declare function keepJob(job: string, key: string): () => void;
|
|
3
4
|
export declare function finishJob(job: string, key: string, mark: number): void;
|
|
4
5
|
export declare function releaseJob(job: string, key: string): void;
|