claude-db 0.8.3 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -1
- package/dist/cli/commands/transfer.js +8 -1
- package/dist/graph/languages/ecmascript.js +21 -1
- package/dist/graph/languages/rules.d.ts +1 -0
- package/dist/graph/scan/extract.js +2 -0
- package/dist/graph/scan/files.js +1 -1
- package/dist/store/adapter.d.ts +1 -0
- package/dist/store/postgres/graph.js +47 -27
- package/dist/store/postgres/index.d.ts +1 -0
- package/dist/store/postgres/index.js +17 -0
- package/dist/store/postgres/insert.js +3 -2
- package/dist/store/postgres/rows.js +16 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,48 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.4
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- **Postgres: `scan` issued one round-trip per symbol and per edge, making it
|
|
8
|
+
unusable against a hosted database.**
|
|
9
|
+
([#16](https://github.com/Avijit07x/claude-db/issues/16))
|
|
10
|
+
`upsertGraph` looped over `scan.symbols`, `scan.edges` and `scan.files`, and
|
|
11
|
+
awaited a separate `INSERT` for each row. Parsing a 639-file React Native repo
|
|
12
|
+
takes 1.5s and produces 50,353 rows; against a Neon instance with a 100ms
|
|
13
|
+
round-trip that is 84 minutes of pure network wait, and the scan never
|
|
14
|
+
completed. Rows are now written 500 per statement, which takes the same repo
|
|
15
|
+
to 20s end to end. SQLite used a prepared statement and MongoDB used
|
|
16
|
+
`bulkWrite` already; only Postgres wrote row by row.
|
|
17
|
+
- **Postgres: changing embedding dimensions permanently disabled vector search,
|
|
18
|
+
and `reembed` could not repair it.**
|
|
19
|
+
([#14](https://github.com/Avijit07x/claude-db/issues/14))
|
|
20
|
+
Installing `@xenova/transformers`, which `doctor` recommends, moves `auto`
|
|
21
|
+
from the 256d builtin embedder to 384d MiniLM. `ensureVectorColumn` only ever
|
|
22
|
+
created the `embedding` column, never widened an existing one, so every write
|
|
23
|
+
was rejected for width and `searchVector` returned nothing on every query.
|
|
24
|
+
Rows were still stamped with the new `embedder`, and `reembed` skipped any row
|
|
25
|
+
whose label already matched, so the corpus could not recover: on one 455-row
|
|
26
|
+
database, 291 rows kept stale 256d vectors, 164 had none, and search silently
|
|
27
|
+
ran keyword-only for weeks. `reembed` now rebuilds the column at the new width
|
|
28
|
+
and re-embeds every row, `embedder` is only written when the vector is stored,
|
|
29
|
+
and rows missing a vector are treated as stale. The Postgres adapter also
|
|
30
|
+
hydrates `embedding` on read, as SQLite and MongoDB already did.
|
|
31
|
+
|
|
32
|
+
### Added
|
|
33
|
+
|
|
34
|
+
- **Rendering a component in `.tsx` now produces a graph edge.**
|
|
35
|
+
([#19](https://github.com/Avijit07x/claude-db/issues/19))
|
|
36
|
+
The TSX spec was TypeScript with a different label, and JSX has no rule, so
|
|
37
|
+
`<Card />` matched nothing: a presentational component had no inbound edges at
|
|
38
|
+
all and `find_usages` reported it as unused. `jsx_opening_element` and
|
|
39
|
+
`jsx_self_closing_element` are now reference rules, filtered to capitalised
|
|
40
|
+
names via a new optional `namePattern` on `ReferenceRule` so host elements
|
|
41
|
+
like `div` and `span` are not mistaken for components. The same rules apply to
|
|
42
|
+
`.jsx`, which resolves to the JavaScript spec. On a 639-file React
|
|
43
|
+
Native repo this adds 7,904 edges, 3,477 of them resolving to a symbol in the
|
|
44
|
+
repo, for 0.2s of extra parse time.
|
|
45
|
+
|
|
3
46
|
## 0.8.3
|
|
4
47
|
|
|
5
48
|
### Fixed
|
|
@@ -12,7 +55,7 @@
|
|
|
12
55
|
`/home/f/work/epiclin_2` was looked up under `-home-f-work-epiclin_2` while
|
|
13
56
|
its transcripts sat in `-home-f-work-epiclin-2`, so `flush`, `adoption` and
|
|
14
57
|
`setup` all reported nothing to ingest. The name is now derived the same way
|
|
15
|
-
Claude Code writes it
|
|
58
|
+
Claude Code writes it, one dash per non-alphanumeric character, which also
|
|
16
59
|
covers paths with spaces, `@`, `~` or accented characters.
|
|
17
60
|
|
|
18
61
|
## 0.8.2
|
|
@@ -108,11 +108,15 @@ export async function cmdReembed(argv = []) {
|
|
|
108
108
|
console.error('No embedder available; nothing to do.');
|
|
109
109
|
process.exit(1);
|
|
110
110
|
}
|
|
111
|
+
let migrated = false;
|
|
112
|
+
if (!scoped && ctx.store.migrateVectorDims) {
|
|
113
|
+
migrated = await ctx.store.migrateVectorDims(embedder.dimensions);
|
|
114
|
+
}
|
|
111
115
|
let updated = 0;
|
|
112
116
|
let skipped = 0;
|
|
113
117
|
const filter = scoped ? { project: resolveProject(undefined) } : {};
|
|
114
118
|
const scanned = await eachObservation(ctx, filter, async (batch) => {
|
|
115
|
-
const stale = batch.filter((obs) => obs.embedder !== embedder.id);
|
|
119
|
+
const stale = batch.filter((obs) => obs.embedder !== embedder.id || !obs.embedding?.length);
|
|
116
120
|
skipped += batch.length - stale.length;
|
|
117
121
|
if (stale.length === 0)
|
|
118
122
|
return;
|
|
@@ -122,6 +126,9 @@ export async function cmdReembed(argv = []) {
|
|
|
122
126
|
process.stderr.write(`\r${updated} re-embedded...`);
|
|
123
127
|
});
|
|
124
128
|
process.stderr.write('\r');
|
|
129
|
+
if (migrated) {
|
|
130
|
+
console.log(`Rebuilt vector storage at ${embedder.dimensions}d.`);
|
|
131
|
+
}
|
|
125
132
|
console.log(`Scanned ${scanned}, re-embedded ${updated} with ${embedder.id}` +
|
|
126
133
|
`${skipped > 0 ? `, ${skipped} already current` : ''}.`);
|
|
127
134
|
}
|
|
@@ -30,11 +30,27 @@ export const typescript = {
|
|
|
30
30
|
definitions: [...SHARED_DEFINITIONS, ...TYPED_DEFINITIONS],
|
|
31
31
|
references: [...SHARED_REFERENCES, ...TYPED_REFERENCES],
|
|
32
32
|
};
|
|
33
|
+
const COMPONENT_NAME = /^[A-Z]/;
|
|
34
|
+
const JSX_REFERENCES = [
|
|
35
|
+
{
|
|
36
|
+
kind: 'jsx_opening_element',
|
|
37
|
+
field: ['name'],
|
|
38
|
+
relation: 'references',
|
|
39
|
+
namePattern: COMPONENT_NAME,
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
kind: 'jsx_self_closing_element',
|
|
43
|
+
field: ['name'],
|
|
44
|
+
relation: 'references',
|
|
45
|
+
namePattern: COMPONENT_NAME,
|
|
46
|
+
},
|
|
47
|
+
];
|
|
33
48
|
export const tsx = {
|
|
34
49
|
...typescript,
|
|
35
50
|
id: 'Tsx',
|
|
36
51
|
label: 'tsx',
|
|
37
52
|
extensions: ['.tsx'],
|
|
53
|
+
references: [...SHARED_REFERENCES, ...TYPED_REFERENCES, ...JSX_REFERENCES],
|
|
38
54
|
};
|
|
39
55
|
export const javascript = {
|
|
40
56
|
id: 'JavaScript',
|
|
@@ -44,5 +60,9 @@ export const javascript = {
|
|
|
44
60
|
...SHARED_DEFINITIONS,
|
|
45
61
|
{ kind: 'class_declaration', field: ['name'], symbol: 'class' },
|
|
46
62
|
],
|
|
47
|
-
references: [
|
|
63
|
+
references: [
|
|
64
|
+
...SHARED_REFERENCES,
|
|
65
|
+
{ kind: 'class_heritage', field: [], relation: 'extends' },
|
|
66
|
+
...JSX_REFERENCES,
|
|
67
|
+
],
|
|
48
68
|
};
|
|
@@ -133,6 +133,8 @@ export function extractFile(file, project) {
|
|
|
133
133
|
continue;
|
|
134
134
|
if (rule.relation === 'references' && name.includes('.'))
|
|
135
135
|
continue;
|
|
136
|
+
if (rule.namePattern && !rule.namePattern.test(name))
|
|
137
|
+
continue;
|
|
136
138
|
const line = target.range().start.line + 1;
|
|
137
139
|
references.push({
|
|
138
140
|
file: file.path,
|
package/dist/graph/scan/files.js
CHANGED
|
@@ -7,7 +7,7 @@ const MAX_BUFFER = 64 * 1024 * 1024;
|
|
|
7
7
|
const MAX_FILE_BYTES = 1024 * 1024;
|
|
8
8
|
export const SCAN_VERSION = 3;
|
|
9
9
|
const RULES_FINGERPRINT = createHash('sha256')
|
|
10
|
-
.update(JSON.stringify(LANGUAGES))
|
|
10
|
+
.update(JSON.stringify(LANGUAGES, (_key, value) => (value instanceof RegExp ? String(value) : value)))
|
|
11
11
|
.update(BASIC_FINGERPRINT)
|
|
12
12
|
.digest('hex')
|
|
13
13
|
.slice(0, 12);
|
package/dist/store/adapter.d.ts
CHANGED
|
@@ -15,6 +15,7 @@ export interface MemoryStore {
|
|
|
15
15
|
clearSummary(id: string): Promise<boolean>;
|
|
16
16
|
insertObservations(observations: Observation[]): Promise<void>;
|
|
17
17
|
getObservations(ids: string[]): Promise<Observation[]>;
|
|
18
|
+
migrateVectorDims?(dims: number): Promise<boolean>;
|
|
18
19
|
remove(filter: RemoveFilter): Promise<number>;
|
|
19
20
|
list(filter: ListFilter): Promise<Observation[]>;
|
|
20
21
|
listProjects(): Promise<ProjectSummary[]>;
|
|
@@ -1,43 +1,63 @@
|
|
|
1
1
|
import { toEdge, toSymbol } from './rows.js';
|
|
2
|
+
const CHUNK = 500;
|
|
3
|
+
function lastById(rows, key) {
|
|
4
|
+
const byId = new Map();
|
|
5
|
+
for (const row of rows)
|
|
6
|
+
byId.set(key(row), row);
|
|
7
|
+
return [...byId.values()];
|
|
8
|
+
}
|
|
9
|
+
function placeholders(count, width) {
|
|
10
|
+
const groups = [];
|
|
11
|
+
for (let row = 0; row < count; row += 1) {
|
|
12
|
+
const holes = [];
|
|
13
|
+
for (let col = 1; col <= width; col += 1)
|
|
14
|
+
holes.push(`$${row * width + col}`);
|
|
15
|
+
groups.push(`(${holes.join(', ')})`);
|
|
16
|
+
}
|
|
17
|
+
return groups.join(', ');
|
|
18
|
+
}
|
|
19
|
+
async function upsertChunked(client, rows, width, sql, columns) {
|
|
20
|
+
for (let start = 0; start < rows.length; start += CHUNK) {
|
|
21
|
+
const slice = rows.slice(start, start + CHUNK);
|
|
22
|
+
const values = [];
|
|
23
|
+
for (const row of slice)
|
|
24
|
+
values.push(...columns(row));
|
|
25
|
+
await client.query(sql(placeholders(slice.length, width)), values);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
2
28
|
export async function upsertGraph(pool, scan) {
|
|
3
29
|
const client = await pool.connect();
|
|
4
30
|
try {
|
|
5
31
|
await client.query('BEGIN');
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
32
|
+
await upsertChunked(client, lastById(scan.symbols, (s) => s.id), 8, (values) => `INSERT INTO symbols (id, project, name, kind, file, line, lang, signature)
|
|
33
|
+
VALUES ${values}
|
|
9
34
|
ON CONFLICT (id) DO UPDATE SET
|
|
10
35
|
name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
|
|
11
36
|
line = EXCLUDED.line, lang = EXCLUDED.lang,
|
|
12
|
-
signature = EXCLUDED.signature`, [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
|
|
13
|
-
|
|
14
|
-
for (const e of scan.edges) {
|
|
15
|
-
await client.query(`INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
|
|
37
|
+
signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
|
|
38
|
+
await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
|
|
16
39
|
relation, confidence, score, file, line)
|
|
17
|
-
VALUES
|
|
40
|
+
VALUES ${values}
|
|
18
41
|
ON CONFLICT (id) DO UPDATE SET
|
|
19
42
|
dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
|
|
20
43
|
relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
|
|
21
|
-
score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, [
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
await client.query(`INSERT INTO scanned_files (project, path, hash, scanned_at)
|
|
37
|
-
VALUES ($1, $2, $3, $4)
|
|
44
|
+
score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
|
|
45
|
+
e.id,
|
|
46
|
+
e.project,
|
|
47
|
+
e.srcId,
|
|
48
|
+
e.srcName,
|
|
49
|
+
e.dstId,
|
|
50
|
+
e.dstName,
|
|
51
|
+
e.relation,
|
|
52
|
+
e.confidence,
|
|
53
|
+
e.score,
|
|
54
|
+
e.file,
|
|
55
|
+
e.line,
|
|
56
|
+
]);
|
|
57
|
+
await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
|
|
58
|
+
VALUES ${values}
|
|
38
59
|
ON CONFLICT (project, path) DO UPDATE SET
|
|
39
|
-
hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, [f.project, f.path, f.hash, f.scannedAt]);
|
|
40
|
-
}
|
|
60
|
+
hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
|
|
41
61
|
await client.query('COMMIT');
|
|
42
62
|
}
|
|
43
63
|
catch (error) {
|
|
@@ -17,6 +17,7 @@ export declare class PostgresStore implements MemoryStore {
|
|
|
17
17
|
writeMeta(): Promise<void>;
|
|
18
18
|
ensureTagsIndexed(): Promise<void>;
|
|
19
19
|
readVectorDims(): Promise<number | null>;
|
|
20
|
+
migrateVectorDims(dims: number): Promise<boolean>;
|
|
20
21
|
private ensureVectorColumn;
|
|
21
22
|
close(): Promise<void>;
|
|
22
23
|
ping(): Promise<boolean>;
|
|
@@ -47,6 +47,23 @@ export class PostgresStore {
|
|
|
47
47
|
async readVectorDims() {
|
|
48
48
|
return metaOps.readVectorDims(this.pool);
|
|
49
49
|
}
|
|
50
|
+
async migrateVectorDims(dims) {
|
|
51
|
+
if (!this.vectorEnabled)
|
|
52
|
+
return false;
|
|
53
|
+
if (!Number.isInteger(dims) || dims < 1 || dims > 16000)
|
|
54
|
+
return false;
|
|
55
|
+
if (this.vectorDims === dims)
|
|
56
|
+
return false;
|
|
57
|
+
await this.pool.query('ALTER TABLE observations DROP COLUMN IF EXISTS embedding');
|
|
58
|
+
await this.pool.query(`ALTER TABLE observations ADD COLUMN embedding vector(${dims})`);
|
|
59
|
+
await this.pool.query('UPDATE observations SET embedder = NULL');
|
|
60
|
+
await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
|
|
61
|
+
ON observations USING hnsw (embedding vector_cosine_ops)`);
|
|
62
|
+
this.vectorDims = await this.readVectorDims();
|
|
63
|
+
this.warnedDims = false;
|
|
64
|
+
await this.writeMeta();
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
50
67
|
async ensureVectorColumn(dims) {
|
|
51
68
|
if (!this.vectorEnabled)
|
|
52
69
|
return false;
|
|
@@ -7,6 +7,7 @@ export async function insertObservations(pool, ensureVectorColumn, observations)
|
|
|
7
7
|
try {
|
|
8
8
|
await client.query('BEGIN');
|
|
9
9
|
for (const obs of observations) {
|
|
10
|
+
const vectorStored = vectors && obs.embedding?.length === width;
|
|
10
11
|
const columns = [
|
|
11
12
|
'id',
|
|
12
13
|
'session_id',
|
|
@@ -31,11 +32,11 @@ export async function insertObservations(pool, ensureVectorColumn, observations)
|
|
|
31
32
|
JSON.stringify(obs.files),
|
|
32
33
|
JSON.stringify(obs.tags),
|
|
33
34
|
obs.createdAt,
|
|
34
|
-
obs.embedder ?? null,
|
|
35
|
+
vectorStored ? (obs.embedder ?? null) : null,
|
|
35
36
|
obs.author ?? null,
|
|
36
37
|
obs.status ?? 'done',
|
|
37
38
|
];
|
|
38
|
-
if (
|
|
39
|
+
if (vectorStored && obs.embedding) {
|
|
39
40
|
columns.push('embedding');
|
|
40
41
|
values.push(`[${obs.embedding.join(',')}]`);
|
|
41
42
|
}
|
|
@@ -11,6 +11,19 @@ export function toSession(row) {
|
|
|
11
11
|
session.summary = row['summary'];
|
|
12
12
|
return session;
|
|
13
13
|
}
|
|
14
|
+
function toVector(value) {
|
|
15
|
+
if (Array.isArray(value))
|
|
16
|
+
return value;
|
|
17
|
+
if (typeof value !== 'string')
|
|
18
|
+
return null;
|
|
19
|
+
const trimmed = value.trim();
|
|
20
|
+
if (!trimmed.startsWith('[') || !trimmed.endsWith(']'))
|
|
21
|
+
return null;
|
|
22
|
+
const inner = trimmed.slice(1, -1);
|
|
23
|
+
if (inner.length === 0)
|
|
24
|
+
return null;
|
|
25
|
+
return inner.split(',').map(Number);
|
|
26
|
+
}
|
|
14
27
|
export function toObservation(row) {
|
|
15
28
|
const obs = {
|
|
16
29
|
id: row['id'],
|
|
@@ -23,6 +36,9 @@ export function toObservation(row) {
|
|
|
23
36
|
tags: row['tags'] ?? [],
|
|
24
37
|
createdAt: Number(row['created_at']),
|
|
25
38
|
};
|
|
39
|
+
const embedding = toVector(row['embedding']);
|
|
40
|
+
if (embedding)
|
|
41
|
+
obs.embedding = embedding;
|
|
26
42
|
if (row['embedder'] != null)
|
|
27
43
|
obs.embedder = row['embedder'];
|
|
28
44
|
if (row['author'] != null)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-db",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.4",
|
|
4
4
|
"description": "Persistent memory for Claude Code. Captures why decisions were made, not just which files changed. Bring your own database: SQLite, MongoDB or Postgres. No cloud, no subscription.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|