claude-db 0.5.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/README.md +74 -45
  3. package/dist/capture/flush.js +16 -8
  4. package/dist/cli/commands/graph.js +14 -2
  5. package/dist/cli/commands/transfer.d.ts +1 -1
  6. package/dist/cli/commands/transfer.js +4 -2
  7. package/dist/cli/index.js +1 -1
  8. package/dist/cli/install.js +1 -0
  9. package/dist/cli/usage.js +1 -1
  10. package/dist/config/schema.d.ts +2 -1
  11. package/dist/config/schema.js +2 -1
  12. package/dist/graph/index.d.ts +2 -2
  13. package/dist/graph/index.js +2 -2
  14. package/dist/graph/query/format.js +7 -3
  15. package/dist/graph/query/index.d.ts +1 -0
  16. package/dist/graph/query/index.js +1 -0
  17. package/dist/graph/query/lookup.d.ts +1 -0
  18. package/dist/graph/query/lookup.js +6 -0
  19. package/dist/graph/query/suggest.d.ts +5 -0
  20. package/dist/graph/query/suggest.js +65 -0
  21. package/dist/graph/scan/files.d.ts +1 -0
  22. package/dist/graph/scan/files.js +8 -2
  23. package/dist/graph/scan/index.d.ts +1 -1
  24. package/dist/graph/scan/index.js +1 -1
  25. package/dist/hooks/grep-symbols.d.ts +3 -0
  26. package/dist/hooks/grep-symbols.js +22 -0
  27. package/dist/hooks/prefer-usages.d.ts +2 -0
  28. package/dist/hooks/prefer-usages.js +58 -0
  29. package/dist/mcp/server.js +6 -170
  30. package/dist/mcp/tools/kinds.d.ts +1 -0
  31. package/dist/mcp/tools/kinds.js +1 -0
  32. package/dist/mcp/tools/memory.d.ts +3 -0
  33. package/dist/mcp/tools/memory.js +44 -0
  34. package/dist/mcp/tools/observations.d.ts +3 -0
  35. package/dist/mcp/tools/observations.js +55 -0
  36. package/dist/mcp/tools/search.d.ts +3 -0
  37. package/dist/mcp/tools/search.js +54 -0
  38. package/dist/mcp/tools/usages.d.ts +3 -0
  39. package/dist/mcp/tools/usages.js +79 -0
  40. package/dist/search/service.js +3 -0
  41. package/dist/search/stopwords.d.ts +2 -0
  42. package/dist/search/stopwords.js +137 -0
  43. package/dist/store/sqlite/filters.js +3 -5
  44. package/dist/usages/find.d.ts +1 -1
  45. package/dist/usages/find.js +5 -3
  46. package/docs/how-it-works.md +12 -12
  47. package/docs/with-and-without.md +123 -0
  48. package/package.json +3 -1
  49. package/docs/mcp-findings.md +0 -391
  50. package/docs/token-economics.md +0 -216
@@ -0,0 +1,137 @@
1
+ const STOPWORDS = new Set([
2
+ 'a',
3
+ 'about',
4
+ 'after',
5
+ 'all',
6
+ 'also',
7
+ 'am',
8
+ 'an',
9
+ 'and',
10
+ 'any',
11
+ 'are',
12
+ 'as',
13
+ 'at',
14
+ 'be',
15
+ 'because',
16
+ 'been',
17
+ 'before',
18
+ 'being',
19
+ 'between',
20
+ 'both',
21
+ 'but',
22
+ 'by',
23
+ 'can',
24
+ 'could',
25
+ 'did',
26
+ 'do',
27
+ 'does',
28
+ 'doing',
29
+ 'done',
30
+ 'down',
31
+ 'during',
32
+ 'each',
33
+ 'even',
34
+ 'for',
35
+ 'from',
36
+ 'further',
37
+ 'get',
38
+ 'got',
39
+ 'had',
40
+ 'has',
41
+ 'have',
42
+ 'having',
43
+ 'he',
44
+ 'her',
45
+ 'here',
46
+ 'hers',
47
+ 'him',
48
+ 'his',
49
+ 'how',
50
+ 'however',
51
+ 'i',
52
+ 'if',
53
+ 'in',
54
+ 'into',
55
+ 'is',
56
+ 'it',
57
+ 'its',
58
+ 'itself',
59
+ 'just',
60
+ 'may',
61
+ 'me',
62
+ 'might',
63
+ 'more',
64
+ 'most',
65
+ 'much',
66
+ 'must',
67
+ 'my',
68
+ 'no',
69
+ 'nor',
70
+ 'not',
71
+ 'now',
72
+ 'of',
73
+ 'off',
74
+ 'on',
75
+ 'once',
76
+ 'only',
77
+ 'or',
78
+ 'other',
79
+ 'our',
80
+ 'out',
81
+ 'over',
82
+ 'own',
83
+ 'said',
84
+ 'same',
85
+ 'she',
86
+ 'should',
87
+ 'so',
88
+ 'some',
89
+ 'such',
90
+ 'than',
91
+ 'that',
92
+ 'the',
93
+ 'their',
94
+ 'them',
95
+ 'then',
96
+ 'there',
97
+ 'these',
98
+ 'they',
99
+ 'this',
100
+ 'those',
101
+ 'through',
102
+ 'to',
103
+ 'too',
104
+ 'under',
105
+ 'until',
106
+ 'up',
107
+ 'use',
108
+ 'used',
109
+ 'using',
110
+ 'very',
111
+ 'was',
112
+ 'we',
113
+ 'were',
114
+ 'what',
115
+ 'when',
116
+ 'where',
117
+ 'which',
118
+ 'while',
119
+ 'who',
120
+ 'whom',
121
+ 'why',
122
+ 'will',
123
+ 'with',
124
+ 'would',
125
+ 'you',
126
+ 'your',
127
+ 'yours',
128
+ ]);
129
+ export function isStopword(token) {
130
+ return STOPWORDS.has(token);
131
+ }
132
+ export function meaningfulTokens(text) {
133
+ return text
134
+ .toLowerCase()
135
+ .split(/[^\p{L}\p{N}_]+/u)
136
+ .filter((token) => token.length > 1 && !isStopword(token));
137
+ }
@@ -1,6 +1,7 @@
1
1
  import { partitionIds } from '../../util/shortid.js';
2
2
  import { resolve } from 'node:path';
3
3
  import { scopeToken } from '../../util/scope.js';
4
+ import { meaningfulTokens } from '../../search/stopwords.js';
4
5
  export function removeWhere(filter) {
5
6
  const conditions = [];
6
7
  const params = [];
@@ -58,13 +59,10 @@ export function appendScope(query, conditions, params, prefix) {
58
59
  }
59
60
  }
60
61
  export function toMatchExpression(text, project) {
61
- const tokens = text
62
- .toLowerCase()
63
- .split(/[^\p{L}\p{N}_]+/u)
64
- .filter((token) => token.length > 1);
62
+ const tokens = meaningfulTokens(text);
65
63
  const scope = project ? `scope:${scopeToken(project)}` : null;
66
64
  if (tokens.length === 0)
67
- return scope;
65
+ return null;
68
66
  const terms = tokens.map((token) => `"${token}"`).join(' OR ');
69
67
  return scope ? `${scope} AND (${terms})` : terms;
70
68
  }
@@ -20,4 +20,4 @@ export interface FindUsagesInput {
20
20
  }
21
21
  export declare function findUsages(input: FindUsagesInput): UsagesResult;
22
22
  export declare function repoRootFor(start: string): string;
23
- export declare function formatUsages(result: UsagesResult): string;
23
+ export declare function formatUsages(result: UsagesResult, suggestions?: string[]): string;
@@ -116,9 +116,11 @@ function isDefinitionLike(symbol, text) {
116
116
  `|(?:export\\s+)?(?:const|let|var)\\s+${s}\\s*[:=]` +
117
117
  `|export\\s*\\{[^}]*\\b${s}\\b)`).test(text);
118
118
  }
119
- export function formatUsages(result) {
120
- if (result.matches.length === 0)
121
- return `No usages of that symbol found under ${result.root}.`;
119
+ export function formatUsages(result, suggestions = []) {
120
+ if (result.matches.length === 0) {
121
+ return (`No usages of that symbol found under ${result.root}.` +
122
+ (suggestions.length > 0 ? `\nDid you mean: ${suggestions.join(', ')}` : ''));
123
+ }
122
124
  const rows = result.matches.map((m) => `${m.file}${m.isMatch ? ':' : '-'}${m.line}${m.isDefinition ? ' [definition?]' : ''} ${m.text.trim()}`);
123
125
  const shown = result.matches.filter((m) => m.isMatch).length;
124
126
  const header = result.truncated
@@ -6,7 +6,7 @@ graph stays honest.
6
6
 
7
7
  ## Capture
8
8
 
9
- Claude Code already writes every session to disk as JSONL — your prompts, its
9
+ Claude Code already writes every session to disk as JSONL: your prompts, its
10
10
  replies, every tool call. claude-db reads that file, so it captures **why** you
11
11
  did something, not just which files changed.
12
12
 
@@ -41,7 +41,7 @@ best match in full on each prompt. Roughly 350 tokens when something relevant
41
41
  exists, zero when it doesn't.
42
42
 
43
43
  **What search returns.** An id, kind, date, title and one line of the matching
44
- body — enough to tell two similarly-titled rows apart without expanding either.
44
+ body, enough to tell two similarly-titled rows apart without expanding either.
45
45
  Full bodies come only from `get_observations`, for ids you picked.
46
46
 
47
47
  One stored memory looks like this:
@@ -71,8 +71,8 @@ already landed, or a finished task would flicker back into the list every time
71
71
  a neighbouring line changed.
72
72
 
73
73
  `claude-db status` lists what is still open, and SessionStart injects the
74
- newest few. That is what lets a brand-new chat answer "what was I doing" —
75
- memory search ranks by relevance, and a prompt like "do the last task" contains
74
+ newest few. That is what lets a brand-new chat answer "what was I doing".
75
+ Memory search ranks by relevance, and a prompt like "do the last task" contains
76
76
  no words worth matching, so recency has to come from somewhere else.
77
77
 
78
78
  Rows captured before this existed default to `done`. Marking a whole existing
@@ -82,7 +82,7 @@ database unfinished would be worse than saying nothing.
82
82
 
83
83
  `claude-db install` copies two things to disk: the `/cdb-scan` skill, and a
84
84
  standing instruction block in `CLAUDE.local.md`. Claude Code only reads skills
85
- from `~/.claude/skills/`, so a copy has to exist there — but a copy drifts as
85
+ from `~/.claude/skills/`, so a copy has to exist there, but a copy drifts as
86
86
  soon as the package updates.
87
87
 
88
88
  The hooks are registered by absolute path into the installed package, so they
@@ -97,7 +97,7 @@ back, and anything `uninstall` removed stays removed.
97
97
  ## The code graph
98
98
 
99
99
  `claude-db scan` parses every supported source file and stores what it finds:
100
- each symbol, and each relationship between symbols — what calls what, what
100
+ each symbol, and each relationship between symbols: what calls what, what
101
101
  imports what, what extends what. Parsing is local and deterministic, costs no
102
102
  tokens, and takes about a second on a mid-size repository. A rescan only
103
103
  re-parses files whose contents changed.
@@ -117,7 +117,7 @@ Shortest path (4 hops):
117
117
  ```
118
118
 
119
119
  **Every edge carries its confidence.** `EXTRACTED` means the relationship was
120
- read literally out of the syntax tree — an import names its own target, so
120
+ read literally out of the syntax tree. An import names its own target, so
121
121
  nothing is guessed. `INFERRED` means the target was matched by name across
122
122
  files, which is wrong when two files export the same identifier, and carries a
123
123
  score saying how sure the match was. Name matching is the one guess this makes,
@@ -131,7 +131,7 @@ have been scanned, but correctness does not depend on it having run.
131
131
 
132
132
  Traversal is keyed on symbol id rather than name. Keying by name would merge
133
133
  every same-named symbol into one node, and a repository with a `run` in each
134
- test file would then grow shortcuts between unrelated code — making the
134
+ test file would then grow shortcuts between unrelated code, making the
135
135
  shortest path a route nothing can actually take.
136
136
 
137
137
  Languages: TypeScript, TSX, JavaScript, Python, Go and Rust. The parser is
@@ -141,8 +141,8 @@ nothing compiles at install and nothing else has to be installed.
141
141
  ## The `/cdb-scan` skill
142
142
 
143
143
  Installed alongside the hooks, it runs two passes: `claude-db scan` for the
144
- graph, then five written notes — stack, layout, conventions, workflows,
145
- architecture — stored under stable keys so a re-run updates them in place.
144
+ graph, then five written notes (stack, layout, conventions, workflows,
145
+ architecture) stored under stable keys so a re-run updates them in place.
146
146
 
147
147
  The graph records what the code _is_; the notes record why it is built that
148
148
  way. Those notes are tagged `inferred`, because they are Claude's reading of
@@ -154,7 +154,7 @@ database is testimony, and a search result must not blur the two.
154
154
  SQLite by default, at `~/.claude-memory/memory.db`. The connection string picks
155
155
  the backend, and `CLAUDE_DB_URL` overrides the config file.
156
156
 
157
- Search is hybrid — keyword plus vector — on all three backends. Ranking, fusion
157
+ Search is hybrid, keyword plus vector, on all three backends. Ranking, fusion
158
158
  and token budgeting live in `src/search` rather than in the adapters, so recall
159
159
  behaves identically no matter which database is plugged in; only retrieval cost
160
160
  changes. Memory is partitioned by project path, so one database serves every
@@ -179,5 +179,5 @@ Atlas indexes are not created by this tool. Add one by hand on the
179
179
  ```
180
180
 
181
181
  `numDimensions` is 256 for the builtin embedder, 384 once
182
- `@xenova/transformers` is installed. Without this index Mongo still works —
182
+ `@xenova/transformers` is installed. Without this index Mongo still works,
183
183
  vector search falls back to scoring candidates in process.
@@ -0,0 +1,123 @@
1
+ # With claude-db and without it
2
+
3
+ Does it actually save tokens? Here is the same work measured both ways, so you
4
+ can decide before you install.
5
+
6
+ Run `npm run bench:ab` from a clone to reproduce all of it on your own
7
+ codebase. Tokens are approximated at 4 characters each.
8
+
9
+ ## The short answer
10
+
11
+ **One lookup pays for about three prompts of recall.**
12
+
13
+ Recall costs ~180 tokens on every prompt. A symbol lookup saves ~600 tokens
14
+ against doing the same work by hand. Ask your codebase something once every few
15
+ prompts and you are ahead. Never ask it anything and you are paying for context
16
+ you did not use.
17
+
18
+ ## Looking up a symbol
19
+
20
+ The question is _who uses this, and how?_ Without claude-db that means a grep,
21
+ then opening the file it points into to see which hits are definitions, which
22
+ are calls, and which are noise. With it, one command returns the same answer
23
+ already classified.
24
+
25
+ Measured on eight real symbols in this repository:
26
+
27
+ ```
28
+ symbol WITH grep +read WITHOUT
29
+ ---------------------------------------------------------
30
+ isSearchable 1734 1228 1153 2381
31
+ observationsFromTurns 2042 2362 7529 9891
32
+ closeObservations 3209 1354 6597 7951
33
+ flushSession 1786 818 4442 5260
34
+ refreshGraph 2640 1143 961 2104
35
+ observationId 2020 2418 572 2990
36
+ toSnippet 1994 1789 704 2493
37
+ openWork 1166 1294 1305 2599
38
+ ---------------------------------------------------------
39
+ TOTAL chars 16591 35669
40
+ TOTAL tokens 4148 8917
41
+ 2.1x
42
+ ```
43
+
44
+ **518 tokens per lookup instead of 1,115.**
45
+
46
+ The average hides something useful. `refreshGraph` and `observationId` are
47
+ cheaper _without_ claude-db, because their defining files are small and grep
48
+ answers them fine. `observationsFromTurns` and `closeObservations` are four to
49
+ five times cheaper with it, because answering those by hand means reading a
50
+ 200-line file, or three of them spread across different modules.
51
+
52
+ The win concentrates on questions that span files, which is also where a wrong
53
+ answer costs you the most.
54
+
55
+ ## Asking why something is the way it is
56
+
57
+ Some questions have no grep equivalent at all. _Why does capture read the
58
+ transcript instead of hooking the tools?_ leaves no trace in the code. The
59
+ closest substitute is `git log -S` plus reading diffs, and that tells you what
60
+ changed, never why.
61
+
62
+ Measured against that substitute the two come out roughly level, but they are
63
+ not answering the same question. The real cost without stored memory is
64
+ explaining it to Claude again yourself, which no benchmark can price.
65
+
66
+ ## What it costs
67
+
68
+ Recall arrives automatically on every prompt, so it has a standing cost. What
69
+ arrives is index lines, never full bodies: an id, a kind, a date, a title, and
70
+ the line that matched. Claude pulls a full body only when a title earns it.
71
+
72
+ ```
73
+ every prompt costs 180 tokens of recall
74
+ every lookup refunds 597 tokens (1,115 by hand, 518 with)
75
+
76
+ so one lookup pays for 3.3 prompts of recall
77
+ ```
78
+
79
+ On 28% of prompts nothing is injected at all, because everything matching is
80
+ already in the conversation.
81
+
82
+ ```
83
+ a session of recall costs lookups to break even
84
+ ------------------------------------------------------
85
+ 1 prompt 180 0.3
86
+ 5 prompts 900 1.5
87
+ 10 prompts 1,800 3.0
88
+ 20 prompts 3,600 6.0
89
+ 60 prompts 10,800 18.1
90
+ ```
91
+
92
+ ## When it does not pay off
93
+
94
+ Worth saying plainly, because it is a real case: a session where you never ask
95
+ anything about the codebase pays for recall and gets nothing back for it.
96
+
97
+ Three things push you the other way. Refactoring, where every rename wants a
98
+ blast-radius check. Returning to a project after a break, where the stored
99
+ reasoning is the point. And any codebase large enough that "who calls this"
100
+ means opening several files rather than one.
101
+
102
+ If your sessions are short and mostly conversational, turn the per-prompt
103
+ recall off and keep the code graph:
104
+
105
+ ```json
106
+ { "inject": { "perPrompt": false } }
107
+ ```
108
+
109
+ Session-start context still arrives, lookups still work, and the standing cost
110
+ goes to zero.
111
+
112
+ ## Reproducing this
113
+
114
+ ```bash
115
+ git clone https://github.com/Avijit07x/claude-db && cd claude-db
116
+ npm install && npm run build
117
+
118
+ npm run bench:ab # this document, on your own repo
119
+ npm run bench:tokens # where the injected tokens go
120
+ ```
121
+
122
+ The grep side excludes markdown, so both sides see the same code rather than
123
+ counting prose that happens to mention a symbol name.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "claude-db",
3
- "version": "0.5.3",
3
+ "version": "0.6.0",
4
4
  "description": "Persistent memory for Claude Code. Captures why decisions were made, not just which files changed. Bring your own database: SQLite, MongoDB or Postgres. No cloud, no subscription.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -35,6 +35,8 @@
35
35
  "local": "node scripts/local.mjs",
36
36
  "try": "node scripts/try.mjs",
37
37
  "bench": "node scripts/bench.mjs",
38
+ "bench:tokens": "node scripts/bench-tokens.mjs",
39
+ "bench:ab": "node scripts/bench-ab.mjs",
38
40
  "postinstall": "node -e \"const v=process.versions.node.split('.').map(Number); if(v[0]<22||(v[0]===22&&v[1]<5)){console.error('\\n claude-db needs Node >= 22.5 (node:sqlite is builtin from there). You have '+process.versions.node+'.\\n');process.exit(1)}\"",
39
41
  "prepublishOnly": "npm run build && npm test",
40
42
  "format": "prettier --write .",