claude-db 0.12.2 → 0.12.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,27 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.12.3
4
+
5
+ ### Fixed
6
+
7
+ - **`distill` no longer stops on Postgres.** A Claude memory file with no `modified` time made it fail with
8
+ `invalid input syntax for type bigint`, so chats waited for days. Times are now whole milliseconds.
9
+ - **Rows saved without a vector get one later.** A row is saved as text only when the embedding model is still
10
+ loading, or when its width differs from the Postgres column. The next chat start fills such rows in the
11
+ background, at most every 6 hours, once an embedder of the right width loads. It never replaces a vector from
12
+ another embedder.
13
+ - **Background jobs keep their output.** It goes to `~/.claude-memory/logs/background.log`, which rolls over at
14
+ 1 MB.
15
+ - **The waiting-chats warning names `claude-db distill --backfill`**, which works through them.
16
+ - **`claude-db languages` lists TypeScript, TSX and JavaScript** as built in.
17
+
18
+ ### Changed
19
+
20
+ - **A grep for a symbol that is not in the graph is faster.** The hook no longer runs a "did you mean" scan it
21
+ never shows.
22
+ - **`claude-db adoption` is faster.** It looks each symbol up once, several at a time, and shows a counter on a
23
+ terminal.
24
+
3
25
  ## 0.12.2
4
26
 
5
27
  ### Fixed
@@ -4,6 +4,7 @@ import { CONFIG_DIR } from '../config/index.js';
4
4
  import { readTranscript, sessionIdsOnDisk, transcriptPathFor } from './transcript.js';
5
5
  import { observationsFromTurns } from './turn-extractor.js';
6
6
  import { closeLandedWork } from './progress.js';
7
+ import { noteMissingVectors } from './vectors.js';
7
8
  export async function flushSession(ctx, sessionId, project, transcriptPath, rebuild = false, finalReply = '') {
8
9
  const path = transcriptPath ?? transcriptPathFor(project, sessionId);
9
10
  const cursor = readCursor(sessionId);
@@ -37,10 +38,15 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
37
38
  return { observations: observations.length, summary, lastReply };
38
39
  }
39
40
  export async function embedObservations(ctx, observations) {
41
+ if (observations.length === 0)
42
+ return;
40
43
  try {
41
44
  const embedder = await ctx.embedder();
42
- if (embedder.dimensions === 0)
45
+ if (embedder.dimensions === 0) {
46
+ if (ctx.config.embeddings.provider !== 'none')
47
+ noteMissingVectors(observations);
43
48
  return;
49
+ }
44
50
  const size = ctx.config.embeddings.batchSize;
45
51
  for (let start = 0; start < observations.length; start += size) {
46
52
  const chunk = observations.slice(start, start + size);
@@ -53,8 +59,11 @@ export async function embedObservations(ctx, observations) {
53
59
  }
54
60
  });
55
61
  }
62
+ const stored = ctx.store.storedVectorDims?.() ?? null;
63
+ noteMissingVectors(observations, stored !== null && stored !== embedder.dimensions);
56
64
  }
57
65
  catch (error) {
66
+ noteMissingVectors(observations);
58
67
  process.stderr.write(`[claude-db] embedding skipped: ` +
59
68
  `${error instanceof Error ? error.message : String(error)}\n`);
60
69
  }
@@ -10,6 +10,7 @@ export { redact } from './redact.js';
10
10
  export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
11
11
  export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
12
12
  export type { FlushResult } from './flush.js';
13
+ export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
13
14
  export { closeLandedWork, openWork } from './progress.js';
14
15
  export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
15
16
  export type { ProjectReingest, SessionReingest } from './reingest.js';
@@ -7,6 +7,7 @@ export { observationsFromTurns } from './turn-extractor.js';
7
7
  export { redact } from './redact.js';
8
8
  export { claimScrub, finishScrub, releaseScrub, scrubSecrets } from './scrub.js';
9
9
  export { flushSession, resetCursor, clearCursor, sweepCursors, summarize, embedObservations, } from './flush.js';
10
+ export { claimReembed, finishReembed, releaseReembed, startReembed, vectorsMissing, } from './vectors.js';
10
11
  export { closeLandedWork, openWork } from './progress.js';
11
12
  export { REINGEST_VERSION, claimReingest, finishReingest, reingestDone, reingestProject, rememberedTranscripts, reingestSession, releaseReingest, } from './reingest.js';
12
13
  export { aiSummary, buildSummaryPrompt, validateSummary } from './summarize.js';
@@ -0,0 +1,7 @@
1
+ import type { Observation } from '../types.js';
2
+ export declare function noteMissingVectors(observations: Observation[], everyRow?: boolean): void;
3
+ export declare const vectorsMissing: (project: string) => boolean;
4
+ export declare function claimReembed(project: string, now?: number): boolean;
5
+ export declare function startReembed(project: string): void;
6
+ export declare function finishReembed(project: string, now?: number): void;
7
+ export declare function releaseReembed(project: string): void;
@@ -0,0 +1,38 @@
1
+ import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { CONFIG_DIR } from '../config/index.js';
4
+ import { claimJob, finishJob, jobMark, releaseJob } from '../util/job-lock.js';
5
+ import { scopeToken } from '../util/scope.js';
6
+ const JOB = 'reembed';
7
+ const FLAGS = 'vectors-missing';
8
+ const REEMBED_EVERY_MS = 6 * 60 * 60 * 1000;
9
+ const flagPath = (project) => join(CONFIG_DIR, FLAGS, `${scopeToken(project)}.flag`);
10
+ export function noteMissingVectors(observations, everyRow = false) {
11
+ const projects = new Set(observations.filter((obs) => everyRow || !obs.embedding?.length).map((obs) => obs.project));
12
+ for (const project of projects) {
13
+ try {
14
+ mkdirSync(join(CONFIG_DIR, FLAGS), { recursive: true });
15
+ writeFileSync(flagPath(project), '');
16
+ }
17
+ catch {
18
+ continue;
19
+ }
20
+ }
21
+ }
22
+ export const vectorsMissing = (project) => existsSync(flagPath(project));
23
+ export function claimReembed(project, now = Date.now()) {
24
+ if (!vectorsMissing(project))
25
+ return false;
26
+ if (now - jobMark(JOB, project) < REEMBED_EVERY_MS)
27
+ return false;
28
+ return claimJob(JOB, project, now);
29
+ }
30
+ export function startReembed(project) {
31
+ rmSync(flagPath(project), { force: true });
32
+ }
33
+ export function finishReembed(project, now = Date.now()) {
34
+ finishJob(JOB, project, now);
35
+ }
36
+ export function releaseReembed(project) {
37
+ releaseJob(JOB, project);
38
+ }
@@ -4,6 +4,8 @@ import { queryGraph } from '../../graph/index.js';
4
4
  import { transcriptsFor } from '../../capture/transcript.js';
5
5
  import { resolveProject } from '../../util/project.js';
6
6
  import { DECLARED, isSymbol, symbolsGreppedIn } from '../../hooks/grep-symbols.js';
7
+ const MAX_SYMBOLS = 2;
8
+ const LOOKUPS_IN_FLIGHT = 8;
7
9
  export async function cmdAdoption() {
8
10
  const project = resolveProject(undefined);
9
11
  const files = transcriptsFor(project);
@@ -45,35 +47,8 @@ export async function cmdAdoption() {
45
47
  }
46
48
  const greps = bash.filter((command) => /\b(?:grep|rg)\b/.test(command));
47
49
  const candidates = greps.map((command) => symbolsGreppedIn(command)).filter((s) => s.length > 0);
48
- const ctx = await createContext();
49
- let fires = 0;
50
- const cache = new Map();
51
- try {
52
- for (const symbols of candidates) {
53
- let hit = false;
54
- for (const symbol of symbols.slice(0, 2)) {
55
- let fired = cache.get(symbol);
56
- if (fired === undefined) {
57
- const answer = await queryGraph(ctx.store, project, {
58
- mode: 'usages',
59
- symbol,
60
- limit: 20,
61
- });
62
- fired =
63
- !answer.empty &&
64
- (isSymbol(symbol) || answer.definitions.some((d) => DECLARED.has(d.kind)));
65
- cache.set(symbol, fired);
66
- }
67
- if (fired)
68
- hit = true;
69
- }
70
- if (hit)
71
- fires++;
72
- }
73
- }
74
- finally {
75
- await ctx.close();
76
- }
50
+ const firing = await symbolsThatFire(project, candidates.flatMap((symbols) => symbols.slice(0, MAX_SYMBOLS)));
51
+ const fires = candidates.filter((symbols) => symbols.slice(0, MAX_SYMBOLS).some((symbol) => firing.has(symbol))).length;
77
52
  const memoryTotal = [...memory.values()].reduce((a, b) => a + b, 0);
78
53
  const pct = (n, of) => (of === 0 ? '0.0%' : `${((n / of) * 100).toFixed(1)}%`);
79
54
  console.log(`Adoption for ${project} (${files.length} transcripts)\n`);
@@ -92,3 +67,40 @@ export async function cmdAdoption() {
92
67
  console.log(`\n ${ratio}`);
93
68
  }
94
69
  }
70
+ async function symbolsThatFire(project, symbols) {
71
+ const unique = [...new Set(symbols)];
72
+ const firing = new Set();
73
+ if (unique.length === 0)
74
+ return firing;
75
+ const ctx = await createContext();
76
+ const tty = process.stderr.isTTY === true;
77
+ let next = 0;
78
+ let done = 0;
79
+ const lookup = async () => {
80
+ while (next < unique.length) {
81
+ const symbol = unique[next++];
82
+ const answer = await queryGraph(ctx.store, project, {
83
+ mode: 'usages',
84
+ symbol,
85
+ limit: 20,
86
+ suggest: false,
87
+ });
88
+ if (!answer.empty &&
89
+ (isSymbol(symbol) || answer.definitions.some((d) => DECLARED.has(d.kind)))) {
90
+ firing.add(symbol);
91
+ }
92
+ done += 1;
93
+ if (tty)
94
+ process.stderr.write(`\rchecking symbols ${done}/${unique.length}`);
95
+ }
96
+ };
97
+ try {
98
+ await Promise.all(Array.from({ length: Math.min(LOOKUPS_IN_FLIGHT, unique.length) }, () => lookup()));
99
+ }
100
+ finally {
101
+ if (tty)
102
+ process.stderr.write('\r\x1b[K');
103
+ await ctx.close();
104
+ }
105
+ return firing;
106
+ }
@@ -1,13 +1,17 @@
1
1
  import { GRAMMARS, PLATFORM, hasGrammar } from '../../graph/grammars.js';
2
- import { languageNames } from '../../graph/index.js';
2
+ import { LANGUAGES, languageNames } from '../../graph/index.js';
3
3
  const SHIPPED = 'Grammars ship with claude-db for each platform, so there is nothing to add or remove.';
4
- const stateOf = (name) => hasGrammar(name)
5
- ? 'read with real syntax'
6
- : `read by pattern: no grammar for ${PLATFORM} in this install`;
4
+ const stateOf = (name) => {
5
+ if (!GRAMMARS.includes(name))
6
+ return 'read with real syntax, built in';
7
+ return hasGrammar(name)
8
+ ? 'read with real syntax'
9
+ : `read by pattern: no grammar for ${PLATFORM} in this install`;
10
+ };
7
11
  function listLanguages() {
8
12
  console.log(`read with real syntax: ${languageNames()}`);
9
- for (const name of GRAMMARS)
10
- console.log(` ${name.padEnd(8)} ${stateOf(name)}`);
13
+ for (const { id, label } of LANGUAGES)
14
+ console.log(` ${label.padEnd(10)} ${stateOf(id)}`);
11
15
  if (GRAMMARS.some((name) => !hasGrammar(name))) {
12
16
  console.log('Reinstall claude-db to add the grammars for this platform.');
13
17
  }
@@ -1,7 +1,7 @@
1
1
  import { BATCH } from '../constants.js';
2
2
  const SESSION_LIMIT = 10000;
3
3
  import { createContext } from '../../context.js';
4
- import { embedObservations } from '../../capture/index.js';
4
+ import { embedObservations, finishReembed, releaseReembed, startReembed, } from '../../capture/index.js';
5
5
  import { loadConfig } from '../../config/index.js';
6
6
  import { readFileSync } from 'node:fs';
7
7
  import { resolveProject } from '../../util/project.js';
@@ -127,39 +127,58 @@ export async function cmdPrune(argv) {
127
127
  }
128
128
  export async function cmdReembed(argv = []) {
129
129
  const scoped = argv.includes('--project') || argv.includes('-p');
130
+ const background = argv.includes('--background');
131
+ const project = resolveProject(undefined);
130
132
  const base = loadConfig();
131
133
  const ctx = await createContext({ embeddings: { ...base.embeddings, timeoutMs: 0 } });
134
+ if (background)
135
+ startReembed(project);
136
+ let finished = false;
132
137
  try {
133
138
  const embedder = await ctx.embedder();
134
139
  if (embedder.dimensions === 0) {
135
140
  console.error('No embedder available; nothing to do.');
136
141
  process.exit(1);
137
142
  }
143
+ const stored = ctx.store.storedVectorDims?.() ?? null;
144
+ if (background && stored !== null && stored !== embedder.dimensions) {
145
+ finished = true;
146
+ return;
147
+ }
138
148
  let migrated = false;
139
149
  if (!scoped && ctx.store.migrateVectorDims) {
140
150
  migrated = await ctx.store.migrateVectorDims(embedder.dimensions);
141
151
  }
142
152
  let updated = 0;
143
153
  let skipped = 0;
144
- const filter = scoped ? { project: resolveProject(undefined) } : {};
154
+ const filter = scoped ? { project } : {};
145
155
  const scanned = await eachObservation(ctx, filter, async (batch) => {
146
- const stale = batch.filter((obs) => obs.embedder !== embedder.id || !obs.embedding?.length);
156
+ const stale = batch.filter((obs) => !obs.embedding?.length || (!background && obs.embedder !== embedder.id));
147
157
  skipped += batch.length - stale.length;
148
158
  if (stale.length === 0)
149
159
  return;
150
160
  await embedObservations(ctx, stale);
151
161
  await ctx.store.insertObservations(stale);
152
162
  updated += stale.length;
153
- process.stderr.write(`\r${updated} re-embedded...`);
163
+ if (!background)
164
+ process.stderr.write(`\r${updated} re-embedded...`);
154
165
  });
155
- process.stderr.write('\r');
166
+ if (!background)
167
+ process.stderr.write('\r');
156
168
  if (migrated) {
157
169
  console.log(`Rebuilt vector storage at ${embedder.dimensions}d.`);
158
170
  }
159
171
  console.log(`Scanned ${scanned}, re-embedded ${updated} with ${embedder.id}` +
160
172
  `${skipped > 0 ? `, ${skipped} already current` : ''}.`);
173
+ finished = true;
161
174
  }
162
175
  finally {
176
+ if (background) {
177
+ if (finished)
178
+ finishReembed(project);
179
+ else
180
+ releaseReembed(project);
181
+ }
163
182
  await ctx.close();
164
183
  }
165
184
  }
@@ -76,7 +76,7 @@ function memoryFiles(dir) {
76
76
  for (const name of names) {
77
77
  try {
78
78
  const path = join(dir, name);
79
- const parsed = parseMemoryFile(readFileSync(path, 'utf8'), statSync(path).mtimeMs);
79
+ const parsed = parseMemoryFile(readFileSync(path, 'utf8'), Math.trunc(statSync(path).mtimeMs));
80
80
  if (parsed)
81
81
  files.push(parsed);
82
82
  }
@@ -6,5 +6,6 @@ export function waitingWarning(waiting, oldestAt, usage, now = Date.now()) {
6
6
  return null;
7
7
  if (usage.used > 0 || usage.pausedUntil > now)
8
8
  return null;
9
- return `warning : ${waiting} chat(s) have waited over a day and no facts were tried today. Run claude-db doctor`;
9
+ return (`warning : ${waiting} chat(s) have waited over a day and no facts were tried today. ` +
10
+ 'Run claude-db distill --backfill to work through them now, or claude-db doctor to check Haiku');
10
11
  }
@@ -20,5 +20,6 @@ export interface GraphQuery {
20
20
  symbol: string;
21
21
  target?: string;
22
22
  limit: number;
23
+ suggest?: boolean;
23
24
  }
24
25
  export declare function queryGraph(store: MemoryStore, project: string, query: GraphQuery): Promise<GraphAnswer>;
@@ -56,7 +56,8 @@ export async function queryGraph(store, project, query) {
56
56
  answer.inbound = [...direct, ...(await throughAliases(store, project, direct, edgeLimit))];
57
57
  answer.outbound = edges.filter((edge) => idSet.has(edge.srcId) && !pointsAtSymbol(edge));
58
58
  answer.empty = answer.definitions.length === 0 && answer.inbound.length === 0;
59
- if (answer.empty)
59
+ if (answer.empty && query.suggest !== false) {
60
60
  answer.suggestions = await suggestFor(store, project, query.symbol);
61
+ }
61
62
  return answer;
62
63
  }
@@ -1,4 +1,4 @@
1
- import { claimReingest, claimScrub, reingestDone, releaseReingest, releaseScrub, } from '../capture/index.js';
1
+ import { claimReembed, claimReingest, claimScrub, reingestDone, releaseReembed, releaseReingest, releaseScrub, } from '../capture/index.js';
2
2
  import { FACTS_JOB } from '../facts/distill.js';
3
3
  import { claimJob, jobMark, releaseJob } from '../util/job-lock.js';
4
4
  import { CLI, PICK_WORKER, runDetached } from './detached.js';
@@ -14,6 +14,10 @@ export function startBackgroundWork(project, database, now = Date.now()) {
14
14
  }
15
15
  if (!reingestDone(project))
16
16
  return;
17
+ if (claimReembed(project, now)) {
18
+ runDetached(CLI, ['reembed', '--project', '--background'], project, () => releaseReembed(project));
19
+ return;
20
+ }
17
21
  if (now - jobMark(FACTS_JOB, project) < FACTS_EVERY_MS)
18
22
  return;
19
23
  if (!claimJob(FACTS_JOB, project, now))
@@ -1,4 +1,5 @@
1
1
  export declare const CLI: string;
2
2
  export declare const PICK_WORKER: string;
3
3
  export declare const TURN_WORKER: string;
4
+ export declare const backgroundLog: () => string;
4
5
  export declare function runDetached(script: string, args: string[], cwd: string, release: () => void): void;
@@ -1,17 +1,52 @@
1
1
  import { spawn } from 'node:child_process';
2
- import { dirname, resolve } from 'node:path';
2
+ import { appendFileSync, closeSync, mkdirSync, openSync, renameSync, statSync } from 'node:fs';
3
+ import { basename, dirname, join, resolve } from 'node:path';
3
4
  import { fileURLToPath } from 'node:url';
5
+ import { CONFIG_DIR } from '../config/dir.js';
4
6
  const DIST = resolve(dirname(fileURLToPath(import.meta.url)), '..');
5
7
  export const CLI = resolve(DIST, 'cli', 'index.js');
6
8
  export const PICK_WORKER = resolve(DIST, 'pick', 'worker.js');
7
9
  export const TURN_WORKER = resolve(DIST, 'hooks', 'turn-save.js');
10
+ const LOG_LIMIT = 1024 * 1024;
11
+ export const backgroundLog = () => join(CONFIG_DIR, 'logs', 'background.log');
12
+ function rotate(path) {
13
+ try {
14
+ if (statSync(path).size > LOG_LIMIT)
15
+ renameSync(path, `${path}.1`);
16
+ }
17
+ catch {
18
+ return;
19
+ }
20
+ }
21
+ function openLog(script, args) {
22
+ try {
23
+ const path = backgroundLog();
24
+ mkdirSync(dirname(path), { recursive: true });
25
+ rotate(path);
26
+ const name = script === CLI ? (args[0] ?? 'cli') : basename(script, '.js');
27
+ appendFileSync(path, `${new Date().toISOString()} ${name} started\n`);
28
+ return openSync(path, 'a');
29
+ }
30
+ catch {
31
+ return 'ignore';
32
+ }
33
+ }
8
34
  export function runDetached(script, args, cwd, release) {
35
+ const log = openLog(script, args);
9
36
  try {
10
- spawn(process.execPath, [script, ...args], { cwd, detached: true, stdio: 'ignore' })
37
+ spawn(process.execPath, [script, ...args], {
38
+ cwd,
39
+ detached: true,
40
+ stdio: ['ignore', log, log],
41
+ })
11
42
  .on('error', release)
12
43
  .unref();
13
44
  }
14
45
  catch {
15
46
  release();
16
47
  }
48
+ finally {
49
+ if (typeof log === 'number')
50
+ closeSync(log);
51
+ }
17
52
  }
@@ -57,7 +57,7 @@ async function answerSymbols(ctx, project, symbols) {
57
57
  store: ctx.store,
58
58
  root: project,
59
59
  project,
60
- query: { mode: 'usages', symbol, limit: EDGE_LIMIT },
60
+ query: { mode: 'usages', symbol, limit: EDGE_LIMIT, suggest: false },
61
61
  refresh: false,
62
62
  text: false,
63
63
  });
@@ -19,6 +19,7 @@ export interface MemoryStore {
19
19
  insertObservations(observations: Observation[]): Promise<void>;
20
20
  getObservations(ids: string[]): Promise<Observation[]>;
21
21
  migrateVectorDims?(dims: number): Promise<boolean>;
22
+ storedVectorDims?(): number | null;
22
23
  remove(filter: RemoveFilter): Promise<number>;
23
24
  list(filter: ListFilter): Promise<Observation[]>;
24
25
  listProjects(): Promise<ProjectSummary[]>;
@@ -18,6 +18,7 @@ export declare class PostgresStore implements MemoryStore {
18
18
  ensureTagsIndexed(): Promise<void>;
19
19
  readVectorDims(): Promise<number | null>;
20
20
  migrateVectorDims(dims: number): Promise<boolean>;
21
+ storedVectorDims(): number | null;
21
22
  private ensureVectorColumn;
22
23
  close(): Promise<void>;
23
24
  ping(): Promise<boolean>;
@@ -65,6 +65,9 @@ export class PostgresStore {
65
65
  await this.writeMeta();
66
66
  return true;
67
67
  }
68
+ storedVectorDims() {
69
+ return this.vectorEnabled ? this.vectorDims : null;
70
+ }
68
71
  async ensureVectorColumn(dims) {
69
72
  if (!this.vectorEnabled)
70
73
  return false;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "claude-db",
3
- "version": "0.12.2",
3
+ "version": "0.12.3",
4
4
  "description": "Free, open-source memory for Claude Code. Remembers why decisions were made and maps your code. Your own database (SQLite, Postgres or MongoDB), no cloud, no subscription.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
@@ -130,10 +130,10 @@
130
130
  "access": "public"
131
131
  },
132
132
  "optionalDependencies": {
133
- "claude-db-grammars-linux-x64": "0.12.2",
134
- "claude-db-grammars-linux-arm64": "0.12.2",
135
- "claude-db-grammars-darwin-x64": "0.12.2",
136
- "claude-db-grammars-darwin-arm64": "0.12.2",
137
- "claude-db-grammars-win32-x64": "0.12.2"
133
+ "claude-db-grammars-linux-x64": "0.12.3",
134
+ "claude-db-grammars-linux-arm64": "0.12.3",
135
+ "claude-db-grammars-darwin-x64": "0.12.3",
136
+ "claude-db-grammars-darwin-arm64": "0.12.3",
137
+ "claude-db-grammars-win32-x64": "0.12.3"
138
138
  }
139
139
  }