claude-db 0.8.5 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/CHANGELOG.md +103 -0
  2. package/dist/capture/flush.js +3 -1
  3. package/dist/capture/progress.js +41 -9
  4. package/dist/capture/transcript.js +8 -2
  5. package/dist/capture/turn-extractor.js +2 -2
  6. package/dist/cli/commands/doctor.js +2 -2
  7. package/dist/cli/commands/graph.js +12 -3
  8. package/dist/cli/commands/insight.js +2 -1
  9. package/dist/cli/commands/maintain.js +2 -2
  10. package/dist/cli/commands/setup.js +3 -3
  11. package/dist/cli/commands/sync.js +9 -3
  12. package/dist/cli/commands/transfer.js +60 -10
  13. package/dist/cli/commands/view.js +3 -2
  14. package/dist/graph/languages/basic.js +5 -0
  15. package/dist/graph/query/lookup.js +7 -3
  16. package/dist/graph/scan/extract.d.ts +2 -0
  17. package/dist/graph/scan/extract.js +40 -20
  18. package/dist/graph/scan/files.d.ts +2 -1
  19. package/dist/graph/scan/files.js +4 -3
  20. package/dist/graph/scan/index.d.ts +1 -0
  21. package/dist/graph/scan/index.js +11 -1
  22. package/dist/graph/scan/resolve.js +17 -6
  23. package/dist/hooks/grep-symbols.js +1 -1
  24. package/dist/hooks/relevance.d.ts +1 -1
  25. package/dist/hooks/relevance.js +3 -1
  26. package/dist/hooks/session-start.js +1 -1
  27. package/dist/hooks/user-prompt.js +3 -2
  28. package/dist/store/adapter.d.ts +1 -0
  29. package/dist/store/mongo/graph.js +7 -0
  30. package/dist/store/mongo/index.d.ts +1 -0
  31. package/dist/store/mongo/index.js +3 -0
  32. package/dist/store/mongo/observations.js +11 -3
  33. package/dist/store/mongo/search.js +7 -5
  34. package/dist/store/mongo/sessions.d.ts +1 -0
  35. package/dist/store/mongo/sessions.js +8 -5
  36. package/dist/store/mongo/vectorSearch.js +1 -1
  37. package/dist/store/postgres/graph.js +7 -2
  38. package/dist/store/postgres/index.d.ts +1 -0
  39. package/dist/store/postgres/index.js +4 -1
  40. package/dist/store/postgres/insert.d.ts +1 -1
  41. package/dist/store/postgres/insert.js +2 -1
  42. package/dist/store/postgres/meta.js +16 -4
  43. package/dist/store/postgres/observations.js +10 -3
  44. package/dist/store/postgres/sessions.d.ts +1 -0
  45. package/dist/store/postgres/sessions.js +4 -0
  46. package/dist/store/sqlite/graph.js +6 -2
  47. package/dist/store/sqlite/index.d.ts +1 -0
  48. package/dist/store/sqlite/index.js +3 -0
  49. package/dist/store/sqlite/observations.js +10 -3
  50. package/dist/store/sqlite/sessions.d.ts +1 -0
  51. package/dist/store/sqlite/sessions.js +4 -0
  52. package/dist/types.d.ts +3 -0
  53. package/dist/usages/find.js +10 -6
  54. package/package.json +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,108 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.9.1
4
+
5
+ ### Fixed
6
+
7
+ - **Commands printed database passwords in plain output.**
8
+ ([#21](https://github.com/Avijit07x/claude-db/pull/21))
9
+ `status`, `doctor`, `install`, `projects` and `reset` interpolated
10
+ `config.database` verbatim, and `view --export` wrote it into the generated
11
+ HTML, so a hosted Postgres or Mongo URL showed its password on screen and in
12
+ any file or bug report built from that output. SECURITY.md already stated
13
+ that these credentials are masked in command output; six call sites were not.
14
+ They now go through the existing `redact()`, as `use` already did, which
15
+ leaves the host readable and a SQLite path untouched. A new lint rule,
16
+ `no-raw-database-url`, fails the build on any future interpolation of
17
+ `config.database` without it.
18
+
19
+ ## 0.9.0
20
+
21
+ A correctness release. Everything below was found by reviewing the whole
22
+ codebase rather than reported, so most of it had been failing quietly.
23
+
24
+ **Stored graphs rebuild on the first scan after upgrading.** Extraction and
25
+ edge ids both changed, so `SCAN_VERSION` is bumped and the old rows are
26
+ discarded rather than left to disagree with the new ones.
27
+
28
+ ### Fixed
29
+
30
+ - **`claude-db merge` could permanently delete observations.** `list()` ordered
31
+ by `created_at` alone while `after` was a strict `>`, so any row sharing a
32
+ timestamp with a batch boundary was stepped over by every caller that pages
33
+ through the store. `merge` copies page by page and then removes the source
34
+ project, deleting exactly the rows its own loop had skipped. Git-seeded
35
+ observations take the commit date, which has one-second resolution, so ties
36
+ are routine. Paging is now a keyset walk over `(created_at, id)`.
37
+ `export` and `sync` were dropping the same rows without deleting them.
38
+
39
+ - **A session's last turn was lost when the prompt was still being written.**
40
+ The flush cursor advanced to the raw end of the transcript even when the file
41
+ ended mid-line, so the next read began inside the JSON, the entry was dropped,
42
+ and the prompt with all its work never appeared again. The cursor now stops at
43
+ the last complete line.
44
+
45
+ - **`find_usages` hid the real caller of any method.** Inbound edges were
46
+ matched on the destination name, but a member call is stored under its
47
+ qualified name, so the actual `repo.save()` caller was fetched and then
48
+ discarded, leaving only the class that declares it.
49
+
50
+ - **A repeated name in one file pointed its edge at the wrong declaration.**
51
+ 0.8.5 gave each declaration its own id, but the resolver still matched by
52
+ name and took the first one in the file, so the second `run()` got an edge to
53
+ the first, stamped at the confidence tier that means "read from the syntax".
54
+
55
+ - **`implements` never produced an edge, and neither did `extends` in `.js` or
56
+ `.jsx`.** Those rules capture a whole clause, whose text includes the keyword,
57
+ and the whitespace guard dropped it.
58
+
59
+ - **An incremental `scan` broke every cross-file edge it touched**, rewriting it
60
+ with no destination at full confidence, until someone ran `--force`.
61
+
62
+ - **Two edges on one line could overwrite each other**, because the edge id
63
+ ignored the relation and the call it came from. `trim(value.trim())` lost the
64
+ direct call entirely.
65
+
66
+ - **A file over 1 MB or containing a NUL made the graph reparse the whole repo
67
+ forever**, and kept its stale symbols indefinitely.
68
+
69
+ - **A standing rule stopped being injected** once a project had more than 100
70
+ preferences, and `view` showed months-old rows as the current state past 500
71
+ observations. Both asked for the oldest N and sorted afterwards.
72
+
73
+ - **Committed work was announced as "not committed yet".** Landed work was
74
+ closed before the new rows were inserted, so a session's final turn was always
75
+ left open. Renamed files were also mis-parsed out of `git status`, closing
76
+ work whose rename had not landed.
77
+
78
+ - **A memory dropped by the prompt budget was suppressed for the rest of the
79
+ session**, having been marked as shown without being rendered.
80
+
81
+ - **An unclosed `<private>` tag was not redacted**, so a pasted credential was
82
+ stored and injected back into context. The README promises it is stripped.
83
+
84
+ - **The grep hook ignored `grep --recursive <symbol>`**, because its
85
+ invert-flag guard also matched `--recursive` and `--devices`.
86
+
87
+ - **`export` and `import` carried no sessions**, so a restored database reported
88
+ "none yet for this project" against a full memory. One truncated line also
89
+ aborted `import` after earlier batches had committed, with no report of what
90
+ had landed.
91
+
92
+ - **Postgres and Mongo diverged from SQLite in four places**: Mongo read a
93
+ zero-row timeline request as unlimited, scored Atlas vector hits on a
94
+ different scale so the relevance floor passed everything, and moved a
95
+ session's start time to when it ended; Postgres kept a stale vector when a row
96
+ was re-inserted without one, and could leave keyword search permanently broken
97
+ if it crashed while rebuilding the `tsv` column.
98
+
99
+ - **`usages --mode path X --target Y` searched for a target named `--target`.**
100
+
101
+ ### Added
102
+
103
+ - `ListFilter` takes `newest` and `afterId`; `EdgeFilter` takes `dstName`;
104
+ stores expose `sessionProjects()`.
105
+
3
106
  ## 0.8.5
4
107
 
5
108
  ### Added
@@ -8,18 +8,20 @@ export async function flushSession(ctx, sessionId, project, transcriptPath, rebu
8
8
  const path = transcriptPath ?? transcriptPathFor(project, sessionId);
9
9
  const cursor = readCursor(sessionId);
10
10
  const { turns, nextOffset } = readTranscript(path, cursor);
11
- await closeLandedWork(ctx.store, project);
12
11
  if (turns.length === 0) {
12
+ await closeLandedWork(ctx.store, project);
13
13
  writeCursor(sessionId, nextOffset);
14
14
  return { observations: 0, summary: null };
15
15
  }
16
16
  const observations = observationsFromTurns(turns, sessionId, project, ctx.config);
17
17
  if (observations.length === 0) {
18
+ await closeLandedWork(ctx.store, project);
18
19
  writeCursor(sessionId, nextOffset);
19
20
  return { observations: 0, summary: null };
20
21
  }
21
22
  await embedObservations(ctx, observations);
22
23
  await ctx.store.insertObservations(observations);
24
+ await closeLandedWork(ctx.store, project);
23
25
  const previous = rebuild ? null : await ctx.store.getSession(sessionId);
24
26
  const summary = summarize(observations, previous?.summary);
25
27
  await ctx.store.upsertSession({
@@ -8,10 +8,21 @@ function uncommittedFiles(project) {
8
8
  encoding: 'utf8',
9
9
  maxBuffer: MAX_BUFFER,
10
10
  });
11
- return new Set(raw
12
- .split('\0')
13
- .filter((entry) => entry.length > 3)
14
- .map((entry) => join(project, entry.slice(3))));
11
+ const files = new Set();
12
+ const fields = raw.split('\0');
13
+ for (let i = 0; i < fields.length; i += 1) {
14
+ const entry = fields[i];
15
+ if (entry === undefined || entry.length <= 3)
16
+ continue;
17
+ files.add(join(project, entry.slice(3)));
18
+ if (/^[RC]/.test(entry) || /^.[RC]/.test(entry)) {
19
+ const origin = fields[i + 1];
20
+ if (origin !== undefined && origin.length > 0)
21
+ files.add(join(project, origin));
22
+ i += 1;
23
+ }
24
+ }
25
+ return files;
15
26
  }
16
27
  catch {
17
28
  return null;
@@ -21,13 +32,34 @@ export async function closeLandedWork(store, project) {
21
32
  const dirty = uncommittedFiles(project);
22
33
  if (!dirty)
23
34
  return 0;
24
- const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT });
25
- const landed = open
26
- .filter((obs) => obs.files.every((file) => !dirty.has(file)))
27
- .map((obs) => obs.id);
35
+ const landed = [];
36
+ let after;
37
+ let afterId;
38
+ for (;;) {
39
+ const batch = await store.list({
40
+ project,
41
+ status: 'open',
42
+ ...(after === undefined ? {} : { after }),
43
+ ...(afterId === undefined ? {} : { afterId }),
44
+ limit: OPEN_LIMIT,
45
+ });
46
+ if (batch.length === 0)
47
+ break;
48
+ for (const obs of batch) {
49
+ if (obs.files.every((file) => !dirty.has(file)))
50
+ landed.push(obs.id);
51
+ }
52
+ const last = batch[batch.length - 1];
53
+ if (!last)
54
+ break;
55
+ after = last.createdAt;
56
+ afterId = last.id;
57
+ if (batch.length < OPEN_LIMIT)
58
+ break;
59
+ }
28
60
  return store.closeObservations(landed);
29
61
  }
30
62
  export async function openWork(store, project) {
31
- const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT });
63
+ const open = await store.list({ project, status: 'open', limit: OPEN_LIMIT, newest: true });
32
64
  return open.sort((a, b) => b.createdAt - a.createdAt);
33
65
  }
@@ -31,8 +31,10 @@ export function readTranscript(path, fromOffset = 0) {
31
31
  }
32
32
  catch { }
33
33
  }
34
+ const lastNewline = raw.lastIndexOf('\n');
35
+ const complete = lastNewline < 0 ? start : start + Buffer.byteLength(raw.slice(0, lastNewline + 1), 'utf8');
34
36
  const turns = groupIntoTurns(entries);
35
- const nextOffset = turns.length > 0 ? (turns[turns.length - 1]?.offset ?? size) : size;
37
+ const nextOffset = turns.length > 0 ? (turns[turns.length - 1]?.offset ?? complete) : complete;
36
38
  return { turns, nextOffset };
37
39
  }
38
40
  finally {
@@ -42,8 +44,12 @@ export function readTranscript(path, fromOffset = 0) {
42
44
  function groupIntoTurns(entries) {
43
45
  const turns = [];
44
46
  let current = null;
47
+ let lastKnown = 0;
45
48
  for (const { entry, offset } of entries) {
46
- const timestamp = Date.parse(entry.timestamp ?? '') || Date.now();
49
+ const parsed = Date.parse(entry.timestamp ?? '');
50
+ const timestamp = Number.isNaN(parsed) ? lastKnown : parsed;
51
+ if (!Number.isNaN(parsed))
52
+ lastKnown = parsed;
47
53
  if (entry.type === 'user') {
48
54
  const text = extractText(entry.message?.content);
49
55
  if (!text || entry.isCompactSummary || isSyntheticPrompt(text))
@@ -8,7 +8,7 @@ export function observationsFromTurns(turns, sessionId, project, config) {
8
8
  .map((turn) => buildObservation(turn, sessionId, project, config));
9
9
  }
10
10
  function withoutPrivate(turn) {
11
- const strip = (text) => text.replace(/<private>[\s\S]*?<\/private>/gi, '[private]');
11
+ const strip = (text) => text.replace(/<private>[\s\S]*?(?:<\/private>|$)/gi, '[private]');
12
12
  return { ...turn, prompt: strip(turn.prompt), reasoning: strip(turn.reasoning) };
13
13
  }
14
14
  function isSubstantive(turn, config) {
@@ -165,7 +165,7 @@ function shortPath(path) {
165
165
  function isExcluded(file, patterns) {
166
166
  return patterns.some((pattern) => file.includes(pattern));
167
167
  }
168
- const PRIVATE_BLOCK = /<private>[\s\S]*?<\/private>/gi;
168
+ const PRIVATE_BLOCK = /<private>[\s\S]*?(?:<\/private>|$)/gi;
169
169
  const PRIVATE_KEY = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
170
170
  export function redact(text) {
171
171
  return text
@@ -2,7 +2,7 @@ import { createContext } from '../../context.js';
2
2
  import { loadConfig } from '../../config/index.js';
3
3
  import { packageVersion } from '../../update.js';
4
4
  import { randomUUID } from 'node:crypto';
5
- import { remember } from '../../capture/index.js';
5
+ import { redact, remember } from '../../capture/index.js';
6
6
  import { resolveProject } from '../../util/project.js';
7
7
  import { settingsPathFor } from '../paths.js';
8
8
  import { existsSync, readFileSync } from 'node:fs';
@@ -16,7 +16,7 @@ export async function cmdDoctor(argv) {
16
16
  const embedder = await ctx.embedder();
17
17
  const vectors = await probeEmbedder(embedder);
18
18
  console.log(`version : ${packageVersion()}`);
19
- console.log(`database : ${ctx.config.database}`);
19
+ console.log(`database : ${redact(ctx.config.database)}`);
20
20
  console.log(`adapter : ${ctx.store.kind}`);
21
21
  console.log(`reachable: ${reachable ? 'yes' : 'no'}`);
22
22
  console.log(`requested: embeddings.provider = ${ctx.config.embeddings.provider}`);
@@ -13,8 +13,15 @@ export async function cmdScan(argv) {
13
13
  const known = force
14
14
  ? new Map()
15
15
  : new Map((await ctx.store.scannedFiles(project)).map((f) => [f.path, f.hash]));
16
+ const existing = force ? [] : await ctx.store.findSymbols({ project, limit: 200000 });
16
17
  const started = Date.now();
17
- const scan = scanRepository({ root, project, known, ...(force ? { force } : {}) });
18
+ const scan = scanRepository({
19
+ root,
20
+ project,
21
+ known,
22
+ ...(force ? { force } : {}),
23
+ ...(existing.length > 0 ? { existing } : {}),
24
+ });
18
25
  if (force)
19
26
  await ctx.store.removeGraph(project);
20
27
  else if (scan.changed.length > 0)
@@ -43,7 +50,8 @@ export async function cmdUsages(argv) {
43
50
  const context = Number(valueOf(argv, '--context') ?? 0);
44
51
  const limit = Number(valueOf(argv, '--limit') ?? 100);
45
52
  const mode = valueOf(argv, '--mode') ?? 'text';
46
- const words = withoutFlags(argv, ['--path', '--context', '--limit', '--mode'])
53
+ const targetFlag = valueOf(argv, '--target');
54
+ const words = withoutFlags(argv, ['--path', '--context', '--limit', '--mode', '--target'])
47
55
  .filter((arg) => arg !== '--regex')
48
56
  .join(' ')
49
57
  .trim();
@@ -69,7 +77,8 @@ export async function cmdUsages(argv) {
69
77
  console.error(`Unknown --mode "${mode}". Use text, usages, explain or path.`);
70
78
  process.exit(1);
71
79
  }
72
- const [symbol, target] = words.split(/\s+/);
80
+ const [symbol, positional] = words.split(/\s+/);
81
+ const target = targetFlag ?? positional;
73
82
  if (mode === 'path' && !target) {
74
83
  console.error('--mode path needs two symbols: claude-db usages --mode path <from> <to>');
75
84
  process.exit(1);
@@ -2,6 +2,7 @@ import { createContext } from '../../context.js';
2
2
  import { eachObservation } from '../commands/transfer.js';
3
3
  import { resolve } from 'node:path';
4
4
  import { resolveProject } from '../../util/project.js';
5
+ import { redact } from '../../capture/turn-extractor.js';
5
6
  export async function cmdStats() {
6
7
  const project = resolveProject(undefined);
7
8
  const ctx = await createContext();
@@ -55,7 +56,7 @@ export async function cmdProjects() {
55
56
  console.log('No memory stored yet.');
56
57
  return;
57
58
  }
58
- console.log(`database: ${ctx.config.database}\n`);
59
+ console.log(`database: ${redact(ctx.config.database)}\n`);
59
60
  for (const entry of projects) {
60
61
  const marker = entry.project === here ? '*' : ' ';
61
62
  const date = new Date(entry.lastActive).toISOString().slice(0, 10);
@@ -2,7 +2,7 @@ import { CONFIG_DIR, loadConfig } from '../../config/index.js';
2
2
  import { basename, join } from 'node:path';
3
3
  import { checkForUpdate } from '../../update.js';
4
4
  import { createContext } from '../../context.js';
5
- import { flushSession, resetCursor, sweepCursors, transcriptsFor } from '../../capture/index.js';
5
+ import { flushSession, redact, resetCursor, sweepCursors, transcriptsFor, } from '../../capture/index.js';
6
6
  import { resolveProject } from '../../util/project.js';
7
7
  import { rmSync } from 'node:fs';
8
8
  export async function cmdReset(argv) {
@@ -11,7 +11,7 @@ export async function cmdReset(argv) {
11
11
  const project = resolveProject(undefined);
12
12
  const ctx = await createContext();
13
13
  try {
14
- const target = scoped ? project : ctx.config.database;
14
+ const target = scoped ? project : redact(ctx.config.database);
15
15
  if (!confirmed) {
16
16
  console.log(`This would delete ${scoped ? "this project's" : 'ALL'} memory from:`);
17
17
  console.log(` ${target}`);
@@ -1,5 +1,5 @@
1
1
  import { toShortId } from '../../util/shortid.js';
2
- import { closeLandedWork, openWork } from '../../capture/index.js';
2
+ import { closeLandedWork, openWork, redact } from '../../capture/index.js';
3
3
  import { CONFIG_PATH } from '../../config/index.js';
4
4
  import { DIST_DIR } from '../constants.js';
5
5
  import { assertStableLocation, install, uninstall } from '../install.js';
@@ -28,7 +28,7 @@ export async function cmdInstall(scope) {
28
28
  console.log(`Guidance : ${instructionsPathFor(scope, project)}`);
29
29
  console.log(`Skill : ${skillPathFor(scope, project)} (/cdb-scan)`);
30
30
  console.log(`Config : ${CONFIG_PATH}`);
31
- console.log(`Database : ${ctx.config.database}`);
31
+ console.log(`Database : ${redact(ctx.config.database)}`);
32
32
  if (embedder.id === 'builtin-hashing') {
33
33
  console.log('Search : keyword + basic vectors');
34
34
  console.log(' for semantic search: npm i -g @xenova/transformers');
@@ -84,7 +84,7 @@ export async function cmdStatus() {
84
84
  }));
85
85
  const active = wiring.filter((entry) => entry.installed);
86
86
  console.log(`project : ${project}`);
87
- console.log(`database : ${ctx.config.database}`);
87
+ console.log(`database : ${redact(ctx.config.database)}`);
88
88
  console.log(`hooks : ${active.length > 0 ? active.map((entry) => entry.label).join(' + ') : 'NOT INSTALLED'}`);
89
89
  const mcpFile = [mcpPathFor('project', project), mcpPathFor('global', project)].find((candidate) => fileMentions(candidate, resolve(DIST_DIR, 'mcp', 'server.js')));
90
90
  console.log(`mcp : ${mcpFile ? `registered (${mcpFile})` : 'NOT REGISTERED'}`);
@@ -79,16 +79,22 @@ async function syncSessions(local, remote) {
79
79
  return moved;
80
80
  }
81
81
  async function eachRemoteObservation(store, visit) {
82
- let after = 0;
82
+ let after;
83
+ let afterId;
83
84
  for (;;) {
84
- const batch = await store.list({ after, limit: BATCH });
85
+ const batch = await store.list({
86
+ ...(after === undefined ? {} : { after }),
87
+ ...(afterId === undefined ? {} : { afterId }),
88
+ limit: BATCH,
89
+ });
85
90
  if (batch.length === 0)
86
91
  return;
87
92
  await visit(batch);
88
93
  const last = batch[batch.length - 1];
89
94
  if (!last)
90
95
  return;
91
- after = last.createdAt === after ? after + 1 : last.createdAt;
96
+ after = last.createdAt;
97
+ afterId = last.id;
92
98
  if (batch.length < BATCH)
93
99
  return;
94
100
  }
@@ -1,4 +1,5 @@
1
1
  import { BATCH } from '../constants.js';
2
+ const SESSION_LIMIT = 10000;
2
3
  import { createContext } from '../../context.js';
3
4
  import { embedObservations } from '../../capture/index.js';
4
5
  import { loadConfig } from '../../config/index.js';
@@ -6,10 +7,16 @@ import { readFileSync } from 'node:fs';
6
7
  import { resolveProject } from '../../util/project.js';
7
8
  import { valueOf } from '../args.js';
8
9
  export async function eachObservation(ctx, filter, visit) {
9
- let after = 0;
10
+ let after;
11
+ let afterId;
10
12
  let total = 0;
11
13
  for (;;) {
12
- const batch = await ctx.store.list({ ...filter, after, limit: BATCH });
14
+ const batch = await ctx.store.list({
15
+ ...filter,
16
+ ...(after === undefined ? {} : { after }),
17
+ ...(afterId === undefined ? {} : { afterId }),
18
+ limit: BATCH,
19
+ });
13
20
  if (batch.length === 0)
14
21
  return total;
15
22
  await visit(batch);
@@ -17,7 +24,8 @@ export async function eachObservation(ctx, filter, visit) {
17
24
  const last = batch[batch.length - 1];
18
25
  if (!last)
19
26
  return total;
20
- after = last.createdAt === after ? after + 1 : last.createdAt;
27
+ after = last.createdAt;
28
+ afterId = last.id;
21
29
  if (batch.length < BATCH)
22
30
  return total;
23
31
  }
@@ -30,7 +38,22 @@ export async function cmdExport(argv) {
30
38
  for (const obs of batch)
31
39
  process.stdout.write(`${JSON.stringify(obs)}\n`);
32
40
  });
33
- process.stderr.write(`${count} observation(s) exported.\n`);
41
+ const projects = all
42
+ ? [
43
+ ...new Set([
44
+ ...(await ctx.store.listProjects()).map((entry) => entry.project),
45
+ ...(await ctx.store.sessionProjects()),
46
+ ]),
47
+ ]
48
+ : [resolveProject(undefined)];
49
+ let sessions = 0;
50
+ for (const project of projects) {
51
+ for (const session of await ctx.store.recentSessions(project, SESSION_LIMIT)) {
52
+ process.stdout.write(`${JSON.stringify({ ...session, _type: 'session' })}\n`);
53
+ sessions += 1;
54
+ }
55
+ }
56
+ process.stderr.write(`${count} observation(s) and ${sessions} session(s) exported.\n`);
34
57
  }
35
58
  finally {
36
59
  await ctx.close();
@@ -46,20 +69,47 @@ export async function cmdImport(path) {
46
69
  .filter((line) => line.trim().length > 0);
47
70
  const ctx = await createContext();
48
71
  let imported = 0;
72
+ let skipped = 0;
73
+ let restored = 0;
49
74
  try {
50
75
  for (let i = 0; i < lines.length; i += BATCH) {
51
- const batch = lines
52
- .slice(i, i + BATCH)
53
- .map((line) => JSON.parse(line))
54
- .filter((obs) => typeof obs.id === 'string' && typeof obs.project === 'string');
55
- await ctx.store.insertObservations(batch);
76
+ const batch = [];
77
+ for (const line of lines.slice(i, i + BATCH)) {
78
+ let parsed;
79
+ try {
80
+ parsed = JSON.parse(line);
81
+ }
82
+ catch {
83
+ skipped += 1;
84
+ continue;
85
+ }
86
+ if (parsed._type === 'session') {
87
+ const { _type, ...session } = parsed;
88
+ void _type;
89
+ if (typeof session.id === 'string' && typeof session.project === 'string') {
90
+ await ctx.store.upsertSession(session);
91
+ restored += 1;
92
+ }
93
+ else
94
+ skipped += 1;
95
+ continue;
96
+ }
97
+ if (typeof parsed.id === 'string' && typeof parsed.project === 'string')
98
+ batch.push(parsed);
99
+ else
100
+ skipped += 1;
101
+ }
102
+ if (batch.length > 0)
103
+ await ctx.store.insertObservations(batch);
56
104
  imported += batch.length;
57
105
  }
58
106
  }
59
107
  finally {
60
108
  await ctx.close();
61
109
  }
62
- console.log(`Imported ${imported} observation(s).`);
110
+ console.log(`Imported ${imported} observation(s)` +
111
+ `${restored > 0 ? `, ${restored} session(s)` : ''}` +
112
+ `${skipped > 0 ? `, skipped ${skipped} unreadable line(s)` : ''}.`);
63
113
  }
64
114
  export async function cmdPrune(argv) {
65
115
  const days = Number(valueOf(argv, '--older-than') ?? NaN);
@@ -6,8 +6,9 @@ import { createContext } from '../../context.js';
6
6
  import { resolveProject } from '../../util/project.js';
7
7
  import { toShortId } from '../../util/shortid.js';
8
8
  import { renderPage } from './view-page.js';
9
+ import { redact } from '../../capture/turn-extractor.js';
9
10
  async function collect(ctx, project) {
10
- const all = (await ctx.store.list({ project, limit: 500 })).sort((a, b) => b.createdAt - a.createdAt);
11
+ const all = (await ctx.store.list({ project, limit: 500, newest: true })).sort((a, b) => b.createdAt - a.createdAt);
11
12
  const kinds = {};
12
13
  for (const obs of all)
13
14
  kinds[obs.kind] = (kinds[obs.kind] ?? 0) + 1;
@@ -25,7 +26,7 @@ async function collect(ctx, project) {
25
26
  }));
26
27
  return {
27
28
  project,
28
- database: ctx.config.database,
29
+ database: redact(ctx.config.database),
29
30
  scannedFiles: (await ctx.store.scannedFiles(project)).length,
30
31
  kinds,
31
32
  rules,
@@ -104,6 +104,11 @@ export const BASIC_FINGERPRINT = [
104
104
  CALLABLE.source,
105
105
  CALL.source,
106
106
  Object.keys(LABELS).join(','),
107
+ Object.entries(KEYWORDS)
108
+ .map(([word, kind]) => `${word}:${kind}`)
109
+ .sort()
110
+ .join(','),
111
+ [...NOT_A_CALL].sort().join(','),
107
112
  ].join('|');
108
113
  export const BASIC_LANGUAGES = Object.entries(LABELS).map(([label, extensions]) => ({
109
114
  id: 'basic',
@@ -29,11 +29,15 @@ export async function queryGraph(store, project, query) {
29
29
  const ids = answer.definitions.map((symbol) => symbol.id);
30
30
  const edges = await store.findEdges({
31
31
  project,
32
- ...(ids.length > 0 ? { srcIds: ids, dstIds: ids } : {}),
32
+ ...(ids.length > 0 ? { srcIds: ids, dstIds: ids } : { dstName: query.symbol }),
33
33
  limit: Math.max(query.limit * 10, 500),
34
34
  });
35
- answer.inbound = edges.filter((edge) => edge.dstName === query.symbol);
36
- answer.outbound = edges.filter((edge) => ids.includes(edge.srcId) && edge.dstName !== query.symbol);
35
+ const idSet = new Set(ids);
36
+ const pointsAtSymbol = (edge) => edge.dstName === query.symbol ||
37
+ edge.dstName.endsWith(`.${query.symbol}`) ||
38
+ (edge.dstId !== '' && idSet.has(edge.dstId));
39
+ answer.inbound = edges.filter(pointsAtSymbol);
40
+ answer.outbound = edges.filter((edge) => idSet.has(edge.srcId) && !pointsAtSymbol(edge));
37
41
  answer.empty = answer.definitions.length === 0 && answer.inbound.length === 0;
38
42
  if (answer.empty)
39
43
  answer.suggestions = await suggestFor(store, project, query.symbol);
@@ -7,6 +7,8 @@ export interface Reference {
7
7
  line: number;
8
8
  from: CodeSymbol | null;
9
9
  weak?: boolean;
10
+ to?: string;
11
+ origin?: string;
10
12
  }
11
13
  export interface Extraction {
12
14
  symbols: CodeSymbol[];
@@ -87,6 +87,18 @@ function enclosing(line, spans) {
87
87
  }
88
88
  return (best ?? fallback)?.symbol ?? null;
89
89
  }
90
+ function namedTargets(node) {
91
+ if (!/\s/.test(node.text().trim()))
92
+ return [node];
93
+ const inner = [];
94
+ for (const kind of ['type_identifier', 'identifier']) {
95
+ try {
96
+ inner.push(...node.findAll({ rule: { kind } }));
97
+ }
98
+ catch { }
99
+ }
100
+ return inner.length > 0 ? inner : [node];
101
+ }
90
102
  export function extractFile(file, project) {
91
103
  if (file.spec.basic)
92
104
  return extractByPattern(file, project);
@@ -132,31 +144,39 @@ export function extractFile(file, project) {
132
144
  relation: 'defines',
133
145
  line: span.start,
134
146
  from: owner,
147
+ to: span.symbol.id,
135
148
  });
136
149
  }
137
150
  for (const rule of file.spec.references) {
138
151
  for (const node of root.findAll({ rule: { kind: rule.kind } })) {
139
- const target = rule.field.length === 0 ? node : resolveField(node, rule.field);
140
- const raw = target?.text();
141
- if (!target || !raw)
142
- continue;
143
- const name = unquote(raw.trim());
144
- if (!name || /\s/.test(name))
145
- continue;
146
- if (rule.relation === 'references' && name.includes('.'))
147
- continue;
148
- if (rule.namePattern && !rule.namePattern.test(name))
152
+ const matched = rule.field.length === 0 ? node : resolveField(node, rule.field);
153
+ if (!matched)
149
154
  continue;
150
- if (rule.excludeParents?.includes(node.parent()?.kind() ?? ''))
151
- continue;
152
- const line = target.range().start.line + 1;
153
- references.push({
154
- file: file.path,
155
- name,
156
- relation: rule.relation,
157
- line,
158
- from: enclosing(line, spans),
159
- });
155
+ const targets = rule.field.length === 0 ? namedTargets(matched) : [matched];
156
+ for (const target of targets) {
157
+ const raw = target.text();
158
+ if (!raw)
159
+ continue;
160
+ const name = unquote(raw.trim());
161
+ if (!name || /\s/.test(name))
162
+ continue;
163
+ if (rule.relation === 'references' && name.includes('.'))
164
+ continue;
165
+ if (rule.namePattern && !rule.namePattern.test(name))
166
+ continue;
167
+ if (rule.excludeParents?.includes(node.parent()?.kind() ?? ''))
168
+ continue;
169
+ const line = target.range().start.line + 1;
170
+ const span = node.range();
171
+ references.push({
172
+ file: file.path,
173
+ name,
174
+ relation: rule.relation,
175
+ line,
176
+ from: enclosing(line, spans),
177
+ origin: `${span.start.line}:${span.start.column}-${span.end.line}:${span.end.column}`,
178
+ });
179
+ }
160
180
  }
161
181
  }
162
182
  return { symbols, references };
@@ -1,11 +1,12 @@
1
1
  import type { LanguageSpec } from '../languages/index.js';
2
2
  export interface SourceFile {
3
+ unreadable?: boolean;
3
4
  path: string;
4
5
  spec: LanguageSpec;
5
6
  source: string;
6
7
  hash: string;
7
8
  }
8
- export declare const SCAN_VERSION = 4;
9
+ export declare const SCAN_VERSION = 5;
9
10
  export declare function hashOf(bytes: Buffer): string;
10
11
  export declare function listFiles(root: string): string[];
11
12
  export declare function sourceFiles(root: string): string[];