claude-db 0.12.3 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/CHANGELOG.md +57 -0
  2. package/dist/capture/cursors.d.ts +8 -0
  3. package/dist/capture/cursors.js +99 -0
  4. package/dist/capture/flush-lock.d.ts +11 -0
  5. package/dist/capture/flush-lock.js +108 -0
  6. package/dist/capture/flush.d.ts +4 -4
  7. package/dist/capture/flush.js +27 -60
  8. package/dist/capture/index.d.ts +3 -3
  9. package/dist/capture/index.js +3 -3
  10. package/dist/capture/progress.js +8 -1
  11. package/dist/capture/reingest.d.ts +1 -0
  12. package/dist/capture/reingest.js +4 -1
  13. package/dist/capture/scrub.d.ts +1 -0
  14. package/dist/capture/scrub.js +4 -1
  15. package/dist/capture/vectors.js +2 -1
  16. package/dist/catchup/gather.js +1 -0
  17. package/dist/cli/commands/daemon.d.ts +1 -0
  18. package/dist/cli/commands/daemon.js +29 -0
  19. package/dist/cli/commands/distill.js +2 -1
  20. package/dist/cli/commands/graph.js +2 -1
  21. package/dist/cli/commands/maintain.js +3 -1
  22. package/dist/cli/commands/sync.js +5 -4
  23. package/dist/cli/commands/transfer.js +4 -3
  24. package/dist/cli/index.js +4 -0
  25. package/dist/cli/usage.js +1 -0
  26. package/dist/config/schema.d.ts +4 -0
  27. package/dist/config/schema.js +6 -0
  28. package/dist/context.d.ts +1 -0
  29. package/dist/context.js +21 -6
  30. package/dist/daemon/client.d.ts +12 -0
  31. package/dist/daemon/client.js +88 -0
  32. package/dist/daemon/paths.d.ts +16 -0
  33. package/dist/daemon/paths.js +19 -0
  34. package/dist/daemon/server.d.ts +2 -0
  35. package/dist/daemon/server.js +225 -0
  36. package/dist/facts/budget.d.ts +1 -0
  37. package/dist/facts/budget.js +4 -0
  38. package/dist/facts/claude-memory.js +7 -2
  39. package/dist/facts/distill.js +6 -3
  40. package/dist/facts/handoff.js +6 -1
  41. package/dist/facts/known.js +3 -1
  42. package/dist/facts/model.d.ts +1 -1
  43. package/dist/facts/model.js +18 -7
  44. package/dist/facts/start.js +2 -2
  45. package/dist/graph/query/answer.js +1 -1
  46. package/dist/graph/query/path.js +69 -37
  47. package/dist/graph/query/refresh.js +5 -1
  48. package/dist/graph/query/suggest.js +1 -2
  49. package/dist/graph/scan/cache.d.ts +1 -0
  50. package/dist/graph/scan/cache.js +1 -1
  51. package/dist/graph/scan/persist.js +12 -16
  52. package/dist/hooks/compact.d.ts +2 -1
  53. package/dist/hooks/compact.js +9 -3
  54. package/dist/hooks/detached.d.ts +1 -0
  55. package/dist/hooks/detached.js +6 -0
  56. package/dist/hooks/graph-presence.d.ts +3 -0
  57. package/dist/hooks/graph-presence.js +42 -0
  58. package/dist/hooks/handlers/prefer-usages.d.ts +2 -0
  59. package/dist/hooks/handlers/prefer-usages.js +113 -0
  60. package/dist/hooks/handlers/session-end.d.ts +2 -0
  61. package/dist/hooks/handlers/session-end.js +62 -0
  62. package/dist/hooks/handlers/session-start.d.ts +2 -0
  63. package/dist/hooks/handlers/session-start.js +104 -0
  64. package/dist/hooks/handlers/turn-end.d.ts +2 -0
  65. package/dist/hooks/handlers/turn-end.js +31 -0
  66. package/dist/hooks/handlers/user-prompt.d.ts +2 -0
  67. package/dist/hooks/handlers/user-prompt.js +55 -0
  68. package/dist/hooks/local.d.ts +2 -0
  69. package/dist/hooks/local.js +23 -0
  70. package/dist/hooks/payload.d.ts +9 -4
  71. package/dist/hooks/payload.js +26 -17
  72. package/dist/hooks/prefer-usages.js +2 -118
  73. package/dist/hooks/route.d.ts +2 -0
  74. package/dist/hooks/route.js +28 -0
  75. package/dist/hooks/run.d.ts +14 -0
  76. package/dist/hooks/run.js +12 -0
  77. package/dist/hooks/save-turn.d.ts +2 -0
  78. package/dist/hooks/save-turn.js +11 -0
  79. package/dist/hooks/session-end.js +2 -63
  80. package/dist/hooks/session-start.js +2 -90
  81. package/dist/hooks/start-legacy.js +1 -1
  82. package/dist/hooks/turn-end.js +2 -25
  83. package/dist/hooks/turn-save.js +10 -15
  84. package/dist/hooks/user-prompt.js +2 -58
  85. package/dist/search/service.d.ts +1 -0
  86. package/dist/search/service.js +16 -6
  87. package/dist/store/adapter.d.ts +11 -7
  88. package/dist/store/each.d.ts +2 -4
  89. package/dist/store/each.js +1 -0
  90. package/dist/store/mongo/graph.d.ts +2 -1
  91. package/dist/store/mongo/graph.js +17 -2
  92. package/dist/store/mongo/index.d.ts +7 -8
  93. package/dist/store/mongo/index.js +13 -2
  94. package/dist/store/mongo/observations.d.ts +2 -1
  95. package/dist/store/mongo/observations.js +38 -4
  96. package/dist/store/postgres/driver.d.ts +10 -3
  97. package/dist/store/postgres/driver.js +12 -0
  98. package/dist/store/postgres/graph.d.ts +5 -6
  99. package/dist/store/postgres/graph.js +78 -57
  100. package/dist/store/postgres/index.d.ts +7 -8
  101. package/dist/store/postgres/index.js +16 -11
  102. package/dist/store/postgres/insert.js +98 -48
  103. package/dist/store/postgres/meta.d.ts +1 -0
  104. package/dist/store/postgres/meta.js +4 -0
  105. package/dist/store/postgres/observations.d.ts +3 -2
  106. package/dist/store/postgres/observations.js +40 -6
  107. package/dist/store/postgres/rows.js +1 -1
  108. package/dist/store/project-resolver.d.ts +3 -0
  109. package/dist/store/project-resolver.js +34 -23
  110. package/dist/store/scoped-store.js +5 -1
  111. package/dist/store/sqlite/graph.d.ts +5 -6
  112. package/dist/store/sqlite/graph.js +61 -35
  113. package/dist/store/sqlite/index.d.ts +7 -8
  114. package/dist/store/sqlite/index.js +11 -2
  115. package/dist/store/sqlite/observations.d.ts +2 -1
  116. package/dist/store/sqlite/observations.js +38 -7
  117. package/dist/types.d.ts +9 -0
  118. package/dist/util/job-lock.d.ts +1 -0
  119. package/dist/util/job-lock.js +50 -10
  120. package/docs/setup-guide.md +5 -0
  121. package/package.json +6 -8
@@ -25,40 +25,56 @@ async function upsertChunked(client, rows, width, sql, columns) {
25
25
  await client.query(sql(placeholders(slice.length, width)), values);
26
26
  }
27
27
  }
28
- export async function upsertGraph(pool, scan) {
28
+ async function writeGraphRows(client, scan) {
29
+ await upsertChunked(client, lastById(scan.symbols, (s) => s.id), 8, (values) => `INSERT INTO symbols (id, project, name, kind, file, line, lang, signature)
30
+ VALUES ${values}
31
+ ON CONFLICT (id) DO UPDATE SET
32
+ name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
33
+ line = EXCLUDED.line, lang = EXCLUDED.lang,
34
+ signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
35
+ await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
36
+ relation, confidence, score, file, line)
37
+ VALUES ${values}
38
+ ON CONFLICT (id) DO UPDATE SET
39
+ dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
40
+ relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
41
+ score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
42
+ e.id,
43
+ e.project,
44
+ e.srcId,
45
+ e.srcName,
46
+ e.dstId,
47
+ e.dstName,
48
+ e.relation,
49
+ e.confidence,
50
+ e.score,
51
+ e.file,
52
+ e.line,
53
+ ]);
54
+ await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
55
+ VALUES ${values}
56
+ ON CONFLICT (project, path) DO UPDATE SET
57
+ hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
58
+ }
59
+ async function deleteGraphRows(client, project, files) {
60
+ if (files !== null && files.length === 0)
61
+ return 0;
62
+ const values = files === null ? [project] : [project, files];
63
+ const where = files === null ? 'project = $1' : 'project = $1 AND file = ANY($2::text[])';
64
+ const cacheWhere = files === null ? 'project = $1' : 'project = $1 AND path = ANY($2::text[])';
65
+ const res = await client.query(`WITH gone AS (DELETE FROM symbols WHERE ${where} RETURNING 1),
66
+ edges AS (DELETE FROM symbol_edges WHERE ${where}),
67
+ cached AS (DELETE FROM scanned_files WHERE ${cacheWhere})
68
+ SELECT COUNT(*)::int AS n FROM gone`, values);
69
+ return Number(res.rows[0]?.['n'] ?? 0);
70
+ }
71
+ async function inTransaction(pool, work) {
29
72
  const client = await pool.connect();
30
73
  try {
31
74
  await client.query('BEGIN');
32
- await upsertChunked(client, lastById(scan.symbols, (s) => s.id), 8, (values) => `INSERT INTO symbols (id, project, name, kind, file, line, lang, signature)
33
- VALUES ${values}
34
- ON CONFLICT (id) DO UPDATE SET
35
- name = EXCLUDED.name, kind = EXCLUDED.kind, file = EXCLUDED.file,
36
- line = EXCLUDED.line, lang = EXCLUDED.lang,
37
- signature = EXCLUDED.signature`, (s) => [s.id, s.project, s.name, s.kind, s.file, s.line, s.lang, s.signature]);
38
- await upsertChunked(client, lastById(scan.edges, (e) => e.id), 11, (values) => `INSERT INTO symbol_edges (id, project, src_id, src_name, dst_id, dst_name,
39
- relation, confidence, score, file, line)
40
- VALUES ${values}
41
- ON CONFLICT (id) DO UPDATE SET
42
- dst_id = EXCLUDED.dst_id, dst_name = EXCLUDED.dst_name,
43
- relation = EXCLUDED.relation, confidence = EXCLUDED.confidence,
44
- score = EXCLUDED.score, file = EXCLUDED.file, line = EXCLUDED.line`, (e) => [
45
- e.id,
46
- e.project,
47
- e.srcId,
48
- e.srcName,
49
- e.dstId,
50
- e.dstName,
51
- e.relation,
52
- e.confidence,
53
- e.score,
54
- e.file,
55
- e.line,
56
- ]);
57
- await upsertChunked(client, lastById(scan.files, (f) => `${f.project}\u0000${f.path}`), 4, (values) => `INSERT INTO scanned_files (project, path, hash, scanned_at)
58
- VALUES ${values}
59
- ON CONFLICT (project, path) DO UPDATE SET
60
- hash = EXCLUDED.hash, scanned_at = EXCLUDED.scanned_at`, (f) => [f.project, f.path, f.hash, f.scannedAt]);
75
+ const result = await work(client);
61
76
  await client.query('COMMIT');
77
+ return result;
62
78
  }
63
79
  catch (error) {
64
80
  await client.query('ROLLBACK');
@@ -68,6 +84,17 @@ export async function upsertGraph(pool, scan) {
68
84
  client.release();
69
85
  }
70
86
  }
87
+ export async function upsertGraph(pool, scan) {
88
+ await inTransaction(pool, (client) => writeGraphRows(client, scan));
89
+ }
90
+ export async function replaceGraph(pool, project, files, scan) {
91
+ await inTransaction(pool, async (client) => {
92
+ await deleteGraphRows(client, project, files);
93
+ await writeGraphRows(client, scan);
94
+ });
95
+ }
96
+ const SYMBOL_COLUMNS = 'id, name, kind, file, line, lang, signature';
97
+ const EDGE_COLUMNS = 'id, src_id, src_name, dst_id, dst_name, relation, confidence, score, file, line';
71
98
  export async function findSymbols(pool, filter) {
72
99
  const values = [filter.project];
73
100
  const conditions = ['project = $1'];
@@ -80,9 +107,13 @@ export async function findSymbols(pool, filter) {
80
107
  conditions.push(`file = $${values.length}`);
81
108
  }
82
109
  values.push(filter.limit ?? 500);
83
- const res = await pool.query(`SELECT * FROM symbols WHERE ${conditions.join(' AND ')}
110
+ const res = await pool.query(`SELECT ${SYMBOL_COLUMNS} FROM symbols WHERE ${conditions.join(' AND ')}
84
111
  ORDER BY file, line LIMIT $${values.length}`, values);
85
- return res.rows.map(toSymbol);
112
+ return res.rows.map((row) => toSymbol({ ...row, project: filter.project }));
113
+ }
114
+ export async function symbolNames(pool, project, limit) {
115
+ const res = await pool.query('SELECT name FROM symbols WHERE project = $1 ORDER BY file, line LIMIT $2', [project, limit]);
116
+ return res.rows.map((row) => row['name']);
86
117
  }
87
118
  export async function findEdges(pool, filter) {
88
119
  const values = [filter.project];
@@ -96,20 +127,29 @@ export async function findEdges(pool, filter) {
96
127
  values.push(ids);
97
128
  sides.push(`${column} = ANY($${values.length}::text[])`);
98
129
  }
130
+ for (const [side, names] of [
131
+ ['src', filter.unresolvedSrcNames],
132
+ ['dst', filter.unresolvedDstNames],
133
+ ]) {
134
+ if (!names || names.length === 0)
135
+ continue;
136
+ values.push(names);
137
+ sides.push(`(${side}_id = '' AND ${side}_name = ANY($${values.length}::text[]))`);
138
+ }
99
139
  let where = sides.length > 0 ? ` AND (${sides.join(' OR ')})` : '';
100
140
  if (filter.dstName !== undefined) {
101
141
  values.push(filter.dstName, `%.${filter.dstName}`);
102
142
  where += ` AND (dst_name = $${values.length - 1} OR dst_name LIKE $${values.length})`;
103
143
  }
104
144
  values.push(filter.limit ?? 2000);
105
- const res = await pool.query(`SELECT * FROM symbol_edges WHERE project = $1${where}
145
+ const res = await pool.query(`SELECT ${EDGE_COLUMNS} FROM symbol_edges WHERE project = $1${where}
106
146
  ORDER BY file, line, id LIMIT $${values.length}`, values);
107
- return res.rows.map(toEdge);
147
+ return res.rows.map((row) => toEdge({ ...row, project: filter.project }));
108
148
  }
109
- export async function scannedFiles(pool, project) {
110
- const res = await pool.query('SELECT * FROM scanned_files WHERE project = $1', [project]);
149
+ export async function scannedFiles(pool, project, limit) {
150
+ const res = await pool.query(`SELECT path, hash, scanned_at FROM scanned_files WHERE project = $1${limit === undefined ? '' : ' LIMIT $2'}`, limit === undefined ? [project] : [project, limit]);
111
151
  return res.rows.map((row) => ({
112
- project: row['project'],
152
+ project,
113
153
  path: row['path'],
114
154
  hash: row['hash'],
115
155
  scannedAt: Number(row['scanned_at']),
@@ -117,24 +157,5 @@ export async function scannedFiles(pool, project) {
117
157
  }
118
158
  export async function removeGraph(pool, project, files) {
119
159
  const scoped = files !== undefined && files.length > 0;
120
- const values = scoped ? [project, files] : [project];
121
- const where = scoped ? 'project = $1 AND file = ANY($2::text[])' : 'project = $1';
122
- const cacheWhere = scoped ? 'project = $1 AND path = ANY($2::text[])' : 'project = $1';
123
- const client = await pool.connect();
124
- try {
125
- await client.query('BEGIN');
126
- const counted = await client.query(`SELECT COUNT(*)::int AS n FROM symbols WHERE ${where}`, values);
127
- await client.query(`DELETE FROM symbols WHERE ${where}`, values);
128
- await client.query(`DELETE FROM symbol_edges WHERE ${where}`, values);
129
- await client.query(`DELETE FROM scanned_files WHERE ${cacheWhere}`, values);
130
- await client.query('COMMIT');
131
- return Number(counted.rows[0]?.['n'] ?? 0);
132
- }
133
- catch (error) {
134
- await client.query('ROLLBACK');
135
- throw error;
136
- }
137
- finally {
138
- client.release();
139
- }
160
+ return deleteGraphRows(pool, project, scoped ? files : null);
140
161
  }
@@ -1,5 +1,5 @@
1
- import type { CodeEdge, CodeSymbol, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
2
- import type { MemoryStore, ProjectSummary } from '../adapter.js';
1
+ import type { CodeEdge, CodeSymbol, CountFilter, EdgeFilter, ListFilter, Observation, ObservationIndexEntry, ProjectFilter, RemoveFilter, ScannedFile, SearchQuery, Session, SymbolFilter, TimelineQuery } from '../../types.js';
2
+ import type { GraphRows, MemoryStore, ProjectSummary } from '../adapter.js';
3
3
  export declare class PostgresStore implements MemoryStore {
4
4
  private readonly pool;
5
5
  readonly kind = "postgres";
@@ -33,6 +33,7 @@ export declare class PostgresStore implements MemoryStore {
33
33
  getObservations(ids: string[]): Promise<Observation[]>;
34
34
  remove(filter: RemoveFilter): Promise<number>;
35
35
  list(filter: ListFilter): Promise<Observation[]>;
36
+ count(filter: CountFilter): Promise<number>;
36
37
  listProjects(): Promise<ProjectSummary[]>;
37
38
  inventory(): Promise<string[]>;
38
39
  searchKeyword(query: SearchQuery): Promise<ObservationIndexEntry[]>;
@@ -41,13 +42,11 @@ export declare class PostgresStore implements MemoryStore {
41
42
  markReplaced(ids: string[]): Promise<number>;
42
43
  timeline(query: TimelineQuery): Promise<ObservationIndexEntry[]>;
43
44
  tryEnableVector(): Promise<boolean>;
44
- upsertGraph(scan: {
45
- symbols: CodeSymbol[];
46
- edges: CodeEdge[];
47
- files: ScannedFile[];
48
- }): Promise<void>;
45
+ upsertGraph(scan: GraphRows): Promise<void>;
46
+ replaceGraph(project: string, files: string[] | null, scan: GraphRows): Promise<void>;
49
47
  findSymbols(filter: SymbolFilter): Promise<CodeSymbol[]>;
48
+ symbolNames(project: string, limit: number): Promise<string[]>;
50
49
  findEdges(filter: EdgeFilter): Promise<CodeEdge[]>;
51
- scannedFiles(project: string): Promise<ScannedFile[]>;
50
+ scannedFiles(project: string, limit?: number): Promise<ScannedFile[]>;
52
51
  removeGraph(project: string, files?: string[]): Promise<number>;
53
52
  }
@@ -6,7 +6,7 @@ import * as observationsOps from './observations.js';
6
6
  import * as sessionsOps from './sessions.js';
7
7
  import * as graphOps from './graph.js';
8
8
  import * as projectLinksOps from './project-links.js';
9
- import { importPg } from './driver.js';
9
+ import { openPool } from './driver.js';
10
10
  import { DDL, SCHEMA_VERSION } from './schema.js';
11
11
  export class PostgresStore {
12
12
  pool;
@@ -18,9 +18,7 @@ export class PostgresStore {
18
18
  this.pool = pool;
19
19
  }
20
20
  static async create(uri) {
21
- const pg = await importPg();
22
- const pool = new pg.Pool({ connectionString: uri });
23
- return new PostgresStore(pool);
21
+ return new PostgresStore(await openPool(uri));
24
22
  }
25
23
  async init() {
26
24
  const meta = await this.readMeta();
@@ -58,8 +56,7 @@ export class PostgresStore {
58
56
  await this.pool.query('ALTER TABLE observations DROP COLUMN IF EXISTS embedding');
59
57
  await this.pool.query(`ALTER TABLE observations ADD COLUMN embedding vector(${dims})`);
60
58
  await this.pool.query('UPDATE observations SET embedder = NULL');
61
- await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
62
- ON observations USING hnsw (embedding vector_cosine_ops)`);
59
+ await metaOps.indexEmbeddings(this.pool);
63
60
  this.vectorDims = await this.readVectorDims();
64
61
  this.warnedDims = false;
65
62
  await this.writeMeta();
@@ -75,8 +72,7 @@ export class PostgresStore {
75
72
  return false;
76
73
  if (this.vectorDims === null) {
77
74
  await this.pool.query(`ALTER TABLE observations ADD COLUMN IF NOT EXISTS embedding vector(${dims})`);
78
- await this.pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
79
- ON observations USING hnsw (embedding vector_cosine_ops)`);
75
+ await metaOps.indexEmbeddings(this.pool);
80
76
  this.vectorDims = await this.readVectorDims();
81
77
  await this.writeMeta();
82
78
  }
@@ -125,7 +121,10 @@ export class PostgresStore {
125
121
  return observationsOps.remove(this.pool, filter);
126
122
  }
127
123
  async list(filter) {
128
- return observationsOps.list(this.pool, filter);
124
+ return observationsOps.list(this.pool, filter, this.vectorDims !== null);
125
+ }
126
+ async count(filter) {
127
+ return observationsOps.count(this.pool, filter);
129
128
  }
130
129
  async listProjects() {
131
130
  return observationsOps.listProjects(this.pool);
@@ -154,14 +153,20 @@ export class PostgresStore {
154
153
  async upsertGraph(scan) {
155
154
  return graphOps.upsertGraph(this.pool, scan);
156
155
  }
156
+ async replaceGraph(project, files, scan) {
157
+ return graphOps.replaceGraph(this.pool, project, files, scan);
158
+ }
157
159
  async findSymbols(filter) {
158
160
  return graphOps.findSymbols(this.pool, filter);
159
161
  }
162
+ async symbolNames(project, limit) {
163
+ return graphOps.symbolNames(this.pool, project, limit);
164
+ }
160
165
  async findEdges(filter) {
161
166
  return graphOps.findEdges(this.pool, filter);
162
167
  }
163
- async scannedFiles(project) {
164
- return graphOps.scannedFiles(this.pool, project);
168
+ async scannedFiles(project, limit) {
169
+ return graphOps.scannedFiles(this.pool, project, limit);
165
170
  }
166
171
  async removeGraph(project, files) {
167
172
  return graphOps.removeGraph(this.pool, project, files);
@@ -1,54 +1,91 @@
1
- export async function insertObservations(pool, ensureVectorColumn, observations, hasVectorColumn = false) {
2
- if (observations.length === 0)
3
- return;
4
- const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
5
- const vectors = width !== undefined && (await ensureVectorColumn(width));
1
+ const COLUMNS = [
2
+ 'id',
3
+ 'session_id',
4
+ 'project',
5
+ 'kind',
6
+ 'title',
7
+ 'body',
8
+ 'files',
9
+ 'tags',
10
+ 'created_at',
11
+ 'embedder',
12
+ 'author',
13
+ 'status',
14
+ ];
15
+ const ROWS_PER_STATEMENT = 400;
16
+ const KEEP_CLOSED = `status = CASE
17
+ WHEN observations.status = 'done' AND EXCLUDED.status = 'open'
18
+ AND observations.files = EXCLUDED.files THEN observations.status
19
+ ELSE EXCLUDED.status END`;
20
+ function rowValues(obs, shape) {
21
+ const values = [
22
+ obs.id,
23
+ obs.sessionId,
24
+ obs.project,
25
+ obs.kind,
26
+ obs.title,
27
+ obs.body,
28
+ JSON.stringify(obs.files),
29
+ JSON.stringify(obs.tags),
30
+ obs.createdAt,
31
+ shape.vectorStored ? (obs.embedder ?? null) : null,
32
+ obs.author ?? null,
33
+ obs.status ?? 'done',
34
+ ];
35
+ if (shape.vectorStored && obs.embedding)
36
+ values.push(`[${obs.embedding.join(',')}]`);
37
+ return values;
38
+ }
39
+ function statement(rows, shape, clearVector) {
40
+ const columns = shape.vectorStored ? [...COLUMNS, 'embedding'] : COLUMNS;
41
+ const values = [];
42
+ const tuples = rows.map((obs) => {
43
+ const start = values.length;
44
+ values.push(...rowValues(obs, shape));
45
+ return `(${columns.map((_, i) => `$${start + i + 1}`).join(', ')})`;
46
+ });
47
+ const updates = columns
48
+ .slice(1)
49
+ .filter((column) => !(shape.vectorKept && column === 'embedder'))
50
+ .map((column) => (column === 'status' ? KEEP_CLOSED : `${column} = EXCLUDED.${column}`))
51
+ .concat(clearVector ? ['embedding = NULL'] : [])
52
+ .join(', ');
53
+ return {
54
+ sql: `INSERT INTO observations (${columns.join(', ')}) VALUES ${tuples.join(', ')}
55
+ ON CONFLICT (id) DO UPDATE SET ${updates}`,
56
+ values,
57
+ };
58
+ }
59
+ function lastPerId(observations) {
60
+ return [...new Map(observations.map((obs) => [obs.id, obs])).values()];
61
+ }
62
+ function insertStatements(observations, width, vectors, hasVectorColumn) {
63
+ const groups = new Map();
64
+ for (const obs of lastPerId(observations)) {
65
+ const shape = {
66
+ vectorStored: vectors && obs.embedding?.length === width,
67
+ vectorKept: !obs.embedding?.length && obs.embedder != null,
68
+ };
69
+ const id = `${shape.vectorStored}:${shape.vectorKept}`;
70
+ const group = groups.get(id) ?? { shape, rows: [] };
71
+ group.rows.push(obs);
72
+ groups.set(id, group);
73
+ }
74
+ const statements = [];
75
+ for (const { shape, rows } of groups.values()) {
76
+ const clearVector = !shape.vectorStored && !shape.vectorKept && (hasVectorColumn || vectors);
77
+ for (let i = 0; i < rows.length; i += ROWS_PER_STATEMENT) {
78
+ statements.push(statement(rows.slice(i, i + ROWS_PER_STATEMENT), shape, clearVector));
79
+ }
80
+ }
81
+ return statements;
82
+ }
83
+ async function inTransaction(pool, statements) {
6
84
  const client = await pool.connect();
7
85
  try {
8
86
  await client.query('BEGIN');
9
- for (const obs of observations) {
10
- const vectorStored = vectors && obs.embedding?.length === width;
11
- const columns = [
12
- 'id',
13
- 'session_id',
14
- 'project',
15
- 'kind',
16
- 'title',
17
- 'body',
18
- 'files',
19
- 'tags',
20
- 'created_at',
21
- 'embedder',
22
- 'author',
23
- 'status',
24
- ];
25
- const values = [
26
- obs.id,
27
- obs.sessionId,
28
- obs.project,
29
- obs.kind,
30
- obs.title,
31
- obs.body,
32
- JSON.stringify(obs.files),
33
- JSON.stringify(obs.tags),
34
- obs.createdAt,
35
- vectorStored ? (obs.embedder ?? null) : null,
36
- obs.author ?? null,
37
- obs.status ?? 'done',
38
- ];
39
- if (vectorStored && obs.embedding) {
40
- columns.push('embedding');
41
- values.push(`[${obs.embedding.join(',')}]`);
42
- }
43
- const holes = values.map((_, i) => `$${i + 1}`).join(', ');
44
- const updates = columns
45
- .slice(1)
46
- .map((column) => `${column} = EXCLUDED.${column}`)
47
- .concat(!vectorStored && (hasVectorColumn || vectors) ? ['embedding = NULL'] : [])
48
- .join(', ');
49
- await client.query(`INSERT INTO observations (${columns.join(', ')}) VALUES (${holes})
50
- ON CONFLICT (id) DO UPDATE SET ${updates}`, values);
51
- }
87
+ for (const { sql, values } of statements)
88
+ await client.query(sql, values);
52
89
  await client.query('COMMIT');
53
90
  }
54
91
  catch (error) {
@@ -59,3 +96,16 @@ export async function insertObservations(pool, ensureVectorColumn, observations,
59
96
  client.release();
60
97
  }
61
98
  }
99
+ export async function insertObservations(pool, ensureVectorColumn, observations, hasVectorColumn = false) {
100
+ if (observations.length === 0)
101
+ return;
102
+ const width = observations.find((obs) => obs.embedding?.length)?.embedding?.length;
103
+ const vectors = width !== undefined && (await ensureVectorColumn(width));
104
+ const statements = insertStatements(observations, width, vectors, hasVectorColumn);
105
+ const [only] = statements;
106
+ if (statements.length === 1 && only) {
107
+ await pool.query(only.sql, only.values);
108
+ return;
109
+ }
110
+ await inTransaction(pool, statements);
111
+ }
@@ -8,3 +8,4 @@ export declare function ensureTagsIndexed(pool: Pool): Promise<void>;
8
8
  export declare function readVectorDims(pool: Pool): Promise<number | null>;
9
9
  export declare function tryEnableVector(pool: Pool): Promise<boolean>;
10
10
  export declare function writeMeta(pool: Pool, vectorEnabled: boolean, vectorDims: number | null): Promise<void>;
11
+ export declare function indexEmbeddings(pool: Pool): Promise<void>;
@@ -65,3 +65,7 @@ export async function writeMeta(pool, vectorEnabled, vectorDims) {
65
65
  vector_enabled = EXCLUDED.vector_enabled,
66
66
  vector_dims = EXCLUDED.vector_dims`, [SCHEMA_VERSION, vectorEnabled, vectorDims]);
67
67
  }
68
+ export async function indexEmbeddings(pool) {
69
+ await pool.query(`CREATE INDEX IF NOT EXISTS idx_obs_embedding
70
+ ON observations USING hnsw (embedding vector_cosine_ops)`);
71
+ }
@@ -1,9 +1,10 @@
1
1
  import type { Pool } from './driver.js';
2
- import type { ListFilter, Observation, ObservationIndexEntry, RemoveFilter, TimelineQuery } from '../../types.js';
2
+ import type { CountFilter, ListFilter, Observation, ObservationIndexEntry, RemoveFilter, TimelineQuery } from '../../types.js';
3
3
  import type { ProjectSummary } from '../adapter.js';
4
4
  export declare function getObservations(pool: Pool, ids: string[]): Promise<Observation[]>;
5
5
  export declare function remove(pool: Pool, filter: RemoveFilter): Promise<number>;
6
- export declare function list(pool: Pool, filter: ListFilter): Promise<Observation[]>;
6
+ export declare function list(pool: Pool, filter: ListFilter, hasVectorColumn?: boolean): Promise<Observation[]>;
7
+ export declare function count(pool: Pool, filter: CountFilter): Promise<number>;
7
8
  export declare function listProjects(pool: Pool): Promise<ProjectSummary[]>;
8
9
  export declare function inventory(pool: Pool): Promise<string[]>;
9
10
  export declare function timeline(pool: Pool, query: TimelineQuery): Promise<ObservationIndexEntry[]>;
@@ -3,6 +3,13 @@ import { partitionIds } from '../../util/shortid.js';
3
3
  import { excludeSessionsClause, projectClause } from './filters.js';
4
4
  import { toIndexEntry, toObservation } from './rows.js';
5
5
  import { noProjects } from '../project-scope.js';
6
+ const COLUMNS = 'id, session_id, project, kind, title, body, files, tags, created_at, embedder, author, status';
7
+ const WITHOUT_BODY = COLUMNS.replace(' body,', '');
8
+ function listColumns(filter) {
9
+ if (filter.vectors)
10
+ return '*';
11
+ return filter.bodies === false ? WITHOUT_BODY : COLUMNS;
12
+ }
6
13
  export async function getObservations(pool, ids) {
7
14
  if (ids.length === 0)
8
15
  return [];
@@ -17,7 +24,7 @@ export async function getObservations(pool, ids) {
17
24
  values.push(`${prefix.replace(/[%_]/g, '')}%`);
18
25
  clauses.push(`id LIKE $${values.length}`);
19
26
  }
20
- const res = await pool.query(`SELECT * FROM observations WHERE ${clauses.join(' OR ')}`, values);
27
+ const res = await pool.query(`SELECT ${COLUMNS} FROM observations WHERE ${clauses.join(' OR ')}`, values);
21
28
  return res.rows.map(toObservation);
22
29
  }
23
30
  export async function remove(pool, filter) {
@@ -76,11 +83,8 @@ export async function remove(pool, filter) {
76
83
  client.release();
77
84
  }
78
85
  }
79
- export async function list(pool, filter) {
80
- if (noProjects(filter.project))
81
- return [];
86
+ function scopeConditions(filter, values) {
82
87
  const conditions = [];
83
- const values = [];
84
88
  const project = projectClause(filter.project, values);
85
89
  if (project)
86
90
  conditions.push(project);
@@ -99,6 +103,17 @@ export async function list(pool, filter) {
99
103
  values.push(filter.status);
100
104
  conditions.push(`status = $${values.length}`);
101
105
  }
106
+ if (filter.tag) {
107
+ values.push(JSON.stringify([filter.tag]));
108
+ conditions.push(`tags @> $${values.length}::jsonb`);
109
+ }
110
+ return conditions;
111
+ }
112
+ export async function list(pool, filter, hasVectorColumn = false) {
113
+ if (noProjects(filter.project))
114
+ return [];
115
+ const values = [];
116
+ const conditions = scopeConditions(filter, values);
102
117
  if (filter.after !== undefined) {
103
118
  if (filter.afterId === undefined) {
104
119
  values.push(filter.after);
@@ -109,13 +124,32 @@ export async function list(pool, filter) {
109
124
  conditions.push(`(created_at > $${values.length - 2} OR (created_at = $${values.length - 1} AND id > $${values.length}))`);
110
125
  }
111
126
  }
127
+ if (filter.vectorStale) {
128
+ const stale = hasVectorColumn
129
+ ? ['embedder IS NULL', 'embedding IS NULL']
130
+ : ['embedder IS NULL'];
131
+ if (filter.vectorStale.embedder !== undefined) {
132
+ values.push(filter.vectorStale.embedder);
133
+ stale.push(`embedder <> $${values.length}`);
134
+ }
135
+ conditions.push(`(${stale.join(' OR ')})`);
136
+ }
112
137
  values.push(filter.limit ?? 1000);
113
138
  const direction = filter.newest ? 'DESC' : 'ASC';
114
- const res = await pool.query(`SELECT * FROM observations
139
+ const res = await pool.query(`SELECT ${listColumns(filter)} FROM observations
115
140
  ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}
116
141
  ORDER BY created_at ${direction}, id ${direction} LIMIT $${values.length}`, values);
117
142
  return res.rows.map(toObservation);
118
143
  }
144
+ export async function count(pool, filter) {
145
+ if (noProjects(filter.project))
146
+ return 0;
147
+ const values = [];
148
+ const conditions = scopeConditions(filter, values);
149
+ const res = await pool.query(`SELECT COUNT(*)::int AS n FROM observations
150
+ ${conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''}`, values);
151
+ return Number(res.rows[0]?.['n'] ?? 0);
152
+ }
119
153
  export async function listProjects(pool) {
120
154
  const res = await pool.query(`SELECT project, COUNT(*)::int AS n, MAX(created_at) AS last
121
155
  FROM observations GROUP BY project ORDER BY last DESC`);
@@ -35,7 +35,7 @@ export function toObservation(row) {
35
35
  project: row['project'],
36
36
  kind: row['kind'],
37
37
  title: row['title'],
38
- body: row['body'],
38
+ body: row['body'] ?? '',
39
39
  files: row['files'] ?? [],
40
40
  tags: row['tags'] ?? [],
41
41
  createdAt: Number(row['created_at']),
@@ -4,12 +4,15 @@ export interface ScopeResolverOptions {
4
4
  remote: string;
5
5
  isFolder?: (value: string) => boolean;
6
6
  keyOf?: (folder: string, remote: string) => string;
7
+ scopeTtlMs?: number;
8
+ now?: () => number;
7
9
  }
8
10
  export interface ProjectScope {
9
11
  key: string;
10
12
  names: string[];
11
13
  }
12
14
  export interface ScopeResolver {
15
+ key(value: string): Promise<string>;
13
16
  resolve(value: string): Promise<ProjectScope>;
14
17
  }
15
18
  export declare function isExistingFolder(value: string): boolean;
@@ -1,7 +1,8 @@
1
1
  import { statSync } from 'node:fs';
2
2
  import { isAbsolute } from 'node:path';
3
3
  import { projectKey } from '../util/project-key.js';
4
- import { orderScope } from './project-scope.js';
4
+ import { isLinkable, orderScope } from './project-scope.js';
5
+ const SCOPE_TTL_MS = 60_000;
5
6
  export function isExistingFolder(value) {
6
7
  if (!isAbsolute(value))
7
8
  return false;
@@ -13,37 +14,47 @@ export function isExistingFolder(value) {
13
14
  }
14
15
  }
15
16
  export function createScopeResolver(options) {
16
- const { store, remote, isFolder = isExistingFolder, keyOf = (folder, name) => projectKey(folder, { remote: name }), } = options;
17
- const keys = new Map();
18
- async function linkedKey(folder) {
19
- const key = keyOf(folder, remote);
17
+ const { store, remote, isFolder = isExistingFolder, keyOf = (folder, name) => projectKey(folder, { remote: name }), scopeTtlMs = SCOPE_TTL_MS, now = Date.now, } = options;
18
+ const scopes = new Map();
19
+ function forgetOthers(folder) {
20
+ for (const cached of [...scopes.keys()])
21
+ if (cached !== folder)
22
+ scopes.delete(cached);
23
+ }
24
+ async function linked(folder) {
20
25
  try {
21
- await store.linkProject(folder, key);
26
+ return await store.projectScope(folder);
22
27
  }
23
28
  catch {
24
- return key;
29
+ return null;
25
30
  }
26
- return key;
27
31
  }
28
- async function namesFor(folder, key) {
29
- try {
30
- return orderScope(folder, [...(await store.projectScope(folder)), key]);
31
- }
32
- catch {
33
- return orderScope(folder, [key]);
32
+ async function lookUp(folder) {
33
+ const key = keyOf(folder, remote);
34
+ let related = await linked(folder);
35
+ if (isLinkable(folder, key) && !related?.includes(key)) {
36
+ const before = related;
37
+ related = await store.linkProject(folder, key).then(() => {
38
+ forgetOthers(folder);
39
+ return linked(folder);
40
+ }, () => before);
34
41
  }
42
+ return { key, names: orderScope(folder, [...(related ?? []), key]) };
43
+ }
44
+ function scopeOf(folder) {
45
+ const cached = scopes.get(folder);
46
+ if (cached && now() - cached.at < scopeTtlMs)
47
+ return cached.scope;
48
+ const scope = lookUp(folder);
49
+ scopes.set(folder, { at: now(), scope });
50
+ return scope;
35
51
  }
36
52
  return {
53
+ async key(value) {
54
+ return isFolder(value) ? (await scopeOf(value)).key : value;
55
+ },
37
56
  async resolve(value) {
38
- if (!isFolder(value))
39
- return { key: value, names: [value] };
40
- let pending = keys.get(value);
41
- if (!pending) {
42
- pending = linkedKey(value);
43
- keys.set(value, pending);
44
- }
45
- const key = await pending;
46
- return { key, names: await namesFor(value, key) };
57
+ return isFolder(value) ? scopeOf(value) : { key: value, names: [value] };
47
58
  },
48
59
  };
49
60
  }