@spences10/pi-context 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -9
- package/dist/index.d.ts +2 -2
- package/dist/index.js +339 -25
- package/dist/index.js.map +1 -1
- package/dist/policy.d.ts +3 -0
- package/dist/policy.js +30 -0
- package/dist/policy.js.map +1 -0
- package/dist/schema.d.ts +2 -0
- package/dist/schema.js +45 -0
- package/dist/schema.js.map +1 -0
- package/dist/schema.sql +49 -0
- package/dist/store.d.ts +26 -66
- package/dist/store.js +326 -269
- package/dist/store.js.map +1 -1
- package/dist/text.d.ts +10 -0
- package/dist/text.js +184 -0
- package/dist/text.js.map +1 -0
- package/dist/types.d.ts +138 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +5 -4
package/dist/store.js
CHANGED
|
@@ -4,65 +4,11 @@ import { existsSync, mkdirSync, statSync } from 'node:fs';
|
|
|
4
4
|
import { homedir } from 'node:os';
|
|
5
5
|
import { dirname, join } from 'node:path';
|
|
6
6
|
import { DatabaseSync } from 'node:sqlite';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
PRAGMA foreign_keys = ON;
|
|
13
|
-
PRAGMA journal_mode = WAL;
|
|
14
|
-
PRAGMA busy_timeout = 5000;
|
|
15
|
-
|
|
16
|
-
CREATE TABLE IF NOT EXISTS context_sources (
|
|
17
|
-
id TEXT PRIMARY KEY,
|
|
18
|
-
session_id TEXT,
|
|
19
|
-
project_path TEXT,
|
|
20
|
-
tool_name TEXT NOT NULL,
|
|
21
|
-
input_summary TEXT,
|
|
22
|
-
created_at INTEGER NOT NULL,
|
|
23
|
-
byte_count INTEGER NOT NULL,
|
|
24
|
-
line_count INTEGER NOT NULL,
|
|
25
|
-
content_hash TEXT NOT NULL,
|
|
26
|
-
preview_byte_count INTEGER NOT NULL DEFAULT 0,
|
|
27
|
-
returned_byte_count INTEGER NOT NULL DEFAULT 0
|
|
28
|
-
);
|
|
29
|
-
|
|
30
|
-
CREATE TABLE IF NOT EXISTS context_chunks (
|
|
31
|
-
id TEXT PRIMARY KEY,
|
|
32
|
-
source_id TEXT NOT NULL REFERENCES context_sources(id) ON DELETE CASCADE,
|
|
33
|
-
ordinal INTEGER NOT NULL,
|
|
34
|
-
title TEXT,
|
|
35
|
-
content TEXT NOT NULL,
|
|
36
|
-
byte_count INTEGER NOT NULL
|
|
37
|
-
);
|
|
38
|
-
|
|
39
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS context_chunks_fts USING fts5(
|
|
40
|
-
title,
|
|
41
|
-
content,
|
|
42
|
-
content='context_chunks',
|
|
43
|
-
content_rowid='rowid'
|
|
44
|
-
);
|
|
45
|
-
|
|
46
|
-
CREATE TRIGGER IF NOT EXISTS context_chunks_ai AFTER INSERT ON context_chunks BEGIN
|
|
47
|
-
INSERT INTO context_chunks_fts(rowid, title, content) VALUES (new.rowid, new.title, new.content);
|
|
48
|
-
END;
|
|
49
|
-
|
|
50
|
-
CREATE TRIGGER IF NOT EXISTS context_chunks_ad AFTER DELETE ON context_chunks BEGIN
|
|
51
|
-
INSERT INTO context_chunks_fts(context_chunks_fts, rowid, title, content)
|
|
52
|
-
VALUES('delete', old.rowid, old.title, old.content);
|
|
53
|
-
END;
|
|
54
|
-
|
|
55
|
-
CREATE TRIGGER IF NOT EXISTS context_chunks_au AFTER UPDATE ON context_chunks BEGIN
|
|
56
|
-
INSERT INTO context_chunks_fts(context_chunks_fts, rowid, title, content)
|
|
57
|
-
VALUES('delete', old.rowid, old.title, old.content);
|
|
58
|
-
INSERT INTO context_chunks_fts(rowid, title, content) VALUES (new.rowid, new.title, new.content);
|
|
59
|
-
END;
|
|
60
|
-
|
|
61
|
-
CREATE INDEX IF NOT EXISTS idx_context_sources_created ON context_sources(created_at);
|
|
62
|
-
CREATE INDEX IF NOT EXISTS idx_context_sources_session ON context_sources(session_id);
|
|
63
|
-
CREATE INDEX IF NOT EXISTS idx_context_sources_project ON context_sources(project_path);
|
|
64
|
-
CREATE INDEX IF NOT EXISTS idx_context_chunks_source ON context_chunks(source_id, ordinal);
|
|
65
|
-
`;
|
|
7
|
+
import { parse_context_retention_policy } from './policy.js';
|
|
8
|
+
import { apply_schema } from './schema.js';
|
|
9
|
+
import { DEFAULT_CONTEXT_MAX_BYTES, DEFAULT_CONTEXT_MAX_LINES, chunk_text, count_lines, escape_fts5_query, make_preview, should_index_text, summarize_source, } from './text.js';
|
|
10
|
+
export { DEFAULT_CONTEXT_RETENTION_DAYS, parse_context_retention_policy, } from './policy.js';
|
|
11
|
+
export { DEFAULT_CONTEXT_MAX_BYTES, DEFAULT_CONTEXT_MAX_LINES, count_lines, escape_fts5_query, make_preview, should_index_text, } from './text.js';
|
|
66
12
|
let global_options = {};
|
|
67
13
|
let global_enabled = false;
|
|
68
14
|
let global_store = null;
|
|
@@ -75,9 +21,12 @@ export function default_context_db_path() {
|
|
|
75
21
|
}
|
|
76
22
|
export function set_context_sidecar_enabled(enabled, options = {}) {
|
|
77
23
|
global_enabled = enabled;
|
|
78
|
-
|
|
79
|
-
|
|
24
|
+
if (!enabled) {
|
|
25
|
+
global_options = {};
|
|
80
26
|
global_store = null;
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
global_options = { ...global_options, ...options };
|
|
81
30
|
}
|
|
82
31
|
export function is_context_sidecar_enabled() {
|
|
83
32
|
return global_enabled;
|
|
@@ -88,6 +37,9 @@ export function get_context_store(options = {}) {
|
|
|
88
37
|
if (!global_store || global_store.db_path !== db_path) {
|
|
89
38
|
global_store = new ContextStore({ ...merged, db_path });
|
|
90
39
|
}
|
|
40
|
+
else {
|
|
41
|
+
global_store.configure(merged);
|
|
42
|
+
}
|
|
91
43
|
return global_store;
|
|
92
44
|
}
|
|
93
45
|
export function maybe_store_context_output(input, options = {}) {
|
|
@@ -95,177 +47,6 @@ export function maybe_store_context_output(input, options = {}) {
|
|
|
95
47
|
return null;
|
|
96
48
|
return get_context_store(options).store(input);
|
|
97
49
|
}
|
|
98
|
-
export function count_lines(text) {
|
|
99
|
-
if (!text)
|
|
100
|
-
return 0;
|
|
101
|
-
return text.split('\n').length;
|
|
102
|
-
}
|
|
103
|
-
export function should_index_text(text, options = {}) {
|
|
104
|
-
const max_bytes = options.max_bytes ?? DEFAULT_CONTEXT_MAX_BYTES;
|
|
105
|
-
const max_lines = options.max_lines ?? DEFAULT_CONTEXT_MAX_LINES;
|
|
106
|
-
return (Buffer.byteLength(text, 'utf8') > max_bytes ||
|
|
107
|
-
count_lines(text) > max_lines);
|
|
108
|
-
}
|
|
109
|
-
export function escape_fts5_query(query) {
|
|
110
|
-
const trimmed = query.trim();
|
|
111
|
-
if (!trimmed)
|
|
112
|
-
return '""';
|
|
113
|
-
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
114
|
-
return trimmed.replace(/"(.*)"/s, (_match, inner) => `"${inner.replace(/"/g, '""')}"`);
|
|
115
|
-
}
|
|
116
|
-
const tokens = trimmed
|
|
117
|
-
.split(/\s+/)
|
|
118
|
-
.map((token) => token.trim())
|
|
119
|
-
.filter(Boolean)
|
|
120
|
-
.map((token) => {
|
|
121
|
-
const is_prefix = token.endsWith('*');
|
|
122
|
-
const base = is_prefix ? token.slice(0, -1) : token;
|
|
123
|
-
const safe = base
|
|
124
|
-
.replace(/["'(){}[\]^:./\\+-]/g, ' ')
|
|
125
|
-
.trim()
|
|
126
|
-
.replace(/\s+/g, ' ');
|
|
127
|
-
if (!safe)
|
|
128
|
-
return '';
|
|
129
|
-
const quoted = `"${safe.replace(/"/g, '""')}"`;
|
|
130
|
-
return is_prefix ? `${quoted}*` : quoted;
|
|
131
|
-
})
|
|
132
|
-
.filter(Boolean);
|
|
133
|
-
return tokens.length > 0 ? tokens.join(' ') : '""';
|
|
134
|
-
}
|
|
135
|
-
export function make_preview(text, max_lines = DEFAULT_PREVIEW_LINES, max_bytes = DEFAULT_PREVIEW_BYTES) {
|
|
136
|
-
const lines = text.split('\n');
|
|
137
|
-
let preview;
|
|
138
|
-
if (lines.length <= max_lines) {
|
|
139
|
-
preview = text;
|
|
140
|
-
}
|
|
141
|
-
else {
|
|
142
|
-
const head_count = Math.ceil(max_lines / 2);
|
|
143
|
-
const tail_count = Math.floor(max_lines / 2);
|
|
144
|
-
const omitted = lines.length - head_count - tail_count;
|
|
145
|
-
preview = [
|
|
146
|
-
...lines.slice(0, head_count),
|
|
147
|
-
``,
|
|
148
|
-
`[... ${omitted} lines omitted; indexed in context sidecar ...]`,
|
|
149
|
-
``,
|
|
150
|
-
...lines.slice(-tail_count),
|
|
151
|
-
].join('\n');
|
|
152
|
-
}
|
|
153
|
-
return take_utf8_bytes(preview, max_bytes);
|
|
154
|
-
}
|
|
155
|
-
function take_utf8_bytes(text, max_bytes) {
|
|
156
|
-
if (Buffer.byteLength(text, 'utf8') <= max_bytes)
|
|
157
|
-
return text;
|
|
158
|
-
let bytes = 0;
|
|
159
|
-
let output = '';
|
|
160
|
-
for (const char of text) {
|
|
161
|
-
const char_bytes = Buffer.byteLength(char, 'utf8');
|
|
162
|
-
if (bytes + char_bytes > max_bytes)
|
|
163
|
-
break;
|
|
164
|
-
bytes += char_bytes;
|
|
165
|
-
output += char;
|
|
166
|
-
}
|
|
167
|
-
return `${output}\n[... preview truncated at ${format_bytes(max_bytes)} ...]`;
|
|
168
|
-
}
|
|
169
|
-
function chunk_text(text, source_id) {
|
|
170
|
-
const paragraphs = text.split(/\n{2,}/);
|
|
171
|
-
const chunks = [];
|
|
172
|
-
let current = '';
|
|
173
|
-
const target_bytes = 4096;
|
|
174
|
-
for (const paragraph of paragraphs) {
|
|
175
|
-
if (Buffer.byteLength(paragraph, 'utf8') > target_bytes) {
|
|
176
|
-
if (current)
|
|
177
|
-
chunks.push(current);
|
|
178
|
-
chunks.push(...split_large_chunk(paragraph, target_bytes));
|
|
179
|
-
current = '';
|
|
180
|
-
continue;
|
|
181
|
-
}
|
|
182
|
-
const next = current ? `${current}\n\n${paragraph}` : paragraph;
|
|
183
|
-
if (Buffer.byteLength(next, 'utf8') > target_bytes && current) {
|
|
184
|
-
chunks.push(current);
|
|
185
|
-
current = paragraph;
|
|
186
|
-
}
|
|
187
|
-
else {
|
|
188
|
-
current = next;
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
if (current)
|
|
192
|
-
chunks.push(current);
|
|
193
|
-
if (chunks.length === 0)
|
|
194
|
-
chunks.push(text);
|
|
195
|
-
return chunks.map((content, index) => ({
|
|
196
|
-
id: `${source_id}_${String(index + 1).padStart(4, '0')}`,
|
|
197
|
-
source_id,
|
|
198
|
-
ordinal: index + 1,
|
|
199
|
-
title: first_non_empty_line(content),
|
|
200
|
-
content,
|
|
201
|
-
byte_count: Buffer.byteLength(content, 'utf8'),
|
|
202
|
-
}));
|
|
203
|
-
}
|
|
204
|
-
function split_large_chunk(text, target_bytes) {
|
|
205
|
-
const chunks = [];
|
|
206
|
-
let current = '';
|
|
207
|
-
for (const line of text.split('\n')) {
|
|
208
|
-
const next = current ? `${current}\n${line}` : line;
|
|
209
|
-
if (Buffer.byteLength(next, 'utf8') <= target_bytes) {
|
|
210
|
-
current = next;
|
|
211
|
-
continue;
|
|
212
|
-
}
|
|
213
|
-
if (current)
|
|
214
|
-
chunks.push(current);
|
|
215
|
-
if (Buffer.byteLength(line, 'utf8') <= target_bytes) {
|
|
216
|
-
current = line;
|
|
217
|
-
continue;
|
|
218
|
-
}
|
|
219
|
-
let rest = line;
|
|
220
|
-
while (Buffer.byteLength(rest, 'utf8') > target_bytes) {
|
|
221
|
-
const [head, tail] = split_utf8_at_byte(rest, target_bytes);
|
|
222
|
-
chunks.push(head);
|
|
223
|
-
rest = tail;
|
|
224
|
-
}
|
|
225
|
-
current = rest;
|
|
226
|
-
}
|
|
227
|
-
if (current)
|
|
228
|
-
chunks.push(current);
|
|
229
|
-
return chunks;
|
|
230
|
-
}
|
|
231
|
-
function split_utf8_at_byte(text, max_bytes) {
|
|
232
|
-
let bytes = 0;
|
|
233
|
-
let index = 0;
|
|
234
|
-
for (const char of text) {
|
|
235
|
-
const char_bytes = Buffer.byteLength(char, 'utf8');
|
|
236
|
-
if (bytes + char_bytes > max_bytes)
|
|
237
|
-
break;
|
|
238
|
-
bytes += char_bytes;
|
|
239
|
-
index += char.length;
|
|
240
|
-
}
|
|
241
|
-
return [text.slice(0, index), text.slice(index)];
|
|
242
|
-
}
|
|
243
|
-
function first_non_empty_line(text) {
|
|
244
|
-
const line = text
|
|
245
|
-
.split('\n')
|
|
246
|
-
.map((value) => value.trim())
|
|
247
|
-
.find(Boolean);
|
|
248
|
-
return line ? line.slice(0, 120) : null;
|
|
249
|
-
}
|
|
250
|
-
function format_bytes(bytes) {
|
|
251
|
-
if (bytes < 1024)
|
|
252
|
-
return `${bytes} B`;
|
|
253
|
-
if (bytes < 1024 * 1024)
|
|
254
|
-
return `${(bytes / 1024).toFixed(1)} KiB`;
|
|
255
|
-
return `${(bytes / 1024 / 1024).toFixed(1)} MiB`;
|
|
256
|
-
}
|
|
257
|
-
function summarize_source(result, tool_name) {
|
|
258
|
-
return [
|
|
259
|
-
`[context-sidecar] Large ${tool_name} output indexed locally`,
|
|
260
|
-
``,
|
|
261
|
-
`Source: ${result.source_id}`,
|
|
262
|
-
`Size: ${format_bytes(result.bytes)}, ${result.lines} lines, ${result.chunk_count} chunks`,
|
|
263
|
-
`Use context_search query:"..." source_id:"${result.source_id}" to inspect it.`,
|
|
264
|
-
`Use context_get source_id:"${result.source_id}" for exact chunks.`,
|
|
265
|
-
``,
|
|
266
|
-
result.preview,
|
|
267
|
-
].join('\n');
|
|
268
|
-
}
|
|
269
50
|
export class ContextStore {
|
|
270
51
|
db_path;
|
|
271
52
|
db;
|
|
@@ -285,7 +66,69 @@ export class ContextStore {
|
|
|
285
66
|
this.db = new DatabaseSync(this.db_path, {
|
|
286
67
|
enableForeignKeyConstraints: true,
|
|
287
68
|
});
|
|
288
|
-
this.db
|
|
69
|
+
apply_schema(this.db);
|
|
70
|
+
}
|
|
71
|
+
configure(options = {}) {
|
|
72
|
+
if (options.project_path !== undefined)
|
|
73
|
+
this.project_path = options.project_path;
|
|
74
|
+
if (options.session_id !== undefined)
|
|
75
|
+
this.session_id = options.session_id;
|
|
76
|
+
if (options.max_bytes !== undefined)
|
|
77
|
+
this.max_bytes = options.max_bytes;
|
|
78
|
+
if (options.max_lines !== undefined)
|
|
79
|
+
this.max_lines = options.max_lines;
|
|
80
|
+
}
|
|
81
|
+
scoped_filter(alias, options = {}) {
|
|
82
|
+
const where = [];
|
|
83
|
+
const params = [];
|
|
84
|
+
if (options.session_id === null) {
|
|
85
|
+
where.push(`${alias}.session_id IS NULL`);
|
|
86
|
+
}
|
|
87
|
+
else if (options.session_id !== undefined) {
|
|
88
|
+
where.push(`${alias}.session_id = ?`);
|
|
89
|
+
params.push(options.session_id);
|
|
90
|
+
}
|
|
91
|
+
else if (!options.global && this.session_id) {
|
|
92
|
+
where.push(`${alias}.session_id = ?`);
|
|
93
|
+
params.push(this.session_id);
|
|
94
|
+
}
|
|
95
|
+
if (options.project_path === null) {
|
|
96
|
+
where.push(`${alias}.project_path IS NULL`);
|
|
97
|
+
}
|
|
98
|
+
else if (options.project_path !== undefined) {
|
|
99
|
+
where.push(`${alias}.project_path = ?`);
|
|
100
|
+
params.push(options.project_path);
|
|
101
|
+
}
|
|
102
|
+
else if (!options.global &&
|
|
103
|
+
where.length === 0 &&
|
|
104
|
+
this.project_path) {
|
|
105
|
+
where.push(`${alias}.project_path = ?`);
|
|
106
|
+
params.push(this.project_path);
|
|
107
|
+
}
|
|
108
|
+
return { where, params };
|
|
109
|
+
}
|
|
110
|
+
find_duplicate_source(content_hash, scope) {
|
|
111
|
+
const scoped = this.scoped_filter('context_sources', scope);
|
|
112
|
+
const filters = [
|
|
113
|
+
'context_sources.content_hash = ?',
|
|
114
|
+
...scoped.where,
|
|
115
|
+
];
|
|
116
|
+
const params = [
|
|
117
|
+
content_hash,
|
|
118
|
+
...scoped.params,
|
|
119
|
+
];
|
|
120
|
+
const row = this.db
|
|
121
|
+
.prepare(`
|
|
122
|
+
SELECT context_sources.id, COUNT(context_chunks.id) as chunk_count
|
|
123
|
+
FROM context_sources
|
|
124
|
+
LEFT JOIN context_chunks ON context_chunks.source_id = context_sources.id
|
|
125
|
+
WHERE ${filters.join(' AND ')}
|
|
126
|
+
GROUP BY context_sources.id
|
|
127
|
+
ORDER BY context_sources.created_at DESC
|
|
128
|
+
LIMIT 1
|
|
129
|
+
`)
|
|
130
|
+
.get(...params);
|
|
131
|
+
return row ?? null;
|
|
289
132
|
}
|
|
290
133
|
store(input) {
|
|
291
134
|
const redaction = redact_text(input.text);
|
|
@@ -298,13 +141,39 @@ export class ContextStore {
|
|
|
298
141
|
return null;
|
|
299
142
|
const bytes = Buffer.byteLength(text, 'utf8');
|
|
300
143
|
const lines = count_lines(text);
|
|
301
|
-
const source_id = `ctx_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
|
|
302
144
|
const created_at = Date.now();
|
|
303
145
|
const content_hash = createHash('sha256')
|
|
304
146
|
.update(text)
|
|
305
147
|
.digest('hex');
|
|
306
|
-
const
|
|
148
|
+
const session_id = input.session_id ?? this.session_id;
|
|
149
|
+
const project_path = input.project_path ?? this.project_path;
|
|
307
150
|
const preview = make_preview(text);
|
|
151
|
+
const duplicate = this.find_duplicate_source(content_hash, {
|
|
152
|
+
session_id,
|
|
153
|
+
project_path,
|
|
154
|
+
});
|
|
155
|
+
if (duplicate) {
|
|
156
|
+
const provisional = {
|
|
157
|
+
source_id: duplicate.id,
|
|
158
|
+
bytes,
|
|
159
|
+
lines,
|
|
160
|
+
preview,
|
|
161
|
+
receipt: '',
|
|
162
|
+
chunk_count: duplicate.chunk_count,
|
|
163
|
+
returned_bytes: 0,
|
|
164
|
+
project_path,
|
|
165
|
+
session_id,
|
|
166
|
+
deduped: true,
|
|
167
|
+
};
|
|
168
|
+
const receipt = summarize_source(provisional, input.tool_name);
|
|
169
|
+
const returned_bytes = Buffer.byteLength(receipt, 'utf8');
|
|
170
|
+
this.db
|
|
171
|
+
.prepare('UPDATE context_sources SET returned_byte_count = returned_byte_count + ? WHERE id = ?')
|
|
172
|
+
.run(returned_bytes, duplicate.id);
|
|
173
|
+
return { ...provisional, receipt, returned_bytes };
|
|
174
|
+
}
|
|
175
|
+
const source_id = `ctx_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
|
|
176
|
+
const chunks = chunk_text(text, source_id);
|
|
308
177
|
const preview_bytes = Buffer.byteLength(preview, 'utf8');
|
|
309
178
|
const insert = this.db.prepare(`
|
|
310
179
|
INSERT INTO context_sources (
|
|
@@ -321,7 +190,7 @@ export class ContextStore {
|
|
|
321
190
|
`);
|
|
322
191
|
this.db.exec('BEGIN');
|
|
323
192
|
try {
|
|
324
|
-
insert.run(source_id,
|
|
193
|
+
insert.run(source_id, session_id, project_path, input.tool_name, input.input_summary ?? null, created_at, bytes, lines, content_hash, preview_bytes);
|
|
325
194
|
for (const chunk of chunks) {
|
|
326
195
|
insert_chunk.run(chunk.id, chunk.source_id, chunk.ordinal, chunk.title, chunk.content, chunk.byte_count);
|
|
327
196
|
}
|
|
@@ -333,6 +202,8 @@ export class ContextStore {
|
|
|
333
202
|
receipt: '',
|
|
334
203
|
chunk_count: chunks.length,
|
|
335
204
|
returned_bytes: 0,
|
|
205
|
+
project_path,
|
|
206
|
+
session_id,
|
|
336
207
|
};
|
|
337
208
|
const receipt = summarize_source(provisional, input.tool_name);
|
|
338
209
|
const returned_bytes = Buffer.byteLength(receipt, 'utf8');
|
|
@@ -345,11 +216,80 @@ export class ContextStore {
|
|
|
345
216
|
throw error;
|
|
346
217
|
}
|
|
347
218
|
}
|
|
219
|
+
list(options = {}) {
|
|
220
|
+
const limit = Math.max(1, Math.min(options.limit ?? 10, 50));
|
|
221
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
222
|
+
const scoped = this.scoped_filter('context_sources', options);
|
|
223
|
+
const filters = [...scoped.where];
|
|
224
|
+
const params = [...scoped.params];
|
|
225
|
+
if (options.source_id) {
|
|
226
|
+
filters.push('context_sources.id = ?');
|
|
227
|
+
params.push(options.source_id);
|
|
228
|
+
}
|
|
229
|
+
if (options.tool_name) {
|
|
230
|
+
filters.push('context_sources.tool_name = ?');
|
|
231
|
+
params.push(options.tool_name);
|
|
232
|
+
}
|
|
233
|
+
if (options.newer_than_days !== undefined) {
|
|
234
|
+
filters.push('context_sources.created_at >= ?');
|
|
235
|
+
params.push(Date.now() - options.newer_than_days * 24 * 60 * 60 * 1000);
|
|
236
|
+
}
|
|
237
|
+
if (options.older_than_days !== undefined) {
|
|
238
|
+
filters.push('context_sources.created_at < ?');
|
|
239
|
+
params.push(Date.now() - options.older_than_days * 24 * 60 * 60 * 1000);
|
|
240
|
+
}
|
|
241
|
+
params.push(limit, offset);
|
|
242
|
+
const where_clause = filters.length
|
|
243
|
+
? `WHERE ${filters.join(' AND ')}`
|
|
244
|
+
: '';
|
|
245
|
+
const stmt = this.db.prepare(`
|
|
246
|
+
SELECT
|
|
247
|
+
context_sources.id as source_id,
|
|
248
|
+
context_sources.created_at,
|
|
249
|
+
context_sources.project_path,
|
|
250
|
+
context_sources.session_id,
|
|
251
|
+
context_sources.tool_name,
|
|
252
|
+
context_sources.input_summary,
|
|
253
|
+
context_sources.byte_count,
|
|
254
|
+
context_sources.line_count,
|
|
255
|
+
COUNT(context_chunks.id) as chunk_count,
|
|
256
|
+
(
|
|
257
|
+
SELECT title FROM context_chunks first_chunk
|
|
258
|
+
WHERE first_chunk.source_id = context_sources.id
|
|
259
|
+
ORDER BY ordinal LIMIT 1
|
|
260
|
+
) as first_chunk_title,
|
|
261
|
+
(
|
|
262
|
+
SELECT substr(content, 1, 240) FROM context_chunks first_chunk
|
|
263
|
+
WHERE first_chunk.source_id = context_sources.id
|
|
264
|
+
ORDER BY ordinal LIMIT 1
|
|
265
|
+
) as preview
|
|
266
|
+
FROM context_sources
|
|
267
|
+
LEFT JOIN context_chunks ON context_chunks.source_id = context_sources.id
|
|
268
|
+
${where_clause}
|
|
269
|
+
GROUP BY context_sources.id
|
|
270
|
+
ORDER BY context_sources.created_at DESC
|
|
271
|
+
LIMIT ? OFFSET ?
|
|
272
|
+
`);
|
|
273
|
+
return stmt.all(...params).map((row) => ({
|
|
274
|
+
source_id: row.source_id,
|
|
275
|
+
created_at: row.created_at,
|
|
276
|
+
project_path: row.project_path,
|
|
277
|
+
session_id: row.session_id,
|
|
278
|
+
tool_name: row.tool_name,
|
|
279
|
+
input_summary: row.input_summary,
|
|
280
|
+
bytes: row.byte_count,
|
|
281
|
+
lines: row.line_count,
|
|
282
|
+
chunk_count: row.chunk_count,
|
|
283
|
+
first_chunk_title: row.first_chunk_title,
|
|
284
|
+
preview: row.preview,
|
|
285
|
+
}));
|
|
286
|
+
}
|
|
348
287
|
search(query, options = {}) {
|
|
349
288
|
const limit = Math.max(1, Math.min(options.limit ?? 5, 25));
|
|
350
289
|
const match = escape_fts5_query(query);
|
|
351
|
-
const
|
|
352
|
-
const
|
|
290
|
+
const scoped = this.scoped_filter('context_sources', options);
|
|
291
|
+
const filters = [...scoped.where];
|
|
292
|
+
const params = [match, ...scoped.params];
|
|
353
293
|
if (options.source_id) {
|
|
354
294
|
filters.push('context_sources.id = ?');
|
|
355
295
|
params.push(options.source_id);
|
|
@@ -394,41 +334,74 @@ export class ContextStore {
|
|
|
394
334
|
rank: row.rank,
|
|
395
335
|
}));
|
|
396
336
|
}
|
|
397
|
-
get(source_id, chunk_id) {
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
337
|
+
get(source_id, chunk_id, options = {}) {
|
|
338
|
+
const scoped = this.scoped_filter('context_sources', options);
|
|
339
|
+
const filters = ['context_chunks.source_id = ?', ...scoped.where];
|
|
340
|
+
const params = [
|
|
341
|
+
source_id,
|
|
342
|
+
...scoped.params,
|
|
343
|
+
];
|
|
344
|
+
if (chunk_id) {
|
|
345
|
+
filters.push('context_chunks.id = ?');
|
|
346
|
+
params.push(chunk_id);
|
|
347
|
+
}
|
|
348
|
+
const stmt = this.db.prepare(`
|
|
349
|
+
SELECT
|
|
350
|
+
context_chunks.id,
|
|
351
|
+
context_chunks.source_id,
|
|
352
|
+
context_chunks.ordinal,
|
|
353
|
+
context_chunks.title,
|
|
354
|
+
context_chunks.content,
|
|
355
|
+
context_chunks.byte_count
|
|
356
|
+
FROM context_chunks
|
|
357
|
+
JOIN context_sources ON context_sources.id = context_chunks.source_id
|
|
358
|
+
WHERE ${filters.join(' AND ')}
|
|
359
|
+
ORDER BY context_chunks.ordinal
|
|
360
|
+
`);
|
|
408
361
|
return stmt.all(...params);
|
|
409
362
|
}
|
|
410
|
-
|
|
363
|
+
count_stats(options) {
|
|
364
|
+
const scoped = this.scoped_filter('context_sources', options);
|
|
365
|
+
const where_clause = scoped.where.length
|
|
366
|
+
? `WHERE ${scoped.where.join(' AND ')}`
|
|
367
|
+
: '';
|
|
411
368
|
const source = this.db
|
|
412
369
|
.prepare(`
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
370
|
+
SELECT
|
|
371
|
+
COUNT(*) as sources,
|
|
372
|
+
COALESCE(SUM(byte_count), 0) as bytes_stored,
|
|
373
|
+
COALESCE(SUM(returned_byte_count), 0) as bytes_returned,
|
|
374
|
+
MIN(created_at) as oldest_created_at,
|
|
375
|
+
MAX(created_at) as newest_created_at
|
|
376
|
+
FROM context_sources
|
|
377
|
+
${where_clause}
|
|
378
|
+
`)
|
|
379
|
+
.get(...scoped.params);
|
|
420
380
|
const chunks = this.db
|
|
421
|
-
.prepare(
|
|
422
|
-
|
|
381
|
+
.prepare(`
|
|
382
|
+
SELECT COUNT(context_chunks.id) as chunks
|
|
383
|
+
FROM context_chunks
|
|
384
|
+
JOIN context_sources ON context_sources.id = context_chunks.source_id
|
|
385
|
+
${where_clause}
|
|
386
|
+
`)
|
|
387
|
+
.get(...scoped.params);
|
|
388
|
+
return { ...source, chunks: chunks.chunks };
|
|
389
|
+
}
|
|
390
|
+
stats(options = { global: true }) {
|
|
391
|
+
const source = this.count_stats(options);
|
|
392
|
+
const global = options.global
|
|
393
|
+
? source
|
|
394
|
+
: this.count_stats({ global: true });
|
|
423
395
|
const bytes_saved = source.bytes_stored - source.bytes_returned;
|
|
424
396
|
const reduction_pct = source.bytes_stored > 0
|
|
425
397
|
? Math.round((bytes_saved / source.bytes_stored) * 1000) / 10
|
|
426
398
|
: 0;
|
|
427
399
|
const db_bytes = file_size(this.db_path);
|
|
428
400
|
const wal_bytes = file_size(`${this.db_path}-wal`);
|
|
401
|
+
const policy = parse_context_retention_policy();
|
|
429
402
|
return {
|
|
430
403
|
sources: source.sources,
|
|
431
|
-
chunks:
|
|
404
|
+
chunks: source.chunks,
|
|
432
405
|
bytes_stored: source.bytes_stored,
|
|
433
406
|
bytes_returned: source.bytes_returned,
|
|
434
407
|
bytes_saved,
|
|
@@ -436,21 +409,105 @@ export class ContextStore {
|
|
|
436
409
|
db_bytes,
|
|
437
410
|
wal_bytes,
|
|
438
411
|
total_bytes: db_bytes + wal_bytes,
|
|
412
|
+
oldest_created_at: source.oldest_created_at,
|
|
413
|
+
newest_created_at: source.newest_created_at,
|
|
414
|
+
retention_days: policy.retention_days,
|
|
415
|
+
purge_on_shutdown: policy.purge_on_shutdown,
|
|
416
|
+
max_mb: policy.max_mb,
|
|
417
|
+
scope_project_path: options.global === true
|
|
418
|
+
? null
|
|
419
|
+
: (options.project_path ?? null),
|
|
420
|
+
scope_session_id: options.global === true ? null : (options.session_id ?? null),
|
|
421
|
+
global_sources: global.sources,
|
|
422
|
+
global_chunks: global.chunks,
|
|
423
|
+
global_bytes_stored: global.bytes_stored,
|
|
424
|
+
global_oldest_created_at: global.oldest_created_at,
|
|
425
|
+
global_newest_created_at: global.newest_created_at,
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
cleanup(policy = parse_context_retention_policy()) {
|
|
429
|
+
let age_deleted = 0;
|
|
430
|
+
if (policy.retention_days !== null) {
|
|
431
|
+
age_deleted = this.purge({
|
|
432
|
+
older_than_days: policy.retention_days,
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
const size_deleted = policy.max_bytes
|
|
436
|
+
? this.purge_to_max_stored_bytes(policy.max_bytes)
|
|
437
|
+
: 0;
|
|
438
|
+
return {
|
|
439
|
+
deleted: age_deleted + size_deleted,
|
|
440
|
+
age_deleted,
|
|
441
|
+
size_deleted,
|
|
442
|
+
policy,
|
|
439
443
|
};
|
|
440
444
|
}
|
|
445
|
+
purge_to_max_stored_bytes(max_bytes) {
|
|
446
|
+
const total_row = this.db
|
|
447
|
+
.prepare('SELECT COALESCE(SUM(byte_count), 0) as bytes FROM context_sources')
|
|
448
|
+
.get();
|
|
449
|
+
let total = total_row.bytes;
|
|
450
|
+
if (total <= max_bytes)
|
|
451
|
+
return 0;
|
|
452
|
+
const rows = this.db
|
|
453
|
+
.prepare('SELECT id, byte_count FROM context_sources ORDER BY created_at ASC')
|
|
454
|
+
.all();
|
|
455
|
+
const delete_source = this.db.prepare('DELETE FROM context_sources WHERE id = ?');
|
|
456
|
+
let deleted = 0;
|
|
457
|
+
for (const row of rows) {
|
|
458
|
+
if (total <= max_bytes)
|
|
459
|
+
break;
|
|
460
|
+
const result = delete_source.run(row.id);
|
|
461
|
+
if (Number(result.changes ?? 0) > 0) {
|
|
462
|
+
deleted += 1;
|
|
463
|
+
total -= row.byte_count;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
return deleted;
|
|
467
|
+
}
|
|
441
468
|
purge(options = {}) {
|
|
469
|
+
return this.purge_with_details(options).deleted;
|
|
470
|
+
}
|
|
471
|
+
purge_with_details(options = {}) {
|
|
472
|
+
const filters = [];
|
|
473
|
+
const params = [];
|
|
442
474
|
if (options.source_id) {
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
475
|
+
filters.push('id = ?');
|
|
476
|
+
params.push(options.source_id);
|
|
477
|
+
}
|
|
478
|
+
if (options.project_path === null) {
|
|
479
|
+
filters.push('project_path IS NULL');
|
|
480
|
+
}
|
|
481
|
+
else if (options.project_path !== undefined) {
|
|
482
|
+
filters.push('project_path = ?');
|
|
483
|
+
params.push(options.project_path);
|
|
484
|
+
}
|
|
485
|
+
if (options.session_id === null) {
|
|
486
|
+
filters.push('session_id IS NULL');
|
|
487
|
+
}
|
|
488
|
+
else if (options.session_id !== undefined) {
|
|
489
|
+
filters.push('session_id = ?');
|
|
490
|
+
params.push(options.session_id);
|
|
491
|
+
}
|
|
492
|
+
const days = options.older_than_days;
|
|
493
|
+
if (days !== undefined) {
|
|
494
|
+
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
|
495
|
+
filters.push('created_at < ?');
|
|
496
|
+
params.push(cutoff);
|
|
497
|
+
}
|
|
498
|
+
if (filters.length === 0) {
|
|
499
|
+
return { deleted: 0 };
|
|
447
500
|
}
|
|
448
|
-
const days = options.older_than_days ?? 14;
|
|
449
|
-
const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
|
|
450
501
|
const result = this.db
|
|
451
|
-
.prepare(
|
|
452
|
-
.run(
|
|
453
|
-
return
|
|
502
|
+
.prepare(`DELETE FROM context_sources WHERE ${filters.join(' AND ')}`)
|
|
503
|
+
.run(...params);
|
|
504
|
+
return {
|
|
505
|
+
deleted: Number(result.changes ?? 0),
|
|
506
|
+
source_id: options.source_id,
|
|
507
|
+
project_path: options.project_path,
|
|
508
|
+
session_id: options.session_id,
|
|
509
|
+
older_than_days: options.older_than_days,
|
|
510
|
+
};
|
|
454
511
|
}
|
|
455
512
|
close() {
|
|
456
513
|
this.db.close();
|