@spences10/pi-context 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/store/chunking-helpers.d.ts +8 -0
- package/dist/store/chunking-helpers.js +114 -0
- package/dist/store/chunking-helpers.js.map +1 -0
- package/dist/store/purge-helpers.d.ts +8 -0
- package/dist/store/purge-helpers.js +80 -0
- package/dist/store/purge-helpers.js.map +1 -0
- package/dist/store/query-helpers.d.ts +24 -0
- package/dist/store/query-helpers.js +125 -0
- package/dist/store/query-helpers.js.map +1 -0
- package/dist/store/schema-helpers.d.ts +3 -0
- package/dist/store/schema-helpers.js +18 -0
- package/dist/store/schema-helpers.js.map +1 -0
- package/dist/store/search-helpers.d.ts +11 -0
- package/dist/store/search-helpers.js +157 -0
- package/dist/store/search-helpers.js.map +1 -0
- package/package.json +6 -4
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { StoreContextInput, StoredContextOutput } from '../types.js';
|
|
3
|
+
import { type StoreScopeState } from './query-helpers.js';
|
|
4
|
+
export interface StoreCaptureLimits {
|
|
5
|
+
max_bytes: number;
|
|
6
|
+
max_lines: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function store_context_input(db: DatabaseSync, state: StoreScopeState, limits: StoreCaptureLimits, input: StoreContextInput): StoredContextOutput | null;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { redact_text } from '@spences10/pi-redact';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { chunk_text, count_lines, make_preview, should_index_text, summarize_source, } from '../text.js';
|
|
4
|
+
import { scoped_filter, } from './query-helpers.js';
|
|
5
|
+
function find_duplicate_source(db, state, content_hash, scope) {
|
|
6
|
+
const scoped = scoped_filter(state, 'context_sources', scope);
|
|
7
|
+
const filters = [
|
|
8
|
+
'context_sources.content_hash = ?',
|
|
9
|
+
...scoped.where,
|
|
10
|
+
];
|
|
11
|
+
const params = [
|
|
12
|
+
content_hash,
|
|
13
|
+
...scoped.params,
|
|
14
|
+
];
|
|
15
|
+
const row = db
|
|
16
|
+
.prepare(`
|
|
17
|
+
SELECT
|
|
18
|
+
context_sources.id,
|
|
19
|
+
COUNT(context_chunks.id) as chunk_count,
|
|
20
|
+
(
|
|
21
|
+
SELECT first_chunk.id FROM context_chunks first_chunk
|
|
22
|
+
WHERE first_chunk.source_id = context_sources.id
|
|
23
|
+
ORDER BY first_chunk.ordinal LIMIT 1
|
|
24
|
+
) as first_chunk_id
|
|
25
|
+
FROM context_sources
|
|
26
|
+
LEFT JOIN context_chunks ON context_chunks.source_id = context_sources.id
|
|
27
|
+
WHERE ${filters.join(' AND ')}
|
|
28
|
+
GROUP BY context_sources.id
|
|
29
|
+
ORDER BY context_sources.created_at DESC
|
|
30
|
+
LIMIT 1
|
|
31
|
+
`)
|
|
32
|
+
.get(...params);
|
|
33
|
+
return row ?? null;
|
|
34
|
+
}
|
|
35
|
+
export function store_context_input(db, state, limits, input) {
|
|
36
|
+
const redaction = redact_text(input.text);
|
|
37
|
+
const text = redaction.redacted;
|
|
38
|
+
if (!input.force && !should_index_text(text, limits))
|
|
39
|
+
return null;
|
|
40
|
+
const bytes = Buffer.byteLength(text, 'utf8');
|
|
41
|
+
const lines = count_lines(text);
|
|
42
|
+
const created_at = Date.now();
|
|
43
|
+
const content_hash = createHash('sha256')
|
|
44
|
+
.update(text)
|
|
45
|
+
.digest('hex');
|
|
46
|
+
const session_id = input.session_id ?? state.session_id;
|
|
47
|
+
const project_path = input.project_path ?? state.project_path;
|
|
48
|
+
const preview = make_preview(text);
|
|
49
|
+
const duplicate = find_duplicate_source(db, state, content_hash, {
|
|
50
|
+
project_path,
|
|
51
|
+
});
|
|
52
|
+
if (duplicate) {
|
|
53
|
+
const provisional = {
|
|
54
|
+
source_id: duplicate.id,
|
|
55
|
+
bytes,
|
|
56
|
+
lines,
|
|
57
|
+
preview,
|
|
58
|
+
receipt: '',
|
|
59
|
+
chunk_count: duplicate.chunk_count,
|
|
60
|
+
first_chunk_id: duplicate.first_chunk_id,
|
|
61
|
+
returned_bytes: 0,
|
|
62
|
+
project_path,
|
|
63
|
+
session_id,
|
|
64
|
+
deduped: true,
|
|
65
|
+
};
|
|
66
|
+
const receipt = summarize_source(provisional, input.tool_name);
|
|
67
|
+
const returned_bytes = Buffer.byteLength(receipt, 'utf8');
|
|
68
|
+
db.prepare('UPDATE context_sources SET returned_byte_count = returned_byte_count + ? WHERE id = ?').run(returned_bytes, duplicate.id);
|
|
69
|
+
return { ...provisional, receipt, returned_bytes };
|
|
70
|
+
}
|
|
71
|
+
const source_id = `ctx_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
|
|
72
|
+
const chunks = chunk_text(text, source_id);
|
|
73
|
+
const preview_bytes = Buffer.byteLength(preview, 'utf8');
|
|
74
|
+
const insert = db.prepare(`
|
|
75
|
+
INSERT INTO context_sources (
|
|
76
|
+
id, session_id, project_path, tool_name, input_summary, created_at,
|
|
77
|
+
byte_count, line_count, content_hash, preview_byte_count, returned_byte_count
|
|
78
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)
|
|
79
|
+
`);
|
|
80
|
+
const insert_chunk = db.prepare(`
|
|
81
|
+
INSERT INTO context_chunks (id, source_id, ordinal, title, content, byte_count)
|
|
82
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
83
|
+
`);
|
|
84
|
+
const update_returned = db.prepare('UPDATE context_sources SET returned_byte_count = ? WHERE id = ?');
|
|
85
|
+
db.exec('BEGIN');
|
|
86
|
+
try {
|
|
87
|
+
insert.run(source_id, session_id, project_path, input.tool_name, input.input_summary ?? null, created_at, bytes, lines, content_hash, preview_bytes);
|
|
88
|
+
for (const chunk of chunks) {
|
|
89
|
+
insert_chunk.run(chunk.id, chunk.source_id, chunk.ordinal, chunk.title, chunk.content, chunk.byte_count);
|
|
90
|
+
}
|
|
91
|
+
const provisional = {
|
|
92
|
+
source_id,
|
|
93
|
+
bytes,
|
|
94
|
+
lines,
|
|
95
|
+
preview,
|
|
96
|
+
receipt: '',
|
|
97
|
+
chunk_count: chunks.length,
|
|
98
|
+
first_chunk_id: chunks[0]?.id ?? null,
|
|
99
|
+
returned_bytes: 0,
|
|
100
|
+
project_path,
|
|
101
|
+
session_id,
|
|
102
|
+
};
|
|
103
|
+
const receipt = summarize_source(provisional, input.tool_name);
|
|
104
|
+
const returned_bytes = Buffer.byteLength(receipt, 'utf8');
|
|
105
|
+
update_returned.run(returned_bytes, source_id);
|
|
106
|
+
db.exec('COMMIT');
|
|
107
|
+
return { ...provisional, receipt, returned_bytes };
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
db.exec('ROLLBACK');
|
|
111
|
+
throw error;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=chunking-helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunking-helpers.js","sourceRoot":"","sources":["../../src/store/chunking-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,GAChB,MAAM,YAAY,CAAC;AAMpB,OAAO,EACN,aAAa,GAEb,MAAM,oBAAoB,CAAC;AAO5B,SAAS,qBAAqB,CAC7B,EAAgB,EAChB,KAAsB,EACtB,YAAoB,EACpB,KAA0B;IAM1B,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,iBAAiB,EAAE,KAAK,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG;QACf,kCAAkC;QAClC,GAAG,MAAM,CAAC,KAAK;KACf,CAAC;IACF,MAAM,MAAM,GAA2B;QACtC,YAAY;QACZ,GAAG,MAAM,CAAC,MAAM;KAChB,CAAC;IACF,MAAM,GAAG,GAAG,EAAE;SACZ,OAAO,CAAC;;;;;;;;;;;WAWA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;;;;GAI7B,CAAC;SACD,GAAG,CAAC,GAAG,MAAM,CAMH,CAAC;IACb,OAAO,GAAG,IAAI,IAAI,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAClC,EAAgB,EAChB,KAAsB,EACtB,MAA0B,EAC1B,KAAwB;IAExB,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC;IAChC,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAElE,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC;SACvC,MAAM,CAAC,IAAI,CAAC;SACZ,MAAM,CAAC,KAAK,CAAC,CAAC;IAChB,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;IACxD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC;IAC9D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,qBAAqB,CAAC,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE;QAChE,YAAY;KACZ,CAAC,CAAC;IACH,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,WAAW,GAAwB;YACxC,SAAS,EAAE,SAAS,CAAC,EAAE;YACvB,KAAK;YACL,KAAK;YACL,OAAO;YACP,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,cAAc,EAAE,SAAS,CAAC,cAAc;YACxC,cAAc,EAAE,CAAC;YACjB,YAAY;YACZ,UAAU;YACV,OAAO,EAAE,IAAI;SACb,CAAC;QACF,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QAC/D,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1D,EAAE,CAAC,OAAO,CACT,uFAAuF,CACvF,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;QACpC,OAAO,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;IACpD,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;IAC/E,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC3C,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;EAKzB,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC;;;EAG/B,CAAC,CAAC;IACH,MAAM,eAAe,GAAG,EAAE,CAAC,OAAO,CACjC,iEAAiE,CACjE,CAAC;IAEF,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACJ,MAAM,CAAC,GAAG,CACT,SAAS,EACT,UAAU,EACV,YAAY,EACZ,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,aAAa,IAAI,IAAI,EAC3B,UAAU,EACV,KAAK,EACL,KAAK,EACL,YAAY,EACZ,aAAa,CACb,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,YAAY,CAAC,GAAG,CACf,KAAK,CAAC,EAAE,EACR,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,UAAU,CAChB,CAAC;QACH,CAAC;QACD,MAAM,WAAW,GAAwB;YACxC,SAAS;YACT,KAAK;YACL,KAAK;YACL,OAAO;YACP,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,MAAM,CAAC,MAAM;YAC1B,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI;YACrC,cAAc,EAAE,CAAC;YACjB,YAAY;YACZ,UAAU;SACV,CAAC;QACF,MAAM,OAAO,GAAG,gBAAgB,CAAC,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QAC/D,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1D,eAAe,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;QAC/C,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,OAAO,EAAE,GAAG,WAAW,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpB,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { ContextCleanupResult, ContextPurgeDetails, ContextRetentionPolicy, ContextScopeOptions } from '../types.js';
|
|
3
|
+
export declare function purge_to_max_stored_bytes(db: DatabaseSync, max_bytes: number): number;
|
|
4
|
+
export declare function purge_context_sources(db: DatabaseSync, options?: ContextScopeOptions & {
|
|
5
|
+
older_than_days?: number;
|
|
6
|
+
source_id?: string;
|
|
7
|
+
}): ContextPurgeDetails;
|
|
8
|
+
export declare function cleanup_context_sources(db: DatabaseSync, policy?: ContextRetentionPolicy): ContextCleanupResult;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { parse_context_retention_policy } from '../policy.js';
|
|
2
|
+
export function purge_to_max_stored_bytes(db, max_bytes) {
|
|
3
|
+
const total_row = db
|
|
4
|
+
.prepare('SELECT COALESCE(SUM(byte_count), 0) as bytes FROM context_sources')
|
|
5
|
+
.get();
|
|
6
|
+
let total = total_row.bytes;
|
|
7
|
+
if (total <= max_bytes)
|
|
8
|
+
return 0;
|
|
9
|
+
const rows = db
|
|
10
|
+
.prepare('SELECT id, byte_count FROM context_sources ORDER BY created_at ASC')
|
|
11
|
+
.all();
|
|
12
|
+
const delete_source = db.prepare('DELETE FROM context_sources WHERE id = ?');
|
|
13
|
+
let deleted = 0;
|
|
14
|
+
for (const row of rows) {
|
|
15
|
+
if (total <= max_bytes)
|
|
16
|
+
break;
|
|
17
|
+
const result = delete_source.run(row.id);
|
|
18
|
+
if (Number(result.changes ?? 0) > 0) {
|
|
19
|
+
deleted += 1;
|
|
20
|
+
total -= row.byte_count;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return deleted;
|
|
24
|
+
}
|
|
25
|
+
export function purge_context_sources(db, options = {}) {
|
|
26
|
+
const filters = [];
|
|
27
|
+
const params = [];
|
|
28
|
+
if (options.source_id) {
|
|
29
|
+
filters.push('id = ?');
|
|
30
|
+
params.push(options.source_id);
|
|
31
|
+
}
|
|
32
|
+
if (options.project_path === null) {
|
|
33
|
+
filters.push('project_path IS NULL');
|
|
34
|
+
}
|
|
35
|
+
else if (options.project_path !== undefined) {
|
|
36
|
+
filters.push('project_path = ?');
|
|
37
|
+
params.push(options.project_path);
|
|
38
|
+
}
|
|
39
|
+
if (options.session_id === null) {
|
|
40
|
+
filters.push('session_id IS NULL');
|
|
41
|
+
}
|
|
42
|
+
else if (options.session_id !== undefined) {
|
|
43
|
+
filters.push('session_id = ?');
|
|
44
|
+
params.push(options.session_id);
|
|
45
|
+
}
|
|
46
|
+
const days = options.older_than_days;
|
|
47
|
+
if (days !== undefined) {
|
|
48
|
+
filters.push('created_at < ?');
|
|
49
|
+
params.push(Date.now() - days * 24 * 60 * 60 * 1000);
|
|
50
|
+
}
|
|
51
|
+
if (filters.length === 0)
|
|
52
|
+
return { deleted: 0 };
|
|
53
|
+
const result = db
|
|
54
|
+
.prepare(`DELETE FROM context_sources WHERE ${filters.join(' AND ')}`)
|
|
55
|
+
.run(...params);
|
|
56
|
+
return {
|
|
57
|
+
deleted: Number(result.changes ?? 0),
|
|
58
|
+
source_id: options.source_id,
|
|
59
|
+
project_path: options.project_path,
|
|
60
|
+
session_id: options.session_id,
|
|
61
|
+
older_than_days: options.older_than_days,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function cleanup_context_sources(db, policy = parse_context_retention_policy()) {
|
|
65
|
+
const age_deleted = policy.retention_days === null
|
|
66
|
+
? 0
|
|
67
|
+
: purge_context_sources(db, {
|
|
68
|
+
older_than_days: policy.retention_days,
|
|
69
|
+
}).deleted;
|
|
70
|
+
const size_deleted = policy.max_bytes
|
|
71
|
+
? purge_to_max_stored_bytes(db, policy.max_bytes)
|
|
72
|
+
: 0;
|
|
73
|
+
return {
|
|
74
|
+
deleted: age_deleted + size_deleted,
|
|
75
|
+
age_deleted,
|
|
76
|
+
size_deleted,
|
|
77
|
+
policy,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=purge-helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"purge-helpers.js","sourceRoot":"","sources":["../../src/store/purge-helpers.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,8BAA8B,EAAE,MAAM,cAAc,CAAC;AAE9D,MAAM,UAAU,yBAAyB,CACxC,EAAgB,EAChB,SAAiB;IAEjB,MAAM,SAAS,GAAG,EAAE;SAClB,OAAO,CACP,mEAAmE,CACnE;SACA,GAAG,EAAuB,CAAC;IAC7B,IAAI,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAC5B,IAAI,KAAK,IAAI,SAAS;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,EAAE;SACb,OAAO,CACP,oEAAoE,CACpE;SACA,GAAG,EAA+C,CAAC;IACrD,MAAM,aAAa,GAAG,EAAE,CAAC,OAAO,CAC/B,0CAA0C,CAC1C,CAAC;IACF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,IAAI,SAAS;YAAE,MAAM;QAC9B,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,CAAC;YACb,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC;QACzB,CAAC;IACF,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,qBAAqB,CACpC,EAAgB,EAChB,UAGI,EAAE;IAEN,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACtC,CAAC;SAAM,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;SAAM,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,eAAe,CAAC;IACrC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAChD,MAAM,MAAM,GAAG,EAAE;SACf,OAAO,CACP,qCAAqC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAC5D;SACA,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACjB,OAAO;QACN,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QACpC,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,eAAe,EAAE,OAAO,CAAC,eAAe;KACxC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CACtC,EAAgB,EAChB,SAAiC,8BAA8B,EAAE;IAEjE,MAAM,WAAW,GAChB,MAAM,CAAC,cAAc,KAAK,IAAI;QAC7B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,qBAAqB,CAAC,EAAE,EAAE;YAC1B,eAAe,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC,CAAC,OAAO,CAAC;IACd,MAAM,YAAY,GAAG,MAAM,CAAC,SAAS;QACpC,CAAC,CAAC,yBAAyB,CAAC,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC;QACjD,CAAC,CAAC,CAAC,CAAC;IACL,OAAO;QACN,OAAO,EAAE,WAAW,GAAG,YAAY;QACnC,WAAW;QACX,YAAY;QACZ,MAAM;KACN,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { ContextListResult, ContextScopeOptions, ScopedFilter } from '../types.js';
|
|
3
|
+
export interface StoreScopeState {
|
|
4
|
+
project_path: string | null;
|
|
5
|
+
session_id: string | null;
|
|
6
|
+
}
|
|
7
|
+
export interface ContextCountStats {
|
|
8
|
+
sources: number;
|
|
9
|
+
chunks: number;
|
|
10
|
+
bytes_stored: number;
|
|
11
|
+
bytes_returned: number;
|
|
12
|
+
oldest_created_at: number | null;
|
|
13
|
+
newest_created_at: number | null;
|
|
14
|
+
}
|
|
15
|
+
export declare function scoped_filter(state: StoreScopeState, alias: string, options?: ContextScopeOptions): ScopedFilter;
|
|
16
|
+
export declare function list_context_sources(db: DatabaseSync, state: StoreScopeState, options?: ContextScopeOptions & {
|
|
17
|
+
source_id?: string;
|
|
18
|
+
tool_name?: string;
|
|
19
|
+
limit?: number;
|
|
20
|
+
offset?: number;
|
|
21
|
+
newer_than_days?: number;
|
|
22
|
+
older_than_days?: number;
|
|
23
|
+
}): ContextListResult[];
|
|
24
|
+
export declare function count_context_stats(db: DatabaseSync, state: StoreScopeState, options: ContextScopeOptions): ContextCountStats;
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
export function scoped_filter(state, alias, options = {}) {
|
|
2
|
+
const where = [];
|
|
3
|
+
const params = [];
|
|
4
|
+
if (options.session_id === null) {
|
|
5
|
+
where.push(`${alias}.session_id IS NULL`);
|
|
6
|
+
}
|
|
7
|
+
else if (options.session_id !== undefined) {
|
|
8
|
+
where.push(`${alias}.session_id = ?`);
|
|
9
|
+
params.push(options.session_id);
|
|
10
|
+
}
|
|
11
|
+
else if (!options.global && state.session_id) {
|
|
12
|
+
where.push(`${alias}.session_id = ?`);
|
|
13
|
+
params.push(state.session_id);
|
|
14
|
+
}
|
|
15
|
+
if (options.project_path === null) {
|
|
16
|
+
where.push(`${alias}.project_path IS NULL`);
|
|
17
|
+
}
|
|
18
|
+
else if (options.project_path !== undefined) {
|
|
19
|
+
where.push(`${alias}.project_path = ?`);
|
|
20
|
+
params.push(options.project_path);
|
|
21
|
+
}
|
|
22
|
+
else if (!options.global &&
|
|
23
|
+
where.length === 0 &&
|
|
24
|
+
state.project_path) {
|
|
25
|
+
where.push(`${alias}.project_path = ?`);
|
|
26
|
+
params.push(state.project_path);
|
|
27
|
+
}
|
|
28
|
+
return { where, params };
|
|
29
|
+
}
|
|
30
|
+
export function list_context_sources(db, state, options = {}) {
|
|
31
|
+
const limit = Math.max(1, Math.min(options.limit ?? 10, 50));
|
|
32
|
+
const offset = Math.max(0, options.offset ?? 0);
|
|
33
|
+
const scoped = scoped_filter(state, 'context_sources', options);
|
|
34
|
+
const filters = [...scoped.where];
|
|
35
|
+
const params = [...scoped.params];
|
|
36
|
+
if (options.source_id) {
|
|
37
|
+
filters.push('context_sources.id = ?');
|
|
38
|
+
params.push(options.source_id);
|
|
39
|
+
}
|
|
40
|
+
if (options.tool_name) {
|
|
41
|
+
filters.push('context_sources.tool_name = ?');
|
|
42
|
+
params.push(options.tool_name);
|
|
43
|
+
}
|
|
44
|
+
if (options.newer_than_days !== undefined) {
|
|
45
|
+
filters.push('context_sources.created_at >= ?');
|
|
46
|
+
params.push(Date.now() - options.newer_than_days * 24 * 60 * 60 * 1000);
|
|
47
|
+
}
|
|
48
|
+
if (options.older_than_days !== undefined) {
|
|
49
|
+
filters.push('context_sources.created_at < ?');
|
|
50
|
+
params.push(Date.now() - options.older_than_days * 24 * 60 * 60 * 1000);
|
|
51
|
+
}
|
|
52
|
+
params.push(limit, offset);
|
|
53
|
+
const where_clause = filters.length
|
|
54
|
+
? `WHERE ${filters.join(' AND ')}`
|
|
55
|
+
: '';
|
|
56
|
+
const stmt = db.prepare(`
|
|
57
|
+
SELECT
|
|
58
|
+
context_sources.id as source_id,
|
|
59
|
+
context_sources.created_at,
|
|
60
|
+
context_sources.project_path,
|
|
61
|
+
context_sources.session_id,
|
|
62
|
+
context_sources.tool_name,
|
|
63
|
+
context_sources.input_summary,
|
|
64
|
+
context_sources.byte_count,
|
|
65
|
+
context_sources.line_count,
|
|
66
|
+
COUNT(context_chunks.id) as chunk_count,
|
|
67
|
+
(
|
|
68
|
+
SELECT title FROM context_chunks first_chunk
|
|
69
|
+
WHERE first_chunk.source_id = context_sources.id
|
|
70
|
+
ORDER BY ordinal LIMIT 1
|
|
71
|
+
) as first_chunk_title,
|
|
72
|
+
(
|
|
73
|
+
SELECT substr(content, 1, 240) FROM context_chunks first_chunk
|
|
74
|
+
WHERE first_chunk.source_id = context_sources.id
|
|
75
|
+
ORDER BY ordinal LIMIT 1
|
|
76
|
+
) as preview
|
|
77
|
+
FROM context_sources
|
|
78
|
+
LEFT JOIN context_chunks ON context_chunks.source_id = context_sources.id
|
|
79
|
+
${where_clause}
|
|
80
|
+
GROUP BY context_sources.id
|
|
81
|
+
ORDER BY context_sources.created_at DESC
|
|
82
|
+
LIMIT ? OFFSET ?
|
|
83
|
+
`);
|
|
84
|
+
return stmt.all(...params).map((row) => ({
|
|
85
|
+
source_id: row.source_id,
|
|
86
|
+
created_at: row.created_at,
|
|
87
|
+
project_path: row.project_path,
|
|
88
|
+
session_id: row.session_id,
|
|
89
|
+
tool_name: row.tool_name,
|
|
90
|
+
input_summary: row.input_summary,
|
|
91
|
+
bytes: row.byte_count,
|
|
92
|
+
lines: row.line_count,
|
|
93
|
+
chunk_count: row.chunk_count,
|
|
94
|
+
first_chunk_title: row.first_chunk_title,
|
|
95
|
+
preview: row.preview,
|
|
96
|
+
}));
|
|
97
|
+
}
|
|
98
|
+
export function count_context_stats(db, state, options) {
|
|
99
|
+
const scoped = scoped_filter(state, 'context_sources', options);
|
|
100
|
+
const where_clause = scoped.where.length
|
|
101
|
+
? `WHERE ${scoped.where.join(' AND ')}`
|
|
102
|
+
: '';
|
|
103
|
+
const source = db
|
|
104
|
+
.prepare(`
|
|
105
|
+
SELECT
|
|
106
|
+
COUNT(*) as sources,
|
|
107
|
+
COALESCE(SUM(byte_count), 0) as bytes_stored,
|
|
108
|
+
COALESCE(SUM(returned_byte_count), 0) as bytes_returned,
|
|
109
|
+
MIN(created_at) as oldest_created_at,
|
|
110
|
+
MAX(created_at) as newest_created_at
|
|
111
|
+
FROM context_sources
|
|
112
|
+
${where_clause}
|
|
113
|
+
`)
|
|
114
|
+
.get(...scoped.params);
|
|
115
|
+
const chunks = db
|
|
116
|
+
.prepare(`
|
|
117
|
+
SELECT COUNT(context_chunks.id) as chunks
|
|
118
|
+
FROM context_chunks
|
|
119
|
+
JOIN context_sources ON context_sources.id = context_chunks.source_id
|
|
120
|
+
${where_clause}
|
|
121
|
+
`)
|
|
122
|
+
.get(...scoped.params);
|
|
123
|
+
return { ...source, chunks: chunks.chunks };
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=query-helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-helpers.js","sourceRoot":"","sources":["../../src/store/query-helpers.ts"],"names":[],"mappings":"AAsBA,MAAM,UAAU,aAAa,CAC5B,KAAsB,EACtB,KAAa,EACb,UAA+B,EAAE;IAEjC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,OAAO,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAC;IAC3C,CAAC;SAAM,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;SAAM,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,OAAO,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,uBAAuB,CAAC,CAAC;IAC7C,CAAC;SAAM,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACnC,CAAC;SAAM,IACN,CAAC,OAAO,CAAC,MAAM;QACf,KAAK,CAAC,MAAM,KAAK,CAAC;QAClB,KAAK,CAAC,YAAY,EACjB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,oBAAoB,CACnC,EAAgB,EAChB,KAAsB,EACtB,UAOI,EAAE;IAEN,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,OAAO,GAAa,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAA2B,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAChD,MAAM,CAAC,IAAI,CACV,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAC1D,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAC/C,MAAM,CAAC,IAAI,CACV,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAC1D,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM;QAClC,CAAC,CAAC,SAAS,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QAClC,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;IAuBrB,YAAY;;;;EAId,CAAC,CAAC;IACH,OAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAA0B,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAClE,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,aAAa,EAAE,GAAG,CAAC,aAAa;QAChC,KAAK,EAAE,GAAG,CAAC,UAAU;QACrB,KAAK,EAAE,GAAG,CAAC,UAAU;QACrB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;QACxC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,mBAAmB,CAClC,EAAgB,EAChB,KAAsB,EACtB,OAA4B;IAE5B,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;QACvC,CAAC,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QACvC,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,MAAM,GAAG,EAAE;SACf,OAAO,CAAC;;;;;;;;KAQN,YAAY;GACd,CAAC;SACD,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAsC,CAAC;IAC7D,MAAM,MAAM,GAAG,EAAE;SACf,OAAO,CAAC;;;;KAIN,YAAY;GACd,CAAC;SACD,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAuB,CAAC;IAC9C,OAAO,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
3
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
4
|
+
import { apply_schema } from '../schema.js';
|
|
5
|
+
export function open_context_database(db_path) {
|
|
6
|
+
const dir = dirname(db_path);
|
|
7
|
+
if (!existsSync(dir))
|
|
8
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
9
|
+
const db = new DatabaseSync(db_path, {
|
|
10
|
+
enableForeignKeyConstraints: true,
|
|
11
|
+
});
|
|
12
|
+
apply_schema(db);
|
|
13
|
+
return db;
|
|
14
|
+
}
|
|
15
|
+
export function file_size(path) {
|
|
16
|
+
return existsSync(path) ? statSync(path).size : 0;
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=schema-helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema-helpers.js","sourceRoot":"","sources":["../../src/store/schema-helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,UAAU,qBAAqB,CAAC,OAAe;IACpD,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QACnB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAClD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,OAAO,EAAE;QACpC,2BAA2B,EAAE,IAAI;KACjC,CAAC,CAAC;IACH,YAAY,CAAC,EAAE,CAAC,CAAC;IACjB,OAAO,EAAE,CAAC;AACX,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACrC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
import type { ContextChunk, ContextChunkSummary, ContextScopeOptions, ContextSearchResult } from '../types.js';
|
|
3
|
+
import { type StoreScopeState } from './query-helpers.js';
|
|
4
|
+
export declare function chunk_reference_to_ordinal(source_id: string, chunk_id: string): number | null;
|
|
5
|
+
export declare function search_context_chunks(db: DatabaseSync, state: StoreScopeState, query: string, options?: ContextScopeOptions & {
|
|
6
|
+
source_id?: string;
|
|
7
|
+
limit?: number;
|
|
8
|
+
tool_name?: string;
|
|
9
|
+
}): ContextSearchResult[];
|
|
10
|
+
export declare function chunk_summary(db: DatabaseSync, state: StoreScopeState, source_id: string): ContextChunkSummary | null;
|
|
11
|
+
export declare function get_context_chunks(db: DatabaseSync, state: StoreScopeState, source_id: string, chunk_id?: string): ContextChunk[];
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { escape_fts5_query, relaxed_fts5_query } from '../text.js';
|
|
2
|
+
import { scoped_filter, } from './query-helpers.js';
|
|
3
|
+
function escape_regexp(value) {
|
|
4
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
5
|
+
}
|
|
6
|
+
export function chunk_reference_to_ordinal(source_id, chunk_id) {
|
|
7
|
+
const trimmed = chunk_id.trim();
|
|
8
|
+
const legacy_match = new RegExp(`^${escape_regexp(source_id)}:chunk:(\\d+)$`).exec(trimmed);
|
|
9
|
+
if (legacy_match) {
|
|
10
|
+
const value = Number.parseInt(legacy_match[1], 10);
|
|
11
|
+
if (!Number.isSafeInteger(value))
|
|
12
|
+
return null;
|
|
13
|
+
return value <= 0 ? 1 : value;
|
|
14
|
+
}
|
|
15
|
+
if (!/^\d+$/.test(trimmed))
|
|
16
|
+
return null;
|
|
17
|
+
const value = Number.parseInt(trimmed, 10);
|
|
18
|
+
return Number.isSafeInteger(value) && value > 0 ? value : null;
|
|
19
|
+
}
|
|
20
|
+
function search_match(db, state, match, options, limit) {
|
|
21
|
+
const scoped = options.source_id
|
|
22
|
+
? { where: [], params: [] }
|
|
23
|
+
: scoped_filter(state, 'context_sources', options);
|
|
24
|
+
const filters = [...scoped.where];
|
|
25
|
+
const params = [match, ...scoped.params];
|
|
26
|
+
if (options.source_id) {
|
|
27
|
+
filters.push('context_sources.id = ?');
|
|
28
|
+
params.push(options.source_id);
|
|
29
|
+
}
|
|
30
|
+
if (options.tool_name) {
|
|
31
|
+
filters.push('context_sources.tool_name = ?');
|
|
32
|
+
params.push(options.tool_name);
|
|
33
|
+
}
|
|
34
|
+
params.push(limit);
|
|
35
|
+
const where_filters = filters.length
|
|
36
|
+
? ` AND ${filters.join(' AND ')}`
|
|
37
|
+
: '';
|
|
38
|
+
const stmt = db.prepare(`
|
|
39
|
+
SELECT
|
|
40
|
+
context_sources.id,
|
|
41
|
+
context_sources.tool_name,
|
|
42
|
+
context_sources.created_at,
|
|
43
|
+
context_sources.byte_count,
|
|
44
|
+
context_sources.line_count,
|
|
45
|
+
context_chunks.id as chunk_id,
|
|
46
|
+
context_chunks.ordinal,
|
|
47
|
+
context_chunks.title,
|
|
48
|
+
context_chunks.content,
|
|
49
|
+
bm25(context_chunks_fts, 5.0, 1.0) as rank
|
|
50
|
+
FROM context_chunks_fts
|
|
51
|
+
JOIN context_chunks ON context_chunks.rowid = context_chunks_fts.rowid
|
|
52
|
+
JOIN context_sources ON context_sources.id = context_chunks.source_id
|
|
53
|
+
WHERE context_chunks_fts MATCH ?${where_filters}
|
|
54
|
+
ORDER BY rank
|
|
55
|
+
LIMIT ?
|
|
56
|
+
`);
|
|
57
|
+
return stmt.all(...params).map((row) => ({
|
|
58
|
+
source_id: row.id,
|
|
59
|
+
chunk_id: row.chunk_id,
|
|
60
|
+
ordinal: row.ordinal,
|
|
61
|
+
title: row.title,
|
|
62
|
+
content: row.content,
|
|
63
|
+
tool_name: row.tool_name,
|
|
64
|
+
created_at: row.created_at,
|
|
65
|
+
bytes: row.byte_count,
|
|
66
|
+
lines: row.line_count,
|
|
67
|
+
rank: row.rank,
|
|
68
|
+
}));
|
|
69
|
+
}
|
|
70
|
+
export function search_context_chunks(db, state, query, options = {}) {
|
|
71
|
+
const limit = Math.max(1, Math.min(options.limit ?? 5, 25));
|
|
72
|
+
const strict = escape_fts5_query(query);
|
|
73
|
+
const relaxed = relaxed_fts5_query(query);
|
|
74
|
+
const results = search_match(db, state, strict, options, limit);
|
|
75
|
+
if (results.length >= limit || !relaxed || relaxed === strict)
|
|
76
|
+
return results;
|
|
77
|
+
const seen = new Set(results.map((result) => result.chunk_id));
|
|
78
|
+
for (const result of search_match(db, state, relaxed, options, limit)) {
|
|
79
|
+
if (seen.has(result.chunk_id))
|
|
80
|
+
continue;
|
|
81
|
+
results.push(result);
|
|
82
|
+
seen.add(result.chunk_id);
|
|
83
|
+
if (results.length >= limit)
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
return results;
|
|
87
|
+
}
|
|
88
|
+
export function chunk_summary(db, state, source_id) {
|
|
89
|
+
const scoped = scoped_filter(state, 'context_sources', {
|
|
90
|
+
global: true,
|
|
91
|
+
});
|
|
92
|
+
const filters = ['context_sources.id = ?', ...scoped.where];
|
|
93
|
+
const params = [
|
|
94
|
+
source_id,
|
|
95
|
+
...scoped.params,
|
|
96
|
+
];
|
|
97
|
+
const row = db
|
|
98
|
+
.prepare(`
|
|
99
|
+
SELECT
|
|
100
|
+
context_sources.id as source_id,
|
|
101
|
+
COUNT(context_chunks.id) as chunk_count,
|
|
102
|
+
(
|
|
103
|
+
SELECT first_chunk.id FROM context_chunks first_chunk
|
|
104
|
+
WHERE first_chunk.source_id = context_sources.id
|
|
105
|
+
ORDER BY first_chunk.ordinal LIMIT 1
|
|
106
|
+
) as first_chunk_id,
|
|
107
|
+
(
|
|
108
|
+
SELECT last_chunk.id FROM context_chunks last_chunk
|
|
109
|
+
WHERE last_chunk.source_id = context_sources.id
|
|
110
|
+
ORDER BY last_chunk.ordinal DESC LIMIT 1
|
|
111
|
+
) as last_chunk_id,
|
|
112
|
+
MIN(context_chunks.ordinal) as first_ordinal,
|
|
113
|
+
MAX(context_chunks.ordinal) as last_ordinal
|
|
114
|
+
FROM context_sources
|
|
115
|
+
LEFT JOIN context_chunks ON context_chunks.source_id = context_sources.id
|
|
116
|
+
WHERE ${filters.join(' AND ')}
|
|
117
|
+
GROUP BY context_sources.id
|
|
118
|
+
`)
|
|
119
|
+
.get(...params);
|
|
120
|
+
return row ?? null;
|
|
121
|
+
}
|
|
122
|
+
export function get_context_chunks(db, state, source_id, chunk_id) {
|
|
123
|
+
const scoped = scoped_filter(state, 'context_sources', {
|
|
124
|
+
global: true,
|
|
125
|
+
});
|
|
126
|
+
const filters = ['context_chunks.source_id = ?', ...scoped.where];
|
|
127
|
+
const params = [
|
|
128
|
+
source_id,
|
|
129
|
+
...scoped.params,
|
|
130
|
+
];
|
|
131
|
+
if (chunk_id) {
|
|
132
|
+
const ordinal = chunk_reference_to_ordinal(source_id, chunk_id);
|
|
133
|
+
if (ordinal) {
|
|
134
|
+
filters.push('context_chunks.ordinal = ?');
|
|
135
|
+
params.push(ordinal);
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
filters.push('context_chunks.id = ?');
|
|
139
|
+
params.push(chunk_id);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
const stmt = db.prepare(`
|
|
143
|
+
SELECT
|
|
144
|
+
context_chunks.id,
|
|
145
|
+
context_chunks.source_id,
|
|
146
|
+
context_chunks.ordinal,
|
|
147
|
+
context_chunks.title,
|
|
148
|
+
context_chunks.content,
|
|
149
|
+
context_chunks.byte_count
|
|
150
|
+
FROM context_chunks
|
|
151
|
+
JOIN context_sources ON context_sources.id = context_chunks.source_id
|
|
152
|
+
WHERE ${filters.join(' AND ')}
|
|
153
|
+
ORDER BY context_chunks.ordinal
|
|
154
|
+
`);
|
|
155
|
+
return stmt.all(...params);
|
|
156
|
+
}
|
|
157
|
+
//# sourceMappingURL=search-helpers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-helpers.js","sourceRoot":"","sources":["../../src/store/search-helpers.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EACN,aAAa,GAEb,MAAM,oBAAoB,CAAC;AAE5B,SAAS,aAAa,CAAC,KAAa;IACnC,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,0BAA0B,CACzC,SAAiB,EACjB,QAAgB;IAEhB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,YAAY,GAAG,IAAI,MAAM,CAC9B,IAAI,aAAa,CAAC,SAAS,CAAC,gBAAgB,CAC5C,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChB,IAAI,YAAY,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC9C,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC3C,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAChE,CAAC;AAED,SAAS,YAAY,CACpB,EAAgB,EAChB,KAAsB,EACtB,KAAa,EACb,OAGC,EACD,KAAa;IAEb,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS;QAC/B,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAA4B,EAAE;QACrD,CAAC,CAAC,aAAa,CAAC,KAAK,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC;IACpD,MAAM,OAAO,GAAa,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,MAAM,GAA2B,CAAC,KAAK,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM;QACnC,CAAC,CAAC,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QACjC,CAAC,CAAC,EAAE,CAAC;IACN,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;;;oCAeW,aAAa;;;EAG/C,CAAC,CAAC;IACH,OAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAA4B,CAAC,GAAG,CACzD,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACT,SAAS,EAAE,GAAG,CAAC,EAAE;QACjB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,UAAU;QACrB,KAAK,EAAE,GAAG,CAAC,UAAU;QACrB,IAAI,EAAE,GAAG,CAAC,IAAI;KACd,CAAC,CACF,CAAC;AACH,CAAC;AAED,MAAM,UAAU,qBAAqB,CACpC,EAAgB,EAChB,KAAsB,EACtB,KAAa,EACb,UAII,EAAE;IAEN,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,MAAM;QAC5D,OAAO,OAAO,CAAC;IAEhB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/D,KAAK,MAAM,MAAM,IAAI,YAAY,CAChC,EAAE,EACF,KAAK,EACL,OAAO,EACP,OAAO,EACP,KAAK,CACL,EAAE,CAAC;QACH,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;YAAE,SAAS;QACxC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACpC,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,aAAa,CAC5B,EAAgB,EAChB,KAAsB,EACtB,SAAiB;IAEjB,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,iBAAiB,EAAE;QACtD,MAAM,EAAE,IAAI;KACZ,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,CAAC,wBAAwB,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5D,MAAM,MAAM,GAA2B;QACtC,SAAS;QACT,GAAG,MAAM,CAAC,MAAM;KAChB,CAAC;IACF,MAAM,GAAG,GAAG,EAAE;SACZ,OAAO,CAAC;;;;;;;;;;;;;;;;;;WAkBA,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;;GAE7B,CAAC;SACD,GAAG,CAAC,GAAG,MAAM,CAAgC,CAAC;IAChD,OAAO,GAAG,IAAI,IAAI,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,EAAgB,EAChB,KAAsB,EACtB,SAAiB,EACjB,QAAiB;IAEjB,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE,iBAAiB,EAAE;QACtD,MAAM,EAAE,IAAI;KACZ,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,CAAC,8BAA8B,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAClE,MAAM,MAAM,GAA2B;QACtC,SAAS;QACT,GAAG,MAAM,CAAC,MAAM;KAChB,CAAC;IACF,IAAI,QAAQ,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,0BAA0B,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,IAAI,OAAO,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;YAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC;IACF,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;UAUf,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;;EAE7B,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAA8B,CAAC;AACzD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spences10/pi-context",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.16",
|
|
4
4
|
"description": "Pi extension for local SQLite context sidecar storage and retrieval of large tool output",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"context",
|
|
@@ -36,8 +36,8 @@
|
|
|
36
36
|
}
|
|
37
37
|
},
|
|
38
38
|
"dependencies": {
|
|
39
|
-
"@spences10/pi-
|
|
40
|
-
"@spences10/pi-
|
|
39
|
+
"@spences10/pi-tui-modal": "0.0.14",
|
|
40
|
+
"@spences10/pi-redact": "0.0.8"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/node": "^25.8.0",
|
|
@@ -63,6 +63,8 @@
|
|
|
63
63
|
"check": "pnpm --filter \"$npm_package_name^...\" run build:self && pnpm run check:self",
|
|
64
64
|
"check:self": "tsc --noEmit",
|
|
65
65
|
"test": "pnpm --filter \"$npm_package_name^...\" run build:self && pnpm run build:self && pnpm run test:self",
|
|
66
|
-
"test:self": "vitest run"
|
|
66
|
+
"test:self": "vitest run",
|
|
67
|
+
"coverage:self": "vitest run --coverage",
|
|
68
|
+
"coverage": "pnpm --filter \"$npm_package_name^...\" run build:self && pnpm run coverage:self"
|
|
67
69
|
}
|
|
68
70
|
}
|