advisers 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh.md +13 -15
- package/README.md +32 -22
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/bin/advisers.js +2769 -183
- package/index.js +1 -1
- package/lib/brief-archive.js +267 -0
- package/lib/client-pack.js +1016 -0
- package/lib/consent.js +390 -0
- package/lib/credentials.js +195 -0
- package/lib/customer-read.js +524 -0
- package/lib/error-codes.js +516 -7
- package/lib/evidence.js +928 -120
- package/lib/kb-extract.js +626 -0
- package/lib/kb-facts.js +429 -0
- package/lib/kb-pages.js +1084 -0
- package/lib/kb.js +1036 -0
- package/lib/ledger-read.js +109 -1
- package/lib/mail-errors.js +46 -0
- package/lib/mail-extract.js +351 -0
- package/lib/mail-imap.js +446 -0
- package/lib/mail-import.js +176 -0
- package/lib/mail-parse.js +223 -0
- package/lib/mail-store.js +731 -0
- package/lib/mail.js +506 -0
- package/lib/mcp-server.js +397 -11
- package/lib/notify.js +265 -7
- package/lib/policy/decision.js +315 -0
- package/lib/policy/egress.js +268 -0
- package/lib/policy/errors.js +71 -0
- package/lib/policy/index.js +99 -0
- package/lib/policy/rules.js +468 -0
- package/lib/policy/surfaces.js +141 -0
- package/lib/receipt-verify.js +283 -26
- package/lib/receivables.js +1653 -0
- package/lib/scheduler.js +453 -9
- package/lib/shell.js +1406 -0
- package/lib/source.js +98 -3
- package/lib/sqlite.js +501 -49
- package/lib/tos/index.js +2 -2
- package/lib/tos/manifest.json +6 -6
- package/lib/tos/privacy.zh-CN.md +33 -20
- package/lib/tos/safety.zh-CN.md +7 -6
- package/lib/tos/summary.zh-CN.md +3 -2
- package/lib/tos/terms.zh-CN.md +31 -16
- package/lib/tui/labels.js +1 -1
- package/lib/tui/render.js +1 -1
- package/lib/urgency.js +467 -0
- package/package.json +4 -4
package/lib/kb.js
ADDED
|
@@ -0,0 +1,1036 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KB module (0.4.0, schema v22): local document ingestion plus deterministic
|
|
3
|
+
* full-text retrieval over the already-opened SqliteStore.
|
|
4
|
+
*
|
|
5
|
+
* Product contract (45-kb-design.md, frozen for 0.4.0):
|
|
6
|
+
* - L0 metadata + L1 extracted chunks live only in the local database; the
|
|
7
|
+
* source file is never copied and an absolute path is never stored or
|
|
8
|
+
* returned (source_name is the basename only);
|
|
9
|
+
* - ingest is bounded and explicit: 50 MB file, 2 MB text, 2000 chunks, and
|
|
10
|
+
* every truncation is reported (truncated / truncatedBy / warnings);
|
|
11
|
+
* - search is deterministic and offline: FTS5 trigram MATCH for every token
|
|
12
|
+
* with at least 3 code points, LIKE fallback for 2-character/short queries
|
|
13
|
+
* with degraded:['short_query_like'], LIKE when the index degraded to
|
|
14
|
+
* unicode61 or is unavailable; FTS syntax is never passed through (tokens are
|
|
15
|
+
* quoted/escaped, a query error falls back to LIKE instead of throwing);
|
|
16
|
+
* - clearance is applied in SQL BEFORE counting/paging: effective secrecy is
|
|
17
|
+
* max(document.secrecy, client.secrecyDefault, matter.secrecy), an
|
|
18
|
+
* unresolvable owner fails closed, and hidden rows are only counted in
|
|
19
|
+
* protectedExcluded;
|
|
20
|
+
* - snippets are at most KB_SNIPPET_CHARS code points and are sanitized
|
|
21
|
+
* (terminal escapes, C0/C1 controls and bidi marks removed); highlights are
|
|
22
|
+
* numeric offsets into the returned snippet, never HTML/Markdown markup;
|
|
23
|
+
* - removal is honest: kb rm tombstones the document and really deletes its
|
|
24
|
+
* chunks (FTS triggers clear the index so MATCH returns 0), kb purge --yes
|
|
25
|
+
* is the physical delete;
|
|
26
|
+
* - every write path commits exactly one audit row with counts/ids only (never
|
|
27
|
+
* a title, filename, path or quote).
|
|
28
|
+
*
|
|
29
|
+
* The caller owns the connection lifecycle and the CLI/MCP surface; this module
|
|
30
|
+
* performs no HOME handling, no migration and no console output. It writes only
|
|
31
|
+
* the v22 KB tables (the schema is owned by lib/sqlite.js).
|
|
32
|
+
*/
|
|
33
|
+
import { randomUUID } from 'node:crypto'
|
|
34
|
+
import { lstatSync } from 'node:fs'
|
|
35
|
+
import { basename, resolve } from 'node:path'
|
|
36
|
+
import { entityError } from './sqlite.js'
|
|
37
|
+
import { sha256FileSync } from './backup.js'
|
|
38
|
+
import { chunkText, extractForIndex, KB_MAX_FILE_BYTES } from './kb-extract.js'
|
|
39
|
+
import { classify, safetyErrorFrom } from './safety/index.js'
|
|
40
|
+
import { stripBidiControls, stripControlChars, stripTerminalSequences } from './text-safe.js'
|
|
41
|
+
|
|
42
|
+
/** Document kinds accepted by kb add (45 section 3.4). */
|
|
43
|
+
export const KB_KINDS = Object.freeze(['quote', 'contract', 'receipt', 'delivery', 'note', 'other'])
|
|
44
|
+
/** Relations accepted by kb link. */
|
|
45
|
+
export const KB_RELATIONS = Object.freeze(['source', 'mentions'])
|
|
46
|
+
/** Local CLI default clearance; MCP/DSH callers must pass 0 until KB is exposed. */
|
|
47
|
+
export const KB_DEFAULT_CLEARANCE = 3
|
|
48
|
+
export const KB_LIST_LIMIT = 50
|
|
49
|
+
export const KB_LIST_LIMIT_MAX = 200
|
|
50
|
+
export const KB_SEARCH_LIMIT = 20
|
|
51
|
+
export const KB_SEARCH_LIMIT_MAX = 200
|
|
52
|
+
export const KB_QUERY_MAX_CHARS = 200
|
|
53
|
+
export const KB_MAX_BATCH_FILES = 200
|
|
54
|
+
/** One document is capped at this many chunks (45 section 3.3). */
|
|
55
|
+
export const KB_MAX_CHUNKS = 2000
|
|
56
|
+
export const KB_SNIPPET_CHARS = 240
|
|
57
|
+
export const KB_CHUNK_PREVIEW_CHARS = 400
|
|
58
|
+
/** Hard cap for the JS clearance pass over matched page rows. */
|
|
59
|
+
export const KB_PAGE_SCAN_MAX = 2000
|
|
60
|
+
|
|
61
|
+
const LIKE_CLAUSE = "LIKE ? ESCAPE '\\'"
|
|
62
|
+
const INSERT_DOCUMENT_SQL = 'INSERT INTO kb_documents(id,kind,title,client_id,project_id,matter_id,source_name,source_sha256,mime,size_bytes,encoding,extractor,status,secrecy,needs_ocr,truncated,text_bytes,chunk_count,safety,created_at,captured_at,deleted_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
|
|
63
|
+
const INSERT_CHUNK_SQL = 'INSERT INTO kb_chunks(id,doc_id,seq,text,char_count) VALUES(?,?,?,?,?)'
|
|
64
|
+
|
|
65
|
+
/* -------------------------------------------------------------------------- */
|
|
66
|
+
/* small helpers */
|
|
67
|
+
/* -------------------------------------------------------------------------- */
|
|
68
|
+
|
|
69
|
+
function nowIso() {
|
|
70
|
+
return new Date().toISOString()
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function newId(prefix) {
|
|
74
|
+
return prefix + randomUUID().replace(/-/g, '').slice(0, 16)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function codePointLength(value) {
|
|
78
|
+
return [...String(value ?? '')].length
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function clampLimit(value, fallback, max) {
|
|
82
|
+
if (value === undefined || value === null) return fallback
|
|
83
|
+
const number = Number(value)
|
|
84
|
+
if (!Number.isInteger(number) || number < 1) return fallback
|
|
85
|
+
return Math.min(number, max)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function normalizeClearance(value) {
|
|
89
|
+
const number = Number(value)
|
|
90
|
+
if (!Number.isFinite(number) || number < 0) return 0
|
|
91
|
+
return Math.floor(number)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function sanitizeLine(value, max = KB_SNIPPET_CHARS) {
|
|
95
|
+
const text = stripControlChars(stripBidiControls(stripTerminalSequences(String(value ?? ''))), { keepNewlines: false, keepTabs: false, replacement: ' ' })
|
|
96
|
+
.replace(/\s+/gu, ' ').trim()
|
|
97
|
+
const chars = Array.from(text)
|
|
98
|
+
if (chars.length <= max) return text
|
|
99
|
+
return chars.slice(0, max).join('')
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function assertStore(store) {
|
|
103
|
+
if (!store || !store.db || typeof store.db.prepare !== 'function' || typeof store.read !== 'function' || typeof store.mutate !== 'function') {
|
|
104
|
+
throw new TypeError('kb requires an opened SqliteStore (KB tables are SQLite-only)')
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function hasSqliteObject(store, name, types = ['table']) {
|
|
109
|
+
const placeholders = types.map(() => '?').join(',')
|
|
110
|
+
const row = store.db.prepare('SELECT name FROM sqlite_master WHERE name = ? AND type IN (' + placeholders + ')').get(name, ...types)
|
|
111
|
+
return Boolean(row)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Throw E_SCHEMA_STALE when the v22 KB tables are absent (pre-0.4 database). */
|
|
115
|
+
export function assertKbSchema(store) {
|
|
116
|
+
assertStore(store)
|
|
117
|
+
if (!hasSqliteObject(store, 'kb_documents')) {
|
|
118
|
+
throw entityError('E_SCHEMA_STALE', 'KB tables are missing; open an upgraded database (schema v22)', 'advisers doctor')
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Tokenizer read back from the kb FTS DDL (null when the index is absent). */
|
|
123
|
+
export function kbFtsTokenizer(store, table = 'kb_fts') {
|
|
124
|
+
assertStore(store)
|
|
125
|
+
const row = store.db.prepare("SELECT sql FROM sqlite_master WHERE name = ? AND type = 'table'").get(String(table))
|
|
126
|
+
if (!row) return null
|
|
127
|
+
return /tokenize\s*=\s*'?trigram/i.test(String(row.sql ?? '')) ? 'trigram' : 'unicode61'
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function validateKind(value) {
|
|
131
|
+
if (value === undefined || value === null || value === '') return 'other'
|
|
132
|
+
const kind = String(value)
|
|
133
|
+
if (!KB_KINDS.includes(kind)) throw entityError('E_KB_INVALID', 'unknown document kind: ' + kind, 'use one of ' + KB_KINDS.join(', '))
|
|
134
|
+
return kind
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function validateSecrecy(value) {
|
|
138
|
+
if (value === undefined || value === null) return 0
|
|
139
|
+
const number = Number(value)
|
|
140
|
+
if (!Number.isInteger(number) || number < 0 || number > 3) throw entityError('E_KB_INVALID', 'secrecy must be an integer between 0 and 3')
|
|
141
|
+
return number
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function validateTitle(value, fallback) {
|
|
145
|
+
const raw = value === undefined || value === null || value === '' ? fallback : String(value)
|
|
146
|
+
const title = sanitizeLine(raw, 200)
|
|
147
|
+
if (!title) throw entityError('E_KB_INVALID', 'title is empty')
|
|
148
|
+
return title
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function titleFromName(name) {
|
|
152
|
+
const base = basename(String(name ?? ''))
|
|
153
|
+
const dot = base.lastIndexOf('.')
|
|
154
|
+
return (dot > 0 ? base.slice(0, dot) : base) || base
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Resolve client/project/matter ownership against the live customer tables. */
|
|
158
|
+
function resolveOwner(store, { clientId, projectId, matterId } = {}) {
|
|
159
|
+
let client = clientId === undefined || clientId === null ? null : String(clientId).trim() || null
|
|
160
|
+
let project = projectId === undefined || projectId === null ? null : String(projectId).trim() || null
|
|
161
|
+
let matter = matterId === undefined || matterId === null ? null : String(matterId).trim() || null
|
|
162
|
+
let projectRow
|
|
163
|
+
let matterRow
|
|
164
|
+
if (project) {
|
|
165
|
+
projectRow = typeof store.getProject === 'function' ? store.getProject(project) : undefined
|
|
166
|
+
if (!projectRow || projectRow.deletedAt) throw entityError('E_PROJECT_NOT_FOUND', 'unknown project: ' + project, 'advisers project ls')
|
|
167
|
+
if (!client) client = projectRow.clientId
|
|
168
|
+
else if (projectRow.clientId !== client) throw entityError('E_PROJECT_CLIENT_MISMATCH', 'project ' + project + ' belongs to client ' + projectRow.clientId)
|
|
169
|
+
}
|
|
170
|
+
if (matter) {
|
|
171
|
+
matterRow = typeof store.getMatter === 'function' ? store.getMatter(matter) : undefined
|
|
172
|
+
if (!matterRow) throw entityError('E_MATTER_NOT_FOUND', 'unknown matter: ' + matter, 'advisers matter ls')
|
|
173
|
+
if (!client && matterRow.clientId) client = matterRow.clientId
|
|
174
|
+
else if (client && matterRow.clientId && matterRow.clientId !== client) throw entityError('E_MATTER_CLIENT_MISMATCH', 'matter ' + matter + ' belongs to client ' + matterRow.clientId)
|
|
175
|
+
if (!project && matterRow.projectId) project = matterRow.projectId
|
|
176
|
+
}
|
|
177
|
+
if (client) {
|
|
178
|
+
const clientRow = typeof store.getClient === 'function' ? store.getClient(client) : undefined
|
|
179
|
+
if (!clientRow || clientRow.deletedAt) throw entityError('E_CLIENT_NOT_FOUND', 'unknown client: ' + client, 'advisers client ls')
|
|
180
|
+
}
|
|
181
|
+
return { clientId: client, projectId: project, matterId: matter }
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function duplicateError(existingId, name) {
|
|
185
|
+
const error = entityError('E_KB_DUPLICATE', 'document already indexed: ' + name, 'use --replace to re-index it')
|
|
186
|
+
error.existingId = existingId
|
|
187
|
+
error.skip = true
|
|
188
|
+
error.file = name
|
|
189
|
+
return error
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function findActiveBySha(store, sha256) {
|
|
193
|
+
return store.db.prepare("SELECT * FROM kb_documents WHERE source_sha256 = ? AND status = 'active' AND deleted_at IS NULL ORDER BY created_at DESC, rowid DESC LIMIT 1").get(String(sha256))
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function classifyIndexText(text, hmacKey) {
|
|
197
|
+
const safety = classify(String(text ?? '').slice(0, 100_000), { surface: 'attachment', hmacKey })
|
|
198
|
+
if (safety.action === 'block') throw safetyErrorFrom(safety, { surface: 'attachment' })
|
|
199
|
+
return {
|
|
200
|
+
action: safety.action,
|
|
201
|
+
code: safety.code ?? null,
|
|
202
|
+
categories: Array.isArray(safety.categories) ? [...safety.categories] : [],
|
|
203
|
+
ruleIds: Array.isArray(safety.ruleIds) ? [...safety.ruleIds] : [],
|
|
204
|
+
severity: safety.severity,
|
|
205
|
+
crisis: Boolean(safety.crisis),
|
|
206
|
+
downgraded: Boolean(safety.downgraded),
|
|
207
|
+
rulesetVersion: safety.rulesetVersion,
|
|
208
|
+
evidenceHash: safety.evidenceHash ?? null,
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function safetyJson(safety) {
|
|
213
|
+
try { return safety ? JSON.stringify(safety) : null } catch { return null }
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Public snake_case row -> JSON-ready camelCase document (no paths ever). */
|
|
217
|
+
export function mapKbDocumentRow(row) {
|
|
218
|
+
if (!row) return null
|
|
219
|
+
return {
|
|
220
|
+
id: String(row.id),
|
|
221
|
+
kind: String(row.kind),
|
|
222
|
+
title: String(row.title),
|
|
223
|
+
status: String(row.status),
|
|
224
|
+
...(row.client_id === null || row.client_id === undefined ? {} : { clientId: String(row.client_id) }),
|
|
225
|
+
...(row.project_id === null || row.project_id === undefined ? {} : { projectId: String(row.project_id) }),
|
|
226
|
+
...(row.matter_id === null || row.matter_id === undefined ? {} : { matterId: String(row.matter_id) }),
|
|
227
|
+
...(row.source_name === null || row.source_name === undefined ? {} : { sourceName: String(row.source_name) }),
|
|
228
|
+
sourceSha256: String(row.source_sha256),
|
|
229
|
+
...(row.mime === null || row.mime === undefined ? {} : { mime: String(row.mime) }),
|
|
230
|
+
sizeBytes: Number(row.size_bytes ?? 0),
|
|
231
|
+
...(row.encoding === null || row.encoding === undefined ? {} : { encoding: String(row.encoding) }),
|
|
232
|
+
extractor: String(row.extractor),
|
|
233
|
+
secrecy: Number(row.secrecy ?? 0),
|
|
234
|
+
needsOcr: Boolean(row.needs_ocr),
|
|
235
|
+
truncated: Boolean(row.truncated),
|
|
236
|
+
textBytes: Number(row.text_bytes ?? 0),
|
|
237
|
+
chunkCount: Number(row.chunk_count ?? 0),
|
|
238
|
+
createdAt: String(row.created_at),
|
|
239
|
+
capturedAt: String(row.captured_at),
|
|
240
|
+
...(row.deleted_at === null || row.deleted_at === undefined ? {} : { deletedAt: String(row.deleted_at) }),
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/* -------------------------------------------------------------------------- */
|
|
245
|
+
/* clearance SQL (filter BEFORE counting/paging) */
|
|
246
|
+
/* -------------------------------------------------------------------------- */
|
|
247
|
+
|
|
248
|
+
function ownershipValidSql(alias) {
|
|
249
|
+
return '(((' + alias + '.client_id IS NULL) OR EXISTS (SELECT 1 FROM clients c1 WHERE c1.id = ' + alias + '.client_id))'
|
|
250
|
+
+ ' AND ((' + alias + '.matter_id IS NULL) OR EXISTS (SELECT 1 FROM matters m1 WHERE m1.id = ' + alias + '.matter_id)))'
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function effectiveSecrecySql(alias) {
|
|
254
|
+
return 'MAX(COALESCE(' + alias + '.secrecy, 0),'
|
|
255
|
+
+ ' COALESCE((SELECT c2.secrecy_default FROM clients c2 WHERE c2.id = ' + alias + '.client_id), 0),'
|
|
256
|
+
+ ' COALESCE((SELECT m2.secrecy FROM matters m2 WHERE m2.id = ' + alias + '.matter_id), 0))'
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function docFiltersSql(alias, filters = {}) {
|
|
260
|
+
const clauses = []
|
|
261
|
+
const params = []
|
|
262
|
+
if (filters.clientId !== undefined && filters.clientId !== null) { clauses.push(alias + '.client_id = ?'); params.push(String(filters.clientId)) }
|
|
263
|
+
if (filters.projectId !== undefined && filters.projectId !== null) { clauses.push(alias + '.project_id = ?'); params.push(String(filters.projectId)) }
|
|
264
|
+
if (filters.matterId !== undefined && filters.matterId !== null) { clauses.push(alias + '.matter_id = ?'); params.push(String(filters.matterId)) }
|
|
265
|
+
if (filters.kind !== undefined && filters.kind !== null) {
|
|
266
|
+
const kind = String(filters.kind)
|
|
267
|
+
if (!KB_KINDS.includes(kind)) throw entityError('E_KB_INVALID', 'unknown document kind: ' + kind)
|
|
268
|
+
clauses.push(alias + '.kind = ?')
|
|
269
|
+
params.push(kind)
|
|
270
|
+
}
|
|
271
|
+
return { sql: clauses.length ? ' AND ' + clauses.join(' AND ') : '', params }
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function scopedDocumentWhere(filters, clearance, { withClearance = true } = {}) {
|
|
275
|
+
const filter = docFiltersSql('d', filters)
|
|
276
|
+
const sql = "d.status = 'active' AND d.deleted_at IS NULL AND " + ownershipValidSql('d')
|
|
277
|
+
+ (withClearance ? ' AND ' + effectiveSecrecySql('d') + ' <= ?' : '') + filter.sql
|
|
278
|
+
return { sql, params: withClearance ? [normalizeClearance(clearance), ...filter.params] : [...filter.params] }
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/* -------------------------------------------------------------------------- */
|
|
282
|
+
/* kb add */
|
|
283
|
+
/* -------------------------------------------------------------------------- */
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Index one local file. Throws typed E_KB_* errors; a duplicate is a skip-level
|
|
287
|
+
* E_KB_DUPLICATE carrying .existingId (the batch helper turns it into skipped).
|
|
288
|
+
*
|
|
289
|
+
* @param {object} store opened writable SqliteStore
|
|
290
|
+
* @param {{file: string, cwd?: string, title?: string, kind?: string, clientId?: string,
|
|
291
|
+
* projectId?: string, matterId?: string, secrecy?: number, replace?: boolean,
|
|
292
|
+
* hmacKey?: string|Buffer}} options
|
|
293
|
+
* @returns {Promise<object>} JSON-ready {ok, command, document, counts, ...}
|
|
294
|
+
*/
|
|
295
|
+
export async function kbAdd(store, options = {}) {
|
|
296
|
+
assertKbSchema(store)
|
|
297
|
+
const { file, cwd = process.cwd(), title, kind, clientId, projectId, matterId, secrecy, replace = false, hmacKey } = options
|
|
298
|
+
if (typeof file !== 'string' || !file.trim()) throw entityError('E_KB_INVALID', 'file path is required')
|
|
299
|
+
const absPath = resolve(cwd, file)
|
|
300
|
+
const name = basename(absPath)
|
|
301
|
+
let info
|
|
302
|
+
try {
|
|
303
|
+
info = lstatSync(absPath)
|
|
304
|
+
} catch (error) {
|
|
305
|
+
throw entityError('E_KB_READ', 'cannot read file ' + name + ' (' + String(error?.code ?? 'ENOENT') + ')', 'check that the path exists and is readable')
|
|
306
|
+
}
|
|
307
|
+
if (info.isSymbolicLink()) throw entityError('E_KB_INVALID', 'symbolic links are not indexed: ' + name)
|
|
308
|
+
if (!info.isFile()) throw entityError('E_KB_INVALID', 'not a regular file: ' + name)
|
|
309
|
+
if (info.size > KB_MAX_FILE_BYTES) throw entityError('E_KB_TOO_LARGE', 'file is larger than the 50 MB KB limit: ' + name)
|
|
310
|
+
const kindValue = validateKind(kind)
|
|
311
|
+
const secrecyValue = validateSecrecy(secrecy)
|
|
312
|
+
const titleValue = validateTitle(title, titleFromName(name))
|
|
313
|
+
const owner = resolveOwner(store, { clientId, projectId, matterId })
|
|
314
|
+
let hashed
|
|
315
|
+
try {
|
|
316
|
+
hashed = sha256FileSync(absPath)
|
|
317
|
+
} catch (error) {
|
|
318
|
+
throw entityError('E_KB_READ', 'cannot hash file ' + name + ' (' + String(error?.code ?? 'EIO') + ')', 'check the file permissions and retry')
|
|
319
|
+
}
|
|
320
|
+
const existing = findActiveBySha(store, hashed.sha256)
|
|
321
|
+
if (existing && !replace) throw duplicateError(existing.id, name)
|
|
322
|
+
let extraction
|
|
323
|
+
try {
|
|
324
|
+
extraction = await extractForIndex(absPath)
|
|
325
|
+
} catch (error) {
|
|
326
|
+
if (error?.code) throw error
|
|
327
|
+
throw entityError('E_KB_INVALID', 'cannot extract text from ' + name + ': ' + failureMessage(error), 'check that the file is not corrupt')
|
|
328
|
+
}
|
|
329
|
+
const safety = extraction.needsOcr && !extraction.text ? null : classifyIndexText(extraction.text, hmacKey)
|
|
330
|
+
const chunked = chunkText(extraction.text, { maxChunks: KB_MAX_CHUNKS })
|
|
331
|
+
const truncated = Boolean(extraction.truncated || chunked.truncated)
|
|
332
|
+
const docId = newId('kb_')
|
|
333
|
+
const capturedAt = nowIso()
|
|
334
|
+
const inserted = store.mutate(() => {
|
|
335
|
+
const duplicate = findActiveBySha(store, hashed.sha256)
|
|
336
|
+
if (duplicate && !replace) throw duplicateError(duplicate.id, name)
|
|
337
|
+
if (duplicate) {
|
|
338
|
+
store.db.prepare("UPDATE kb_documents SET status = 'deleted', deleted_at = ? WHERE id = ?").run(capturedAt, duplicate.id)
|
|
339
|
+
store.db.prepare('DELETE FROM kb_chunks WHERE doc_id = ?').run(duplicate.id)
|
|
340
|
+
}
|
|
341
|
+
store.db.prepare(INSERT_DOCUMENT_SQL).run(
|
|
342
|
+
docId, kindValue, titleValue, owner.clientId, owner.projectId, owner.matterId,
|
|
343
|
+
String(extraction.sourceName), hashed.sha256, extraction.mime, Number(extraction.sizeBytes),
|
|
344
|
+
extraction.encoding, extraction.extractor, 'active', secrecyValue,
|
|
345
|
+
extraction.needsOcr ? 1 : 0, truncated ? 1 : 0, Number(extraction.textBytes),
|
|
346
|
+
chunked.chunks.length, safetyJson(safety), capturedAt, capturedAt, null,
|
|
347
|
+
)
|
|
348
|
+
const insertChunk = store.db.prepare(INSERT_CHUNK_SQL)
|
|
349
|
+
for (const chunk of chunked.chunks) {
|
|
350
|
+
insertChunk.run('chk_' + randomUUID().replace(/-/g, '').slice(0, 16), docId, chunk.seq, chunk.text, chunk.charCount)
|
|
351
|
+
}
|
|
352
|
+
return { replacedId: duplicate ? duplicate.id : null }
|
|
353
|
+
}, {
|
|
354
|
+
actor: 'local',
|
|
355
|
+
action: 'kb.add',
|
|
356
|
+
allowed: true,
|
|
357
|
+
reason: 'doc ' + docId + ', ' + chunked.chunks.length + ' chunks, ' + Number(extraction.textBytes) + ' bytes' + (replace ? ' (replace)' : ''),
|
|
358
|
+
})
|
|
359
|
+
const row = store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(docId)
|
|
360
|
+
return {
|
|
361
|
+
ok: true,
|
|
362
|
+
command: 'kb add',
|
|
363
|
+
document: mapKbDocumentRow(row),
|
|
364
|
+
counts: { added: 1, chunks: chunked.chunks.length, textBytes: Number(extraction.textBytes) },
|
|
365
|
+
truncated,
|
|
366
|
+
truncatedBy: [...extraction.truncatedBy, ...(chunked.truncated ? ['chunk_limit'] : [])],
|
|
367
|
+
needsOcr: Boolean(extraction.needsOcr),
|
|
368
|
+
encoding: extraction.encoding,
|
|
369
|
+
extractor: extraction.extractor,
|
|
370
|
+
warnings: [...extraction.warnings],
|
|
371
|
+
...(inserted.result.replacedId ? { replacedId: inserted.result.replacedId } : {}),
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function failureMessage(error) {
|
|
376
|
+
const raw = error?.code ? String(error.message ?? '') : String(error?.message ?? error)
|
|
377
|
+
const text = stripControlChars(stripTerminalSequences(raw), { keepNewlines: false, keepTabs: false, replacement: ' ' }).replace(/\s+/gu, ' ').trim()
|
|
378
|
+
return Array.from(text).slice(0, 300).join('')
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Index a batch of files, aggregating per-file outcomes into the CLI JSON shape.
|
|
383
|
+
* A duplicate is skipped (ok:false only when every file failed).
|
|
384
|
+
*/
|
|
385
|
+
export async function kbAddMany(store, options = {}) {
|
|
386
|
+
const { files, ...rest } = options
|
|
387
|
+
if (!Array.isArray(files) || files.length === 0) throw entityError('E_KB_INVALID', 'at least one file is required')
|
|
388
|
+
if (files.length > KB_MAX_BATCH_FILES) throw entityError('E_KB_LIMIT', 'at most ' + KB_MAX_BATCH_FILES + ' files per batch')
|
|
389
|
+
const added = []
|
|
390
|
+
const skipped = []
|
|
391
|
+
const failed = []
|
|
392
|
+
let chunks = 0
|
|
393
|
+
let textBytes = 0
|
|
394
|
+
for (const file of files) {
|
|
395
|
+
const fileName = basename(String(file))
|
|
396
|
+
try {
|
|
397
|
+
const result = await kbAdd(store, { ...rest, file })
|
|
398
|
+
added.push(result.document)
|
|
399
|
+
chunks += result.counts.chunks
|
|
400
|
+
textBytes += result.counts.textBytes
|
|
401
|
+
} catch (error) {
|
|
402
|
+
if (error?.code === 'E_KB_DUPLICATE') {
|
|
403
|
+
skipped.push({ file: fileName, code: 'E_KB_DUPLICATE', existingId: error.existingId ?? null })
|
|
404
|
+
} else {
|
|
405
|
+
failed.push({ file: fileName, code: error?.code ?? 'E_INTERNAL', message: failureMessage(error) })
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return {
|
|
410
|
+
ok: added.length > 0 || failed.length === 0,
|
|
411
|
+
command: 'kb add',
|
|
412
|
+
counts: { added: added.length, skipped: skipped.length, failed: failed.length, chunks, textBytes },
|
|
413
|
+
added,
|
|
414
|
+
skipped,
|
|
415
|
+
failed,
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/* -------------------------------------------------------------------------- */
|
|
420
|
+
/* kb ls / kb show */
|
|
421
|
+
/* -------------------------------------------------------------------------- */
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* List documents with the clearance filter applied before counting/paging.
|
|
425
|
+
* Default is active documents; all:true includes tombstones.
|
|
426
|
+
*/
|
|
427
|
+
export function kbList(store, options = {}) {
|
|
428
|
+
assertKbSchema(store)
|
|
429
|
+
const { clientId, projectId, matterId, kind, all = false, limit, clearance } = options
|
|
430
|
+
const level = normalizeClearance(clearance === undefined ? KB_DEFAULT_CLEARANCE : clearance)
|
|
431
|
+
const pageLimit = clampLimit(limit, KB_LIST_LIMIT, KB_LIST_LIMIT_MAX)
|
|
432
|
+
const statusSql = all === true ? '1 = 1' : "d.status = 'active' AND d.deleted_at IS NULL"
|
|
433
|
+
const filter = docFiltersSql('d', { clientId, projectId, matterId, kind })
|
|
434
|
+
const ownership = ownershipValidSql('d')
|
|
435
|
+
const effective = effectiveSecrecySql('d')
|
|
436
|
+
return store.read(() => {
|
|
437
|
+
const visibleWhere = statusSql + ' AND ' + ownership + ' AND ' + effective + ' <= ?' + filter.sql
|
|
438
|
+
const probeWhere = statusSql + ' AND ' + ownership + filter.sql
|
|
439
|
+
const visibleParams = [level, ...filter.params]
|
|
440
|
+
const total = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_documents d WHERE ' + visibleWhere).get(...visibleParams)?.n ?? 0)
|
|
441
|
+
const probeTotal = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_documents d WHERE ' + probeWhere).get(...filter.params)?.n ?? 0)
|
|
442
|
+
const rows = store.db.prepare('SELECT d.* FROM kb_documents d WHERE ' + visibleWhere + ' ORDER BY d.created_at DESC, d.rowid DESC LIMIT ?')
|
|
443
|
+
.all(...visibleParams, pageLimit)
|
|
444
|
+
return {
|
|
445
|
+
ok: true,
|
|
446
|
+
command: 'kb ls',
|
|
447
|
+
documents: rows.map(mapKbDocumentRow),
|
|
448
|
+
total,
|
|
449
|
+
truncated: total > rows.length,
|
|
450
|
+
limit: pageLimit,
|
|
451
|
+
clearance: level,
|
|
452
|
+
protectedExcluded: Math.max(0, probeTotal - total),
|
|
453
|
+
}
|
|
454
|
+
})
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/** One document plus (optionally) a bounded chunk preview. */
|
|
458
|
+
export function kbShow(store, id, options = {}) {
|
|
459
|
+
assertKbSchema(store)
|
|
460
|
+
const key = typeof id === 'string' ? id.trim() : ''
|
|
461
|
+
if (!key) throw entityError('E_KB_INVALID', 'document id is required')
|
|
462
|
+
const clearance = normalizeClearance(options.clearance === undefined ? KB_DEFAULT_CLEARANCE : options.clearance)
|
|
463
|
+
const chunkLimit = clampLimit(options.chunks, 0, 20)
|
|
464
|
+
return store.read(() => {
|
|
465
|
+
const row = store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(key)
|
|
466
|
+
if (!row || row.status !== 'active' || row.deleted_at !== null) throw entityError('E_KB_NOT_FOUND', 'unknown document: ' + key, 'advisers kb ls')
|
|
467
|
+
const level = Number(store.db.prepare('SELECT ' + effectiveSecrecySql('d') + ' AS secrecy FROM kb_documents d WHERE d.id = ?').get(key)?.secrecy ?? Number.POSITIVE_INFINITY)
|
|
468
|
+
if (!Number.isFinite(level) || level > clearance) throw entityError('E_KB_NOT_FOUND', 'unknown document: ' + key)
|
|
469
|
+
const chunks = []
|
|
470
|
+
let chunksTotal = 0
|
|
471
|
+
if (chunkLimit > 0) {
|
|
472
|
+
const rows = store.db.prepare('SELECT id, seq, text, char_count FROM kb_chunks WHERE doc_id = ? ORDER BY seq LIMIT ?').all(key, chunkLimit)
|
|
473
|
+
for (const chunk of rows) {
|
|
474
|
+
const text = Array.from(String(chunk.text ?? ''))
|
|
475
|
+
const preview = text.slice(0, KB_CHUNK_PREVIEW_CHARS).join('')
|
|
476
|
+
chunks.push({ id: String(chunk.id), seq: Number(chunk.seq), charCount: Number(chunk.char_count), text: sanitizeLine(preview, KB_CHUNK_PREVIEW_CHARS), truncated: text.length > KB_CHUNK_PREVIEW_CHARS })
|
|
477
|
+
}
|
|
478
|
+
chunksTotal = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_chunks WHERE doc_id = ?').get(key)?.n ?? 0)
|
|
479
|
+
}
|
|
480
|
+
return { ok: true, command: 'kb show', document: mapKbDocumentRow(row), chunks, chunksTotal, clearance }
|
|
481
|
+
})
|
|
482
|
+
}
|
|
483
|
+
/* -------------------------------------------------------------------------- */
|
|
484
|
+
/* kb rm / kb purge */
|
|
485
|
+
/* -------------------------------------------------------------------------- */
|
|
486
|
+
|
|
487
|
+
/** Tombstone one document and physically delete its chunks (searchable -> 0). */
|
|
488
|
+
export function kbRm(store, id) {
|
|
489
|
+
assertKbSchema(store)
|
|
490
|
+
const key = typeof id === 'string' ? id.trim() : ''
|
|
491
|
+
if (!key) throw entityError('E_KB_INVALID', 'document id is required')
|
|
492
|
+
const audit = { actor: 'local', action: 'kb.rm', allowed: true, reason: '' }
|
|
493
|
+
const outcome = store.mutate(() => {
|
|
494
|
+
const row = store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(key)
|
|
495
|
+
if (!row || row.status !== 'active' || row.deleted_at !== null) throw entityError('E_KB_NOT_FOUND', 'unknown document: ' + key, 'advisers kb ls')
|
|
496
|
+
const chunks = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_chunks WHERE doc_id = ?').get(key)?.n ?? 0)
|
|
497
|
+
store.db.prepare('DELETE FROM kb_chunks WHERE doc_id = ?').run(key)
|
|
498
|
+
store.db.prepare("UPDATE kb_documents SET status = 'deleted', deleted_at = ? WHERE id = ?").run(nowIso(), key)
|
|
499
|
+
audit.reason = 'doc ' + key + ', ' + chunks + ' chunks removed'
|
|
500
|
+
return { document: mapKbDocumentRow(store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(key)), chunks }
|
|
501
|
+
}, audit)
|
|
502
|
+
return { ok: true, command: 'kb rm', removed: true, document: outcome.result.document, counts: { chunks: outcome.result.chunks } }
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* purge one document: dry-run by default (yes:false), physical delete with
|
|
507
|
+
* yes:true. Deleting chunks clears the FTS entries through the sync triggers.
|
|
508
|
+
*/
|
|
509
|
+
export function kbPurge(store, id, options = {}) {
|
|
510
|
+
assertKbSchema(store)
|
|
511
|
+
const key = typeof id === 'string' ? id.trim() : ''
|
|
512
|
+
if (!key) throw entityError('E_KB_INVALID', 'document id is required')
|
|
513
|
+
const yes = options.yes === true
|
|
514
|
+
if (!yes) {
|
|
515
|
+
return store.read(() => {
|
|
516
|
+
const row = store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(key)
|
|
517
|
+
if (!row) throw entityError('E_KB_NOT_FOUND', 'unknown document: ' + key, 'advisers kb ls')
|
|
518
|
+
const chunks = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_chunks WHERE doc_id = ?').get(key)?.n ?? 0)
|
|
519
|
+
const factLinks = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_fact_links WHERE doc_id = ?').get(key)?.n ?? 0)
|
|
520
|
+
return { ok: true, command: 'kb purge', dryRun: true, purged: false, document: mapKbDocumentRow(row), counts: { chunks, factLinks }, hint: 'pass --yes to delete permanently' }
|
|
521
|
+
})
|
|
522
|
+
}
|
|
523
|
+
const audit = { actor: 'local', action: 'kb.purge', allowed: true, reason: '' }
|
|
524
|
+
const outcome = store.mutate(() => {
|
|
525
|
+
const row = store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(key)
|
|
526
|
+
if (!row) throw entityError('E_KB_NOT_FOUND', 'unknown document: ' + key, 'advisers kb ls')
|
|
527
|
+
const chunks = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_chunks WHERE doc_id = ?').get(key)?.n ?? 0)
|
|
528
|
+
const factLinks = Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_fact_links WHERE doc_id = ?').get(key)?.n ?? 0)
|
|
529
|
+
store.db.prepare('DELETE FROM kb_chunks WHERE doc_id = ?').run(key)
|
|
530
|
+
store.db.prepare('DELETE FROM kb_fact_links WHERE doc_id = ?').run(key)
|
|
531
|
+
store.db.prepare('DELETE FROM kb_documents WHERE id = ?').run(key)
|
|
532
|
+
audit.reason = 'doc ' + key + ', ' + chunks + ' chunks, ' + factLinks + ' links'
|
|
533
|
+
return { counts: { chunks, factLinks }, document: mapKbDocumentRow(row) }
|
|
534
|
+
}, audit)
|
|
535
|
+
return { ok: true, command: 'kb purge', dryRun: false, purged: true, document: outcome.result.document, counts: { chunks: outcome.result.counts.chunks, factLinks: outcome.result.counts.factLinks } }
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
/* -------------------------------------------------------------------------- */
|
|
539
|
+
/* kb reindex / verify */
|
|
540
|
+
/* -------------------------------------------------------------------------- */
|
|
541
|
+
|
|
542
|
+
const KB_FTS_SPECS = Object.freeze([
|
|
543
|
+
{ name: 'kb_fts', source: 'kb_chunks', triggers: ['kb_chunks_ai', 'kb_chunks_ad', 'kb_chunks_au'] },
|
|
544
|
+
{ name: 'kb_pages_fts', source: 'kb_pages', triggers: ['kb_pages_ai', 'kb_pages_ad', 'kb_pages_au'] },
|
|
545
|
+
])
|
|
546
|
+
|
|
547
|
+
/** External-content FTS integrity: presence, triggers, count parity, byte check. */
|
|
548
|
+
export function verifyKbFts(store) {
|
|
549
|
+
assertKbSchema(store)
|
|
550
|
+
const report = { ok: true, tokenizer: kbFtsTokenizer(store), degraded: [], tables: {} }
|
|
551
|
+
for (const spec of KB_FTS_SPECS) {
|
|
552
|
+
const entry = { present: false, ok: false, source: spec.source, triggers: false, count: 0, sourceCount: 0, error: null }
|
|
553
|
+
entry.present = hasSqliteObject(store, spec.name)
|
|
554
|
+
if (!entry.present) {
|
|
555
|
+
entry.error = 'index missing'
|
|
556
|
+
report.ok = false
|
|
557
|
+
report.degraded.push('fts_unavailable')
|
|
558
|
+
report.tables[spec.name] = entry
|
|
559
|
+
continue
|
|
560
|
+
}
|
|
561
|
+
try { entry.sourceCount = Number(store.db.prepare('SELECT COUNT(*) AS n FROM ' + spec.source).get()?.n ?? 0) } catch (error) { entry.error = String(error?.message ?? error).slice(0, 200) }
|
|
562
|
+
// The FTS5 shadow table (<name>_docsize) holds one row per indexed row;
|
|
563
|
+
// counting the virtual table itself reads through external content and
|
|
564
|
+
// cannot see a lost index. rank=1 makes integrity-check compare the index
|
|
565
|
+
// against kb_chunks / kb_pages (not just internal consistency).
|
|
566
|
+
try { entry.count = Number(store.db.prepare('SELECT COUNT(*) AS n FROM ' + spec.name + '_docsize').get()?.n ?? 0) } catch (error) { entry.error = String(error?.message ?? error).slice(0, 200) }
|
|
567
|
+
entry.triggers = spec.triggers.every((trigger) => hasSqliteObject(store, trigger, ['trigger']))
|
|
568
|
+
entry.ok = entry.error === null && entry.triggers
|
|
569
|
+
if (entry.ok) {
|
|
570
|
+
try {
|
|
571
|
+
store.db.prepare('INSERT INTO ' + spec.name + '(' + spec.name + ", rank) VALUES('integrity-check', 1)").run()
|
|
572
|
+
} catch (error) {
|
|
573
|
+
entry.ok = false
|
|
574
|
+
entry.error = String(error?.message ?? error).slice(0, 200)
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
if (entry.ok && entry.count !== entry.sourceCount) {
|
|
578
|
+
entry.ok = false
|
|
579
|
+
entry.error = 'index count ' + entry.count + ' != source count ' + entry.sourceCount
|
|
580
|
+
}
|
|
581
|
+
if (!entry.ok) report.ok = false
|
|
582
|
+
report.tables[spec.name] = entry
|
|
583
|
+
}
|
|
584
|
+
if (report.tokenizer === 'unicode61') report.degraded.push('fts_unicode61')
|
|
585
|
+
report.degraded = [...new Set(report.degraded)].sort()
|
|
586
|
+
return report
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Verify (default) or rebuild the two KB FTS indexes. Rebuilding re-indexes
|
|
591
|
+
* kb_chunks / kb_pages rows only; the source file is never re-read (its path is
|
|
592
|
+
* not stored). Writes exactly one audit row when anything is rebuilt.
|
|
593
|
+
*/
|
|
594
|
+
export function kbReindex(store, options = {}) {
|
|
595
|
+
assertKbSchema(store)
|
|
596
|
+
const force = options.rebuild === true
|
|
597
|
+
let integrity = verifyKbFts(store)
|
|
598
|
+
const needed = force || !integrity.ok
|
|
599
|
+
const rebuilt = []
|
|
600
|
+
if (needed) {
|
|
601
|
+
const audit = { actor: 'local', action: 'kb.reindex', allowed: true, reason: '' }
|
|
602
|
+
store.mutate(() => {
|
|
603
|
+
for (const spec of KB_FTS_SPECS) {
|
|
604
|
+
if (!hasSqliteObject(store, spec.name)) continue
|
|
605
|
+
store.db.exec("INSERT INTO " + spec.name + '(' + spec.name + ") VALUES('rebuild')")
|
|
606
|
+
rebuilt.push(spec.name)
|
|
607
|
+
}
|
|
608
|
+
audit.reason = 'rebuilt ' + rebuilt.length + ' index(es): ' + rebuilt.join(',')
|
|
609
|
+
}, audit)
|
|
610
|
+
integrity = verifyKbFts(store)
|
|
611
|
+
}
|
|
612
|
+
const counts = {
|
|
613
|
+
documents: Number(store.db.prepare("SELECT COUNT(*) AS n FROM kb_documents WHERE status = 'active' AND deleted_at IS NULL").get()?.n ?? 0),
|
|
614
|
+
chunks: Number(store.db.prepare('SELECT COUNT(*) AS n FROM kb_chunks').get()?.n ?? 0),
|
|
615
|
+
pages: Number(store.db.prepare("SELECT COUNT(*) AS n FROM kb_pages WHERE status = 'generated' AND deleted_at IS NULL").get()?.n ?? 0),
|
|
616
|
+
}
|
|
617
|
+
return { ok: integrity.ok, command: 'kb reindex', rebuilt, integrity, counts, degraded: integrity.degraded }
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
/* -------------------------------------------------------------------------- */
|
|
621
|
+
/* kb link / unlink (explicit fact <-> document provenance) */
|
|
622
|
+
/* -------------------------------------------------------------------------- */
|
|
623
|
+
|
|
624
|
+
function assertActiveDocument(store, docId) {
|
|
625
|
+
const row = store.db.prepare('SELECT * FROM kb_documents WHERE id = ?').get(String(docId))
|
|
626
|
+
if (!row || row.status !== 'active' || row.deleted_at !== null) throw entityError('E_KB_NOT_FOUND', 'unknown document: ' + String(docId), 'advisers kb ls')
|
|
627
|
+
return row
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/** Link one fact to one document (and optionally a chunk). */
|
|
631
|
+
export function kbLink(store, { docId, factId, relation = 'source', chunkId = null } = {}) {
|
|
632
|
+
assertKbSchema(store)
|
|
633
|
+
const doc = String(docId ?? '').trim()
|
|
634
|
+
const fact = String(factId ?? '').trim()
|
|
635
|
+
if (!doc) throw entityError('E_KB_INVALID', 'document id is required')
|
|
636
|
+
if (!fact) throw entityError('E_KB_INVALID', 'fact id is required')
|
|
637
|
+
const rel = String(relation)
|
|
638
|
+
if (!KB_RELATIONS.includes(rel)) throw entityError('E_KB_INVALID', 'unknown relation: ' + rel, 'use one of ' + KB_RELATIONS.join(', '))
|
|
639
|
+
const audit = { actor: 'local', action: 'kb.link', allowed: true, reason: '' }
|
|
640
|
+
const outcome = store.mutate(() => {
|
|
641
|
+
assertActiveDocument(store, doc)
|
|
642
|
+
const factRow = typeof store.getFact === 'function' ? store.getFact(fact) : undefined
|
|
643
|
+
if (!factRow) throw entityError('E_FACT_NOT_FOUND', 'unknown fact: ' + fact, 'advisers fact ls')
|
|
644
|
+
let chunk = null
|
|
645
|
+
if (chunkId !== null && chunkId !== undefined && chunkId !== '') {
|
|
646
|
+
const chunkRow = store.db.prepare('SELECT id FROM kb_chunks WHERE id = ? AND doc_id = ?').get(String(chunkId), doc)
|
|
647
|
+
if (!chunkRow) throw entityError('E_KB_INVALID', 'chunk does not belong to the document: ' + String(chunkId))
|
|
648
|
+
chunk = String(chunkId)
|
|
649
|
+
}
|
|
650
|
+
const createdAt = nowIso()
|
|
651
|
+
store.db.prepare('INSERT OR REPLACE INTO kb_fact_links(fact_id, doc_id, chunk_id, relation, created_at) VALUES(?,?,?,?,?)').run(fact, doc, chunk, rel, createdAt)
|
|
652
|
+
audit.reason = 'fact ' + fact + ' -> doc ' + doc + ' (' + rel + ')'
|
|
653
|
+
return { factId: fact, docId: doc, chunkId: chunk, relation: rel, createdAt }
|
|
654
|
+
}, audit)
|
|
655
|
+
return { ok: true, command: 'kb link', link: outcome.result }
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/** Remove one explicit fact <-> document link (idempotent; returns removed count). */
|
|
659
|
+
export function kbUnlink(store, { docId, factId, relation = null } = {}) {
|
|
660
|
+
assertKbSchema(store)
|
|
661
|
+
const doc = String(docId ?? '').trim()
|
|
662
|
+
const fact = String(factId ?? '').trim()
|
|
663
|
+
if (!doc) throw entityError('E_KB_INVALID', 'document id is required')
|
|
664
|
+
if (!fact) throw entityError('E_KB_INVALID', 'fact id is required')
|
|
665
|
+
const audit = { actor: 'local', action: 'kb.unlink', allowed: true, reason: '' }
|
|
666
|
+
const outcome = store.mutate(() => {
|
|
667
|
+
assertActiveDocument(store, doc)
|
|
668
|
+
const removed = relation === null || relation === undefined
|
|
669
|
+
? Number(store.db.prepare('DELETE FROM kb_fact_links WHERE fact_id = ? AND doc_id = ?').run(fact, doc).changes ?? 0)
|
|
670
|
+
: Number(store.db.prepare('DELETE FROM kb_fact_links WHERE fact_id = ? AND doc_id = ? AND relation = ?').run(fact, doc, String(relation)).changes ?? 0)
|
|
671
|
+
audit.reason = 'fact ' + fact + ' x doc ' + doc + ': ' + removed + ' link(s)'
|
|
672
|
+
return removed
|
|
673
|
+
}, audit)
|
|
674
|
+
return { ok: true, command: 'kb unlink', removed: Number(outcome.result ?? 0) }
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/* -------------------------------------------------------------------------- */
|
|
678
|
+
/* search */
|
|
679
|
+
/* -------------------------------------------------------------------------- */
|
|
680
|
+
|
|
681
|
+
const CJK_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af]/
|
|
682
|
+
|
|
683
|
+
function hasCjk(text) {
|
|
684
|
+
return CJK_RE.test(String(text ?? ''))
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
function likePattern(token) {
|
|
688
|
+
return '%' + String(token).replace(/[\\%_]/g, (char) => '\\' + char) + '%'
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Decide MATCH vs LIKE for one query. Matches 45 section 3.5:
|
|
693
|
+
* trigram + every token >= 3 code points -> MATCH; otherwise LIKE with an
|
|
694
|
+
* explicit degraded reason (never a silent behavioral switch).
|
|
695
|
+
*/
|
|
696
|
+
export function selectKbSearchPlan(store, tokens, table = 'kb_fts') {
|
|
697
|
+
const tokenizer = kbFtsTokenizer(store, table)
|
|
698
|
+
const degraded = []
|
|
699
|
+
if (!Array.isArray(tokens) || tokens.length === 0) return { mode: 'like', degraded }
|
|
700
|
+
if (tokenizer === null) { degraded.push('fts_unavailable'); return { mode: 'like', degraded } }
|
|
701
|
+
if (tokenizer === 'trigram') {
|
|
702
|
+
if (tokens.some((token) => codePointLength(token) < 3)) { degraded.push('short_query_like'); return { mode: 'like', degraded } }
|
|
703
|
+
const ftsQuery = tokens.map((token) => '"' + String(token).replace(/"/g, ' ') + '"').join(' AND ')
|
|
704
|
+
return { mode: 'match', ftsQuery, degraded }
|
|
705
|
+
}
|
|
706
|
+
if (tokens.some((token) => hasCjk(token))) degraded.push('fts_unicode61')
|
|
707
|
+
if (tokens.some((token) => codePointLength(token) < 3)) degraded.push('short_query_like')
|
|
708
|
+
return { mode: 'like', degraded }
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/** Sanitize a query: strip C0/C1 controls, collapse whitespace, cap length. */
|
|
712
|
+
export function normalizeKbQuery(query) {
|
|
713
|
+
if (typeof query !== 'string') throw entityError('E_KB_INVALID', 'search query is required')
|
|
714
|
+
const cleaned = stripControlChars(query, { keepNewlines: false, keepTabs: false, replacement: ' ' }).replace(/\s+/gu, ' ').trim()
|
|
715
|
+
if (!cleaned) throw entityError('E_USAGE', 'search query is empty')
|
|
716
|
+
if (codePointLength(cleaned) > KB_QUERY_MAX_CHARS) throw entityError('E_QUERY_TOO_LONG', 'search query is longer than ' + KB_QUERY_MAX_CHARS + ' characters')
|
|
717
|
+
const tokens = cleaned.split(' ').filter(Boolean)
|
|
718
|
+
if (tokens.length === 0) throw entityError('E_USAGE', 'search query is empty')
|
|
719
|
+
return { query: cleaned, tokens }
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* Build a bounded, sanitized snippet plus numeric highlight ranges.
|
|
724
|
+
* highlights offsets are UTF-16 code-unit indexes into the returned snippet
|
|
725
|
+
* (String.slice semantics); callers render highlights themselves, never as
|
|
726
|
+
* injected HTML/Markdown.
|
|
727
|
+
*/
|
|
728
|
+
export function makeSnippet(text, tokens) {
|
|
729
|
+
const cleaned = stripControlChars(stripBidiControls(stripTerminalSequences(String(text ?? ''))), { keepNewlines: false, keepTabs: false, replacement: ' ' })
|
|
730
|
+
.replace(/\s+/gu, ' ').trim()
|
|
731
|
+
if (!cleaned) return { snippet: '', highlights: [] }
|
|
732
|
+
const lowered = cleaned.toLowerCase()
|
|
733
|
+
const comparable = lowered.length === cleaned.length ? lowered : cleaned
|
|
734
|
+
const needles = tokens.map((token) => String(token).toLowerCase()).filter(Boolean)
|
|
735
|
+
let anchor = -1
|
|
736
|
+
for (const needle of needles) {
|
|
737
|
+
const index = comparable.indexOf(needle)
|
|
738
|
+
if (index !== -1 && (anchor === -1 || index < anchor)) anchor = index
|
|
739
|
+
}
|
|
740
|
+
const chars = Array.from(cleaned)
|
|
741
|
+
const charIndex = anchor <= 0 ? 0 : Array.from(cleaned.slice(0, anchor)).length
|
|
742
|
+
let start = charIndex > 100 ? charIndex - 80 : 0
|
|
743
|
+
let windowChars = chars.slice(start, start + KB_SNIPPET_CHARS)
|
|
744
|
+
if (windowChars.length < KB_SNIPPET_CHARS && chars.length > KB_SNIPPET_CHARS) {
|
|
745
|
+
start = Math.max(0, chars.length - KB_SNIPPET_CHARS)
|
|
746
|
+
windowChars = chars.slice(start, start + KB_SNIPPET_CHARS)
|
|
747
|
+
}
|
|
748
|
+
const snippet = windowChars.join('')
|
|
749
|
+
const snippetLower = snippet.toLowerCase()
|
|
750
|
+
const haystack = snippetLower.length === snippet.length ? snippetLower : snippet
|
|
751
|
+
const ranges = []
|
|
752
|
+
for (const needle of needles) {
|
|
753
|
+
let from = 0
|
|
754
|
+
for (let count = 0; count < 8; count += 1) {
|
|
755
|
+
const index = haystack.indexOf(needle, from)
|
|
756
|
+
if (index === -1) break
|
|
757
|
+
ranges.push({ start: index, end: index + needle.length })
|
|
758
|
+
from = index + needle.length
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
ranges.sort((a, b) => (a.start - b.start) || (a.end - b.end))
|
|
762
|
+
const highlights = []
|
|
763
|
+
for (const range of ranges) {
|
|
764
|
+
const last = highlights[highlights.length - 1]
|
|
765
|
+
if (last && range.start <= last.end) last.end = Math.max(last.end, range.end)
|
|
766
|
+
else highlights.push({ start: range.start, end: range.end })
|
|
767
|
+
}
|
|
768
|
+
return { snippet, highlights }
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
function countRows(store, sql, params) {
|
|
772
|
+
return Number(store.db.prepare(sql).get(...params)?.n ?? 0)
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
function loadScopeMaps(store) {
|
|
776
|
+
const clients = new Map(store.db.prepare('SELECT id, secrecy_default FROM clients').all().map((row) => [String(row.id), Number(row.secrecy_default ?? 0)]))
|
|
777
|
+
const matters = new Map(store.db.prepare('SELECT id, secrecy, client_id FROM matters').all().map((row) => [String(row.id), { secrecy: Number(row.secrecy ?? 0), clientId: row.client_id === null || row.client_id === undefined ? null : String(row.client_id) }]))
|
|
778
|
+
const projects = new Map(store.db.prepare('SELECT id, client_id FROM projects').all().map((row) => [String(row.id), row.client_id === null || row.client_id === undefined ? null : String(row.client_id)]))
|
|
779
|
+
return { clients, matters, projects }
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
function pageOwner(row, maps) {
|
|
783
|
+
let data = null
|
|
784
|
+
try { data = JSON.parse(String(row.json ?? '')) } catch { data = null }
|
|
785
|
+
if (data && data.owner && typeof data.owner === 'object') {
|
|
786
|
+
return {
|
|
787
|
+
clientId: data.owner.clientId === undefined || data.owner.clientId === null ? null : String(data.owner.clientId),
|
|
788
|
+
matterId: data.owner.matterId === undefined || data.owner.matterId === null ? null : String(data.owner.matterId),
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
const kind = String(row.kind)
|
|
792
|
+
if (kind === 'client' || kind === 'timeline') return { clientId: String(row.subject_id), matterId: null }
|
|
793
|
+
if (kind === 'project') {
|
|
794
|
+
const clientId = maps.projects.get(String(row.subject_id))
|
|
795
|
+
return clientId === undefined ? null : { clientId, matterId: null }
|
|
796
|
+
}
|
|
797
|
+
return null
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
function ownerSecrecy(maps, owner) {
|
|
801
|
+
if (!owner) return Number.POSITIVE_INFINITY
|
|
802
|
+
let level = 0
|
|
803
|
+
if (owner.clientId) {
|
|
804
|
+
const value = maps.clients.get(String(owner.clientId))
|
|
805
|
+
if (value === undefined) return Number.POSITIVE_INFINITY
|
|
806
|
+
level = Math.max(level, value)
|
|
807
|
+
}
|
|
808
|
+
if (owner.matterId) {
|
|
809
|
+
const matter = maps.matters.get(String(owner.matterId))
|
|
810
|
+
if (!matter) return Number.POSITIVE_INFINITY
|
|
811
|
+
level = Math.max(level, matter.secrecy)
|
|
812
|
+
if (matter.clientId) {
|
|
813
|
+
const value = maps.clients.get(String(matter.clientId))
|
|
814
|
+
if (value === undefined) return Number.POSITIVE_INFINITY
|
|
815
|
+
level = Math.max(level, value)
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
return level
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function documentHitFromRow(row, tokens, matchPath, score) {
|
|
822
|
+
const hit = {
|
|
823
|
+
kind: 'kb_chunk',
|
|
824
|
+
ref: { kind: 'kb_chunk', id: String(row.chunk_id) },
|
|
825
|
+
docId: String(row.doc_id),
|
|
826
|
+
title: sanitizeLine(row.doc_title, 200),
|
|
827
|
+
snippet: '',
|
|
828
|
+
highlights: [],
|
|
829
|
+
matchPath,
|
|
830
|
+
at: String(row.created_at ?? ''),
|
|
831
|
+
}
|
|
832
|
+
const snippet = makeSnippet(row.chunk_text, tokens)
|
|
833
|
+
hit.snippet = snippet.snippet
|
|
834
|
+
hit.highlights = snippet.highlights
|
|
835
|
+
if (row.client_id !== null && row.client_id !== undefined) hit.clientId = String(row.client_id)
|
|
836
|
+
if (row.project_id !== null && row.project_id !== undefined) hit.projectId = String(row.project_id)
|
|
837
|
+
if (row.matter_id !== null && row.matter_id !== undefined) hit.matterId = String(row.matter_id)
|
|
838
|
+
if (score !== undefined && score !== null && Number.isFinite(Number(score))) hit.score = Number(score)
|
|
839
|
+
return hit
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
function searchDocuments(store, tokens, state, { limit, clearance, filters }) {
|
|
843
|
+
const patterns = tokens.map(likePattern)
|
|
844
|
+
const titleAnd = tokens.map(() => 'd.title ' + LIKE_CLAUSE).join(' AND ')
|
|
845
|
+
const chunkAnd = tokens.map(() => 'c.text ' + LIKE_CLAUSE).join(' AND ')
|
|
846
|
+
const visible = scopedDocumentWhere(filters, clearance)
|
|
847
|
+
const probe = scopedDocumentWhere(filters, clearance, { withClearance: false })
|
|
848
|
+
|
|
849
|
+
const titleTotal = countRows(store, 'SELECT COUNT(*) AS n FROM kb_documents d WHERE ' + visible.sql + ' AND (' + titleAnd + ')', [...visible.params, ...patterns])
|
|
850
|
+
const titleProbe = countRows(store, 'SELECT COUNT(*) AS n FROM kb_documents d WHERE ' + probe.sql + ' AND (' + titleAnd + ')', [...probe.params, ...patterns])
|
|
851
|
+
const titleRows = titleTotal > 0
|
|
852
|
+
? store.db.prepare('SELECT d.* FROM kb_documents d WHERE ' + visible.sql + ' AND (' + titleAnd + ') ORDER BY d.created_at DESC, d.id ASC LIMIT ?').all(...visible.params, ...patterns, limit)
|
|
853
|
+
: []
|
|
854
|
+
const titleHits = titleRows.map((row) => ({
|
|
855
|
+
kind: 'kb_doc',
|
|
856
|
+
ref: { kind: 'kb_doc', id: String(row.id) },
|
|
857
|
+
docId: String(row.id),
|
|
858
|
+
title: sanitizeLine(row.title, 200),
|
|
859
|
+
snippet: '',
|
|
860
|
+
highlights: [],
|
|
861
|
+
matchPath: 'title',
|
|
862
|
+
at: String(row.created_at ?? ''),
|
|
863
|
+
...(row.client_id === null || row.client_id === undefined ? {} : { clientId: String(row.client_id) }),
|
|
864
|
+
...(row.project_id === null || row.project_id === undefined ? {} : { projectId: String(row.project_id) }),
|
|
865
|
+
...(row.matter_id === null || row.matter_id === undefined ? {} : { matterId: String(row.matter_id) }),
|
|
866
|
+
}))
|
|
867
|
+
|
|
868
|
+
let chunkTotal = 0
|
|
869
|
+
let chunkProbe = 0
|
|
870
|
+
let chunkHits = []
|
|
871
|
+
if (state.plan.mode === 'match') {
|
|
872
|
+
try {
|
|
873
|
+
const matchParams = [state.plan.ftsQuery]
|
|
874
|
+
chunkTotal = countRows(store, 'SELECT COUNT(*) AS n FROM kb_fts JOIN kb_chunks c ON c.rowid = kb_fts.rowid JOIN kb_documents d ON d.id = c.doc_id WHERE kb_fts MATCH ? AND ' + visible.sql, [...matchParams, ...visible.params])
|
|
875
|
+
chunkProbe = countRows(store, 'SELECT COUNT(*) AS n FROM kb_fts JOIN kb_chunks c ON c.rowid = kb_fts.rowid JOIN kb_documents d ON d.id = c.doc_id WHERE kb_fts MATCH ? AND ' + probe.sql, [...matchParams, ...probe.params])
|
|
876
|
+
chunkHits = store.db.prepare('SELECT c.id AS chunk_id, c.text AS chunk_text, d.id AS doc_id, d.title AS doc_title, d.client_id, d.project_id, d.matter_id, d.created_at, rank AS fts_rank FROM kb_fts JOIN kb_chunks c ON c.rowid = kb_fts.rowid JOIN kb_documents d ON d.id = c.doc_id WHERE kb_fts MATCH ? AND ' + visible.sql + ' ORDER BY rank, c.id LIMIT ?')
|
|
877
|
+
.all(...matchParams, ...visible.params, limit)
|
|
878
|
+
.map((row) => documentHitFromRow(row, tokens, 'match', Number(row.fts_rank) === 0 ? 0 : Number(row.fts_rank)))
|
|
879
|
+
} catch (error) {
|
|
880
|
+
state.degraded.add('fts_query_fallback')
|
|
881
|
+
state.plan = { mode: 'like', degraded: [] }
|
|
882
|
+
chunkTotal = 0
|
|
883
|
+
chunkProbe = 0
|
|
884
|
+
chunkHits = []
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
if (state.plan.mode === 'like') {
|
|
888
|
+
chunkTotal = countRows(store, 'SELECT COUNT(*) AS n FROM kb_chunks c JOIN kb_documents d ON d.id = c.doc_id WHERE ' + visible.sql + ' AND (' + chunkAnd + ')', [...visible.params, ...patterns])
|
|
889
|
+
chunkProbe = countRows(store, 'SELECT COUNT(*) AS n FROM kb_chunks c JOIN kb_documents d ON d.id = c.doc_id WHERE ' + probe.sql + ' AND (' + chunkAnd + ')', [...probe.params, ...patterns])
|
|
890
|
+
chunkHits = store.db.prepare('SELECT c.id AS chunk_id, c.text AS chunk_text, d.id AS doc_id, d.title AS doc_title, d.client_id, d.project_id, d.matter_id, d.created_at FROM kb_chunks c JOIN kb_documents d ON d.id = c.doc_id WHERE ' + visible.sql + ' AND (' + chunkAnd + ') ORDER BY c.rowid DESC LIMIT ?')
|
|
891
|
+
.all(...visible.params, ...patterns, limit)
|
|
892
|
+
.map((row) => documentHitFromRow(row, tokens, 'like'))
|
|
893
|
+
}
|
|
894
|
+
return {
|
|
895
|
+
titleHits,
|
|
896
|
+
chunkHits,
|
|
897
|
+
titleTotal,
|
|
898
|
+
chunkTotal,
|
|
899
|
+
protectedExcluded: Math.max(0, titleProbe - titleTotal) + Math.max(0, chunkProbe - chunkTotal),
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
function searchPages(store, tokens, state, { limit, clearance, filters }) {
|
|
904
|
+
const patterns = tokens.map(likePattern)
|
|
905
|
+
const titleAnd = tokens.map(() => 'p.title ' + LIKE_CLAUSE).join(' AND ')
|
|
906
|
+
const bodyAnd = tokens.map(() => 'p.markdown ' + LIKE_CLAUSE).join(' AND ')
|
|
907
|
+
const base = "p.status = 'generated' AND p.deleted_at IS NULL"
|
|
908
|
+
const scanLimit = Math.min(KB_PAGE_SCAN_MAX, Math.max(200, limit * 10))
|
|
909
|
+
let rows = []
|
|
910
|
+
if (state.plan.mode === 'match') {
|
|
911
|
+
try {
|
|
912
|
+
rows = store.db.prepare('SELECT p.* FROM kb_pages_fts JOIN kb_pages p ON p.rowid = kb_pages_fts.rowid WHERE kb_pages_fts MATCH ? AND ' + base + ' LIMIT ?').all(state.plan.ftsQuery, scanLimit)
|
|
913
|
+
} catch (error) {
|
|
914
|
+
state.degraded.add('fts_query_fallback')
|
|
915
|
+
state.plan = { mode: 'like', degraded: [] }
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
if (state.plan.mode === 'like') {
|
|
919
|
+
rows = store.db.prepare('SELECT p.* FROM kb_pages p WHERE ' + base + ' AND ((' + titleAnd + ') OR (' + bodyAnd + ')) ORDER BY p.generated_at DESC, p.id ASC LIMIT ?')
|
|
920
|
+
.all(...patterns, ...patterns, scanLimit)
|
|
921
|
+
}
|
|
922
|
+
if (rows.length >= scanLimit) state.degraded.add('page_scan_capped')
|
|
923
|
+
const maps = loadScopeMaps(store)
|
|
924
|
+
const hits = []
|
|
925
|
+
let protectedExcluded = 0
|
|
926
|
+
for (const row of rows) {
|
|
927
|
+
const owner = pageOwner(row, maps)
|
|
928
|
+
const level = ownerSecrecy(maps, owner)
|
|
929
|
+
if (!Number.isFinite(level) || level > clearance) { protectedExcluded += 1; continue }
|
|
930
|
+
if (filters.clientId !== undefined && filters.clientId !== null && (!owner || owner.clientId !== String(filters.clientId))) continue
|
|
931
|
+
if (filters.matterId !== undefined && filters.matterId !== null && (!owner || owner.matterId !== String(filters.matterId))) continue
|
|
932
|
+
const snippet = makeSnippet(row.markdown, tokens)
|
|
933
|
+
hits.push({
|
|
934
|
+
kind: 'kb_page',
|
|
935
|
+
ref: { kind: 'kb_page', id: String(row.id) },
|
|
936
|
+
pageId: String(row.id),
|
|
937
|
+
title: sanitizeLine(row.title, 200),
|
|
938
|
+
snippet: snippet.snippet,
|
|
939
|
+
highlights: snippet.highlights,
|
|
940
|
+
matchPath: 'page',
|
|
941
|
+
generatedAt: String(row.generated_at ?? ''),
|
|
942
|
+
...(owner && owner.clientId ? { clientId: owner.clientId } : {}),
|
|
943
|
+
...(owner && owner.matterId ? { matterId: owner.matterId } : {}),
|
|
944
|
+
})
|
|
945
|
+
}
|
|
946
|
+
return { hits: hits.slice(0, limit), total: hits.length, protectedExcluded }
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* Unified KB search (scope kb | pages | all).
|
|
951
|
+
*
|
|
952
|
+
* Shape matches 45 section 3.5. counts.turns/facts/preferences stay 0 here:
|
|
953
|
+
* the CLI search command owns those engines and merges the counts.
|
|
954
|
+
*/
|
|
955
|
+
export function kbSearch(store, query, options = {}) {
|
|
956
|
+
assertKbSchema(store)
|
|
957
|
+
const parsed = normalizeKbQuery(query)
|
|
958
|
+
const scope = options.scope === undefined ? 'kb' : String(options.scope)
|
|
959
|
+
if (!['kb', 'pages', 'all'].includes(scope)) throw entityError('E_KB_INVALID', 'unknown search scope: ' + scope, 'use kb, pages or all')
|
|
960
|
+
const limit = clampLimit(options.limit, KB_SEARCH_LIMIT, KB_SEARCH_LIMIT_MAX)
|
|
961
|
+
const clearance = normalizeClearance(options.clearance === undefined ? KB_DEFAULT_CLEARANCE : options.clearance)
|
|
962
|
+
const filters = { clientId: options.clientId, projectId: options.projectId, matterId: options.matterId, kind: options.kind }
|
|
963
|
+
return store.read(() => {
|
|
964
|
+
const initial = selectKbSearchPlan(store, parsed.tokens)
|
|
965
|
+
const state = { plan: initial, degraded: new Set(initial.degraded) }
|
|
966
|
+
const counts = { kbDocs: 0, kbChunks: 0, kbPages: 0, turns: 0, facts: 0, preferences: 0 }
|
|
967
|
+
let protectedExcluded = 0
|
|
968
|
+
let titleHits = []
|
|
969
|
+
let chunkHits = []
|
|
970
|
+
let pageHits = []
|
|
971
|
+
if (scope === 'kb' || scope === 'all') {
|
|
972
|
+
const documents = searchDocuments(store, parsed.tokens, state, { limit, clearance, filters })
|
|
973
|
+
titleHits = documents.titleHits
|
|
974
|
+
chunkHits = documents.chunkHits
|
|
975
|
+
counts.kbDocs = documents.titleTotal
|
|
976
|
+
counts.kbChunks = documents.chunkTotal
|
|
977
|
+
protectedExcluded += documents.protectedExcluded
|
|
978
|
+
}
|
|
979
|
+
if (scope === 'pages' || scope === 'all') {
|
|
980
|
+
const pages = searchPages(store, parsed.tokens, state, { limit, clearance, filters })
|
|
981
|
+
pageHits = pages.hits
|
|
982
|
+
counts.kbPages = pages.total
|
|
983
|
+
protectedExcluded += pages.protectedExcluded
|
|
984
|
+
}
|
|
985
|
+
const hits = [...titleHits, ...chunkHits, ...pageHits].slice(0, limit)
|
|
986
|
+
const total = counts.kbDocs + counts.kbChunks + counts.kbPages
|
|
987
|
+
return {
|
|
988
|
+
ok: true,
|
|
989
|
+
command: 'search',
|
|
990
|
+
query: parsed.query,
|
|
991
|
+
scope,
|
|
992
|
+
limit,
|
|
993
|
+
clearance,
|
|
994
|
+
degraded: [...state.degraded].sort(),
|
|
995
|
+
counts,
|
|
996
|
+
hits,
|
|
997
|
+
total,
|
|
998
|
+
truncated: total > hits.length,
|
|
999
|
+
protectedExcluded,
|
|
1000
|
+
}
|
|
1001
|
+
})
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
export default Object.freeze({
|
|
1005
|
+
KB_KINDS,
|
|
1006
|
+
KB_RELATIONS,
|
|
1007
|
+
KB_DEFAULT_CLEARANCE,
|
|
1008
|
+
KB_LIST_LIMIT,
|
|
1009
|
+
KB_LIST_LIMIT_MAX,
|
|
1010
|
+
KB_SEARCH_LIMIT,
|
|
1011
|
+
KB_SEARCH_LIMIT_MAX,
|
|
1012
|
+
KB_QUERY_MAX_CHARS,
|
|
1013
|
+
KB_MAX_BATCH_FILES,
|
|
1014
|
+
KB_MAX_CHUNKS,
|
|
1015
|
+
KB_SNIPPET_CHARS,
|
|
1016
|
+
KB_CHUNK_PREVIEW_CHARS,
|
|
1017
|
+
KB_PAGE_SCAN_MAX,
|
|
1018
|
+
kbAdd,
|
|
1019
|
+
kbAddMany,
|
|
1020
|
+
kbList,
|
|
1021
|
+
kbShow,
|
|
1022
|
+
kbRm,
|
|
1023
|
+
kbPurge,
|
|
1024
|
+
kbReindex,
|
|
1025
|
+
verifyKbFts,
|
|
1026
|
+
kbLink,
|
|
1027
|
+
kbUnlink,
|
|
1028
|
+
kbSearch,
|
|
1029
|
+
normalizeKbQuery,
|
|
1030
|
+
selectKbSearchPlan,
|
|
1031
|
+
makeSnippet,
|
|
1032
|
+
kbFtsTokenizer,
|
|
1033
|
+
assertKbSchema,
|
|
1034
|
+
mapKbDocumentRow,
|
|
1035
|
+
})
|
|
1036
|
+
|