@jacebenson/jsn 4.3.10-dev.67 → 4.3.11-dev.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/jsn.js +7 -1
- package/package.json +2 -2
- package/src/app.js +10 -23
- package/src/auth/browser-session.js +21 -0
- package/src/auth/credential-state.js +124 -0
- package/src/auth/credential-store.js +177 -0
- package/src/auth/oauth.js +66 -0
- package/src/auth.js +11 -378
- package/src/cli.js +3 -1
- package/src/commands/docs/db.js +22 -2
- package/src/commands/docs/docs.js +13 -13
- package/src/commands/docs/embed.js +33 -33
- package/src/commands/docs/ingest.js +58 -58
- package/src/commands/docs/refresh.js +91 -91
- package/src/commands/docs/search.js +108 -108
- package/src/commands/docs/serve.js +4 -1
- package/src/commands/setup.js +27 -25
- package/src/commands/version.js +16 -2
- package/src/helpers.js +1 -1
- package/src/paginated-search.js +162 -161
- package/src/prompts.js +15 -9
- package/src/sdk.js +79 -92
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import fs from 'node:fs';
|
|
4
4
|
import { createHash } from 'node:crypto';
|
|
5
5
|
import { parseFrontmatter } from '../../frontmatter.js';
|
|
6
|
-
import {
|
|
6
|
+
import { withDocsDb, initDocsSchema, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
|
|
7
7
|
import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
|
|
8
8
|
|
|
9
9
|
export function ingestDocs(opts = {}) {
|
|
@@ -14,69 +14,69 @@ export function ingestDocs(opts = {}) {
|
|
|
14
14
|
// Fresh build each run.
|
|
15
15
|
if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
return withDocsDb({ dbPath }, (db) => {
|
|
18
|
+
initDocsSchema(db);
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
const insert = db.prepare(`
|
|
21
|
+
INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
|
|
22
|
+
VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
|
|
23
|
+
`);
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
let count = 0;
|
|
26
|
+
let errors = 0;
|
|
27
|
+
const startedAt = Date.now();
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
try {
|
|
32
|
-
const stat = fs.statSync(file);
|
|
33
|
-
const raw = fs.readFileSync(file, 'utf8');
|
|
34
|
-
const hash = createHash('sha256').update(raw).digest('hex');
|
|
35
|
-
let data = {};
|
|
36
|
-
let body = raw;
|
|
29
|
+
db.exec('BEGIN');
|
|
30
|
+
for (const { file, rel } of walkDocs(roots)) {
|
|
37
31
|
try {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
32
|
+
const stat = fs.statSync(file);
|
|
33
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
34
|
+
const hash = createHash('sha256').update(raw).digest('hex');
|
|
35
|
+
let data = {};
|
|
36
|
+
let body = raw;
|
|
37
|
+
try {
|
|
38
|
+
const parsed = parseFrontmatter(raw);
|
|
39
|
+
data = parsed.data || {};
|
|
40
|
+
body = parsed.content || '';
|
|
41
|
+
} catch {
|
|
42
|
+
// Malformed frontmatter: keep raw body, empty metadata.
|
|
43
|
+
}
|
|
44
|
+
const hrr = embed
|
|
45
|
+
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
46
|
+
: null;
|
|
47
|
+
insert.run({
|
|
48
|
+
path: rel,
|
|
49
|
+
title: data.title ?? null,
|
|
50
|
+
release: data.release ?? null,
|
|
51
|
+
bundle: data.bundle ?? null,
|
|
52
|
+
doc_type: data.doc_type ?? null,
|
|
53
|
+
locale: data.locale ?? null,
|
|
54
|
+
frontmatter: JSON.stringify(data),
|
|
55
|
+
body,
|
|
56
|
+
size: stat.size,
|
|
57
|
+
mtime: Math.floor(stat.mtimeMs),
|
|
58
|
+
hash,
|
|
59
|
+
hrr_vector: hrr,
|
|
60
|
+
});
|
|
61
|
+
count++;
|
|
62
|
+
if (count % 5000 === 0) {
|
|
63
|
+
process.stderr.write(` ...${count} files\n`);
|
|
64
|
+
}
|
|
65
|
+
} catch (err) {
|
|
66
|
+
errors++;
|
|
67
|
+
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
43
68
|
}
|
|
44
|
-
const hrr = embed
|
|
45
|
-
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
46
|
-
: null;
|
|
47
|
-
insert.run({
|
|
48
|
-
path: rel,
|
|
49
|
-
title: data.title ?? null,
|
|
50
|
-
release: data.release ?? null,
|
|
51
|
-
bundle: data.bundle ?? null,
|
|
52
|
-
doc_type: data.doc_type ?? null,
|
|
53
|
-
locale: data.locale ?? null,
|
|
54
|
-
frontmatter: JSON.stringify(data),
|
|
55
|
-
body,
|
|
56
|
-
size: stat.size,
|
|
57
|
-
mtime: Math.floor(stat.mtimeMs),
|
|
58
|
-
hash,
|
|
59
|
-
hrr_vector: hrr,
|
|
60
|
-
});
|
|
61
|
-
count++;
|
|
62
|
-
if (count % 5000 === 0) {
|
|
63
|
-
process.stderr.write(` ...${count} files\n`);
|
|
64
|
-
}
|
|
65
|
-
} catch (err) {
|
|
66
|
-
errors++;
|
|
67
|
-
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
68
69
|
}
|
|
69
|
-
|
|
70
|
-
db.exec('COMMIT');
|
|
70
|
+
db.exec('COMMIT');
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
closeDocsDb(db);
|
|
72
|
+
db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
|
|
73
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
74
|
+
db.exec("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)");
|
|
75
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('hrr_dim', ?)").run(String(DEFAULT_DIM));
|
|
76
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
|
|
77
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('embeddings', ?)").run(String(embed));
|
|
79
78
|
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
const secs = ((Date.now() - startedAt) / 1000).toFixed(1);
|
|
80
|
+
return { count, errors, secs, dbPath, embedded: embed };
|
|
81
|
+
});
|
|
82
82
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import fs from 'node:fs';
|
|
5
5
|
import { createHash } from 'node:crypto';
|
|
6
6
|
import { parseFrontmatter } from '../../frontmatter.js';
|
|
7
|
-
import {
|
|
7
|
+
import { withDocsDb, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
|
|
8
8
|
import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
|
|
9
9
|
|
|
10
10
|
export function refreshDocs(opts = {}) {
|
|
@@ -15,108 +15,108 @@ export function refreshDocs(opts = {}) {
|
|
|
15
15
|
throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first for the initial build.`);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
return withDocsDb({ dbPath }, (db) => {
|
|
19
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
const cols = db.prepare('PRAGMA table_info(docs)').all().map((c) => c.name);
|
|
22
|
+
if (!cols.includes('hash')) db.exec('ALTER TABLE docs ADD COLUMN hash TEXT');
|
|
23
|
+
if (!cols.includes('hrr_vector')) db.exec('ALTER TABLE docs ADD COLUMN hrr_vector BLOB');
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
const embeddingsEnabled =
|
|
26
|
+
db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hrr_vector IS NOT NULL').get().n > 0 ||
|
|
27
|
+
!cols.includes('hrr_vector');
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
const isBackfill = db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hash IS NOT NULL').get().n === 0
|
|
30
|
+
&& db.prepare('SELECT COUNT(*) AS n FROM docs').get().n > 0;
|
|
31
|
+
const setHash = db.prepare('UPDATE docs SET hash = ? WHERE id = ?');
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
const existing = new Map();
|
|
34
|
+
for (const row of db.prepare('SELECT id, path, hash FROM docs').all()) {
|
|
35
|
+
existing.set(row.path, { id: row.id, hash: row.hash });
|
|
36
|
+
}
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
let added = 0, updated = 0, removed = 0, unchanged = 0, errors = 0;
|
|
48
|
-
const seen = new Set();
|
|
49
|
-
const started = Date.now();
|
|
50
|
-
|
|
51
|
-
const files = [...walkDocs(roots)];
|
|
52
|
-
|
|
53
|
-
db.exec('BEGIN');
|
|
54
|
-
for (const { file, rel } of files) {
|
|
55
|
-
try {
|
|
56
|
-
seen.add(rel);
|
|
57
|
-
const raw = fs.readFileSync(file, 'utf8');
|
|
58
|
-
const hash = createHash('sha256').update(raw).digest('hex');
|
|
59
|
-
|
|
60
|
-
const prev = existing.get(rel);
|
|
61
|
-
if (prev && prev.hash === hash) {
|
|
62
|
-
unchanged++;
|
|
63
|
-
continue;
|
|
64
|
-
}
|
|
38
|
+
const upsert = db.prepare(`
|
|
39
|
+
INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
|
|
40
|
+
VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
|
|
41
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
42
|
+
title=@title, release=@release, bundle=@bundle, doc_type=@doc_type, locale=@locale,
|
|
43
|
+
frontmatter=@frontmatter, body=@body, size=@size, mtime=@mtime, hash=@hash, hrr_vector=@hrr_vector
|
|
44
|
+
`);
|
|
45
|
+
const del = db.prepare('DELETE FROM docs WHERE id = ?');
|
|
65
46
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
47
|
+
let added = 0, updated = 0, removed = 0, unchanged = 0, errors = 0;
|
|
48
|
+
const seen = new Set();
|
|
49
|
+
const started = Date.now();
|
|
50
|
+
|
|
51
|
+
const files = [...walkDocs(roots)];
|
|
71
52
|
|
|
72
|
-
|
|
73
|
-
|
|
53
|
+
db.exec('BEGIN');
|
|
54
|
+
for (const { file, rel } of files) {
|
|
74
55
|
try {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
56
|
+
seen.add(rel);
|
|
57
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
58
|
+
const hash = createHash('sha256').update(raw).digest('hex');
|
|
59
|
+
|
|
60
|
+
const prev = existing.get(rel);
|
|
61
|
+
if (prev && prev.hash === hash) {
|
|
62
|
+
unchanged++;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (isBackfill && prev) {
|
|
67
|
+
setHash.run(hash, prev.id);
|
|
68
|
+
unchanged++;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const stat = fs.statSync(file);
|
|
73
|
+
let data = {}, body = raw;
|
|
74
|
+
try {
|
|
75
|
+
const parsed = parseFrontmatter(raw);
|
|
76
|
+
data = parsed.data || {};
|
|
77
|
+
body = parsed.content || '';
|
|
78
|
+
} catch {
|
|
79
|
+
// keep raw body on malformed frontmatter
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const hrr = embeddingsEnabled
|
|
83
|
+
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
84
|
+
: null;
|
|
85
|
+
|
|
86
|
+
upsert.run({
|
|
87
|
+
path: rel,
|
|
88
|
+
title: data.title ?? null,
|
|
89
|
+
release: data.release ?? null,
|
|
90
|
+
bundle: data.bundle ?? null,
|
|
91
|
+
doc_type: data.doc_type ?? null,
|
|
92
|
+
locale: data.locale ?? null,
|
|
93
|
+
frontmatter: JSON.stringify(data),
|
|
94
|
+
body,
|
|
95
|
+
size: stat.size,
|
|
96
|
+
mtime: Math.floor(stat.mtimeMs),
|
|
97
|
+
hash,
|
|
98
|
+
hrr_vector: hrr,
|
|
99
|
+
});
|
|
100
|
+
if (prev) updated++; else added++;
|
|
101
|
+
} catch (err) {
|
|
102
|
+
errors++;
|
|
103
|
+
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
80
104
|
}
|
|
81
|
-
|
|
82
|
-
const hrr = embeddingsEnabled
|
|
83
|
-
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
84
|
-
: null;
|
|
85
|
-
|
|
86
|
-
upsert.run({
|
|
87
|
-
path: rel,
|
|
88
|
-
title: data.title ?? null,
|
|
89
|
-
release: data.release ?? null,
|
|
90
|
-
bundle: data.bundle ?? null,
|
|
91
|
-
doc_type: data.doc_type ?? null,
|
|
92
|
-
locale: data.locale ?? null,
|
|
93
|
-
frontmatter: JSON.stringify(data),
|
|
94
|
-
body,
|
|
95
|
-
size: stat.size,
|
|
96
|
-
mtime: Math.floor(stat.mtimeMs),
|
|
97
|
-
hash,
|
|
98
|
-
hrr_vector: hrr,
|
|
99
|
-
});
|
|
100
|
-
if (prev) updated++; else added++;
|
|
101
|
-
} catch (err) {
|
|
102
|
-
errors++;
|
|
103
|
-
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
104
105
|
}
|
|
105
|
-
}
|
|
106
106
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
107
|
+
for (const [rel, row] of existing) {
|
|
108
|
+
if (!seen.has(rel)) {
|
|
109
|
+
del.run(row.id);
|
|
110
|
+
removed++;
|
|
111
|
+
}
|
|
111
112
|
}
|
|
112
|
-
|
|
113
|
-
db.exec('COMMIT');
|
|
113
|
+
db.exec('COMMIT');
|
|
114
114
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
closeDocsDb(db);
|
|
115
|
+
db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
|
|
116
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
117
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
|
|
119
118
|
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
const secs = ((Date.now() - started) / 1000).toFixed(1);
|
|
120
|
+
return { added, updated, removed, unchanged, errors, secs, dbPath, embeddingsEnabled };
|
|
121
|
+
});
|
|
122
122
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// CLI search against the docs SQLite database.
|
|
2
2
|
|
|
3
3
|
import fs from 'node:fs';
|
|
4
|
-
import { getDocsDbPath, hasEmbeddings, openDocsDb, closeDocsDb, getMeta } from './db.js';
|
|
4
|
+
import { getDocsDbPath, hasEmbeddings, openDocsDb, closeDocsDb, withDocsDb, getMeta } from './db.js';
|
|
5
5
|
import { encodeText, similarity, bytesToPhases, DEFAULT_DIM } from './hrr.js';
|
|
6
6
|
|
|
7
7
|
function safeParse(s) {
|
|
@@ -50,119 +50,119 @@ export function searchDocs(opts = {}) {
|
|
|
50
50
|
throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first.`);
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
} else {
|
|
94
|
-
const candParams = { q, cand: (limit + offset) * 4 + 20 };
|
|
95
|
-
if (bundle) candParams.bundle = bundle;
|
|
96
|
-
if (docType) candParams.docType = docType;
|
|
97
|
-
const candidates = db
|
|
98
|
-
.prepare(
|
|
99
|
-
`SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release, d.hrr_vector,
|
|
100
|
-
json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
|
|
101
|
-
snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
|
|
102
|
-
bm25(docs_fts) AS bm25
|
|
103
|
-
FROM docs_fts
|
|
104
|
-
JOIN docs d ON d.id = docs_fts.rowid
|
|
105
|
-
WHERE docs_fts MATCH @q${where}
|
|
106
|
-
ORDER BY bm25
|
|
107
|
-
LIMIT @cand`
|
|
108
|
-
)
|
|
109
|
-
.all(candParams);
|
|
110
|
-
|
|
111
|
-
if (candidates.length === 0) {
|
|
112
|
-
rows = [];
|
|
53
|
+
return withDocsDb({ dbPath }, (db) => {
|
|
54
|
+
const limit = Math.min(parseInt(opts.limit, 10) || 20, 1000);
|
|
55
|
+
const offset = parseInt(opts.offset, 10) || 0;
|
|
56
|
+
const bundle = opts.bundle ? String(opts.bundle) : null;
|
|
57
|
+
const docType = opts.docType ? String(opts.docType) : null;
|
|
58
|
+
const mode = opts.mode === 'keyword' ? 'keyword' : 'hybrid';
|
|
59
|
+
const useHybrid = mode === 'hybrid' && hasEmbeddings(db);
|
|
60
|
+
|
|
61
|
+
const filters = [];
|
|
62
|
+
const params = { q, limit, offset };
|
|
63
|
+
if (bundle) {
|
|
64
|
+
filters.push('d.bundle = @bundle');
|
|
65
|
+
params.bundle = bundle;
|
|
66
|
+
}
|
|
67
|
+
if (docType) {
|
|
68
|
+
filters.push('d.doc_type = @docType');
|
|
69
|
+
params.docType = docType;
|
|
70
|
+
}
|
|
71
|
+
const where = filters.length ? ' AND ' + filters.join(' AND ') : '';
|
|
72
|
+
|
|
73
|
+
// Total matching docs (for pagination)
|
|
74
|
+
const total = db
|
|
75
|
+
.prepare(`SELECT COUNT(*) AS n FROM docs_fts JOIN docs d ON d.id = docs_fts.rowid WHERE docs_fts MATCH @q${where}`)
|
|
76
|
+
.get({ q, ...(bundle && { bundle }), ...(docType && { docType }) }).n;
|
|
77
|
+
|
|
78
|
+
let rows;
|
|
79
|
+
if (!useHybrid) {
|
|
80
|
+
rows = db
|
|
81
|
+
.prepare(
|
|
82
|
+
`SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release,
|
|
83
|
+
json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
|
|
84
|
+
snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
|
|
85
|
+
bm25(docs_fts) AS score
|
|
86
|
+
FROM docs_fts
|
|
87
|
+
JOIN docs d ON d.id = docs_fts.rowid
|
|
88
|
+
WHERE docs_fts MATCH @q${where}
|
|
89
|
+
ORDER BY score
|
|
90
|
+
LIMIT @limit OFFSET @offset`
|
|
91
|
+
)
|
|
92
|
+
.all(params);
|
|
113
93
|
} else {
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
94
|
+
const candParams = { q, cand: (limit + offset) * 4 + 20 };
|
|
95
|
+
if (bundle) candParams.bundle = bundle;
|
|
96
|
+
if (docType) candParams.docType = docType;
|
|
97
|
+
const candidates = db
|
|
98
|
+
.prepare(
|
|
99
|
+
`SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release, d.hrr_vector,
|
|
100
|
+
json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
|
|
101
|
+
snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
|
|
102
|
+
bm25(docs_fts) AS bm25
|
|
103
|
+
FROM docs_fts
|
|
104
|
+
JOIN docs d ON d.id = docs_fts.rowid
|
|
105
|
+
WHERE docs_fts MATCH @q${where}
|
|
106
|
+
ORDER BY bm25
|
|
107
|
+
LIMIT @cand`
|
|
108
|
+
)
|
|
109
|
+
.all(candParams);
|
|
110
|
+
|
|
111
|
+
if (candidates.length === 0) {
|
|
112
|
+
rows = [];
|
|
113
|
+
} else {
|
|
114
|
+
const hrrDim = (() => {
|
|
115
|
+
const value = getMeta(db, 'hrr_dim');
|
|
116
|
+
const n = parseInt(value, 10);
|
|
117
|
+
return Number.isInteger(n) ? n : DEFAULT_DIM;
|
|
118
|
+
})();
|
|
119
|
+
const queryVec = encodeText(q, hrrDim);
|
|
120
|
+
const queryTokens = new Set(q.toLowerCase().split(/\s+/).filter(Boolean));
|
|
121
|
+
|
|
122
|
+
const bmVals = candidates.map((c) => c.bm25);
|
|
123
|
+
const bmMin = Math.min(...bmVals);
|
|
124
|
+
const bmMax = Math.max(...bmVals);
|
|
125
|
+
const bmRange = bmMax - bmMin || 1;
|
|
126
|
+
|
|
127
|
+
const FTS_W = 0.4, JACCARD_W = 0.3, HRR_W = 0.3;
|
|
128
|
+
|
|
129
|
+
for (const c of candidates) {
|
|
130
|
+
const ftsScore = 1 - (c.bm25 - bmMin) / bmRange;
|
|
131
|
+
const contentTokens = new Set(
|
|
132
|
+
`${c.title || ''} ${c.path || ''}`.toLowerCase().split(/\s+/).filter(Boolean)
|
|
133
|
+
);
|
|
134
|
+
let inter = 0;
|
|
135
|
+
for (const t of queryTokens) if (contentTokens.has(t)) inter++;
|
|
136
|
+
const union = queryTokens.size + contentTokens.size - inter;
|
|
137
|
+
const jaccard = union ? inter / union : 0;
|
|
138
|
+
|
|
139
|
+
let hrrSim = 0.5;
|
|
140
|
+
if (c.hrr_vector) {
|
|
141
|
+
hrrSim = (similarity(queryVec, bytesToPhases(c.hrr_vector)) + 1) / 2;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
c.score = FTS_W * ftsScore + JACCARD_W * jaccard + HRR_W * hrrSim;
|
|
145
|
+
delete c.hrr_vector;
|
|
146
|
+
delete c.bm25;
|
|
142
147
|
}
|
|
143
148
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
delete c.bm25;
|
|
149
|
+
candidates.sort((a, b) => b.score - a.score);
|
|
150
|
+
rows = candidates.slice(offset, offset + limit);
|
|
147
151
|
}
|
|
148
|
-
|
|
149
|
-
candidates.sort((a, b) => b.score - a.score);
|
|
150
|
-
rows = candidates.slice(offset, offset + limit);
|
|
151
152
|
}
|
|
152
|
-
}
|
|
153
153
|
|
|
154
|
-
closeDocsDb(db);
|
|
155
154
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
155
|
+
// Post-process canonical_url JSON strings that may contain escaped content.
|
|
156
|
+
for (const row of rows) {
|
|
157
|
+
if (row.canonical_url) row.canonical_url = safeParse(row.canonical_url) || row.canonical_url;
|
|
158
|
+
}
|
|
160
159
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
160
|
+
return {
|
|
161
|
+
query: q,
|
|
162
|
+
mode: useHybrid ? 'hybrid' : 'keyword',
|
|
163
|
+
total,
|
|
164
|
+
count: rows.length,
|
|
165
|
+
results: rows,
|
|
166
|
+
};
|
|
167
|
+
});
|
|
168
168
|
}
|
|
@@ -5,7 +5,7 @@ import path from 'node:path';
|
|
|
5
5
|
import http from 'node:http';
|
|
6
6
|
import { URL } from 'node:url';
|
|
7
7
|
import Database from 'better-sqlite3';
|
|
8
|
-
import { getDocsDbPath, hasEmbeddings } from './db.js';
|
|
8
|
+
import { getDocsDbPath, hasEmbeddings, closeDocsDb } from './db.js';
|
|
9
9
|
import { encodeText, similarity, bytesToPhases, DEFAULT_DIM } from './hrr.js';
|
|
10
10
|
|
|
11
11
|
const PUBLIC_DIR = path.join(import.meta.dirname, 'public');
|
|
@@ -236,6 +236,8 @@ export function serveDocs(opts = {}) {
|
|
|
236
236
|
}
|
|
237
237
|
});
|
|
238
238
|
|
|
239
|
+
server.once('close', () => closeDocsDb(db));
|
|
240
|
+
|
|
239
241
|
return new Promise((resolve, reject) => {
|
|
240
242
|
let currentPort = parseInt(port, 10);
|
|
241
243
|
const maxAttempts = 10;
|
|
@@ -263,6 +265,7 @@ export function serveDocs(opts = {}) {
|
|
|
263
265
|
err.message = `Could not find an open port between ${port} and ${currentPort}. Try specifying one with --port.`;
|
|
264
266
|
}
|
|
265
267
|
resolved = true;
|
|
268
|
+
closeDocsDb(db);
|
|
266
269
|
reject(err);
|
|
267
270
|
}
|
|
268
271
|
|