@jacebenson/jsn 4.3.10-dev.67 → 4.3.11-dev.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@
3
3
  import fs from 'node:fs';
4
4
  import { createHash } from 'node:crypto';
5
5
  import { parseFrontmatter } from '../../frontmatter.js';
6
- import { openDocsDb, closeDocsDb, initDocsSchema, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
6
+ import { withDocsDb, initDocsSchema, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
7
7
  import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
8
8
 
9
9
  export function ingestDocs(opts = {}) {
@@ -14,69 +14,69 @@ export function ingestDocs(opts = {}) {
14
14
  // Fresh build each run.
15
15
  if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
16
16
 
17
- const db = openDocsDb({ dbPath });
18
- initDocsSchema(db);
17
+ return withDocsDb({ dbPath }, (db) => {
18
+ initDocsSchema(db);
19
19
 
20
- const insert = db.prepare(`
21
- INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
22
- VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
23
- `);
20
+ const insert = db.prepare(`
21
+ INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
22
+ VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
23
+ `);
24
24
 
25
- let count = 0;
26
- let errors = 0;
27
- const startedAt = Date.now();
25
+ let count = 0;
26
+ let errors = 0;
27
+ const startedAt = Date.now();
28
28
 
29
- db.exec('BEGIN');
30
- for (const { file, rel } of walkDocs(roots)) {
31
- try {
32
- const stat = fs.statSync(file);
33
- const raw = fs.readFileSync(file, 'utf8');
34
- const hash = createHash('sha256').update(raw).digest('hex');
35
- let data = {};
36
- let body = raw;
29
+ db.exec('BEGIN');
30
+ for (const { file, rel } of walkDocs(roots)) {
37
31
  try {
38
- const parsed = parseFrontmatter(raw);
39
- data = parsed.data || {};
40
- body = parsed.content || '';
41
- } catch {
42
- // Malformed frontmatter: keep raw body, empty metadata.
32
+ const stat = fs.statSync(file);
33
+ const raw = fs.readFileSync(file, 'utf8');
34
+ const hash = createHash('sha256').update(raw).digest('hex');
35
+ let data = {};
36
+ let body = raw;
37
+ try {
38
+ const parsed = parseFrontmatter(raw);
39
+ data = parsed.data || {};
40
+ body = parsed.content || '';
41
+ } catch {
42
+ // Malformed frontmatter: keep raw body, empty metadata.
43
+ }
44
+ const hrr = embed
45
+ ? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
46
+ : null;
47
+ insert.run({
48
+ path: rel,
49
+ title: data.title ?? null,
50
+ release: data.release ?? null,
51
+ bundle: data.bundle ?? null,
52
+ doc_type: data.doc_type ?? null,
53
+ locale: data.locale ?? null,
54
+ frontmatter: JSON.stringify(data),
55
+ body,
56
+ size: stat.size,
57
+ mtime: Math.floor(stat.mtimeMs),
58
+ hash,
59
+ hrr_vector: hrr,
60
+ });
61
+ count++;
62
+ if (count % 5000 === 0) {
63
+ process.stderr.write(` ...${count} files\n`);
64
+ }
65
+ } catch (err) {
66
+ errors++;
67
+ if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
43
68
  }
44
- const hrr = embed
45
- ? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
46
- : null;
47
- insert.run({
48
- path: rel,
49
- title: data.title ?? null,
50
- release: data.release ?? null,
51
- bundle: data.bundle ?? null,
52
- doc_type: data.doc_type ?? null,
53
- locale: data.locale ?? null,
54
- frontmatter: JSON.stringify(data),
55
- body,
56
- size: stat.size,
57
- mtime: Math.floor(stat.mtimeMs),
58
- hash,
59
- hrr_vector: hrr,
60
- });
61
- count++;
62
- if (count % 5000 === 0) {
63
- process.stderr.write(` ...${count} files\n`);
64
- }
65
- } catch (err) {
66
- errors++;
67
- if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
68
69
  }
69
- }
70
- db.exec('COMMIT');
70
+ db.exec('COMMIT');
71
71
 
72
- db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
73
- db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
74
- db.exec("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)");
75
- db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('hrr_dim', ?)").run(String(DEFAULT_DIM));
76
- db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
77
- db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('embeddings', ?)").run(String(embed));
78
- closeDocsDb(db);
72
+ db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
73
+ db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
74
+ db.exec("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)");
75
+ db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('hrr_dim', ?)").run(String(DEFAULT_DIM));
76
+ db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
77
+ db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('embeddings', ?)").run(String(embed));
79
78
 
80
- const secs = ((Date.now() - startedAt) / 1000).toFixed(1);
81
- return { count, errors, secs, dbPath, embedded: embed };
79
+ const secs = ((Date.now() - startedAt) / 1000).toFixed(1);
80
+ return { count, errors, secs, dbPath, embedded: embed };
81
+ });
82
82
  }
@@ -4,7 +4,7 @@
4
4
  import fs from 'node:fs';
5
5
  import { createHash } from 'node:crypto';
6
6
  import { parseFrontmatter } from '../../frontmatter.js';
7
- import { openDocsDb, closeDocsDb, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
7
+ import { withDocsDb, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
8
8
  import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
9
9
 
10
10
  export function refreshDocs(opts = {}) {
@@ -15,108 +15,108 @@ export function refreshDocs(opts = {}) {
15
15
  throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first for the initial build.`);
16
16
  }
17
17
 
18
- const db = openDocsDb({ dbPath });
19
- db.exec('PRAGMA journal_mode = WAL');
18
+ return withDocsDb({ dbPath }, (db) => {
19
+ db.exec('PRAGMA journal_mode = WAL');
20
20
 
21
- const cols = db.prepare('PRAGMA table_info(docs)').all().map((c) => c.name);
22
- if (!cols.includes('hash')) db.exec('ALTER TABLE docs ADD COLUMN hash TEXT');
23
- if (!cols.includes('hrr_vector')) db.exec('ALTER TABLE docs ADD COLUMN hrr_vector BLOB');
21
+ const cols = db.prepare('PRAGMA table_info(docs)').all().map((c) => c.name);
22
+ if (!cols.includes('hash')) db.exec('ALTER TABLE docs ADD COLUMN hash TEXT');
23
+ if (!cols.includes('hrr_vector')) db.exec('ALTER TABLE docs ADD COLUMN hrr_vector BLOB');
24
24
 
25
- const embeddingsEnabled =
26
- db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hrr_vector IS NOT NULL').get().n > 0 ||
27
- !cols.includes('hrr_vector');
25
+ const embeddingsEnabled =
26
+ db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hrr_vector IS NOT NULL').get().n > 0 ||
27
+ !cols.includes('hrr_vector');
28
28
 
29
- const isBackfill = db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hash IS NOT NULL').get().n === 0
30
- && db.prepare('SELECT COUNT(*) AS n FROM docs').get().n > 0;
31
- const setHash = db.prepare('UPDATE docs SET hash = ? WHERE id = ?');
29
+ const isBackfill = db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hash IS NOT NULL').get().n === 0
30
+ && db.prepare('SELECT COUNT(*) AS n FROM docs').get().n > 0;
31
+ const setHash = db.prepare('UPDATE docs SET hash = ? WHERE id = ?');
32
32
 
33
- const existing = new Map();
34
- for (const row of db.prepare('SELECT id, path, hash FROM docs').all()) {
35
- existing.set(row.path, { id: row.id, hash: row.hash });
36
- }
33
+ const existing = new Map();
34
+ for (const row of db.prepare('SELECT id, path, hash FROM docs').all()) {
35
+ existing.set(row.path, { id: row.id, hash: row.hash });
36
+ }
37
37
 
38
- const upsert = db.prepare(`
39
- INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
40
- VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
41
- ON CONFLICT(path) DO UPDATE SET
42
- title=@title, release=@release, bundle=@bundle, doc_type=@doc_type, locale=@locale,
43
- frontmatter=@frontmatter, body=@body, size=@size, mtime=@mtime, hash=@hash, hrr_vector=@hrr_vector
44
- `);
45
- const del = db.prepare('DELETE FROM docs WHERE id = ?');
46
-
47
- let added = 0, updated = 0, removed = 0, unchanged = 0, errors = 0;
48
- const seen = new Set();
49
- const started = Date.now();
50
-
51
- const files = [...walkDocs(roots)];
52
-
53
- db.exec('BEGIN');
54
- for (const { file, rel } of files) {
55
- try {
56
- seen.add(rel);
57
- const raw = fs.readFileSync(file, 'utf8');
58
- const hash = createHash('sha256').update(raw).digest('hex');
59
-
60
- const prev = existing.get(rel);
61
- if (prev && prev.hash === hash) {
62
- unchanged++;
63
- continue;
64
- }
38
+ const upsert = db.prepare(`
39
+ INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
40
+ VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
41
+ ON CONFLICT(path) DO UPDATE SET
42
+ title=@title, release=@release, bundle=@bundle, doc_type=@doc_type, locale=@locale,
43
+ frontmatter=@frontmatter, body=@body, size=@size, mtime=@mtime, hash=@hash, hrr_vector=@hrr_vector
44
+ `);
45
+ const del = db.prepare('DELETE FROM docs WHERE id = ?');
65
46
 
66
- if (isBackfill && prev) {
67
- setHash.run(hash, prev.id);
68
- unchanged++;
69
- continue;
70
- }
47
+ let added = 0, updated = 0, removed = 0, unchanged = 0, errors = 0;
48
+ const seen = new Set();
49
+ const started = Date.now();
50
+
51
+ const files = [...walkDocs(roots)];
71
52
 
72
- const stat = fs.statSync(file);
73
- let data = {}, body = raw;
53
+ db.exec('BEGIN');
54
+ for (const { file, rel } of files) {
74
55
  try {
75
- const parsed = parseFrontmatter(raw);
76
- data = parsed.data || {};
77
- body = parsed.content || '';
78
- } catch {
79
- // keep raw body on malformed frontmatter
56
+ seen.add(rel);
57
+ const raw = fs.readFileSync(file, 'utf8');
58
+ const hash = createHash('sha256').update(raw).digest('hex');
59
+
60
+ const prev = existing.get(rel);
61
+ if (prev && prev.hash === hash) {
62
+ unchanged++;
63
+ continue;
64
+ }
65
+
66
+ if (isBackfill && prev) {
67
+ setHash.run(hash, prev.id);
68
+ unchanged++;
69
+ continue;
70
+ }
71
+
72
+ const stat = fs.statSync(file);
73
+ let data = {}, body = raw;
74
+ try {
75
+ const parsed = parseFrontmatter(raw);
76
+ data = parsed.data || {};
77
+ body = parsed.content || '';
78
+ } catch {
79
+ // keep raw body on malformed frontmatter
80
+ }
81
+
82
+ const hrr = embeddingsEnabled
83
+ ? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
84
+ : null;
85
+
86
+ upsert.run({
87
+ path: rel,
88
+ title: data.title ?? null,
89
+ release: data.release ?? null,
90
+ bundle: data.bundle ?? null,
91
+ doc_type: data.doc_type ?? null,
92
+ locale: data.locale ?? null,
93
+ frontmatter: JSON.stringify(data),
94
+ body,
95
+ size: stat.size,
96
+ mtime: Math.floor(stat.mtimeMs),
97
+ hash,
98
+ hrr_vector: hrr,
99
+ });
100
+ if (prev) updated++; else added++;
101
+ } catch (err) {
102
+ errors++;
103
+ if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
80
104
  }
81
-
82
- const hrr = embeddingsEnabled
83
- ? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
84
- : null;
85
-
86
- upsert.run({
87
- path: rel,
88
- title: data.title ?? null,
89
- release: data.release ?? null,
90
- bundle: data.bundle ?? null,
91
- doc_type: data.doc_type ?? null,
92
- locale: data.locale ?? null,
93
- frontmatter: JSON.stringify(data),
94
- body,
95
- size: stat.size,
96
- mtime: Math.floor(stat.mtimeMs),
97
- hash,
98
- hrr_vector: hrr,
99
- });
100
- if (prev) updated++; else added++;
101
- } catch (err) {
102
- errors++;
103
- if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
104
105
  }
105
- }
106
106
 
107
- for (const [rel, row] of existing) {
108
- if (!seen.has(rel)) {
109
- del.run(row.id);
110
- removed++;
107
+ for (const [rel, row] of existing) {
108
+ if (!seen.has(rel)) {
109
+ del.run(row.id);
110
+ removed++;
111
+ }
111
112
  }
112
- }
113
- db.exec('COMMIT');
113
+ db.exec('COMMIT');
114
114
 
115
- db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
116
- db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
117
- db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
118
- closeDocsDb(db);
115
+ db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
116
+ db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
117
+ db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
119
118
 
120
- const secs = ((Date.now() - started) / 1000).toFixed(1);
121
- return { added, updated, removed, unchanged, errors, secs, dbPath, embeddingsEnabled };
119
+ const secs = ((Date.now() - started) / 1000).toFixed(1);
120
+ return { added, updated, removed, unchanged, errors, secs, dbPath, embeddingsEnabled };
121
+ });
122
122
  }
@@ -1,7 +1,7 @@
1
1
  // CLI search against the docs SQLite database.
2
2
 
3
3
  import fs from 'node:fs';
4
- import { getDocsDbPath, hasEmbeddings, openDocsDb, closeDocsDb, getMeta } from './db.js';
4
+ import { getDocsDbPath, hasEmbeddings, openDocsDb, closeDocsDb, withDocsDb, getMeta } from './db.js';
5
5
  import { encodeText, similarity, bytesToPhases, DEFAULT_DIM } from './hrr.js';
6
6
 
7
7
  function safeParse(s) {
@@ -50,119 +50,119 @@ export function searchDocs(opts = {}) {
50
50
  throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first.`);
51
51
  }
52
52
 
53
- const db = openDocsDb({ dbPath });
54
- const limit = Math.min(parseInt(opts.limit, 10) || 20, 1000);
55
- const offset = parseInt(opts.offset, 10) || 0;
56
- const bundle = opts.bundle ? String(opts.bundle) : null;
57
- const docType = opts.docType ? String(opts.docType) : null;
58
- const mode = opts.mode === 'keyword' ? 'keyword' : 'hybrid';
59
- const useHybrid = mode === 'hybrid' && hasEmbeddings(db);
60
-
61
- const filters = [];
62
- const params = { q, limit, offset };
63
- if (bundle) {
64
- filters.push('d.bundle = @bundle');
65
- params.bundle = bundle;
66
- }
67
- if (docType) {
68
- filters.push('d.doc_type = @docType');
69
- params.docType = docType;
70
- }
71
- const where = filters.length ? ' AND ' + filters.join(' AND ') : '';
72
-
73
- // Total matching docs (for pagination)
74
- const total = db
75
- .prepare(`SELECT COUNT(*) AS n FROM docs_fts JOIN docs d ON d.id = docs_fts.rowid WHERE docs_fts MATCH @q${where}`)
76
- .get({ q, ...(bundle && { bundle }), ...(docType && { docType }) }).n;
77
-
78
- let rows;
79
- if (!useHybrid) {
80
- rows = db
81
- .prepare(
82
- `SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release,
83
- json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
84
- snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
85
- bm25(docs_fts) AS score
86
- FROM docs_fts
87
- JOIN docs d ON d.id = docs_fts.rowid
88
- WHERE docs_fts MATCH @q${where}
89
- ORDER BY score
90
- LIMIT @limit OFFSET @offset`
91
- )
92
- .all(params);
93
- } else {
94
- const candParams = { q, cand: (limit + offset) * 4 + 20 };
95
- if (bundle) candParams.bundle = bundle;
96
- if (docType) candParams.docType = docType;
97
- const candidates = db
98
- .prepare(
99
- `SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release, d.hrr_vector,
100
- json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
101
- snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
102
- bm25(docs_fts) AS bm25
103
- FROM docs_fts
104
- JOIN docs d ON d.id = docs_fts.rowid
105
- WHERE docs_fts MATCH @q${where}
106
- ORDER BY bm25
107
- LIMIT @cand`
108
- )
109
- .all(candParams);
110
-
111
- if (candidates.length === 0) {
112
- rows = [];
53
+ return withDocsDb({ dbPath }, (db) => {
54
+ const limit = Math.min(parseInt(opts.limit, 10) || 20, 1000);
55
+ const offset = parseInt(opts.offset, 10) || 0;
56
+ const bundle = opts.bundle ? String(opts.bundle) : null;
57
+ const docType = opts.docType ? String(opts.docType) : null;
58
+ const mode = opts.mode === 'keyword' ? 'keyword' : 'hybrid';
59
+ const useHybrid = mode === 'hybrid' && hasEmbeddings(db);
60
+
61
+ const filters = [];
62
+ const params = { q, limit, offset };
63
+ if (bundle) {
64
+ filters.push('d.bundle = @bundle');
65
+ params.bundle = bundle;
66
+ }
67
+ if (docType) {
68
+ filters.push('d.doc_type = @docType');
69
+ params.docType = docType;
70
+ }
71
+ const where = filters.length ? ' AND ' + filters.join(' AND ') : '';
72
+
73
+ // Total matching docs (for pagination)
74
+ const total = db
75
+ .prepare(`SELECT COUNT(*) AS n FROM docs_fts JOIN docs d ON d.id = docs_fts.rowid WHERE docs_fts MATCH @q${where}`)
76
+ .get({ q, ...(bundle && { bundle }), ...(docType && { docType }) }).n;
77
+
78
+ let rows;
79
+ if (!useHybrid) {
80
+ rows = db
81
+ .prepare(
82
+ `SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release,
83
+ json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
84
+ snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
85
+ bm25(docs_fts) AS score
86
+ FROM docs_fts
87
+ JOIN docs d ON d.id = docs_fts.rowid
88
+ WHERE docs_fts MATCH @q${where}
89
+ ORDER BY score
90
+ LIMIT @limit OFFSET @offset`
91
+ )
92
+ .all(params);
113
93
  } else {
114
- const hrrDim = (() => {
115
- const value = getMeta(db, 'hrr_dim');
116
- const n = parseInt(value, 10);
117
- return Number.isInteger(n) ? n : DEFAULT_DIM;
118
- })();
119
- const queryVec = encodeText(q, hrrDim);
120
- const queryTokens = new Set(q.toLowerCase().split(/\s+/).filter(Boolean));
121
-
122
- const bmVals = candidates.map((c) => c.bm25);
123
- const bmMin = Math.min(...bmVals);
124
- const bmMax = Math.max(...bmVals);
125
- const bmRange = bmMax - bmMin || 1;
126
-
127
- const FTS_W = 0.4, JACCARD_W = 0.3, HRR_W = 0.3;
128
-
129
- for (const c of candidates) {
130
- const ftsScore = 1 - (c.bm25 - bmMin) / bmRange;
131
- const contentTokens = new Set(
132
- `${c.title || ''} ${c.path || ''}`.toLowerCase().split(/\s+/).filter(Boolean)
133
- );
134
- let inter = 0;
135
- for (const t of queryTokens) if (contentTokens.has(t)) inter++;
136
- const union = queryTokens.size + contentTokens.size - inter;
137
- const jaccard = union ? inter / union : 0;
138
-
139
- let hrrSim = 0.5;
140
- if (c.hrr_vector) {
141
- hrrSim = (similarity(queryVec, bytesToPhases(c.hrr_vector)) + 1) / 2;
94
+ const candParams = { q, cand: (limit + offset) * 4 + 20 };
95
+ if (bundle) candParams.bundle = bundle;
96
+ if (docType) candParams.docType = docType;
97
+ const candidates = db
98
+ .prepare(
99
+ `SELECT d.id, d.path, d.title, d.bundle, d.doc_type, d.release, d.hrr_vector,
100
+ json_extract(d.frontmatter, '$.canonical_url') AS canonical_url,
101
+ snippet(docs_fts, 1, '[', ']', ' … ', 12) AS snippet,
102
+ bm25(docs_fts) AS bm25
103
+ FROM docs_fts
104
+ JOIN docs d ON d.id = docs_fts.rowid
105
+ WHERE docs_fts MATCH @q${where}
106
+ ORDER BY bm25
107
+ LIMIT @cand`
108
+ )
109
+ .all(candParams);
110
+
111
+ if (candidates.length === 0) {
112
+ rows = [];
113
+ } else {
114
+ const hrrDim = (() => {
115
+ const value = getMeta(db, 'hrr_dim');
116
+ const n = parseInt(value, 10);
117
+ return Number.isInteger(n) ? n : DEFAULT_DIM;
118
+ })();
119
+ const queryVec = encodeText(q, hrrDim);
120
+ const queryTokens = new Set(q.toLowerCase().split(/\s+/).filter(Boolean));
121
+
122
+ const bmVals = candidates.map((c) => c.bm25);
123
+ const bmMin = Math.min(...bmVals);
124
+ const bmMax = Math.max(...bmVals);
125
+ const bmRange = bmMax - bmMin || 1;
126
+
127
+ const FTS_W = 0.4, JACCARD_W = 0.3, HRR_W = 0.3;
128
+
129
+ for (const c of candidates) {
130
+ const ftsScore = 1 - (c.bm25 - bmMin) / bmRange;
131
+ const contentTokens = new Set(
132
+ `${c.title || ''} ${c.path || ''}`.toLowerCase().split(/\s+/).filter(Boolean)
133
+ );
134
+ let inter = 0;
135
+ for (const t of queryTokens) if (contentTokens.has(t)) inter++;
136
+ const union = queryTokens.size + contentTokens.size - inter;
137
+ const jaccard = union ? inter / union : 0;
138
+
139
+ let hrrSim = 0.5;
140
+ if (c.hrr_vector) {
141
+ hrrSim = (similarity(queryVec, bytesToPhases(c.hrr_vector)) + 1) / 2;
142
+ }
143
+
144
+ c.score = FTS_W * ftsScore + JACCARD_W * jaccard + HRR_W * hrrSim;
145
+ delete c.hrr_vector;
146
+ delete c.bm25;
142
147
  }
143
148
 
144
- c.score = FTS_W * ftsScore + JACCARD_W * jaccard + HRR_W * hrrSim;
145
- delete c.hrr_vector;
146
- delete c.bm25;
149
+ candidates.sort((a, b) => b.score - a.score);
150
+ rows = candidates.slice(offset, offset + limit);
147
151
  }
148
-
149
- candidates.sort((a, b) => b.score - a.score);
150
- rows = candidates.slice(offset, offset + limit);
151
152
  }
152
- }
153
153
 
154
- closeDocsDb(db);
155
154
 
156
- // Post-process canonical_url JSON strings that may contain escaped content.
157
- for (const row of rows) {
158
- if (row.canonical_url) row.canonical_url = safeParse(row.canonical_url) || row.canonical_url;
159
- }
155
+ // Post-process canonical_url JSON strings that may contain escaped content.
156
+ for (const row of rows) {
157
+ if (row.canonical_url) row.canonical_url = safeParse(row.canonical_url) || row.canonical_url;
158
+ }
160
159
 
161
- return {
162
- query: q,
163
- mode: useHybrid ? 'hybrid' : 'keyword',
164
- total,
165
- count: rows.length,
166
- results: rows,
167
- };
160
+ return {
161
+ query: q,
162
+ mode: useHybrid ? 'hybrid' : 'keyword',
163
+ total,
164
+ count: rows.length,
165
+ results: rows,
166
+ };
167
+ });
168
168
  }
@@ -5,7 +5,7 @@ import path from 'node:path';
5
5
  import http from 'node:http';
6
6
  import { URL } from 'node:url';
7
7
  import Database from 'better-sqlite3';
8
- import { getDocsDbPath, hasEmbeddings } from './db.js';
8
+ import { getDocsDbPath, hasEmbeddings, closeDocsDb } from './db.js';
9
9
  import { encodeText, similarity, bytesToPhases, DEFAULT_DIM } from './hrr.js';
10
10
 
11
11
  const PUBLIC_DIR = path.join(import.meta.dirname, 'public');
@@ -236,6 +236,8 @@ export function serveDocs(opts = {}) {
236
236
  }
237
237
  });
238
238
 
239
+ server.once('close', () => closeDocsDb(db));
240
+
239
241
  return new Promise((resolve, reject) => {
240
242
  let currentPort = parseInt(port, 10);
241
243
  const maxAttempts = 10;
@@ -263,6 +265,7 @@ export function serveDocs(opts = {}) {
263
265
  err.message = `Could not find an open port between ${port} and ${currentPort}. Try specifying one with --port.`;
264
266
  }
265
267
  resolved = true;
268
+ closeDocsDb(db);
266
269
  reject(err);
267
270
  }
268
271