@jacebenson/jsn 4.3.11 → 4.3.12-dev.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/jsn.js +7 -1
- package/package.json +3 -3
- package/src/app.js +10 -23
- package/src/auth/browser-session.js +21 -0
- package/src/auth/credential-state.js +124 -0
- package/src/auth/credential-store.js +177 -0
- package/src/auth/oauth.js +66 -0
- package/src/auth.js +11 -378
- package/src/cli-adapter.js +12 -4
- package/src/cli.js +3 -1
- package/src/commands/_generic.js +13 -9
- package/src/commands/_ticket.js +14 -11
- package/src/commands/docs/db.js +22 -2
- package/src/commands/docs/docs.js +13 -13
- package/src/commands/docs/embed.js +33 -33
- package/src/commands/docs/ingest.js +58 -58
- package/src/commands/docs/refresh.js +91 -91
- package/src/commands/docs/search.js +108 -108
- package/src/commands/docs/serve.js +4 -1
- package/src/commands/groups.js +11 -8
- package/src/commands/records.js +11 -16
- package/src/commands/requests.js +1 -1
- package/src/commands/setup.js +27 -25
- package/src/commands/tickets.js +14 -6
- package/src/commands/users.js +11 -8
- package/src/commands/version.js +16 -2
- package/src/frontmatter.js +5 -3
- package/src/helpers.js +45 -1
- package/src/paginated-search.js +162 -161
- package/src/prompts.js +15 -9
- package/src/sdk.js +79 -92
|
@@ -4,16 +4,6 @@ import fs from 'node:fs';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import net from 'node:net';
|
|
6
6
|
import process from 'node:process';
|
|
7
|
-
import {
|
|
8
|
-
getDocsDbPath, getDocsSourceDir, getDocsSourceMarkdownDir, docsDbExists,
|
|
9
|
-
openDocsDb, closeDocsDb, getDocsIndexStats,
|
|
10
|
-
} from './db.js';
|
|
11
|
-
import { syncDocs } from './sync.js';
|
|
12
|
-
import { refreshDocs } from './refresh.js';
|
|
13
|
-
import { embedDocs } from './embed.js';
|
|
14
|
-
import { searchDocs, getDocByIdOrPath } from './search.js';
|
|
15
|
-
import { serveDocs } from './serve.js';
|
|
16
|
-
import { syncCommunity, listCommunityDocs } from './community.js';
|
|
17
7
|
import { declareCapabilities } from '../../capabilities.js';
|
|
18
8
|
import { formatCommandList } from '../../helpers.js';
|
|
19
9
|
|
|
@@ -39,6 +29,8 @@ export function docsCmd(wrap) {
|
|
|
39
29
|
.option('no-embed', { type: 'boolean', describe: 'Skip HRR semantic embeddings' })
|
|
40
30
|
.option('no-ingest', { type: 'boolean', describe: 'Clone/pull only; do not rebuild index' }),
|
|
41
31
|
handler: wrap(async (argv, app) => {
|
|
32
|
+
const { getDocsDbPath, getDocsSourceMarkdownDir } = await import('./db.js');
|
|
33
|
+
const { syncDocs } = await import('./sync.js');
|
|
42
34
|
const result = syncDocs({
|
|
43
35
|
markdownDir: getDocsSourceMarkdownDir(),
|
|
44
36
|
dbPath: getDocsDbPath(),
|
|
@@ -52,6 +44,7 @@ export function docsCmd(wrap) {
|
|
|
52
44
|
command: 'status',
|
|
53
45
|
describe: 'Show docs database status',
|
|
54
46
|
handler: wrap(async (argv, app) => {
|
|
47
|
+
const { getDocsDbPath, getDocsSourceDir, getDocsSourceMarkdownDir, docsDbExists, withDocsDb, getDocsIndexStats } = await import('./db.js');
|
|
55
48
|
const dbPath = getDocsDbPath();
|
|
56
49
|
const repoDir = getDocsSourceDir();
|
|
57
50
|
const markdownDir = getDocsSourceMarkdownDir();
|
|
@@ -60,12 +53,10 @@ export function docsCmd(wrap) {
|
|
|
60
53
|
|
|
61
54
|
let state, embedded, documents, bundles, docTypes, syncedAt, dbSize;
|
|
62
55
|
if (dbExists) {
|
|
63
|
-
const
|
|
64
|
-
const stats = getDocsIndexStats(db);
|
|
56
|
+
const stats = withDocsDb({}, (db) => getDocsIndexStats(db));
|
|
65
57
|
({ documents, bundles, docTypes, syncedAt } = stats);
|
|
66
58
|
const stat = fs.statSync(dbPath);
|
|
67
59
|
dbSize = stat.size;
|
|
68
|
-
closeDocsDb(db);
|
|
69
60
|
|
|
70
61
|
embedded = stats.embeddedCount >= documents && documents > 0;
|
|
71
62
|
if (embedded) state = 'ready';
|
|
@@ -108,6 +99,7 @@ export function docsCmd(wrap) {
|
|
|
108
99
|
.option('doc-type', { type: 'string', describe: 'Filter by doc type' })
|
|
109
100
|
.option('mode', { type: 'string', choices: ['keyword', 'hybrid'], default: 'hybrid', describe: 'Search mode' }),
|
|
110
101
|
handler: wrap(async (argv, app) => {
|
|
102
|
+
const { searchDocs } = await import('./search.js');
|
|
111
103
|
const result = searchDocs({
|
|
112
104
|
query: argv.query,
|
|
113
105
|
limit: argv.limit,
|
|
@@ -154,6 +146,7 @@ export function docsCmd(wrap) {
|
|
|
154
146
|
.positional('id-or-path', { type: 'string', describe: 'Document id (number) or file path' })
|
|
155
147
|
.option('lines', { type: 'number', default: 80, describe: 'Lines of body to show in terminal (use --json for full content)' }),
|
|
156
148
|
handler: wrap(async (argv, app) => {
|
|
149
|
+
const { getDocByIdOrPath } = await import('./search.js');
|
|
157
150
|
const idOrPath = argv['id-or-path'];
|
|
158
151
|
const row = getDocByIdOrPath(idOrPath);
|
|
159
152
|
|
|
@@ -192,6 +185,7 @@ export function docsCmd(wrap) {
|
|
|
192
185
|
.option('host', { type: 'string', default: '127.0.0.1', describe: 'Bind address' })
|
|
193
186
|
.option('expose', { type: 'boolean', default: false, describe: 'Bind to 0.0.0.0 to expose on the network' }),
|
|
194
187
|
handler: wrap(async (argv, _app) => {
|
|
188
|
+
const { serveDocs } = await import('./serve.js');
|
|
195
189
|
try {
|
|
196
190
|
const host = argv.expose ? '0.0.0.0' : argv.host;
|
|
197
191
|
await serveDocs({ port: argv.port, host });
|
|
@@ -205,6 +199,7 @@ export function docsCmd(wrap) {
|
|
|
205
199
|
command: 'embed',
|
|
206
200
|
describe: 'Backfill or refresh HRR semantic embeddings',
|
|
207
201
|
handler: wrap(async (argv, app) => {
|
|
202
|
+
const { embedDocs } = await import('./embed.js');
|
|
208
203
|
const result = embedDocs();
|
|
209
204
|
app.ok(result, { summary: `Embedded ${result.embedded} of ${result.total} docs` });
|
|
210
205
|
}),
|
|
@@ -213,6 +208,8 @@ export function docsCmd(wrap) {
|
|
|
213
208
|
command: 'refresh',
|
|
214
209
|
describe: 'Incrementally refresh docs.db from existing source clone',
|
|
215
210
|
handler: wrap(async (argv, app) => {
|
|
211
|
+
const { getDocsSourceMarkdownDir } = await import('./db.js');
|
|
212
|
+
const { refreshDocs } = await import('./refresh.js');
|
|
216
213
|
const result = refreshDocs({ docsDir: getDocsSourceMarkdownDir() });
|
|
217
214
|
app.ok(result, { summary: `Refresh complete: +${result.added} ~${result.updated} -${result.removed} =${result.unchanged}` });
|
|
218
215
|
}),
|
|
@@ -230,6 +227,7 @@ export function docsCmd(wrap) {
|
|
|
230
227
|
.option('sources', { type: 'string', describe: 'Path to a community-sources.json (default: bundled)' })
|
|
231
228
|
.option('delay-ms', { type: 'number', describe: 'Delay between article fetches (default: 500)' }),
|
|
232
229
|
handler: wrap(async (argv, app) => {
|
|
230
|
+
const { syncCommunity } = await import('./community.js');
|
|
233
231
|
const result = await syncCommunity({
|
|
234
232
|
sourcesFile: argv.sources,
|
|
235
233
|
expand: argv.expand === true,
|
|
@@ -245,6 +243,7 @@ export function docsCmd(wrap) {
|
|
|
245
243
|
command: 'list',
|
|
246
244
|
describe: 'List community docs currently in the index',
|
|
247
245
|
handler: wrap(async (argv, app) => {
|
|
246
|
+
const { listCommunityDocs } = await import('./community.js');
|
|
248
247
|
const result = listCommunityDocs();
|
|
249
248
|
const lines = result.docs.map((d) => `[${d.id}] ${d.title || d.path} (${d.doc_type})`);
|
|
250
249
|
app.ok({
|
|
@@ -258,6 +257,7 @@ export function docsCmd(wrap) {
|
|
|
258
257
|
});
|
|
259
258
|
},
|
|
260
259
|
handler: wrap(async (argv, app) => {
|
|
260
|
+
const { getDocsDbPath } = await import('./db.js');
|
|
261
261
|
const subcommands = argv.subcommands || [];
|
|
262
262
|
const summary = 'jsn docs — ServiceNow documentation search (sync, status, search, show, serve).';
|
|
263
263
|
app.ok({
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Populate HRR phase-vector embeddings for semantic reranking.
|
|
2
2
|
|
|
3
3
|
import fs from 'node:fs';
|
|
4
|
-
import {
|
|
4
|
+
import { withDocsDb, getDocsDbPath } from './db.js';
|
|
5
5
|
import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
|
|
6
6
|
|
|
7
7
|
export function embedDocs(opts = {}) {
|
|
@@ -11,36 +11,36 @@ export function embedDocs(opts = {}) {
|
|
|
11
11
|
throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first.`);
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
14
|
+
return withDocsDb({ dbPath }, (db) => {
|
|
15
|
+
|
|
16
|
+
const cols = db.prepare('PRAGMA table_info(docs)').all();
|
|
17
|
+
if (!cols.some((c) => c.name === 'hrr_vector')) {
|
|
18
|
+
db.exec('ALTER TABLE docs ADD COLUMN hrr_vector BLOB');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
db.exec("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)");
|
|
22
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('hrr_dim', ?)").run(String(DEFAULT_DIM));
|
|
23
|
+
|
|
24
|
+
const rows = db.prepare('SELECT id, title, body FROM docs WHERE hrr_vector IS NULL').all();
|
|
25
|
+
const total = db.prepare('SELECT COUNT(*) AS n FROM docs').get().n;
|
|
26
|
+
|
|
27
|
+
const update = db.prepare('UPDATE docs SET hrr_vector = ? WHERE id = ?');
|
|
28
|
+
const started = Date.now();
|
|
29
|
+
let n = 0;
|
|
30
|
+
|
|
31
|
+
db.exec('BEGIN');
|
|
32
|
+
for (const row of rows) {
|
|
33
|
+
const surface = docSurface(row.title, row.body);
|
|
34
|
+
const vec = encodeText(surface, DEFAULT_DIM);
|
|
35
|
+
update.run(phasesToBytes(vec), row.id);
|
|
36
|
+
n++;
|
|
37
|
+
if (n % 5000 === 0) process.stderr.write(` ...${n}\n`);
|
|
38
|
+
}
|
|
39
|
+
db.exec('COMMIT');
|
|
40
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
41
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('embeddings', ?)").run('true');
|
|
42
|
+
|
|
43
|
+
const secs = ((Date.now() - started) / 1000).toFixed(1);
|
|
44
|
+
return { embedded: n, total, secs, dbPath };
|
|
45
|
+
});
|
|
46
46
|
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import fs from 'node:fs';
|
|
4
4
|
import { createHash } from 'node:crypto';
|
|
5
5
|
import { parseFrontmatter } from '../../frontmatter.js';
|
|
6
|
-
import {
|
|
6
|
+
import { withDocsDb, initDocsSchema, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
|
|
7
7
|
import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
|
|
8
8
|
|
|
9
9
|
export function ingestDocs(opts = {}) {
|
|
@@ -14,69 +14,69 @@ export function ingestDocs(opts = {}) {
|
|
|
14
14
|
// Fresh build each run.
|
|
15
15
|
if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
return withDocsDb({ dbPath }, (db) => {
|
|
18
|
+
initDocsSchema(db);
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
const insert = db.prepare(`
|
|
21
|
+
INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
|
|
22
|
+
VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
|
|
23
|
+
`);
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
let count = 0;
|
|
26
|
+
let errors = 0;
|
|
27
|
+
const startedAt = Date.now();
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
try {
|
|
32
|
-
const stat = fs.statSync(file);
|
|
33
|
-
const raw = fs.readFileSync(file, 'utf8');
|
|
34
|
-
const hash = createHash('sha256').update(raw).digest('hex');
|
|
35
|
-
let data = {};
|
|
36
|
-
let body = raw;
|
|
29
|
+
db.exec('BEGIN');
|
|
30
|
+
for (const { file, rel } of walkDocs(roots)) {
|
|
37
31
|
try {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
32
|
+
const stat = fs.statSync(file);
|
|
33
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
34
|
+
const hash = createHash('sha256').update(raw).digest('hex');
|
|
35
|
+
let data = {};
|
|
36
|
+
let body = raw;
|
|
37
|
+
try {
|
|
38
|
+
const parsed = parseFrontmatter(raw);
|
|
39
|
+
data = parsed.data || {};
|
|
40
|
+
body = parsed.content || '';
|
|
41
|
+
} catch {
|
|
42
|
+
// Malformed frontmatter: keep raw body, empty metadata.
|
|
43
|
+
}
|
|
44
|
+
const hrr = embed
|
|
45
|
+
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
46
|
+
: null;
|
|
47
|
+
insert.run({
|
|
48
|
+
path: rel,
|
|
49
|
+
title: data.title ?? null,
|
|
50
|
+
release: data.release ?? null,
|
|
51
|
+
bundle: data.bundle ?? null,
|
|
52
|
+
doc_type: data.doc_type ?? null,
|
|
53
|
+
locale: data.locale ?? null,
|
|
54
|
+
frontmatter: JSON.stringify(data),
|
|
55
|
+
body,
|
|
56
|
+
size: stat.size,
|
|
57
|
+
mtime: Math.floor(stat.mtimeMs),
|
|
58
|
+
hash,
|
|
59
|
+
hrr_vector: hrr,
|
|
60
|
+
});
|
|
61
|
+
count++;
|
|
62
|
+
if (count % 5000 === 0) {
|
|
63
|
+
process.stderr.write(` ...${count} files\n`);
|
|
64
|
+
}
|
|
65
|
+
} catch (err) {
|
|
66
|
+
errors++;
|
|
67
|
+
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
43
68
|
}
|
|
44
|
-
const hrr = embed
|
|
45
|
-
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
46
|
-
: null;
|
|
47
|
-
insert.run({
|
|
48
|
-
path: rel,
|
|
49
|
-
title: data.title ?? null,
|
|
50
|
-
release: data.release ?? null,
|
|
51
|
-
bundle: data.bundle ?? null,
|
|
52
|
-
doc_type: data.doc_type ?? null,
|
|
53
|
-
locale: data.locale ?? null,
|
|
54
|
-
frontmatter: JSON.stringify(data),
|
|
55
|
-
body,
|
|
56
|
-
size: stat.size,
|
|
57
|
-
mtime: Math.floor(stat.mtimeMs),
|
|
58
|
-
hash,
|
|
59
|
-
hrr_vector: hrr,
|
|
60
|
-
});
|
|
61
|
-
count++;
|
|
62
|
-
if (count % 5000 === 0) {
|
|
63
|
-
process.stderr.write(` ...${count} files\n`);
|
|
64
|
-
}
|
|
65
|
-
} catch (err) {
|
|
66
|
-
errors++;
|
|
67
|
-
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
68
69
|
}
|
|
69
|
-
|
|
70
|
-
db.exec('COMMIT');
|
|
70
|
+
db.exec('COMMIT');
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
closeDocsDb(db);
|
|
72
|
+
db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
|
|
73
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
74
|
+
db.exec("CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)");
|
|
75
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('hrr_dim', ?)").run(String(DEFAULT_DIM));
|
|
76
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
|
|
77
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('embeddings', ?)").run(String(embed));
|
|
79
78
|
|
|
80
|
-
|
|
81
|
-
|
|
79
|
+
const secs = ((Date.now() - startedAt) / 1000).toFixed(1);
|
|
80
|
+
return { count, errors, secs, dbPath, embedded: embed };
|
|
81
|
+
});
|
|
82
82
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import fs from 'node:fs';
|
|
5
5
|
import { createHash } from 'node:crypto';
|
|
6
6
|
import { parseFrontmatter } from '../../frontmatter.js';
|
|
7
|
-
import {
|
|
7
|
+
import { withDocsDb, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
|
|
8
8
|
import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
|
|
9
9
|
|
|
10
10
|
export function refreshDocs(opts = {}) {
|
|
@@ -15,108 +15,108 @@ export function refreshDocs(opts = {}) {
|
|
|
15
15
|
throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first for the initial build.`);
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
return withDocsDb({ dbPath }, (db) => {
|
|
19
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
20
20
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
const cols = db.prepare('PRAGMA table_info(docs)').all().map((c) => c.name);
|
|
22
|
+
if (!cols.includes('hash')) db.exec('ALTER TABLE docs ADD COLUMN hash TEXT');
|
|
23
|
+
if (!cols.includes('hrr_vector')) db.exec('ALTER TABLE docs ADD COLUMN hrr_vector BLOB');
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
const embeddingsEnabled =
|
|
26
|
+
db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hrr_vector IS NOT NULL').get().n > 0 ||
|
|
27
|
+
!cols.includes('hrr_vector');
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
const isBackfill = db.prepare('SELECT COUNT(*) AS n FROM docs WHERE hash IS NOT NULL').get().n === 0
|
|
30
|
+
&& db.prepare('SELECT COUNT(*) AS n FROM docs').get().n > 0;
|
|
31
|
+
const setHash = db.prepare('UPDATE docs SET hash = ? WHERE id = ?');
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
const existing = new Map();
|
|
34
|
+
for (const row of db.prepare('SELECT id, path, hash FROM docs').all()) {
|
|
35
|
+
existing.set(row.path, { id: row.id, hash: row.hash });
|
|
36
|
+
}
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
let added = 0, updated = 0, removed = 0, unchanged = 0, errors = 0;
|
|
48
|
-
const seen = new Set();
|
|
49
|
-
const started = Date.now();
|
|
50
|
-
|
|
51
|
-
const files = [...walkDocs(roots)];
|
|
52
|
-
|
|
53
|
-
db.exec('BEGIN');
|
|
54
|
-
for (const { file, rel } of files) {
|
|
55
|
-
try {
|
|
56
|
-
seen.add(rel);
|
|
57
|
-
const raw = fs.readFileSync(file, 'utf8');
|
|
58
|
-
const hash = createHash('sha256').update(raw).digest('hex');
|
|
59
|
-
|
|
60
|
-
const prev = existing.get(rel);
|
|
61
|
-
if (prev && prev.hash === hash) {
|
|
62
|
-
unchanged++;
|
|
63
|
-
continue;
|
|
64
|
-
}
|
|
38
|
+
const upsert = db.prepare(`
|
|
39
|
+
INSERT INTO docs (path, title, release, bundle, doc_type, locale, frontmatter, body, size, mtime, hash, hrr_vector)
|
|
40
|
+
VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
|
|
41
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
42
|
+
title=@title, release=@release, bundle=@bundle, doc_type=@doc_type, locale=@locale,
|
|
43
|
+
frontmatter=@frontmatter, body=@body, size=@size, mtime=@mtime, hash=@hash, hrr_vector=@hrr_vector
|
|
44
|
+
`);
|
|
45
|
+
const del = db.prepare('DELETE FROM docs WHERE id = ?');
|
|
65
46
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
47
|
+
let added = 0, updated = 0, removed = 0, unchanged = 0, errors = 0;
|
|
48
|
+
const seen = new Set();
|
|
49
|
+
const started = Date.now();
|
|
50
|
+
|
|
51
|
+
const files = [...walkDocs(roots)];
|
|
71
52
|
|
|
72
|
-
|
|
73
|
-
|
|
53
|
+
db.exec('BEGIN');
|
|
54
|
+
for (const { file, rel } of files) {
|
|
74
55
|
try {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
56
|
+
seen.add(rel);
|
|
57
|
+
const raw = fs.readFileSync(file, 'utf8');
|
|
58
|
+
const hash = createHash('sha256').update(raw).digest('hex');
|
|
59
|
+
|
|
60
|
+
const prev = existing.get(rel);
|
|
61
|
+
if (prev && prev.hash === hash) {
|
|
62
|
+
unchanged++;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (isBackfill && prev) {
|
|
67
|
+
setHash.run(hash, prev.id);
|
|
68
|
+
unchanged++;
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const stat = fs.statSync(file);
|
|
73
|
+
let data = {}, body = raw;
|
|
74
|
+
try {
|
|
75
|
+
const parsed = parseFrontmatter(raw);
|
|
76
|
+
data = parsed.data || {};
|
|
77
|
+
body = parsed.content || '';
|
|
78
|
+
} catch {
|
|
79
|
+
// keep raw body on malformed frontmatter
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const hrr = embeddingsEnabled
|
|
83
|
+
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
84
|
+
: null;
|
|
85
|
+
|
|
86
|
+
upsert.run({
|
|
87
|
+
path: rel,
|
|
88
|
+
title: data.title ?? null,
|
|
89
|
+
release: data.release ?? null,
|
|
90
|
+
bundle: data.bundle ?? null,
|
|
91
|
+
doc_type: data.doc_type ?? null,
|
|
92
|
+
locale: data.locale ?? null,
|
|
93
|
+
frontmatter: JSON.stringify(data),
|
|
94
|
+
body,
|
|
95
|
+
size: stat.size,
|
|
96
|
+
mtime: Math.floor(stat.mtimeMs),
|
|
97
|
+
hash,
|
|
98
|
+
hrr_vector: hrr,
|
|
99
|
+
});
|
|
100
|
+
if (prev) updated++; else added++;
|
|
101
|
+
} catch (err) {
|
|
102
|
+
errors++;
|
|
103
|
+
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
80
104
|
}
|
|
81
|
-
|
|
82
|
-
const hrr = embeddingsEnabled
|
|
83
|
-
? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
|
|
84
|
-
: null;
|
|
85
|
-
|
|
86
|
-
upsert.run({
|
|
87
|
-
path: rel,
|
|
88
|
-
title: data.title ?? null,
|
|
89
|
-
release: data.release ?? null,
|
|
90
|
-
bundle: data.bundle ?? null,
|
|
91
|
-
doc_type: data.doc_type ?? null,
|
|
92
|
-
locale: data.locale ?? null,
|
|
93
|
-
frontmatter: JSON.stringify(data),
|
|
94
|
-
body,
|
|
95
|
-
size: stat.size,
|
|
96
|
-
mtime: Math.floor(stat.mtimeMs),
|
|
97
|
-
hash,
|
|
98
|
-
hrr_vector: hrr,
|
|
99
|
-
});
|
|
100
|
-
if (prev) updated++; else added++;
|
|
101
|
-
} catch (err) {
|
|
102
|
-
errors++;
|
|
103
|
-
if (errors <= 10) process.stderr.write(`Skipped ${file}: ${err.message}\n`);
|
|
104
105
|
}
|
|
105
|
-
}
|
|
106
106
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
107
|
+
for (const [rel, row] of existing) {
|
|
108
|
+
if (!seen.has(rel)) {
|
|
109
|
+
del.run(row.id);
|
|
110
|
+
removed++;
|
|
111
|
+
}
|
|
111
112
|
}
|
|
112
|
-
|
|
113
|
-
db.exec('COMMIT');
|
|
113
|
+
db.exec('COMMIT');
|
|
114
114
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
closeDocsDb(db);
|
|
115
|
+
db.exec("INSERT INTO docs_fts(docs_fts) VALUES('optimize');");
|
|
116
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
117
|
+
db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES('synced_at', ?)").run(String(Date.now()));
|
|
119
118
|
|
|
120
|
-
|
|
121
|
-
|
|
119
|
+
const secs = ((Date.now() - started) / 1000).toFixed(1);
|
|
120
|
+
return { added, updated, removed, unchanged, errors, secs, dbPath, embeddingsEnabled };
|
|
121
|
+
});
|
|
122
122
|
}
|