@jacebenson/jsn 4.0.0 → 4.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,343 @@
1
+ // Fetch ServiceNow Community pages, convert them to markdown, and index them
2
+ // into the local docs DB alongside the ServiceNowDocs clone.
3
+ //
4
+ // Community content is written to <cache>/servicenow-cli/docs/community/ as
5
+ // markdown with frontmatter, then picked up by ingest/refresh via walkDocs().
6
+
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+ import process from 'node:process';
10
+ import TurndownService from 'turndown';
11
+ import matter from 'gray-matter';
12
+ import Database from 'better-sqlite3';
13
+ import { getDocsCommunityDir, getDocsDbPath, docsDbExists } from './db.js';
14
+ import { refreshDocs } from './refresh.js';
15
+ import { ingestDocs } from './ingest.js';
16
+
17
+ const DEFAULT_SOURCES_FILE = path.join(import.meta.dirname, 'community-sources.json');
18
+ const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36';
19
+
20
+ const turndown = new TurndownService({
21
+ headingStyle: 'atx',
22
+ codeBlockStyle: 'fenced',
23
+ bulletListMarker: '-',
24
+ });
25
+
26
+ // --- Extraction helpers (pure, exported for tests) ---
27
+
28
+ export function extractBodyHtml(html) {
29
+ const marker = 'lia-message-body-content';
30
+ const start = html.indexOf(marker);
31
+ if (start === -1) return null;
32
+ const open = html.indexOf('>', start) + 1;
33
+ if (open <= 0) return null;
34
+ // Depth-count the <div> nesting from the body content element.
35
+ let depth = 1;
36
+ let i = open;
37
+ while (i < html.length && depth > 0) {
38
+ const nextOpen = html.indexOf('<div', i);
39
+ const nextClose = html.indexOf('</div>', i);
40
+ if (nextOpen === -1 && nextClose === -1) break;
41
+ if (nextClose === -1 || (nextOpen !== -1 && nextOpen < nextClose)) {
42
+ depth++;
43
+ i = nextOpen + 4;
44
+ } else {
45
+ depth--;
46
+ i = nextClose + 6;
47
+ }
48
+ }
49
+ return html.slice(open, i);
50
+ }
51
+
52
+ export function extractMeta(html) {
53
+ const title = (() => {
54
+ const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/);
55
+ if (h1) return stripTags(h1[1]).trim();
56
+ const t = html.match(/<title>([\s\S]*?)<\/title>/);
57
+ return t ? t[1].trim().replace(/\s*-\s*ServiceNow Community\s*$/i, '') : null;
58
+ })();
59
+
60
+ const author = (() => {
61
+ const m = html.match(/<span\s+content="([^"]+)"\s+itemprop="name"[^>]*>/);
62
+ return m ? m[1] : null;
63
+ })();
64
+
65
+ const authorUrl = (() => {
66
+ const m = html.match(/class="[^"]*lia-user-name-link[^"]*"[^>]*href="([^"]+)"/);
67
+ return m ? m[1] : null;
68
+ })();
69
+
70
+ return { title, author, authorUrl };
71
+ }
72
+
73
+ export function stripTags(s) {
74
+ return decodeEntities(String(s).replace(/<[^>]*>/g, ''));
75
+ }
76
+
77
+ function decodeEntities(s) {
78
+ return s
79
+ .replace(/&amp;/g, '&')
80
+ .replace(/&lt;/g, '<')
81
+ .replace(/&gt;/g, '>')
82
+ .replace(/&quot;/g, '"')
83
+ .replace(/&#39;/g, "'")
84
+ .replace(/&nbsp;/g, ' ');
85
+ }
86
+
87
+ // --- HTML -> markdown ---
88
+
89
+ export function htmlToMarkdown(html) {
90
+ return turndown.turndown(html).replace(/\n{3,}/g, '\n\n').trim() + '\n';
91
+ }
92
+
93
+ // --- Community page fetch ---
94
+
95
+ async function fetchPage(url, { retries = 3, timeoutMs = 30000 } = {}) {
96
+ let lastErr;
97
+ for (let attempt = 0; attempt <= retries; attempt++) {
98
+ const controller = new AbortController();
99
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
100
+ try {
101
+ const res = await fetch(url, {
102
+ headers: { 'user-agent': UA, accept: 'text/html' },
103
+ signal: controller.signal,
104
+ redirect: 'follow',
105
+ });
106
+ if (res.status === 429) {
107
+ const wait = 5000 * (attempt + 1);
108
+ process.stderr.write(` rate limited (429), waiting ${wait / 1000}s...\n`);
109
+ await sleep(wait);
110
+ continue;
111
+ }
112
+ if (res.status >= 400) {
113
+ lastErr = new Error(`HTTP ${res.status} for ${url}`);
114
+ continue;
115
+ }
116
+ return await res.text();
117
+ } catch (err) {
118
+ lastErr = err;
119
+ if (attempt < retries) await sleep(1500 * (attempt + 1));
120
+ } finally {
121
+ clearTimeout(timer);
122
+ }
123
+ }
124
+ throw lastErr;
125
+ }
126
+
127
+ function sleep(ms) {
128
+ return new Promise((resolve) => setTimeout(resolve, ms));
129
+ }
130
+
131
+ // --- Link extraction for --expand ---
132
+
133
+ export function extractCommunityLinks(html, baseUrl) {
134
+ const bodyHtml = extractBodyHtml(html) || html;
135
+ const out = new Set();
136
+ const re = /href="([^"]+)"/g;
137
+ let m;
138
+ while ((m = re.exec(bodyHtml))) {
139
+ let u = m[1].replace(/&amp;/g, '&');
140
+ if (u.startsWith('#')) continue;
141
+ try {
142
+ u = new URL(u, baseUrl).toString();
143
+ } catch {
144
+ continue;
145
+ }
146
+ // Only new Khoros community article/blog pages: /community/<section>/<slug>/t[abdp]-p/<id>
147
+ const km = u.match(/^https?:\/\/www\.servicenow\.com\/community\/[a-z0-9-]+\/[^/]+\/t[abdp]-p\/\d+/);
148
+ if (!km) continue;
149
+ // Strip query/hash for dedupe.
150
+ const clean = km[0];
151
+ if (/\/user\/|viewprofilepage|notifymoderator|kudos|printpage|\/s\//.test(clean)) continue;
152
+ out.add(clean);
153
+ }
154
+ return [...out];
155
+ }
156
+
157
+ // --- File naming ---
158
+
159
+ export function slugify(title, fallback = 'untitled') {
160
+ const s = String(title || '')
161
+ .toLowerCase()
162
+ .replace(/[^a-z0-9]+/g, '-')
163
+ .replace(/^-+|-+$/g, '')
164
+ .slice(0, 80);
165
+ return s || fallback;
166
+ }
167
+
168
+ function frontmatter(body, data) {
169
+ return matter.stringify(body, data);
170
+ }
171
+
172
+ // --- Write a community doc to disk ---
173
+
174
+ export function writeCommunityDoc({ author, title, url, bodyMd, docType, sourceUrl, published, fetchedAt, communityDir } = {}) {
175
+ const baseDir = communityDir || getDocsCommunityDir();
176
+ const authorDir = path.join(baseDir, slugify(author, 'unknown'));
177
+ fs.mkdirSync(authorDir, { recursive: true });
178
+ // Include the post id (t[abdp]-p/<id>) so identical titles across posts
179
+ // never collide on disk. e.g. 456-articles-blogs-videos.../ba-p/2292127
180
+ const id = (url.match(/\/(t[abdp]-p\/\d+)\/?$/) || [])[1] || null;
181
+ const idPart = id ? id.replace(/[^a-z0-9]+/g, '-') : null;
182
+ const titleSlug = slugify(title, 'untitled');
183
+ const slug = idPart ? `${titleSlug}-${idPart}` : titleSlug;
184
+ const rel = path.join(authorDir, `${slug}.md`);
185
+ const fm = {
186
+ title,
187
+ author,
188
+ bundle: 'community',
189
+ doc_type: docType,
190
+ canonical_url: url,
191
+ ...(sourceUrl ? { source_url: sourceUrl } : {}),
192
+ ...(published ? { published } : {}),
193
+ ...(fetchedAt ? { fetched_at: fetchedAt } : {}),
194
+ };
195
+ const md = frontmatter(bodyMd, fm);
196
+ fs.writeFileSync(rel, md);
197
+ return { file: rel, rel: path.relative(baseDir, rel).split(path.sep).join('/') };
198
+ }
199
+
200
+ // --- Sync entry points ---
201
+
202
+ function loadSources(sourcesFile) {
203
+ const file = sourcesFile || DEFAULT_SOURCES_FILE;
204
+ if (!fs.existsSync(file)) {
205
+ throw new Error(`Community sources file not found: ${file}`);
206
+ }
207
+ const data = JSON.parse(fs.readFileSync(file, 'utf8'));
208
+ if (!Array.isArray(data.sources)) {
209
+ throw new Error(`Invalid community sources file ${file}: expected { "sources": [...] }`);
210
+ }
211
+ return data.sources;
212
+ }
213
+
214
+ export async function syncCommunity(opts = {}) {
215
+ const sources = loadSources(opts.sourcesFile);
216
+ const expand = opts.expand === true;
217
+ const force = opts.force === true;
218
+ const delayMs = opts.delayMs != null ? opts.delayMs : 500;
219
+ const dbPath = opts.dbPath || getDocsDbPath();
220
+
221
+ const results = { hubs: [], articles: [], skipped: [], errors: [] };
222
+
223
+ for (const source of sources) {
224
+ const url = source.url || source.hub_url;
225
+ if (!url) {
226
+ results.errors.push({ source: source.name || '?', error: 'missing url' });
227
+ continue;
228
+ }
229
+ process.stderr.write(`Fetching hub: ${url}\n`);
230
+ try {
231
+ const html = await fetchPage(url);
232
+ const meta = extractMeta(html);
233
+ const title = meta.title || source.name || url;
234
+ const author = meta.author || source.author || 'ServiceNow Community';
235
+ const authorUrl = meta.authorUrl || source.author_url;
236
+ const bodyHtml = extractBodyHtml(html);
237
+ if (!bodyHtml) {
238
+ results.errors.push({ url, error: 'no article body found' });
239
+ continue;
240
+ }
241
+ const bodyMd = htmlToMarkdown(bodyHtml);
242
+ const written = writeCommunityDoc({
243
+ author,
244
+ title,
245
+ url,
246
+ bodyMd,
247
+ docType: 'hub',
248
+ fetchedAt: new Date().toISOString(),
249
+ });
250
+ results.hubs.push({ title, author, authorUrl, url, file: written.rel });
251
+
252
+ if (expand) {
253
+ const links = extractCommunityLinks(html, url);
254
+ const urlIndex = buildUrlIndex();
255
+ process.stderr.write(` ${links.length} linked article(s) found\n`);
256
+ for (const link of links) {
257
+ // Skip already-downloaded files unless --force.
258
+ const existing = findExisting(link, urlIndex);
259
+ if (existing && !force) {
260
+ results.skipped.push({ url: link, reason: 'already downloaded' });
261
+ continue;
262
+ }
263
+ process.stderr.write(` Fetching: ${link}\n`);
264
+ try {
265
+ const pageHtml = await fetchPage(link);
266
+ const pageMeta = extractMeta(pageHtml);
267
+ const pageBody = extractBodyHtml(pageHtml);
268
+ if (!pageBody) {
269
+ results.skipped.push({ url: link, reason: 'no article body' });
270
+ continue;
271
+ }
272
+ const md = htmlToMarkdown(pageBody);
273
+ const wrote = writeCommunityDoc({
274
+ author: pageMeta.author || author,
275
+ title: pageMeta.title || link,
276
+ url: link,
277
+ bodyMd: md,
278
+ docType: 'article',
279
+ sourceUrl: url,
280
+ fetchedAt: new Date().toISOString(),
281
+ });
282
+ results.articles.push({ title: pageMeta.title, url: link, file: wrote.rel });
283
+ } catch (err) {
284
+ results.errors.push({ url: link, error: err.message });
285
+ }
286
+ if (delayMs > 0) await sleep(delayMs);
287
+ }
288
+ }
289
+ if (delayMs > 0) await sleep(delayMs);
290
+ } catch (err) {
291
+ results.errors.push({ url, error: err.message });
292
+ }
293
+ }
294
+
295
+ // Refresh the index so new community docs are searchable. Fall back to a
296
+ // full ingest when no DB exists yet.
297
+ const refreshResult = docsDbExists()
298
+ ? refreshDocs({ dbPath })
299
+ : ingestDocs({ dbPath, roots: [{ dir: getDocsCommunityDir(), prefix: 'community/' }], embed: true });
300
+
301
+ return { ...results, refresh: refreshResult };
302
+ }
303
+
304
+ function buildUrlIndex() {
305
+ const communityDir = getDocsCommunityDir();
306
+ if (!fs.existsSync(communityDir)) return new Map();
307
+ const index = new Map();
308
+ const walk = (dir) => {
309
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
310
+ const full = path.join(dir, entry.name);
311
+ if (entry.isDirectory()) walk(full);
312
+ else if (entry.name.toLowerCase().endsWith('.md')) {
313
+ try {
314
+ const parsed = matter(fs.readFileSync(full, 'utf8'));
315
+ if (parsed.data.canonical_url) index.set(parsed.data.canonical_url, full);
316
+ } catch { /* ignore */ }
317
+ }
318
+ }
319
+ };
320
+ walk(communityDir);
321
+ return index;
322
+ }
323
+
324
+ function findExisting(url, urlIndex) {
325
+ if (!urlIndex) return null;
326
+ return urlIndex.get(url) || null;
327
+ }
328
+
329
+ export function listCommunityDocs({ dbPath } = {}) {
330
+ const dbFile = dbPath || getDocsDbPath();
331
+ if (!fs.existsSync(dbFile)) return { total: 0, docs: [] };
332
+ const db = new Database(dbFile, { readonly: true });
333
+ const rows = db
334
+ .prepare(`SELECT id, path, title, doc_type, frontmatter FROM docs WHERE bundle = 'community' ORDER BY path`)
335
+ .all()
336
+ .map((r) => {
337
+ let fm = {};
338
+ try { fm = JSON.parse(r.frontmatter || '{}'); } catch { /* ignore */ }
339
+ return { id: r.id, path: r.path, title: r.title || fm.title, doc_type: r.doc_type, author: fm.author, canonical_url: fm.canonical_url };
340
+ });
341
+ db.close();
342
+ return { total: rows.length, docs: rows };
343
+ }
@@ -28,12 +28,48 @@ export function getDocsSourceMarkdownDir() {
28
28
  return path.join(getDocsSourceDir(), 'markdown');
29
29
  }
30
30
 
31
+ export function getDocsCommunityDir() {
32
+ return path.join(getDocsCacheDir(), 'community');
33
+ }
34
+
31
35
  export function ensureDocsCacheDir() {
32
36
  const dir = getDocsCacheDir();
33
37
  fs.mkdirSync(dir, { recursive: true });
34
38
  return dir;
35
39
  }
36
40
 
41
+ /**
42
+ * Walk one or more markdown source roots, yielding { file, rel } pairs.
43
+ * `rel` is the path stored in the docs table (unique). Roots are passed as
44
+ * { dir, prefix } so community content can live under a `community/` prefix
45
+ * without colliding with the ServiceNowDocs markdown tree.
46
+ */
47
+ export function* walkDocs(roots) {
48
+ function* walk(dir, root, prefix) {
49
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
50
+ const full = path.join(dir, entry.name);
51
+ if (entry.isDirectory()) {
52
+ yield* walk(full, root, prefix);
53
+ } else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) {
54
+ const rel = prefix + path.relative(root, full).split(path.sep).join('/');
55
+ yield { file: full, rel };
56
+ }
57
+ }
58
+ }
59
+ for (const root of roots) {
60
+ if (!fs.existsSync(root.dir)) continue;
61
+ yield* walk(root.dir, root.dir, root.prefix || '');
62
+ }
63
+ }
64
+
65
+ export function defaultDocsRoots(opts = {}) {
66
+ const communityDir = opts.communityDir || getDocsCommunityDir();
67
+ return [
68
+ { dir: opts.docsDir || getDocsSourceMarkdownDir(), prefix: '' },
69
+ { dir: communityDir, prefix: 'community/' },
70
+ ];
71
+ }
72
+
37
73
  export function openDocsDb(opts = {}) {
38
74
  const dbPath = opts.dbPath || getDocsDbPath();
39
75
  ensureDocsCacheDir();
@@ -13,6 +13,7 @@ import { refreshDocs } from './refresh.js';
13
13
  import { embedDocs } from './embed.js';
14
14
  import { searchDocs } from './search.js';
15
15
  import { serveDocs } from './serve.js';
16
+ import { syncCommunity, listCommunityDocs } from './community.js';
16
17
 
17
18
  function formatBytes(n) {
18
19
  if (n == null) return '-';
@@ -226,6 +227,45 @@ export function docsCmd(wrap) {
226
227
  const result = refreshDocs({ docsDir: getDocsSourceMarkdownDir() });
227
228
  app.ok(result, { summary: `Refresh complete: +${result.added} ~${result.updated} -${result.removed} =${result.unchanged}` });
228
229
  }),
230
+ })
231
+ .command({
232
+ command: 'community <action>',
233
+ describe: 'Sync ServiceNow Community articles into the searchable docs index',
234
+ builder: (y) => y
235
+ .command({
236
+ command: 'sync',
237
+ describe: 'Fetch community hub pages (+ linked articles with --expand) and index them',
238
+ builder: (yy) => yy
239
+ .option('expand', { type: 'boolean', describe: 'Also fetch every linked community article found on the hub pages' })
240
+ .option('force', { type: 'boolean', describe: 'Re-fetch articles already downloaded' })
241
+ .option('sources', { type: 'string', describe: 'Path to a community-sources.json (default: bundled)' })
242
+ .option('delay-ms', { type: 'number', describe: 'Delay between article fetches (default: 500)' }),
243
+ handler: wrap(async (argv, app) => {
244
+ const result = await syncCommunity({
245
+ sourcesFile: argv.sources,
246
+ expand: argv.expand === true,
247
+ force: argv.force === true,
248
+ delayMs: argv['delay-ms'],
249
+ });
250
+ app.ok(result, {
251
+ summary: `Community sync: ${result.hubs.length} hub(s), ${result.articles.length} article(s), ${result.skipped.length} skipped, ${result.errors.length} error(s)`,
252
+ });
253
+ }),
254
+ })
255
+ .command({
256
+ command: 'list',
257
+ describe: 'List community docs currently in the index',
258
+ handler: wrap(async (argv, app) => {
259
+ const result = listCommunityDocs();
260
+ const lines = result.docs.map((d) => `[${d.id}] ${d.title || d.path} (${d.doc_type})`);
261
+ app.ok({
262
+ _formatted: lines.length ? lines.join('\n') : '(no community docs indexed yet)',
263
+ total: result.total,
264
+ docs: result.docs,
265
+ }, { summary: `${result.total} community doc(s) indexed` });
266
+ }),
267
+ })
268
+ .demandCommand(1, 'Specify an action: sync or list'),
229
269
  });
230
270
  },
231
271
  handler: () => {
@@ -236,6 +276,7 @@ export function docsCmd(wrap) {
236
276
  process.stdout.write(' search Full-text + semantic search across the local docs index\n');
237
277
  process.stdout.write(' show Display a full documentation page by id or path\n');
238
278
  process.stdout.write(' serve Start a local web UI for browsing and searching docs\n');
279
+ process.stdout.write(' community Sync ServiceNow Community articles (hubs + linked posts) into the index\n');
239
280
  process.stdout.write('\n');
240
281
  process.stdout.write('Tip: use "jsn docs serve --expose" to bind to 0.0.0.0 and share on your network.\n');
241
282
  process.stdout.write('\n');
@@ -1,20 +1,15 @@
1
1
  // Ingest ServiceNow markdown docs into a SQLite database with FTS5 full-text search.
2
2
 
3
3
  import fs from 'node:fs';
4
- import path from 'node:path';
5
4
  import { createHash } from 'node:crypto';
6
5
  import matter from 'gray-matter';
7
- import { openDocsDb, closeDocsDb, initDocsSchema, getDocsDbPath } from './db.js';
6
+ import { openDocsDb, closeDocsDb, initDocsSchema, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
8
7
  import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
9
8
 
10
9
  export function ingestDocs(opts = {}) {
11
- const docsDir = opts.docsDir;
12
10
  const dbPath = opts.dbPath || getDocsDbPath();
13
11
  const embed = opts.embed !== false;
14
-
15
- if (!fs.existsSync(docsDir)) {
16
- throw new Error(`Docs directory not found: ${docsDir}`);
17
- }
12
+ const roots = opts.roots || defaultDocsRoots(opts);
18
13
 
19
14
  // Fresh build each run.
20
15
  if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
@@ -27,20 +22,12 @@ export function ingestDocs(opts = {}) {
27
22
  VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
28
23
  `);
29
24
 
30
- function* walk(dir) {
31
- for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
32
- const full = path.join(dir, entry.name);
33
- if (entry.isDirectory()) yield* walk(full);
34
- else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) yield full;
35
- }
36
- }
37
-
38
25
  let count = 0;
39
26
  let errors = 0;
40
27
  const startedAt = Date.now();
41
28
 
42
29
  db.exec('BEGIN');
43
- for (const file of [...walk(docsDir)]) {
30
+ for (const { file, rel } of walkDocs(roots)) {
44
31
  try {
45
32
  const stat = fs.statSync(file);
46
33
  const raw = fs.readFileSync(file, 'utf8');
@@ -58,7 +45,7 @@ export function ingestDocs(opts = {}) {
58
45
  ? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
59
46
  : null;
60
47
  insert.run({
61
- path: path.relative(docsDir, file).split(path.sep).join('/'),
48
+ path: rel,
62
49
  title: data.title ?? null,
63
50
  release: data.release ?? null,
64
51
  bundle: data.bundle ?? null,
@@ -2,22 +2,18 @@
2
2
  // after a ServiceNow docs update (e.g. `git pull` in the docs repo).
3
3
 
4
4
  import fs from 'node:fs';
5
- import path from 'node:path';
6
5
  import { createHash } from 'node:crypto';
7
6
  import matter from 'gray-matter';
8
- import { openDocsDb, closeDocsDb, getDocsDbPath } from './db.js';
7
+ import { openDocsDb, closeDocsDb, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
9
8
  import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
10
9
 
11
10
  export function refreshDocs(opts = {}) {
12
- const docsDir = opts.docsDir;
13
11
  const dbPath = opts.dbPath || getDocsDbPath();
12
+ const roots = opts.roots || defaultDocsRoots(opts);
14
13
 
15
14
  if (!fs.existsSync(dbPath)) {
16
15
  throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first for the initial build.`);
17
16
  }
18
- if (!fs.existsSync(docsDir)) {
19
- throw new Error(`Docs directory not found: ${docsDir}`);
20
- }
21
17
 
22
18
  const db = openDocsDb({ dbPath });
23
19
  db.exec('PRAGMA journal_mode = WAL');
@@ -34,14 +30,6 @@ export function refreshDocs(opts = {}) {
34
30
  && db.prepare('SELECT COUNT(*) AS n FROM docs').get().n > 0;
35
31
  const setHash = db.prepare('UPDATE docs SET hash = ? WHERE id = ?');
36
32
 
37
- function* walk(dir) {
38
- for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
39
- const full = path.join(dir, entry.name);
40
- if (entry.isDirectory()) yield* walk(full);
41
- else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) yield full;
42
- }
43
- }
44
-
45
33
  const existing = new Map();
46
34
  for (const row of db.prepare('SELECT id, path, hash FROM docs').all()) {
47
35
  existing.set(row.path, { id: row.id, hash: row.hash });
@@ -60,12 +48,11 @@ export function refreshDocs(opts = {}) {
60
48
  const seen = new Set();
61
49
  const started = Date.now();
62
50
 
63
- const files = [...walk(docsDir)];
51
+ const files = [...walkDocs(roots)];
64
52
 
65
53
  db.exec('BEGIN');
66
- for (const file of files) {
54
+ for (const { file, rel } of files) {
67
55
  try {
68
- const rel = path.relative(docsDir, file).split(path.sep).join('/');
69
56
  seen.add(rel);
70
57
  const raw = fs.readFileSync(file, 'utf8');
71
58
  const hash = createHash('sha256').update(raw).digest('hex');
@@ -46,7 +46,7 @@ export function searchDocs(opts = {}) {
46
46
 
47
47
  // Total matching docs (for pagination)
48
48
  const total = db
49
- .prepare(`SELECT COUNT(*) AS n FROM docs_fts WHERE docs_fts MATCH @q${where}`)
49
+ .prepare(`SELECT COUNT(*) AS n FROM docs_fts JOIN docs d ON d.id = docs_fts.rowid WHERE docs_fts MATCH @q${where}`)
50
50
  .get({ q, ...(bundle && { bundle }), ...(docType && { docType }) }).n;
51
51
 
52
52
  let rows;
@@ -1,4 +1,4 @@
1
- import { formatRecordForDisplay, getStringField, resolveFieldsParam } from '../helpers.js';
1
+ import { formatRecordForDisplay, getStringField, resolveFieldsParam, assertSafeExactMatch } from '../helpers.js';
2
2
 
3
3
  export function groupRolesCmd(wrap) {
4
4
  return {
@@ -42,6 +42,8 @@ export function groupRolesCmd(wrap) {
42
42
  handler: wrap(async (argv, app) => {
43
43
  const groupName = argv.group;
44
44
  const roleName = argv.role;
45
+ assertSafeExactMatch(groupName);
46
+ assertSafeExactMatch(roleName);
45
47
 
46
48
  // Resolve group sys_id
47
49
  const isGroupSysID = groupName.length === 32 && /^[0-9a-fA-F]+$/.test(groupName);
@@ -100,6 +102,8 @@ export function groupRolesCmd(wrap) {
100
102
  handler: wrap(async (argv, app) => {
101
103
  const groupName = argv.group;
102
104
  const roleName = argv.role;
105
+ assertSafeExactMatch(groupName);
106
+ assertSafeExactMatch(roleName);
103
107
 
104
108
  // Resolve group sys_id
105
109
  const isGroupSysID = groupName.length === 32 && /^[0-9a-fA-F]+$/.test(groupName);
@@ -1,4 +1,4 @@
1
- import { formatRecordForDisplay, getStringField, interactiveList, resolveFieldsParam } from '../helpers.js';
1
+ import { formatRecordForDisplay, getStringField, interactiveList, resolveFieldsParam, assertSafeExactMatch } from '../helpers.js';
2
2
 
3
3
  export function groupsCmd(wrap) {
4
4
  return {
@@ -32,13 +32,13 @@ export function groupsCmd(wrap) {
32
32
  cp.set('sysparm_query', `group=${sysID}`); cp.set('sysparm_limit', '0'); cp.set('sysparm_fields', 'sys_id');
33
33
  const members = await app.sdk.list('sys_user_grmember', cp);
34
34
  memberCount = members.length;
35
- } catch {}
35
+ } catch { /* non-critical */ }
36
36
  try {
37
37
  const rp = new URLSearchParams();
38
38
  rp.set('sysparm_query', `group=${sysID}`); rp.set('sysparm_limit', '0'); rp.set('sysparm_fields', 'sys_id');
39
39
  const roles = await app.sdk.list('sys_group_has_role', rp);
40
40
  roleCount = roles.length;
41
- } catch {}
41
+ } catch { /* non-critical */ }
42
42
 
43
43
  picked._context = { instance_url: app.getEffectiveInstance(), table: 'sys_user_group' };
44
44
  return app.ok(picked, {
@@ -72,6 +72,7 @@ export function groupsCmd(wrap) {
72
72
  describe: 'Show a group by name or sys_id',
73
73
  handler: wrap(async (argv, app) => {
74
74
  const id = argv.name;
75
+ assertSafeExactMatch(id);
75
76
  const isSysID = id.length === 32 && /^[0-9a-fA-F]+$/.test(id);
76
77
  const params = new URLSearchParams();
77
78
  params.set('sysparm_query', isSysID ? `sys_id=${id}` : `name=${id}`);
@@ -117,6 +118,7 @@ export function groupsCmd(wrap) {
117
118
  .option('data', { type: 'string', demandOption: true, describe: 'JSON data to update' }),
118
119
  handler: wrap(async (argv, app) => {
119
120
  const id = argv.name;
121
+ assertSafeExactMatch(id);
120
122
  const recordData = JSON.parse(argv.data);
121
123
  const isSysID = id.length === 32 && /^[0-9a-fA-F]+$/.test(id);
122
124
  const params = new URLSearchParams();
@@ -142,6 +144,7 @@ export function groupsCmd(wrap) {
142
144
  describe: 'Delete a group by name or sys_id',
143
145
  handler: wrap(async (argv, app) => {
144
146
  const id = argv.name;
147
+ assertSafeExactMatch(id);
145
148
  const isSysID = id.length === 32 && /^[0-9a-fA-F]+$/.test(id);
146
149
  const params = new URLSearchParams();
147
150
  params.set('sysparm_query', isSysID ? `sys_id=${id}` : `name=${id}`);
@@ -1,4 +1,4 @@
1
- import { formatRecordForDisplay, buildQuerySuffix, parseDataArg, getStringField, interactiveList, resolveFieldsParam, checkDerivedFields } from '../helpers.js';
1
+ import { formatRecordForDisplay, buildQuerySuffix, parseDataArg, getStringField, interactiveList, resolveFieldsParam, checkDerivedFields, confirmDelete } from '../helpers.js';
2
2
 
3
3
  const tableDefaultColumns = {
4
4
  incident: ['number', 'short_description', 'priority', 'state', 'assigned_to'],
@@ -115,6 +115,7 @@ export function recordsCmd(wrap) {
115
115
  .option('sys-id', { type: 'string', demandOption: true, describe: 'Record sys_id' })
116
116
  .option('columns', { alias: ['c', 'fields'], type: 'string', describe: 'Comma-separated columns (e.g. "number,short_description")' }),
117
117
  handler: wrap(async (argv, app) => {
118
+ assertSafeExactMatch(argv['sys-id']);
118
119
  const params = new URLSearchParams();
119
120
  params.set('sysparm_query', `sys_id=${argv['sys-id']}`);
120
121
  params.set('sysparm_limit', '1');
@@ -169,8 +170,10 @@ export function recordsCmd(wrap) {
169
170
  describe: 'Delete a record',
170
171
  builder: (y) => y
171
172
  .option('table', { type: 'string', demandOption: true, describe: 'Table name' })
172
- .option('sys-id', { type: 'string', demandOption: true, describe: 'Record sys_id' }),
173
+ .option('sys-id', { type: 'string', demandOption: true, describe: 'Record sys_id' })
174
+ .option('force', { type: 'boolean', default: false, describe: 'Skip confirmation' }),
173
175
  handler: wrap(async (argv, app) => {
176
+ await confirmDelete(app, argv, `Delete record from ${argv.table} (${argv['sys-id']})`);
174
177
  await app.sdk.delete(argv.table, argv['sys-id']);
175
178
  app.ok({ message: 'Record deleted', table: argv.table, sys_id: argv['sys-id'] }, { summary: `Deleted record from ${argv.table}` });
176
179
  }),
@@ -1,5 +1,5 @@
1
1
  import { buildTicketCommands } from './_ticket.js';
2
- import { getStringField } from '../helpers.js';
2
+ import { getStringField, assertSafeExactMatch } from '../helpers.js';
3
3
 
4
4
  const requestDefaultColumns = ['number', 'short_description', 'stage', 'assigned_to'];
5
5
 
@@ -25,6 +25,7 @@ export function requestsCmd(wrap) {
25
25
 
26
26
  async function enrichedShowHandler(argv, app) {
27
27
  const number = argv.number;
28
+ assertSafeExactMatch(number);
28
29
  const params = new URLSearchParams();
29
30
  params.set('sysparm_query', `number=${number}`);
30
31
  params.set('sysparm_display_value', 'all');