@jacebenson/jsn 3.4.3 → 4.0.1

package/src/commands/dev/forms.js

@@ -22,15 +22,6 @@ function getIntValue(record, key) {
   return 0;
 }
 
-function getBoolValue(record, key) {
-  if (!record || typeof record !== 'object') return false;
-  const val = record[key];
-  if (typeof val === 'boolean') return val;
-  if (typeof val === 'string') return val === 'true';
-  if (typeof val === 'object' && val.value != null) return String(val.value) === 'true';
-  return false;
-}
-
 async function showForm(app, table, viewName) {
   // 1) Look up the view
   const viewParams = new URLSearchParams();
@@ -88,8 +79,6 @@ async function showForm(app, table, viewName) {
 
   sectionsOut.sort((a, b) => a.position - b.position);
 
-  const totalElements = sectionsOut.reduce((sum, s) => sum + s.elements.length, 0);
-
   // Build formatted output
   const lines = [];
   lines.push(`Form: ${table} → ${viewName}`);

package/src/commands/dev/import.js

@@ -20,7 +20,7 @@ export const importCmd = (wrap) => buildDevCmd('import', 'sys_import_set', ['imp
         const rows = await app.sdk.list('sys_import_set_row', rp);
         console.log(`  Import rows: ${rows.length}`);
         console.log(`    → jsn records list --table sys_import_set_row --query "sys_import_set=${sysID}" --limit 200`);
-      } catch (err) {
+      } catch {
         console.log(`  Import rows: (unavailable)`);
       }
     },

package/src/commands/dev/lists.js

@@ -17,7 +17,7 @@ export function listsCmd(wrap) {
           handler: wrap(async (argv, app) => {
             app.requireInstance();
             let table = argv.table;
-            let viewSysID = null;
+            let viewSysID;
 
             // If no table, pick one interactively
             if (!table) {
@@ -76,7 +76,7 @@ export function listsCmd(wrap) {
             const table = argv.table;
 
             // Resolve view sys_id
-            let viewSysID = '';
+            let viewSysID;
             const vp = new URLSearchParams();
             vp.set('sysparm_limit', '1'); vp.set('sysparm_fields', 'sys_id');
             vp.set('sysparm_query', `name=${argv.view}^name=${table}`);

package/src/commands/dev/updatesets.js

@@ -164,21 +164,6 @@ export function updateSetsCmd(wrap) {
             });
           }),
         })
-        .command({
-          command: 'yolo',
-          aliases: ['silence'],
-          describe: 'Silence the "Default update set" warning for this profile',
-          handler: wrap(async (argv, app) => {
-            const name = app.context.profileName;
-            if (!name) {
-              throw new Error('No active profile to mark as yolo. Run jsn setup first.');
-            }
-            app.config.profiles[name].yolo = true;
-            const { saveConfig } = await import('../../config.js');
-            saveConfig(app.config);
-            app.ok({ profile: name, yolo: true }, { summary: 'Default update set warning silenced for this profile' });
-          }),
-        })
         .command({
           command: 'export <name>',
           describe: 'Export an update set to XML',
@@ -225,7 +210,6 @@ export function updateSetsCmd(wrap) {
         console.log('  create         Create a new update set (auto-sets as current)');
         console.log('  export <name>  Export an update set to XML');
         console.log('  complete <name>  Mark an update set as complete (coming soon)');
-        console.log('  yolo           Silence the "Default update set" warning');
         console.log('\nRun "jsn updatesets <command> --help" for details.');
         console.log('\nTip: Create an update set first:');
         console.log('  jsn updatesets create --name "My Feature"');

package/src/commands/docs/community-sources.json

@@ -0,0 +1,18 @@
+{
+  "sources": [
+    {
+      "name": "maik-skoddow-knowledge-sources",
+      "author": "Maik Skoddow",
+      "author_url": "https://www.servicenow.com/community/user/viewprofilepage/user-id/116698",
+      "url": "https://www.servicenow.com/community/training-and-certifications/knowledge-sources-to-go/ta-p/2310130",
+      "expand": true
+    },
+    {
+      "name": "mark-roethof-experiences",
+      "author": "Mark Roethof",
+      "author_url": "https://www.servicenow.com/community/user/viewprofilepage/user-id/201738",
+      "url": "https://www.servicenow.com/community/developer-blog/456-articles-blogs-videos-podcasts-share-projects-experiences/ba-p/2292127",
+      "expand": true
+    }
+  ]
+}

package/src/commands/docs/community.js

@@ -0,0 +1,343 @@
+// Fetch ServiceNow Community pages, convert them to markdown, and index them
+// into the local docs DB alongside the ServiceNowDocs clone.
+//
+// Community content is written to <cache>/servicenow-cli/docs/community/ as
+// markdown with frontmatter, then picked up by ingest/refresh via walkDocs().
+
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import TurndownService from 'turndown';
+import matter from 'gray-matter';
+import Database from 'better-sqlite3';
+import { getDocsCommunityDir, getDocsDbPath, docsDbExists } from './db.js';
+import { refreshDocs } from './refresh.js';
+import { ingestDocs } from './ingest.js';
+
+const DEFAULT_SOURCES_FILE = path.join(import.meta.dirname, 'community-sources.json');
+const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36';
+
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+  bulletListMarker: '-',
+});
+
+// --- Extraction helpers (pure, exported for tests) ---
+
+export function extractBodyHtml(html) {
+  const marker = 'lia-message-body-content';
+  const start = html.indexOf(marker);
+  if (start === -1) return null;
+  const open = html.indexOf('>', start) + 1;
+  if (open <= 0) return null;
+  // Depth-count the <div> nesting from the body content element.
+  let depth = 1;
+  let i = open;
+  while (i < html.length && depth > 0) {
+    const nextOpen = html.indexOf('<div', i);
+    const nextClose = html.indexOf('</div>', i);
+    if (nextOpen === -1 && nextClose === -1) break;
+    if (nextClose === -1 || (nextOpen !== -1 && nextOpen < nextClose)) {
+      depth++;
+      i = nextOpen + 4;
+    } else {
+      depth--;
+      i = nextClose + 6;
+    }
+  }
+  return html.slice(open, i);
+}
+
+export function extractMeta(html) {
+  const title = (() => {
+    const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/);
+    if (h1) return stripTags(h1[1]).trim();
+    const t = html.match(/<title>([\s\S]*?)<\/title>/);
+    return t ? t[1].trim().replace(/\s*-\s*ServiceNow Community\s*$/i, '') : null;
+  })();
+
+  const author = (() => {
+    const m = html.match(/<span\s+content="([^"]+)"\s+itemprop="name"[^>]*>/);
+    return m ? m[1] : null;
+  })();
+
+  const authorUrl = (() => {
+    const m = html.match(/class="[^"]*lia-user-name-link[^"]*"[^>]*href="([^"]+)"/);
+    return m ? m[1] : null;
+  })();
+
+  return { title, author, authorUrl };
+}
+
+export function stripTags(s) {
+  return decodeEntities(String(s).replace(/<[^>]*>/g, ''));
+}
+
+function decodeEntities(s) {
+  return s
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ');
+}
+
+// --- HTML -> markdown ---
+
+export function htmlToMarkdown(html) {
+  return turndown.turndown(html).replace(/\n{3,}/g, '\n\n').trim() + '\n';
+}
+
+// --- Community page fetch ---
+
+async function fetchPage(url, { retries = 3, timeoutMs = 30000 } = {}) {
+  let lastErr;
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const res = await fetch(url, {
+        headers: { 'user-agent': UA, accept: 'text/html' },
+        signal: controller.signal,
+        redirect: 'follow',
+      });
+      if (res.status === 429) {
+        const wait = 5000 * (attempt + 1);
+        process.stderr.write(`  rate limited (429), waiting ${wait / 1000}s...\n`);
+        await sleep(wait);
+        continue;
+      }
+      if (res.status >= 400) {
+        lastErr = new Error(`HTTP ${res.status} for ${url}`);
+        continue;
+      }
+      return await res.text();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < retries) await sleep(1500 * (attempt + 1));
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+  throw lastErr;
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// --- Link extraction for --expand ---
+
+export function extractCommunityLinks(html, baseUrl) {
+  const bodyHtml = extractBodyHtml(html) || html;
+  const out = new Set();
+  const re = /href="([^"]+)"/g;
+  let m;
+  while ((m = re.exec(bodyHtml))) {
+    let u = m[1].replace(/&amp;/g, '&');
+    if (u.startsWith('#')) continue;
+    try {
+      u = new URL(u, baseUrl).toString();
+    } catch {
+      continue;
+    }
+    // Only new Khoros community article/blog pages: /community/<section>/<slug>/t[abdp]-p/<id>
+    const km = u.match(/^https?:\/\/www\.servicenow\.com\/community\/[a-z0-9-]+\/[^/]+\/t[abdp]-p\/\d+/);
+    if (!km) continue;
+    // Strip query/hash for dedupe.
+    const clean = km[0];
+    if (/\/user\/|viewprofilepage|notifymoderator|kudos|printpage|\/s\//.test(clean)) continue;
+    out.add(clean);
+  }
+  return [...out];
+}
+
+// --- File naming ---
+
+export function slugify(title, fallback = 'untitled') {
+  const s = String(title || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80);
+  return s || fallback;
+}
+
+function frontmatter(body, data) {
+  return matter.stringify(body, data);
+}
+
+// --- Write a community doc to disk ---
+
+export function writeCommunityDoc({ author, title, url, bodyMd, docType, sourceUrl, published, fetchedAt, communityDir } = {}) {
+  const baseDir = communityDir || getDocsCommunityDir();
+  const authorDir = path.join(baseDir, slugify(author, 'unknown'));
+  fs.mkdirSync(authorDir, { recursive: true });
+  // Include the post id (t[abdp]-p/<id>) so identical titles across posts
+  // never collide on disk. e.g. 456-articles-blogs-videos.../ba-p/2292127
+  const id = (url.match(/\/(t[abdp]-p\/\d+)\/?$/) || [])[1] || null;
+  const idPart = id ? id.replace(/[^a-z0-9]+/g, '-') : null;
+  const titleSlug = slugify(title, 'untitled');
+  const slug = idPart ? `${titleSlug}-${idPart}` : titleSlug;
+  const rel = path.join(authorDir, `${slug}.md`);
+  const fm = {
+    title,
+    author,
+    bundle: 'community',
+    doc_type: docType,
+    canonical_url: url,
+    ...(sourceUrl ? { source_url: sourceUrl } : {}),
+    ...(published ? { published } : {}),
+    ...(fetchedAt ? { fetched_at: fetchedAt } : {}),
+  };
+  const md = frontmatter(bodyMd, fm);
+  fs.writeFileSync(rel, md);
+  return { file: rel, rel: path.relative(baseDir, rel).split(path.sep).join('/') };
+}
+
+// --- Sync entry points ---
+
+function loadSources(sourcesFile) {
+  const file = sourcesFile || DEFAULT_SOURCES_FILE;
+  if (!fs.existsSync(file)) {
+    throw new Error(`Community sources file not found: ${file}`);
+  }
+  const data = JSON.parse(fs.readFileSync(file, 'utf8'));
+  if (!Array.isArray(data.sources)) {
+    throw new Error(`Invalid community sources file ${file}: expected { "sources": [...] }`);
+  }
+  return data.sources;
+}
+
+export async function syncCommunity(opts = {}) {
+  const sources = loadSources(opts.sourcesFile);
+  const expand = opts.expand === true;
+  const force = opts.force === true;
+  const delayMs = opts.delayMs != null ? opts.delayMs : 500;
+  const dbPath = opts.dbPath || getDocsDbPath();
+
+  const results = { hubs: [], articles: [], skipped: [], errors: [] };
+
+  for (const source of sources) {
+    const url = source.url || source.hub_url;
+    if (!url) {
+      results.errors.push({ source: source.name || '?', error: 'missing url' });
+      continue;
+    }
+    process.stderr.write(`Fetching hub: ${url}\n`);
+    try {
+      const html = await fetchPage(url);
+      const meta = extractMeta(html);
+      const title = meta.title || source.name || url;
+      const author = meta.author || source.author || 'ServiceNow Community';
+      const authorUrl = meta.authorUrl || source.author_url;
+      const bodyHtml = extractBodyHtml(html);
+      if (!bodyHtml) {
+        results.errors.push({ url, error: 'no article body found' });
+        continue;
+      }
+      const bodyMd = htmlToMarkdown(bodyHtml);
+      const written = writeCommunityDoc({
+        author,
+        title,
+        url,
+        bodyMd,
+        docType: 'hub',
+        fetchedAt: new Date().toISOString(),
+      });
+      results.hubs.push({ title, author, authorUrl, url, file: written.rel });
+
+      if (expand) {
+        const links = extractCommunityLinks(html, url);
+        const urlIndex = buildUrlIndex();
+        process.stderr.write(`  ${links.length} linked article(s) found\n`);
+        for (const link of links) {
+          // Skip already-downloaded files unless --force.
+          const existing = findExisting(link, urlIndex);
+          if (existing && !force) {
+            results.skipped.push({ url: link, reason: 'already downloaded' });
+            continue;
+          }
+          process.stderr.write(`  Fetching: ${link}\n`);
+          try {
+            const pageHtml = await fetchPage(link);
+            const pageMeta = extractMeta(pageHtml);
+            const pageBody = extractBodyHtml(pageHtml);
+            if (!pageBody) {
+              results.skipped.push({ url: link, reason: 'no article body' });
+              continue;
+            }
+            const md = htmlToMarkdown(pageBody);
+            const wrote = writeCommunityDoc({
+              author: pageMeta.author || author,
+              title: pageMeta.title || link,
+              url: link,
+              bodyMd: md,
+              docType: 'article',
+              sourceUrl: url,
+              fetchedAt: new Date().toISOString(),
+            });
+            results.articles.push({ title: pageMeta.title, url: link, file: wrote.rel });
+          } catch (err) {
+            results.errors.push({ url: link, error: err.message });
+          }
+          if (delayMs > 0) await sleep(delayMs);
+        }
+      }
+      if (delayMs > 0) await sleep(delayMs);
+    } catch (err) {
+      results.errors.push({ url, error: err.message });
+    }
+  }
+
+  // Refresh the index so new community docs are searchable. Fall back to a
+  // full ingest when no DB exists yet.
+  const refreshResult = docsDbExists()
+    ? refreshDocs({ dbPath })
+    : ingestDocs({ dbPath, roots: [{ dir: getDocsCommunityDir(), prefix: 'community/' }], embed: true });
+
+  return { ...results, refresh: refreshResult };
+}
+
+function buildUrlIndex() {
+  const communityDir = getDocsCommunityDir();
+  if (!fs.existsSync(communityDir)) return new Map();
+  const index = new Map();
+  const walk = (dir) => {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) walk(full);
+      else if (entry.name.toLowerCase().endsWith('.md')) {
+        try {
+          const parsed = matter(fs.readFileSync(full, 'utf8'));
+          if (parsed.data.canonical_url) index.set(parsed.data.canonical_url, full);
+        } catch { /* ignore */ }
+      }
+    }
+  };
+  walk(communityDir);
+  return index;
+}
+
+function findExisting(url, urlIndex) {
+  if (!urlIndex) return null;
+  return urlIndex.get(url) || null;
+}
+
+export function listCommunityDocs({ dbPath } = {}) {
+  const dbFile = dbPath || getDocsDbPath();
+  if (!fs.existsSync(dbFile)) return { total: 0, docs: [] };
+  const db = new Database(dbFile, { readonly: true });
+  const rows = db
+    .prepare(`SELECT id, path, title, doc_type, frontmatter FROM docs WHERE bundle = 'community' ORDER BY path`)
+    .all()
+    .map((r) => {
+      let fm = {};
+      try { fm = JSON.parse(r.frontmatter || '{}'); } catch { /* ignore */ }
+      return { id: r.id, path: r.path, title: r.title || fm.title, doc_type: r.doc_type, author: fm.author, canonical_url: fm.canonical_url };
+    });
+  db.close();
+  return { total: rows.length, docs: rows };
+}

package/src/commands/docs/db.js

@@ -3,7 +3,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
-import { DatabaseSync } from 'node:sqlite';
+import Database from 'better-sqlite3';
 
 const DOCS_DIR_NAME = 'servicenow-cli/docs';
 
@@ -28,16 +28,52 @@ export function getDocsSourceMarkdownDir() {
   return path.join(getDocsSourceDir(), 'markdown');
 }
 
+export function getDocsCommunityDir() {
+  return path.join(getDocsCacheDir(), 'community');
+}
+
 export function ensureDocsCacheDir() {
   const dir = getDocsCacheDir();
   fs.mkdirSync(dir, { recursive: true });
   return dir;
 }
 
+/**
+ * Walk one or more markdown source roots, yielding { file, rel } pairs.
+ * `rel` is the path stored in the docs table (unique). Roots are passed as
+ * { dir, prefix } so community content can live under a `community/` prefix
+ * without colliding with the ServiceNowDocs markdown tree.
+ */
+export function* walkDocs(roots) {
+  function* walk(dir, root, prefix) {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        yield* walk(full, root, prefix);
+      } else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) {
+        const rel = prefix + path.relative(root, full).split(path.sep).join('/');
+        yield { file: full, rel };
+      }
+    }
+  }
+  for (const root of roots) {
+    if (!fs.existsSync(root.dir)) continue;
+    yield* walk(root.dir, root.dir, root.prefix || '');
+  }
+}
+
+export function defaultDocsRoots(opts = {}) {
+  const communityDir = opts.communityDir || getDocsCommunityDir();
+  return [
+    { dir: opts.docsDir || getDocsSourceMarkdownDir(), prefix: '' },
+    { dir: communityDir, prefix: 'community/' },
+  ];
+}
+
 export function openDocsDb(opts = {}) {
   const dbPath = opts.dbPath || getDocsDbPath();
   ensureDocsCacheDir();
-  const db = new DatabaseSync(dbPath);
+  const db = new Database(dbPath);
   db.exec('PRAGMA journal_mode = WAL');
   return db;
 }

package/src/commands/docs/docs.js

@@ -13,6 +13,7 @@ import { refreshDocs } from './refresh.js';
 import { embedDocs } from './embed.js';
 import { searchDocs } from './search.js';
 import { serveDocs } from './serve.js';
+import { syncCommunity, listCommunityDocs } from './community.js';
 
 function formatBytes(n) {
   if (n == null) return '-';
@@ -226,6 +227,45 @@ export function docsCmd(wrap) {
             const result = refreshDocs({ docsDir: getDocsSourceMarkdownDir() });
             app.ok(result, { summary: `Refresh complete: +${result.added} ~${result.updated} -${result.removed} =${result.unchanged}` });
           }),
+        })
+        .command({
+          command: 'community <action>',
+          describe: 'Sync ServiceNow Community articles into the searchable docs index',
+          builder: (y) => y
+            .command({
+              command: 'sync',
+              describe: 'Fetch community hub pages (+ linked articles with --expand) and index them',
+              builder: (yy) => yy
+                .option('expand', { type: 'boolean', describe: 'Also fetch every linked community article found on the hub pages' })
+                .option('force', { type: 'boolean', describe: 'Re-fetch articles already downloaded' })
+                .option('sources', { type: 'string', describe: 'Path to a community-sources.json (default: bundled)' })
+                .option('delay-ms', { type: 'number', describe: 'Delay between article fetches (default: 500)' }),
+              handler: wrap(async (argv, app) => {
+                const result = await syncCommunity({
+                  sourcesFile: argv.sources,
+                  expand: argv.expand === true,
+                  force: argv.force === true,
+                  delayMs: argv['delay-ms'],
+                });
+                app.ok(result, {
+                  summary: `Community sync: ${result.hubs.length} hub(s), ${result.articles.length} article(s), ${result.skipped.length} skipped, ${result.errors.length} error(s)`,
+                });
+              }),
+            })
+            .command({
+              command: 'list',
+              describe: 'List community docs currently in the index',
+              handler: wrap(async (argv, app) => {
+                const result = listCommunityDocs();
+                const lines = result.docs.map((d) => `[${d.id}] ${d.title || d.path} (${d.doc_type})`);
+                app.ok({
+                  _formatted: lines.length ? lines.join('\n') : '(no community docs indexed yet)',
+                  total: result.total,
+                  docs: result.docs,
+                }, { summary: `${result.total} community doc(s) indexed` });
+              }),
+            })
+            .demandCommand(1, 'Specify an action: sync or list'),
         });
     },
     handler: () => {
@@ -236,6 +276,7 @@ export function docsCmd(wrap) {
       process.stdout.write('  search    Full-text + semantic search across the local docs index\n');
       process.stdout.write('  show      Display a full documentation page by id or path\n');
       process.stdout.write('  serve     Start a local web UI for browsing and searching docs\n');
+      process.stdout.write('  community Sync ServiceNow Community articles (hubs + linked posts) into the index\n');
       process.stdout.write('\n');
       process.stdout.write('Tip: use "jsn docs serve --expose" to bind to 0.0.0.0 and share on your network.\n');
       process.stdout.write('\n');

package/src/commands/docs/ingest.js

@@ -1,20 +1,15 @@
 // Ingest ServiceNow markdown docs into a SQLite database with FTS5 full-text search.
 
 import fs from 'node:fs';
-import path from 'node:path';
 import { createHash } from 'node:crypto';
 import matter from 'gray-matter';
-import { openDocsDb, closeDocsDb, initDocsSchema, getDocsDbPath } from './db.js';
+import { openDocsDb, closeDocsDb, initDocsSchema, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
 import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
 
 export function ingestDocs(opts = {}) {
-  const docsDir = opts.docsDir;
   const dbPath = opts.dbPath || getDocsDbPath();
   const embed = opts.embed !== false;
-
-  if (!fs.existsSync(docsDir)) {
-    throw new Error(`Docs directory not found: ${docsDir}`);
-  }
+  const roots = opts.roots || defaultDocsRoots(opts);
 
   // Fresh build each run.
   if (fs.existsSync(dbPath)) fs.rmSync(dbPath);
@@ -27,20 +22,12 @@ export function ingestDocs(opts = {}) {
     VALUES (@path, @title, @release, @bundle, @doc_type, @locale, @frontmatter, @body, @size, @mtime, @hash, @hrr_vector)
   `);
 
-  function* walk(dir) {
-    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-      const full = path.join(dir, entry.name);
-      if (entry.isDirectory()) yield* walk(full);
-      else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) yield full;
-    }
-  }
-
   let count = 0;
   let errors = 0;
   const startedAt = Date.now();
 
   db.exec('BEGIN');
-  for (const file of [...walk(docsDir)]) {
+  for (const { file, rel } of walkDocs(roots)) {
     try {
       const stat = fs.statSync(file);
       const raw = fs.readFileSync(file, 'utf8');
@@ -58,7 +45,7 @@ export function ingestDocs(opts = {}) {
         ? phasesToBytes(encodeText(docSurface(data.title ?? null, body), DEFAULT_DIM))
         : null;
       insert.run({
-        path: path.relative(docsDir, file).split(path.sep).join('/'),
+        path: rel,
         title: data.title ?? null,
         release: data.release ?? null,
         bundle: data.bundle ?? null,

package/src/commands/docs/refresh.js

@@ -2,22 +2,18 @@
 // after a ServiceNow docs update (e.g. `git pull` in the docs repo).
 
 import fs from 'node:fs';
-import path from 'node:path';
 import { createHash } from 'node:crypto';
 import matter from 'gray-matter';
-import { openDocsDb, closeDocsDb, getDocsDbPath } from './db.js';
+import { openDocsDb, closeDocsDb, getDocsDbPath, walkDocs, defaultDocsRoots } from './db.js';
 import { encodeText, phasesToBytes, docSurface, DEFAULT_DIM } from './hrr.js';
 
 export function refreshDocs(opts = {}) {
-  const docsDir = opts.docsDir;
   const dbPath = opts.dbPath || getDocsDbPath();
+  const roots = opts.roots || defaultDocsRoots(opts);
 
   if (!fs.existsSync(dbPath)) {
     throw new Error(`Database not found: ${dbPath}. Run "jsn docs sync" first for the initial build.`);
   }
-  if (!fs.existsSync(docsDir)) {
-    throw new Error(`Docs directory not found: ${docsDir}`);
-  }
 
   const db = openDocsDb({ dbPath });
   db.exec('PRAGMA journal_mode = WAL');
@@ -34,14 +30,6 @@ export function refreshDocs(opts = {}) {
     && db.prepare('SELECT COUNT(*) AS n FROM docs').get().n > 0;
   const setHash = db.prepare('UPDATE docs SET hash = ? WHERE id = ?');
 
-  function* walk(dir) {
-    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-      const full = path.join(dir, entry.name);
-      if (entry.isDirectory()) yield* walk(full);
-      else if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) yield full;
-    }
-  }
-
   const existing = new Map();
   for (const row of db.prepare('SELECT id, path, hash FROM docs').all()) {
     existing.set(row.path, { id: row.id, hash: row.hash });
@@ -60,12 +48,11 @@ export function refreshDocs(opts = {}) {
   const seen = new Set();
   const started = Date.now();
 
-  const files = [...walk(docsDir)];
+  const files = [...walkDocs(roots)];
 
   db.exec('BEGIN');
-  for (const file of files) {
+  for (const { file, rel } of files) {
     try {
-      const rel = path.relative(docsDir, file).split(path.sep).join('/');
       seen.add(rel);
       const raw = fs.readFileSync(file, 'utf8');
       const hash = createHash('sha256').update(raw).digest('hex');