botdocs 0.4.0 → 0.6.0

package/dist-client/assets/rag-engine-B9wYRzqT.js

@@ -0,0 +1 @@
+var t=Object.defineProperty,e=(e,n,o)=>((e,n,o)=>n in e?t(e,n,{enumerable:!0,configurable:!0,writable:!0,value:o}):e[n]=o)(e,"symbol"!=typeof n?n+"":n,o);import{g as n}from"./chatbox-Dw_HFrfR.js";import"../bundle.js";function o(t,e){if(t.length!==e.length)throw new Error("Vectors must have the same length");let n=0,o=0,s=0;for(let r=0;r<t.length;r++)n+=t[r]*e[r],o+=t[r]*t[r],s+=e[r]*e[r];return o=Math.sqrt(o),s=Math.sqrt(s),0===o||0===s?0:n/(o*s)}const s=new Set(["a","an","and","are","as","at","be","by","for","from","has","he","in","is","it","its","of","on","that","the","to","was","were","will","with"]);function r(t){return(t.toLowerCase().match(/[a-z0-9][\w.-]*/g)??[]).map(t=>t.replace(/[.-]+$/,"")).filter(t=>t.length>0&&!s.has(t))}class i{constructor(t){e(this,"termFreqs",new Map),e(this,"docFreq",new Map),e(this,"docLength",new Map),e(this,"avgDocLength",0),e(this,"docIds",[]);let n=0;for(const e of t){const t=r(e.text);this.docIds.push(e.id),this.docLength.set(e.id,t.length),n+=t.length;const o=new Map;for(const e of t)o.set(e,(o.get(e)??0)+1);this.termFreqs.set(e.id,o);for(const e of o.keys())this.docFreq.set(e,(this.docFreq.get(e)??0)+1)}this.avgDocLength=t.length>0?n/t.length:0}idf(t){const e=this.docFreq.get(t)??0;return Math.log(1+(this.docIds.length-e+.5)/(e+.5))}score(t,e){const n=this.termFreqs.get(e);if(!n)return 0;const o=this.docLength.get(e)??0;let s=0;for(const r of t){const t=n.get(r);if(!t)continue;const e=2.5*t,i=t+1.5*(.25+.75*o/(this.avgDocLength||1));s+=this.idf(r)*(e/i)}return s}scoreAll(t){const e=r(t),n=new Map;for(const o of this.docIds)n.set(o,this.score(e,o));return n}}function c(t){var e;return function(t,e){return!e.startsWith("/")||e.startsWith("//")?e:t+("/"===e?"index.html":e.slice(1))}((null==(e=document.querySelector('meta[name="botdocs-root"]'))?void 0:e.content)??"/",t)}class a{constructor(){e(this,"vectorDB",null),e(this,"bm25Index",null),e(this,"isLoading",!1)}async loadVectorDB(){if(!this.vectorDB)if(this.isLoading)for(;this.isLoading;)await new Promise(t=>setTimeout(t,100));else{this.isLoading=!0;try{console.log("Loading vector database...");const t=await fetch(c("/vector-db.json"));this.setDatabase(await t.json()),console.log(`Loaded ${this.vectorDB.chunks.length} chunks`)}finally{this.isLoading=!1}}}setDatabase(t){this.vectorDB=t,this.bm25Index=null}getBM25Index(){return this.bm25Index||(this.bm25Index=new i(this.vectorDB.chunks.map(t=>({id:t.id,text:t.text})))),this.bm25Index}async search(t,e,n=5,s=0){this.vectorDB||await this.loadVectorDB();const r=this.vectorDB.chunks.map(e=>({chunk:e,score:o(t,e.embedding)})),i=[...r].sort((t,e)=>e.score-t.score),c=new Map(i.map((t,e)=>[t.chunk.id,e])),a=this.getBM25Index().scoreAll(e),h=[...a.entries()].sort((t,e)=>e[1]-t[1]),l=new Map(h.map(([t],e)=>[t,e]));return r.map(({chunk:t,score:e})=>({chunk:t,score:e,rrfScore:1/(60+c.get(t.id)+1)+1/(60+l.get(t.id)+1),bm25Score:a.get(t.id)??0})).filter(t=>t.score>=s).filter(t=>t.bm25Score>0||t.score>=.9).sort((t,e)=>e.rrfScore-t.rrfScore).slice(0,n).map(({chunk:t,score:e})=>({chunk:t,score:e}))}isReady(){return null!==this.vectorDB}getInfo(){return this.vectorDB?{chunkCount:this.vectorDB.chunks.length,dimension:this.vectorDB.dimension,model:this.vectorDB.model}:null}}let h=null;function l(){return h||(h=new a),h}function u(t){const e=[],n=t.split("\n");let o=[],s=null;const r=()=>{if(0===o.length)return;const t=o.join(" ").trim();if(o=[],t)for(const n of function(t){const e=[],n=t.replace(/`[^`]*`/g,t=>(e.push(t),`\0${e.length-1}\0`));return(n.match(/[^.!?]+[.!?]*(?:\s+|$)/g)||[n]).map(t=>t.replace(/\0(\d+)\0/g,(t,n)=>e[Number(n)])).map(t=>t.trim()).filter(Boolean)}(t))n&&e.push({text:n,kind:"prose"})};let i=0;for(;i<n.length;){const t=n[i];if(t.trim().startsWith("```")){r(),s=null;const o=[t];for(i++;i<n.length&&!n[i].trim().startsWith("```");)o.push(n[i]),i++;i<n.length&&o.push(n[i]),e.push({text:o.join("\n"),kind:"code"}),i++;continue}if(/^#{1,6}\s+/.test(t.trim()))r(),s=null,i++;else{if(/^\s*([-*+]|\d+\.)\s+/.test(t)){r();const n=t.trim();n&&(e.push({text:n,kind:"list"}),s=e.length-1),i++;continue}""!==t.trim()?null===s?(o.push(t.trim()),i++):(e[s].text+=" "+t.trim(),i++):(r(),s=null,i++)}}return r(),e}function d(t){let e,n=t;do{e=n,n=n.replace(/!?\[([^\]]*)\]\([^)]*\)/g,"$1")}while(n!==e);return n}let m=!1,f=5,g=.75;const p=.85;async function w(){return n().isModelCached()}async function b(t){if(!m)try{const e=l();await e.loadVectorDB(),t&&t(5);const o=n();await o.initialize(e=>{t&&t(5+.95*e)}),m=!0,t&&t(100)}catch(e){throw console.error("Search initialization failed:",e),e}}async function v(t,e={}){if(!m)throw new Error("Search system not initialized");const s=e.topK??f,r=e.minScore??g;try{const e=n(),i=await e.embed(t),c=l(),a=await c.search(i,t,s,r),h=a.slice(0,3),m=await Promise.all(h.map(t=>async function(t,e,n,s=3){const r=t.trim(),i=u(r);if(0===i.length)return{snippet:r,topSegment:r,topScore:0};const c=await n(i.map(t=>t.text)),a=i.map((t,n)=>({...t,index:n,score:o(e,c[n])})).sort((t,e)=>e.score-t.score);return{snippet:a.slice(0,Math.min(s,a.length)).sort((t,e)=>t.index-e.index).map(t=>t.text).join("\n\n"),topSegment:a[0].text,topScore:a[0].score}}(t.chunk.text,i,t=>e.embedBatch(t)))),f=function(t,e,n,o){if(0===e.length)return[{content:"I couldn't find anything about that in the documentation. Try rephrasing your question or browse the navigation menu to explore available topics."}];const s=/^how (to|do|can)/i.test(t),r=/^what (is|are)/i.test(t),i=/^why/i.test(t),c=[],a=n.map((t,e)=>({result:t,extract:o[e]})).filter(({result:t,extract:e})=>t.score>p&&e.topScore>p);if(a.length>1){let t="**Quick answer**\n\n";for(const{result:e,extract:n}of a){const o=x(e.chunk.metadata);t+=`- ${d(n.topSegment)} ([${y(e.chunk.metadata)}](${o}))\n`}c.push({content:t.trim()})}let h;h=s?"Here's how you can do that:":r?"Let me explain:":i?"Here's what the docs say about that:":1===e.length?"I found this relevant section:":`I found ${n.length} relevant sections:`;if(c.push({content:h}),n.forEach((t,n)=>{const{chunk:s,score:r}=t,i=s.metadata.heading||s.metadata.title,a=x(s.metadata);let h=`### ${i}\n\n${o[n].snippet}\n\n→ [View full section](${a})`;r>p&&(h+=" *(highly relevant)*"),1===e.length&&(h+="\n\n*Have a follow-up question? Just ask!*"),c.push({content:h})}),e.length>3){const t=e.length-3;c.push({content:`💡 **Found ${t} more related section${t>1?"s":""}** — check the sources below for more details.`})}const l=e.map(t=>({title:y(t.chunk.metadata),url:x(t.chunk.metadata)})),u=Array.from(new Map(l.map(t=>[t.url,t])).values());return c[c.length-1].citations=u,c}(t,a,h,m);return{messages:f}}catch(i){throw console.error("Search query error:",i),i}}function x(t){let e=c(t.url);return t.headingId&&(e+=`#${t.headingId}`),e}function y(t){return t.heading?`${t.title} → ${t.heading}`:t.title}function k(){return m}function D(t){f=t}function B(t){g=t}export{b as initializeRAG,w as isModelCached,k as isRAGReady,v as queryRAG,B as setMinScore,D as setTopK};

package/dist-client/bundle.js

@@ -1 +1 @@
-const e={},t=function(t,n,o){let r=Promise.resolve();if(n&&n.length>0){let t=function(e){return Promise.all(e.map(e=>Promise.resolve(e).then(e=>({status:"fulfilled",value:e}),e=>({status:"rejected",reason:e}))))};document.getElementsByTagName("link");const o=document.querySelector("meta[property=csp-nonce]"),c=(null==o?void 0:o.nonce)||(null==o?void 0:o.getAttribute("nonce"));r=t(n.map(t=>{if((t=function(e){return"/"+e}(t))in e)return;e[t]=!0;const n=t.endsWith(".css"),o=n?'[rel="stylesheet"]':"";if(document.querySelector(`link[href="${t}"]${o}`))return;const r=document.createElement("link");return r.rel=n?"stylesheet":"modulepreload",n||(r.as="script"),r.crossOrigin="",r.href=t,c&&r.setAttribute("nonce",c),document.head.appendChild(r),n?new Promise((e,n)=>{r.addEventListener("load",e),r.addEventListener("error",()=>n(new Error(`Unable to preload CSS for ${t}`)))}):void 0}))}function c(e){const t=new Event("vite:preloadError",{cancelable:!0});if(t.payload=e,window.dispatchEvent(t),!t.defaultPrevented)throw e}return r.then(e=>{for(const t of e||[])"rejected"===t.status&&c(t.reason);return t().catch(c)})},n="botdocs-theme";function o(){const e="dark"===document.documentElement.getAttribute("data-theme")?"light":"dark";r(e),localStorage.setItem(n,e)}function r(e){document.documentElement.setAttribute("data-theme",e)}document.addEventListener("DOMContentLoaded",()=>{!function(){const e=localStorage.getItem(n),t=window.matchMedia("(prefers-color-scheme: dark)").matches;r(e||(t?"dark":"light"));const c=document.querySelector(".theme-toggle");c&&c.addEventListener("click",o),window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change",e=>{localStorage.getItem(n)||r(e.matches?"dark":"light")})}(),function(){const e=window.location.pathname;document.querySelectorAll(".sidebar-nav a").forEach(t=>{const n=t.getAttribute("href");(n===e||"/"===e&&"/index.html"===n)&&t.classList.add("active")}),document.querySelectorAll('a[href^="#"]').forEach(e=>{e.addEventListener("click",t=>{t.preventDefault();const n=e.getAttribute("href");if(n&&n.length>1){const e=document.querySelector(n);e&&e.scrollIntoView({behavior:"smooth"})}})})}();document.getElementById("chat-widget")&&t(async()=>{const{initChat:e}=await import("./assets/chatbox-P1j6YP1y.js").then(e=>e.c);return{initChat:e}},[]).then(({initChat:e})=>{e()})});export{t as _};
+const e={},t=function(t,n,o){let r=Promise.resolve();if(n&&n.length>0){let t=function(e){return Promise.all(e.map(e=>Promise.resolve(e).then(e=>({status:"fulfilled",value:e}),e=>({status:"rejected",reason:e}))))};document.getElementsByTagName("link");const o=document.querySelector("meta[property=csp-nonce]"),c=(null==o?void 0:o.nonce)||(null==o?void 0:o.getAttribute("nonce"));r=t(n.map(t=>{if((t=function(e){return"/"+e}(t))in e)return;e[t]=!0;const n=t.endsWith(".css"),o=n?'[rel="stylesheet"]':"";if(document.querySelector(`link[href="${t}"]${o}`))return;const r=document.createElement("link");return r.rel=n?"stylesheet":"modulepreload",n||(r.as="script"),r.crossOrigin="",r.href=t,c&&r.setAttribute("nonce",c),document.head.appendChild(r),n?new Promise((e,n)=>{r.addEventListener("load",e),r.addEventListener("error",()=>n(new Error(`Unable to preload CSS for ${t}`)))}):void 0}))}function c(e){const t=new Event("vite:preloadError",{cancelable:!0});if(t.payload=e,window.dispatchEvent(t),!t.defaultPrevented)throw e}return r.then(e=>{for(const t of e||[])"rejected"===t.status&&c(t.reason);return t().catch(c)})},n="botdocs-theme";function o(){const e="dark"===document.documentElement.getAttribute("data-theme")?"light":"dark";r(e),localStorage.setItem(n,e)}function r(e){document.documentElement.setAttribute("data-theme",e)}document.addEventListener("DOMContentLoaded",()=>{!function(){const e=localStorage.getItem(n),t=window.matchMedia("(prefers-color-scheme: dark)").matches;r(e||(t?"dark":"light"));const c=document.querySelector(".theme-toggle");c&&c.addEventListener("click",o),window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change",e=>{localStorage.getItem(n)||r(e.matches?"dark":"light")})}(),function(){const e=window.location.pathname.replace(/\/$/,"/index.html");document.querySelectorAll(".sidebar-nav a").forEach(t=>{t.pathname===e&&t.classList.add("active")}),document.querySelectorAll('a[href^="#"]').forEach(e=>{e.addEventListener("click",t=>{t.preventDefault();const n=e.getAttribute("href");if(n&&n.length>1){const e=document.querySelector(n);e&&e.scrollIntoView({behavior:"smooth"})}})})}();document.getElementById("chat-widget")&&t(async()=>{const{initChat:e}=await import("./assets/chatbox-Dw_HFrfR.js").then(e=>e.c);return{initChat:e}},[]).then(({initChat:e})=>{e()})});export{t as _};

package/man/botdocs.1

@@ -1,4 +1,4 @@
-.TH BOTDOCS 1 "2026-03-28" "botdocs 0.3.0" "User Commands"
+.TH BOTDOCS 1 "2026-08-15" "botdocs 0.4.0" "User Commands"
 .SH NAME
 botdocs \- convert markdown documentation into a static website with AI chatbot
 .SH SYNOPSIS
@@ -14,6 +14,10 @@ Transformers.js running entirely in the browser with no backend required.
 The build pipeline scans markdown files, parses front matter, converts to HTML,
 generates vector embeddings for semantic search, and bundles everything into a
 self-contained static site.
+.PP
+No server-side inference is performed. On first use, visitors to the generated
+site are asked for consent, with a disclosure of what runs locally, before the
+embedding model downloads to their browser.
 .SH ARGUMENTS
 .TP
 .I input
@@ -40,6 +44,16 @@ Theme to use for the generated site. Available themes:
 .BR \-v ", " \-\-verbose
 Enable verbose logging during the build process.
 .TP
+.BR \-w ", " \-\-watch
+Rebuild when markdown or config files change, and serve the generated site
+on a local preview server for live editing. Press Ctrl+C to stop.
+.TP
+.BR \-p ", " \-\-port " " \fInumber\fR
+Port for the
+.B \-\-watch
+preview server. Defaults to
+.BR 3000 .
+.TP
 .BR \-V ", " \-\-version
 Output the version number.
 .TP
@@ -54,17 +68,76 @@ file can be placed in the input directory to configure the build:
 .nf
 {
   "title": "My Documentation",
-  "chat": { "enabled": true },
+  "description": "Project docs",
+  "theme": "classic",
+  "customCss": "custom.css",
+  "attribution": true,
+  "baseUrl": "https://example.com/docs/",
+  "chat": {
+    "enabled": true,
+    "welcomeMessage": "Ask me anything about the docs!"
+  },
   "build": {
     "chunkSize": 500,
     "chunkOverlap": 50,
-    "topK": 5
+    "minChunkSize": 15,
+    "topK": 3,
+    "minScore": 0.75
   }
 }
 .fi
 .RE
 .PP
 CLI options override configuration file settings.
+.PP
+.B customCss
+is a path to a CSS file, resolved relative to the config file's directory.
+It is appended after the theme CSS in the bundled stylesheet, so
+same-specificity selectors override the theme without
+.BR !important .
+.PP
+.B attribution
+controls the "Built with Botdocs" footer link and defaults to
+.BR true .
+.PP
+.B baseUrl
+is the canonical URL where the site is hosted (e.g. a GitHub Pages URL).
+When set, each page emits
+.B rel=canonical
+and Open Graph/Twitter card metadata, and a
+.B sitemap.xml
+listing every generated page and a
+.B robots.txt
+pointing at it are written to the output directory, and
+.B llms.txt
+links become absolute.
+When unset, these tags are omitted and no sitemap or robots.txt is generated.
+.PP
+Every build copies each page's markdown source next to its HTML and writes an
+.B llms.txt
+index linking to those
+.B .md
+files, so agents can read the docs without parsing HTML.
+.PP
+.B build.minChunkSize
+folds chunks smaller than this many estimated tokens into a neighboring
+chunk instead of keeping them as standalone, low-signal retrieval
+candidates (e.g. a heading followed by a single short line). Defaults to
+.BR 15 .
+.PP
+.B build.minScore
+is the minimum cosine similarity (0-1) a chunk must reach to be returned
+from search at all, regardless of
+.BR topK .
+Chat search combines vector similarity with BM25 keyword matching
+(Reciprocal Rank Fusion); this threshold is applied to the raw vector
+score so an off-topic query returns fewer or no results instead of always
+padding out to
+.BR topK .
+Defaults to
+.BR 0.75 ,
+since the e5 embedding model has a fairly high similarity floor even for
+unrelated text.
 .SH EXAMPLES
 Build a documentation site from a directory:
 .PP
@@ -97,6 +170,14 @@ Use the material theme:
 botdocs ./docs -t material
 .fi
 .RE
+.PP
+Edit docs with live preview (rebuild on save, serve on port 4000):
+.PP
+.RS 4
+.nf
+botdocs ./docs -w -p 4000
+.fi
+.RE
 .SH EXIT STATUS
 .TP
 .B 0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botdocs",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "CLI tool that converts markdown documentation into a static website with an optional client-side AI chatbot",
   "author": "usr-wwelsh <https://wwel.sh>",
   "license": "MIT",
@@ -24,6 +24,11 @@
     "build": "tsc",
     "build:client": "vite build",
     "dev": "tsc --watch",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p tsconfig.client.json && tsc --noEmit -p tsconfig.test.json",
+    "test": "tsx --test 'src/**/*.test.ts'",
+    "test:integration": "npm run build && tsx --test 'test/integration/**/*.test.ts'",
+    "ci": "npm run build && npm run build:client && npm run typecheck && npm run lint && npm test && npm run test:integration",
     "prepublishOnly": "npm run build && npm run build:client"
   },
   "keywords": [
@@ -53,10 +58,14 @@
     "shiki": "^1.0.0"
   },
   "devDependencies": {
+    "@eslint/js": "^10.0.1",
     "@types/markdown-it": "^14.0.0",
     "@types/node": "^20.11.0",
+    "eslint": "^10.8.1",
     "terser": "^5.36.0",
+    "tsx": "^4.23.12",
     "typescript": "^5.3.0",
+    "typescript-eslint": "^8.67.0",
     "vite": "^6.0.0"
   },
   "overrides": {

package/src/styles/chat.css

@@ -282,6 +282,19 @@
   border-bottom-color: var(--link-color);
 }
 
+.message-content ul {
+  margin: 0.5rem 0;
+  padding-left: 1.25rem;
+}
+
+.message-content li {
+  margin-bottom: 0.375rem;
+}
+
+.message-content li:last-child {
+  margin-bottom: 0;
+}
+
 .message-content hr {
   border: none;
   border-top: 1px solid var(--border-color);

package/src/templates/layout.html

@@ -5,6 +5,15 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <meta name="description" content="{{description}}">
   <title>{{title}} - {{siteTitle}}</title>
+  <meta property="og:title" content="{{title}} - {{siteTitle}}">
+  <meta property="og:description" content="{{description}}">
+  <meta property="og:type" content="website">
+  {{#if ogUrl}}
+  <meta property="og:url" content="{{ogUrl}}">
+  <link rel="canonical" href="{{ogUrl}}">
+  {{/if}}
+  <meta name="twitter:card" content="summary">
+  <meta name="botdocs-root" content="{{root}}">
   <script>
     // Apply theme immediately to prevent flash
     (function() {
@@ -14,7 +23,7 @@
       document.documentElement.setAttribute('data-theme', theme);
     })();
   </script>
-  <link rel="stylesheet" href="/assets/css/bundle.css">
+  <link rel="stylesheet" href="{{root}}assets/css/bundle.css">
 </head>
 <body>
   <div class="layout">
@@ -68,6 +77,9 @@
     {{/if}}
   </div>
 
-  <script type="module" src="/assets/js/bundle.js"></script>
+  {{#if chatEnabled}}
+  <script>window.__BOTDOCS_SEARCH_CONFIG__ = {{searchConfigJson}};</script>
+  {{/if}}
+  <script type="module" src="{{root}}assets/js/bundle.js"></script>
 </body>
 </html>