hdoc-tools 0.57.5 → 0.60.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Hornbill Docs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/hdoc-build-db.js

@@ -17,6 +17,10 @@
 			"doc_lastmod",
 			"doc_status",
 			"doc_keywords",
+			// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
+			"doc_type UNINDEXED",
+			// Raw markdown source of the document (HTML sources are converted)
+			"doc_md UNINDEXED",
 		],
 		hdoc_meta: [
 			"resource_url",
@@ -39,6 +43,16 @@
 			"location_url",
 			"http_code INTEGER UNINDEXED",
 		],
+		// Semantic search embeddings, generated by hdoc-build-embeddings.js.
+		// The docs service merges these into its sqlite-vec index at publish
+		hdoc_embeddings: [
+			"resource_url",
+			"chunk_seq INTEGER",
+			"chunk_text",
+			"embedding BLOB",
+			"model",
+			"dims INTEGER",
+		],
 	};
 
 	exports.create_table = (db, table, cols, virtual, fts5) => {
@@ -138,6 +152,14 @@
 
 		if (!book_config.tags) book_config.tags = [];
 
+		// Book-level content type from hdocbook.json bookType:
+		// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
+		// Bound as BigInt: better-sqlite3 binds JS numbers as REAL, and FTS5
+		// columns have no affinity to coerce them back to INTEGER
+		const book_doc_type = BigInt(
+			Number.isInteger(book_config.bookType) ? book_config.bookType : 0,
+		);
+
 		// Build a prepared statement from a schema entry once, reusing it for every row.
 		// Previously insert_record() called db.prepare() on every single insert.
 		const make_stmt = (table) => {
@@ -183,6 +205,8 @@
 							file.lastmod,
 							file.status,
 							file.keywords,
+							book_doc_type,
+							file.doc_md ?? "",
 						);
 						inserted_row_id = info.lastInsertRowid;
 					} catch (e) {

package/hdoc-build-embeddings.js

@@ -0,0 +1,202 @@
+(() => {
+	const path = require("node:path");
+	const fs = require("node:fs");
+	const crypto = require("node:crypto");
+
+	// Model identity - must match what the docs service query embedder uses
+	// (see embedder.h in esp-docs-service). The service refuses embeddings
+	// built with a different model or dimension count.
+	const MODEL_ID = "all-MiniLM-L6-v2";
+	const HF_MODEL = "Xenova/all-MiniLM-L6-v2";
+	const DIMS = 384;
+
+	// SHA-256 of onnx/model.onnx as downloaded from Hugging Face. Matches the
+	// vendored copy the docs service runs (search_model/model.onnx). Verified
+	// after download, before any embeddings are generated - a hash mismatch
+	// (tampered/corrupt/re-published model) aborts the embedding step.
+	const MODEL_SHA256 = "759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e";
+
+	// Chunking parameters (word based). MiniLM attends to ~256 wordpiece
+	// tokens; 180 words plus the title prefix stays comfortably inside that.
+	const CHUNK_WORDS = 180;
+	const CHUNK_OVERLAP_WORDS = 30;
+	const MAX_CHUNKS_PER_DOC = 40;
+	const EMBED_BATCH_SIZE = 16;
+
+	// Maximum characters of chunk text stored for use as a result snippet
+	const STORED_CHUNK_CHARS = 500;
+
+	// Split document plain text into overlapping word-window chunks
+	const chunk_text = (text) => {
+		const words = text.split(/\s+/).filter((w) => w.length > 0);
+		if (words.length === 0) return [];
+
+		const chunks = [];
+		let start = 0;
+		while (start < words.length && chunks.length < MAX_CHUNKS_PER_DOC) {
+			const end = Math.min(start + CHUNK_WORDS, words.length);
+			chunks.push(words.slice(start, end).join(" "));
+			if (end >= words.length) break;
+			start = end - CHUNK_OVERLAP_WORDS;
+		}
+		return chunks;
+	};
+
+	// Compute the resource_url for an index record. Must stay in step with
+	// the equivalent logic in populate_index (hdoc-build-db.js), as the docs
+	// service joins hdoc_embeddings to hdoc_index on this value.
+	const resource_url_for = (file) => {
+		let index_path_name = file.relative_path.replaceAll("\\", "/");
+		if (
+			index_path_name.endsWith("/index.md") ||
+			index_path_name.endsWith("/index.html") ||
+			index_path_name.endsWith("/index.htm")
+		) {
+			index_path_name = index_path_name.substring(0, index_path_name.lastIndexOf("/"));
+		}
+		index_path_name = `/${index_path_name.replace(path.extname(file.relative_path), "")}`;
+		return file.index_html.id !== null
+			? `${index_path_name}#${file.index_html.id}`
+			: index_path_name;
+	};
+
+	// Generate semantic search embeddings for every indexed document and
+	// write them to the hdoc_embeddings table. Non-fatal on failure - a book
+	// without embeddings remains fully searchable via keyword search.
+	exports.populate_embeddings = async (db, doc_id, index_records, verbose = false) => {
+		const response = {
+			success: false,
+			error: "",
+			doc_count: 0,
+			chunk_count: 0,
+		};
+
+		let transformers = null;
+		try {
+			// @huggingface/transformers is ESM-only, load it dynamically
+			transformers = await import("@huggingface/transformers");
+		} catch (e) {
+			response.error = `Could not load @huggingface/transformers: ${e.message}`;
+			return response;
+		}
+
+		// Keep the model cache inside the package, like the puppeteer browser
+		transformers.env.cacheDir = path.join(__dirname, ".model-cache");
+
+		console.log(`\nGenerating semantic search embeddings (${MODEL_ID})...`);
+
+		let extractor = null;
+		try {
+			// fp32 to match the model the docs service runs at query time
+			extractor = await transformers.pipeline("feature-extraction", HF_MODEL, {
+				dtype: "fp32",
+			});
+		} catch (e) {
+			response.error = `Could not load embedding model ${HF_MODEL}: ${e.message}`;
+			return response;
+		}
+
+		// Verify the downloaded/cached model file against the pinned hash
+		// before generating any embeddings
+		const model_file = path.join(transformers.env.cacheDir, HF_MODEL, "onnx", "model.onnx");
+		try {
+			const hash = crypto
+				.createHash("sha256")
+				.update(fs.readFileSync(model_file))
+				.digest("hex");
+			if (hash !== MODEL_SHA256) {
+				response.error = `Embedding model checksum mismatch for ${model_file}: got ${hash}, expected ${MODEL_SHA256}. Delete the .model-cache folder to re-download, or update MODEL_SHA256 if the pinned model has intentionally changed.`;
+				return response;
+			}
+		} catch (e) {
+			response.error = `Could not verify embedding model checksum: ${e.message}`;
+			return response;
+		}
+
+		// Build the chunk list across all documents
+		const chunks = [];
+		for (const file of index_records) {
+			if (file.inline) continue;
+			if (file.status === "draft") continue;
+			if (!file.index_html || !file.index_html.text) continue;
+
+			const url = resource_url_for(file);
+			const title = (file.index_html.fm_props && file.index_html.fm_props.title) || "";
+
+			const doc_chunks = chunk_text(file.index_html.text);
+			if (doc_chunks.length === 0) continue;
+
+			response.doc_count++;
+			for (let seq = 0; seq < doc_chunks.length; seq++) {
+				chunks.push({
+					url: url,
+					seq: seq,
+					text: doc_chunks[seq],
+					// Title provides context for the embedding but is not
+					// duplicated into the stored snippet text
+					embed_input: title ? `${title}\n${doc_chunks[seq]}` : doc_chunks[seq],
+				});
+			}
+		}
+
+		if (chunks.length === 0) {
+			response.success = true;
+			console.log("No documents to embed.");
+			return response;
+		}
+
+		const stmt = db.prepare(
+			"INSERT INTO hdoc_embeddings (resource_url, chunk_seq, chunk_text, embedding, model, dims) VALUES (?, ?, ?, ?, ?, ?)",
+		);
+		const insert_batch = db.transaction((rows) => {
+			for (const row of rows) {
+				stmt.run(row.url, row.seq, row.text.substring(0, STORED_CHUNK_CHARS), row.embedding, MODEL_ID, DIMS);
+			}
+		});
+
+		const start_time = Date.now();
+
+		try {
+			for (let i = 0; i < chunks.length; i += EMBED_BATCH_SIZE) {
+				const batch = chunks.slice(i, i + EMBED_BATCH_SIZE);
+
+				const output = await extractor(
+					batch.map((c) => c.embed_input),
+					{ pooling: "mean", normalize: true },
+				);
+
+				// output is a [batch, DIMS] tensor over one flat Float32Array
+				const data = output.data;
+				const dims = output.dims[output.dims.length - 1];
+				if (dims !== DIMS) {
+					response.error = `Embedding model returned ${dims} dimensions, expected ${DIMS}`;
+					return response;
+				}
+
+				for (let j = 0; j < batch.length; j++) {
+					// Copy the row out of the shared tensor buffer
+					const vec = new Float32Array(data.slice(j * DIMS, (j + 1) * DIMS));
+					batch[j].embedding = Buffer.from(vec.buffer, 0, DIMS * 4);
+				}
+
+				insert_batch(batch);
+				response.chunk_count += batch.length;
+
+				if (verbose || i % (EMBED_BATCH_SIZE * 10) === 0) {
+					process.stdout.write(`\rEmbedding chunks: ${Math.min(i + EMBED_BATCH_SIZE, chunks.length)}/${chunks.length}`);
+				}
+			}
+		} catch (e) {
+			response.error = `Embedding generation failed: ${e.message}`;
+			return response;
+		}
+
+		const secs = Math.round((Date.now() - start_time) / 1000);
+		console.log(
+			`\rEmbedding Build Complete: ${response.chunk_count} chunks across ${response.doc_count} documents in ${secs}s.`,
+		);
+
+		response.success = true;
+		return response;
+	};
+})();

package/hdoc-build.js

@@ -8,11 +8,22 @@
 	const hdoc_validate_config = require(path.join(__dirname, "hdoc-validate-config.js"));
 	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
 	const hdoc_build_db = require(path.join(__dirname, "hdoc-build-db.js"));
+	const hdoc_build_embeddings = require(path.join(__dirname, "hdoc-build-embeddings.js"));
 	const hdoc_build_onyx = require(path.join(__dirname, "hdoc-build-onyx.js"));
 	const hdoc_build_pdf = require(path.join(__dirname, "hdoc-build-pdf.js"));
 	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
 	const hdoc_mermaid = require(path.join(__dirname, "hdoc-mermaid.js"));
 	const archiver = require("archiver");
+	const TurndownService = require("turndown");
+	const turndown_gfm = require("turndown-plugin-gfm");
+
+	// HTML -> Markdown converter, used to populate the doc_md index column for
+	// static HTML sources (markdown sources store their raw markdown directly).
+	const turndown = new TurndownService({
+		headingStyle: "atx",
+		codeBlockStyle: "fenced",
+	});
+	turndown.use(turndown_gfm.gfm);
 
 	const h_tags_to_search = ["h1", "h2", "h3"];
 	const image_extensions = ["png", "svg", "jpg"];
@@ -267,6 +278,7 @@
 		const fm_headers = [];
 		let doc_title = "";
 		let doc_type = "Article";
+		let doc_md = "";
 		let fm_status = false;
 		// Used only for static HTML: tracks whether the source file already had a
 		// frontmatter comment block that needs replacing in the output.
@@ -295,6 +307,10 @@
 				}
 			}
 
+			// Raw markdown for the doc_md index column: full post-include content,
+			// minus the YAML frontmatter block.
+			doc_md = md_txt.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, "").trim();
+
 			// Point the shared md instance at the current file before rendering
 			// so the highlight callback logs and reports against the right path.
 			currentMdFilePath = file_path;
@@ -410,6 +426,15 @@
 			html_txt = fs.readFileSync(file_path.path, "utf8");
 			html_txt = html_txt.replace(/\r/gm, ""); // Remove CR's so we're just dealing with newlines
 
+			// Best-guess markdown conversion of the HTML source for the doc_md
+			// index column (turndown drops the frontmatter comment automatically)
+			try {
+				doc_md = turndown.turndown(html_txt).trim();
+			} catch (e) {
+				console.error(`[WARNING] HTML to Markdown conversion failed for ${file_path.relativePath}: ${e}`);
+				doc_md = "";
+			}
+
 			// Check if we have a frontmatter comment
 			html_fm = hdoc.getHTMLFrontmatterHeader(html_txt);
 
@@ -797,6 +822,7 @@
 				inline: inline_content,
 				status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
 				keywords: index_data.fm_props.keywords ? index_data.fm_props.keywords : '',
+				doc_md: doc_md,
 			});
 		}
 
@@ -1041,6 +1067,7 @@
 		gen_exclude,
 		build_version = "",
 		output_links = true,
+		build_embeddings = true,
 	) => {
 		if (github_api_token !== "") {
 			git_token = github_api_token;
@@ -1533,6 +1560,21 @@
 			}
 		}
 
+		// Generate semantic search embeddings into the index. A failure here is
+		// not fatal - the book simply ships without semantic search coverage
+		if (!validate && build_embeddings) {
+			const embeddings = await hdoc_build_embeddings.populate_embeddings(
+				db.db,
+				doc_id,
+				index_records,
+				verbose,
+			);
+			if (!embeddings.success) {
+				console.error(`\nWARNING: Semantic embeddings not generated: ${embeddings.error}`);
+				console.error("The book will be keyword-searchable only. Use --no-embeddings to silence this warning.");
+			}
+		}
+
 		if (!validate) {
 			try {
 				const zip_path = path.join(work_path, `${doc_id}.zip`);

package/hdoc-bump.js

@@ -1,123 +1,123 @@
-(() => {
-	const fs = require("node:fs");
-	const path = require("node:path");
-
-	exports.run = (source_path, bump_type) => {
-		if (
-			bump_type !== "patch" &&
-			bump_type !== "minor" &&
-			bump_type !== "major"
-		) {
-			console.error(`Unsupported bump type: ${bump_type}`);
-			process.exit(1);
-		}
-		console.log(`Bumping ${bump_type} book version...\n`);
-
-		// Get document ID
-		const hdocbook_project_config_path = path.join(
-			source_path,
-			"hdocbook-project.json",
-		);
-		let hdocbook_project;
-		try {
-			hdocbook_project = require(hdocbook_project_config_path);
-		} catch (e) {
-			console.error("File not found: hdocbook-project.json:");
-			console.error(e, "\n");
-			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
-			process.exit(1);
-		}
-		const doc_id = hdocbook_project.docId;
-
-		const book_path = path.join(source_path, doc_id);
-		const hdocbook_path = path.join(book_path, "hdocbook.json");
-
-		let hdocbook_config;
-		try {
-			hdocbook_config = require(hdocbook_path);
-		} catch (e) {
-			console.error("File not found: hdocbook.json");
-			console.error(e, "\n");
-			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
-			process.exit(1);
-		}
-		const initial_version = hdocbook_config.version;
-		const hdocbook_version = hdocbook_config.version.split(".");
-		if (hdocbook_version.length !== 3) {
-			console.error(
-				`Book version does not appear to be in a semantic versioning format: ${initial_version}`,
-			);
-			process.exit(1);
-		}
-
-		if (Number.isNaN(hdocbook_version[0])) {
-			console.error(
-				`Existing major version is not a number: ${hdocbook_version[0]}`,
-			);
-			process.exit(1);
-		}
-		if (Number.isNaN(hdocbook_version[1])) {
-			console.error(
-				`Existing minor version is not a number: ${hdocbook_version[1]}`,
-			);
-			process.exit(1);
-		}
-		if (Number.isNaN(hdocbook_version[2])) {
-			console.error(
-				`Existing patch version is not a number: ${hdocbook_version[2]}`,
-			);
-			process.exit(1);
-		}
-
-		switch (bump_type) {
-			case "major":
-				try {
-					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10) + 1;
-					hdocbook_version[1] = 0;
-					hdocbook_version[2] = 0;
-				} catch (e) {
-					console.error("Failed to update major version:");
-					console.error(e);
-					process.exit(1);
-				}
-				break;
-			case "minor":
-				try {
-					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
-					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10) + 1;
-					hdocbook_version[2] = 0;
-				} catch (e) {
-					console.error("Failed to update minor version:");
-					console.error(e);
-					process.exit(1);
-				}
-				break;
-			default:
-				//case "patch" catered for with this default
-				try {
-					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
-					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10);
-					hdocbook_version[2] = Number.parseInt(hdocbook_version[2], 10) + 1;
-				} catch (e) {
-					console.error("Failed to update patch version:");
-					console.error(e);
-					process.exit(1);
-				}
-				break;
-		}
-
-		hdocbook_config.version = hdocbook_version.join(".");
-
-		try {
-			fs.writeFileSync(hdocbook_path, JSON.stringify(hdocbook_config, null, 2));
-		} catch (e) {
-			console.error("Error writing bumped version to book config:", e);
-			process.exit(1);
-		}
-
-		console.log(
-			`Book version updated from ${initial_version} to ${hdocbook_config.version}\n`,
-		);
-		return true;
-	};
-})();
+(() => {
+	const fs = require("node:fs");
+	const path = require("node:path");
+
+	exports.run = (source_path, bump_type) => {
+		if (
+			bump_type !== "patch" &&
+			bump_type !== "minor" &&
+			bump_type !== "major"
+		) {
+			console.error(`Unsupported bump type: ${bump_type}`);
+			process.exit(1);
+		}
+		console.log(`Bumping ${bump_type} book version...\n`);
+
+		// Get document ID
+		const hdocbook_project_config_path = path.join(
+			source_path,
+			"hdocbook-project.json",
+		);
+		let hdocbook_project;
+		try {
+			hdocbook_project = require(hdocbook_project_config_path);
+		} catch (e) {
+			console.error("File not found: hdocbook-project.json:");
+			console.error(e, "\n");
+			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
+			process.exit(1);
+		}
+		const doc_id = hdocbook_project.docId;
+
+		const book_path = path.join(source_path, doc_id);
+		const hdocbook_path = path.join(book_path, "hdocbook.json");
+
+		let hdocbook_config;
+		try {
+			hdocbook_config = require(hdocbook_path);
+		} catch (e) {
+			console.error("File not found: hdocbook.json");
+			console.error(e, "\n");
+			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
+			process.exit(1);
+		}
+		const initial_version = hdocbook_config.version;
+		const hdocbook_version = hdocbook_config.version.split(".");
+		if (hdocbook_version.length !== 3) {
+			console.error(
+				`Book version does not appear to be in a semantic versioning format: ${initial_version}`,
+			);
+			process.exit(1);
+		}
+
+		if (Number.isNaN(hdocbook_version[0])) {
+			console.error(
+				`Existing major version is not a number: ${hdocbook_version[0]}`,
+			);
+			process.exit(1);
+		}
+		if (Number.isNaN(hdocbook_version[1])) {
+			console.error(
+				`Existing minor version is not a number: ${hdocbook_version[1]}`,
+			);
+			process.exit(1);
+		}
+		if (Number.isNaN(hdocbook_version[2])) {
+			console.error(
+				`Existing patch version is not a number: ${hdocbook_version[2]}`,
+			);
+			process.exit(1);
+		}
+
+		switch (bump_type) {
+			case "major":
+				try {
+					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10) + 1;
+					hdocbook_version[1] = 0;
+					hdocbook_version[2] = 0;
+				} catch (e) {
+					console.error("Failed to update major version:");
+					console.error(e);
+					process.exit(1);
+				}
+				break;
+			case "minor":
+				try {
+					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
+					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10) + 1;
+					hdocbook_version[2] = 0;
+				} catch (e) {
+					console.error("Failed to update minor version:");
+					console.error(e);
+					process.exit(1);
+				}
+				break;
+			default:
+				//case "patch" catered for with this default
+				try {
+					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
+					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10);
+					hdocbook_version[2] = Number.parseInt(hdocbook_version[2], 10) + 1;
+				} catch (e) {
+					console.error("Failed to update patch version:");
+					console.error(e);
+					process.exit(1);
+				}
+				break;
+		}
+
+		hdocbook_config.version = hdocbook_version.join(".");
+
+		try {
+			fs.writeFileSync(hdocbook_path, JSON.stringify(hdocbook_config, null, 2));
+		} catch (e) {
+			console.error("Error writing bumped version to book config:", e);
+			process.exit(1);
+		}
+
+		console.log(
+			`Book version updated from ${initial_version} to ${hdocbook_config.version}\n`,
+		);
+		return true;
+	};
+})();

package/hdoc-db.js

@@ -94,6 +94,13 @@
 			}
 		}
 
+		// Page chrome - the document-header block (breadcrumb bar, edit link,
+		// title, "Article / date / N minutes to read / N contributors") is
+		// rendered into every page and would otherwise pollute search index
+		// term frequencies and embedding chunks. The title is indexed
+		// separately from frontmatter, so nothing of value is lost.
+		$(".document-header").remove();
+
 		// Full-document plain text for search indexing
 		const text = $("body").text();
 

package/hdoc-help.js

@@ -13,6 +13,7 @@ Commands
      - Use the '--set-version 1.2.3' argument to set the version number of the built book. 
      - Use the '--no-color' argument to remove any color control characters from the output.
      - Use the '--no-links' argument to skip link output to CLI during validation.
+     - Use the '--no-embeddings' argument to skip semantic search embedding generation.
 
  - createDocs
     Creates folder structure and markdown documents as defined in the HDocBook navigation item links

package/hdoc-validate-config.js

@@ -6,6 +6,7 @@
 	const hdocbook_schema = require(path.join(__dirname, "schemas", "hdocbook.schema.json"));
 	const valid_audience = hdocbook_schema.properties.audience.items.enum;
 	const valid_product_families = hdocbook_schema.properties.productFamily.enum;
+	const valid_book_types = hdocbook_schema.properties.bookType.enum;
 
 	const project_schema = require(path.join(__dirname, "schemas", "hdocbook-project.schema.json"));
 	const project_top_keys = Object.keys(project_schema.properties);
@@ -192,7 +193,7 @@
 		const file = 'hdocbook.json';
 
 		check_extra_keys(config,
-			['docId', 'title', 'description', 'publicSource', 'version', 'productFamily',
+			['docId', 'title', 'description', 'publicSource', 'version', 'productFamily', 'bookType',
 			 'coverImage', 'tags', 'audience', 'languages', 'readingTime', 'navigation', 'inline'],
 			'', file, errors);
 
@@ -267,6 +268,10 @@
 			}
 		}
 
+		if (config.bookType !== undefined && !valid_book_types.includes(config.bookType)) {
+			errors.push(`${file}: "bookType" must be an integer, one of: ${valid_book_types.join(', ')} (0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref)`);
+		}
+
 		if (config.coverImage !== undefined && typeof config.coverImage !== 'string') {
 			errors.push(`${file}: "coverImage" must be a string`);
 		}