hdoc-tools 0.57.5 → 0.59.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Hornbill Docs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/hdoc-build-db.js

@@ -39,6 +39,16 @@
 			"location_url",
 			"http_code INTEGER UNINDEXED",
 		],
+		// Semantic search embeddings, generated by hdoc-build-embeddings.js.
+		// The docs service merges these into its sqlite-vec index at publish
+		hdoc_embeddings: [
+			"resource_url",
+			"chunk_seq INTEGER",
+			"chunk_text",
+			"embedding BLOB",
+			"model",
+			"dims INTEGER",
+		],
 	};
 
 	exports.create_table = (db, table, cols, virtual, fts5) => {

package/hdoc-build-embeddings.js

@@ -0,0 +1,202 @@
+(() => {
+	const path = require("node:path");
+	const fs = require("node:fs");
+	const crypto = require("node:crypto");
+
+	// Model identity - must match what the docs service query embedder uses
+	// (see embedder.h in esp-docs-service). The service refuses embeddings
+	// built with a different model or dimension count.
+	const MODEL_ID = "all-MiniLM-L6-v2";
+	const HF_MODEL = "Xenova/all-MiniLM-L6-v2";
+	const DIMS = 384;
+
+	// SHA-256 of onnx/model.onnx as downloaded from Hugging Face. Matches the
+	// vendored copy the docs service runs (search_model/model.onnx). Verified
+	// after download, before any embeddings are generated - a hash mismatch
+	// (tampered/corrupt/re-published model) aborts the embedding step.
+	const MODEL_SHA256 = "759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e";
+
+	// Chunking parameters (word based). MiniLM attends to ~256 wordpiece
+	// tokens; 180 words plus the title prefix stays comfortably inside that.
+	const CHUNK_WORDS = 180;
+	const CHUNK_OVERLAP_WORDS = 30;
+	const MAX_CHUNKS_PER_DOC = 40;
+	const EMBED_BATCH_SIZE = 16;
+
+	// Maximum characters of chunk text stored for use as a result snippet
+	const STORED_CHUNK_CHARS = 500;
+
+	// Split document plain text into overlapping word-window chunks
+	const chunk_text = (text) => {
+		const words = text.split(/\s+/).filter((w) => w.length > 0);
+		if (words.length === 0) return [];
+
+		const chunks = [];
+		let start = 0;
+		while (start < words.length && chunks.length < MAX_CHUNKS_PER_DOC) {
+			const end = Math.min(start + CHUNK_WORDS, words.length);
+			chunks.push(words.slice(start, end).join(" "));
+			if (end >= words.length) break;
+			start = end - CHUNK_OVERLAP_WORDS;
+		}
+		return chunks;
+	};
+
+	// Compute the resource_url for an index record. Must stay in step with
+	// the equivalent logic in populate_index (hdoc-build-db.js), as the docs
+	// service joins hdoc_embeddings to hdoc_index on this value.
+	const resource_url_for = (file) => {
+		let index_path_name = file.relative_path.replaceAll("\\", "/");
+		if (
+			index_path_name.endsWith("/index.md") ||
+			index_path_name.endsWith("/index.html") ||
+			index_path_name.endsWith("/index.htm")
+		) {
+			index_path_name = index_path_name.substring(0, index_path_name.lastIndexOf("/"));
+		}
+		index_path_name = `/${index_path_name.replace(path.extname(file.relative_path), "")}`;
+		return file.index_html.id !== null
+			? `${index_path_name}#${file.index_html.id}`
+			: index_path_name;
+	};
+
+	// Generate semantic search embeddings for every indexed document and
+	// write them to the hdoc_embeddings table. Non-fatal on failure - a book
+	// without embeddings remains fully searchable via keyword search.
+	exports.populate_embeddings = async (db, doc_id, index_records, verbose = false) => {
+		const response = {
+			success: false,
+			error: "",
+			doc_count: 0,
+			chunk_count: 0,
+		};
+
+		let transformers = null;
+		try {
+			// @huggingface/transformers is ESM-only, load it dynamically
+			transformers = await import("@huggingface/transformers");
+		} catch (e) {
+			response.error = `Could not load @huggingface/transformers: ${e.message}`;
+			return response;
+		}
+
+		// Keep the model cache inside the package, like the puppeteer browser
+		transformers.env.cacheDir = path.join(__dirname, ".model-cache");
+
+		console.log(`\nGenerating semantic search embeddings (${MODEL_ID})...`);
+
+		let extractor = null;
+		try {
+			// fp32 to match the model the docs service runs at query time
+			extractor = await transformers.pipeline("feature-extraction", HF_MODEL, {
+				dtype: "fp32",
+			});
+		} catch (e) {
+			response.error = `Could not load embedding model ${HF_MODEL}: ${e.message}`;
+			return response;
+		}
+
+		// Verify the downloaded/cached model file against the pinned hash
+		// before generating any embeddings
+		const model_file = path.join(transformers.env.cacheDir, HF_MODEL, "onnx", "model.onnx");
+		try {
+			const hash = crypto
+				.createHash("sha256")
+				.update(fs.readFileSync(model_file))
+				.digest("hex");
+			if (hash !== MODEL_SHA256) {
+				response.error = `Embedding model checksum mismatch for ${model_file}: got ${hash}, expected ${MODEL_SHA256}. Delete the .model-cache folder to re-download, or update MODEL_SHA256 if the pinned model has intentionally changed.`;
+				return response;
+			}
+		} catch (e) {
+			response.error = `Could not verify embedding model checksum: ${e.message}`;
+			return response;
+		}
+
+		// Build the chunk list across all documents
+		const chunks = [];
+		for (const file of index_records) {
+			if (file.inline) continue;
+			if (file.status === "draft") continue;
+			if (!file.index_html || !file.index_html.text) continue;
+
+			const url = resource_url_for(file);
+			const title = (file.index_html.fm_props && file.index_html.fm_props.title) || "";
+
+			const doc_chunks = chunk_text(file.index_html.text);
+			if (doc_chunks.length === 0) continue;
+
+			response.doc_count++;
+			for (let seq = 0; seq < doc_chunks.length; seq++) {
+				chunks.push({
+					url: url,
+					seq: seq,
+					text: doc_chunks[seq],
+					// Title provides context for the embedding but is not
+					// duplicated into the stored snippet text
+					embed_input: title ? `${title}\n${doc_chunks[seq]}` : doc_chunks[seq],
+				});
+			}
+		}
+
+		if (chunks.length === 0) {
+			response.success = true;
+			console.log("No documents to embed.");
+			return response;
+		}
+
+		const stmt = db.prepare(
+			"INSERT INTO hdoc_embeddings (resource_url, chunk_seq, chunk_text, embedding, model, dims) VALUES (?, ?, ?, ?, ?, ?)",
+		);
+		const insert_batch = db.transaction((rows) => {
+			for (const row of rows) {
+				stmt.run(row.url, row.seq, row.text.substring(0, STORED_CHUNK_CHARS), row.embedding, MODEL_ID, DIMS);
+			}
+		});
+
+		const start_time = Date.now();
+
+		try {
+			for (let i = 0; i < chunks.length; i += EMBED_BATCH_SIZE) {
+				const batch = chunks.slice(i, i + EMBED_BATCH_SIZE);
+
+				const output = await extractor(
+					batch.map((c) => c.embed_input),
+					{ pooling: "mean", normalize: true },
+				);
+
+				// output is a [batch, DIMS] tensor over one flat Float32Array
+				const data = output.data;
+				const dims = output.dims[output.dims.length - 1];
+				if (dims !== DIMS) {
+					response.error = `Embedding model returned ${dims} dimensions, expected ${DIMS}`;
+					return response;
+				}
+
+				for (let j = 0; j < batch.length; j++) {
+					// Copy the row out of the shared tensor buffer
+					const vec = new Float32Array(data.slice(j * DIMS, (j + 1) * DIMS));
+					batch[j].embedding = Buffer.from(vec.buffer, 0, DIMS * 4);
+				}
+
+				insert_batch(batch);
+				response.chunk_count += batch.length;
+
+				if (verbose || i % (EMBED_BATCH_SIZE * 10) === 0) {
+					process.stdout.write(`\rEmbedding chunks: ${Math.min(i + EMBED_BATCH_SIZE, chunks.length)}/${chunks.length}`);
+				}
+			}
+		} catch (e) {
+			response.error = `Embedding generation failed: ${e.message}`;
+			return response;
+		}
+
+		const secs = Math.round((Date.now() - start_time) / 1000);
+		console.log(
+			`\rEmbedding Build Complete: ${response.chunk_count} chunks across ${response.doc_count} documents in ${secs}s.`,
+		);
+
+		response.success = true;
+		return response;
+	};
+})();

package/hdoc-build.js

@@ -8,6 +8,7 @@
 	const hdoc_validate_config = require(path.join(__dirname, "hdoc-validate-config.js"));
 	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
 	const hdoc_build_db = require(path.join(__dirname, "hdoc-build-db.js"));
+	const hdoc_build_embeddings = require(path.join(__dirname, "hdoc-build-embeddings.js"));
 	const hdoc_build_onyx = require(path.join(__dirname, "hdoc-build-onyx.js"));
 	const hdoc_build_pdf = require(path.join(__dirname, "hdoc-build-pdf.js"));
 	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
@@ -1041,6 +1042,7 @@
 		gen_exclude,
 		build_version = "",
 		output_links = true,
+		build_embeddings = true,
 	) => {
 		if (github_api_token !== "") {
 			git_token = github_api_token;
@@ -1533,6 +1535,21 @@
 			}
 		}
 
+		// Generate semantic search embeddings into the index. A failure here is
+		// not fatal - the book simply ships without semantic search coverage
+		if (!validate && build_embeddings) {
+			const embeddings = await hdoc_build_embeddings.populate_embeddings(
+				db.db,
+				doc_id,
+				index_records,
+				verbose,
+			);
+			if (!embeddings.success) {
+				console.error(`\nWARNING: Semantic embeddings not generated: ${embeddings.error}`);
+				console.error("The book will be keyword-searchable only. Use --no-embeddings to silence this warning.");
+			}
+		}
+
 		if (!validate) {
 			try {
 				const zip_path = path.join(work_path, `${doc_id}.zip`);

package/hdoc-bump.js

@@ -1,123 +1,123 @@
-(() => {
-	const fs = require("node:fs");
-	const path = require("node:path");
-
-	exports.run = (source_path, bump_type) => {
-		if (
-			bump_type !== "patch" &&
-			bump_type !== "minor" &&
-			bump_type !== "major"
-		) {
-			console.error(`Unsupported bump type: ${bump_type}`);
-			process.exit(1);
-		}
-		console.log(`Bumping ${bump_type} book version...\n`);
-
-		// Get document ID
-		const hdocbook_project_config_path = path.join(
-			source_path,
-			"hdocbook-project.json",
-		);
-		let hdocbook_project;
-		try {
-			hdocbook_project = require(hdocbook_project_config_path);
-		} catch (e) {
-			console.error("File not found: hdocbook-project.json:");
-			console.error(e, "\n");
-			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
-			process.exit(1);
-		}
-		const doc_id = hdocbook_project.docId;
-
-		const book_path = path.join(source_path, doc_id);
-		const hdocbook_path = path.join(book_path, "hdocbook.json");
-
-		let hdocbook_config;
-		try {
-			hdocbook_config = require(hdocbook_path);
-		} catch (e) {
-			console.error("File not found: hdocbook.json");
-			console.error(e, "\n");
-			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
-			process.exit(1);
-		}
-		const initial_version = hdocbook_config.version;
-		const hdocbook_version = hdocbook_config.version.split(".");
-		if (hdocbook_version.length !== 3) {
-			console.error(
-				`Book version does not appear to be in a semantic versioning format: ${initial_version}`,
-			);
-			process.exit(1);
-		}
-
-		if (Number.isNaN(hdocbook_version[0])) {
-			console.error(
-				`Existing major version is not a number: ${hdocbook_version[0]}`,
-			);
-			process.exit(1);
-		}
-		if (Number.isNaN(hdocbook_version[1])) {
-			console.error(
-				`Existing minor version is not a number: ${hdocbook_version[1]}`,
-			);
-			process.exit(1);
-		}
-		if (Number.isNaN(hdocbook_version[2])) {
-			console.error(
-				`Existing patch version is not a number: ${hdocbook_version[2]}`,
-			);
-			process.exit(1);
-		}
-
-		switch (bump_type) {
-			case "major":
-				try {
-					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10) + 1;
-					hdocbook_version[1] = 0;
-					hdocbook_version[2] = 0;
-				} catch (e) {
-					console.error("Failed to update major version:");
-					console.error(e);
-					process.exit(1);
-				}
-				break;
-			case "minor":
-				try {
-					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
-					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10) + 1;
-					hdocbook_version[2] = 0;
-				} catch (e) {
-					console.error("Failed to update minor version:");
-					console.error(e);
-					process.exit(1);
-				}
-				break;
-			default:
-				//case "patch" catered for with this default
-				try {
-					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
-					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10);
-					hdocbook_version[2] = Number.parseInt(hdocbook_version[2], 10) + 1;
-				} catch (e) {
-					console.error("Failed to update patch version:");
-					console.error(e);
-					process.exit(1);
-				}
-				break;
-		}
-
-		hdocbook_config.version = hdocbook_version.join(".");
-
-		try {
-			fs.writeFileSync(hdocbook_path, JSON.stringify(hdocbook_config, null, 2));
-		} catch (e) {
-			console.error("Error writing bumped version to book config:", e);
-			process.exit(1);
-		}
-
-		console.log(
-			`Book version updated from ${initial_version} to ${hdocbook_config.version}\n`,
-		);
-		return true;
-	};
-})();
+(() => {
+	const fs = require("node:fs");
+	const path = require("node:path");
+
+	exports.run = (source_path, bump_type) => {
+		if (
+			bump_type !== "patch" &&
+			bump_type !== "minor" &&
+			bump_type !== "major"
+		) {
+			console.error(`Unsupported bump type: ${bump_type}`);
+			process.exit(1);
+		}
+		console.log(`Bumping ${bump_type} book version...\n`);
+
+		// Get document ID
+		const hdocbook_project_config_path = path.join(
+			source_path,
+			"hdocbook-project.json",
+		);
+		let hdocbook_project;
+		try {
+			hdocbook_project = require(hdocbook_project_config_path);
+		} catch (e) {
+			console.error("File not found: hdocbook-project.json:");
+			console.error(e, "\n");
+			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
+			process.exit(1);
+		}
+		const doc_id = hdocbook_project.docId;
+
+		const book_path = path.join(source_path, doc_id);
+		const hdocbook_path = path.join(book_path, "hdocbook.json");
+
+		let hdocbook_config;
+		try {
+			hdocbook_config = require(hdocbook_path);
+		} catch (e) {
+			console.error("File not found: hdocbook.json");
+			console.error(e, "\n");
+			console.error("hdoc bump needs to be run in the root of a HDoc Book.\n");
+			process.exit(1);
+		}
+		const initial_version = hdocbook_config.version;
+		const hdocbook_version = hdocbook_config.version.split(".");
+		if (hdocbook_version.length !== 3) {
+			console.error(
+				`Book version does not appear to be in a semantic versioning format: ${initial_version}`,
+			);
+			process.exit(1);
+		}
+
+		if (Number.isNaN(hdocbook_version[0])) {
+			console.error(
+				`Existing major version is not a number: ${hdocbook_version[0]}`,
+			);
+			process.exit(1);
+		}
+		if (Number.isNaN(hdocbook_version[1])) {
+			console.error(
+				`Existing minor version is not a number: ${hdocbook_version[1]}`,
+			);
+			process.exit(1);
+		}
+		if (Number.isNaN(hdocbook_version[2])) {
+			console.error(
+				`Existing patch version is not a number: ${hdocbook_version[2]}`,
+			);
+			process.exit(1);
+		}
+
+		switch (bump_type) {
+			case "major":
+				try {
+					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10) + 1;
+					hdocbook_version[1] = 0;
+					hdocbook_version[2] = 0;
+				} catch (e) {
+					console.error("Failed to update major version:");
+					console.error(e);
+					process.exit(1);
+				}
+				break;
+			case "minor":
+				try {
+					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
+					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10) + 1;
+					hdocbook_version[2] = 0;
+				} catch (e) {
+					console.error("Failed to update minor version:");
+					console.error(e);
+					process.exit(1);
+				}
+				break;
+			default:
+				//case "patch" catered for with this default
+				try {
+					hdocbook_version[0] = Number.parseInt(hdocbook_version[0], 10);
+					hdocbook_version[1] = Number.parseInt(hdocbook_version[1], 10);
+					hdocbook_version[2] = Number.parseInt(hdocbook_version[2], 10) + 1;
+				} catch (e) {
+					console.error("Failed to update patch version:");
+					console.error(e);
+					process.exit(1);
+				}
+				break;
+		}
+
+		hdocbook_config.version = hdocbook_version.join(".");
+
+		try {
+			fs.writeFileSync(hdocbook_path, JSON.stringify(hdocbook_config, null, 2));
+		} catch (e) {
+			console.error("Error writing bumped version to book config:", e);
+			process.exit(1);
+		}
+
+		console.log(
+			`Book version updated from ${initial_version} to ${hdocbook_config.version}\n`,
+		);
+		return true;
+	};
+})();

package/hdoc-db.js

@@ -94,6 +94,13 @@
 			}
 		}
 
+		// Page chrome - the document-header block (breadcrumb bar, edit link,
+		// title, "Article / date / N minutes to read / N contributors") is
+		// rendered into every page and would otherwise pollute search index
+		// term frequencies and embedding chunks. The title is indexed
+		// separately from frontmatter, so nothing of value is lost.
+		$(".document-header").remove();
+
 		// Full-document plain text for search indexing
 		const text = $("body").text();
 

package/hdoc-help.js

@@ -13,6 +13,7 @@ Commands
      - Use the '--set-version 1.2.3' argument to set the version number of the built book. 
      - Use the '--no-color' argument to remove any color control characters from the output.
      - Use the '--no-links' argument to skip link output to CLI during validation.
+     - Use the '--no-embeddings' argument to skip semantic search embedding generation.
 
  - createDocs
     Creates folder structure and markdown documents as defined in the HDocBook navigation item links

package/hdoc-ver.js

@@ -1,43 +1,43 @@
-(() => {
-	const fs = require("node:fs");
-	const path = require("node:path");
-
-	exports.run = (source_path) => {
-		console.log("Retrieving book version...\n");
-
-		// Get document ID
-		const hdocbook_project_config_path = path.join(
-			source_path,
-			"hdocbook-project.json",
-		);
-		let hdocbook_project;
-		try {
-			hdocbook_project = require(hdocbook_project_config_path);
-		} catch (e) {
-			console.error("File not found: hdocbook-project.json:");
-			console.log(e, "\n");
-			console.error("hdoc ver needs to be run in the root of a HDoc Book.\n");
-			process.exit(1);
-		}
-		const doc_id = hdocbook_project.docId;
-
-		const book_path = path.join(source_path, doc_id);
-		const hdocbook_path = path.join(book_path, "hdocbook.json");
-
-		let hdocbook_config;
-		try {
-			hdocbook_config = require(hdocbook_path);
-		} catch (e) {
-			console.error("File not found: hdocbook.json");
-			console.log(e, "\n");
-			console.error("hdoc ver needs to be run in the root of a HDoc Book.\n");
-			process.exit(1);
-		}
-		if (hdocbook_config.version && hdocbook_config.version !== "") {
-			console.log(`Book version: ${hdocbook_config.version}\n`);
-		} else {
-			console.error("Error - this book has no version defined.\n");
-			process.exit(1);
-		}
-	};
-})();
+(() => {
+	const fs = require("node:fs");
+	const path = require("node:path");
+
+	exports.run = (source_path) => {
+		console.log("Retrieving book version...\n");
+
+		// Get document ID
+		const hdocbook_project_config_path = path.join(
+			source_path,
+			"hdocbook-project.json",
+		);
+		let hdocbook_project;
+		try {
+			hdocbook_project = require(hdocbook_project_config_path);
+		} catch (e) {
+			console.error("File not found: hdocbook-project.json:");
+			console.log(e, "\n");
+			console.error("hdoc ver needs to be run in the root of a HDoc Book.\n");
+			process.exit(1);
+		}
+		const doc_id = hdocbook_project.docId;
+
+		const book_path = path.join(source_path, doc_id);
+		const hdocbook_path = path.join(book_path, "hdocbook.json");
+
+		let hdocbook_config;
+		try {
+			hdocbook_config = require(hdocbook_path);
+		} catch (e) {
+			console.error("File not found: hdocbook.json");
+			console.log(e, "\n");
+			console.error("hdoc ver needs to be run in the root of a HDoc Book.\n");
+			process.exit(1);
+		}
+		if (hdocbook_config.version && hdocbook_config.version !== "") {
+			console.log(`Book version: ${hdocbook_config.version}\n`);
+		} else {
+			console.error("Error - this book has no version defined.\n");
+			process.exit(1);
+		}
+	};
+})();

package/hdoc.js

@@ -163,6 +163,7 @@
 	let bump_type = "patch"; // To generate spellcheck exclusions for all files
 	let output_links = true;
 	let onyx_index = false;
+	let build_embeddings = true;
 
 	// Get options from command args
 	for (let x = 0; x < process.argv.length; x++) {
@@ -212,6 +213,8 @@
 			output_links = false;
 		} else if (process.argv[x].toLowerCase() === "--onyx") {
 			onyx_index = true;
+		} else if (process.argv[x].toLowerCase() === "--no-embeddings") {
+			build_embeddings = false;
 		} else if (process.argv[x].toLowerCase() === "--quiet") {
 			// handled at startup via _quietMode
 		}
@@ -270,7 +273,7 @@
 			gen_exclude,
 			build_version,
 			output_links,
-			onyx_index,
+			build_embeddings,
 		);
 	} else if (command.toLowerCase() === "createdocs") {
 		const creator = require(path.join(__dirname, "hdoc-create.js"));
@@ -285,7 +288,7 @@
 			gen_exclude,
 			build_version,
 			output_links,
-			onyx_index,
+			build_embeddings,
 		);
 	} else if (command.toLowerCase() === "stats") {
 		const stats = require(path.join(__dirname, "hdoc-stats.js"));