hdoc-tools 0.60.1 → 0.62.0

package/hdoc-build-db.js

@@ -1,275 +1,275 @@
-(() => {
-	const path = require("node:path");
-	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
-	const database = require("better-sqlite3");
-
-	const db_schema = {
-		hdoc_index: [
-			"resource_url UNINDEXED",
-			"book_id",
-			"book_audience UNINDEXED",
-			"book_tags",
-			"doc_title",
-			"doc_content",
-			"doc_preview UNINDEXED",
-			"doc_family_id",
-			"doc_md5_hash UNINDEXED",
-			"doc_lastmod",
-			"doc_status",
-			"doc_keywords",
-			// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
-			"doc_type UNINDEXED",
-			// Raw markdown source of the document (HTML sources are converted)
-			"doc_md UNINDEXED",
-		],
-		hdoc_meta: [
-			"resource_url",
-			"book_id",
-			"contributor_count INTEGER UNINDEXED",
-			"edit_url UNINDEXED",
-			"last_commit UNINDEXED",
-			"pdf_size INTEGER UNINDEXED",
-		],
-		hdoc_contributors: [
-			"resource_url",
-			"book_id",
-			"login",
-			"name",
-			"avatar UNINDEXED",
-			"url UNINDEXED",
-		],
-		hdoc_redirects: [
-			"resource_url",
-			"location_url",
-			"http_code INTEGER UNINDEXED",
-		],
-		// Semantic search embeddings, generated by hdoc-build-embeddings.js.
-		// The docs service merges these into its sqlite-vec index at publish
-		hdoc_embeddings: [
-			"resource_url",
-			"chunk_seq INTEGER",
-			"chunk_text",
-			"embedding BLOB",
-			"model",
-			"dims INTEGER",
-		],
-	};
-
-	exports.create_table = (db, table, cols, virtual, fts5) => {
-		const table_created = hdoc_index.create_table(
-			db,
-			table,
-			cols,
-			virtual,
-			fts5,
-		);
-		if (table_created !== null) {
-			return `\nError creating table: ${table_created}`;
-		}
-		console.log(`\nTable created: ${table}`);
-		return null;
-	};
-
-	exports.create_db = function (db_path, doc_id) {
-		const response = {
-			error: null,
-			db: null,
-		};
-		console.log("\nPerforming SQlite index creation...");
-		const db_name = path.join(db_path, doc_id, "index.db");
-		response.db = new database(db_name);
-		response.db.pragma('encoding="UTF-8"');
-		console.log(`\nDatabase created: ${db_name}`);
-
-		// Create tables
-		for (const key in db_schema) {
-			if (Object.hasOwn(db_schema, key)) {
-				const virtual = key === "hdoc_index";
-				const fts5 = key === "hdoc_index";
-				const table_created_error = this.create_table(
-					response.db,
-					key,
-					db_schema[key],
-					virtual,
-					fts5,
-				);
-				if (table_created_error !== null) {
-					console.error(table_created_error);
-				}
-			}
-		}
-		return response;
-	};
-
-	exports.populate_redirects = (db, redirect_records, _verbose = false) => {
-		const response = {
-			success: true,
-			errors: [],
-			index_success_count: 0,
-		};
-
-		// Prepare once, insert all in one transaction
-		const cols = db_schema.hdoc_redirects.map(c => c.replace(/\b(UNINDEXED|INTEGER)\b/g, "").trim());
-		const stmt = db.prepare(`INSERT INTO hdoc_redirects (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`);
-
-		const run_all = db.transaction(() => {
-			for (const record of redirect_records) {
-				try {
-					stmt.run(record.url, record.location ? record.location : "", record.code);
-					response.index_success_count++;
-				} catch (e) {
-					response.success = false;
-					response.errors.push(`Redirect record creation failed - ${record.url}: ${e}`);
-				}
-			}
-		});
-
-		try {
-			run_all();
-		} catch (e) {
-			response.success = false;
-			response.errors.push(`Redirect index transaction failed: ${e}`);
-		}
-
-		console.log(
-			`\nRedirect Index Build Complete: ${response.index_success_count} document records created.`,
-		);
-		return response;
-	};
-
-	exports.populate_index = (
-		db,
-		doc_id,
-		book_config,
-		index_records,
-		verbose = false,
-	) => {
-		const response = {
-			success: false,
-			index_success_count: 0,
-			error: "",
-		};
-
-		if (!book_config.tags) book_config.tags = [];
-
-		// Book-level content type from hdocbook.json bookType:
-		// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
-		// Bound as BigInt: better-sqlite3 binds JS numbers as REAL, and FTS5
-		// columns have no affinity to coerce them back to INTEGER
-		const book_doc_type = BigInt(
-			Number.isInteger(book_config.bookType) ? book_config.bookType : 0,
-		);
-
-		// Build a prepared statement from a schema entry once, reusing it for every row.
-		// Previously insert_record() called db.prepare() on every single insert.
-		const make_stmt = (table) => {
-			const cols = db_schema[table].map(c => c.replace(/\b(UNINDEXED|INTEGER)\b/g, "").trim());
-			return db.prepare(`INSERT INTO ${table} (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`);
-		};
-		const stmt_index = make_stmt("hdoc_index");
-		const stmt_meta = make_stmt("hdoc_meta");
-		const stmt_contrib = make_stmt("hdoc_contributors");
-
-		// A single transaction batches all disk flushes into one — critical for
-		// FTS5 which otherwise re-indexes on every individual insert.
-		const run_all = db.transaction(() => {
-			let curr_file = "";
-			for (const file of index_records) {
-				let index_path_name = file.relative_path.replaceAll("\\", "/");
-				if (
-					index_path_name.endsWith("/index.md") ||
-					index_path_name.endsWith("/index.html") ||
-					index_path_name.endsWith("/index.htm")
-				) {
-					index_path_name = index_path_name.substring(0, index_path_name.lastIndexOf("/"));
-				}
-				index_path_name = `/${index_path_name.replace(path.extname(file.relative_path), "")}`;
-
-				let inserted_row_id = null;
-				const index_content_path = file.index_html.id !== null
-					? `${index_path_name}#${file.index_html.id}`
-					: index_path_name;
-
-				if (!file.inline) {
-					try {
-						const info = stmt_index.run(
-							index_content_path,
-							doc_id,
-							book_config.audience.join(","),
-							book_config.tags.join(","),
-							file.index_html.fm_props.title,
-							file.index_html.text,
-							file.index_html.preview,
-							book_config.productFamily,
-							file.md5,
-							file.lastmod,
-							file.status,
-							file.keywords,
-							book_doc_type,
-							file.doc_md ?? "",
-						);
-						inserted_row_id = info.lastInsertRowid;
-					} catch (e) {
-						console.error(`Index record creation failed - ${doc_id}/${file.index_html.fm_props.title}: ${e}`);
-						continue;
-					}
-				}
-
-				if (curr_file === index_path_name) continue;
-				curr_file = index_path_name;
-
-				try {
-					const meta_info = stmt_meta.run(
-						index_path_name,
-						doc_id,
-						file.metadata.contributor_count,
-						file.metadata.edit_url,
-						file.metadata.last_commit,
-						file.pdf_size,
-					);
-					if (verbose) {
-						console.log(`Inserted index record ${inserted_row_id}: ${doc_id} - ${file.index_html.fm_props.title}`);
-						console.log(`Inserted index metadata record for index ID: ${meta_info.lastInsertRowid}`);
-					}
-				} catch (e) {
-					console.error(`Index metadata record creation failed - ${doc_id}/${inserted_row_id}/${file.index_html.fm_props.title}: ${e}`);
-					continue;
-				}
-
-				for (const contrib of file.contributors) {
-					try {
-						const cont_info = stmt_contrib.run(
-							index_path_name,
-							doc_id,
-							contrib.login,
-							contrib.name,
-							contrib.avatar_url,
-							contrib.html_url,
-						);
-						if (verbose) {
-							console.log(`Inserted document contributor record ${cont_info.lastInsertRowid}`);
-						}
-					} catch (e) {
-						console.error(`Index document contributor record creation failed - ${doc_id}/${inserted_row_id}/${file.index_html.fm_props.title}: ${e}`);
-					}
-				}
-
-				response.index_success_count++;
-			}
-		});
-
-		try {
-			run_all();
-		} catch (e) {
-			response.error = e.message;
-			console.error(`Index build transaction failed: ${e}`);
-			return response;
-		}
-
-		response.success = true;
-		console.log(
-			`\nIndex Build Complete: ${response.index_success_count} document index records created.`,
-		);
-		return response;
-	};
-})();
+(() => {
+	const path = require("node:path");
+	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
+	const database = require("better-sqlite3");
+
+	const db_schema = {
+		hdoc_index: [
+			"resource_url UNINDEXED",
+			"book_id",
+			"book_audience UNINDEXED",
+			"book_tags",
+			"doc_title",
+			"doc_content",
+			"doc_preview UNINDEXED",
+			"doc_family_id",
+			"doc_md5_hash UNINDEXED",
+			"doc_lastmod",
+			"doc_status",
+			"doc_keywords",
+			// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
+			"doc_type UNINDEXED",
+			// Raw markdown source of the document (HTML sources are converted)
+			"doc_md UNINDEXED",
+		],
+		hdoc_meta: [
+			"resource_url",
+			"book_id",
+			"contributor_count INTEGER UNINDEXED",
+			"edit_url UNINDEXED",
+			"last_commit UNINDEXED",
+			"pdf_size INTEGER UNINDEXED",
+		],
+		hdoc_contributors: [
+			"resource_url",
+			"book_id",
+			"login",
+			"name",
+			"avatar UNINDEXED",
+			"url UNINDEXED",
+		],
+		hdoc_redirects: [
+			"resource_url",
+			"location_url",
+			"http_code INTEGER UNINDEXED",
+		],
+		// Semantic search embeddings, generated by hdoc-build-embeddings.js.
+		// The docs service merges these into its sqlite-vec index at publish
+		hdoc_embeddings: [
+			"resource_url",
+			"chunk_seq INTEGER",
+			"chunk_text",
+			"embedding BLOB",
+			"model",
+			"dims INTEGER",
+		],
+	};
+
+	exports.create_table = (db, table, cols, virtual, fts5) => {
+		const table_created = hdoc_index.create_table(
+			db,
+			table,
+			cols,
+			virtual,
+			fts5,
+		);
+		if (table_created !== null) {
+			return `\nError creating table: ${table_created}`;
+		}
+		console.log(`\nTable created: ${table}`);
+		return null;
+	};
+
+	exports.create_db = function (db_path, doc_id) {
+		const response = {
+			error: null,
+			db: null,
+		};
+		console.log("\nPerforming SQlite index creation...");
+		const db_name = path.join(db_path, doc_id, "index.db");
+		response.db = new database(db_name);
+		response.db.pragma('encoding="UTF-8"');
+		console.log(`\nDatabase created: ${db_name}`);
+
+		// Create tables
+		for (const key in db_schema) {
+			if (Object.hasOwn(db_schema, key)) {
+				const virtual = key === "hdoc_index";
+				const fts5 = key === "hdoc_index";
+				const table_created_error = this.create_table(
+					response.db,
+					key,
+					db_schema[key],
+					virtual,
+					fts5,
+				);
+				if (table_created_error !== null) {
+					console.error(table_created_error);
+				}
+			}
+		}
+		return response;
+	};
+
+	exports.populate_redirects = (db, redirect_records, _verbose = false) => {
+		const response = {
+			success: true,
+			errors: [],
+			index_success_count: 0,
+		};
+
+		// Prepare once, insert all in one transaction
+		const cols = db_schema.hdoc_redirects.map(c => c.replace(/\b(UNINDEXED|INTEGER)\b/g, "").trim());
+		const stmt = db.prepare(`INSERT INTO hdoc_redirects (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`);
+
+		const run_all = db.transaction(() => {
+			for (const record of redirect_records) {
+				try {
+					stmt.run(record.url, record.location ? record.location : "", record.code);
+					response.index_success_count++;
+				} catch (e) {
+					response.success = false;
+					response.errors.push(`Redirect record creation failed - ${record.url}: ${e}`);
+				}
+			}
+		});
+
+		try {
+			run_all();
+		} catch (e) {
+			response.success = false;
+			response.errors.push(`Redirect index transaction failed: ${e}`);
+		}
+
+		console.log(
+			`\nRedirect Index Build Complete: ${response.index_success_count} document records created.`,
+		);
+		return response;
+	};
+
+	exports.populate_index = (
+		db,
+		doc_id,
+		book_config,
+		index_records,
+		verbose = false,
+	) => {
+		const response = {
+			success: false,
+			index_success_count: 0,
+			error: "",
+		};
+
+		if (!book_config.tags) book_config.tags = [];
+
+		// Book-level content type from hdocbook.json bookType:
+		// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
+		// Bound as BigInt: better-sqlite3 binds JS numbers as REAL, and FTS5
+		// columns have no affinity to coerce them back to INTEGER
+		const book_doc_type = BigInt(
+			Number.isInteger(book_config.bookType) ? book_config.bookType : 0,
+		);
+
+		// Build a prepared statement from a schema entry once, reusing it for every row.
+		// Previously insert_record() called db.prepare() on every single insert.
+		const make_stmt = (table) => {
+			const cols = db_schema[table].map(c => c.replace(/\b(UNINDEXED|INTEGER)\b/g, "").trim());
+			return db.prepare(`INSERT INTO ${table} (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`);
+		};
+		const stmt_index = make_stmt("hdoc_index");
+		const stmt_meta = make_stmt("hdoc_meta");
+		const stmt_contrib = make_stmt("hdoc_contributors");
+
+		// A single transaction batches all disk flushes into one — critical for
+		// FTS5 which otherwise re-indexes on every individual insert.
+		const run_all = db.transaction(() => {
+			let curr_file = "";
+			for (const file of index_records) {
+				let index_path_name = file.relative_path.replaceAll("\\", "/");
+				if (
+					index_path_name.endsWith("/index.md") ||
+					index_path_name.endsWith("/index.html") ||
+					index_path_name.endsWith("/index.htm")
+				) {
+					index_path_name = index_path_name.substring(0, index_path_name.lastIndexOf("/"));
+				}
+				index_path_name = `/${index_path_name.replace(path.extname(file.relative_path), "")}`;
+
+				let inserted_row_id = null;
+				const index_content_path = file.index_html.id !== null
+					? `${index_path_name}#${file.index_html.id}`
+					: index_path_name;
+
+				if (!file.inline) {
+					try {
+						const info = stmt_index.run(
+							index_content_path,
+							doc_id,
+							book_config.audience.join(","),
+							book_config.tags.join(","),
+							file.index_html.fm_props.title,
+							file.index_html.text,
+							file.index_html.preview,
+							book_config.productFamily,
+							file.md5,
+							file.lastmod,
+							file.status,
+							file.keywords,
+							book_doc_type,
+							file.doc_md ?? "",
+						);
+						inserted_row_id = info.lastInsertRowid;
+					} catch (e) {
+						console.error(`Index record creation failed - ${doc_id}/${file.index_html.fm_props.title}: ${e}`);
+						continue;
+					}
+				}
+
+				if (curr_file === index_path_name) continue;
+				curr_file = index_path_name;
+
+				try {
+					const meta_info = stmt_meta.run(
+						index_path_name,
+						doc_id,
+						file.metadata.contributor_count,
+						file.metadata.edit_url,
+						file.metadata.last_commit,
+						file.pdf_size,
+					);
+					if (verbose) {
+						console.log(`Inserted index record ${inserted_row_id}: ${doc_id} - ${file.index_html.fm_props.title}`);
+						console.log(`Inserted index metadata record for index ID: ${meta_info.lastInsertRowid}`);
+					}
+				} catch (e) {
+					console.error(`Index metadata record creation failed - ${doc_id}/${inserted_row_id}/${file.index_html.fm_props.title}: ${e}`);
+					continue;
+				}
+
+				for (const contrib of file.contributors) {
+					try {
+						const cont_info = stmt_contrib.run(
+							index_path_name,
+							doc_id,
+							contrib.login,
+							contrib.name,
+							contrib.avatar_url,
+							contrib.html_url,
+						);
+						if (verbose) {
+							console.log(`Inserted document contributor record ${cont_info.lastInsertRowid}`);
+						}
+					} catch (e) {
+						console.error(`Index document contributor record creation failed - ${doc_id}/${inserted_row_id}/${file.index_html.fm_props.title}: ${e}`);
+					}
+				}
+
+				response.index_success_count++;
+			}
+		});
+
+		try {
+			run_all();
+		} catch (e) {
+			response.error = e.message;
+			console.error(`Index build transaction failed: ${e}`);
+			return response;
+		}
+
+		response.success = true;
+		console.log(
+			`\nIndex Build Complete: ${response.index_success_count} document index records created.`,
+		);
+		return response;
+	};
+})();