hdoc-tools 0.59.0 → 0.60.0

package/hdoc-build-db.js

@@ -17,6 +17,10 @@
 			"doc_lastmod",
 			"doc_status",
 			"doc_keywords",
+			// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
+			"doc_type UNINDEXED",
+			// Raw markdown source of the document (HTML sources are converted)
+			"doc_md UNINDEXED",
 		],
 		hdoc_meta: [
 			"resource_url",
@@ -148,6 +152,14 @@
 
 		if (!book_config.tags) book_config.tags = [];
 
+		// Book-level content type from hdocbook.json bookType:
+		// 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref
+		// Bound as BigInt: better-sqlite3 binds JS numbers as REAL, and FTS5
+		// columns have no affinity to coerce them back to INTEGER
+		const book_doc_type = BigInt(
+			Number.isInteger(book_config.bookType) ? book_config.bookType : 0,
+		);
+
 		// Build a prepared statement from a schema entry once, reusing it for every row.
 		// Previously insert_record() called db.prepare() on every single insert.
 		const make_stmt = (table) => {
@@ -193,6 +205,8 @@
 							file.lastmod,
 							file.status,
 							file.keywords,
+							book_doc_type,
+							file.doc_md ?? "",
 						);
 						inserted_row_id = info.lastInsertRowid;
 					} catch (e) {

package/hdoc-build.js

@@ -14,6 +14,16 @@
 	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
 	const hdoc_mermaid = require(path.join(__dirname, "hdoc-mermaid.js"));
 	const archiver = require("archiver");
+	const TurndownService = require("turndown");
+	const turndown_gfm = require("turndown-plugin-gfm");
+
+	// HTML -> Markdown converter, used to populate the doc_md index column for
+	// static HTML sources (markdown sources store their raw markdown directly).
+	const turndown = new TurndownService({
+		headingStyle: "atx",
+		codeBlockStyle: "fenced",
+	});
+	turndown.use(turndown_gfm.gfm);
 
 	const h_tags_to_search = ["h1", "h2", "h3"];
 	const image_extensions = ["png", "svg", "jpg"];
@@ -268,6 +278,7 @@
 		const fm_headers = [];
 		let doc_title = "";
 		let doc_type = "Article";
+		let doc_md = "";
 		let fm_status = false;
 		// Used only for static HTML: tracks whether the source file already had a
 		// frontmatter comment block that needs replacing in the output.
@@ -296,6 +307,10 @@
 				}
 			}
 
+			// Raw markdown for the doc_md index column: full post-include content,
+			// minus the YAML frontmatter block.
+			doc_md = md_txt.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, "").trim();
+
 			// Point the shared md instance at the current file before rendering
 			// so the highlight callback logs and reports against the right path.
 			currentMdFilePath = file_path;
@@ -411,6 +426,15 @@
 			html_txt = fs.readFileSync(file_path.path, "utf8");
 			html_txt = html_txt.replace(/\r/gm, ""); // Remove CR's so we're just dealing with newlines
 
+			// Best-guess markdown conversion of the HTML source for the doc_md
+			// index column (turndown drops the frontmatter comment automatically)
+			try {
+				doc_md = turndown.turndown(html_txt).trim();
+			} catch (e) {
+				console.error(`[WARNING] HTML to Markdown conversion failed for ${file_path.relativePath}: ${e}`);
+				doc_md = "";
+			}
+
 			// Check if we have a frontmatter comment
 			html_fm = hdoc.getHTMLFrontmatterHeader(html_txt);
 
@@ -798,6 +822,7 @@
 				inline: inline_content,
 				status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
 				keywords: index_data.fm_props.keywords ? index_data.fm_props.keywords : '',
+				doc_md: doc_md,
 			});
 		}
 

package/hdoc-validate-config.js

@@ -6,6 +6,7 @@
 	const hdocbook_schema = require(path.join(__dirname, "schemas", "hdocbook.schema.json"));
 	const valid_audience = hdocbook_schema.properties.audience.items.enum;
 	const valid_product_families = hdocbook_schema.properties.productFamily.enum;
+	const valid_book_types = hdocbook_schema.properties.bookType.enum;
 
 	const project_schema = require(path.join(__dirname, "schemas", "hdocbook-project.schema.json"));
 	const project_top_keys = Object.keys(project_schema.properties);
@@ -192,7 +193,7 @@
 		const file = 'hdocbook.json';
 
 		check_extra_keys(config,
-			['docId', 'title', 'description', 'publicSource', 'version', 'productFamily',
+			['docId', 'title', 'description', 'publicSource', 'version', 'productFamily', 'bookType',
 			 'coverImage', 'tags', 'audience', 'languages', 'readingTime', 'navigation', 'inline'],
 			'', file, errors);
 
@@ -267,6 +268,10 @@
 			}
 		}
 
+		if (config.bookType !== undefined && !valid_book_types.includes(config.bookType)) {
+			errors.push(`${file}: "bookType" must be an integer, one of: ${valid_book_types.join(', ')} (0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref)`);
+		}
+
 		if (config.coverImage !== undefined && typeof config.coverImage !== 'string') {
 			errors.push(`${file}: "coverImage" must be a string`);
 		}

package/npm-shrinkwrap.json

@@ -22,7 +22,9 @@
 				"markdown-it-container": "4.0.0",
 				"markdown-it-front-matter": "0.2.4",
 				"mermaid": "11.15.0",
-				"puppeteer": "25.1.0"
+				"puppeteer": "25.1.0",
+				"turndown": "7.2.1",
+				"turndown-plugin-gfm": "1.0.2"
 			},
 			"bin": {
 				"hdoc": "hdoc.js"
@@ -600,6 +602,12 @@
 				"@chevrotain/types": "~11.1.1"
 			}
 		},
+		"node_modules/@mixmark-io/domino": {
+			"version": "2.2.0",
+			"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
+			"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
+			"license": "BSD-2-Clause"
+		},
 		"node_modules/@protobufjs/aspromise": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
@@ -4413,6 +4421,21 @@
 				"node": "*"
 			}
 		},
+		"node_modules/turndown": {
+			"version": "7.2.1",
+			"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.1.tgz",
+			"integrity": "sha512-7YiPJw6rLClQL3oUKN3KgMaXeJJ2lAyZItclgKDurqnH61so4k4IH/qwmMva0zpuJc/FhRExBBnk7EbeFANlgQ==",
+			"license": "MIT",
+			"dependencies": {
+				"@mixmark-io/domino": "^2.2.0"
+			}
+		},
+		"node_modules/turndown-plugin-gfm": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
+			"integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
+			"license": "MIT"
+		},
 		"node_modules/type-fest": {
 			"version": "0.13.1",
 			"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz",

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "hdoc-tools",
-	"version": "0.59.0",
+	"version": "0.60.0",
 	"description": "Hornbill HDocBook Development Support Tool",
 	"main": "hdoc.js",
 	"bin": {
@@ -61,6 +61,8 @@
 		"markdown-it-container": "4.0.0",
 		"markdown-it-front-matter": "0.2.4",
 		"mermaid": "11.15.0",
-		"puppeteer": "25.1.0"
+		"puppeteer": "25.1.0",
+		"turndown": "7.2.1",
+		"turndown-plugin-gfm": "1.0.2"
 	}
 }

package/schemas/hdocbook.schema.json

@@ -50,6 +50,11 @@
         "hdocs"
       ]
     },
+    "bookType": {
+      "type": "integer",
+      "description": "The type of content this book contains: 0=document, 1=api_ref, 2=db_ref, 3=etl_ref, 4=mcp_ref. Defaults to 0 when omitted.",
+      "enum": [0, 1, 2, 3, 4]
+    },
     "coverImage": {
       "type": "string",
       "description": "Root-relative path to a cover image used for gallery and social sharing. Optional."

package/templates/init/_hdocbook/hdocbook.json

@@ -4,6 +4,7 @@
 	"description": "--description--",
 	"publicSource": "--publicSource--",
 	"productFamily": "hdocs",
+	"bookType": 0,
 	"version": "0.0.1",
 	"audience": [
 		"public"