@dominikcz/greg 0.9.41 → 0.9.44

package/src/lib/MarkdownDocs/searchServer.js

@@ -29,7 +29,7 @@
 
 import { createServer } from 'node:http';
 import { existsSync, readFileSync } from 'node:fs';
-import { resolve } from 'node:path';
+import { resolve, dirname } from 'node:path';
 import { statSync } from 'node:fs';
 import { buildFuseResult } from './searchIndexBuilder.js';
 import { loadGregConfig } from './loadGregConfig.js';
@@ -45,6 +45,24 @@ function fmtMs(ms) {
 	return `${ms.toFixed(1)}ms`;
 }
 
+function parseBoolean(value, fallback = false) {
+	if (value == null) return fallback;
+	const normalized = String(value).trim().toLowerCase();
+	if (!normalized) return fallback;
+	if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
+	if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
+	return fallback;
+}
+
+function tokenizeQuery(value) {
+	return String(value || '')
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}\s-]+/gu, ' ')
+		.split(/[\s-]+/)
+		.map((part) => part.trim())
+		.filter((part) => part.length >= 3);
+}
+
 // ── CLI argument parser ───────────────────────────────────────────────────────
 function parseArgs(argv) {
 	const args = {};
@@ -63,10 +81,18 @@ function parseArgs(argv) {
 const args = parseArgs(process.argv.slice(2));
 const gregConfig = await loadGregConfig();
 const configuredFuzzy = gregConfig?.search?.fuzzy ?? {};
+const configuredServer = gregConfig?.search?.server ?? {};
 const port = parseInt(args.port ?? process.env.GREG_SEARCH_PORT ?? '3100', 10);
 const host = String(args.host ?? process.env.GREG_SEARCH_HOST ?? 'localhost');
 const url = String(args.url ?? process.env.GREG_SEARCH_URL ?? '/api/search');
 const index = resolve(String(args.index ?? process.env.GREG_SEARCH_INDEX ?? 'dist/search-index.json'));
+const shardsManifest = resolve(
+	String(
+		args['shards-manifest']
+		?? process.env.GREG_SEARCH_SHARDS_MANIFEST
+		?? resolve(dirname(index), 'search-index', 'manifest.json'),
+	),
+);
 const corsOrigin = String(args['cors-origin'] ?? process.env.GREG_SEARCH_CORS_ORIGIN ?? '*');
 const corsMethods = String(args['cors-methods'] ?? process.env.GREG_SEARCH_CORS_METHODS ?? 'GET, OPTIONS');
 const corsHeaders = String(args['cors-headers'] ?? process.env.GREG_SEARCH_CORS_HEADERS ?? 'Content-Type');
@@ -89,6 +115,25 @@ const ignoreLocationRaw =
 	configuredFuzzy.ignoreLocation;
 const ignoreLocation =
 	String(ignoreLocationRaw ?? 'true').toLowerCase() !== 'false';
+const preloadShards = parseBoolean(
+	args['preload-shards'] ?? process.env.GREG_SEARCH_PRELOAD_SHARDS ?? configuredServer.preloadShards,
+	true,
+);
+const maxLoadedShardsRaw = args['max-loaded-shards'] ?? process.env.GREG_SEARCH_MAX_LOADED_SHARDS ?? configuredServer.maxLoadedShards;
+const configuredMaxLoadedShards = (() => {
+	if (maxLoadedShardsRaw == null || String(maxLoadedShardsRaw).trim() === '') return null;
+	const parsed = Number.parseInt(String(maxLoadedShardsRaw), 10);
+	if (!Number.isFinite(parsed) || parsed < 1 || parsed > 512) return null;
+	return parsed;
+})();
+const shardCandidatesRaw = args['shard-candidates'] ?? process.env.GREG_SEARCH_SHARD_CANDIDATES ?? configuredServer.shardCandidates;
+const shardCandidates = (() => {
+	const fallback = 6;
+	if (shardCandidatesRaw == null || String(shardCandidatesRaw).trim() === '') return fallback;
+	const parsed = Number.parseInt(String(shardCandidatesRaw), 10);
+	if (!Number.isFinite(parsed) || parsed < 1 || parsed > 512) return fallback;
+	return parsed;
+})();
 
 function getCorsHeaders(req) {
 	const reflectedOrigin = req.headers.origin ? String(req.headers.origin) : '*';
@@ -141,6 +186,17 @@ let sectionCount = 0;
 let loadParseMs = 0;
 let fuseBuildMs = 0;
 let indexMtimeMs = -1;
+let shardedMode = false;
+let shardManifest = null;
+let shardManifestDir = '';
+let shardLoadMs = 0;
+let shardBuildMs = 0;
+let loadedShardCount = 0;
+let cacheTick = 0;
+let shardPreloadMs = 0;
+let maxLoadedShards = configuredMaxLoadedShards ?? 8;
+/** @type {Map<string, { fuse: Fuse<any>; lastUsedAt: number }>} */
+const shardCache = new Map();
 
 function buildFuse(searchData) {
 	return new Fuse(searchData, {
@@ -157,6 +213,121 @@ function buildFuse(searchData) {
 	});
 }
 
+function loadShardManifest() {
+	if (!existsSync(shardsManifest)) return false;
+	let parsed;
+	try {
+		parsed = JSON.parse(readFileSync(shardsManifest, 'utf-8'));
+	} catch (/** @type {any} */ e) {
+		console.warn(`[greg-search] Failed to parse shards manifest (${shardsManifest}): ${e.message}`);
+		return false;
+	}
+	if (!parsed || parsed.format !== 'greg-search-shards-v1' || !Array.isArray(parsed.files)) {
+		return false;
+	}
+	shardManifest = parsed;
+	shardManifestDir = dirname(shardsManifest);
+	shardedMode = true;
+	return true;
+}
+
+function markShardUsage(key) {
+	const cached = shardCache.get(key);
+	if (!cached) return;
+	cached.lastUsedAt = ++cacheTick;
+}
+
+function evictShardsIfNeeded() {
+	while (shardCache.size > maxLoadedShards) {
+		let oldestKey = null;
+		let oldestTick = Number.POSITIVE_INFINITY;
+		for (const [key, value] of shardCache.entries()) {
+			if (value.lastUsedAt < oldestTick) {
+				oldestTick = value.lastUsedAt;
+				oldestKey = key;
+			}
+		}
+		if (!oldestKey) break;
+		shardCache.delete(oldestKey);
+	}
+}
+
+function getShardFuse(entry) {
+	const cached = shardCache.get(entry.key);
+	if (cached) {
+		markShardUsage(entry.key);
+		return cached.fuse;
+	}
+
+	const filePath = resolve(shardManifestDir, entry.file);
+	const loadT0 = process.hrtime.bigint();
+	const shardData = JSON.parse(readFileSync(filePath, 'utf-8'));
+	shardLoadMs += msSince(loadT0);
+
+	const buildT0 = process.hrtime.bigint();
+	const shardFuse = buildFuse(shardData);
+	shardBuildMs += msSince(buildT0);
+
+	shardCache.set(entry.key, {
+		fuse: shardFuse,
+		lastUsedAt: ++cacheTick,
+	});
+	loadedShardCount += 1;
+	evictShardsIfNeeded();
+	return shardFuse;
+}
+
+function searchAcrossShards(query, limit) {
+	if (!query || !shardManifest?.files?.length) return [];
+	const terms = tokenizeQuery(query);
+	const entries = shardManifest.files;
+	const scores = entries.map((entry) => {
+		const hintSet = new Set(Array.isArray(entry.hints) ? entry.hints : []);
+		let score = 0;
+		for (const term of terms) {
+			if (hintSet.has(term)) score += 1;
+		}
+		return { entry, score };
+	});
+
+	const hasHintMatch = scores.some((item) => item.score > 0);
+	const prioritizedEntries = hasHintMatch
+		? scores
+			.sort((a, b) => b.score - a.score)
+			.slice(0, Math.min(shardCandidates, entries.length))
+			.map((item) => item.entry)
+		: entries;
+
+	const merged = [];
+	const searched = new Set();
+	for (const entry of prioritizedEntries) {
+		searched.add(entry.key);
+		const shardFuse = getShardFuse(entry);
+		const shardResults = shardFuse.search(query, { limit });
+		if (shardResults.length) merged.push(...shardResults);
+	}
+
+	if (hasHintMatch && merged.length === 0) {
+		for (const entry of entries) {
+			if (searched.has(entry.key)) continue;
+			const shardFuse = getShardFuse(entry);
+			const shardResults = shardFuse.search(query, { limit });
+			if (shardResults.length) merged.push(...shardResults);
+		}
+	}
+	merged.sort((a, b) => (a.score ?? 1) - (b.score ?? 1));
+	return merged.slice(0, limit).map(buildFuseResult);
+}
+
+function preloadAllShards() {
+	if (!shardedMode || !shardManifest?.files?.length) return;
+	const t0 = process.hrtime.bigint();
+	for (const entry of shardManifest.files) {
+		getShardFuse(entry);
+	}
+	shardPreloadMs = msSince(t0);
+}
+
 function loadIndex(force = false) {
 	let stats;
 	try {
@@ -190,7 +361,16 @@ function loadIndex(force = false) {
 	return true;
 }
 
-loadIndex(true);
+if (loadShardManifest()) {
+	const manifest = shardManifest;
+	console.log(`[greg-search] Sharded mode enabled (${manifest.files.length} shards).`);
+	maxLoadedShards = configuredMaxLoadedShards ?? manifest.files.length;
+	if (preloadShards) {
+		preloadAllShards();
+	}
+} else {
+	loadIndex(true);
+}
 
 // ── HTTP server ───────────────────────────────────────────────────────────────
 const server = createServer((req, res) => {
@@ -225,9 +405,15 @@ const server = createServer((req, res) => {
 		.map(v => v.trim())
 		.filter(Boolean);
 
-	loadIndex(false);
-
-	let results = q ? fuse.search(q, { limit }).map(buildFuseResult) : [];
+	let results = [];
+	if (q) {
+		if (shardedMode) {
+			results = searchAcrossShards(q, limit);
+		} else {
+			loadIndex(false);
+			results = fuse.search(q, { limit }).map(buildFuseResult);
+		}
+	}
 	if (localeRoot && baseRoot) {
 		results = results.filter((result) =>
 			isPathInLocale(result.id, localeRoot, baseRoot, localeRoots),
@@ -247,11 +433,24 @@ const server = createServer((req, res) => {
 server.listen(port, host, () => {
 	console.log(`[greg-search] Listening on http://${host}:${port}${url}`);
 	const startupMs = msSince(startupT0);
-	console.log(
-		`[greg-search] Startup summary: load+parse=${fmtMs(loadParseMs)}, ` +
-		`fuse-index=${fmtMs(fuseBuildMs)}, total=${fmtMs(startupMs)}, ` +
-		`docs=${data.length}, sections=${sectionCount}`,
-	);
+	if (shardedMode) {
+		console.log(
+			`[greg-search] Startup summary: mode=sharded, total=${fmtMs(startupMs)}, ` +
+			`docs=${shardManifest.totalDocs ?? 'unknown'}, shards=${shardManifest.files.length}, ` +
+			`max-loaded-shards=${maxLoadedShards}, preload=${preloadShards}, shard-candidates=${shardCandidates}`,
+		);
+		console.log(
+			`[greg-search] Runtime shard stats: loaded=${loadedShardCount}, ` +
+			`cache-size=${shardCache.size}, preload=${fmtMs(shardPreloadMs)}, ` +
+			`load+parse=${fmtMs(shardLoadMs)}, fuse-index=${fmtMs(shardBuildMs)}`,
+		);
+	} else {
+		console.log(
+			`[greg-search] Startup summary: load+parse=${fmtMs(loadParseMs)}, ` +
+			`fuse-index=${fmtMs(fuseBuildMs)}, total=${fmtMs(startupMs)}, ` +
+			`docs=${data.length}, sections=${sectionCount}`,
+		);
+	}
 	console.log(
 		`[greg-search] CORS: origin=${corsOrigin}, methods="${corsMethods}", ` +
 		`headers="${corsHeaders}", max-age=${corsMaxAge}`,

package/src/lib/MarkdownDocs/vitePluginCopyDocs.js

@@ -114,9 +114,12 @@ export function vitePluginCopyDocs({ docsDir = 'docs', srcDir = '/docs', staticD
                     ? (url === rootPrefix || url.startsWith(rootPrefix + '/'))
                     : url.startsWith('/');
                 if (isDocsPath) {
-                    const rel = url.slice(rootPrefix.length).replace(/^\//, '');
+                    let rel;
+                    try { rel = decodeURIComponent(url.slice(rootPrefix.length).replace(/^\//, '')); }
+                    catch { next(); return; }
                     for (const dir of (Array.isArray(docsDir) ? docsDir : [docsDir])) {
                         const filePath = path.resolve(root, dir, rel);
+                        if (!filePath.startsWith(path.resolve(root, dir) + path.sep)) continue;
                         if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) {
                             res.setHeader('Content-Type', getContentType(filePath));
                             res.end(fs.readFileSync(filePath));
@@ -128,7 +131,11 @@ export function vitePluginCopyDocs({ docsDir = 'docs', srcDir = '/docs', staticD
                 // Extra static dirs (snippets etc.)
                 for (const dir of staticDirs) {
                     if (url.startsWith('/' + dir + '/') || url === '/' + dir) {
-                        const filePath = path.resolve(root, url.slice(1));
+                        let rel;
+                        try { rel = decodeURIComponent(url.slice(1)); }
+                        catch { next(); return; }
+                        const filePath = path.resolve(root, rel);
+                        if (!filePath.startsWith(path.resolve(root, dir) + path.sep)) continue;
                         if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) {
                             res.setHeader('Content-Type', 'text/plain; charset=utf-8');
                             res.end(fs.readFileSync(filePath, 'utf8'));

package/src/lib/MarkdownDocs/vitePluginSearchIndex.js

@@ -1,6 +1,161 @@
-import { resolve } from 'node:path';
+import fs from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { once } from 'node:events';
 import { buildSearchIndex, invalidateSearchIndexCache } from './searchIndexBuilder.js';
 
+async function writeChunk(target, chunk) {
+	if (target.write(chunk)) return;
+	await once(target, 'drain');
+}
+
+
+async function streamIndexJson(target, index) {
+	await writeChunk(target, '[');
+	for (let i = 0; i < index.length; i++) {
+		if (i > 0) await writeChunk(target, ',');
+		await writeChunk(target, JSON.stringify(index[i]));
+	}
+	await writeChunk(target, ']');
+}
+
+async function writeIndexToFile(filePath, index) {
+	await fs.promises.mkdir(dirname(filePath), { recursive: true });
+	const stream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+	try {
+		await streamIndexJson(stream, index);
+	} finally {
+		stream.end();
+	}
+	await once(stream, 'finish');
+}
+
+async function writeIndexToResponse(res, index) {
+	await streamIndexJson(res, index);
+	res.end();
+}
+
+function resolveOutputDir(config) {
+	return resolve(config.root, config.build.outDir);
+}
+
+function resolveShardCount() {
+	const DEFAULT_SHARDS = 32;
+	const raw = process.env.GREG_SEARCH_SHARDS;
+	if (raw == null || String(raw).trim() === '') return DEFAULT_SHARDS;
+	const normalized = String(raw).trim().toLowerCase();
+	if (['0', 'false', 'off', 'no'].includes(normalized)) return 0;
+	const parsed = Number.parseInt(String(raw), 10);
+	if (!Number.isFinite(parsed) || parsed < 2 || parsed > 512) return DEFAULT_SHARDS;
+	return parsed;
+}
+
+function hashRouteId(id) {
+	let hash = 5381;
+	const text = String(id || '');
+	for (let i = 0; i < text.length; i++) {
+		hash = ((hash << 5) + hash) ^ text.charCodeAt(i);
+	}
+	return hash >>> 0;
+}
+
+function padShardNumber(value, width = 3) {
+	return String(value).padStart(width, '0');
+}
+
+function formatBytes(bytes) {
+	const value = Number(bytes) || 0;
+	if (value < 1024) return `${value} B`;
+	const units = ['KB', 'MB', 'GB', 'TB'];
+	let current = value / 1024;
+	let index = 0;
+	while (current >= 1024 && index < units.length - 1) {
+		current /= 1024;
+		index += 1;
+	}
+	return `${current.toFixed(2)} ${units[index]}`;
+}
+
+function tokenizeHintText(value) {
+	return String(value || '')
+		.toLowerCase()
+		.replace(/[^\p{L}\p{N}\s-]+/gu, ' ')
+		.split(/[\s-]+/)
+		.map((part) => part.trim())
+		.filter((part) => part.length >= 3);
+}
+
+function buildShardHints(docs, maxTerms = 1200) {
+	const terms = new Set();
+	for (const doc of docs) {
+		for (const token of tokenizeHintText(doc?.title)) {
+			terms.add(token);
+			if (terms.size >= maxTerms) return Array.from(terms);
+		}
+		for (const section of (doc?.sections || [])) {
+			for (const token of tokenizeHintText(section?.heading)) {
+				terms.add(token);
+				if (terms.size >= maxTerms) return Array.from(terms);
+			}
+		}
+	}
+	return Array.from(terms);
+}
+
+async function writeShardedIndex(outDir, index, logger) {
+	const shardCount = resolveShardCount();
+	if (shardCount < 2) {
+		if (logger?.info) {
+			logger.info('[vite-plugin-search-index] Skipping sharded index generation (GREG_SEARCH_SHARDS disabled).');
+		}
+		return;
+	}
+	const shardDir = resolve(outDir, 'search-index');
+	const shardBuckets = Array.from({ length: shardCount }, () => []);
+
+	for (const entry of index) {
+		const bucket = hashRouteId(entry?.id) % shardCount;
+		shardBuckets[bucket].push(entry);
+	}
+
+	const files = [];
+	let writtenDocs = 0;
+	let totalBytes = 0;
+	for (let i = 0; i < shardBuckets.length; i++) {
+		const docs = shardBuckets[i];
+		const key = padShardNumber(i);
+		const file = `shard-${key}.json`;
+		const absFile = resolve(shardDir, file);
+		await writeIndexToFile(absFile, docs);
+		const size = fs.statSync(absFile).size;
+		const hints = buildShardHints(docs);
+		files.push({ key, file, docs: docs.length, size, hints });
+		writtenDocs += docs.length;
+		totalBytes += size;
+	}
+
+	const manifest = {
+		format: 'greg-search-shards-v1',
+		createdAt: new Date().toISOString(),
+		shardCount,
+		totalDocs: writtenDocs,
+		totalBytes,
+		files,
+	};
+
+	await fs.promises.mkdir(shardDir, { recursive: true });
+	await fs.promises.writeFile(
+		resolve(shardDir, 'manifest.json'),
+		JSON.stringify(manifest, null, 2),
+		'utf8',
+	);
+
+	if (logger?.info) {
+		logger.info(
+			`[vite-plugin-search-index] Wrote sharded search index (${shardCount} shards, ${formatBytes(totalBytes)}).`,
+		);
+	}
+}
+
 /**
  * Vite plugin: serves /search-index.json in dev and emits it as a build asset.
  *
@@ -13,6 +168,7 @@ import { buildSearchIndex, invalidateSearchIndexCache } from './searchIndexBuild
  */
 export function vitePluginSearchIndex({ docsDir = 'docs', srcDir = '/docs' } = {}) {
 	let resolvedDocsDirs;
+	let resolvedOutDir;
 
 	return {
 		name: 'vite-plugin-search-index',
@@ -20,6 +176,7 @@ export function vitePluginSearchIndex({ docsDir = 'docs', srcDir = '/docs' } = {
 		configResolved(config) {
 			const dirs = Array.isArray(docsDir) ? docsDir : [docsDir];
 			resolvedDocsDirs = dirs.map(d => resolve(config.root, d));
+			resolvedOutDir = resolveOutputDir(config);
 		},
 
 		// Dev-server: answer GET /search-index.json with the cached index
@@ -37,21 +194,22 @@ export function vitePluginSearchIndex({ docsDir = 'docs', srcDir = '/docs' } = {
 						'Content-Type': 'application/json; charset=utf-8',
 						'Cache-Control': 'no-cache',
 					});
-					res.end(JSON.stringify(index));
+					await writeIndexToResponse(res, index);
 				} catch (err) {
 					next(err);
 				}
 			});
 		},
 
-		// Production build: emit search-index.json as a static asset
-		async generateBundle() {
+		// Production build: write full search-index.json directly to outDir.
+		// Streaming avoids creating one giant JSON string in memory.
+		async closeBundle() {
 			const index = await buildSearchIndex(resolvedDocsDirs, srcDir);
-			this.emitFile({
-				type: 'asset',
-				fileName: 'search-index.json',
-				source: JSON.stringify(index),
-			});
+			const outFile = resolve(resolvedOutDir, 'search-index.json');
+			await writeIndexToFile(outFile, index);
+			const fullSize = fs.statSync(outFile).size;
+			this.info(`[vite-plugin-search-index] Wrote search-index.json (${formatBytes(fullSize)}).`);
+			await writeShardedIndex(resolvedOutDir, index, this);
 		},
 	};
 }

package/types/index.d.ts

@@ -445,6 +445,15 @@ export type GregConfig = {
             /** Ignore match position in text. Default: true */
             ignoreLocation?: boolean;
         };
+        /** Standalone `greg search-server` performance tuning options. */
+        server?: {
+            /** Preload all shard indexes on startup. Default: true */
+            preloadShards?: boolean;
+            /** Max number of shard Fuse indexes kept in memory. */
+            maxLoadedShards?: number;
+            /** Number of candidate shards searched first per query. Default: 6 */
+            shardCandidates?: number;
+        };
         /** AI knowledge-base (RAG) configuration. */
         ai?: AiConfig;
     };