@dominikcz/greg 0.9.27

package/src/lib/MarkdownDocs/ai/promptBuilder.js

@@ -0,0 +1,33 @@
+export function buildSystemPrompt(character, chunks, baseUrl = '') {
+	const context = chunks.map((c, i) => {
+		const anchor = c.sectionAnchor ? `#${c.sectionAnchor}` : '';
+		const link = `${baseUrl}${c.pageId}${anchor}`;
+		const heading = c.sectionHeading ? ` › ${c.sectionHeading}` : '';
+		return (
+			`[${i + 1}] Page: "${c.pageTitle}"${heading}\n` +
+			`    Link: ${link}\n` +
+			`    ${c.content}`
+		);
+	}).join('\n\n');
+
+	return `${character.systemPrompt}
+
+STRICT RULES — follow these without exception:
+- Base your answer EXCLUSIVELY on the DOCUMENTATION CONTEXT provided below.
+- If the context does not contain enough information to fully answer, say so clearly instead of guessing.
+- ALWAYS include at least one inline markdown link citation: [Section Title](link)
+- Do NOT invent, hallucinate, or add information absent from the context.
+- Respond in the same language the user used in their question.
+- Format your response in markdown.
+
+DOCUMENTATION CONTEXT:
+${context}`;
+}
+
+export function buildMessages(character, chunks, userQuery, conversationHistory = [], baseUrl = '') {
+	return [
+		{ role: 'system', content: buildSystemPrompt(character, chunks, baseUrl) },
+		...conversationHistory,
+		{ role: 'user', content: userQuery },
+	];
+}

package/src/lib/MarkdownDocs/ai/promptBuilder.ts

@@ -0,0 +1,53 @@
+import type { AiCharacter, ChatMessage, RetrievedChunk } from './types.js';
+
+/**
+ * Build the system prompt by combining the character's persona with
+ * numbered documentation context blocks. Each block includes the source
+ * link so the LLM can cite it inline.
+ */
+export function buildSystemPrompt(
+	character: AiCharacter,
+	chunks: RetrievedChunk[],
+	baseUrl = '',
+): string {
+	const context = chunks.map((c, i) => {
+		const anchor = c.sectionAnchor ? `#${c.sectionAnchor}` : '';
+		const link = `${baseUrl}${c.pageId}${anchor}`;
+		const heading = c.sectionHeading ? ` › ${c.sectionHeading}` : '';
+		return (
+			`[${i + 1}] Page: "${c.pageTitle}"${heading}\n` +
+			`    Link: ${link}\n` +
+			`    ${c.content}`
+		);
+	}).join('\n\n');
+
+	return `${character.systemPrompt}
+
+STRICT RULES — follow these without exception:
+- Base your answer EXCLUSIVELY on the DOCUMENTATION CONTEXT provided below.
+- If the context does not contain enough information to fully answer, say so clearly instead of guessing.
+- ALWAYS include at least one inline markdown link citation: [Section Title](link)
+- Do NOT invent, hallucinate, or add information absent from the context.
+- Respond in the same language the user used in their question.
+- Format your response in markdown.
+
+DOCUMENTATION CONTEXT:
+${context}`;
+}
+
+/**
+ * Build the full messages array for an LLM call.
+ */
+export function buildMessages(
+	character: AiCharacter,
+	chunks: RetrievedChunk[],
+	userQuery: string,
+	conversationHistory: ChatMessage[] = [],
+	baseUrl = '',
+): ChatMessage[] {
+	return [
+		{ role: 'system', content: buildSystemPrompt(character, chunks, baseUrl) },
+		...conversationHistory,
+		{ role: 'user', content: userQuery },
+	];
+}

package/src/lib/MarkdownDocs/ai/ragPipeline.js

@@ -0,0 +1,54 @@
+import { buildMessages } from './promptBuilder.js';
+import { extractSources } from './docLinker.js';
+
+export class RagPipeline {
+	constructor(provider, store, characters) {
+		this.provider = provider;
+		this.store = store;
+		this.characters = characters;
+	}
+
+	getCharacter(characterId) {
+		if (characterId) {
+			const found = this.characters.find(c => c.id === characterId);
+			if (found) return found;
+		}
+		return this.characters[0];
+	}
+
+	async ask(query, characterId, locale, options = {}) {
+		const topK = options.topK ?? 8;
+		const character = this.getCharacter(characterId);
+		const baseUrl = options.baseUrl ?? '';
+
+		let chunks = await this.store.search(query, topK * 2);
+
+		if (locale && locale !== '/') {
+			const prefix = locale.endsWith('/') ? locale : locale + '/';
+			const localeChunks = chunks.filter(
+				c => c.pageId === locale || c.pageId.startsWith(prefix),
+			);
+			if (localeChunks.length > 0) chunks = localeChunks;
+		}
+
+		chunks = chunks.slice(0, topK);
+
+		if (chunks.length === 0) {
+			return {
+				answer: 'I could not find any relevant documentation to answer your question. Try rephrasing or look through the documentation directly.',
+				sources: [],
+				character: character.id,
+			};
+		}
+
+		const messages = buildMessages(character, chunks, query, [], baseUrl);
+		const answer = await this.provider.chat(messages, options.llm);
+		const sources = extractSources(chunks);
+
+		return { answer, sources, character: character.id };
+	}
+
+	getCharacters() {
+		return this.characters;
+	}
+}

package/src/lib/MarkdownDocs/ai/ragPipeline.ts

@@ -0,0 +1,106 @@
+import type { AiProvider } from './aiProvider.js';
+import type { ChunkStore } from './chunkStore.js';
+import type { AiCharacter, AiProviderOptions, AiResponse } from './types.js';
+import { buildMessages } from './promptBuilder.js';
+import { extractSources } from './docLinker.js';
+
+export type RagOptions = {
+	/** Number of chunks to retrieve from the store. Default: 8 */
+	topK?: number;
+	/** LLM generation parameters */
+	llm?: AiProviderOptions;
+	/**
+	 * Base URL prepended to doc links in the system prompt context.
+	 * Useful for absolute links when the AI server runs on a different origin.
+	 * Default: '' (relative links)
+	 */
+	baseUrl?: string;
+};
+
+/**
+ * Retrieval-Augmented Generation pipeline.
+ *
+ * Orchestrates the full ask() flow:
+ *   1. Retrieve relevant chunks from the store (BM25 or embedding search)
+ *   2. Optionally filter by locale
+ *   3. Build system prompt with character persona + retrieved context
+ *   4. Call the LLM provider
+ *   5. Extract source citations
+ *   6. Return AiResponse
+ */
+export class RagPipeline {
+	private readonly characters: AiCharacter[];
+
+	constructor(
+		private readonly provider: AiProvider,
+		private readonly store: ChunkStore,
+		characters: AiCharacter[],
+	) {
+		this.characters = characters;
+	}
+
+	/** Find a character by id, or fall back to the first available. */
+	private getCharacter(characterId?: string): AiCharacter {
+		if (characterId) {
+			const found = this.characters.find(c => c.id === characterId);
+			if (found) return found;
+		}
+		return this.characters[0];
+	}
+
+	/**
+	 * Run a RAG query and return the AI response.
+	 *
+	 * @param query       User's question (plain text)
+	 * @param characterId Persona to use (falls back to first character if unknown)
+	 * @param locale      Locale path prefix for results filtering (e.g. '/pl/')
+	 * @param options     Retrieval and LLM tuning
+	 */
+	async ask(
+		query: string,
+		characterId?: string,
+		locale?: string,
+		options: RagOptions = {},
+	): Promise<AiResponse> {
+		const topK = options.topK ?? 8;
+		const character = this.getCharacter(characterId);
+		const baseUrl = options.baseUrl ?? '';
+
+		// 1. Retrieve — fetch more than topK to have room to filter
+		let chunks = await this.store.search(query, topK * 2);
+
+		// 2. Locale filtering — prefer locale-matching chunks
+		if (locale && locale !== '/') {
+			const prefix = locale.endsWith('/') ? locale : locale + '/';
+			const localeChunks = chunks.filter(
+				c => c.pageId === locale || c.pageId.startsWith(prefix),
+			);
+			// Only apply locale filter when it yields results
+			if (localeChunks.length > 0) chunks = localeChunks;
+		}
+
+		chunks = chunks.slice(0, topK);
+
+		if (chunks.length === 0) {
+			return {
+				answer: 'I could not find any relevant documentation to answer your question. Try rephrasing or look through the documentation directly.',
+				sources: [],
+				character: character.id,
+			};
+		}
+
+		// 3. Build prompt and call LLM
+		const messages = buildMessages(character, chunks, query, [], baseUrl);
+		const answer = await this.provider.chat(messages, options.llm);
+
+		// 4. Extract source citations from retrieved chunks
+		const sources = extractSources(chunks);
+
+		return { answer, sources, character: character.id };
+	}
+
+	/** Return the list of available characters (used by the /characters endpoint). */
+	getCharacters(): AiCharacter[] {
+		return this.characters;
+	}
+}

package/src/lib/MarkdownDocs/ai/stores/memoryStore.js

@@ -0,0 +1,88 @@
+const K1 = 1.5;
+const B = 0.75;
+
+function tokenize(text) {
+	return text
+		.toLowerCase()
+		.replace(/[^\w\s]/g, ' ')
+		.split(/\s+/)
+		.filter(t => t.length > 1);
+}
+
+export class MemoryStore {
+	constructor() {
+		this.chunks = [];
+		this.invertedIndex = new Map();
+		this.avgLen = 0;
+	}
+
+	async index(chunks) {
+		this.invertedIndex = new Map();
+		this.chunks = chunks.map((chunk, i) => {
+			const tokens = tokenize(
+				`${chunk.pageTitle} ${chunk.pageTitle} ${chunk.sectionHeading} ${chunk.sectionHeading} ${chunk.content}`,
+			);
+			const terms = new Map();
+			for (const t of tokens) {
+				terms.set(t, (terms.get(t) ?? 0) + 1);
+			}
+			for (const t of terms.keys()) {
+				if (!this.invertedIndex.has(t)) {
+					this.invertedIndex.set(t, new Set());
+				}
+				this.invertedIndex.get(t).add(i);
+			}
+			return { ...chunk, _terms: terms, _len: tokens.length };
+		});
+
+		const totalLen = this.chunks.reduce((s, c) => s + c._len, 0);
+		this.avgLen = this.chunks.length > 0 ? totalLen / this.chunks.length : 1;
+	}
+
+	async search(query, limit = 8) {
+		if (this.chunks.length === 0) return [];
+
+		const queryTerms = tokenize(query);
+		if (queryTerms.length === 0) return [];
+
+		const N = this.chunks.length;
+		const scores = new Float64Array(N);
+
+		for (const term of queryTerms) {
+			const docs = this.invertedIndex.get(term);
+			if (!docs) continue;
+
+			const df = docs.size;
+			const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+			for (const i of docs) {
+				const chunk = this.chunks[i];
+				const tf = chunk._terms.get(term) ?? 0;
+				const len = chunk._len;
+				const tfBm25 = (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * (len / this.avgLen)));
+				scores[i] += idf * tfBm25;
+			}
+		}
+
+		let maxScore = 0;
+		for (let i = 0; i < N; i++) {
+			if (scores[i] > maxScore) maxScore = scores[i];
+		}
+		if (maxScore === 0) return [];
+
+		const results = [];
+		for (let i = 0; i < N; i++) {
+			if (scores[i] > 0) {
+				const { _terms: _t, _len: _l, ...chunk } = this.chunks[i];
+				results.push({ ...chunk, score: scores[i] / maxScore });
+			}
+		}
+
+		results.sort((a, b) => b.score - a.score);
+		return results.slice(0, limit);
+	}
+
+	size() {
+		return this.chunks.length;
+	}
+}

package/src/lib/MarkdownDocs/ai/stores/memoryStore.ts

@@ -0,0 +1,112 @@
+import type { ChunkStore } from '../chunkStore.js';
+import type { DocChunk, RetrievedChunk } from '../types.js';
+
+/**
+ * In-memory BM25 chunk store.
+ *
+ * Implements Okapi BM25 full-text ranking — no external dependencies.
+ * Suitable for small to medium documentation sets (< ~50k chunks).
+ *
+ * BM25 parameters:
+ *   k1 = 1.5  (term frequency saturation — prevents long docs dominating)
+ *   b  = 0.75 (document-length normalization)
+ */
+
+const K1 = 1.5;
+const B = 0.75;
+
+type IndexedChunk = DocChunk & {
+	/** Term → raw frequency within this chunk's searchable text */
+	_terms: Map<string, number>;
+	/** Total token count (used for length normalization) */
+	_len: number;
+};
+
+function tokenize(text: string): string[] {
+	return text
+		.toLowerCase()
+		.replace(/[^\w\s]/g, ' ')
+		.split(/\s+/)
+		.filter(t => t.length > 1);
+}
+
+export class MemoryStore implements ChunkStore {
+	private chunks: IndexedChunk[] = [];
+	/** term → set of chunk-array indices containing that term */
+	private invertedIndex = new Map<string, Set<number>>();
+	private avgLen = 0;
+
+	async index(chunks: DocChunk[]): Promise<void> {
+		this.invertedIndex = new Map();
+		this.chunks = chunks.map((chunk, i) => {
+			// Title and heading carry more signal — include them in the token bag
+			const tokens = tokenize(
+				`${chunk.pageTitle} ${chunk.pageTitle} ${chunk.sectionHeading} ${chunk.sectionHeading} ${chunk.content}`,
+			);
+			const terms = new Map<string, number>();
+			for (const t of tokens) {
+				terms.set(t, (terms.get(t) ?? 0) + 1);
+			}
+			for (const t of terms.keys()) {
+				if (!this.invertedIndex.has(t)) {
+					this.invertedIndex.set(t, new Set());
+				}
+				this.invertedIndex.get(t)!.add(i);
+			}
+			return { ...chunk, _terms: terms, _len: tokens.length };
+		});
+
+		const totalLen = this.chunks.reduce((s, c) => s + c._len, 0);
+		this.avgLen = this.chunks.length > 0 ? totalLen / this.chunks.length : 1;
+	}
+
+	async search(query: string, limit = 8): Promise<RetrievedChunk[]> {
+		if (this.chunks.length === 0) return [];
+
+		const queryTerms = tokenize(query);
+		if (queryTerms.length === 0) return [];
+
+		const N = this.chunks.length;
+		const scores = new Float64Array(N);
+
+		for (const term of queryTerms) {
+			const docs = this.invertedIndex.get(term);
+			if (!docs) continue;
+
+			// IDF: Robertson–Sparck Jones formula with smoothing
+			const df = docs.size;
+			const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+			for (const i of docs) {
+				const chunk = this.chunks[i];
+				const tf = chunk._terms.get(term) ?? 0;
+				const len = chunk._len;
+				// BM25 TF with length normalization
+				const tfBm25 = (tf * (K1 + 1)) / (tf + K1 * (1 - B + B * (len / this.avgLen)));
+				scores[i] += idf * tfBm25;
+			}
+		}
+
+		// Collect and normalize scores
+		let maxScore = 0;
+		for (let i = 0; i < N; i++) {
+			if (scores[i] > maxScore) maxScore = scores[i];
+		}
+		if (maxScore === 0) return [];
+
+		const results: RetrievedChunk[] = [];
+		for (let i = 0; i < N; i++) {
+			if (scores[i] > 0) {
+				const { _terms: _t, _len: _l, ...chunk } = this.chunks[i];
+				results.push({ ...chunk, score: scores[i] / maxScore });
+			}
+		}
+
+		results.sort((a, b) => b.score - a.score);
+		return results.slice(0, limit);
+	}
+
+	size(): number {
+		return this.chunks.length;
+	}
+}