@lon-ask/dockit 0.1.0

package/apps/server/src/infrastructure/search/json/JsonSearchEngine.ts

@@ -0,0 +1,164 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { parse } from 'node-html-parser';
+import type { ISearchEngine } from '../../../core/ports/ISearchEngine.js';
+import type { IEntryReadModel } from '../../../core/ports/IEntryReadModel.js';
+import type { SearchResult, GlobalSearchResult, HtmlFile } from '../../../core/domain/types.js';
+import { DATA_ROOT } from '../../../services/paths.js';
+
+const STOP_WORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'do', 'does', 'did', 'doing', 'have', 'has', 'had', 'having', 'will', 'would', 'shall',
+  'should', 'can', 'could', 'may', 'might', 'must', 'to', 'of', 'in', 'for', 'on', 'with',
+  'at', 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
+  'between', 'out', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here',
+  'there', 'when', 'where', 'why', 'how', 'what', 'which', 'who', 'whom', 'this', 'that',
+  'these', 'those', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you',
+  'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her',
+  'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves',
+  'not', 'no', 'nor', 'so', 'if', 'about', 'up', 'down', 'just', 'only', 'own', 'same',
+  'than', 'too', 'very', 'some', 'any', 'each', 'every', 'all', 'both', 'few', 'more',
+  'most', 'other', 'such', 'also', 'get', 'got', 'like', 'make', 'made', 'use', 'used',
+  'using', 'create', 'new', 'way', 'need', 'want', 'know', 'tell', 'say', 'said', 'go',
+  'went', 'come', 'see', 'look', 'find', 'give', 'take', 'put', 'set', 'let', 'keep',
+  'work', 'call', 'try', 'ask', 'show', 'think', 'help', 'run', 'move', 'live', 'believe',
+]);
+
+function countOccurrences(text: string, term: string): number {
+  let count = 0;
+  let idx = text.indexOf(term);
+  while (idx !== -1) {
+    count++;
+    idx = text.indexOf(term, idx + term.length);
+  }
+  return count;
+}
+
+export class JsonSearchEngine implements ISearchEngine {
+  readonly capability = 'json' as const;
+
+  constructor(private readonly entryReadModel: IEntryReadModel) {}
+
+  async buildIndex(entryId: string, htmlFiles: HtmlFile[], log: (msg: string) => void): Promise<void> {
+    log(`Building search index for ${htmlFiles.length} files`);
+    const entryDir = path.join(DATA_ROOT, entryId);
+    const bundleDir = path.join(entryDir, 'bundle');
+    const indexPath = path.join(entryDir, 'index.json');
+    const index: SearchResult[] = [];
+
+    for (const file of htmlFiles) {
+      try {
+        const html = fs.readFileSync(file.fullPath, 'utf-8');
+        const root = parse(html);
+
+        const title = root.querySelector('title')?.text.trim()
+          || root.querySelector('h1')?.text.trim()
+          || path.basename(file.relativePath, '.html');
+
+        const headings: string[] = [];
+        root.querySelectorAll('h1, h2, h3, h4').forEach((el) => {
+          const text = el.text.trim();
+          if (text) headings.push(text);
+        });
+
+        const bodyEl = root.querySelector('body');
+        const bodyText = bodyEl ? bodyEl.text.replace(/\s+/g, ' ').trim() : '';
+        const snippet = bodyText.slice(0, 300);
+
+        index.push({
+          path: file.relativePath,
+          title,
+          headings,
+          snippet,
+        });
+      } catch (err) {
+        log(`  Warning: could not parse ${file.relativePath}: ${(err as Error).message}`);
+      }
+    }
+
+    fs.mkdirSync(path.dirname(indexPath), { recursive: true });
+    fs.writeFileSync(indexPath, JSON.stringify(index, null, 2), 'utf-8');
+    log(`Search index written to ${indexPath} with ${index.length} entries`);
+  }
+
+  async search(entryId: string, query: string, limit = 20): Promise<SearchResult[]> {
+    const indexPath = path.join(DATA_ROOT, entryId, 'index.json');
+    if (!fs.existsSync(indexPath)) return [];
+
+    const index: SearchResult[] = JSON.parse(fs.readFileSync(indexPath, 'utf-8'));
+    return this.scoreAndFilter(index, query, limit);
+  }
+
+  async globalSearch(query: string, limit = 30): Promise<GlobalSearchResult[]> {
+    const readyEntries = await this.entryReadModel.listReadyEntries();
+
+    const allResults: GlobalSearchResult[] = [];
+    for (const entry of readyEntries) {
+      const results = await this.search(entry.id, query, 10);
+      for (const r of results) {
+        allResults.push({
+          ...r,
+          entryId: entry.id,
+          entryName: entry.name,
+          entryVersion: entry.version,
+        });
+      }
+    }
+
+    allResults.sort((a, b) => ((b as any).score || 0) - ((a as any).score || 0));
+    return allResults.slice(0, limit);
+  }
+
+  private scoreAndFilter(index: SearchResult[], query: string, maxResults: number): SearchResult[] {
+    const allTerms = query.toLowerCase().split(/\s+/).filter(Boolean);
+    const terms = allTerms.filter((t) => !STOP_WORDS.has(t));
+
+    if (terms.length === 0) return index.slice(0, maxResults);
+
+    const docFreq: Record<string, number> = {};
+    const totalDocs = index.length;
+    for (const term of terms) {
+      docFreq[term] = 0;
+      for (const item of index) {
+        const searchText = [item.title, ...item.headings, item.snippet].join(' ').toLowerCase();
+        if (searchText.includes(term)) {
+          docFreq[term]++;
+        }
+      }
+    }
+
+    return index
+      .map((item) => {
+        const titleLower = item.title.toLowerCase();
+        const headingsLower = item.headings.map((h) => h.toLowerCase());
+        const snippetLower = item.snippet.toLowerCase();
+
+        let score = 0;
+
+        for (const term of terms) {
+          const idf = Math.log(totalDocs / (1 + docFreq[term]));
+
+          const titleCount = countOccurrences(titleLower, term);
+          if (titleCount > 0) score += titleCount * idf * 10;
+
+          let headingCount = 0;
+          for (const heading of headingsLower) {
+            if (heading.includes(term)) headingCount++;
+          }
+          if (headingCount > 0) score += Math.min(headingCount, 5) * idf * 3;
+
+          const snippetCount = countOccurrences(snippetLower, term);
+          if (snippetCount > 0) score += Math.log(1 + snippetCount) * idf;
+        }
+
+        const allTermsInTitle = terms.every((t) => titleLower.includes(t));
+        if (allTermsInTitle) score += 20 * terms.length;
+
+        return { item, score };
+      })
+      .filter(({ score }) => score > 0)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, maxResults)
+      .map(({ item }) => item);
+  }
+}

package/apps/server/src/infrastructure/search/vector/EmbeddingService.ts

@@ -0,0 +1,23 @@
+export class EmbeddingService {
+  private initialized = false;
+  private embedFn: ((texts: string[]) => Promise<number[][]>) | null = null;
+
+  private async init() {
+    if (this.initialized) return;
+
+    const mod = await import('@dockit/embeddings');
+    // Configure for bundled offline mode by default.
+    // env.cacheDir defaults to <package>/model/; allowRemoteModels defaults to true
+    // (permits download if model not yet cached). For air-gapped environments,
+    // call mod.configure({ offline: true }) before first embed().
+    mod.configure();
+    this.embedFn = mod.embed;
+    this.initialized = true;
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    await this.init();
+    if (!this.embedFn) throw new Error('Embedding service not initialized');
+    return this.embedFn(texts);
+  }
+}

package/apps/server/src/infrastructure/search/vector/VectorSearchEngine.ts

@@ -0,0 +1,480 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { parse, HTMLElement } from 'node-html-parser';
+import type { ISearchEngine } from '../../../core/ports/ISearchEngine.js';
+import type { IEntryReadModel } from '../../../core/ports/IEntryReadModel.js';
+import type { SearchResult, GlobalSearchResult, HtmlFile } from '../../../core/domain/types.js';
+import { DATA_ROOT } from '../../../services/paths.js';
+import { EmbeddingService } from './EmbeddingService.js';
+import type { Connection, Table } from '@lancedb/lancedb';
+
+const LANCE_DB_DIR = path.join(DATA_ROOT, '.lancedb');
+const VECTOR_DIM = 384;
+const MAX_EMBED_CHARS = 2000;
+const MAX_SNIPPET_CHARS = 500;
+const MIN_CHUNK_CHARS = 50;
+const RRF_K = 25;
+const FTS_WEAK_WEIGHT = 0.7;
+const FTS_STRONG_WEIGHT = 2.0;
+const FTS_MIN_SCORE_RATIO = 0.3;
+const FTS_CONFIDENCE_RATIO = 1.3;
+const PARALLEL_QUERY_LIMIT = 40;
+
+interface Chunk {
+  primaryTitle: string;
+  sectionTitle: string;
+  text: string;
+  headingPath: string[];
+}
+
+interface LanceDoc {
+  path: string;
+  primaryTitle: string;
+  sectionTitle: string;
+  content: string;
+  searchText: string;
+  embedText: string;
+  headings: string;
+  entryId: string;
+  vector: Float32Array;
+}
+
+interface LanceDbQueryResult {
+  path: string;
+  primaryTitle: string;
+  sectionTitle: string;
+  content: string;
+  headings: string;
+  entryId: string;
+  vector: Float32Array;
+  _distance: number;
+  _score?: number;
+  _query?: string;
+}
+
+export class VectorSearchEngine implements ISearchEngine {
+  readonly capability = 'vector' as const;
+  private embeddingService: EmbeddingService;
+  private dbPromise: Promise<Connection> | null = null;
+  private entryReadModel: IEntryReadModel;
+
+  constructor(entryReadModel: IEntryReadModel, embeddingService?: EmbeddingService) {
+    this.entryReadModel = entryReadModel;
+    this.embeddingService = embeddingService ?? new EmbeddingService();
+  }
+
+  private async getDb(): Promise<Connection> {
+    if (!this.dbPromise) {
+      const lancedb = await import('@lancedb/lancedb');
+      this.dbPromise = lancedb.connect(LANCE_DB_DIR);
+    }
+    return this.dbPromise;
+  }
+
+  async buildIndex(entryId: string, htmlFiles: HtmlFile[], log: (msg: string) => void): Promise<void> {
+    log(`Building vector search index for ${htmlFiles.length} files`);
+    const db = await this.getDb();
+
+    const tableName = this.sanitizeTableName(entryId);
+    try {
+      const names = await db.tableNames();
+      if (names.includes(tableName)) {
+        await db.dropTable(tableName);
+        log(`Dropped existing table ${tableName}`);
+      }
+    } catch {
+      // Table may not exist
+    }
+
+    const allChunks: LanceDoc[] = [];
+
+    for (const file of htmlFiles) {
+      try {
+        const html = fs.readFileSync(file.fullPath, 'utf-8');
+        const root = parse(html);
+
+        const primaryTitle = root.querySelector('title')?.text.trim()
+          || root.querySelector('h1')?.text.trim()
+          || path.basename(file.relativePath, '.html');
+
+        const chunks = chunkDocument(root, primaryTitle);
+
+        if (chunks.length === 0) {
+          // No sections found, treat whole document as one chunk
+          const bodyEl = root.querySelector('body');
+          const bodyText = bodyEl ? bodyEl.text.replace(/\s+/g, ' ').trim() : '';
+          const embedText = `${primaryTitle}. ${primaryTitle}. ${bodyText.replace(/\s+/g, ' ').trim()}`.substring(0, MAX_EMBED_CHARS);
+          const snippet = bodyText.substring(0, MAX_SNIPPET_CHARS);
+
+          allChunks.push({
+            path: file.relativePath,
+            primaryTitle,
+            sectionTitle: primaryTitle,
+            content: snippet,
+            searchText: `${primaryTitle}. ${primaryTitle}. ${bodyText.replace(/\s+/g, ' ').trim()}`,
+            embedText,
+            headings: primaryTitle,
+            entryId,
+            vector: new Float32Array(VECTOR_DIM),
+          });
+        } else {
+          for (const chunk of chunks) {
+            const embedText = `${primaryTitle}. ${primaryTitle}. ${chunk.sectionTitle}. ${chunk.text.replace(/\s+/g, ' ').trim()}`
+              .substring(0, MAX_EMBED_CHARS);
+            const searchText = `${primaryTitle}. ${primaryTitle}. ${chunk.sectionTitle}. ${chunk.text.replace(/\s+/g, ' ').trim()}`;
+            const snippet = chunk.text.replace(/\s+/g, ' ').trim().substring(0, MAX_SNIPPET_CHARS);
+
+            allChunks.push({
+              path: file.relativePath,
+              primaryTitle,
+              sectionTitle: chunk.sectionTitle,
+              content: snippet,
+              searchText,
+              embedText,
+              headings: [...chunk.headingPath, chunk.sectionTitle].join(' | '),
+              entryId,
+              vector: new Float32Array(VECTOR_DIM),
+            });
+          }
+        }
+      } catch (err) {
+        log(`  Warning: could not parse ${file.relativePath}: ${(err as Error).message}`);
+      }
+    }
+
+    if (allChunks.length === 0) {
+      log('No documents to index');
+      return;
+    }
+
+    log(`Created ${allChunks.length} chunks across ${htmlFiles.length} files`);
+
+    // Batch embed all chunks
+    const batchSize = 32;
+    const totalChunks = allChunks.length;
+
+    for (let i = 0; i < totalChunks; i += batchSize) {
+      const batch = allChunks.slice(i, i + batchSize);
+      const texts = batch.map((d) => d.embedText);
+      const embeddings = await this.embeddingService.embed(texts);
+      for (let j = 0; j < batch.length; j++) {
+        batch[j].vector = new Float32Array(embeddings[j]);
+      }
+      if (i % 128 === 0 || i + batchSize >= totalChunks) {
+        log(`Embedded ${Math.min(i + batchSize, totalChunks)}/${totalChunks} chunks`);
+      }
+    }
+
+    // Create LanceDB table
+    // LanceDB types require Record<string, unknown> for createTable; Float32Array vectors don't satisfy this
+    const table = await db.createTable(tableName, allChunks as any[], {
+      mode: 'overwrite',
+    });
+    log(`Created table ${tableName} with ${allChunks.length} rows`);
+
+    // Create vector index with cosine distance
+    try {
+      const lancedb = await import('@lancedb/lancedb');
+      await table.createIndex('vector', {
+        config: lancedb.Index.ivfPq({ distanceType: 'cosine' }),
+      });
+      log(`Created vector index (cosine) on ${tableName}`);
+    } catch (err) {
+      log(`  Warning: could not create vector index: ${(err as Error).message}`);
+    }
+
+    // Create FTS index on searchText column (includes title for better keyword matching)
+    try {
+      const lancedb = await import('@lancedb/lancedb');
+      await table.createIndex('searchText', {
+        config: lancedb.Index.fts(),
+      });
+      log(`Created FTS index on ${tableName}`);
+    } catch (err) {
+      log(`  Warning: could not create FTS index: ${(err as Error).message}`);
+    }
+  }
+
+  async search(entryId: string, query: string, limit = 20): Promise<SearchResult[]> {
+    const db = await this.getDb();
+    const tableName = this.sanitizeTableName(entryId);
+
+    let table: Table;
+    try {
+      table = await db.openTable(tableName);
+    } catch {
+      return [];
+    }
+
+    const results = await this.hybridSearch(table, query, limit);
+    return results;
+  }
+
+  async globalSearch(query: string, limit = 30): Promise<GlobalSearchResult[]> {
+    const db = await this.getDb();
+    const readyEntries = await this.entryReadModel.listReadyEntries();
+
+    if (readyEntries.length === 0) return [];
+
+    // Search all entries in parallel
+    const fetchLimit = Math.min(5, Math.ceil(limit / readyEntries.length));
+    const perEntry = Math.max(5, fetchLimit);
+
+    const entryResults = await Promise.all(
+      readyEntries.map(async (entry) => {
+        try {
+          const table = await db.openTable(this.sanitizeTableName(entry.id));
+          const results = await this.hybridSearch(table, query, perEntry);
+          return results.map((r) => ({
+            ...r,
+            entryId: entry.id,
+            entryName: entry.name,
+            entryVersion: entry.version,
+          }));
+        } catch {
+          return [] as GlobalSearchResult[];
+        }
+      })
+    );
+
+    // Flatten and re-sort by RRF methodology
+    // All results already have internal ordering, just merge and limit
+    const allResults = entryResults.flat();
+    return this.deduplicateByPath(allResults).slice(0, limit);
+  }
+
+  private async hybridSearch(table: Table, query: string, limit: number): Promise<SearchResult[]> {
+    const queryEmbedding = await this.embeddingService.embed([query]);
+    const queryVector = new Float32Array(queryEmbedding[0]);
+
+    // Run vector and FTS queries in parallel
+    const [vecResults, ftsResults] = await Promise.allSettled([
+      table
+        .query()
+        .nearestTo(queryVector)
+        .distanceType('cosine')
+        .limit(PARALLEL_QUERY_LIMIT)
+        .toArray(),
+      table
+        .query()
+        .fullTextSearch(query, { columns: ['searchText'] })
+        .limit(PARALLEL_QUERY_LIMIT)
+        .toArray(),
+    ]);
+
+    const vec = vecResults.status === 'fulfilled' ? (vecResults.value as LanceDbQueryResult[]) : [];
+    const fts = ftsResults.status === 'fulfilled' ? (ftsResults.value as LanceDbQueryResult[]) : [];
+
+    if (vec.length === 0 && fts.length === 0) return [];
+
+    // If only one query succeeded, use its results directly
+    if (vec.length === 0) {
+      return this.deduplicateByPath(
+        fts.map((r: LanceDbQueryResult) => ({
+          path: r.path,
+          title: r.primaryTitle || r.sectionTitle,
+          headings: r.headings ? r.headings.split(' | ') : [],
+          snippet: r.content,
+        }))
+      ).slice(0, limit);
+    }
+
+    if (fts.length === 0) {
+      return this.deduplicateByPath(
+        vec.map((r: LanceDbQueryResult) => ({
+          path: r.path,
+          title: r.primaryTitle || r.sectionTitle,
+          headings: r.headings ? r.headings.split(' | ') : [],
+          snippet: r.content,
+        }))
+      ).slice(0, limit);
+    }
+
+    // Hybrid fusion: Reciprocal Rank Fusion
+    const fused = this.hybridFuse(vec, fts, limit);
+    return fused;
+  }
+
+  private hybridFuse(vecResults: LanceDbQueryResult[], ftsResults: LanceDbQueryResult[], limit: number): SearchResult[] {
+    // Deduplicate: keep only best chunk per path BEFORE RRF fusion.
+    const dedupVec = this.dedupBest(vecResults, (r) => r._distance ?? Infinity, 'asc');
+    let dedupFts = this.dedupBest(ftsResults, (r) => r._score ?? 0, 'desc');
+
+    // Filter FTS results by minimum relevance threshold
+    if (dedupFts.length > 0) {
+      const maxScore = dedupFts[0]._score ?? 0;
+      const minScore = maxScore * FTS_MIN_SCORE_RATIO;
+      dedupFts = dedupFts.filter((r) => (r._score ?? 0) >= minScore);
+    }
+
+    // Dynamic FTS weight: if FTS is confident (clear score gap between #1 and others),
+    // weight FTS higher. If scores are similar, FTS is uncertain, rely more on vector.
+    let ftsWeight = FTS_WEAK_WEIGHT;
+    if (dedupFts.length >= 2) {
+      const maxScore = dedupFts[0]._score ?? 0;
+      const secondScore = dedupFts[1]._score ?? 0;
+      if (secondScore > 0 && maxScore / secondScore > FTS_CONFIDENCE_RATIO) {
+        ftsWeight = FTS_STRONG_WEIGHT;
+      }
+    } else if (dedupFts.length === 1) {
+      ftsWeight = FTS_STRONG_WEIGHT; // Single result = high confidence
+    }
+
+    const scores = new Map<string, { path: string; title: string; headings: string[]; snippet: string; score: number }>();
+
+    // Apply RRF from vector results
+    dedupVec.forEach((r, i) => {
+      const path = r.path as string;
+      const rrfScore = 1 / (RRF_K + i + 1);
+      this.addScore(scores, path, r.primaryTitle || r.sectionTitle, r.headings, r.content, rrfScore);
+    });
+
+    // Apply RRF from FTS results (dynamically weighted, with title match boosting)
+    dedupFts.forEach((r, i) => {
+      const path = r.path as string;
+      let rrfScore = ftsWeight / (RRF_K + i + 1);
+
+      // Title match boost: if query terms appear in title, extra 50%
+      const queryTerms = (r._query || '').toLowerCase().split(/\s+/).filter((t: string) => t.length > 2);
+      const sectionTitle = (r.sectionTitle || '').toLowerCase();
+      const primaryTitle = (r.primaryTitle || '').toLowerCase();
+      const titleMatch = queryTerms.some(
+        (t: string) => sectionTitle.includes(t) || primaryTitle.includes(t)
+      );
+      if (titleMatch) {
+        rrfScore *= 1.5;
+      }
+
+      this.addScore(scores, path, r.primaryTitle || r.sectionTitle, r.headings, r.content, rrfScore);
+    });
+
+    // Sort by fused RRF score descending
+    return [...scores.values()]
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit)
+      .map(({ score, ...rest }) => rest);
+  }
+
+  private addScore(
+    map: Map<string, { path: string; title: string; headings: string[]; snippet: string; score: number }>,
+    path: string,
+    title: string,
+    headings: string,
+    snippet: string,
+    score: number,
+  ): void {
+    const current = map.get(path);
+    if (!current) {
+      map.set(path, {
+        path,
+        title,
+        headings: headings ? headings.split(' | ') : [],
+        snippet,
+        score,
+      });
+    } else {
+      current.score += score;
+      // Use FTS-chosen content (more likely to have keyword match in snippet)
+      if (score > 0 && snippet) {
+        current.snippet = snippet;
+        current.title = title;
+      }
+    }
+  }
+
+  private dedupBest<T extends { path: string }>(
+    results: T[],
+    scoreFn: (r: T) => number,
+    order: 'asc' | 'desc',
+  ): T[] {
+    const best = new Map<string, { item: T; score: number }>();
+    for (const r of results) {
+      const s = scoreFn(r);
+      const existing = best.get(r.path);
+      if (
+        !existing ||
+        (order === 'asc' && s < existing.score) ||
+        (order === 'desc' && s > existing.score)
+      ) {
+        best.set(r.path, { item: r, score: s });
+      }
+    }
+    return [...best.values()].map((v) => v.item);
+  }
+
+  private deduplicateByPath<T extends { path: string }>(results: T[]): T[] {
+    const seen = new Set<string>();
+    return results.filter((r) => {
+      if (seen.has(r.path)) return false;
+      seen.add(r.path);
+      return true;
+    });
+  }
+
+  private sanitizeTableName(entryId: string): string {
+    return entryId.replace(/[^a-zA-Z0-9_]/g, '_');
+  }
+}
+
+function chunkDocument(root: ReturnType<typeof parse>, pageTitle: string): Chunk[] {
+  const chunks: Chunk[] = [];
+  const body = root.querySelector('body');
+  if (!body) return chunks;
+
+  const headingStack: string[] = [];
+  let currentSectionHeading = pageTitle;
+  let currentText = '';
+
+  const headingSelector = 'h1, h2, h3, h4';
+
+  // Collect all heading and text elements in document order
+  const elements = body.querySelectorAll(
+    `${headingSelector}, p, div, section, article, ul, ol, dl, pre, blockquote, table, figure`
+  );
+
+  for (const el of elements) {
+    const tagName = el.tagName?.toLowerCase();
+    const headingMatch = tagName?.match(/^h([1-4])$/);
+
+    if (headingMatch) {
+      // Save previous chunk if it has enough content
+      if (currentText.trim().length >= MIN_CHUNK_CHARS) {
+        chunks.push({
+          primaryTitle: pageTitle,
+          sectionTitle: currentSectionHeading,
+          text: currentText.replace(/\s+/g, ' ').trim().substring(0, MAX_EMBED_CHARS),
+          headingPath: [...headingStack],
+        });
+      }
+
+      // Start new section
+      const level = parseInt(headingMatch[1]);
+      const headingText = el.text.trim();
+      currentSectionHeading = headingText || currentSectionHeading;
+
+      // Adjust heading stack
+      while (headingStack.length >= level) headingStack.pop();
+      headingStack.push(headingText || pageTitle);
+
+      currentText = '';
+    } else {
+      // Accumulate text
+      const text = el.text?.trim();
+      if (text) {
+        currentText += ' ' + text;
+      }
+    }
+  }
+
+  // Save the last chunk
+  if (currentText.trim().length >= MIN_CHUNK_CHARS) {
+    chunks.push({
+      primaryTitle: pageTitle,
+      sectionTitle: currentSectionHeading,
+      text: currentText.replace(/\s+/g, ' ').trim().substring(0, MAX_EMBED_CHARS),
+      headingPath: [...headingStack],
+    });
+  }
+
+  return chunks;
+}

package/apps/server/src/infrastructure/source-processors/AntoraSourceProcessor.ts

@@ -0,0 +1,14 @@
+import path from 'node:path';
+import type { ISourceProcessor } from '../../core/ports/ISourceProcessor.js';
+import type { Source, AntoraSourceConfig } from '../../core/domain/types.js';
+import { buildAntoraSource } from '../../services/antora.js';
+
+export class AntoraSourceProcessor implements ISourceProcessor {
+  readonly sourceType = 'antora' as const;
+
+  async process(source: Source, _sourceDir: string, entryDir: string, entryId: string, log: (msg: string) => void): Promise<string> {
+    const config = source.config as AntoraSourceConfig;
+    const workDir = path.join(entryDir, 'antora', source.id);
+    return buildAntoraSource(config, entryId, workDir, log);
+  }
+}

package/apps/server/src/infrastructure/source-processors/AsciidocSourceProcessor.ts

@@ -0,0 +1,12 @@
+import type { ISourceProcessor } from '../../core/ports/ISourceProcessor.js';
+import type { Source, AsciidocSourceConfig } from '../../core/domain/types.js';
+import { buildAsciidocSource } from '../../services/asciidoc.js';
+
+export class AsciidocSourceProcessor implements ISourceProcessor {
+  readonly sourceType = 'asciidoc' as const;
+
+  async process(source: Source, sourceDir: string, _entryDir: string, _entryId: string, log: (msg: string) => void): Promise<string> {
+    await buildAsciidocSource(source.config as AsciidocSourceConfig, sourceDir, log);
+    return sourceDir;
+  }
+}