crewly 1.5.17 → 1.5.19

package/dist/cli/backend/src/services/knowledge/fts5-index.service.js

@@ -0,0 +1,220 @@
+/**
+ * FTS5 Index Service
+ *
+ * Manages a SQLite FTS5 full-text search index for the Wiki-First
+ * Knowledge System. Provides upsert, remove, search, and rebuild
+ * operations over a `knowledge_fts` virtual table using Porter stemming
+ * and Unicode61 tokenization.
+ *
+ * The database file is stored at `<knowledgePath>/index.fts5.db`.
+ *
+ * @module services/knowledge/fts5-index.service
+ */
+import * as path from 'path';
+import { existsSync, mkdirSync } from 'fs';
+import { LoggerService } from '../core/logger.service.js';
+// ---------------------------------------------------------------------------
+// Lazy module loading
+// ---------------------------------------------------------------------------
+/** Lazy-loaded better-sqlite3 module reference. */
+let _BetterSqlite3 = null;
+/**
+ * Load better-sqlite3 on demand. Throws a clear error if the native module
+ * is not available.
+ */
+function getBetterSqlite3() {
+    if (!_BetterSqlite3) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            _BetterSqlite3 = require('better-sqlite3');
+        }
+        catch (err) {
+            throw new Error('better-sqlite3 native module failed to load. Run `npm rebuild better-sqlite3` to fix. ' +
+                `Original error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return _BetterSqlite3;
+}
+// ---------------------------------------------------------------------------
+// Service
+// ---------------------------------------------------------------------------
+/**
+ * Fts5IndexService manages a SQLite FTS5 full-text search database.
+ *
+ * It lazily initializes the database and FTS5 virtual table on first use,
+ * and provides methods to upsert, remove, search, and rebuild the index.
+ *
+ * @example
+ * ```typescript
+ * const fts5 = new Fts5IndexService('/path/to/knowledge');
+ * fts5.upsertDocument({ id: 'doc-1', title: 'Setup', tags: 'devops', content: '...', category: 'SOPs' });
+ * const results = fts5.search('setup devops');
+ * ```
+ */
+export class Fts5IndexService {
+    db = null;
+    dbPath;
+    logger;
+    /**
+     * Create an Fts5IndexService.
+     *
+     * @param knowledgePath - Directory where the FTS5 database will be stored
+     */
+    constructor(knowledgePath) {
+        this.dbPath = path.join(knowledgePath, 'index.fts5.db');
+        this.logger = LoggerService.getInstance().createComponentLogger('Fts5IndexService');
+    }
+    /**
+     * Get or lazily initialize the SQLite database connection.
+     * Creates the parent directory and FTS5 virtual table if they do not exist.
+     *
+     * @returns The better-sqlite3 Database instance
+     */
+    getDb() {
+        if (this.db) {
+            return this.db;
+        }
+        // Ensure parent directory exists
+        const dir = path.dirname(this.dbPath);
+        if (!existsSync(dir)) {
+            mkdirSync(dir, { recursive: true });
+        }
+        const Database = getBetterSqlite3();
+        this.db = new Database(this.dbPath);
+        // Performance pragmas
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('synchronous = NORMAL');
+        this.initSchema();
+        this.logger.info('FTS5 index database opened', { path: this.dbPath });
+        return this.db;
+    }
+    /**
+     * Create the FTS5 virtual table if it does not already exist.
+     * Uses Porter stemming and Unicode61 tokenization for broad language support.
+     */
+    initSchema() {
+        const db = this.db;
+        db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts USING fts5(
+        id UNINDEXED,
+        title,
+        tags,
+        content,
+        category UNINDEXED,
+        tokenize='porter unicode61'
+      );
+    `);
+    }
+    /**
+     * Upsert a document into the FTS5 index.
+     * Removes any existing row with the same ID before inserting.
+     *
+     * @param doc - The document to upsert
+     */
+    upsertDocument(doc) {
+        const db = this.getDb();
+        // FTS5 tables do not support ON CONFLICT, so delete-then-insert
+        const deleteStmt = db.prepare('DELETE FROM knowledge_fts WHERE id = ?');
+        const insertStmt = db.prepare('INSERT INTO knowledge_fts (id, title, tags, content, category) VALUES (?, ?, ?, ?, ?)');
+        const txn = db.transaction(() => {
+            deleteStmt.run(doc.id);
+            insertStmt.run(doc.id, doc.title, doc.tags, doc.content, doc.category);
+        });
+        txn();
+    }
+    /**
+     * Remove a document from the FTS5 index by its ID.
+     *
+     * @param id - The document identifier to remove
+     */
+    removeDocument(id) {
+        const db = this.getDb();
+        db.prepare('DELETE FROM knowledge_fts WHERE id = ?').run(id);
+    }
+    /**
+     * Search the FTS5 index using a MATCH query with BM25 ranking.
+     *
+     * @param query - The search query string (FTS5 MATCH syntax)
+     * @param options - Optional search parameters
+     * @param options.category - Filter results by category
+     * @param options.limit - Maximum number of results (default 10)
+     * @returns Array of matching documents sorted by relevance
+     */
+    search(query, options) {
+        if (!query || query.trim().length === 0) {
+            return [];
+        }
+        const db = this.getDb();
+        const limit = options?.limit ?? 10;
+        // Sanitize query: remove characters that break FTS5 MATCH syntax
+        const sanitized = query.replace(/['"*(){}[\]^~\\]/g, ' ').trim();
+        if (sanitized.length === 0) {
+            return [];
+        }
+        try {
+            if (options?.category) {
+                const stmt = db.prepare(`
+          SELECT id, title, tags, content, category, rank
+          FROM knowledge_fts
+          WHERE knowledge_fts MATCH ?
+            AND category = ?
+          ORDER BY rank
+          LIMIT ?
+        `);
+                return stmt.all(sanitized, options.category, limit);
+            }
+            const stmt = db.prepare(`
+        SELECT id, title, tags, content, category, rank
+        FROM knowledge_fts
+        WHERE knowledge_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `);
+            return stmt.all(sanitized, limit);
+        }
+        catch (err) {
+            this.logger.warn('FTS5 search query failed, returning empty results', {
+                query: sanitized,
+                error: err instanceof Error ? err.message : String(err),
+            });
+            return [];
+        }
+    }
+    /**
+     * Rebuild the entire FTS5 index from scratch.
+     * Drops all existing rows and reinserts the provided documents in a single transaction.
+     *
+     * @param documents - The complete set of documents to index
+     */
+    rebuildIndex(documents) {
+        const db = this.getDb();
+        const deleteAll = db.prepare('DELETE FROM knowledge_fts');
+        const insertStmt = db.prepare('INSERT INTO knowledge_fts (id, title, tags, content, category) VALUES (?, ?, ?, ?, ?)');
+        const txn = db.transaction(() => {
+            deleteAll.run();
+            for (const doc of documents) {
+                insertStmt.run(doc.id, doc.title, doc.tags, doc.content, doc.category);
+            }
+        });
+        txn();
+        this.logger.info('FTS5 index rebuilt', { documentCount: documents.length });
+    }
+    /**
+     * Close the database connection and release resources.
+     */
+    close() {
+        if (this.db) {
+            try {
+                this.db.close();
+                this.logger.debug('FTS5 index database closed', { path: this.dbPath });
+            }
+            catch (err) {
+                this.logger.warn('Error closing FTS5 database', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+            this.db = null;
+        }
+    }
+}
+//# sourceMappingURL=fts5-index.service.js.map

package/dist/cli/backend/src/services/knowledge/fts5-index.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts5-index.service.js","sourceRoot":"","sources":["../../../../../../backend/src/services/knowledge/fts5-index.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAwB,MAAM,2BAA2B,CAAC;AAEhF,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,mDAAmD;AACnD,IAAI,cAAc,GAA2C,IAAI,CAAC;AAElE;;;GAGG;AACH,SAAS,gBAAgB;IACvB,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,8DAA8D;YAC9D,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,wFAAwF;gBACtF,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxE,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,cAAe,CAAC;AACzB,CAAC;AA2CD,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E;;;;;;;;;;;;GAYG;AACH,MAAM,OAAO,gBAAgB;IACnB,EAAE,GAAwB,IAAI,CAAC;IACtB,MAAM,CAAS;IACf,MAAM,CAAkB;IAEzC;;;;OAIG;IACH,YAAY,aAAqB;QAC/B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;IACtF,CAAC;IAED;;;;;OAKG;IACK,KAAK;QACX,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,EAAE,CAAC;QACjB,CAAC;QAED,iCAAiC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,QAAQ,GAAG,gBAAgB,EAAE,CAAC;QACpC,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpC,sBAAsB;QACtB,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAEvC,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,UAAU;QAChB,MAAM,EAAE,GAAG,IAAI,CAAC,EAAG,CAAC;QACpB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;KASP,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,cAAc,CAAC,GAAgB;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAExB,gEAAgE;QAChE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,uFAAuF,CACxF,CAAC;QAEF,MAAM,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC9B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;QACH,GAAG,EAAE,CAAC;IACR,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,EAAU;QACvB,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACxB,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,KAAa,EAAE,OAA+C;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QAEnC,iEAAiE;QACjE,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACjE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC;YACH,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;gBACtB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;SAOvB,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAsB,CAAC;YAC3E,CAAC;YAED,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;OAMvB,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAsB,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mDAAmD,EAAE;gBACpE,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,SAAwB;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,uFAAuF,CACxF,CAAC;QAEF,MAAM,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC9B,SAAS,CAAC,GAAG,EAAE,CAAC;YAChB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzE,CAAC;QACH,CAAC,CAAC,CAAC;QACH,GAAG,EAAE,CAAC;QAEN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,aAAa,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE;oBAC9C,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;CACF"}

package/dist/cli/backend/src/services/knowledge/fts5-search-strategy.d.ts

@@ -0,0 +1,56 @@
+/**
+ * FTS5 Search Strategy
+ *
+ * Implements the KnowledgeSearchStrategy interface using the SQLite FTS5
+ * full-text search index. Maps FTS5 BM25-ranked results to the existing
+ * ScoredDocument format used by KnowledgeSearchService.
+ *
+ * @module services/knowledge/fts5-search-strategy
+ */
+import type { KnowledgeSearchStrategy, ScoredDocument } from './knowledge-search.service.js';
+import type { KnowledgeDocumentSummary } from '../../types/knowledge.types.js';
+import { WikiCompilerService } from './wiki-compiler.service.js';
+/**
+ * FTS5-based search strategy for the Wiki-First Knowledge System.
+ *
+ * This strategy provides pure full-text search via SQLite FTS5 with BM25 ranking.
+ *
+ * Note on semantic fallback (design doc Section 10): The 10% semantic fallback
+ * weight is achieved at the MemoryService level, which runs both FTS5 knowledge
+ * search AND vector-based semantic search in parallel. This strategy intentionally
+ * does not mix semantic results — that composition happens upstream in the recall
+ * pipeline (MemoryService.recall -> searchKnowledgeDocuments + semanticSearch).
+ *
+ * Uses SQLite FTS5 MATCH with BM25 ranking for fast, zero-cost retrieval.
+ * Results are mapped back to the KnowledgeDocumentSummary objects provided
+ * by the caller, falling back to minimal summaries when a document is found
+ * in the FTS5 index but not in the provided documents array.
+ *
+ * @example
+ * ```typescript
+ * const strategy = new Fts5SearchStrategy(wikiCompiler);
+ * const results = await strategy.search('deployment guide', documents);
+ * ```
+ */
+export declare class Fts5SearchStrategy implements KnowledgeSearchStrategy {
+    private readonly wikiCompiler;
+    private readonly logger;
+    /**
+     * Create an Fts5SearchStrategy.
+     *
+     * @param wikiCompiler - The WikiCompilerService instance providing FTS5 search
+     */
+    constructor(wikiCompiler: WikiCompilerService);
+    /**
+     * Search for documents matching the query using FTS5 full-text search.
+     *
+     * FTS5 BM25 rank values are negative (more negative = more relevant),
+     * so scores are converted to positive values for the ScoredDocument interface.
+     *
+     * @param query - The search query text
+     * @param documents - Candidate document summaries to match against
+     * @returns Documents sorted by descending relevance score
+     */
+    search(query: string, documents: KnowledgeDocumentSummary[]): Promise<ScoredDocument[]>;
+}
+//# sourceMappingURL=fts5-search-strategy.d.ts.map

package/dist/cli/backend/src/services/knowledge/fts5-search-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts5-search-strategy.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/knowledge/fts5-search-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC7F,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAGjE;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,kBAAmB,YAAW,uBAAuB;IAChE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IACnD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IAEzC;;;;OAIG;gBACS,YAAY,EAAE,mBAAmB;IAK7C;;;;;;;;;OASG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,wBAAwB,EAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;CAyC9F"}

package/dist/cli/backend/src/services/knowledge/fts5-search-strategy.js

@@ -0,0 +1,91 @@
+/**
+ * FTS5 Search Strategy
+ *
+ * Implements the KnowledgeSearchStrategy interface using the SQLite FTS5
+ * full-text search index. Maps FTS5 BM25-ranked results to the existing
+ * ScoredDocument format used by KnowledgeSearchService.
+ *
+ * @module services/knowledge/fts5-search-strategy
+ */
+import { LoggerService } from '../core/logger.service.js';
+/**
+ * FTS5-based search strategy for the Wiki-First Knowledge System.
+ *
+ * This strategy provides pure full-text search via SQLite FTS5 with BM25 ranking.
+ *
+ * Note on semantic fallback (design doc Section 10): The 10% semantic fallback
+ * weight is achieved at the MemoryService level, which runs both FTS5 knowledge
+ * search AND vector-based semantic search in parallel. This strategy intentionally
+ * does not mix semantic results — that composition happens upstream in the recall
+ * pipeline (MemoryService.recall -> searchKnowledgeDocuments + semanticSearch).
+ *
+ * Uses SQLite FTS5 MATCH with BM25 ranking for fast, zero-cost retrieval.
+ * Results are mapped back to the KnowledgeDocumentSummary objects provided
+ * by the caller, falling back to minimal summaries when a document is found
+ * in the FTS5 index but not in the provided documents array.
+ *
+ * @example
+ * ```typescript
+ * const strategy = new Fts5SearchStrategy(wikiCompiler);
+ * const results = await strategy.search('deployment guide', documents);
+ * ```
+ */
+export class Fts5SearchStrategy {
+    wikiCompiler;
+    logger;
+    /**
+     * Create an Fts5SearchStrategy.
+     *
+     * @param wikiCompiler - The WikiCompilerService instance providing FTS5 search
+     */
+    constructor(wikiCompiler) {
+        this.wikiCompiler = wikiCompiler;
+        this.logger = LoggerService.getInstance().createComponentLogger('Fts5SearchStrategy');
+    }
+    /**
+     * Search for documents matching the query using FTS5 full-text search.
+     *
+     * FTS5 BM25 rank values are negative (more negative = more relevant),
+     * so scores are converted to positive values for the ScoredDocument interface.
+     *
+     * @param query - The search query text
+     * @param documents - Candidate document summaries to match against
+     * @returns Documents sorted by descending relevance score
+     */
+    async search(query, documents) {
+        const ftsResults = this.wikiCompiler.search(query, { limit: 10 });
+        if (ftsResults.length === 0) {
+            this.logger.debug('FTS5 search returned no results', { query });
+            return [];
+        }
+        // Build a lookup map from the provided documents array
+        const docById = new Map(documents.map((d) => [d.id, d]));
+        const scored = [];
+        for (const ftsResult of ftsResults) {
+            // FTS5 rank is negative; convert to positive score (higher = more relevant)
+            const score = Math.abs(ftsResult.rank);
+            const existingDoc = docById.get(ftsResult.id);
+            if (existingDoc) {
+                scored.push({ document: existingDoc, score });
+            }
+            else {
+                // Create a minimal summary for documents in FTS5 but not in the provided list
+                const minimalDoc = {
+                    id: ftsResult.id,
+                    title: ftsResult.title,
+                    category: ftsResult.category,
+                    tags: ftsResult.tags ? ftsResult.tags.split(/,\s*/) : [],
+                    preview: ftsResult.content.slice(0, 200),
+                    scope: 'global',
+                    createdBy: 'system',
+                    updatedBy: 'system',
+                    createdAt: new Date().toISOString(),
+                    updatedAt: new Date().toISOString(),
+                };
+                scored.push({ document: minimalDoc, score });
+            }
+        }
+        return scored.sort((a, b) => b.score - a.score);
+    }
+}
+//# sourceMappingURL=fts5-search-strategy.js.map

package/dist/cli/backend/src/services/knowledge/fts5-search-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fts5-search-strategy.js","sourceRoot":"","sources":["../../../../../../backend/src/services/knowledge/fts5-search-strategy.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,EAAE,aAAa,EAAwB,MAAM,2BAA2B,CAAC;AAEhF;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,OAAO,kBAAkB;IACZ,YAAY,CAAsB;IAClC,MAAM,CAAkB;IAEzC;;;;OAIG;IACH,YAAY,YAAiC;QAC3C,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,oBAAoB,CAAC,CAAC;IACxF,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,SAAqC;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAElE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,uDAAuD;QACvD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEzD,MAAM,MAAM,GAAqB,EAAE,CAAC;QAEpC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,4EAA4E;YAC5E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAEvC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAE9C,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,8EAA8E;gBAC9E,MAAM,UAAU,GAA6B;oBAC3C,EAAE,EAAE,SAAS,CAAC,EAAE;oBAChB,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,QAAQ,EAAE,SAAS,CAAC,QAAQ;oBAC5B,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;oBACxD,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBACxC,KAAK,EAAE,QAAQ;oBACf,SAAS,EAAE,QAAQ;oBACnB,SAAS,EAAE,QAAQ;oBACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC;CACF"}

package/dist/cli/backend/src/services/knowledge/wiki-compiler.service.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Wiki Compiler Service
+ *
+ * Reads Markdown files with YAML frontmatter from the `.crewly/knowledge/`
+ * directory and indexes them into the FTS5 full-text search database.
+ * Provides reindexing, single-file indexing, and search capabilities.
+ *
+ * @module services/knowledge/wiki-compiler.service
+ */
+import { Fts5IndexService, type FtsSearchResult } from './fts5-index.service.js';
+/**
+ * Parsed frontmatter result containing extracted metadata and the body text.
+ */
+interface ParsedFrontmatter {
+    /** Key-value pairs extracted from YAML frontmatter */
+    frontmatter: Record<string, string>;
+    /** The markdown body after the frontmatter block */
+    body: string;
+}
+/**
+ * Parse YAML frontmatter from a Markdown file content string.
+ * Handles simple key: value pairs and bracket-delimited arrays.
+ * Does not require an external YAML parsing dependency.
+ *
+ * @param content - Raw file content including frontmatter
+ * @returns Parsed frontmatter object and body text
+ */
+export declare function parseFrontmatter(content: string): ParsedFrontmatter;
+/**
+ * WikiCompilerService scans `.md` files in a knowledge directory, parses
+ * their YAML frontmatter, and indexes the content into an FTS5 database
+ * for fast full-text search.
+ *
+ * @example
+ * ```typescript
+ * const compiler = new WikiCompilerService('/Users/me/.crewly/knowledge');
+ * const { indexed, errors } = await compiler.reindex();
+ * const results = compiler.search('deployment guide');
+ * ```
+ */
+export declare class WikiCompilerService {
+    private readonly fts5;
+    private readonly knowledgePath;
+    private readonly logger;
+    /**
+     * Create a WikiCompilerService.
+     *
+     * @param knowledgePath - Directory containing `.md` knowledge files
+     */
+    constructor(knowledgePath: string);
+    /**
+     * Get the path to the knowledge directory.
+     *
+     * @returns The absolute path to the knowledge directory
+     */
+    getKnowledgePath(): string;
+    /**
+     * Recursively find all `.md` files under the knowledge directory.
+     *
+     * @returns Array of absolute file paths
+     */
+    getMarkdownFiles(): Promise<string[]>;
+    /**
+     * Scan all `.md` files in the knowledge directory (recursively), parse
+     * their frontmatter, and rebuild the FTS5 index from scratch.
+     *
+     * @returns An object with the count of indexed documents and any errors encountered
+     */
+    reindex(): Promise<{
+        indexed: number;
+        errors: string[];
+    }>;
+    /**
+     * Index a single markdown file into the FTS5 database.
+     *
+     * @param filePath - Absolute path to the markdown file
+     */
+    indexFile(filePath: string): Promise<void>;
+    /**
+     * Search the knowledge base using FTS5 full-text search.
+     *
+     * @param query - The search query string
+     * @param options - Optional search parameters (category filter, result limit)
+     * @returns Array of matching documents sorted by BM25 relevance
+     */
+    search(query: string, options?: {
+        category?: string;
+        limit?: number;
+    }): FtsSearchResult[];
+    /**
+     * Get direct access to the underlying FTS5 index service.
+     *
+     * @returns The Fts5IndexService instance
+     */
+    getFts5(): Fts5IndexService;
+    /**
+     * Close the underlying FTS5 database connection.
+     */
+    close(): void;
+    /**
+     * Recursively find all `.md` files under the knowledge directory.
+     *
+     * @returns Array of absolute file paths
+     */
+    private globMarkdownFiles;
+    /**
+     * Read and parse a single markdown file into an FTS5 document shape.
+     *
+     * @param filePath - Absolute path to the markdown file
+     * @returns Parsed document ready for FTS5 indexing
+     */
+    private parseFile;
+}
+export {};
+//# sourceMappingURL=wiki-compiler.service.d.ts.map

package/dist/cli/backend/src/services/knowledge/wiki-compiler.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wiki-compiler.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/knowledge/wiki-compiler.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,EAAE,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAMjF;;GAEG;AACH,UAAU,iBAAiB;IACzB,sDAAsD;IACtD,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,oDAAoD;IACpD,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,iBAAiB,CA4BnE;AAkBD;;;;;;;;;;;GAWG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IAEzC;;;;OAIG;gBACS,aAAa,EAAE,MAAM;IAMjC;;;;OAIG;IACH,gBAAgB,IAAI,MAAM;IAI1B;;;;OAIG;IACG,gBAAgB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAI3C;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IA4B/D;;;;OAIG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhD;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe,EAAE;IAIzF;;;;OAIG;IACH,OAAO,IAAI,gBAAgB;IAI3B;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;;OAIG;YACW,iBAAiB;IAyB/B;;;;;OAKG;YACW,SAAS;CAcxB"}