@remnic/core 1.1.31 → 9.3.516

package/src/search/embed-helper.ts

@@ -1,5 +1,7 @@
 import { log } from "../logger.js";
 import type { PluginConfig } from "../types.js";
+import { isAbortError } from "../abort-error.js";
+import { withTimeoutSignal } from "./abort.js";
 
 type ProviderConfig = {
   type: "openai" | "local";
@@ -38,23 +40,29 @@ export class EmbedHelper {
   /**
    * Embed a single text string. Returns null if no provider is available.
    */
-  async embed(text: string): Promise<number[] | null> {
+  async embed(text: string, options: { signal?: AbortSignal } = {}): Promise<number[] | null> {
     const provider = this.getProvider();
     if (!provider) return null;
-    return this.callEmbed(text, provider);
+    return this.callEmbed(text, provider, options.signal);
   }
 
   /**
    * Embed a batch of texts. Returns an array parallel to input; entries are null on failure.
    */
-  async embedBatch(texts: string[], batchSize = 32): Promise<(number[] | null)[]> {
+  async embedBatch(
+    texts: string[],
+    batchSize = 32,
+    options: { signal?: AbortSignal } = {},
+  ): Promise<(number[] | null)[]> {
     const provider = this.getProvider();
     if (!provider) return texts.map(() => null);
 
     const results: (number[] | null)[] = new Array(texts.length).fill(null);
     for (let i = 0; i < texts.length; i += batchSize) {
       const batch = texts.slice(i, i + batchSize);
-      const batchResults = await Promise.all(batch.map((t) => this.callEmbed(t, provider)));
+      const batchResults = await Promise.all(
+        batch.map((t) => this.callEmbed(t, provider, options.signal)),
+      );
       for (let j = 0; j < batchResults.length; j++) {
         results[i + j] = batchResults[j];
       }
@@ -114,7 +122,11 @@ export class EmbedHelper {
     return null;
   }
 
-  private async callEmbed(input: string, provider: ProviderConfig): Promise<number[] | null> {
+  private async callEmbed(
+    input: string,
+    provider: ProviderConfig,
+    signal?: AbortSignal,
+  ): Promise<number[] | null> {
     try {
       const res = await fetch(provider.endpoint, {
         method: "POST",
@@ -124,7 +136,7 @@ export class EmbedHelper {
           input: input.slice(0, 8000),
           encoding_format: "float",
         }),
-        signal: AbortSignal.timeout(30_000),
+        signal: withTimeoutSignal(signal, 30_000),
       });
       if (!res.ok) {
         log.debug(`EmbedHelper request failed: ${provider.type} ${res.status}`);
@@ -135,6 +147,7 @@ export class EmbedHelper {
       if (!Array.isArray(vector)) return null;
       return vector.map((n: unknown) => { const v = Number(n); return Number.isFinite(v) ? v : 0; });
     } catch (err) {
+      if (isAbortError(err)) throw err;
       log.debug(`EmbedHelper error: ${err}`);
       return null;
     }

package/src/search/factory.ts

@@ -161,9 +161,10 @@ export function createConversationIndexRuntime(
   },
 ): ConversationIndexRuntime {
   const qmd = createConversationSearchBackend(config) as ConversationQmdRuntime | undefined;
-  const faiss =
-    config.conversationIndexEnabled && config.conversationIndexBackend === "faiss"
-      ? new FaissConversationIndexAdapter({
+  let faiss: FaissConversationIndexAdapter | undefined;
+  if (config.conversationIndexEnabled && config.conversationIndexBackend === "faiss") {
+    try {
+      faiss = new FaissConversationIndexAdapter({
           memoryDir: config.memoryDir,
           scriptPath: config.conversationIndexFaissScriptPath,
           pythonBin: config.conversationIndexFaissPythonBin,
@@ -174,8 +175,11 @@ export function createConversationIndexRuntime(
           healthTimeoutMs: config.conversationIndexFaissHealthTimeoutMs,
           maxBatchSize: config.conversationIndexFaissMaxBatchSize,
           maxSearchK: config.conversationIndexFaissMaxSearchK,
-        })
-      : undefined;
+      });
+    } catch (err) {
+      log.warn(`Conversation index FAISS adapter disabled: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
 
   const backend = createConversationIndexBackend({
     enabled: config.conversationIndexEnabled,

package/src/search/lancedb-backend.ts

@@ -2,6 +2,7 @@ import { log } from "../logger.js";
 import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions, SearchResult } from "./port.js";
 import type { EmbedHelper } from "./embed-helper.js";
 import { scanMemoryDir } from "./document-scanner.js";
+import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
 
 export interface LanceDbBackendOptions {
   dbPath: string;
@@ -60,24 +61,26 @@ export class LanceDbBackend implements SearchBackend {
     _collection?: string,
     maxResults?: number,
     _options?: SearchQueryOptions,
-    _execution?: SearchExecutionOptions,
+    execution?: SearchExecutionOptions,
   ): Promise<SearchResult[]> {
-    return this.hybridSearch(query, _collection, maxResults);
+    return this.hybridSearch(query, _collection, maxResults, execution);
   }
 
-  async searchGlobal(query: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+  async searchGlobal(query: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
     const limit = maxResults ?? 10;
     if (!this.available) return [];
 
     try {
+      throwIfSearchAborted(execution, "LanceDbBackend global search aborted");
       const db = await this.ensureDb();
       const tableNames = await db.tableNames();
       const allResults: SearchResult[] = [];
 
       for (const name of tableNames) {
+        throwIfSearchAborted(execution, "LanceDbBackend global search aborted");
         try {
           const table = await db.openTable(name);
-          const results = await this.searchTable(table, query, "hybrid", limit);
+          const results = await this.searchTable(table, query, "hybrid", limit, execution);
           allResults.push(...results);
         } catch {
           // Skip tables that fail
@@ -92,33 +95,42 @@ export class LanceDbBackend implements SearchBackend {
     }
   }
 
-  async bm25Search(query: string, collection?: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+  async bm25Search(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+    if (isSearchAborted(execution)) return [];
     const table = await this.ensureTableForCollection(collection ?? this.collection);
+    if (isSearchAborted(execution)) return [];
     if (!table) return [];
-    return this.searchTable(table, query, "fts", maxResults ?? 10);
+    return this.searchTable(table, query, "fts", maxResults ?? 10, execution);
   }
 
-  async vectorSearch(query: string, collection?: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+  async vectorSearch(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+    if (isSearchAborted(execution)) return [];
     const table = await this.ensureTableForCollection(collection ?? this.collection);
+    if (isSearchAborted(execution)) return [];
     if (!table) return [];
-    return this.searchTable(table, query, "vector", maxResults ?? 10);
+    return this.searchTable(table, query, "vector", maxResults ?? 10, execution);
   }
 
-  async hybridSearch(query: string, collection?: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+  async hybridSearch(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+    if (isSearchAborted(execution)) return [];
     const table = await this.ensureTableForCollection(collection ?? this.collection);
+    if (isSearchAborted(execution)) return [];
     if (!table) return [];
-    return this.searchTable(table, query, "hybrid", maxResults ?? 10);
+    return this.searchTable(table, query, "hybrid", maxResults ?? 10, execution);
   }
 
   async update(execution?: SearchExecutionOptions): Promise<void> {
     await this.updateCollection(this.collection, execution);
   }
 
-  async updateCollection(collection: string, _execution?: SearchExecutionOptions): Promise<void> {
+  async updateCollection(collection: string, execution?: SearchExecutionOptions): Promise<void> {
+    if (isSearchAborted(execution)) return;
     const table = await this.ensureTableForCollection(collection);
+    if (isSearchAborted(execution)) return;
     if (!table) return;
 
     const docs = await scanMemoryDir(this.memoryDir);
+    if (isSearchAborted(execution)) return;
     if (docs.length === 0) {
       // Clear stale data when no docs remain
       try {
@@ -131,27 +143,40 @@ export class LanceDbBackend implements SearchBackend {
       return;
     }
 
+    const existingVectors = new Map<string, number[]>();
+    try {
+      const existingRows = await table.query().select(["docid", "vector"]).toArray();
+      for (const row of existingRows ?? []) {
+        if (isSearchAborted(execution)) return;
+        const docid = row.docid;
+        if (typeof docid !== "string") continue;
+        const vector = row.vector;
+        if (!vector || typeof vector !== "object") continue;
+        existingVectors.set(docid, Array.from(vector as ArrayLike<number>));
+      }
+    } catch {
+      // Vector preservation is best-effort; refresh can proceed without it.
+    }
+
     const rows = docs.map((d) => ({
       docid: d.docid,
       path: d.path,
       content: d.content,
       snippet: d.snippet,
-      vector: new Array(this.embeddingDimension).fill(0),
+      vector: existingVectors.get(d.docid) ?? new Array(this.embeddingDimension).fill(0),
     }));
 
     try {
-      // Overwrite with fresh data
-      const db = await this.ensureDb();
-      await db.dropTable(collection).catch(() => {});
-      if (collection === this.collection) this.table = null;
-      const newTable = await db.createTable(collection, rows);
+      if (isSearchAborted(execution)) return;
+      await table.add(rows, { mode: "overwrite" });
+      if (isSearchAborted(execution)) return;
       // Create FTS index on content column
       try {
-        await newTable.createIndex("content", { config: this.lanceIndex.fts() });
+        await table.createIndex("content", { config: this.lanceIndex.fts() });
       } catch {
         // FTS index creation may fail on some platforms — degrade gracefully
       }
-      if (collection === this.collection) this.table = newTable;
+      if (collection === this.collection) this.table = table;
     } catch (err) {
       log.debug(`LanceDbBackend update failed: ${err}`);
     }
@@ -193,7 +218,10 @@ export class LanceDbBackend implements SearchBackend {
     }
   }
 
-  async ensureCollection(_memoryDir: string): Promise<"present" | "missing" | "unknown" | "skipped"> {
+  async ensureCollection(
+    _memoryDir: string,
+    _execution?: SearchExecutionOptions,
+  ): Promise<"present" | "missing" | "unknown" | "skipped"> {
     try {
       await this.ensureTable();
       return "present";
@@ -286,28 +314,41 @@ export class LanceDbBackend implements SearchBackend {
     return this.table;
   }
 
-  private async searchTable(table: any, query: string, mode: "fts" | "vector" | "hybrid", limit: number): Promise<SearchResult[]> {
+  private async searchTable(
+    table: any,
+    query: string,
+    mode: "fts" | "vector" | "hybrid",
+    limit: number,
+    execution?: SearchExecutionOptions,
+  ): Promise<SearchResult[]> {
     try {
+      throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
       if (mode === "fts") {
         const results = await table.search(query, "fts").limit(limit).toArray();
+        throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       }
 
       if (mode === "vector") {
-        const vec = await this.embedHelper.embed(query);
+        const vec = await this.embedHelper.embed(query, { signal: execution?.signal });
+        throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         if (!vec) {
           // Fall back to FTS
           const results = await table.search(query, "fts").limit(limit).toArray();
+          throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
           return this.mapRows(results);
         }
         const results = await table.search(vec).limit(limit).toArray();
+        throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       }
 
       // hybrid — try FTS+vector with RRF reranking
-      const vec = await this.embedHelper.embed(query);
+      const vec = await this.embedHelper.embed(query, { signal: execution?.signal });
+      throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
       if (!vec) {
         const results = await table.search(query, "fts").limit(limit).toArray();
+        throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       }
 
@@ -317,10 +358,12 @@ export class LanceDbBackend implements SearchBackend {
           .vector(vec)
           .limit(limit)
           .toArray();
+        throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       } catch {
         // Hybrid may not be supported in all LanceDB versions — fall back to vector
         const results = await table.search(vec).limit(limit).toArray();
+        throwIfSearchAborted(execution, `LanceDbBackend ${mode} search aborted`);
         return this.mapRows(results);
       }
     } catch (err) {

package/src/search/meilisearch-backend.ts

@@ -1,6 +1,7 @@
 import { log } from "../logger.js";
 import type { SearchBackend, SearchExecutionOptions, SearchQueryOptions, SearchResult } from "./port.js";
 import { scanMemoryDir } from "./document-scanner.js";
+import { isSearchAborted, throwIfSearchAborted } from "./abort.js";
 
 export interface MeilisearchBackendOptions {
   host: string;
@@ -63,23 +64,27 @@ export class MeilisearchBackend implements SearchBackend {
     collection?: string,
     maxResults?: number,
     _options?: SearchQueryOptions,
-    _execution?: SearchExecutionOptions,
+    execution?: SearchExecutionOptions,
   ): Promise<SearchResult[]> {
+    if (isSearchAborted(execution)) return [];
     // Try hybrid first; fall back to plain FTS only if hybrid throws (e.g. no embedder configured)
     try {
-      return await this.doSearch(query, maxResults ?? 10, { hybrid: { semanticRatio: 0.5, embedder: "default" } }, collection, true);
+      return await this.doSearch(query, maxResults ?? 10, { hybrid: { semanticRatio: 0.5, embedder: "default" } }, collection, true, execution);
     } catch {
-      return this.bm25Search(query, collection, maxResults);
+      if (isSearchAborted(execution)) return [];
+      return this.bm25Search(query, collection, maxResults, execution);
     }
   }
 
-  async searchGlobal(query: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+  async searchGlobal(query: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
     const limit = maxResults ?? 10;
     if (!this.available) return [];
 
     try {
+      throwIfSearchAborted(execution, "MeilisearchBackend global search aborted");
       const client = await this.ensureClient();
       const indexes = await client.getIndexes();
+      throwIfSearchAborted(execution, "MeilisearchBackend global search aborted");
       const queries = (indexes.results ?? []).map((idx: any) => ({
         indexUid: idx.uid,
         q: query,
@@ -89,6 +94,7 @@ export class MeilisearchBackend implements SearchBackend {
       if (queries.length === 0) return [];
 
       const multiResult = await client.multiSearch({ queries });
+      throwIfSearchAborted(execution, "MeilisearchBackend global search aborted");
       const allResults: SearchResult[] = [];
       for (const result of multiResult.results ?? []) {
         allResults.push(...this.mapHits(result.hits ?? []));
@@ -101,29 +107,32 @@ export class MeilisearchBackend implements SearchBackend {
     }
   }
 
-  async bm25Search(query: string, collection?: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
-    return this.doSearch(query, maxResults ?? 10, undefined, collection);
+  async bm25Search(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+    return this.doSearch(query, maxResults ?? 10, undefined, collection, false, execution);
   }
 
-  async vectorSearch(query: string, collection?: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
-    return this.doSearch(query, maxResults ?? 10, { hybrid: { semanticRatio: 1.0, embedder: "default" } }, collection);
+  async vectorSearch(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+    return this.doSearch(query, maxResults ?? 10, { hybrid: { semanticRatio: 1.0, embedder: "default" } }, collection, false, execution);
   }
 
-  async hybridSearch(query: string, collection?: string, maxResults?: number, _execution?: SearchExecutionOptions): Promise<SearchResult[]> {
-    return this.doSearch(query, maxResults ?? 10, { hybrid: { semanticRatio: 0.5, embedder: "default" } }, collection);
+  async hybridSearch(query: string, collection?: string, maxResults?: number, execution?: SearchExecutionOptions): Promise<SearchResult[]> {
+    return this.doSearch(query, maxResults ?? 10, { hybrid: { semanticRatio: 0.5, embedder: "default" } }, collection, false, execution);
   }
 
   async update(execution?: SearchExecutionOptions): Promise<void> {
     await this.updateCollection(this.collection, execution);
   }
 
-  async updateCollection(collection: string, _execution?: SearchExecutionOptions): Promise<void> {
+  async updateCollection(collection: string, execution?: SearchExecutionOptions): Promise<void> {
     if (!this.autoIndex || !this.memoryDir) return;
     if (!this.available) return;
+    if (isSearchAborted(execution)) return;
 
     try {
       const client = await this.ensureClient();
+      if (isSearchAborted(execution)) return;
       const docs = await scanMemoryDir(this.memoryDir);
+      if (isSearchAborted(execution)) return;
       const index = client.index(collection);
 
       const meilDocs = docs.map((d) => ({
@@ -134,8 +143,10 @@ export class MeilisearchBackend implements SearchBackend {
       }));
 
       // Upsert current docs and wait for the task to complete
+      if (isSearchAborted(execution)) return;
       const addTask = await index.addDocuments(meilDocs, { primaryKey: "id" });
       await client.waitForTask(addTask.taskUid, { timeOutMs: this.timeoutMs });
+      if (isSearchAborted(execution)) return;
 
       // Remove docs that no longer exist on disk (paginated to handle large indexes)
       const currentIds = new Set(docs.map((d) => d.docid));
@@ -145,6 +156,7 @@ export class MeilisearchBackend implements SearchBackend {
         let staleIds: string[] = [];
         let hasMore = true;
         while (hasMore) {
+          if (isSearchAborted(execution)) return;
           const page = await index.getDocuments({ limit: PAGE_SIZE, offset, fields: ["id"] });
           const results = page.results ?? [];
           for (const doc of results) {
@@ -155,6 +167,7 @@ export class MeilisearchBackend implements SearchBackend {
           hasMore = results.length === PAGE_SIZE;
         }
         if (staleIds.length > 0) {
+          if (isSearchAborted(execution)) return;
           const delTask = await index.deleteDocuments(staleIds);
           await client.waitForTask(delTask.taskUid, { timeOutMs: this.timeoutMs });
         }
@@ -176,7 +189,10 @@ export class MeilisearchBackend implements SearchBackend {
     // manages embeddings server-side per index (collection).
   }
 
-  async ensureCollection(_memoryDir: string): Promise<"present" | "missing" | "unknown" | "skipped"> {
+  async ensureCollection(
+    _memoryDir: string,
+    _execution?: SearchExecutionOptions,
+  ): Promise<"present" | "missing" | "unknown" | "skipped"> {
     if (!this.available) return "skipped";
     try {
       const client = await this.ensureClient();
@@ -207,12 +223,22 @@ export class MeilisearchBackend implements SearchBackend {
     return this.client;
   }
 
-  private async doSearch(query: string, limit: number, extra?: Record<string, unknown>, collection?: string, rethrow = false): Promise<SearchResult[]> {
+  private async doSearch(
+    query: string,
+    limit: number,
+    extra?: Record<string, unknown>,
+    collection?: string,
+    rethrow = false,
+    execution?: SearchExecutionOptions,
+  ): Promise<SearchResult[]> {
     if (!this.available) return [];
+    if (isSearchAborted(execution)) return [];
     try {
       const client = await this.ensureClient();
+      throwIfSearchAborted(execution, "MeilisearchBackend search aborted");
       const index = client.index(collection ?? this.collection);
       const result = await index.search(query, { limit, showRankingScore: true, ...extra });
+      throwIfSearchAborted(execution, "MeilisearchBackend search aborted");
       return this.mapHits(result.hits ?? []);
     } catch (err) {
       log.debug(`MeilisearchBackend search failed: ${err}`);

package/src/search/noop-backend.ts

@@ -51,7 +51,7 @@ export class NoopSearchBackend implements SearchBackend {
   async embed(): Promise<void> {}
   async embedCollection(_collection: string): Promise<void> {}
 
-  async ensureCollection(_memoryDir: string): Promise<"skipped"> {
+  async ensureCollection(_memoryDir: string, _execution?: SearchExecutionOptions): Promise<"skipped"> {
     return "skipped";
   }
 }

package/src/search/orama-backend.test.ts

@@ -0,0 +1,27 @@
+import assert from "node:assert/strict";
+import path from "node:path";
+import test from "node:test";
+
+import { resolveOramaCollectionDbFilePath } from "./orama-backend.js";
+
+test("Orama collection filenames cannot escape dbPath", () => {
+  const dbPath = path.join("/tmp", "remnic-orama-db");
+
+  assert.equal(
+    resolveOramaCollectionDbFilePath(dbPath, "openclaw-engram"),
+    path.join(dbPath, "openclaw-engram.msp"),
+  );
+  for (const collection of [
+    "../outside",
+    "nested/name",
+    "",
+    ".hidden",
+    "collection name",
+  ]) {
+    assert.throws(
+      () => resolveOramaCollectionDbFilePath(dbPath, collection),
+      /Invalid Orama collection/,
+      collection,
+    );
+  }
+});