@lincoln504/pi-research 0.1.13 → 1.0.1

package/src/knowledge/index.ts

@@ -0,0 +1,126 @@
+import * as fs from 'node:fs';
+import { KnowledgeStore } from './store.ts';
+import { WriterQueue } from './writer-queue.ts';
+import { Chunker } from './chunker.ts';
+import { getConfig, validateConfig, getDbDir, type Config } from '../config.ts';
+import { logger } from '../logger.ts';
+import { getModelChunkConfig } from './model-config.ts';
+import type { MigrationStrategy } from './migration.ts';
+import type { 
+  IEmbedder, 
+  IKnowledgeStore 
+} from '../core/interfaces/knowledge-interfaces.ts';
+
+export { SUPPORTED_MODELS, getModelEmbedderConfig, getModelChunkConfig } from './model-config.ts';
+
+/** Migration strategy for model changes (read from config) */
+function getMigrationStrategy(config: Config): MigrationStrategy | undefined {
+  const strategy = config.MIGRATION_STRATEGY;
+  if (!strategy) return undefined;
+  
+  const validStrategies: MigrationStrategy[] = ['drop', 're-embed', 'backup'];
+  if (validStrategies.includes(strategy as MigrationStrategy)) {
+    return strategy as MigrationStrategy;
+  }
+  
+  logger.warn(`[knowledge] Invalid migration strategy '${strategy}'. Valid options: drop, re-embed, backup. Falling back to default (backup).`);
+  return undefined;
+}
+
+/**
+ * Result of knowledge store initialization
+ */
+export interface KnowledgeStoreComponents {
+  embedder: IEmbedder;
+  store: IKnowledgeStore;
+  writerQueue: WriterQueue;
+}
+
+const MAX_INIT_RETRIES = 5;
+const BASE_RETRY_DELAY_MS = 1000;
+const MAX_RETRY_DELAY_MS = 30000;
+
+/**
+ * Initialize knowledge store components.
+ */
+export async function createKnowledgeStoreComponents(
+  embedderFactory: () => Promise<IEmbedder>,
+  reconnectFactory?: () => Promise<IEmbedder>,
+  withLock?: <T>(fn: () => Promise<T>) => Promise<T>,
+  config: Config = getConfig(),
+  workspace: string = process.cwd()
+): Promise<KnowledgeStoreComponents | null> {
+  if (config.KNOWLEDGE_STORE_MODE === 'none') {
+    logger.debug('[knowledge] Knowledge store is disabled in configuration');
+    return null;
+  }
+  validateConfig(config);
+
+  for (let attempt = 1; attempt <= MAX_INIT_RETRIES; attempt++) {
+    let store: KnowledgeStore | undefined;
+    try {
+      logger.info(`[knowledge] Creating Knowledge Store components (attempt ${attempt}/${MAX_INIT_RETRIES})...`);
+
+      const embedder = await embedderFactory();
+      const migrationStrategy = getMigrationStrategy(config);
+
+      store = new KnowledgeStore({
+        dbDir: getDbDir(config, workspace),
+        embedder,
+        modelName: config.EMBEDDING_MODEL,
+        migrationStrategy: migrationStrategy,
+        reconnectFactory,
+        withLock,
+        workspace,
+        knowledgeMode: config.KNOWLEDGE_STORE_MODE,
+        ttlDays: config.KNOWLEDGE_STORE_CACHE_TTL_DAYS,
+        maxServeAgeDays: config.KNOWLEDGE_STORE_MAX_SERVE_AGE_DAYS,
+      });
+
+      const chunkCfg = getModelChunkConfig(config.EMBEDDING_MODEL);
+      const chunkOverlap = Math.round(chunkCfg.chunkSize * chunkCfg.overlapPct);
+      const chunker = new Chunker({ targetSize: chunkCfg.chunkSize, overlap: chunkOverlap });
+      const writerQueue = new WriterQueue({ store, chunker });
+
+      await store.initialize();
+
+      return { embedder, store, writerQueue };
+    } catch (err) {
+      // Close any partially-opened store before retrying/giving up: store.initialize()
+      // can throw AFTER lancedb.connect() set its db handle, and discarding the object
+      // without close() leaks the LanceDB connection/file handles (up to MAX_INIT_RETRIES
+      // times per failing startup).
+      if (store) {
+        try { await store.close(); } catch { /* best-effort cleanup */ }
+      }
+      const givingUp = attempt === MAX_INIT_RETRIES;
+      if (givingUp) {
+        throw err;
+      }
+      const backoffDelay = Math.min(BASE_RETRY_DELAY_MS * Math.pow(2, attempt - 1), MAX_RETRY_DELAY_MS);
+      const totalDelay = backoffDelay + Math.random() * 500;
+      logger.warn(`[knowledge] Attempt ${attempt}/${MAX_INIT_RETRIES} failed. Retrying in ${(totalDelay / 1000).toFixed(1)}s...`, err);
+      await new Promise(resolve => setTimeout(resolve, totalDelay));
+    }
+  }
+
+  throw new Error('Failed to create knowledge store components');
+}
+
+/**
+ * Force delete the entire knowledge store directory.
+ * Used for system-level clearing or recovery from corruption.
+ */
+export async function forceDeleteKnowledgeStore(config?: Config, workspace?: string): Promise<void> {
+  const dbDir = getDbDir(config, workspace);
+  if (fs.existsSync(dbDir)) {
+    try {
+      fs.rmSync(dbDir, { recursive: true, force: true });
+      fs.mkdirSync(dbDir, { recursive: true });
+      logger.info(`[knowledge] Knowledge store at ${dbDir} forcefully deleted and recreated.`);
+    } catch (err) {
+      logger.error(`[knowledge] Failed to forcefully delete directory ${dbDir}:`, err);
+      throw err;
+    }
+  }
+}

package/src/knowledge/lancedb-loader.ts

@@ -0,0 +1,35 @@
+/**
+ * Lazy loader for the @lancedb/lancedb native module.
+ *
+ * The knowledge store is only reached behind a dynamic import() (see
+ * infrastructure/service-initialization.ts, which loads the store service via
+ * `await import(...)` inside a lazy service factory). That source-level lazy
+ * boundary is, however, defeated by the CLI build: esbuild bundles src/cli.ts
+ * into a single ESM file with `packages: 'external'` and no code-splitting, and
+ * in that mode it HOISTS every *static* `import` — including transitive ones —
+ * to the top of the bundle. A top-of-module `import * as lancedb from
+ * '@lancedb/lancedb'` therefore becomes an eager top-level import in dist/cli.mjs,
+ * evaluated before main() runs. lancedb's entry eagerly loads its native binding,
+ * so on any platform where that binding is absent (Intel macOS — no darwin-x64
+ * binding is published; musl/Alpine; or an npm optional-dependency install miss,
+ * npm/cli#4828) the whole bundle fails to evaluate and even `pi-research --help`
+ * / `status` exit 1.
+ *
+ * Loading lancedb through a runtime import() instead keeps esbuild from hoisting
+ * it (a dynamic import stays a deferred call), so the native binding is required
+ * only on first actual store access — exactly where
+ * KnowledgeStoreService.initialize() already catches a missing native stack
+ * (isNativeStackUnavailableError) and degrades the store to DISABLED. Help,
+ * version, status, and browser-only research then work with no binding present.
+ *
+ * Modules that need lancedb *types* should keep `import type * as lancedb from
+ * '@lancedb/lancedb'` (type imports are erased and never hoisted); only runtime
+ * value access (connect, Index, rerankers, …) must go through getLancedb().
+ */
+
+let _lancedb: typeof import('@lancedb/lancedb') | undefined;
+
+/** Resolve the lancedb module, memoizing after the first load. */
+export async function getLancedb(): Promise<typeof import('@lancedb/lancedb')> {
+  return (_lancedb ??= await import('@lancedb/lancedb'));
+}

package/src/knowledge/migration.ts

@@ -0,0 +1,17 @@
+/**
+ * Simplified migration system for Knowledge Store.
+ *
+ * For a local research tool, we only need 3 strategies:
+ * - 'drop': Recreate table (fast, simple, data loss)
+ * - 're-embed': Preserve data by re-embedding with new model
+ * - 'backup': Rename old data and start fresh (safe, no data loss)
+ */
+
+export type MigrationStrategy = 'drop' | 're-embed' | 'backup';
+
+export interface MigrationResult {
+  strategy: MigrationStrategy;
+  success: boolean;
+  documentsProcessed: number;
+  error?: string;
+}

package/src/knowledge/model-config.ts

@@ -0,0 +1,132 @@
+/**
+ * Embedding Model Configuration
+ *
+ * Pure lookup table for supported embedding models.
+ * Extracted to a standalone module so both knowledge/index.ts and
+ * infrastructure/embedding/embedding-factory.ts can import it without
+ * creating a circular dependency between those two modules.
+ */
+
+interface ModelConfig {
+  pooling: 'mean' | 'cls' | 'last_token';
+  queryPrefix?: string;
+  chunkSize: number;
+  overlapPct: number;
+  maxTokens?: number;
+  batchSize?: number;
+  charsPerToken?: number;
+  documentPrefix?: string;
+  useCache?: boolean;
+  multilingual: boolean;
+}
+
+const MODEL_CONFIG: Record<string, ModelConfig> = {
+  'Xenova/multilingual-e5-small': {
+    pooling: 'mean',
+    chunkSize: 1500,
+    overlapPct: 0.15,
+    queryPrefix: 'query: ',
+    documentPrefix: 'passage: ',
+    charsPerToken: 3.5,
+    multilingual: true,
+  },
+  'Xenova/multilingual-e5-base': {
+    pooling: 'mean',
+    chunkSize: 1500,
+    overlapPct: 0.15,
+    queryPrefix: 'query: ',
+    documentPrefix: 'passage: ',
+    charsPerToken: 3.5,
+    multilingual: true,
+  },
+  'Xenova/bge-m3': {
+    pooling: 'cls',
+    chunkSize: 1500,
+    overlapPct: 0.15,
+    charsPerToken: 3.5,
+    multilingual: true,
+  },
+  'onnx-community/embeddinggemma-300m-ONNX': {
+    pooling: 'mean',
+    chunkSize: 1600,
+    overlapPct: 0.15,
+    queryPrefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query.\nQuery: ',
+    charsPerToken: 3.5,
+    multilingual: true,
+  },
+  'onnx-community/Qwen3-Embedding-0.6B-ONNX': {
+    pooling: 'last_token',
+    queryPrefix: 'Instruct: Given a web search query, retrieve relevant passages.\nQuery: ',
+    chunkSize: 1200,
+    overlapPct: 0.15,
+    maxTokens: 512,
+    batchSize: 2,
+    charsPerToken: 2.5,
+    useCache: false,
+    multilingual: true,
+  },
+  'Xenova/all-MiniLM-L6-v2': {
+    pooling: 'mean',
+    chunkSize: 800,
+    overlapPct: 0.15,
+    maxTokens: 256,
+    multilingual: false,
+  },
+  'Xenova/bge-small-en-v1.5': {
+    pooling: 'cls',
+    chunkSize: 1500,
+    overlapPct: 0.15,
+    multilingual: false,
+  },
+  'Xenova/all-mpnet-base-v2': {
+    pooling: 'mean',
+    chunkSize: 1200,
+    overlapPct: 0.15,
+    maxTokens: 384,
+    multilingual: false,
+  },
+  'onnx-community/granite-embedding-small-english-r2-ONNX': {
+    pooling: 'cls',
+    chunkSize: 1800,
+    overlapPct: 0.15,
+    maxTokens: 512,
+    batchSize: 8,
+    multilingual: false,
+  },
+};
+
+export const SUPPORTED_MODELS: ReadonlyArray<{ id: string; multilingual: boolean }> =
+  Object.entries(MODEL_CONFIG).map(([id, cfg]) => ({ id, multilingual: cfg.multilingual }));
+
+const DEFAULT_CHUNK_SIZE = 1200;
+const DEFAULT_OVERLAP_PCT = 0.15;
+
+export function getModelEmbedderConfig(modelId: string): {
+  pooling: 'mean' | 'cls' | 'last_token';
+  queryPrefix?: string;
+  documentPrefix?: string;
+  maxTokens?: number;
+  batchSize?: number;
+  charsPerToken?: number;
+  useCache?: boolean;
+} {
+  const cfg = MODEL_CONFIG[modelId];
+  return cfg
+    ? {
+        pooling: cfg.pooling,
+        queryPrefix: cfg.queryPrefix,
+        documentPrefix: cfg.documentPrefix,
+        maxTokens: cfg.maxTokens,
+        batchSize: cfg.batchSize,
+        charsPerToken: cfg.charsPerToken,
+        useCache: cfg.useCache,
+      }
+    : { pooling: 'mean' };
+}
+
+export function getModelChunkConfig(modelId: string): { chunkSize: number; overlapPct: number } {
+  const cfg = MODEL_CONFIG[modelId];
+  return cfg
+    ? { chunkSize: cfg.chunkSize, overlapPct: cfg.overlapPct }
+    : { chunkSize: DEFAULT_CHUNK_SIZE, overlapPct: DEFAULT_OVERLAP_PCT };
+}

package/src/knowledge/onnx-env.ts

@@ -0,0 +1,48 @@
+/**
+ * ONNX / transformers.js environment setup.
+ *
+ * `@huggingface/transformers` (which in turn loads the native onnxruntime-node
+ * binding) is pulled in LAZILY via ./transformers-loader.ts, never as a static
+ * import — otherwise esbuild would hoist it to the top of the CLI bundle and crash
+ * startup on any platform lacking the binding (see transformers-loader.ts). Both
+ * accessors below are async so the native ML stack loads only when the embedder is
+ * actually constructed — i.e. when the knowledge store is enabled and used.
+ */
+
+import { HFEnv } from '../core/interfaces/knowledge-interfaces.ts';
+import { logger } from '../logger.ts';
+import { getModelCacheDir } from './embedder-utils.ts';
+import { getTransformers } from './transformers-loader.ts';
+
+/**
+ * Get the HuggingFace env object (loads transformers lazily on first call).
+ */
+export async function getHFEnv() {
+  return (await getTransformers()).env;
+}
+
+/**
+ * Initialize the ONNX environment
+ */
+let onnxInitialized = false;
+export async function initializeONNXEnv(): Promise<void> {
+  if (onnxInitialized) return;
+
+  const hfEnv = (await getTransformers()).env;
+  hfEnv.cacheDir = getModelCacheDir();
+
+  try {
+    const envObj = hfEnv as unknown as HFEnv;
+    if (envObj.backends?.onnx) {
+      envObj.backends.onnx.logLevel = 'error';
+    }
+    // Fallback for older transformers.js versions if any
+    if ((envObj as any).onnx) {
+      (envObj as any).onnx.logLevel = 'error';
+    }
+  } catch (e) {
+    logger.debug('[embedder] Failed to set ONNX logLevel:', e);
+  }
+
+  onnxInitialized = true;
+}

package/src/knowledge/store-operations.ts

@@ -0,0 +1,291 @@
+/**
+ * Knowledge Store Operations
+ *
+ * Core database operations for the KnowledgeStore.
+ */
+
+// Type-only import: this module references lancedb solely in type positions
+// (lancedb.Table / lancedb.rerankers.*). A type import is erased at build time,
+// so esbuild never hoists a native-loading `import` into the CLI bundle. See
+// ./lancedb-loader.ts for why that matters.
+import type * as lancedb from '@lancedb/lancedb';
+import { logger } from '../logger.ts';
+import { metrics } from '../utils/metrics.ts';
+import { StoreDocument } from '../core/interfaces/knowledge-interfaces.ts';
+
+/**
+ * Add documents to the store
+ */
+export async function addDocumentsToStore(
+  table: lancedb.Table,
+  docs: StoreDocument[],
+  embedder: { embedMany(texts: string[]): Promise<(Float32Array | number[])[]>; getDimension?(): number | null },
+  isClosing: () => boolean,
+  workspace: string,
+  isGlobal: boolean
+): Promise<void> {
+  if (docs.length === 0) return;
+  if (isClosing()) {
+    logger.warn('[store] Ignoring addDocuments during close');
+    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
+    return;
+  }
+
+  const startTime = Date.now();
+
+  try {
+    const vectors = await embedder.embedMany(docs.map(d => d.text));
+
+    // Guard against a broken embedder silently writing corrupt vectors: a
+    // wrong-count, empty, or ragged result would otherwise land in the
+    // FixedSizeList column and surface only as an opaque Arrow error or, worse,
+    // as silently degraded search. Fail loudly here instead.
+    if (vectors.length !== docs.length) {
+      throw new Error(
+        `[store] embedder returned ${vectors.length} vectors for ${docs.length} documents`,
+      );
+    }
+    // Expected width: the embedder's declared dimension, falling back to the
+    // first vector's width. Every vector must match it exactly — a uniformly
+    // wrong width is just as corrupting as a ragged one.
+    const declaredDim = embedder.getDimension?.();
+    const dim = declaredDim && declaredDim > 0 ? declaredDim : (vectors[0]?.length ?? 0);
+    if (dim === 0) {
+      throw new Error('[store] embedder returned empty/zero-width vectors');
+    }
+    for (let i = 0; i < vectors.length; i++) {
+      const v = vectors[i];
+      if (!v || v.length !== dim) {
+        throw new Error(
+          `[store] embedder returned a wrong-width vector at index ${i} (got ${v?.length ?? 0}, expected ${dim})`,
+        );
+      }
+      // A correct-width vector can still be poisoned: a WebGPU device-lost
+      // mid-inference, or a truncated server response coerced through
+      // Float32Array, yields NaN/Inf entries that pass every length check but
+      // silently corrupt the ANN index (NaN distances break ordering). Scan
+      // here — this is the last gate before the vector is persisted.
+      for (let j = 0; j < v.length; j++) {
+        if (!Number.isFinite(v[j]!)) {
+          throw new Error(
+            `[store] embedder returned a non-finite value (${v[j]}) at vector ${i}, position ${j}`,
+          );
+        }
+      }
+    }
+
+    const data = docs.map((doc, i) => ({
+      vector: vectors[i]!,
+      url: doc.url,
+      text: doc.text,
+      content: doc.content ?? null,
+      metadata: JSON.stringify(doc.metadata),
+      workspace,
+      is_global: isGlobal,
+      ingestion_type: doc.ingestion_type || (doc.metadata['ingestionType'] as string) || 'unknown',
+      timestamp: BigInt(doc.timestamp),
+    }));
+
+    await table.add(data);
+    const duration = Date.now() - startTime;
+    metrics.observe('knowledge_store_add_documents_duration_ms', duration);
+    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'success' });
+    metrics.increment('knowledge_store_chunks_added_total', docs.length);
+    logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [workspace=${workspace}, global=${isGlobal}]`);
+  } catch (err) {
+    const duration = Date.now() - startTime;
+    metrics.observe('knowledge_store_add_documents_duration_ms', duration, { status: 'error' });
+    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
+    logger.error('[store] Failed to add documents:', err);
+    throw err;
+  }
+}
+
+/**
+ * Search for documents in the store using hybrid search.
+ *
+ * Semantic search (vector similarity) targets the 'text' field (synthesis descriptions)
+ * since vectors are computed from description chunks. Keyword search (BM25/FTS) targets
+ * BOTH 'text' and 'content' fields — 'text' for description keywords and 'content'
+ * for full article body text. Results are fused via Reciprocal Rank Fusion (RRF).
+ */
+export async function searchStore(
+  table: lancedb.Table,
+  embedder: { embed(query: string): Promise<Float32Array | number[]> },
+  query: string,
+  getReranker: () => Promise<lancedb.rerankers.RRFReranker>,
+  limit: number,
+  scopeFilter?: string
+): Promise<StoreDocument[]> {
+  const startTime = Date.now();
+
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment('knowledge_store_search_total', 1, { status: 'empty' });
+    return [];
+  }
+
+  const vector = await embedder.embed(query);
+
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  const results = await table
+    .query()
+    .nearestTo(vector)
+    .where(filter)
+    .fullTextSearch(query, { columns: ['text', 'content'] })
+    .rerank(await getReranker())
+    .limit(limit)
+    .toArray();
+
+  const filteredResults = results
+    .map(r => {
+      let metadata: Record<string, unknown>;
+      try {
+        metadata = JSON.parse(r.metadata as string) as Record<string, unknown>;
+      } catch {
+        logger.debug('[store] Corrupted metadata in searchStore, skipping row');
+        return null;
+      }
+      return {
+        url: r.url as string,
+        text: r.text as string,
+        content: (r.content as string | null) ?? undefined,
+        metadata,
+        timestamp: Number(r.timestamp),
+      };
+    })
+    .filter((doc): doc is NonNullable<typeof doc> => doc !== null);
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_search_duration_ms', duration);
+  metrics.increment('knowledge_store_search_total', 1, { status: 'success' });
+  metrics.increment('knowledge_store_search_results_total', filteredResults.length);
+
+  return filteredResults;
+}
+
+/**
+ * Find documents by URL
+ */
+export async function findDocumentsByUrl(
+  table: lancedb.Table,
+  url: string,
+  scopeFilter?: string
+): Promise<StoreDocument[]> {
+  const startTime = Date.now();
+
+  // Escape single quotes in URL to prevent SQL injection
+  const escapedUrl = url.replace(/'/g, "''");
+
+  let filter = `url = '${escapedUrl}'`;
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  const results = await table
+    .query()
+    .where(filter)
+    .limit(1000)
+    .toArray();
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_query_duration_ms', duration, { operation: 'find_by_url' });
+  metrics.increment('knowledge_store_query_total', 1, { operation: 'find_by_url' });
+
+  if (results.length === 1000) {
+    logger.warn(`[store] findByUrl hit 1000-chunk cap for ${url} - some chunks may be missing`);
+    metrics.increment('knowledge_store_query_cap_hits_total', 1);
+  }
+
+  return results.map(r => {
+    let metadata: Record<string, unknown> = {};
+    try { 
+      metadata = JSON.parse(r.metadata as string); 
+    } catch {
+      logger.debug('[store] Corrupted metadata in findDocumentsByUrl, using empty object');
+    }
+    return {
+    url: r.url as string,
+    text: r.text as string,
+    content: (r.content as string | null) ?? undefined,
+    metadata,
+    timestamp: Number(r.timestamp),
+    };
+  });
+}
+
+/**
+ * Find URLs relevant to a query using hybrid search.
+ *
+ * Semantic search (vector) targets the 'text' field (descriptions).
+ * Keyword search (BM25/FTS) targets both 'text' and 'content' fields.
+ * Results are fused via RRF reranking.
+ */
+export async function findRelevantUrls(
+  table: lancedb.Table,
+  embedder: { embed(query: string): Promise<Float32Array | number[]> },
+  query: string,
+  getReranker: () => Promise<lancedb.rerankers.RRFReranker>,
+  limit: number,
+  scopeFilter?: string
+): Promise<{ url: string; description: string; provenance?: string }[]> {
+  const startTime = Date.now();
+  const rowCount = scopeFilter ? await table.countRows(scopeFilter) : await table.countRows();
+  if (rowCount === 0) {
+    metrics.increment('knowledge_store_find_urls_total', 1, { status: 'empty' });
+    return [];
+  }
+
+  const vector = await embedder.embed(query);
+
+  let filter = "ingestion_type = 'synthesis-description'";
+  if (scopeFilter) {
+    filter = `(${filter}) AND (${scopeFilter})`;
+  }
+
+  const results = await table
+    .query()
+    .nearestTo(vector)
+    .where(filter)
+    .fullTextSearch(query, { columns: ['text', 'content'] })
+    .rerank(await getReranker())
+    .limit(limit)
+    .toArray();
+
+  const entries: { url: string; description: string; provenance?: string }[] = [];
+  const seen = new Set<string>();
+  for (const r of results) {
+    const url = r.url as string;
+    if (seen.has(url)) continue;
+    seen.add(url);
+
+    let description = '';
+    let provenance = 'description-unverified';
+    try {
+      const meta = JSON.parse(r.metadata as string);
+      description = meta.description as string ?? '';
+      // Surface provenance metadata so consumers can prefer verified entries
+      if (meta.provenance) provenance = meta.provenance;
+      else if (meta.hasContent === true) provenance = 'scraped-verified';
+    } catch { logger.debug('[store] Corrupted metadata in findRelevantUrls, skipping row'); }
+    
+    if (!description) description = (r.text as string ?? '').substring(0, 300);
+    
+    // Add store scope to provenance
+    const scope = r.is_global === true ? 'Global Store' : `Local Project (${r.workspace || 'unknown'})`;
+    const finalProvenance = `${provenance} [${scope}]`;
+    
+    entries.push({ url, description, provenance: finalProvenance });
+  }
+
+  const duration = Date.now() - startTime;
+  metrics.observe('knowledge_store_find_urls_duration_ms', duration);
+  metrics.increment('knowledge_store_find_urls_total', 1, { status: 'success' });
+  metrics.increment('knowledge_store_urls_found_total', entries.length);
+
+  return entries;
+}