@unbrained/pm-cli 2026.6.6 → 2026.6.7

package/dist/cli/commands/search.js

@@ -1,8 +1,8 @@
 
-!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="1091bd00-cd43-5ff5-bc98-41259fc28722")}catch(e){}}();
+!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="29fa7999-bb88-5675-8f5c-d953b2a80e6f")}catch(e){}}();
 import fs from "node:fs/promises";
 import path from "node:path";
-import { toNonEmptyStringOrUndefined } from "../../core/shared/primitives.js";
+import { coerceNumberInRange, toNonEmptyStringOrUndefined } from "../../core/shared/primitives.js";
 import { isPathWithinDirectory } from "../../core/fs/path-utils.js";
 import { getActiveExtensionRegistrations, runActiveOnReadHooks } from "../../core/extensions/index.js";
 import { collectRegisteredItemFieldNames } from "../../core/extensions/item-fields.js";
@@ -10,6 +10,7 @@ import { resolveRegisteredSearchProvider, resolveRegisteredVectorStoreAdapter, }
 import { resolveItemTypeRegistry } from "../../core/item/type-registry.js";
 import { parseLimit, parsePriority, parseType } from "../shared-parsers.js";
 import { executeEmbeddingRequest, resolveEmbeddingProviders, } from "../../core/search/providers.js";
+import { buildDeterministicQueryExpansions, mergeQueryExpansions, normalizeQueryExpansionOutput, normalizeRerankOutput, rerankCandidatesWithEmbeddings, resolveQueryExpansionConfig, resolveRerankConfig, } from "../../core/search/relevance.js";
 import { resolveSettingsWithSemanticRuntimeDefaults } from "../../core/search/semantic-defaults.js";
 import { executeVectorQuery, resolveVectorStores, } from "../../core/search/vector-stores.js";
 import { readVectorizationStatusLedger } from "../../core/search/cache.js";
@@ -209,6 +210,9 @@ function parseTitleExact(raw) {
 function parsePhraseExact(raw) {
     return raw === true;
 }
+function parseSemanticWeightOverride(raw) {
+    return coerceNumberInRange(raw, 0, 1) ?? undefined;
+}
 function normalizeSearchPhrase(value) {
     return value
         .toLowerCase()
@@ -631,7 +635,7 @@ export function resolveSearchTuning(settings) {
         linked_content_weight: resolveWeight(tuning.linked_content_weight, defaults.linked_content_weight),
     };
 }
-function emptySearchResult(query, mode, options, includeLinked, scoreThreshold, hybridSemanticWeight, projection, warnings) {
+function emptySearchResult(query, mode, options, includeLinked, scoreThreshold, hybridSemanticWeight, queryExpansion, rerank, projection, warnings) {
     const projectionFields = projection.mode === "full" ? null : [...projection.fields];
     return {
         query: query.trim(),
@@ -651,6 +655,11 @@ function emptySearchResult(query, mode, options, includeLinked, scoreThreshold,
             phrase_exact: options.phraseExact === true,
             score_threshold: scoreThreshold,
             hybrid_semantic_weight: mode === "hybrid" ? hybridSemanticWeight : null,
+            query_expansion_enabled: queryExpansion.enabled,
+            query_expansion_provider: queryExpansion.provider,
+            rerank_enabled: rerank.enabled,
+            rerank_model: rerank.model,
+            rerank_top_k: rerank.top_k,
             limit: options.limit ?? null,
         },
         projection: {
@@ -674,27 +683,45 @@ function requireSemanticDependencies(requestedMode, providerResolution, vectorRe
     };
 }
 const toOptionalNonEmptyString = toNonEmptyStringOrUndefined;
-function resolveExtensionSearchProvider(settings) {
+function resolveExtensionSearchProviderByName(providerName) {
     const registrations = getActiveExtensionRegistrations();
-    const providerName = toOptionalNonEmptyString(settings.search?.provider);
     const resolved = resolveRegisteredSearchProvider(registrations, providerName);
     if (!resolved) {
         return null;
     }
     const runtimeDefinition = resolved.runtime_definition ?? resolved.definition;
     const query = runtimeDefinition.query;
-    if (typeof query !== "function") {
-        return null;
-    }
+    const queryExpansion = runtimeDefinition.queryExpansion ??
+        runtimeDefinition.query_expansion;
+    const rerank = runtimeDefinition.rerank;
     const registeredName = toOptionalNonEmptyString(runtimeDefinition.name) ??
         toOptionalNonEmptyString(resolved.definition.name) ??
         providerName;
     if (!registeredName) {
         return null;
     }
-    return {
+    const hooks = {
         providerName: registeredName,
-        query: query,
+        ...(typeof query === "function" ? { query: query } : {}),
+        ...(typeof queryExpansion === "function"
+            ? { queryExpansion: queryExpansion }
+            : {}),
+        ...(typeof rerank === "function" ? { rerank: rerank } : {}),
+    };
+    if (!hooks.query && !hooks.queryExpansion && !hooks.rerank) {
+        return null;
+    }
+    return hooks;
+}
+function resolveExtensionSearchProvider(settings) {
+    const providerName = toOptionalNonEmptyString(settings.search?.provider);
+    const resolved = resolveExtensionSearchProviderByName(providerName);
+    if (!resolved?.query) {
+        return null;
+    }
+    return {
+        providerName: resolved.providerName,
+        query: resolved.query,
     };
 }
 function resolveExtensionVectorAdapter(settings) {
@@ -802,42 +829,187 @@ function combineHybridHits(filteredById, semanticScores, keywordHits, hybridSema
     })
         .filter((entry) => entry !== null);
 }
+function mergeVectorHitsById(vectorHitGroups) {
+    const bestById = new Map();
+    for (const group of vectorHitGroups) {
+        for (const hit of group) {
+            const existing = bestById.get(hit.id);
+            if (!existing || hit.score > existing.score) {
+                bestById.set(hit.id, hit);
+            }
+        }
+    }
+    const merged = [...bestById.values()];
+    merged.sort((left, right) => {
+        if (left.score !== right.score) {
+            return right.score - left.score;
+        }
+        return left.id.localeCompare(right.id);
+    });
+    return merged;
+}
+function buildRerankCorpus(document) {
+    const metadata = document.metadata;
+    const tags = Array.isArray(metadata?.tags) ? metadata.tags.join(" ") : "";
+    return [
+        metadata?.title,
+        metadata?.description,
+        metadata?.type,
+        metadata?.status,
+        tags,
+        document.body,
+    ]
+        .map((entry) => (typeof entry === "string" ? entry.trim() : ""))
+        .filter((entry) => entry.length > 0)
+        .join("\n");
+}
 async function computeSemanticOrHybridHits(context) {
     const semanticLimit = context.limit ?? context.maxResults;
     const embeddingOptions = context.embeddingTimeoutMs !== undefined ? { timeout_ms: context.embeddingTimeoutMs } : {};
     const vectorQueryOptions = context.vectorQueryTimeoutMs !== undefined ? { timeout_ms: context.vectorQueryTimeoutMs } : {};
-    const queryVectors = await executeEmbeddingRequest(context.provider, context.query.trim(), embeddingOptions);
-    const semanticVector = queryVectors[0];
-    let vectorHits;
-    if (context.extensionVectorAdapter?.query) {
-        try {
-            vectorHits = await Promise.resolve(context.extensionVectorAdapter.query({
-                vector: semanticVector,
-                limit: semanticLimit,
-                settings: context.settings,
-            }));
-        }
-        catch (error) {
-            if (!context.vectorStore) {
-                throw new PmCliError(`Extension vector adapter query failed and no built-in fallback store is configured (${error instanceof Error ? error.message : String(error)})`, EXIT_CODE.GENERIC_FAILURE);
+    const queryTrimmed = context.query.trim();
+    const baseExpandedQueries = context.queryExpansion.enabled
+        ? buildDeterministicQueryExpansions(queryTrimmed, context.queryExpansion.max_queries)
+        : [queryTrimmed];
+    let expandedQueries = baseExpandedQueries.length > 0 ? baseExpandedQueries : [queryTrimmed];
+    if (context.queryExpansion.enabled) {
+        if (context.queryExpansionExtension?.expand) {
+            try {
+                const rawExpansion = await Promise.resolve(context.queryExpansionExtension.expand({
+                    query: queryTrimmed,
+                    mode: context.requestedMode,
+                    settings: context.settings,
+                }));
+                const extensionExpansion = normalizeQueryExpansionOutput(rawExpansion);
+                expandedQueries = mergeQueryExpansions(expandedQueries, extensionExpansion, context.queryExpansion.max_queries);
+            }
+            catch {
+                context.warnings.push(`search_query_expansion_provider_failed:${context.queryExpansionExtension.providerName}:using_builtin`);
             }
-            vectorHits = await executeVectorQuery(context.vectorStore, semanticVector, semanticLimit, vectorQueryOptions);
+        }
+        else if (context.queryExpansion.provider &&
+            context.queryExpansion.provider !== "openai" &&
+            context.queryExpansion.provider !== "ollama") {
+            context.warnings.push(`search_query_expansion_provider_unavailable:${context.queryExpansion.provider}:using_builtin`);
         }
     }
-    else if (context.vectorStore) {
-        vectorHits = await executeVectorQuery(context.vectorStore, semanticVector, semanticLimit, vectorQueryOptions);
-    }
-    else {
+    const queryVectors = await executeEmbeddingRequest(context.provider, expandedQueries, embeddingOptions);
+    const executeVectorQueryWithFallback = async (semanticVector) => {
+        if (context.extensionVectorAdapter?.query) {
+            try {
+                return await Promise.resolve(context.extensionVectorAdapter.query({
+                    vector: semanticVector,
+                    limit: semanticLimit,
+                    settings: context.settings,
+                }));
+            }
+            catch (error) {
+                if (!context.vectorStore) {
+                    throw new PmCliError(`Extension vector adapter query failed and no built-in fallback store is configured (${error instanceof Error ? error.message : String(error)})`, EXIT_CODE.GENERIC_FAILURE);
+                }
+                return await executeVectorQuery(context.vectorStore, semanticVector, semanticLimit, vectorQueryOptions);
+            }
+        }
+        if (context.vectorStore) {
+            return await executeVectorQuery(context.vectorStore, semanticVector, semanticLimit, vectorQueryOptions);
+        }
         throw new PmCliError("Semantic search requires either a configured vector store or an extension vector adapter query handler", EXIT_CODE.USAGE);
-    }
+    };
+    const queryVectorGroups = await Promise.all(queryVectors.map(async (semanticVector) => await executeVectorQueryWithFallback(semanticVector)));
+    const vectorHits = mergeVectorHitsById(queryVectorGroups);
     const filteredById = new Map(context.filteredDocuments.map((document) => [document.metadata.id, document]));
     const { semanticHits, semanticScores } = buildSemanticHits(vectorHits, filteredById);
     const vectorMatchCount = semanticScores.size;
     if (context.requestedMode === "semantic") {
         return { hits: semanticHits, vectorMatchCount };
     }
+    let hybridHits = combineHybridHits(filteredById, semanticScores, context.keywordHits, context.hybridSemanticWeight);
+    if (context.rerank.enabled && hybridHits.length > 1) {
+        const sortedForCandidates = [...hybridHits].sort((left, right) => {
+            if (left.score !== right.score) {
+                return right.score - left.score;
+            }
+            return left.item.id.localeCompare(right.item.id);
+        });
+        const candidateHits = sortedForCandidates.slice(0, context.rerank.top_k);
+        const candidateContexts = candidateHits
+            .map((hit) => {
+            const document = filteredById.get(hit.item.id);
+            if (!document) {
+                return null;
+            }
+            return { hit, text: buildRerankCorpus(document) };
+        })
+            .filter((entry) => entry !== null);
+        const rerankCandidates = candidateContexts.map((entry) => ({
+            id: entry.hit.item.id,
+            text: entry.text,
+        }));
+        let rerankScores = null;
+        if (context.rerankExtension?.rerank) {
+            try {
+                const rawRerank = await Promise.resolve(context.rerankExtension.rerank({
+                    query: queryTrimmed,
+                    mode: "hybrid",
+                    model: context.rerank.model,
+                    top_k: context.rerank.top_k,
+                    settings: context.settings,
+                    candidates: candidateContexts.map((entry) => ({
+                        id: entry.hit.item.id,
+                        text: entry.text,
+                        score: entry.hit.score,
+                    })),
+                }));
+                const normalizedRerank = normalizeRerankOutput(rawRerank);
+                if (normalizedRerank.length > 0) {
+                    rerankScores = new Map(normalizedRerank.map((entry) => [entry.id, entry.score]));
+                }
+                else {
+                    context.warnings.push(`search_rerank_provider_invalid_response:${context.rerankExtension.providerName}:using_builtin`);
+                }
+            }
+            catch {
+                context.warnings.push(`search_rerank_provider_failed:${context.rerankExtension.providerName}:using_builtin`);
+            }
+        }
+        if (!rerankScores) {
+            try {
+                rerankScores = await rerankCandidatesWithEmbeddings(context.provider, context.rerank.model, queryTrimmed, rerankCandidates, context.embeddingTimeoutMs);
+            }
+            catch {
+                context.warnings.push("search_rerank_failed:using_hybrid_scores");
+            }
+        }
+        if (rerankScores && rerankScores.size > 0) {
+            const rerankedIds = new Set(rerankScores.keys());
+            hybridHits = hybridHits.map((hit) => {
+                const rerankScore = rerankScores.get(hit.item.id);
+                if (rerankScore === undefined) {
+                    return hit;
+                }
+                const matchedFields = new Set(hit.matched_fields);
+                matchedFields.add("rerank");
+                return {
+                    ...hit,
+                    score: rerankScore,
+                    matched_fields: [...matchedFields].sort((left, right) => left.localeCompare(right)),
+                };
+            });
+            hybridHits.sort((left, right) => {
+                const leftWasReranked = rerankedIds.has(left.item.id);
+                const rightWasReranked = rerankedIds.has(right.item.id);
+                if (leftWasReranked !== rightWasReranked) {
+                    return leftWasReranked ? -1 : 1;
+                }
+                if (left.score !== right.score) {
+                    return right.score - left.score;
+                }
+                return left.item.id.localeCompare(right.item.id);
+            });
+        }
+    }
     return {
-        hits: combineHybridHits(filteredById, semanticScores, context.keywordHits, context.hybridSemanticWeight),
+        hits: hybridHits,
         vectorMatchCount,
     };
 }
@@ -973,18 +1145,33 @@ export async function runSearch(query, options, global) {
     const typeRegistry = resolveItemTypeRegistry(settings, getActiveExtensionRegistrations());
     const maxResults = resolveSearchMaxResults(settings);
     const scoreThreshold = resolveSearchScoreThreshold(settings);
-    const hybridSemanticWeight = resolveHybridSemanticWeight(settings);
+    const semanticWeightProvided = options.semanticWeight !== undefined;
+    const semanticWeightOverride = parseSemanticWeightOverride(options.semanticWeight);
+    const hybridSemanticWeight = semanticWeightOverride ?? resolveHybridSemanticWeight(settings);
     const tuning = resolveSearchTuning(settings);
     const providerResolution = resolveEmbeddingProviders(settings);
     const vectorResolution = resolveVectorStores(settings);
     const extensionSearchProvider = resolveExtensionSearchProvider(settings);
     const extensionVectorAdapter = resolveExtensionVectorAdapter(settings);
+    const queryExpansion = resolveQueryExpansionConfig(settings, providerResolution.active?.name ?? null);
+    const rerank = resolveRerankConfig(settings, providerResolution.active?.model ?? toOptionalNonEmptyString(settings.search?.embedding_model) ?? "text-embedding-3-small");
+    const queryExpansionProvider = resolveExtensionSearchProviderByName(queryExpansion.provider ?? undefined);
+    const queryExpansionExtension = queryExpansionProvider?.queryExpansion
+        ? { providerName: queryExpansionProvider.providerName, expand: queryExpansionProvider.queryExpansion }
+        : null;
+    const rerankProvider = resolveExtensionSearchProviderByName(toOptionalNonEmptyString(settings.search?.provider));
+    const rerankExtension = rerankProvider?.rerank
+        ? { providerName: rerankProvider.providerName, rerank: rerankProvider.rerank }
+        : null;
     let effectiveMode = parseMode(options.mode, {
         hasProvider: providerResolution.active !== null || extensionSearchProvider !== null,
         hasVectorStore: vectorResolution.active !== null || extensionVectorAdapter !== null,
     });
     const loadedDocuments = await loadDocuments(pmRoot, settings.item_format ?? "toon", typeRegistry.type_to_folder, settings.schema);
     const warnings = loadedDocuments.warnings;
+    if (effectiveMode === "hybrid" && semanticWeightProvided && semanticWeightOverride === undefined) {
+        warnings.push("search_hybrid_semantic_weight_override_invalid:using_settings_default");
+    }
     const allDocuments = loadedDocuments.documents;
     const metadataFilteredDocuments = applyFilters(allDocuments, options, typeRegistry, runtimeFieldFilters, statusFilter);
     const filteredDocuments = applyExactQueryFilters(metadataFilteredDocuments, normalizedQuery, {
@@ -992,7 +1179,7 @@ export async function runSearch(query, options, global) {
         phraseExact,
     });
     if (effectiveMode === "keyword" && (filteredDocuments.length === 0 || limit === 0)) {
-        return emptySearchResult(query, effectiveMode, options, includeLinked, scoreThreshold, hybridSemanticWeight, projection, warnings);
+        return emptySearchResult(query, effectiveMode, options, includeLinked, scoreThreshold, hybridSemanticWeight, queryExpansion, rerank, projection, warnings);
     }
     const projectRoot = process.cwd();
     const globalRoot = resolveGlobalPmRoot(projectRoot);
@@ -1028,7 +1215,7 @@ export async function runSearch(query, options, global) {
                 requireSemanticDependencies(effectiveMode, providerResolution, vectorResolution, extensionVectorAdapter !== null);
             }
             if (filteredDocuments.length === 0 || limit === 0) {
-                return emptySearchResult(query, effectiveMode, options, includeLinked, scoreThreshold, hybridSemanticWeight, projection, warnings);
+                return emptySearchResult(query, effectiveMode, options, includeLinked, scoreThreshold, hybridSemanticWeight, queryExpansion, rerank, projection, warnings);
             }
             const filteredById = new Map(filteredDocuments.map((document) => [document.metadata.id, document]));
             const canUseBuiltInSemantic = providerResolution.active !== null && (vectorResolution.active !== null || extensionVectorAdapter !== null);
@@ -1064,6 +1251,11 @@ export async function runSearch(query, options, global) {
                     provider,
                     vectorStore,
                     extensionVectorAdapter,
+                    queryExpansion,
+                    queryExpansionExtension,
+                    rerank,
+                    rerankExtension,
+                    warnings,
                     settings,
                     ...(implicitHybridMode
                         ? {
@@ -1124,6 +1316,11 @@ export async function runSearch(query, options, global) {
             phrase_exact: phraseExact,
             score_threshold: scoreThreshold,
             hybrid_semantic_weight: effectiveMode === "hybrid" ? hybridSemanticWeight : null,
+            query_expansion_enabled: queryExpansion.enabled,
+            query_expansion_provider: queryExpansion.provider,
+            rerank_enabled: rerank.enabled,
+            rerank_model: rerank.model,
+            rerank_top_k: rerank.top_k,
             limit: options.limit ?? null,
             runtime_filters: runtimeFieldFilters,
         },
@@ -1136,4 +1333,4 @@ export async function runSearch(query, options, global) {
     };
 }
 //# sourceMappingURL=search.js.map
-//# debugId=1091bd00-cd43-5ff5-bc98-41259fc28722
+//# debugId=29fa7999-bb88-5675-8f5c-d953b2a80e6f