@hesed/search 0.2.0 → 0.2.1

package/README.md

@@ -20,7 +20,7 @@ $ npm install -g @hesed/search
 $ search COMMAND
 running command...
 $ search (--version)
-@hesed/search/0.2.0 linux-x64 node-v24.16.0
+@hesed/search/0.2.1 linux-x64 node-v24.16.0
 $ search --help [COMMAND]
 USAGE
   $ search COMMAND
@@ -62,5 +62,5 @@ EXAMPLES
   $ search search "update jira" --details
 ```
 
-_See code: [src/commands/search.ts](https://github.com/hesedcasa/search/blob/v0.2.0/src/commands/search.ts)_
+_See code: [src/commands/search.ts](https://github.com/hesedcasa/search/blob/v0.2.1/src/commands/search.ts)_
 <!-- commandsstop -->

package/dist/commands/search.js

@@ -1,5 +1,4 @@
-import { Args, Command, CommandHelp, Flags, toConfiguredId, ux } from '@oclif/core';
-import { isMiniLMModelCached, MiniLMCommandEmbedder } from '../embedders/minilm.js';
+import { Args, Command, CommandHelp, Flags, toConfiguredId } from '@oclif/core';
 import { searchCommands } from '../search-logic.js';
 export default class Search extends Command {
     static args = {
@@ -19,14 +18,7 @@ export default class Search extends Command {
     async run() {
         const { args, flags } = await this.parse(Search);
         const allCommands = this.config.commands.filter((c) => !c.hidden && c.pluginName !== '@oclif/plugin-plugins');
-        const loader = createModelLoader(this.jsonEnabled() || isMiniLMModelCached());
-        let scored;
-        try {
-            scored = (await searchCommands(args.query, allCommands, getSearchOptions(this.config, loader.onProgress))).slice(0, flags.limit);
-        }
-        finally {
-            loader.stop();
-        }
+        const scored = (await searchCommands(args.query, allCommands)).slice(0, flags.limit);
         const results = scored.map((entry) => {
             const { cmd } = entry;
             const configuredId = toConfiguredId(cmd.id, this.config);
@@ -85,61 +77,3 @@ export default class Search extends Command {
         }
     }
 }
-function getSearchOptions(config, onLoadProgress) {
-    const testConfig = config;
-    if (!testConfig.searchEmbedder)
-        return { embedder: new MiniLMCommandEmbedder({ onLoadProgress }) };
-    return { embedder: testConfig.searchEmbedder };
-}
-function createModelLoader(silent) {
-    let started = false;
-    let stopped = false;
-    const start = (status) => {
-        if (silent || stopped)
-            return;
-        if (started) {
-            ux.action.status = status;
-            return;
-        }
-        ux.action.start('Loading semantic search model', status);
-        started = true;
-    };
-    const stop = () => {
-        if (silent || stopped)
-            return;
-        stopped = true;
-        if (started)
-            ux.action.stop();
-    };
-    return {
-        onProgress(progress) {
-            if (progress.status === 'ready') {
-                stop();
-                return;
-            }
-            if (progress.status === 'progress_total' && typeof progress.progress === 'number') {
-                start(`${Math.round(progress.progress)}% ${formatBytes(progress.loaded)} / ${formatBytes(progress.total)}`);
-                return;
-            }
-            if (progress.status === 'download') {
-                start(formatProgressFile(progress.file));
-            }
-        },
-        stop,
-    };
-}
-function formatProgressFile(file) {
-    return file?.split('/').at(-1);
-}
-function formatBytes(value) {
-    if (!value || value <= 0)
-        return '?';
-    const units = ['B', 'KB', 'MB', 'GB'];
-    let size = value;
-    let unit = 0;
-    while (size >= 1024 && unit < units.length - 1) {
-        size /= 1024;
-        unit++;
-    }
-    return `${size.toFixed(size >= 10 || unit === 0 ? 0 : 1)} ${units[unit]}`;
-}

package/dist/search-logic.d.ts

@@ -8,19 +8,5 @@ export type ScoredCommand<T extends SearchableCommand = SearchableCommand> = {
     cmd: T;
     score: number;
 };
-export type CommandEmbedder = {
-    embed(texts: string[]): Promise<number[][]>;
-};
-export type CommandReranker<T extends SearchableCommand = SearchableCommand> = {
-    rerank(query: string, commands: Array<ScoredCommand<T>>): Promise<Array<ScoredCommand<T>>>;
-};
-export type SearchCommandsOptions<T extends SearchableCommand = SearchableCommand> = {
-    embedder?: CommandEmbedder;
-    lexicalWeight?: number;
-    minSemanticScore?: number;
-    reranker?: CommandReranker<T>;
-    semantic?: boolean;
-    semanticWeight?: number;
-};
-export declare function searchCommands<T extends SearchableCommand>(query: string, commands: T[], options?: SearchCommandsOptions<T>): Promise<Array<ScoredCommand<T>>>;
+export declare function searchCommands<T extends SearchableCommand>(query: string, commands: T[]): Promise<Array<ScoredCommand<T>>>;
 export declare function searchCommandsLexically<T extends SearchableCommand>(query: string, commands: T[], haystack?: string[]): Array<ScoredCommand<T>>;

package/dist/search-logic.js

@@ -1,39 +1,18 @@
-import UFuzzy from '@leeoniya/ufuzzy';
-import { MiniLMCommandEmbedder } from './embedders/minilm.js';
-const DEFAULT_MIN_SEMANTIC_SCORE = 0.22;
-const DEFAULT_SEMANTIC_WEIGHT = 0.75;
-const DEFAULT_LEXICAL_WEIGHT = 0.25;
-let defaultEmbedderPromise;
-export async function searchCommands(query, commands, options = {}) {
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+const { Index } = require('flexsearch');
+export async function searchCommands(query, commands) {
     const normalizedQuery = query.trim();
     if (normalizedQuery.length === 0 || commands.length === 0)
         return [];
     const haystack = commands.map((command) => commandSearchText(command));
-    const lexical = searchCommandsLexically(normalizedQuery, commands, haystack);
-    if (options.semantic === false)
-        return lexical;
-    let semantic;
-    try {
-        semantic = await searchCommandsSemantically(normalizedQuery, commands, haystack, options);
-    }
-    catch {
-        return lexical;
-    }
-    if (semantic.length === 0)
-        return lexical;
-    const merged = mergeScores(semantic, lexical, {
-        lexicalWeight: options.lexicalWeight ?? DEFAULT_LEXICAL_WEIGHT,
-        minSemanticScore: options.minSemanticScore ?? DEFAULT_MIN_SEMANTIC_SCORE,
-        semanticWeight: options.semanticWeight ?? DEFAULT_SEMANTIC_WEIGHT,
-    });
-    return options.reranker ? options.reranker.rerank(normalizedQuery, merged) : merged;
+    return searchCommandsLexically(normalizedQuery, commands, haystack);
 }
 export function searchCommandsLexically(query, commands, haystack = commands.map((command) => commandSearchText(command))) {
-    const uf = new UFuzzy({ intraIns: Infinity });
-    const [idxs, , order] = uf.search(haystack, query, 0, Infinity);
-    if (idxs && idxs.length > 0) {
-        const ranked = order ?? idxs.map((_, i) => i);
-        return ranked.map((oi, rank) => ({ cmd: commands[idxs[oi]], score: rank }));
+    const index = createCommandSearchIndex(haystack);
+    const idxs = index.search(query, { limit: commands.length, suggest: true });
+    if (idxs.length > 0) {
+        return idxs.map((idx, rank) => ({ cmd: commands[Number(idx)], score: rank }));
     }
     // Multi-token fallback: score each command by how many individual query
     // tokens it matches. Handles queries containing unknown alias words (e.g.
@@ -43,16 +22,23 @@ export function searchCommandsLexically(query, commands, haystack = commands.map
         return [];
     const hitCount = new Map();
     for (const token of tokens) {
-        const [tIdxs] = uf.search(haystack, token, 0, Infinity);
-        if (tIdxs) {
-            for (const idx of tIdxs)
-                hitCount.set(idx, (hitCount.get(idx) ?? 0) + 1);
+        const tIdxs = index.search(token, { limit: commands.length, suggest: true });
+        for (const idx of tIdxs) {
+            const commandIndex = Number(idx);
+            hitCount.set(commandIndex, (hitCount.get(commandIndex) ?? 0) + 1);
         }
     }
     return [...hitCount.entries()]
         .sort((a, b) => b[1] - a[1] || a[0] - b[0])
         .map(([idx, hits]) => ({ cmd: commands[idx], score: tokens.length - hits }));
 }
+function createCommandSearchIndex(haystack) {
+    const index = new Index({ encoder: 'LatinAdvanced', resolution: 9, tokenize: 'forward' });
+    for (const [idx, text] of haystack.entries()) {
+        index.add(idx, text);
+    }
+    return index;
+}
 function commandSearchText(command) {
     return [command.id, command.summary ?? command.description ?? '', command.pluginName ?? '']
         .filter(Boolean)
@@ -60,47 +46,3 @@ function commandSearchText(command) {
         .replaceAll(/\s+/g, ' ')
         .trim();
 }
-async function searchCommandsSemantically(query, commands, haystack, options) {
-    const embedder = options.embedder ?? (await getDefaultEmbedder());
-    const [queryEmbedding, ...commandEmbeddings] = await embedder.embed([query, ...haystack]);
-    return commandEmbeddings
-        .map((embedding, idx) => ({
-        cmd: commands[idx],
-        score: cosineSimilarity(queryEmbedding, embedding),
-    }))
-        .filter(({ score }) => Number.isFinite(score))
-        .sort((a, b) => b.score - a.score);
-}
-async function getDefaultEmbedder() {
-    defaultEmbedderPromise ??= Promise.resolve(new MiniLMCommandEmbedder());
-    return defaultEmbedderPromise;
-}
-function mergeScores(semantic, lexical, options) {
-    const lexicalById = new Map(lexical.map((entry, idx) => [entry.cmd.id, lexicalScore(idx, lexical.length)]));
-    const lexicalIds = new Set(lexical.map((entry) => entry.cmd.id));
-    return semantic
-        .filter((entry) => entry.score >= options.minSemanticScore || lexicalIds.has(entry.cmd.id))
-        .map((entry) => {
-        const finalScore = entry.score * options.semanticWeight + (lexicalById.get(entry.cmd.id) ?? 0) * options.lexicalWeight;
-        return { cmd: entry.cmd, score: 1 - finalScore };
-    })
-        .sort((a, b) => a.score - b.score);
-}
-function lexicalScore(index, length) {
-    if (length <= 1)
-        return 1;
-    return 1 - index / (length - 1);
-}
-function cosineSimilarity(a, b) {
-    let dot = 0;
-    let aMagnitude = 0;
-    let bMagnitude = 0;
-    for (let i = 0; i < Math.min(a.length, b.length); i++) {
-        dot += a[i] * b[i];
-        aMagnitude += a[i] * a[i];
-        bMagnitude += b[i] * b[i];
-    }
-    if (aMagnitude === 0 || bMagnitude === 0)
-        return 0;
-    return dot / (Math.sqrt(aMagnitude) * Math.sqrt(bMagnitude));
-}

package/oclif.manifest.json

@@ -58,5 +58,5 @@
       ]
     }
   },
-  "version": "0.2.0"
+  "version": "0.2.1"
 }

package/package.json

@@ -1,16 +1,15 @@
 {
   "name": "@hesed/search",
   "description": "Intelligence search plugin",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "author": "Hesed",
   "bin": {
     "permission": "./bin/run.js"
   },
   "bugs": "https://github.com/hesedcasa/search/issues",
   "dependencies": {
-    "@huggingface/transformers": "^4.2.0",
-    "@leeoniya/ufuzzy": "^1.0.19",
-    "@oclif/core": "^4"
+    "@oclif/core": "^4",
+    "flexsearch": "^0.8.212"
   },
   "devDependencies": {
     "@eslint/compat": "^1",

package/dist/embedders/minilm.d.ts

@@ -1,22 +0,0 @@
-import { type CommandEmbedder } from '../search-logic.js';
-export type ModelLoadProgress = {
-    file?: string;
-    loaded?: number;
-    model?: string;
-    name?: string;
-    progress?: number;
-    status: string;
-    task?: string;
-    total?: number;
-};
-export declare class MiniLMCommandEmbedder implements CommandEmbedder {
-    private readonly options;
-    private extractorPromise;
-    constructor(options?: {
-        onLoadProgress?: (progress: ModelLoadProgress) => void;
-    });
-    embed(texts: string[]): Promise<number[][]>;
-    private getExtractor;
-}
-export declare function isMiniLMModelCached(): boolean;
-export declare function getMiniLMModelCachePath(): string;

package/dist/embedders/minilm.js

@@ -1,49 +0,0 @@
-import { existsSync } from 'node:fs';
-import { createRequire } from 'node:module';
-import path from 'node:path';
-const MINILM_MODEL = 'Xenova/paraphrase-MiniLM-L3-v2';
-const MODEL_CACHE_FILES = ['config.json', 'tokenizer.json', 'tokenizer_config.json', 'onnx/model.onnx'];
-// eslint-disable-next-line no-new-func
-const importTransformers = new Function('specifier', 'return import(specifier)');
-const require = createRequire(import.meta.url);
-export class MiniLMCommandEmbedder {
-    options;
-    extractorPromise;
-    constructor(options = {}) {
-        this.options = options;
-    }
-    async embed(texts) {
-        if (texts.length === 0)
-            return [];
-        const extractor = await this.getExtractor();
-        const output = await extractor(texts, { normalize: true, pooling: 'mean' });
-        return tensorToRows(output.data, output.dims);
-    }
-    async getExtractor() {
-        this.extractorPromise ??= importTransformers('@huggingface/transformers').then(async ({ pipeline }) => {
-            const pipelineOptions = {};
-            // eslint-disable-next-line camelcase
-            pipelineOptions.progress_callback = this.options.onLoadProgress;
-            const extractor = await pipeline('feature-extraction', MINILM_MODEL, pipelineOptions);
-            return extractor;
-        });
-        return this.extractorPromise;
-    }
-}
-export function isMiniLMModelCached() {
-    const cachePath = getMiniLMModelCachePath();
-    return MODEL_CACHE_FILES.every((file) => existsSync(path.join(cachePath, file)));
-}
-export function getMiniLMModelCachePath() {
-    const entrypointPath = require.resolve('@huggingface/transformers');
-    const packagePath = path.dirname(path.dirname(entrypointPath));
-    return path.join(packagePath, '.cache', ...MINILM_MODEL.split('/'));
-}
-function tensorToRows(data, dims) {
-    const rows = dims[0] ?? 0;
-    const columns = dims[1] ?? data.length;
-    return Array.from({ length: rows }, (_, row) => {
-        const rowData = data.slice(row * columns, (row + 1) * columns);
-        return Array.isArray(rowData) ? rowData : [...rowData];
-    });
-}