@autorag/librarian 2.10.3 → 2.10.5

package/dist/minsync/client.d.ts

@@ -38,6 +38,14 @@ export declare class MinSyncClient {
     private writeIdentity;
     private identityMismatch;
     sync(force?: boolean, signal?: AbortSignal): Promise<MinSyncSyncResult>;
+    /**
+     * The tokenizer to run this sync with: always {@link MINSYNC_LEXICAL_LANGUAGE}, which fails every sync when the
+     * Kiwi native library is missing. A workspace that is new, or tokenized differently, then gets whitespace
+     * tokenization instead and the sync reports `lexicalFallback`. An index already built with the multilingual
+     * tokenizer is left alone: switching it would re-embed every chunk, and the probe can miss a library that the OS
+     * loader finds on its own.
+     */
+    private resolveLexicalLanguage;
     private lockHeldResult;
     private checkAndSync;
     query(text: string, topK: number, mode?: MinSyncQueryMode, signal?: AbortSignal): Promise<readonly MinSyncQueryHit[]>;

package/dist/minsync/embedder-config.d.ts

@@ -1,3 +1,4 @@
+import type { MinSyncLexicalLanguage } from "./lexical-language.js";
 import type { MinSyncEmbedderConfig } from "./types.js";
 export interface MinSyncEmbeddingIdentity {
     readonly provider: string;
@@ -23,6 +24,11 @@ export declare function minSyncConfigPath(workspacePath: string): string;
 export declare function minSyncEmbeddingIdentityPath(workspacePath: string): string;
 export declare function configuredMaxChunkSize(workspacePath: string): number | undefined;
 export declare function configuredVectorDimension(workspacePath: string): number | undefined;
+/**
+ * The BM25 tokenizer in `[lexical].language`, or undefined when there is no config. A config without the section
+ * (written before MinSync had one) behaves as MinSync's default, `simple`.
+ */
+export declare function configuredLexicalLanguage(workspacePath: string): string | undefined;
 /**
  * Atomically rewrite allowlisted embedder fields in MinSync config.toml.
  * Reads the existing file, merges the allowlisted fields from `embedder`,
@@ -32,9 +38,13 @@ export declare function configuredVectorDimension(workspacePath: string): number
  * Allowlisted [embedder] fields: id, base_url, query_prefix, passage_prefix,
  * batch_size, max_retries, max_concurrent, timeout_seconds (from timeoutMs / 1000 ceil).
  * Dimension is written under [vectorstore.options] as `dimension`.
+ * `managed.maxChunkSize` goes to [chunker.options] and `managed.lexicalLanguage` to [lexical].language. MinSync
+ * itself records the tokenizer an index was built with in its cursor and, when [lexical].language differs, deletes
+ * and re-indexes every chunk on the next sync, so changing it needs no `--full` from the caller.
  *
  * Returns true if the file was rewritten, false if the file does not exist.
  */
-export declare function rewriteEmbedderConfig(workspacePath: string, embedder: MinSyncEmbedderConfig, chunkerConfig?: {
+export declare function rewriteEmbedderConfig(workspacePath: string, embedder: MinSyncEmbedderConfig, managed?: {
     readonly maxChunkSize?: number;
+    readonly lexicalLanguage?: MinSyncLexicalLanguage;
 }): boolean;

package/dist/minsync/installer.d.ts

@@ -1,4 +1,4 @@
-export declare const MINSYNC_VERSION = "0.4.8";
+export declare const MINSYNC_VERSION = "0.4.10";
 export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
 export interface MinSyncReleaseAsset {
     readonly name: string;

package/dist/minsync/kiwi.d.ts

@@ -0,0 +1,17 @@
+/** What the probe reads, injectable so every platform's layout can be checked from any host. */
+export interface KiwiProbeEnvironment {
+    readonly env: NodeJS.ProcessEnv;
+    readonly platform: NodeJS.Platform;
+    readonly home: string | undefined;
+    readonly exists: (path: string) => boolean;
+}
+/**
+ * MinSync's `ko` tokenizer loads the Kiwi native library and its base model at run time (through `kiwi-rs`); neither
+ * ships with MinSync. Without them every sync that needs Korean tokenization fails, so AutoRAG checks first.
+ *
+ * This mirrors the filesystem part of `kiwi-rs`'s discovery (`KIWI_LIBRARY_PATH` / `KIWI_MODEL_PATH`, then the
+ * per-user `~/.local/kiwi` install and the well-known system directories). Names left to the OS loader search path
+ * are not probed, so a library found only that way reads as missing; point `KIWI_LIBRARY_PATH` at it to fix that.
+ * kiwi-rs treats an explicit path as authoritative, and so does this probe.
+ */
+export declare function kiwiAvailable(environment?: KiwiProbeEnvironment): boolean;

package/dist/minsync/lexical-language.d.ts

@@ -0,0 +1,18 @@
+/**
+ * The BM25 tokenizer presets AutoRAG sets in MinSync's `[lexical].language`. MinSync also has `ko`, `ja`, `zh` and
+ * `ar`, which AutoRAG never selects.
+ */
+export type MinSyncLexicalLanguage = "simple" | "multilingual";
+/**
+ * The tokenizer AutoRAG always asks MinSync for, whatever the global `languages` setting says.
+ *
+ * `simple` splits on whitespace only, so a Korean word with a particle (인증서를) never matches its stem (인증서).
+ * `ko` fixes that by indexing only Kiwi's morphemes, but then a compound (연구목표) exists only as 연구 and 목표 and an
+ * exact-term query loses to documents that merely contain both pieces. `multilingual` indexes the original
+ * whitespace-separated words as well as every analyzer's tokens, so exact terms and inflected forms both match. On
+ * a sample of real Korean documents (60 stem queries with inflected forms, 40 exact-term queries, BM25 only) it had the
+ * best recall on both and kept most of `simple`'s top-5 precision on exact terms, where `ko` lost most of it.
+ *
+ * The preset needs the Kiwi native library; see `kiwi.ts`.
+ */
+export declare const MINSYNC_LEXICAL_LANGUAGE: MinSyncLexicalLanguage;

package/dist/minsync/types.d.ts

@@ -1,4 +1,5 @@
 /** Embedder configuration for MinSync vector indexing. Shared by config and method layers. */
+import type { MinSyncLexicalLanguage } from "./lexical-language.js";
 export interface MinSyncEmbedderConfig {
     /** Select an AutoRAG-owned embedding runtime profile. */
     readonly profile?: "qwen3-embedding-0.6b";
@@ -41,6 +42,14 @@ export interface MinSyncSyncResult {
     readonly stagingExcluded?: readonly string[];
     /** Files MinSync skipped (exit 6, `partial`); the sync itself completed and the next sync retries them. */
     readonly filesFailed?: readonly MinSyncFileFailure[];
+    /**
+     * Set when the multilingual tokenizer needs the Kiwi native library and it was not found, so the index was
+     * built with whitespace tokenization instead. The sync itself succeeded.
+     */
+    readonly lexicalFallback?: {
+        readonly requested: MinSyncLexicalLanguage;
+        readonly used: MinSyncLexicalLanguage;
+    };
 }
 export interface MinSyncFileFailure {
     readonly path: string;

package/dist/web/search/types.d.ts

@@ -32,10 +32,6 @@ export declare const SEARCH_PROVIDER_OPTIONS: readonly [{
     readonly value: "xai";
     readonly label: "xAI";
     readonly description: "xAI Responses web_search via the agent's xAI model credentials (XAI_API_KEY)";
-}, {
-    readonly value: "perplexity";
-    readonly label: "Perplexity";
-    readonly description: "Anonymous Perplexity ask endpoint; no key or signup required";
 }, {
     readonly value: "parallel";
     readonly label: "Parallel";
@@ -44,14 +40,18 @@ export declare const SEARCH_PROVIDER_OPTIONS: readonly [{
     readonly value: "searxng";
     readonly label: "SearXNG";
     readonly description: "Advanced: requires a self-hosted SEARXNG_ENDPOINT; never part of the default experience";
-}, {
-    readonly value: "startpage";
-    readonly label: "Startpage";
-    readonly description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged";
 }, {
     readonly value: "duckduckgo";
     readonly label: "DuckDuckGo";
     readonly description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs";
+}, {
+    readonly value: "perplexity";
+    readonly label: "Perplexity";
+    readonly description: "Anonymous Perplexity ask endpoint; no key or signup required";
+}, {
+    readonly value: "startpage";
+    readonly label: "Startpage";
+    readonly description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged";
 }, {
     readonly value: "ecosia";
     readonly label: "Ecosia";
@@ -78,8 +78,12 @@ export type SearchProviderId = Exclude<(typeof SEARCH_PROVIDER_OPTIONS)[number][
 /**
  * Auto-resolution priority order. Derived from {@link SEARCH_PROVIDER_OPTIONS}
  * (minus `auto`) so any dropdown/setting and `resolveProviderChain()` share
- * one source of truth and never drift apart: credential-free and
- * model-credential-reusing providers first, then the scraped engines, with
+ * one source of truth and never drift apart: model-credential-reusing
+ * providers first, then the keyless routes that live probing shows to be
+ * healthy and fast (`parallel`, `duckduckgo`; `searxng` sits between them
+ * and is skipped unless an endpoint is configured), then the routes that
+ * are routinely walled for anonymous clients (`perplexity` sign-up wall,
+ * `startpage` Anubis PoW, `ecosia`/`google`/`mojeek` bot challenges), with
  * the public fan-out last (explicit selection only).
  */
 export declare const SEARCH_PROVIDER_ORDER: readonly SearchProviderId[];
@@ -100,10 +104,6 @@ export declare const SEARCH_PROVIDER_CHOICES: ({
     readonly value: "xai";
     readonly label: "xAI";
     readonly description: "xAI Responses web_search via the agent's xAI model credentials (XAI_API_KEY)";
-} | {
-    readonly value: "perplexity";
-    readonly label: "Perplexity";
-    readonly description: "Anonymous Perplexity ask endpoint; no key or signup required";
 } | {
     readonly value: "parallel";
     readonly label: "Parallel";
@@ -112,14 +112,18 @@ export declare const SEARCH_PROVIDER_CHOICES: ({
     readonly value: "searxng";
     readonly label: "SearXNG";
     readonly description: "Advanced: requires a self-hosted SEARXNG_ENDPOINT; never part of the default experience";
-} | {
-    readonly value: "startpage";
-    readonly label: "Startpage";
-    readonly description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged";
 } | {
     readonly value: "duckduckgo";
     readonly label: "DuckDuckGo";
     readonly description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs";
+} | {
+    readonly value: "perplexity";
+    readonly label: "Perplexity";
+    readonly description: "Anonymous Perplexity ask endpoint; no key or signup required";
+} | {
+    readonly value: "startpage";
+    readonly label: "Startpage";
+    readonly description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged";
 } | {
     readonly value: "ecosia";
     readonly label: "Ecosia";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.10.3",
+  "version": "2.10.5",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {
@@ -55,6 +55,7 @@
     "@earendil-works/pi-ai": "^1.2.0",
     "@earendil-works/pi-coding-agent": "1.2.0",
     "@earendil-works/pi-tui": "1.2.0",
+    "@llblab/pi-telegram": "0.57.2",
     "@modelcontextprotocol/server": "^2.3.1",
     "@openrouter/sdk": "^1.4.25",
     "chardet": "^2.2.0",

package/skills/autorag-doctor/SKILL.md

@@ -223,6 +223,7 @@ to retry those files once without re-indexing healthy ones.
 | `minsync-sync-failed` | MinSync indexing failed; the message carries MinSync's own reason | Fix the reported cause, then `autorag refresh --method minsync` |
 | `minsync-file-skipped` | MinSync skipped one file (unreadable, unparsable, or rejected by the embedder) and kept its older chunks; retried on the next sync | Fix that file's permissions/content, then `autorag refresh --method minsync` |
 | `minsync-lock-held` | Another `minsync` run holds `.minsync/lock`, often an orphan left by a killed host; retryable | Wait for it to finish or stop that `minsync` process, then `autorag refresh --method minsync` |
+| `lexical-tokenizer-unavailable` | Warning: the BM25 tokenizer needs the Kiwi library and model, which were not found, so MinSync tokenizes on whitespace and `인증서를` does not match `인증서` | Install Kiwi (set `KIWI_LIBRARY_PATH` and `KIWI_MODEL_PATH`, or install under `~/.local/kiwi`), then `autorag refresh --method minsync` |
 | `jikji-unavailable` | Jikji binary missing or install failed | Check cargo, retry refresh |
 | `everything-index-failed` | Windows Everything instance could not start or index; message carries ES exit code and stderr | Fix the reported cause, `autorag refresh --method everything --json` |
 | `embedding-identity-mismatch` | Indexed vectors use a different embedder | `autorag index rebuild --method minsync` |

package/skills/autorag-lite-setup/SKILL.md

@@ -169,7 +169,7 @@ AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite refresh --jso
   are missing or broken, run a full refresh or return to setup rather than
   silently degrading to lexical-only search.
 - MinSync's default embedder is in-process native Qwen3 embeddings
-  (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.8): no
+  (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.10): no
   embedder flags, no API key, and no external daemon (such as Ollama) are
   needed, and no corpus text leaves the machine. For gateway-profiled
   embeddings, prefetch with `autorag models prefetch --profile qwen3-embedding-0.6b`.

package/skills/autorag-setup/SKILL.md

@@ -190,7 +190,7 @@ told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
 index every copy.
 
 MinSync's default embedder is in-process native Qwen3 embeddings
-(`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.8). The
+(`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.10). The
 default needs no embedder flags, no API key, and no external daemon (such as
 Ollama) — all embeddings run in-process locally and privately. For workspaces
 using the loopback llama-server gateway, prefetch the verified model with