@autorag/librarian 2.13.1 → 2.13.3

package/NOTICE

@@ -6,22 +6,22 @@ legacy/ package license: Apache-2.0
 
 Third-party production dependencies:
 
-@earendil-works/pi-agent-core@1.2.0
+@earendil-works/pi-agent-core@1.4.0
 License: MIT
 
-@earendil-works/pi-ai@1.2.0
+@earendil-works/pi-ai@1.4.0
 License: MIT
 
-@earendil-works/pi-coding-agent@1.2.0
+@earendil-works/pi-coding-agent@1.4.0
 License: MIT
 
-@earendil-works/pi-tui@1.2.0
+@earendil-works/pi-tui@1.4.0
 License: MIT
 
-@llblab/pi-telegram@0.57.2
+@llblab/pi-telegram@0.58.3
 License: MIT
 
-@modelcontextprotocol/server@2.3.1
+@modelcontextprotocol/server@2.4.0
 License: Apache-2.0
 
 @parcel/watcher@2.6.0
@@ -45,10 +45,10 @@ License: MIT
 jszip@3.10.2
 License: (MIT OR GPL-3.0-or-later)
 
-kordoc@4.21.30
+kordoc@4.24.19
 License: MIT
 
-mailparser@3.9.38
+mailparser@3.13.0
 License: MIT
 
 pdfjs-dist@6.4.299
@@ -60,7 +60,7 @@ License: MIT
 pi-discord-remote@0.2.4
 License: MIT
 
-puppeteer-core@25.13.0
+puppeteer-core@25.16.0
 License: Apache-2.0
 
 slides-grab@1.5.2

package/README.md

@@ -458,12 +458,16 @@ try {
 terminates the MinSync, Jikji and datasource child processes (and their descendants). The
 refresh then rejects with the signal's reason and is recorded as `failed`; files already
 mirrored are kept and not re-parsed on the next run. `onProgress` reports `parsed`
-(`processed`/`total` are files, `currentVirtualPath` is the file about to be processed) and
-`datasources` (`processed`/`total` are datasource skills). During the parse phase
-`.autorag/refresh-progress.json` also carries the running `sourceFiles.total` and
-`parsedCounts`, refreshed every 25 files rather than per file. The marker is replaced with a
-retried rename (an in-place overwrite if a reader keeps the destination held, as on Windows).
-Without these options a refresh behaves as before.
+(`processed`/`total` are source files, `currentVirtualPath` is the file about to be processed),
+`minsync` (`processed`/`total` are files processed by MinSync), and `datasources`
+(`processed`/`total` are datasource skills). `.autorag/refresh-progress.json` carries the running
+`sourceFiles.total` and `parsedCounts` during parsing (refreshed every 25 files rather than per
+file), and `minsync: { synced, total }` on every MinSync batch or heartbeat. Each progress write
+also advances `updatedAt`; between events a stale timestamp means "phase still running", not
+"stuck". Use the marker's pid through `getRefreshStatus().progress.ownerAlive` (or
+`isRefreshOwnerAlive`) for liveness. The marker is replaced with a retried rename (an in-place
+overwrite if a reader keeps the destination held, as on Windows). Without these options a refresh
+behaves as before.
 
 ---
 
@@ -656,8 +660,7 @@ AutoRAG Agent connects to external tools and communication platforms using dedic
 | **Telegram** | `telegram` | [`telecrawl`](https://github.com/openclaw/telecrawl) CLI | Local-first desktop archive + FTS5 | Lexical FTS5; BM25 + Semantic + Hybrid by default via [`sidecar`](docs/datasource-skills.md#hybrid-search-for-chat-crawlers-sidecar) |
 | **Slack** | `slack` | [`slacrawl`](https://github.com/openclaw/slacrawl) CLI | Local workspace/channel/thread archive + FTS5 | Lexical FTS5; BM25 + Semantic + Hybrid by default via [`sidecar`](docs/datasource-skills.md#hybrid-search-for-chat-crawlers-sidecar) |
 | **Notion** | `notion` | [`notcrawl`](https://github.com/openclaw/notcrawl) CLI | Local page/database/block archive + FTS5 | Lexical FTS5 |
-| **Email Archives** | `mailcrawl` | [`mailcrawl`](https://github.com/NomaDamas/mailcrawl) CLI | Local Gmail, IMAP, and Maildir storage | BM25, Semantic, Hybrid |
-| **Local Mail Export**| `mail-export` | Built-in `.mbox` / `.eml` parser | Local filesystem mailboxes | Lexical |
+| **Email Archives** | `mailcrawl` | [`mailcrawl`](https://github.com/NomaDamas/mailcrawl) CLI | Local Gmail, IMAP, Maildir, and `.eml`/`.mbox` mailbox backups (`--source files`) | BM25, Semantic, Hybrid |
 | **Obsidian Vaults** | `obsidian` | Indexed with MinSync in its own workspace (no external CLI) | Markdown notes, folder-scopable | Lexical (BM25), Semantic, Scoped |
 | **GitHub** | `github` | GitHub REST API | In-memory fetched Issues and Pull Requests | Lexical, Scoped |
 | **GitHub Gists** | `github-gist` | GitHub REST API | Incremental local index of own-account public and permitted secret Gist content/metadata; token not stored | Lexical (BM25), Local Semantic, Scoped |

package/dist/agent/agent.d.ts

@@ -9,7 +9,7 @@ import { type EverythingClientOptions, type EverythingSearchRequest, type Everyt
 import { type FSearchClientOptions, type FSearchFailureReason, type FSearchSearchRequest, type FSearchSearchResult } from "../fsearch/index.js";
 import { type JikjiFailureReason, type JikjiOptions, type JikjiPrepareOptions } from "../jikji/index.js";
 import { type LanguageTag } from "../language.js";
-import { type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
+import { type MinSyncSyncProgress, type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
 import { type ParsedMirrorSyncOptions, type ParsedMirrorSyncResult } from "../mirror/sync.js";
 import { type DefaultParserRegistryOptions } from "../parser/index.js";
 import { RetrievalEngine } from "../retrieval/engine.js";
@@ -87,10 +87,10 @@ export type RefreshMethod = "parsed" | "minsync" | "datasources" | "jikji" | "ev
 /**
  * Progress of a refresh phase that can count its work. `parsed` counts source files
  * (`currentVirtualPath` is the opaque virtual path of the file about to be processed);
- * `datasources` counts configured datasource skills. Other phases do not report counts.
+ * `minsync` counts files processed by MinSync; `datasources` counts configured datasource skills.
  */
 export interface AutoRAGRefreshProgress {
-    readonly phase: "parsed" | "datasources";
+    readonly phase: "parsed" | "minsync" | "datasources";
     /** Items finished so far. Advances by one per item and ends equal to `total`. */
     readonly processed: number;
     readonly total: number;
@@ -110,7 +110,7 @@ export interface AutoRAGRefreshOptions {
      * reason and records the run as failed; mirror entries written before the abort are kept.
      */
     readonly signal?: AbortSignal;
-    /** Called while the parse and datasource phases run. A throwing callback is ignored; it never fails the refresh. */
+    /** Called while the parse, MinSync and datasource phases run. A throwing callback is ignored; it never fails the refresh. */
     readonly onProgress?: (progress: AutoRAGRefreshProgress) => void;
     /**
      * Per-root `jikji prepare` controls: a hard `timeoutMs` wall per root and a
@@ -125,6 +125,8 @@ export interface AutoRAGMinSyncRefreshResult {
     readonly reason?: string;
     /** Count of parsed documents excluded from the MinSync index by file name. */
     readonly stagingExcludedCount?: number;
+    /** Count of files MinSync skipped in a partial sync; per-file details remain in diagnostics. */
+    readonly filesSkippedCount?: number;
     readonly diagnostics?: readonly SearchDocumentDiagnostic[];
 }
 export interface AutoRAGRefreshResult extends Omit<ParsedMirrorSyncResult, "diagnostics"> {
@@ -179,6 +181,8 @@ export interface AutoRAGRefreshStatus {
         readonly parsedCounts?: RefreshProgressCounts;
         readonly minsync?: {
             readonly synced?: number;
+            readonly total?: number;
+            readonly filesSkipped?: number;
         };
         readonly ownerAlive?: boolean;
     };
@@ -743,7 +747,9 @@ export declare class AutoRAGAgent {
      * are enabled, with where similar past questions were answered (from
      * retrieval memory) in the state. Returns the datasource ids and whether the
      * web is to be searched with every query before the rerank; a failed check
-     * searches neither.
+     * searches neither. It runs while the route check is still in flight, so it
+     * returns its diagnostics instead of recording them (see
+     * {@link SearchDatasourcePlan}).
      */
     private selectSearchDatasources;
     /**
@@ -896,7 +902,7 @@ export declare class AutoRAGAgent {
      */
     private scanMirrorStaleness;
     private exactDuplicateExclusions;
-    syncMinSync(force?: boolean, signal?: AbortSignal): Promise<MinSyncSyncResult>;
+    syncMinSync(force?: boolean, signal?: AbortSignal, onProgress?: (progress: MinSyncSyncProgress) => void): Promise<MinSyncSyncResult>;
     /** Provider for the Windows-only everything_search tool. */
     searchEverything(request: EverythingSearchRequest): Promise<EverythingSearchResult>;
     /**

package/dist/agent/refresh-progress.d.ts

@@ -1,3 +1,5 @@
+/** Persist display progress every 25 files; this is not the engine's recovery checkpoint. */
+export declare const REFRESH_PROGRESS_EVERY = 25;
 export type RefreshProgressPhase = "parsed" | "minsync" | "datasources" | "jikji" | "everything" | "fsearch" | "finalizing";
 export interface RefreshProgressCounts {
     readonly scanned: number;
@@ -20,6 +22,8 @@ export interface PersistedRefreshProgress {
     readonly parsedCounts?: RefreshProgressCounts;
     readonly minsync?: {
         readonly synced?: number;
+        readonly total?: number;
+        readonly filesSkipped?: number;
     };
     readonly error?: string;
 }

package/dist/agent/search-documents.d.ts

@@ -9,13 +9,15 @@ export type SearchDocumentDiagnosticSeverity = "info" | "warning" | "error";
  * (#21/#22) does not require another source-breaking type change; codes not yet
  * emitted are still valid members.
  */
-export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-staging-excluded" | "minsync-file-skipped" | "minsync-sync-failed" | "minsync-lock-held" | "embedder-unavailable" | "embedding-identity-mismatch" | "lexical-tokenizer-unavailable" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "parser-warning" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "pdf-extract-thin" | "retrieval-method-failed" | "rerank-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "everything-index-failed" | "fsearch-index-failed" | "fsearch-binary-missing" | "refresh-failed" | "refresh-interrupted" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "no-final-answer" | "model-request-failed" | "tool-call-burst" | "query-routed" | "query-route-fallback" | "query-decomposition-failed" | "follow-up-skipped" | "follow-up-check-fallback" | "datasources-selected" | "datasource-selection-fallback" | "web-datasource-id-taken" | "web-search-failed" | "citation-without-result" | "line-reference-unverified" | "citation-irrelevant" | "citation-relevance-fallback" | "evidence-judged" | "evidence-judgment-fallback" | "memory-search-fallback" | "self-config-unavailable" | "self-config-rolled-back" | "organize-skill-unavailable" | "embedding-unavailable";
+export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-staging-excluded" | "minsync-file-skipped" | "minsync-sync-failed" | "minsync-lock-held" | "embedder-unavailable" | "embedding-identity-mismatch" | "lexical-tokenizer-unavailable" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "parser-warning" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "pdf-extract-thin" | "retrieval-method-failed" | "rerank-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "everything-index-failed" | "fsearch-index-failed" | "fsearch-binary-missing" | "refresh-failed" | "refresh-interrupted" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "no-final-answer" | "model-request-failed" | "tool-call-burst" | "query-routed" | "query-route-fallback" | "query-decomposition-failed" | "follow-up-skipped" | "follow-up-check-fallback" | "datasources-selected" | "datasource-selection-fallback" | "web-datasource-id-taken" | "web-search-failed" | "citation-without-result" | "line-reference-unverified" | "citation-irrelevant" | "citation-relevance-fallback" | "evidence-judged" | "evidence-judgment-fallback" | "memory-search-fallback" | "self-config-unavailable" | "self-config-rolled-back" | "organize-skill-unavailable" | "embedding-unavailable";
 export interface SearchDocumentDiagnostic {
     readonly code: SearchDocumentDiagnosticCode;
     readonly severity: SearchDocumentDiagnosticSeverity;
     readonly message: string;
     /** Component label (e.g. "sanitizer", "minsync") or opaque virtual path — never a real filesystem path. */
     readonly source?: string;
+    /** Structured cause for a component diagnostic, such as a changed source revision. */
+    readonly reason?: string;
 }
 export interface SearchDocumentEvidence {
     readonly excerpt: string;

package/dist/agent/search-progress.d.ts

@@ -49,6 +49,11 @@ export declare class SearchProgressTracker {
     /** Starts (or restarts) a step that the caller closes with {@link endStep}. */
     beginStep(id: string, label: string): void;
     endStep(id: string, status: Exclude<SearchProgressStatus, "running">, detail?: string): void;
+    /**
+     * Removes a step that turned out not to belong to this search: speculative work whose branch was not taken.
+     * A job still running under that id settles into nothing, so the step does not come back.
+     */
+    discardStep(id: string): void;
     /**
      * Runs `work` as step `id`: running while it is in flight, done with
      * `detail(value)` when it resolves, failed with the error's own message when