@autorag/librarian 2.10.1 → 2.10.3

package/dist/minsync/client.d.ts

@@ -37,7 +37,9 @@ export declare class MinSyncClient {
     private readIdentity;
     private writeIdentity;
     private identityMismatch;
-    sync(force?: boolean): Promise<MinSyncSyncResult>;
+    sync(force?: boolean, signal?: AbortSignal): Promise<MinSyncSyncResult>;
+    private lockHeldResult;
+    private checkAndSync;
     query(text: string, topK: number, mode?: MinSyncQueryMode, signal?: AbortSignal): Promise<readonly MinSyncQueryHit[]>;
     private spawn;
 }

package/dist/minsync/index.d.ts

@@ -5,5 +5,5 @@ export { type EnsureMinSyncBinaryOptions, ensureMinSyncBinary, executableName, f
 export { ensureLocalEmbedder, LocalEmbedderError, type LocalEmbedderPreflightOptions, } from "./local-embedder.js";
 export { MinSyncHybridMethod, type MinSyncHybridMethodOptions, MinSyncVectorMethod, type MinSyncVectorMethodOptions, } from "./method.js";
 export { MINSYNC_FILES_SUBDIR, MINSYNC_SUBDIR, minSyncDocumentPath, minSyncWorkspaceRoot } from "./paths.js";
-export type { MinSyncChunkerConfig, MinSyncDiagnostic, MinSyncEmbedderConfig, MinSyncOptions, MinSyncQueryHit, MinSyncSyncResult, } from "./types.js";
+export type { MinSyncChunkerConfig, MinSyncDiagnostic, MinSyncEmbedderConfig, MinSyncFileFailure, MinSyncOptions, MinSyncQueryHit, MinSyncSyncResult, } from "./types.js";
 export { buildMinSyncPathMap, type MinSyncWorkspaceEntry, type MinSyncWorkspaceSyncResult, syncMinSyncWorkspace, } from "./workspace.js";

package/dist/minsync/installer.d.ts

@@ -1,4 +1,4 @@
-export declare const MINSYNC_VERSION = "0.4.7";
+export declare const MINSYNC_VERSION = "0.4.8";
 export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
 export interface MinSyncReleaseAsset {
     readonly name: string;

package/dist/minsync/method.d.ts

@@ -34,7 +34,7 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
     private readonly scopedTopK;
     constructor(options: MinSyncVectorMethodOptions);
     describe(): RetrievalMethodDescriptor;
-    sync(force?: boolean): Promise<MinSyncSyncResult>;
+    sync(force?: boolean, signal?: AbortSignal): Promise<MinSyncSyncResult>;
     private syncUnlocked;
     /** Report unavailable binaries without treating auto-install as already failed. */
     isBinaryMissing(): boolean;

package/dist/minsync/types.d.ts

@@ -39,8 +39,15 @@ export interface MinSyncSyncResult {
      * unindexable document is never dropped silently.
      */
     readonly stagingExcluded?: readonly string[];
+    /** Files MinSync skipped (exit 6, `partial`); the sync itself completed and the next sync retries them. */
+    readonly filesFailed?: readonly MinSyncFileFailure[];
 }
-export type MinSyncDiagnosticCode = "embedder-unavailable" | "sync-failed" | "no-hit" | "embedding-identity-mismatch" | "migration-required";
+export interface MinSyncFileFailure {
+    readonly path: string;
+    readonly stage: string;
+    readonly error: string;
+}
+export type MinSyncDiagnosticCode = "embedder-unavailable" | "sync-failed" | "minsync-lock-held" | "no-hit" | "embedding-identity-mismatch" | "migration-required";
 export interface MinSyncDiagnostic {
     readonly code: MinSyncDiagnosticCode;
     readonly message: string;

package/dist/mirror/atomic-write.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Write `content` to `path` through a sibling temp file and an atomic rename.
+ * On any failure the temp file is removed before the error is rethrown, so a full
+ * disk or an I/O error never leaves `*.tmp` orphans in the mirror.
+ */
+export declare function writeFileAtomic(path: string, content: string): void;

package/dist/mirror/index-store.d.ts

@@ -8,11 +8,14 @@ export interface ParsedMirrorEntry {
     readonly updatedAt: string;
 }
 /** Why refresh deliberately left a source out of the parsed mirror. */
-export type ParsedMirrorSkipReason = "duplicate-excluded" | "user-excluded" | "parser-skipped" | "parser-failed";
+export type ParsedMirrorSkipReason = "duplicate-excluded" | "user-excluded" | "parser-skipped" | "parser-failed" | "source-unreadable";
 /**
  * A source the last refresh decided not to mirror, pinned to the exact version it
  * decided on. Staleness checks compare against this record instead of re-deriving
  * the decision, so a deliberately skipped file is not reported as a new source.
+ *
+ * Retryable skips (`parser-failed`, `source-unreadable`) also carry the parser stamp, so refresh
+ * can honor the record without reading or parsing the file again.
  */
 export interface ParsedMirrorSkipEntry {
     readonly virtualPath: string;
@@ -21,6 +24,10 @@ export interface ParsedMirrorSkipEntry {
     readonly sourceMtimeNs: number;
     readonly sourceSizeBytes: number;
     readonly updatedAt: string;
+    /** `Parser.name` the skip was decided with. */
+    readonly parserName?: string;
+    /** `Parser.version` the skip was decided with. Absent on records written before version stamps. */
+    readonly parserVersion?: string;
 }
 export interface ParsedMirrorIndex {
     readonly version: 1;

package/dist/mirror/index.d.ts

@@ -1,5 +1,5 @@
 export type { ParsedMirrorEntry, ParsedMirrorIndex } from "./index-store.js";
 export { emptyMirrorIndex, loadMirrorIndex, saveMirrorIndex } from "./index-store.js";
 export { isParsedRefreshComplete, parsedMirrorIndexPath, parsedMirrorRoot, parsedOutputPath, refreshReadinessPath, } from "./paths.js";
-export type { ParsedMirrorDiagnostic, ParsedMirrorDiagnosticCode, ParsedMirrorSyncOptions, ParsedMirrorSyncResult, } from "./sync.js";
-export { DEFAULT_MAX_SOURCE_BYTES, detectMirrorStaleness, isPathExcluded, syncParsedMirrors } from "./sync.js";
+export type { ParsedMirrorDiagnostic, ParsedMirrorDiagnosticCode, ParsedMirrorProgress, ParsedMirrorSyncOptions, ParsedMirrorSyncResult, } from "./sync.js";
+export { DEFAULT_MAX_SOURCE_BYTES, DEFAULT_TRANSIENT_RETRIES, detectMirrorStaleness, isPathExcluded, MirrorWorkspaceError, syncParsedMirrors, } from "./sync.js";

package/dist/mirror/sync.d.ts

@@ -2,6 +2,12 @@ import { type DefaultParserRegistryOptions } from "../parser/defaults.js";
 import type { ParserRegistry } from "../parser/registry.js";
 /** Refuse to fully load/parse sources larger than this. Keeps Node heap stable on home folders. */
 export declare const DEFAULT_MAX_SOURCE_BYTES: number;
+/** A refresh may delete at most this share of the indexed documents (above {@link MASS_DELETE_FLOOR}). */
+export declare const DEFAULT_MAX_DELETE_RATIO = 0.3;
+/** Directory entries examined per search root before the walk stops. */
+export declare const DEFAULT_MAX_SCANNED_ENTRIES = 2000000;
+/** Deepest directory level (search root = 0) the walk descends into. */
+export declare const DEFAULT_MAX_WALK_DEPTH = 64;
 export interface ParsedMirrorSyncOptions {
     readonly root: string;
     readonly searchPaths: readonly string[];
@@ -14,8 +20,48 @@ export interface ParsedMirrorSyncOptions {
     readonly excludeSourcePaths?: ReadonlySet<string>;
     /** Absolute source paths or directories omitted by the user. */
     readonly userExcludedSourcePaths?: ReadonlySet<string>;
+    /**
+     * Retry sources that previously failed to parse or read once, ignoring their skip records.
+     * Healthy, unchanged mirrors are still left alone (unlike `force`).
+     */
+    readonly retrySkipped?: boolean;
+    /**
+     * Stops the sync between files and kills an in-flight parse worker. The sync rejects with the signal's
+     * reason after checkpointing the mirror, so finished files are not re-parsed next time.
+     */
+    readonly signal?: AbortSignal;
+    /** Called before each file and once more with `processed === total` when the parse phase ends. */
+    readonly onProgress?: (progress: ParsedMirrorProgress) => void;
+    /**
+     * Largest share of previously indexed documents one refresh may delete because their source
+     * disappeared. A larger deletion (above 100 documents) is suppressed with `mass-delete-suppressed`
+     * unless `force` is set. Default {@link DEFAULT_MAX_DELETE_RATIO}.
+     */
+    readonly maxDeleteRatio?: number;
+    /** Directory entries examined per search root before the walk stops with `walk-truncated`. */
+    readonly maxScannedEntries?: number;
+    /** Deepest directory level the walk descends into (search root = 0), else `walk-truncated`. */
+    readonly maxDepth?: number;
+    /**
+     * Retries for a transient I/O error (`EMFILE`, `EBUSY`, `EIO`, `ETIMEDOUT`) on one
+     * file's read or mirror write before the file is recorded as skipped. Default 3.
+     */
+    readonly transientRetries?: number;
+    /** Base delay between transient retries; doubles per retry. Default 50ms. */
+    readonly transientRetryBackoffMs?: number;
 }
-export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "source-vanished" | "source-unreadable" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning";
+/** Parse-phase progress. Every scanned file counts as processed, whether parsed, unchanged, skipped or excluded. */
+export interface ParsedMirrorProgress {
+    readonly processed: number;
+    readonly total: number;
+    /** Running totals for the files processed so far; the final event carries the sync result's counts. */
+    readonly written: number;
+    readonly deleted: number;
+    readonly skipped: number;
+    /** The file about to be processed; absent on the final event. */
+    readonly currentVirtualPath?: string;
+}
+export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable";
 /**
  * Refresh diagnostic. `source` is an opaque virtual path, never a real filesystem path.
  * `message` keeps the underlying error text, which may include a filesystem path.
@@ -35,6 +81,16 @@ export interface ParsedMirrorSyncResult {
     readonly indexPath: string;
     readonly diagnostics: readonly ParsedMirrorDiagnostic[];
 }
+/** Default retries for a transient per-file I/O error. Mirrors the `refresh.transientRetries` config default. */
+export declare const DEFAULT_TRANSIENT_RETRIES = 3;
+/**
+ * A failure that makes continuing the refresh pointless: the workspace volume is full or the
+ * workspace is not writable. Per-file failures never use this class.
+ */
+export declare class MirrorWorkspaceError extends Error {
+    readonly code: string | undefined;
+    constructor(workspace: string, cause: unknown);
+}
 export declare function syncParsedMirrors(options: ParsedMirrorSyncOptions): Promise<ParsedMirrorSyncResult>;
 /**
  * Cheap, parse-free staleness check: compares current source files (by mtime and

package/dist/p2p/simplex-server.d.ts

@@ -91,7 +91,6 @@ export interface StartSimplexPeerServerOptions {
     readonly injectionClassifier?: boolean;
     readonly injectionClassifierModel?: InjectionClassifierModel;
     readonly buildPeerResponse?: PeerResponseBuilder;
-    readonly searchTimeoutMs?: number;
     readonly pseudonymize?: boolean;
     readonly logger?: (entry: SimplexPeerLog) => void;
 }

package/dist/parser/defaults.d.ts

@@ -7,6 +7,14 @@ export interface DefaultParserRegistryOptions {
     readonly kordoc?: KordocParserOptions;
     /** Global document languages; selects the OCR engine when OCR is enabled. */
     readonly languages?: readonly LanguageTag[];
+    /**
+     * Hard per-file parse deadline in milliseconds (default 120000). kordoc, pptx and eml parse in a worker
+     * thread during a refresh; a parse still running at the deadline has its worker terminated and the file
+     * is skipped as `parser-failed`. A synchronous spin cannot be interrupted any other way.
+     */
+    readonly fileTimeoutMs?: number;
+    /** V8 old-generation heap limit in MB for the parse worker. An out-of-memory parse becomes a `parser-failed` skip. */
+    readonly maxHeapMb?: number;
 }
 /** Workspace-relative directory for Tesseract traineddata, so models never land in the process working directory. */
 export declare function tesseractCachePathFor(workspacePath: string): string;

package/dist/parser/errors.d.ts

@@ -3,3 +3,13 @@ export declare class ParseError extends Error {
     readonly virtualPath: string;
     constructor(parserName: string, virtualPath: string, cause: unknown);
 }
+export type ParseIsolationFailureCode = "PARSE_TIMEOUT" | "WORKER_OOM" | "WORKER_CRASHED";
+/**
+ * Why an isolated parse worker was lost. Carried as the `cause` of a
+ * {@link ParseError} so a timeout, an out-of-memory kill, and a crash stay
+ * distinguishable from a parser's own failure.
+ */
+export declare class ParseIsolationError extends Error {
+    readonly code: ParseIsolationFailureCode;
+    constructor(code: ParseIsolationFailureCode, message: string);
+}

package/dist/parser/isolated.d.ts

@@ -0,0 +1,77 @@
+import type { DefaultParserRegistryOptions } from "./defaults.js";
+import type { ParseInput, ParseOutput } from "./types.js";
+/** Per-file parse deadline when `parserOptions.fileTimeoutMs` is not set. */
+export declare const DEFAULT_PARSE_FILE_TIMEOUT_MS = 120000;
+/**
+ * Parsers that run third-party or format-parsing code which can spin, exhaust
+ * the heap, or crash on a hostile file. Plain text decoding stays in-process.
+ */
+export declare const ISOLATED_PARSER_NAMES: ReadonlySet<string>;
+/** The structured-cloneable subset of {@link DefaultParserRegistryOptions} a worker can rebuild its registry from. */
+export interface WorkerParserOptions {
+    readonly languages?: DefaultParserRegistryOptions["languages"];
+    readonly ocr?: DefaultParserRegistryOptions["ocr"];
+    readonly kordoc?: {
+        readonly ocr?: boolean | "force";
+    };
+}
+export interface ParseWorkerRequest {
+    readonly id: number;
+    readonly parserName: string;
+    readonly virtualPath: string;
+    readonly sourcePath?: string;
+    readonly bytes: Uint8Array;
+}
+export type ParseWorkerResponse = {
+    readonly id: number;
+    readonly ok: true;
+    readonly output: ParseOutput;
+} | {
+    readonly id: number;
+    readonly ok: false;
+    readonly kind: "parse-error";
+    readonly parserName: string;
+    readonly message: string;
+    readonly stack?: string;
+} | {
+    readonly id: number;
+    readonly ok: false;
+    readonly kind: "error";
+    readonly message: string;
+    readonly stack?: string;
+};
+/**
+ * Parser options a worker thread can receive, or `undefined` when they hold a
+ * function (an injected `kordoc.parse` or OCR provider) that cannot cross the
+ * thread boundary. Such options stay in-process.
+ */
+export declare function workerParserOptions(options: DefaultParserRegistryOptions): WorkerParserOptions | undefined;
+/** Rejects a non-positive or non-finite `fileTimeoutMs` / `maxHeapMb` before any file is parsed. */
+export declare function validateIsolationOptions(options: Pick<DefaultParserRegistryOptions, "fileTimeoutMs" | "maxHeapMb">): void;
+export interface IsolatedParseRunnerOptions {
+    readonly parserOptions: WorkerParserOptions;
+    readonly fileTimeoutMs?: number;
+    /** V8 `resourceLimits.maxOldGenerationSizeMb` for the worker. Unset keeps Node's default. */
+    readonly maxHeapMb?: number;
+    /** Override the worker entry (tests). */
+    readonly workerUrl?: URL;
+}
+/**
+ * Runs parsers inside a worker thread so one file can be hard-killed. A
+ * `Promise.race` deadline cannot interrupt a synchronous spin, but
+ * `Worker.terminate()` can; a heap limit turns an out-of-memory parse into a
+ * catchable worker error instead of a host crash.
+ *
+ * One worker serves files in sequence and is reused until it is lost (deadline,
+ * out of memory, crash, abort); the next file then gets a fresh worker. Every
+ * worker-side loss rejects with a {@link ParseError} whose cause is a
+ * {@link ParseIsolationError}, so the caller's existing per-file skip applies.
+ */
+export declare class IsolatedParseRunner {
+    #private;
+    constructor(options: IsolatedParseRunnerOptions);
+    /** Parse one file. Rejects with the signal's reason when `signal` aborts, after killing the worker. */
+    parse(parserName: string, input: ParseInput, signal?: AbortSignal): Promise<ParseOutput>;
+    /** Terminate the worker, if one is alive. The runner stays usable and spawns a fresh worker on demand. */
+    close(): Promise<void>;
+}

package/dist/parser/kordoc.d.ts

@@ -54,6 +54,7 @@ export declare const KORDOC_IMAGE_EXTENSIONS: readonly [".png", ".jpg", ".jpeg",
 export declare class KordocParser extends Parser {
     #private;
     readonly name = "kordoc";
+    readonly version: string;
     readonly extensions: readonly string[];
     constructor(options?: KordocParserOptions);
     parse(input: ParseInput): Promise<ParseOutput>;

package/dist/parser/parse-worker.d.ts

@@ -0,0 +1 @@
+export {};