@autorag/librarian 2.13.4 → 2.13.6

package/dist/mcp/server.d.ts

@@ -1,9 +1,12 @@
 import { McpServer } from "@modelcontextprotocol/server";
 import type { AutoRAGLite } from "../core.js";
+import { type CloudFileProbe } from "../filesystem/cloud-placeholder.js";
 export interface AutoRAGMcpServerOptions {
     readonly readOnly?: boolean;
     readonly tools?: readonly string[];
     /** Test/embedded override; defaults to the current Node platform. */
     readonly platform?: NodeJS.Platform;
+    /** Test/embedded override for the cloud probe `autorag.duplicates` uses; defaults to this platform's probe. */
+    readonly cloudProbe?: CloudFileProbe;
 }
 export declare function createAutoRAGMcpServer(lite: AutoRAGLite, options?: AutoRAGMcpServerOptions): McpServer;

package/dist/minsync/binary.d.ts

@@ -1,4 +1,5 @@
 import { type EnsureMinSyncBinaryOptions } from "./installer.js";
+import { type OutdatedMinSyncBinary } from "./version.js";
 export interface ResolveMinSyncBinaryOptions {
     /** Workspace project root; the cached binary lives under `<root>/.autorag/bin`. */
     readonly root: string;
@@ -9,13 +10,17 @@ export interface ResolveMinSyncBinaryOptions {
     /** False for query paths: they use an existing binary or fail, and never wait on `cargo install`. */
     readonly install?: boolean;
 }
-/** Resolve the `minsync` executable from PATH directories. Returns the first match or undefined. */
-export declare function lookupMinSyncInPath(env: NodeJS.ProcessEnv): string | undefined;
+/**
+ * Resolve the `minsync` executable from PATH directories: the first executable that is not older than
+ * {@link MINSYNC_MIN_VERSION}. Older ones are skipped and, when `outdated` is given, recorded there.
+ */
+export declare function lookupMinSyncInPath(env: NodeJS.ProcessEnv, outdated?: OutdatedMinSyncBinary[]): string | undefined;
 /**
  * Resolve the MinSync CLI from the user's global PATH first, then the workspace cache, and finally the
- * verified auto-install fallback. Throws {@link MinSyncRequiredError}: MinSync is required, so there is
- * no "no binary" value to degrade on.
+ * verified auto-install fallback. A binary older than {@link MINSYNC_MIN_VERSION} is never run: it is skipped on
+ * PATH, and a cached one is replaced by the install. Throws {@link MinSyncRequiredError}: MinSync is required, so
+ * there is no "no binary" value to degrade on. An explicit `binaryPath` is used as given and never probed.
  */
 export declare function resolveMinSyncBinary(options: ResolveMinSyncBinaryOptions): Promise<string>;
-/** True when no `minsync` is resolvable right now without installing one. */
+/** True when no usable `minsync` is resolvable right now without installing one. */
 export declare function isMinSyncBinaryMissing(options: Pick<ResolveMinSyncBinaryOptions, "root" | "binaryPath">): boolean;

package/dist/minsync/client.d.ts

@@ -5,8 +5,11 @@ export interface MinSyncClientOptions {
     readonly embedder?: MinSyncEmbedderConfig;
     readonly maxChunkSize?: number;
 }
-/** MinSync's native query modes: vector, BM25, and hybrid. */
-export type MinSyncQueryMode = "vector" | "bm25" | "hybrid";
+/**
+ * MinSync's native query modes: dense vector, learned-sparse (SPLADE, which replaced BM25 in MinSync 0.6), and
+ * their hybrid fusion.
+ */
+export type MinSyncQueryMode = "vector" | "sparse" | "hybrid";
 export declare class MinSyncQueryError extends Error {
     readonly code: number | null;
     readonly stderr: string;
@@ -26,6 +29,8 @@ export declare class MinSyncClient {
     /**
      * What MinSync is told to embed with. No `id`, or a `native:` id, is MinSync's own Qwen3 engine, which
      * MinSync provisions and runs (in-process, or through a `minsync serve` daemon it finds on its own).
+     * {@link SPARSE_ONLY_EMBEDDER_ID} is a sparse-only index: only the id is passed on, since a dimension or an
+     * endpoint left over from a dense setup would describe vectors MinSync no longer builds.
      * Anything else is an endpoint the operator configured, passed through untouched.
      */
     private effectiveEmbedder;
@@ -33,14 +38,6 @@ export declare class MinSyncClient {
     private writeIdentity;
     private identityMismatch;
     sync(force?: boolean, signal?: AbortSignal, onProgress?: (progress: MinSyncSyncProgress) => void): Promise<MinSyncSyncResult>;
-    /**
-     * The tokenizer to run this sync with: always {@link MINSYNC_LEXICAL_LANGUAGE}, which fails every sync when the
-     * Kiwi native library is missing. A workspace that is new, or tokenized differently, then gets whitespace
-     * tokenization instead and the sync reports `lexicalFallback`. An index already built with the multilingual
-     * tokenizer is left alone: switching it would re-embed every chunk, and the probe can miss a library that the OS
-     * loader finds on its own.
-     */
-    private resolveLexicalLanguage;
     private lockHeldResult;
     private checkAndSync;
     query(text: string, topK: number, mode?: MinSyncQueryMode, signal?: AbortSignal): Promise<readonly MinSyncQueryHit[]>;

package/dist/minsync/embedder-config.d.ts

@@ -1,4 +1,3 @@
-import type { MinSyncLexicalLanguage } from "./lexical-language.js";
 import type { MinSyncEmbedderConfig } from "./types.js";
 export interface MinSyncEmbeddingIdentity {
     readonly provider: string;
@@ -13,6 +12,15 @@ export declare const EMBEDDING_IDENTITY_FILE = "autorag-embedding-identity.json"
 export declare const DEFAULT_MINSYNC_EMBEDDER_ID = "native:Qwen/Qwen3-Embedding-0.6B";
 export declare const DEFAULT_MINSYNC_EMBEDDER_DIMENSION = 1024;
 export declare const DEFAULT_MINSYNC_EMBEDDER_CONFIG: MinSyncEmbedderConfig;
+/**
+ * `minSync.embedder.id` for a sparse-only index: MinSync builds SPLADE sparse vectors and no dense ones, so nothing
+ * downloads or runs a dense embedding model. Every MinSync search is then a sparse search.
+ */
+export declare const SPARSE_ONLY_EMBEDDER_ID = "none";
+/** True when the configured embedder asks for a sparse-only MinSync index. */
+export declare function isSparseOnlyEmbedder(embedder: MinSyncEmbedderConfig | undefined): boolean;
+/** The `[embedder].id` an initialized workspace was built with, or undefined when there is no config. */
+export declare function configuredEmbedderId(workspacePath: string): string | undefined;
 /**
  * MinSync config.toml lives at `<workspace>/.minsync/config.toml`.
  * After `minsync init` runs, we rewrite allowlisted embedder fields so
@@ -24,11 +32,6 @@ export declare function minSyncConfigPath(workspacePath: string): string;
 export declare function minSyncEmbeddingIdentityPath(workspacePath: string): string;
 export declare function configuredMaxChunkSize(workspacePath: string): number | undefined;
 export declare function configuredVectorDimension(workspacePath: string): number | undefined;
-/**
- * The BM25 tokenizer in `[lexical].language`, or undefined when there is no config. A config without the section
- * (written before MinSync had one) behaves as MinSync's default, `simple`.
- */
-export declare function configuredLexicalLanguage(workspacePath: string): string | undefined;
 /**
  * Atomically rewrite allowlisted embedder fields in MinSync config.toml.
  * Reads the existing file, merges the allowlisted fields from `embedder`,
@@ -38,13 +41,11 @@ export declare function configuredLexicalLanguage(workspacePath: string): string
  * Allowlisted [embedder] fields: id, base_url, query_prefix, passage_prefix,
  * batch_size, max_retries, max_concurrent, timeout_seconds (from timeoutMs / 1000 ceil).
  * Dimension is written under [vectorstore.options] as `dimension`.
- * `managed.maxChunkSize` goes to [chunker.options] and `managed.lexicalLanguage` to [lexical].language. MinSync
- * itself records the tokenizer an index was built with in its cursor and, when [lexical].language differs, deletes
- * and re-indexes every chunk on the next sync, so changing it needs no `--full` from the caller.
+ * `managed.maxChunkSize` goes to [chunker.options]. The `[lexical]` table of a BM25-era workspace is removed: MinSync
+ * 0.6 replaced BM25 with SPLADE sparse retrieval and migrates such an index in place on its next sync.
  *
  * Returns true if the file was rewritten, false if the file does not exist.
  */
 export declare function rewriteEmbedderConfig(workspacePath: string, embedder: MinSyncEmbedderConfig, managed?: {
     readonly maxChunkSize?: number;
-    readonly lexicalLanguage?: MinSyncLexicalLanguage;
 }): boolean;

package/dist/minsync/errors.d.ts

@@ -1,3 +1,4 @@
+import { type OutdatedMinSyncBinary } from "./version.js";
 /**
  * MinSync is the only retrieval method that searches document content
  * semantically, so AutoRAG treats it as required: when it cannot be used there
@@ -10,5 +11,7 @@ export declare class MinSyncRequiredError extends Error {
 }
 /** Raised when the `minsync` executable cannot be resolved. Path-free on purpose. */
 export declare function minSyncBinaryMissingError(): MinSyncRequiredError;
+/** Raised when every `minsync` found is older than {@link MINSYNC_MIN_VERSION} and none may be installed now. */
+export declare function minSyncOutdatedError(found: readonly OutdatedMinSyncBinary[]): MinSyncRequiredError;
 /** Raised when auto-install ran and failed; carries the installer's own message. */
 export declare function minSyncInstallFailedError(cause: unknown): MinSyncRequiredError;

package/dist/minsync/index.d.ts

@@ -1,5 +1,5 @@
 export { MinSyncClient, type MinSyncClientOptions, MinSyncQueryError, type MinSyncQueryMode, } from "./client.js";
-export { configuredVectorDimension, DEFAULT_MINSYNC_EMBEDDER_CONFIG, DEFAULT_MINSYNC_EMBEDDER_DIMENSION, DEFAULT_MINSYNC_EMBEDDER_ID, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE, type MinSyncEmbeddingIdentity, minSyncConfigPath, minSyncEmbeddingIdentityPath, rewriteEmbedderConfig, } from "./embedder-config.js";
+export { configuredVectorDimension, DEFAULT_MINSYNC_EMBEDDER_CONFIG, DEFAULT_MINSYNC_EMBEDDER_DIMENSION, DEFAULT_MINSYNC_EMBEDDER_ID, isSparseOnlyEmbedder, MINSYNC_CONFIG_DIR, MINSYNC_CONFIG_FILE, type MinSyncEmbeddingIdentity, minSyncConfigPath, minSyncEmbeddingIdentityPath, rewriteEmbedderConfig, SPARSE_ONLY_EMBEDDER_ID, } from "./embedder-config.js";
 export { MinSyncRequiredError } from "./errors.js";
 export { type EnsureMinSyncBinaryOptions, ensureMinSyncBinary, executableName, fetchLatestMinSyncRelease, type InstalledMinSyncBinary, MINSYNC_VERSION, type MinSyncRelease, type MinSyncReleaseAsset, MinSyncReleaseError, selectReleaseAsset, } from "./installer.js";
 export { ensureLocalEmbedder, LocalEmbedderError, type LocalEmbedderPreflightOptions, } from "./local-embedder.js";

package/dist/minsync/installer.d.ts

@@ -1,4 +1,4 @@
-export declare const MINSYNC_VERSION = "0.5.1";
+export declare const MINSYNC_VERSION = "0.6.0";
 export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
 export interface MinSyncReleaseAsset {
     readonly name: string;

package/dist/minsync/method.d.ts

@@ -50,6 +50,7 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
     }): Promise<string>;
 }
 export declare class MinSyncHybridMethod extends MinSyncVectorMethod {
+    private readonly sparseOnly;
     constructor(options: MinSyncHybridMethodOptions);
     describe(): RetrievalMethodDescriptor;
 }

package/dist/minsync/types.d.ts

@@ -1,5 +1,4 @@
 /** Embedder configuration for MinSync vector indexing. Shared by config and method layers. */
-import type { MinSyncLexicalLanguage } from "./lexical-language.js";
 export interface MinSyncEmbedderConfig {
     readonly id?: string;
     readonly baseUrl?: string;
@@ -44,14 +43,6 @@ export interface MinSyncSyncResult {
     readonly stagingExcluded?: readonly string[];
     /** Files MinSync skipped (exit 6, `partial`); the sync itself completed and the next sync retries them. */
     readonly filesFailed?: readonly MinSyncFileFailure[];
-    /**
-     * Set when the multilingual tokenizer needs the Kiwi native library and it was not found, so the index was
-     * built with whitespace tokenization instead. The sync itself succeeded.
-     */
-    readonly lexicalFallback?: {
-        readonly requested: MinSyncLexicalLanguage;
-        readonly used: MinSyncLexicalLanguage;
-    };
 }
 export interface MinSyncFileFailure {
     readonly path: string;

package/dist/minsync/version.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Oldest `minsync` AutoRAG will run. 0.6.0 replaced the BM25 full-text index with SPLADE sparse retrieval: AutoRAG
+ * queries `--mode sparse` and no longer passes `init --language`, which older builds reject.
+ */
+export declare const MINSYNC_MIN_VERSION = "0.6.0";
+/** A `minsync` that was found but is older than {@link MINSYNC_MIN_VERSION}, so it is never run. */
+export interface OutdatedMinSyncBinary {
+    readonly path: string;
+    readonly version: string;
+}
+/** Version from `minsync --version` output, or `undefined` when the output is not a MinSync version line. */
+export declare function parseMinSyncVersion(output: string): string | undefined;
+/** True when `version` is at or above {@link MINSYNC_MIN_VERSION}. Numeric, per component. */
+export declare function isSupportedMinSyncVersion(version: string): boolean;
+/**
+ * The version of the `minsync` at `binaryPath` when it is older than {@link MINSYNC_MIN_VERSION}, else undefined.
+ * A binary whose `--version` cannot be run or is not a MinSync version line is not called outdated, so a custom
+ * build is never locked out; the stage protocol check still guards it.
+ */
+export declare function outdatedMinSyncVersion(binaryPath: string): string | undefined;

package/dist/mirror/cloud-fingerprints.d.ts

@@ -0,0 +1,30 @@
+import type { ExactDuplicateFingerprint } from "../dupey/filter.js";
+export declare const CLOUD_FINGERPRINTS_FILE = "cloud-fingerprints.json";
+/**
+ * A cloud or network file's dupey identity, taken once while the file was on disk during its single download
+ * (or its single read over the network) and reused afterwards. This is what lets the exact-duplicate exclusion
+ * group such a file with a scanned copy without reading the released placeholder or the share again.
+ */
+export interface CloudFingerprint {
+    /** The mirror's own source identity at fingerprint time (`size:mtimeNs`), so an unchanged file is not re-read. */
+    readonly sourceVersion: string;
+    /** Canonical-text hash; equals `content_hash` from `dupey scan --json`. */
+    readonly exact: string;
+    /** Characters dupey extracted; `0` means no text, so {@link exact} identifies nothing. */
+    readonly chars: number;
+    /** The document's own last-modified time, or `null` when dupey records none. */
+    readonly modified: string | null;
+    readonly revision: string | null;
+}
+/** AutoRAG-owned store of cloud fingerprints, keyed by absolute source path. Not part of the MinSync stage schema. */
+export interface CloudFingerprintStore {
+    readonly version: 1;
+    readonly entries: Readonly<Record<string, CloudFingerprint>>;
+}
+export declare function cloudFingerprintStorePath(root: string): string;
+export declare function emptyCloudFingerprints(): CloudFingerprintStore;
+/** Load the stored fingerprints; a missing or unreadable store reads as empty, never an error. */
+export declare function loadCloudFingerprints(root: string): CloudFingerprintStore;
+export declare function saveCloudFingerprints(root: string, entries: Readonly<Record<string, CloudFingerprint>>): void;
+/** View the store as the exclusion merge's fingerprint map (absolute path to exact hash and modified time). */
+export declare function cloudFingerprintExclusions(store: CloudFingerprintStore): ReadonlyMap<string, ExactDuplicateFingerprint>;

package/dist/mirror/cloud.d.ts

@@ -0,0 +1,79 @@
+import { type CloudFileProbe, type CloudProvider } from "../filesystem/cloud-placeholder.js";
+/** Placeholders one refresh may download before the rest wait for the next refresh. */
+export declare const DEFAULT_CLOUD_MAX_FILES = 200;
+/** Placeholder bytes one refresh may download. The first file is always allowed so a large one cannot starve. */
+export declare const DEFAULT_CLOUD_MAX_BYTES: number;
+/**
+ * Retries after the first failed release. A just-hydrated file is often briefly held by the provider, so the
+ * release is refused once and succeeds on a retry; a short doubling wait usually clears it.
+ */
+export declare const DEFAULT_CLOUD_EVICT_RETRIES = 3;
+/** First wait before retrying an evict; doubles each retry (500, 1000, 2000 ms by default). */
+export declare const DEFAULT_CLOUD_EVICT_BACKOFF_MS = 500;
+export interface CloudGateOptions {
+    readonly probe?: CloudFileProbe;
+    readonly maxFiles?: number;
+    readonly maxBytes?: number;
+    readonly evictAfterParse?: boolean;
+    /** Retries after the first failed evict; `0` tries once. Default {@link DEFAULT_CLOUD_EVICT_RETRIES}. */
+    readonly evictRetries?: number;
+    /** First retry wait in ms, doubled each retry. Default {@link DEFAULT_CLOUD_EVICT_BACKOFF_MS}. */
+    readonly evictBackoffMs?: number;
+    /** Wait implementation; tests inject one that records instead of sleeping. */
+    readonly sleep?: (ms: number) => Promise<void>;
+}
+export type CloudReleaseResult = {
+    readonly ok: true;
+} | {
+    readonly ok: false;
+    readonly message: string;
+};
+/**
+ * What a refresh does with one file that needs parsing:
+ * - `local`: not a placeholder; read it as always.
+ * - `download`: a placeholder the probe can release, inside the budget; read it, then release it again.
+ * - `skip`: a placeholder whose provider the probe cannot release; do not read it.
+ * - `defer`: a releasable placeholder over this refresh's budget; leave it for the next refresh.
+ */
+export type CloudDecision = {
+    readonly kind: "local";
+} | {
+    readonly kind: "download";
+} | {
+    readonly kind: "skip";
+    readonly provider: CloudProvider | undefined;
+} | {
+    readonly kind: "defer";
+};
+/**
+ * One refresh's view of cloud placeholders. Reading a placeholder downloads it, so the refresh asks here
+ * before every read; without a probe (a platform AutoRAG has no cloud handling for) every file is `local`.
+ */
+export declare class CloudGate {
+    private readonly probe;
+    private readonly maxFiles;
+    private readonly maxBytes;
+    private readonly evictRetries;
+    private readonly evictBackoffMs;
+    private readonly sleep;
+    private readonly providerByDirectory;
+    private placeholders;
+    private downloadedFiles;
+    private downloadedBytes;
+    /** Whether a downloaded cloud file is released again after it is parsed. */
+    readonly evictAfterParse: boolean;
+    constructor(options?: CloudGateOptions);
+    /** Look up, in one batch, which of the files about to be read are placeholders. */
+    classify(paths: readonly string[]): Promise<void>;
+    isPlaceholder(path: string): boolean;
+    /** Decide, and when the answer is `download`, spend this refresh's budget on `sizeBytes`. */
+    decide(path: string, sizeBytes: number, directory: string): Promise<CloudDecision>;
+    /**
+     * Release the local copy of a file this refresh downloaded. A failed attempt is retried with doubling waits;
+     * once the retries are spent the last error is returned and the file is left on disk. Giving up is final for
+     * this refresh: the file stays indexed, and an unchanged file is not processed again, so nothing retries it later.
+     */
+    release(path: string): Promise<CloudReleaseResult>;
+    /** Providers are resolved once per directory: a File Provider domain covers the whole subtree. */
+    private providerIn;
+}

package/dist/mirror/index.d.ts

@@ -1,3 +1,7 @@
+export type { CloudEvictResult, CloudFileProbe, CloudProvider, CloudRoot, CloudRootKind, } from "../filesystem/cloud-placeholder.js";
+export { DEFAULT_CLOUD_MAX_BYTES, DEFAULT_CLOUD_MAX_FILES } from "./cloud.js";
+export type { CloudFingerprint, CloudFingerprintStore } from "./cloud-fingerprints.js";
+export { cloudFingerprintExclusions, cloudFingerprintStorePath, emptyCloudFingerprints, loadCloudFingerprints, saveCloudFingerprints, } from "./cloud-fingerprints.js";
 export type { ParsedMirrorEntry, ParsedMirrorIndex } from "./index-store.js";
 export { emptyMirrorIndex, loadMirrorIndex } from "./index-store.js";
 export { isParsedRefreshComplete, parsedMirrorRoot, parsedOutputPath, refreshReadinessPath, } from "./paths.js";

package/dist/mirror/sync.d.ts

@@ -1,3 +1,5 @@
+import { type DupeyCliOptions } from "../dupey/cli.js";
+import { type CloudFileProbe } from "../filesystem/cloud-placeholder.js";
 import { type DefaultParserRegistryOptions } from "../parser/defaults.js";
 import type { ParserRegistry } from "../parser/registry.js";
 /** Refuse to fully load/parse sources larger than this. Keeps Node heap stable on home folders. */
@@ -8,6 +10,12 @@ export declare const DEFAULT_MAX_DELETE_RATIO = 0.3;
 export declare const DEFAULT_MAX_SCANNED_ENTRIES = 2000000;
 /** Deepest directory level (search root = 0) the walk descends into. */
 export declare const DEFAULT_MAX_WALK_DEPTH = 64;
+/**
+ * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+ * runs. Small enough that a corpus becomes searchable after its first slice,
+ * large enough to amortize one index sync's startup.
+ */
+export declare const DEFAULT_MIRROR_BATCH_SIZE = 25;
 export interface ParsedMirrorSyncOptions {
     readonly root: string;
     readonly searchPaths: readonly string[];
@@ -34,6 +42,20 @@ export interface ParsedMirrorSyncOptions {
     readonly signal?: AbortSignal;
     /** Called before each file and once more with `processed === total` when the parse phase ends. */
     readonly onProgress?: (progress: ParsedMirrorProgress) => void;
+    /**
+     * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+     * runs. Default {@link DEFAULT_MIRROR_BATCH_SIZE}.
+     */
+    readonly batchSize?: number;
+    /**
+     * Runs after each batch of sources is parsed and committed to the mirror, and
+     * before the next batch is read, so a caller can make the batch's results
+     * searchable (stage them and sync the index). A thrown error aborts the
+     * refresh; returning `stop` ends the loop after this batch. Deletions of
+     * genuinely-removed sources wait for the final batch, so a mid-refresh call
+     * never drops a not-yet-reached document from the index.
+     */
+    readonly onBatch?: (batch: ParsedMirrorBatch) => Promise<ParsedMirrorBatchOutcome | undefined>;
     /**
      * Largest share of previously indexed documents one refresh may delete because their source
      * disappeared. A larger deletion (above 100 documents) is suppressed with `mass-delete-suppressed`
@@ -51,6 +73,36 @@ export interface ParsedMirrorSyncOptions {
     readonly transientRetries?: number;
     /** Base delay between transient retries; doubles per retry. Default 50ms. */
     readonly transientRetryBackoffMs?: number;
+    /**
+     * Cloud-aware handling of File Provider placeholders (files whose data is not on disk; reading one
+     * downloads it) and the roots — cloud or network — that other readers must skip or exclude. `undefined`
+     * means the platform default: macOS probes with `stat` and releases through a Foundation call, other
+     * platforms read every file as before. Tests inject their own probe.
+     */
+    readonly cloudProbe?: CloudFileProbe;
+    /** Most placeholders one refresh may download. Default {@link DEFAULT_CLOUD_MAX_FILES}. */
+    readonly cloudMaxFiles?: number;
+    /** Most placeholder bytes one refresh may download; the first file is always allowed. Default {@link DEFAULT_CLOUD_MAX_BYTES}. */
+    readonly cloudMaxBytes?: number;
+    /** Release a downloaded cloud file again once it is parsed and indexed. Default true. */
+    readonly cloudEvictAfterParse?: boolean;
+    /** Retries after a failed release (`evictUbiquitousItem`), with doubling waits; `0` tries once. Default 3. */
+    readonly cloudEvictRetries?: number;
+    /** First wait before retrying a failed release, doubled each retry. Default 500 ms. */
+    readonly cloudEvictBackoffMs?: number;
+    /** Wait used between release retries. Tests inject a recorder so they never really sleep. */
+    readonly cloudSleep?: (ms: number) => Promise<void>;
+    /**
+     * Record a dupey fingerprint for every cloud or network file parsed in this refresh, while it is on disk,
+     * into an AutoRAG-owned store under the workspace. The stored identity lets the next refresh's
+     * exact-duplicate exclusion group the file without reading its released placeholder or its share again.
+     * Absent keeps no fingerprints. Needs {@link ParsedMirrorSyncOptions.cloudProbe} for the machine's roots.
+     */
+    readonly cloudFingerprints?: CloudFingerprintSyncOptions;
+}
+/** How a refresh fingerprints cloud and network files: the dupey CLI (or a test double) that reads the exact hash. */
+export interface CloudFingerprintSyncOptions {
+    readonly dupey?: DupeyCliOptions;
 }
 /** Parse-phase progress. Every scanned file counts as processed, whether parsed, unchanged, skipped or excluded. */
 export interface ParsedMirrorProgress {
@@ -63,7 +115,7 @@ export interface ParsedMirrorProgress {
     /** The file about to be processed; absent on the final event. */
     readonly currentVirtualPath?: string;
 }
-export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable";
+export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "cloud-download-deferred" | "cloud-placeholder-skipped" | "cloud-evict-failed" | "cloud-fingerprint-failed";
 /**
  * Refresh diagnostic. `source` is an opaque virtual path, never a real filesystem path.
  * `message` keeps the underlying error text, which may include a filesystem path.
@@ -83,6 +135,32 @@ export interface ParsedMirrorSyncResult {
     readonly indexPath: string;
     readonly diagnostics: readonly ParsedMirrorDiagnostic[];
 }
+/**
+ * One batch of the source listing, parsed and committed to the mirror before
+ * {@link ParsedMirrorSyncOptions.onBatch} runs. The last batch is the only one
+ * whose run reflects deletions of genuinely-removed sources.
+ */
+export interface ParsedMirrorBatch {
+    /** 1-based index of this batch among the batches handed to `onBatch`. */
+    readonly index: number;
+    /**
+     * Batches the listing was divided into. A batch that wrote and deleted no
+     * mirror output is not handed to `onBatch`; the final batch always is.
+     */
+    readonly total: number;
+    /** Virtual paths the refresh attempted in this batch. */
+    readonly virtualPaths: readonly string[];
+    /** The last batch: genuine deletions have by now been applied to the mirror. */
+    readonly final: boolean;
+}
+/**
+ * What {@link ParsedMirrorSyncOptions.onBatch} asks the refresh to do next.
+ * A thrown error aborts the refresh; `stop` ends the loop after this batch, so a
+ * systemic index failure never reads sources it knows it cannot index.
+ */
+export interface ParsedMirrorBatchOutcome {
+    readonly stop?: boolean;
+}
 /** Default retries for a transient per-file I/O error. Mirrors the `refresh.transientRetries` config default. */
 export declare const DEFAULT_TRANSIENT_RETRIES = 3;
 /**

package/dist/parser/csv.d.ts

@@ -0,0 +1,26 @@
+import { type ParseInput, type ParseOutput, Parser } from "./types.js";
+/** Delimited-text extensions the CSV parser owns. */
+export declare const DELIMITED_EXTENSIONS: readonly [".csv", ".tsv"];
+export interface CsvParserOptions {
+    /** Rows kept per file, header included. Default 100,000. */
+    readonly maxRows?: number;
+}
+/**
+ * Parses `.csv` and `.tsv` files into the same row lines as workbooks (see `spreadsheet-render.ts`):
+ * the file name stands in for the sheet name, titles above a table become its caption, and each value
+ * is paired with its column label when the header is unambiguous.
+ *
+ * Bytes are decoded like plain text (UTF-8 with or without BOM, or CP949/EUC-KR as Korean Excel saves
+ * CSV). Fields follow RFC 4180: quoted fields may hold separators, line breaks and doubled quotes, and
+ * rows may have different lengths. `.tsv` is tab-separated; a `.csv` uses Excel's `sep=` line when present,
+ * otherwise the separator (`,` `;` tab `|`) that splits the first lines most consistently outside quotes.
+ * Values stay exactly as written: no number, date or formula conversion.
+ */
+export declare class CsvParser extends Parser {
+    #private;
+    readonly name = "csv";
+    readonly extensions: readonly [".csv", ".tsv"];
+    readonly version = "2";
+    constructor(options?: CsvParserOptions);
+    parse(input: ParseInput): Promise<ParseOutput>;
+}

package/dist/parser/defaults.d.ts

@@ -8,7 +8,7 @@ export interface DefaultParserRegistryOptions {
     /** Global document languages; selects the OCR engine when OCR is enabled. */
     readonly languages?: readonly LanguageTag[];
     /**
-     * Hard per-file parse deadline in milliseconds (default 120000). kordoc, pptx and eml parse in a worker
+     * Hard per-file parse deadline in milliseconds (default 120000). kordoc, spreadsheet, csv, pptx and eml parse in a worker
      * thread during a refresh; a parse still running at the deadline has its worker terminated and the file
      * is skipped as `parser-failed`. A synchronous spin cannot be interrupted any other way.
      */

package/dist/parser/index.d.ts

@@ -1,3 +1,4 @@
+export { CsvParser, type CsvParserOptions, DELIMITED_EXTENSIONS } from "./csv.js";
 export { createDefaultParserRegistry, type DefaultParserRegistryOptions, resolveParserOptions, } from "./defaults.js";
 export { EmlParser } from "./eml.js";
 export { ParseError } from "./errors.js";
@@ -6,4 +7,5 @@ export { BUILTIN_OCR_LANGUAGES, createTesseractOcrProvider, type OcrOptions, typ
 export { PptxParser } from "./office.js";
 export { PlainTextParser } from "./plain-text.js";
 export { ParserRegistry } from "./registry.js";
+export { SPREADSHEET_EXTENSIONS, SpreadsheetParser, type SpreadsheetParserOptions } from "./spreadsheet.js";
 export { type ParseDiagnostic, type ParseDiagnosticCode, type ParseInput, type ParseOutput, Parser, } from "./types.js";

package/dist/parser/kordoc.d.ts

@@ -37,15 +37,18 @@ export interface KordocParserOptions {
     readonly images?: boolean;
     readonly parse?: KordocParseFn;
 }
-/** Document extensions kordoc parses. `.pptx` and `.eml` stay on their own parsers. */
-export declare const KORDOC_EXTENSIONS: readonly [".hwp", ".hwpx", ".hml", ".hwpml", ".pdf", ".docx", ".xlsx", ".xls"];
+/**
+ * Document extensions kordoc parses. `.pptx` and `.eml` stay on their own parsers, and workbooks
+ * (`.xlsx`, `.xls`, …) go to the spreadsheet parser, which keeps every value next to its labels.
+ */
+export declare const KORDOC_EXTENSIONS: readonly [".hwp", ".hwpx", ".hml", ".hwpml", ".pdf", ".docx"];
 /**
  * Standalone image formats kordoc OCRs directly. `.bmp` and `.tiff` return
  * `UNSUPPORTED_FORMAT` from kordoc, so they are deliberately not claimed.
  */
 export declare const KORDOC_IMAGE_EXTENSIONS: readonly [".png", ".jpg", ".jpeg", ".webp"];
 /**
- * Parses HWP/HWPX/HWPML, PDF, DOCX and XLSX/XLS through the kordoc library.
+ * Parses HWP/HWPX/HWPML, PDF and DOCX through the kordoc library.
  *
  * kordoc reports failures as a result object; AutoRAG's contract is a thrown
  * `ParseError`, so the code and message are forwarded verbatim (AGENTS.md error