@autorag/librarian 2.13.5 → 2.13.7

package/dist/mcp/server.d.ts

@@ -1,12 +1,9 @@
 import { McpServer } from "@modelcontextprotocol/server";
 import type { AutoRAGLite } from "../core.js";
-import { type CloudFileProbe } from "../filesystem/cloud-placeholder.js";
 export interface AutoRAGMcpServerOptions {
     readonly readOnly?: boolean;
     readonly tools?: readonly string[];
     /** Test/embedded override; defaults to the current Node platform. */
     readonly platform?: NodeJS.Platform;
-    /** Test/embedded override for the cloud probe `autorag.duplicates` uses; defaults to this platform's probe. */
-    readonly cloudProbe?: CloudFileProbe;
 }
 export declare function createAutoRAGMcpServer(lite: AutoRAGLite, options?: AutoRAGMcpServerOptions): McpServer;

package/dist/minsync/installer.d.ts

@@ -1,4 +1,4 @@
-export declare const MINSYNC_VERSION = "0.6.0";
+export declare const MINSYNC_VERSION = "0.7.3";
 export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
 export interface MinSyncReleaseAsset {
     readonly name: string;

package/dist/minsync/version.d.ts

@@ -1,8 +1,9 @@
 /**
  * Oldest `minsync` AutoRAG will run. 0.6.0 replaced the BM25 full-text index with SPLADE sparse retrieval: AutoRAG
- * queries `--mode sparse` and no longer passes `init --language`, which older builds reject.
+ * queries `--mode sparse` and no longer passes `init --language`, which older builds reject. 0.7.1 answers queries
+ * from every committed batch, also while the first sync is still running or was interrupted.
  */
-export declare const MINSYNC_MIN_VERSION = "0.6.0";
+export declare const MINSYNC_MIN_VERSION = "0.7.1";
 /** A `minsync` that was found but is older than {@link MINSYNC_MIN_VERSION}, so it is never run. */
 export interface OutdatedMinSyncBinary {
     readonly path: string;

package/dist/mirror/cloud-fingerprints.d.ts

@@ -8,17 +8,18 @@ export declare const CLOUD_FINGERPRINTS_FILE = "cloud-fingerprints.json";
 export interface CloudFingerprint {
     /** The mirror's own source identity at fingerprint time (`size:mtimeNs`), so an unchanged file is not re-read. */
     readonly sourceVersion: string;
-    /** Canonical-text hash; equals `content_hash` from `dupey scan --json`. */
+    /** Exact hash; equals `content_hash` from `dupey scan --json` (dupey's file-byte hash when it extracts no text). */
     readonly exact: string;
-    /** Characters dupey extracted; `0` means no text, so {@link exact} identifies nothing. */
-    readonly chars: number;
     /** The document's own last-modified time, or `null` when dupey records none. */
     readonly modified: string | null;
     readonly revision: string | null;
 }
-/** AutoRAG-owned store of cloud fingerprints, keyed by absolute source path. Not part of the MinSync stage schema. */
+/**
+ * AutoRAG-owned store of cloud fingerprints, keyed by absolute source path. Not part of the MinSync stage schema.
+ * Version 2 holds hashes from dupey 0.2.0 or later (the floor AutoRAG enforces); version 1 was written by 0.1.2.
+ */
 export interface CloudFingerprintStore {
-    readonly version: 1;
+    readonly version: 2;
     readonly entries: Readonly<Record<string, CloudFingerprint>>;
 }
 export declare function cloudFingerprintStorePath(root: string): string;

package/dist/mirror/cloud.d.ts

@@ -1,7 +1,7 @@
 import { type CloudFileProbe, type CloudProvider } from "../filesystem/cloud-placeholder.js";
-/** Placeholders one refresh may download before the rest wait for the next refresh. */
+/** Placeholders one batch may download; a placeholder over the budget starts the next batch. */
 export declare const DEFAULT_CLOUD_MAX_FILES = 200;
-/** Placeholder bytes one refresh may download. The first file is always allowed so a large one cannot starve. */
+/** Placeholder bytes one batch may download. A batch's first file is always allowed, so a large one cannot starve. */
 export declare const DEFAULT_CLOUD_MAX_BYTES: number;
 /**
  * Retries after the first failed release. A just-hydrated file is often briefly held by the provider, so the
@@ -31,9 +31,10 @@ export type CloudReleaseResult = {
 /**
  * What a refresh does with one file that needs parsing:
  * - `local`: not a placeholder; read it as always.
- * - `download`: a placeholder the probe can release, inside the budget; read it, then release it again.
+ * - `download`: a placeholder the probe can release, inside the batch's budget; read it, then release it again.
  * - `skip`: a placeholder whose provider the probe cannot release; do not read it.
- * - `defer`: a releasable placeholder over this refresh's budget; leave it for the next refresh.
+ * - `next-batch`: a releasable placeholder over this batch's budget; end the batch before it and read it first
+ *   in the next one, whose budget starts fresh.
  */
 export type CloudDecision = {
     readonly kind: "local";
@@ -43,7 +44,7 @@ export type CloudDecision = {
     readonly kind: "skip";
     readonly provider: CloudProvider | undefined;
 } | {
-    readonly kind: "defer";
+    readonly kind: "next-batch";
 };
 /**
  * One refresh's view of cloud placeholders. Reading a placeholder downloads it, so the refresh asks here
@@ -66,7 +67,9 @@ export declare class CloudGate {
     /** Look up, in one batch, which of the files about to be read are placeholders. */
     classify(paths: readonly string[]): Promise<void>;
     isPlaceholder(path: string): boolean;
-    /** Decide, and when the answer is `download`, spend this refresh's budget on `sizeBytes`. */
+    /** Give the next batch a fresh download budget. */
+    startBatch(): void;
+    /** Decide, and when the answer is `download`, spend this batch's budget on `sizeBytes`. */
     decide(path: string, sizeBytes: number, directory: string): Promise<CloudDecision>;
     /**
      * Release the local copy of a file this refresh downloaded. A failed attempt is retried with doubling waits;

package/dist/mirror/sync.d.ts

@@ -10,6 +10,12 @@ export declare const DEFAULT_MAX_DELETE_RATIO = 0.3;
 export declare const DEFAULT_MAX_SCANNED_ENTRIES = 2000000;
 /** Deepest directory level (search root = 0) the walk descends into. */
 export declare const DEFAULT_MAX_WALK_DEPTH = 64;
+/**
+ * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+ * runs. Small enough that the first batch starts the index sync soon after parsing
+ * begins; later syncs fold every batch committed meanwhile into one run.
+ */
+export declare const DEFAULT_MIRROR_BATCH_SIZE = 25;
 export interface ParsedMirrorSyncOptions {
     readonly root: string;
     readonly searchPaths: readonly string[];
@@ -36,6 +42,23 @@ export interface ParsedMirrorSyncOptions {
     readonly signal?: AbortSignal;
     /** Called before each file and once more with `processed === total` when the parse phase ends. */
     readonly onProgress?: (progress: ParsedMirrorProgress) => void;
+    /**
+     * Most sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+     * runs; a batch ends sooner when a cloud placeholder would exceed its download budget.
+     * Default {@link DEFAULT_MIRROR_BATCH_SIZE}.
+     */
+    readonly batchSize?: number;
+    /**
+     * Runs after each batch of sources is parsed and committed to the mirror, so
+     * a caller can make the batch's results searchable (sync the index). The next
+     * batch is read once the returned promise settles: a caller that wants parsing
+     * to carry on returns at once and indexes in the background, and waits only on
+     * the final batch. A thrown error aborts the
+     * refresh; returning `stop` ends the loop after this batch. Deletions of
+     * genuinely-removed sources wait for the final batch, so a mid-refresh call
+     * never drops a not-yet-reached document from the index.
+     */
+    readonly onBatch?: (batch: ParsedMirrorBatch) => Promise<ParsedMirrorBatchOutcome | undefined>;
     /**
      * Largest share of previously indexed documents one refresh may delete because their source
      * disappeared. A larger deletion (above 100 documents) is suppressed with `mass-delete-suppressed`
@@ -60,9 +83,15 @@ export interface ParsedMirrorSyncOptions {
      * platforms read every file as before. Tests inject their own probe.
      */
     readonly cloudProbe?: CloudFileProbe;
-    /** Most placeholders one refresh may download. Default {@link DEFAULT_CLOUD_MAX_FILES}. */
+    /**
+     * Most placeholders one batch may download; the next placeholder starts a new batch, so every placeholder is
+     * still read in the same refresh. Default {@link DEFAULT_CLOUD_MAX_FILES}.
+     */
     readonly cloudMaxFiles?: number;
-    /** Most placeholder bytes one refresh may download; the first file is always allowed. Default {@link DEFAULT_CLOUD_MAX_BYTES}. */
+    /**
+     * Most placeholder bytes one batch may download; a batch's first placeholder is always allowed, and the next
+     * one over the budget starts a new batch. Default {@link DEFAULT_CLOUD_MAX_BYTES}.
+     */
     readonly cloudMaxBytes?: number;
     /** Release a downloaded cloud file again once it is parsed and indexed. Default true. */
     readonly cloudEvictAfterParse?: boolean;
@@ -95,7 +124,7 @@ export interface ParsedMirrorProgress {
     /** The file about to be processed; absent on the final event. */
     readonly currentVirtualPath?: string;
 }
-export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "cloud-download-deferred" | "cloud-placeholder-skipped" | "cloud-evict-failed" | "cloud-fingerprint-failed";
+export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "cloud-placeholder-skipped" | "cloud-evict-failed" | "cloud-fingerprint-failed";
 /**
  * Refresh diagnostic. `source` is an opaque virtual path, never a real filesystem path.
  * `message` keeps the underlying error text, which may include a filesystem path.
@@ -115,6 +144,27 @@ export interface ParsedMirrorSyncResult {
     readonly indexPath: string;
     readonly diagnostics: readonly ParsedMirrorDiagnostic[];
 }
+/**
+ * One batch of the source listing, parsed and committed to the mirror before
+ * {@link ParsedMirrorSyncOptions.onBatch} runs. The last batch is the only one
+ * whose run reflects deletions of genuinely-removed sources.
+ */
+export interface ParsedMirrorBatch {
+    /** 1-based index of this batch among the batches handed to `onBatch`. */
+    readonly index: number;
+    /** Virtual paths the refresh attempted in this batch. */
+    readonly virtualPaths: readonly string[];
+    /** The last batch: genuine deletions have by now been applied to the mirror. */
+    readonly final: boolean;
+}
+/**
+ * What {@link ParsedMirrorSyncOptions.onBatch} asks the refresh to do next.
+ * A thrown error aborts the refresh; `stop` ends the loop after this batch, so a
+ * systemic index failure never reads sources it knows it cannot index.
+ */
+export interface ParsedMirrorBatchOutcome {
+    readonly stop?: boolean;
+}
 /** Default retries for a transient per-file I/O error. Mirrors the `refresh.transientRetries` config default. */
 export declare const DEFAULT_TRANSIENT_RETRIES = 3;
 /**