@autorag/librarian 2.13.6 → 2.13.8

package/dist/mcp/server.d.ts

@@ -1,12 +1,9 @@
 import { McpServer } from "@modelcontextprotocol/server";
 import type { AutoRAGLite } from "../core.js";
-import { type CloudFileProbe } from "../filesystem/cloud-placeholder.js";
 export interface AutoRAGMcpServerOptions {
     readonly readOnly?: boolean;
     readonly tools?: readonly string[];
     /** Test/embedded override; defaults to the current Node platform. */
     readonly platform?: NodeJS.Platform;
-    /** Test/embedded override for the cloud probe `autorag.duplicates` uses; defaults to this platform's probe. */
-    readonly cloudProbe?: CloudFileProbe;
 }
 export declare function createAutoRAGMcpServer(lite: AutoRAGLite, options?: AutoRAGMcpServerOptions): McpServer;

package/dist/minsync/installer.d.ts

@@ -1,4 +1,4 @@
-export declare const MINSYNC_VERSION = "0.6.0";
+export declare const MINSYNC_VERSION = "0.7.3";
 export declare const MINSYNC_CARGO_INSTALL_TIMEOUT_MS: number;
 export interface MinSyncReleaseAsset {
     readonly name: string;

package/dist/minsync/method.d.ts

@@ -1,3 +1,4 @@
+import { type RecordCut } from "../retrieval/record-cut.js";
 import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
 import type { MinSyncQueryMode } from "./client.js";
 import type { EnsureMinSyncBinaryOptions } from "./installer.js";
@@ -16,6 +17,11 @@ export interface MinSyncVectorMethodOptions {
     readonly defaultTopK?: number;
     /** Fetch cap applied when a scope narrows the query. Default 100. */
     readonly scopedTopK?: number;
+    /**
+     * Fuse deeper and cut the hits by document, so one long document cannot take every one of `topK` slots
+     * (see {@link RecordCut}). Unset asks MinSync for `topK` and cuts by chunk.
+     */
+    readonly recordCut?: RecordCut;
 }
 export type MinSyncHybridMethodOptions = Omit<MinSyncVectorMethodOptions, "mode">;
 export declare class MinSyncVectorMethod implements RetrievalMethod {
@@ -29,6 +35,7 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
     private readonly mode;
     private readonly defaultTopK;
     private readonly scopedTopK;
+    private readonly recordCut;
     constructor(options: MinSyncVectorMethodOptions);
     describe(): RetrievalMethodDescriptor;
     sync(force?: boolean, signal?: AbortSignal, onProgress?: (progress: MinSyncSyncProgress) => void): Promise<MinSyncSyncResult>;

package/dist/minsync/version.d.ts

@@ -1,8 +1,9 @@
 /**
  * Oldest `minsync` AutoRAG will run. 0.6.0 replaced the BM25 full-text index with SPLADE sparse retrieval: AutoRAG
- * queries `--mode sparse` and no longer passes `init --language`, which older builds reject.
+ * queries `--mode sparse` and no longer passes `init --language`, which older builds reject. 0.7.1 answers queries
+ * from every committed batch, also while the first sync is still running or was interrupted.
  */
-export declare const MINSYNC_MIN_VERSION = "0.6.0";
+export declare const MINSYNC_MIN_VERSION = "0.7.1";
 /** A `minsync` that was found but is older than {@link MINSYNC_MIN_VERSION}, so it is never run. */
 export interface OutdatedMinSyncBinary {
     readonly path: string;

package/dist/mirror/cloud-fingerprints.d.ts

@@ -8,17 +8,18 @@ export declare const CLOUD_FINGERPRINTS_FILE = "cloud-fingerprints.json";
 export interface CloudFingerprint {
     /** The mirror's own source identity at fingerprint time (`size:mtimeNs`), so an unchanged file is not re-read. */
     readonly sourceVersion: string;
-    /** Canonical-text hash; equals `content_hash` from `dupey scan --json`. */
+    /** Exact hash; equals `content_hash` from `dupey scan --json` (dupey's file-byte hash when it extracts no text). */
     readonly exact: string;
-    /** Characters dupey extracted; `0` means no text, so {@link exact} identifies nothing. */
-    readonly chars: number;
     /** The document's own last-modified time, or `null` when dupey records none. */
     readonly modified: string | null;
     readonly revision: string | null;
 }
-/** AutoRAG-owned store of cloud fingerprints, keyed by absolute source path. Not part of the MinSync stage schema. */
+/**
+ * AutoRAG-owned store of cloud fingerprints, keyed by absolute source path. Not part of the MinSync stage schema.
+ * Version 2 holds hashes from dupey 0.2.0 or later (the floor AutoRAG enforces); version 1 was written by 0.1.2.
+ */
 export interface CloudFingerprintStore {
-    readonly version: 1;
+    readonly version: 2;
     readonly entries: Readonly<Record<string, CloudFingerprint>>;
 }
 export declare function cloudFingerprintStorePath(root: string): string;

package/dist/mirror/cloud.d.ts

@@ -1,7 +1,7 @@
 import { type CloudFileProbe, type CloudProvider } from "../filesystem/cloud-placeholder.js";
-/** Placeholders one refresh may download before the rest wait for the next refresh. */
+/** Placeholders one batch may download; a placeholder over the budget starts the next batch. */
 export declare const DEFAULT_CLOUD_MAX_FILES = 200;
-/** Placeholder bytes one refresh may download. The first file is always allowed so a large one cannot starve. */
+/** Placeholder bytes one batch may download. A batch's first file is always allowed, so a large one cannot starve. */
 export declare const DEFAULT_CLOUD_MAX_BYTES: number;
 /**
  * Retries after the first failed release. A just-hydrated file is often briefly held by the provider, so the
@@ -31,9 +31,10 @@ export type CloudReleaseResult = {
 /**
  * What a refresh does with one file that needs parsing:
  * - `local`: not a placeholder; read it as always.
- * - `download`: a placeholder the probe can release, inside the budget; read it, then release it again.
+ * - `download`: a placeholder the probe can release, inside the batch's budget; read it, then release it again.
  * - `skip`: a placeholder whose provider the probe cannot release; do not read it.
- * - `defer`: a releasable placeholder over this refresh's budget; leave it for the next refresh.
+ * - `next-batch`: a releasable placeholder over this batch's budget; end the batch before it and read it first
+ *   in the next one, whose budget starts fresh.
  */
 export type CloudDecision = {
     readonly kind: "local";
@@ -43,7 +44,7 @@ export type CloudDecision = {
     readonly kind: "skip";
     readonly provider: CloudProvider | undefined;
 } | {
-    readonly kind: "defer";
+    readonly kind: "next-batch";
 };
 /**
  * One refresh's view of cloud placeholders. Reading a placeholder downloads it, so the refresh asks here
@@ -66,7 +67,9 @@ export declare class CloudGate {
     /** Look up, in one batch, which of the files about to be read are placeholders. */
     classify(paths: readonly string[]): Promise<void>;
     isPlaceholder(path: string): boolean;
-    /** Decide, and when the answer is `download`, spend this refresh's budget on `sizeBytes`. */
+    /** Give the next batch a fresh download budget. */
+    startBatch(): void;
+    /** Decide, and when the answer is `download`, spend this batch's budget on `sizeBytes`. */
     decide(path: string, sizeBytes: number, directory: string): Promise<CloudDecision>;
     /**
      * Release the local copy of a file this refresh downloaded. A failed attempt is retried with doubling waits;

package/dist/mirror/sync.d.ts

@@ -12,8 +12,8 @@ export declare const DEFAULT_MAX_SCANNED_ENTRIES = 2000000;
 export declare const DEFAULT_MAX_WALK_DEPTH = 64;
 /**
  * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
- * runs. Small enough that a corpus becomes searchable after its first slice,
- * large enough to amortize one index sync's startup.
+ * runs. Small enough that the first batch starts the index sync soon after parsing
+ * begins; later syncs fold every batch committed meanwhile into one run.
  */
 export declare const DEFAULT_MIRROR_BATCH_SIZE = 25;
 export interface ParsedMirrorSyncOptions {
@@ -28,6 +28,15 @@ export interface ParsedMirrorSyncOptions {
     readonly maxSourceBytes?: number;
     /** Absolute source paths omitted from parsed mirrors (for example exact duplicate copies). */
     readonly excludeSourcePaths?: ReadonlySet<string>;
+    /**
+     * Read-only freshness checks only: omit the sources the last refresh already recorded as
+     * `duplicate-excluded` instead of requiring a fresh duplicate scan. Duplicate selection needs
+     * a content scan of every root, which is the dominant cost of a status or retrieve call.
+     * A recorded copy is only omitted while its mtime and size still match the record, so an
+     * edited copy is still reported stale. Copies that became duplicates after the last refresh
+     * are not known here; see `AutoRAGAgent.scanMirrorStaleness`.
+     */
+    readonly reuseRecordedDuplicateExclusions?: boolean;
     /** Absolute source paths or directories omitted by the user. */
     readonly userExcludedSourcePaths?: ReadonlySet<string>;
     /**
@@ -43,14 +52,17 @@ export interface ParsedMirrorSyncOptions {
     /** Called before each file and once more with `processed === total` when the parse phase ends. */
     readonly onProgress?: (progress: ParsedMirrorProgress) => void;
     /**
-     * Sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
-     * runs. Default {@link DEFAULT_MIRROR_BATCH_SIZE}.
+     * Most sources parsed and committed per batch before {@link ParsedMirrorSyncOptions.onBatch}
+     * runs; a batch ends sooner when a cloud placeholder would exceed its download budget.
+     * Default {@link DEFAULT_MIRROR_BATCH_SIZE}.
      */
     readonly batchSize?: number;
     /**
-     * Runs after each batch of sources is parsed and committed to the mirror, and
-     * before the next batch is read, so a caller can make the batch's results
-     * searchable (stage them and sync the index). A thrown error aborts the
+     * Runs after each batch of sources is parsed and committed to the mirror, so
+     * a caller can make the batch's results searchable (sync the index). The next
+     * batch is read once the returned promise settles: a caller that wants parsing
+     * to carry on returns at once and indexes in the background, and waits only on
+     * the final batch. A thrown error aborts the
      * refresh; returning `stop` ends the loop after this batch. Deletions of
      * genuinely-removed sources wait for the final batch, so a mid-refresh call
      * never drops a not-yet-reached document from the index.
@@ -80,9 +92,15 @@ export interface ParsedMirrorSyncOptions {
      * platforms read every file as before. Tests inject their own probe.
      */
     readonly cloudProbe?: CloudFileProbe;
-    /** Most placeholders one refresh may download. Default {@link DEFAULT_CLOUD_MAX_FILES}. */
+    /**
+     * Most placeholders one batch may download; the next placeholder starts a new batch, so every placeholder is
+     * still read in the same refresh. Default {@link DEFAULT_CLOUD_MAX_FILES}.
+     */
     readonly cloudMaxFiles?: number;
-    /** Most placeholder bytes one refresh may download; the first file is always allowed. Default {@link DEFAULT_CLOUD_MAX_BYTES}. */
+    /**
+     * Most placeholder bytes one batch may download; a batch's first placeholder is always allowed, and the next
+     * one over the budget starts a new batch. Default {@link DEFAULT_CLOUD_MAX_BYTES}.
+     */
     readonly cloudMaxBytes?: number;
     /** Release a downloaded cloud file again once it is parsed and indexed. Default true. */
     readonly cloudEvictAfterParse?: boolean;
@@ -115,7 +133,7 @@ export interface ParsedMirrorProgress {
     /** The file about to be processed; absent on the final event. */
     readonly currentVirtualPath?: string;
 }
-export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "cloud-download-deferred" | "cloud-placeholder-skipped" | "cloud-evict-failed" | "cloud-fingerprint-failed";
+export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-changed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index" | "pdf-extract-thin" | "parser-warning" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "cloud-placeholder-skipped" | "cloud-evict-failed" | "cloud-fingerprint-failed";
 /**
  * Refresh diagnostic. `source` is an opaque virtual path, never a real filesystem path.
  * `message` keeps the underlying error text, which may include a filesystem path.
@@ -143,11 +161,6 @@ export interface ParsedMirrorSyncResult {
 export interface ParsedMirrorBatch {
     /** 1-based index of this batch among the batches handed to `onBatch`. */
     readonly index: number;
-    /**
-     * Batches the listing was divided into. A batch that wrote and deleted no
-     * mirror output is not handed to `onBatch`; the final batch always is.
-     */
-    readonly total: number;
     /** Virtual paths the refresh attempted in this batch. */
     readonly virtualPaths: readonly string[];
     /** The last batch: genuine deletions have by now been applied to the mirror. */