@autorag/librarian 2.13.4 → 2.13.6

package/dist/filesystem/cloud-placeholder.d.ts

@@ -1,26 +1,318 @@
-/** Darwin st_flags bit used by iCloud/File Provider data-less placeholders. */
-export declare const UF_DATALESS = 32768;
+/**
+ * Darwin `SF_DATALESS` (st_flags): the file's data lives only in the cloud. `UF_DATALESS` is not a flag;
+ * `0x8000` is `UF_HIDDEN`, which dotfiles carry, so it must never be read as a placeholder marker.
+ */
+export declare const SF_DATALESS = 1073741824;
 export declare const FILE_PROVIDER_XATTR = "com.apple.file-provider-domain-id";
-/** Windows FILE_ATTRIBUTE_* bits commonly used by Cloud Files placeholders. */
-export declare const WINDOWS_FILE_ATTRIBUTE_OFFLINE = 4096;
-export declare const WINDOWS_FILE_ATTRIBUTE_RECALL_ON_OPEN = 262144;
+/** Windows FILE_ATTRIBUTE_* bits carried by Cloud Files placeholders and their pin state. */
+export declare const WINDOWS_FILE_ATTRIBUTE_PINNED = 524288;
+export declare const WINDOWS_FILE_ATTRIBUTE_UNPINNED = 1048576;
 export declare const WINDOWS_FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 4194304;
-export type FilesystemKind = "local" | "file-provider";
-export interface FilesystemClassification {
-    readonly kind: FilesystemKind;
-    readonly provider?: string;
-    readonly reason: "path-marker" | "file-provider-xattr" | "cloud-volume" | "placeholder-attribute" | "local";
-}
 export declare function pathLooksLikeCloudRoot(target: string): boolean;
 export declare function homeCloudRoots(home?: string): readonly string[];
-export declare function classifyFilesystemRoot(target: string): Promise<FilesystemClassification>;
-export declare function isDatalessPlaceholder(target: string): Promise<boolean>;
-export interface MaterializedWalk {
-    readonly materialized: readonly string[];
-    readonly skippedDataless: number;
-}
-export declare function listMaterializedFiles(root: string, options?: {
-    readonly limit?: number;
+/** Whether `st_flags` (the decimal number `stat -f %f` prints) marks a data-less File Provider placeholder. */
+export declare function datalessFromDarwinFlags(flags: number): boolean;
+/**
+ * Whether a decimal `FileAttributes` word marks a Cloud Files placeholder whose bytes live only in the cloud.
+ * Measured on Windows 11 with OneDrive: an online-only placeholder carries `FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS`;
+ * `FILE_ATTRIBUTE_OFFLINE` is NOT set on it, so an earlier check that required the offline bit missed every
+ * real placeholder (#2043).
+ */
+export declare function datalessFromWindowsAttributes(attributes: number): boolean;
+/** Which cloud backs a path. Whether AutoRAG may release a provider's files again is up to {@link CloudFileProbe.canRelease}. */
+export type CloudProvider = "icloud" | "google-drive" | "onedrive" | "dropbox" | "file-provider";
+export declare function cloudProviderFromPath(target: string): CloudProvider;
+/** Provider named by a File Provider domain id such as `com.apple.CloudDocs.iCloudDriveFileProvider/<uuid>`. */
+export declare function cloudProviderFromDomain(domain: string): CloudProvider;
+/** One entry of a cloud-root listing: its name and whether it is a directory. Never carries file contents. */
+export interface CloudDirEntry {
+    readonly name: string;
+    readonly isDirectory: () => boolean;
+}
+/**
+ * Whether `target` is `parent` itself or lies under it. Pure: compares normalized paths only.
+ * Case-insensitive on Windows, where a drive letter and file names may differ in case.
+ */
+export declare function isPathInside(target: string, parent: string): boolean;
+/**
+ * What kind of non-physical location a machine root is:
+ * - `file-provider`: a macOS File Provider directory (a `~/Library/CloudStorage` mount, iCloud Drive, or a
+ *   directory carrying `com.apple.file-provider-domain-id`, such as the iCloud-synced `~/Desktop`/`~/Documents`).
+ * - `cloud-files`: a Windows Cloud Files sync root (OneDrive, Dropbox, iCloud for Windows, …).
+ * - `network`: a network share or virtual filesystem — an SMB/NFS/AFP/WebDAV/FUSE mount, a mapped network
+ *   drive, a UNC path, or a Windows network/FUSE volume.
+ */
+export type CloudRootKind = "file-provider" | "cloud-files" | "network";
+/** One detected non-physical root on this machine: its absolute path and what it is. */
+export interface CloudRoot {
+    readonly path: string;
+    readonly kind: CloudRootKind;
+}
+/**
+ * Best-effort canonical absolute path: symlinks are resolved when the path exists, and the input is returned
+ * unchanged otherwise (a path that does not exist, or one whose lookup fails). macOS reports a mount point at
+ * its real path — `mount_webdav … /var/…` shows up as `/private/var/…` — so both sides of a root comparison
+ * are canonicalized. Resolving a path reads directory metadata only, never file contents.
+ */
+export declare function canonicalPath(value: string): string;
+/**
+ * Rewrites a walked path into the same canonical space as {@link CloudRoot.path}: the search root's own
+ * symlinks are resolved once, so a file reached through `/var/…` compares against a mount root that `mount`
+ * reports at `/private/var/…`. The result is a comparison key, not a path to open: a path under a rewritten
+ * search root comes back in the comparable (`/`-separated) form, and a path under no such search root is
+ * returned unchanged. Pure apart from one `realpath` per search root.
+ */
+export declare function canonicalPathMapper(searchRoots: readonly string[]): (path: string) => string;
+/** What a search root's relationship to this machine's non-physical roots is. */
+export interface CloudScope {
+    /** The search root is a cloud or network folder (or lies inside one): readers that open contents skip it. */
+    readonly insideCloudRoot: boolean;
+    /** Cloud or network folders nested under the search root: readers that must not read them exclude these. */
+    readonly nestedCloudRoots: readonly CloudRoot[];
+}
+/**
+ * Classify a search root against the machine's non-physical roots. Jikji `prepare`, the dupey exact-duplicate
+ * scan, and the cloud fingerprints all use this: a root inside a cloud or network folder is skipped entirely
+ * (reading a cloud placeholder downloads it; a network file is re-read over the wire), and folders nested
+ * inside a root are excluded. Both sides are canonicalized, so a search root or a mount reported at its real
+ * path (`/private/var/…`) still matches; paths that do not exist compare as given.
+ */
+export declare function cloudScope(searchRoot: string, cloudRoots: readonly CloudRoot[]): CloudScope;
+export type CloudEvictResult = {
+    readonly ok: true;
+} | {
+    readonly ok: false;
+    readonly message: string;
+};
+/**
+ * What a refresh needs to know about cloud files, and the one action it may take on them. Every method
+ * is read-only except {@link CloudFileProbe.evict}, which only drops the local copy of a file.
+ */
+export interface CloudFileProbe {
+    /**
+     * Absolute roots of the non-physical folders on this machine (each File Provider mount and File
+     * Provider–managed directory on macOS; the registered Cloud Files sync roots, network/FUSE volumes, and the
+     * search roots that are UNC paths on Windows). `searchRoots` are the roots a refresh is about to walk, so a
+     * folder provider that is not under a known mount (iCloud Desktop/Documents) can be found through the
+     * `com.apple.file-provider-domain-id` xattr. Listing reads directory metadata only, never file contents.
+     * Readers that open originals — Jikji's prepare pass, the dupey exact-duplicate scan, and the cloud
+     * fingerprints — use this to skip or exclude them.
+     */
+    cloudRoots(searchRoots: readonly string[]): Promise<readonly CloudRoot[]>;
+    /** The subset of `paths` whose data is not on disk. Reading one of these downloads it. */
+    datalessAmong(paths: readonly string[]): Promise<ReadonlySet<string>>;
+    /** Provider behind a placeholder, or `undefined` when the path is not under a File Provider domain. */
+    providerOf(path: string): Promise<CloudProvider | undefined>;
+    /**
+     * Whether {@link CloudFileProbe.evict} is verified to release this provider's files on this platform. A refresh
+     * downloads a placeholder only when this is true; any other placeholder is skipped unread.
+     */
+    canRelease(provider: CloudProvider | undefined): boolean;
+    /** Release the local copy of a downloaded file; the file stays in the cloud and keeps its metadata. */
+    evict(path: string): Promise<CloudEvictResult>;
+}
+export interface CloudExecOptions {
     readonly timeoutMs?: number;
-}): Promise<MaterializedWalk>;
-export declare function relativeCloudHint(root: string, filePath: string): string;
+    readonly maxBuffer?: number;
+}
+/** Runs a helper process. Injected by tests so they never need a real File Provider mount. */
+export type CloudExec = (file: string, args: readonly string[], options?: CloudExecOptions) => Promise<{
+    readonly stdout: string;
+    readonly stderr: string;
+}>;
+/** Paths per `xattr` invocation: keeps argv well under the OS limit while batching whole trees. */
+export declare const XATTR_BATCH = 200;
+/** One line of `/sbin/mount`: `<device> on <mountpoint> (<fstype>, <option>, …)`. */
+export interface DarwinMount {
+    readonly device: string;
+    readonly mountPoint: string;
+    readonly fileSystem: string;
+    readonly options: readonly string[];
+}
+/**
+ * Parses `/sbin/mount` output. The mount point is everything between the first ` on ` and the final ` (`, so a
+ * path or device name containing spaces survives; the parenthesised tail is `<fstype>, <option>, …`.
+ */
+export declare function parseDarwinMounts(stdout: string): DarwinMount[];
+/**
+ * Mount points of the volumes that are not local storage: a share (SMB/NFS/AFP/WebDAV), a FUSE mount, or any
+ * other filesystem that does not carry the `local` option. `autofs` (the `map auto_home` automounter) also
+ * lacks `local` but is not a share, so it is skipped. Mount points are canonicalized by the caller.
+ */
+export declare function networkMountPoints(mounts: readonly DarwinMount[]): readonly string[];
+/**
+ * Parses `xattr -v -p <attr> <path>…` stdout. Each path that carries the attribute is printed as
+ * `<path as given>: <value>`; paths without it print nothing to stdout (the `No such xattr` note goes to
+ * stderr and the process exits nonzero). A line is matched to its candidate by prefix, longest candidate
+ * first, so a path that contains `: ` is not mis-split.
+ */
+export declare function parseXattrValues(stdout: string, candidates: readonly string[]): ReadonlyMap<string, string>;
+export interface DarwinCloudProbeOptions {
+    /** Process runner for `stat`, the release helper, `/sbin/mount`, and `xattr`. Injected by tests. */
+    readonly exec?: CloudExec;
+    /** Directory listing for {@link DarwinCloudProbe.cloudRoots}; injected by tests so they need no real mounts. */
+    readonly readDir?: (path: string) => Promise<readonly CloudDirEntry[]>;
+    /** Home directory holding `Library/CloudStorage` and `Library/Mobile Documents`; defaults to the user's home. */
+    readonly home?: string;
+}
+/**
+ * The macOS probe. `stat -f %f` reads `st_flags`, which Node's `fs.stat` does not expose; a placeholder is
+ * the file whose `SF_DATALESS` bit is set. Released with {@link DarwinCloudProbe.evict}: a Foundation
+ * `evictUbiquitousItem` call through `osascript -l JavaScript`, which drops the local copy of iCloud,
+ * Google Drive, OneDrive, and Dropbox placeholders alike (measured; `brctl evict` works only for iCloud, so
+ * not used).
+ */
+export declare class DarwinCloudProbe implements CloudFileProbe {
+    private readonly exec;
+    private readonly readDir;
+    private readonly home;
+    constructor(options?: DarwinCloudProbeOptions);
+    /**
+     * The machine's non-physical roots, without ever reading file contents:
+     * - each File Provider mount is its own directory directly under `~/Library/CloudStorage` (for example
+     *   `GoogleDrive-user`, `OneDrive-Work`, `Dropbox`), and iCloud Drive's own root is `~/Library/Mobile
+     *   Documents`;
+     * - every mount from `/sbin/mount` that lacks the `local` option is a network share, WebDAV, or FUSE
+     *   mount (`autofs` is not), reported by `mount` at its real path;
+     * - a directory managed by a File Provider that is not under a known mount — the iCloud-synced `~/Desktop`
+     *   and `~/Documents` carry `com.apple.file-provider-domain-id` — is found on each search root and its
+     *   ancestors (`inside`) and on each direct child directory (`nested`), batched through `xattr`.
+     *
+     * A helper that fails (no `mount`, no `xattr`, a hung share) contributes no roots rather than failing the
+     * refresh: the roots found by the other sources stand.
+     */
+    cloudRoots(searchRoots?: readonly string[]): Promise<readonly CloudRoot[]>;
+    /** Mount points of the volumes `/sbin/mount` reports without the `local` option; `[]` when `mount` fails. */
+    private networkMountPoints;
+    /**
+     * Directories carrying `com.apple.file-provider-domain-id` among each search root's ancestors and direct
+     * child directories. One directory listing per search root and as few `xattr` processes as the path count
+     * allows; a failure of either contributes no directories.
+     */
+    private fileProviderDirectories;
+    /**
+     * The paths carrying {@link FILE_PROVIDER_XATTR}. `xattr` exits nonzero as soon as one path lacks the
+     * attribute, so the values it printed before that are read off the rejected helper's stdout; a helper that
+     * produced nothing usable contributes no paths.
+     */
+    private xattrValues;
+    private tryReadDir;
+    datalessAmong(paths: readonly string[]): Promise<ReadonlySet<string>>;
+    providerOf(path: string): Promise<CloudProvider | undefined>;
+    canRelease(provider: CloudProvider | undefined): boolean;
+    evict(path: string): Promise<CloudEvictResult>;
+}
+/** Paths per PowerShell process: long cloud paths still keep the command line under the Windows limit. */
+export declare const WINDOWS_ATTRIBUTE_BATCH = 100;
+export interface PowerShellResult {
+    readonly ok: boolean;
+    readonly stdout: string;
+    readonly stderr: string;
+}
+/** Runs one PowerShell process. The default is {@link runPowerShell}. */
+export type PowerShellRunner = (script: string, timeoutMs: number) => Promise<PowerShellResult>;
+/** One local sync root registered by a Cloud Files provider, with the name Explorer shows for it. */
+export interface WindowsSyncRoot {
+    readonly path: string;
+    readonly label: string;
+}
+/**
+ * Reads `FileAttributes` for a batch of paths in one process: one decimal attribute word per line, in batch
+ * order, and `ERR` for a path that could not be read. The list is passed as a PowerShell array literal.
+ */
+export declare function buildWindowsAttributeScript(paths: readonly string[]): string;
+/** Dumps every Cloud Files sync root and its Explorer display name, one `path<TAB>label` line each. */
+export declare function buildWindowsSyncRootsScript(): string;
+/**
+ * Opens one file with a Cloud Files oplock and dehydrates it synchronously with `CfDehydratePlaceholder`.
+ * Prints `OPEN=` and `DEHYDRATE=` as hex HRESULTs and, on failure, `MSG=` with the OS message. Dehydration
+ * is provider-agnostic and does not change the file's pin state; it refuses a file the provider has not
+ * finished uploading, which is exactly the refusal a refresh wants to surface.
+ */
+export declare function buildWindowsDehydrateScript(path: string): string;
+/** Maps one attribute word per output line back to the batch paths, in order; unparseable lines are dropped. */
+export declare function parseWindowsAttributeOutput(stdout: string, paths: readonly string[]): Map<string, number>;
+/** Parses the `path<TAB>label` lines of {@link buildWindowsSyncRootsScript} into sync roots. */
+export declare function parseWindowsSyncRoots(stdout: string): WindowsSyncRoot[];
+/** Provider a sync root's Explorer display name belongs to, from the names the providers register. */
+export declare function providerFromWindowsSyncRoot(label: string): CloudProvider;
+/** `Win32_LogicalDisk.DriveType` for a mapped network drive. */
+export declare const WINDOWS_DRIVE_TYPE_NETWORK = 4;
+/** One logical disk as `Win32_LogicalDisk` reports it. */
+export interface WindowsLogicalDisk {
+    /** The drive letter, for example `C:`. */
+    readonly deviceId: string;
+    readonly driveType: number;
+    readonly fileSystem: string;
+    readonly volumeName: string;
+}
+/** Dumps every logical disk, one `deviceId<TAB>driveType<TAB>fileSystem<TAB>volumeName` line each. */
+export declare function buildWindowsDiskScript(): string;
+/** Parses the tab-separated lines of {@link buildWindowsDiskScript}. */
+export declare function parseWindowsLogicalDisks(stdout: string): WindowsLogicalDisk[];
+/**
+ * The non-physical roots among a disk list:
+ * - `DriveType 4` is a mapped network drive (`X:\`), always `network`;
+ * - a FUSE-compatible volume (WinFsp, Dokan, rclone) is a virtual filesystem, treated as `network`;
+ * - Google Drive for desktop mounts a drive letter the Cloud Files registry does not list, recognized by its
+ *   `GoogleDriveFS` filesystem or `Google Drive` volume name — **unverified**: no Windows host with Google
+ *   Drive for desktop installed has been measured.
+ */
+export declare function windowsRootsFromDisks(disks: readonly WindowsLogicalDisk[]): readonly CloudRoot[];
+/**
+ * The share root of a UNC path — `\\server\share` for `\\server\share\folder`. Returns `undefined` for a local
+ * path and for a device path (`\\?\…`, `\\.\…`), which is not a share.
+ */
+export declare function uncShareRoot(target: string): string | undefined;
+/**
+ * Maps the `OPEN`/`DEHYDRATE`/`MSG` lines of {@link buildWindowsDehydrateScript} to an eviction result.
+ * `HRESULT 0` is success; anything else (including a process that produced no lines) is a failure whose
+ * message is the OS text when the script printed one, otherwise the process stderr.
+ */
+export declare function parseWindowsDehydrateOutput(result: PowerShellResult): CloudEvictResult;
+export interface WindowsCloudProbeOptions {
+    /** Runs PowerShell; tests inject one that returns canned output instead of spawning a process. */
+    readonly run?: PowerShellRunner;
+}
+/**
+ * The Windows probe. A Cloud Files placeholder carries `FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS`; the provider is the
+ * registered sync root that contains the path (else the path's own marker); the local copy is released with
+ * `CfDehydratePlaceholder`. Release is allowed for every Cloud Files sync root because the platform itself refuses
+ * to dehydrate a file with unsynced changes or a pinned file (measured: 0x80070179 and 0x80070188). Verified against
+ * a synthetic cfapi provider only; OneDrive, Dropbox and other commercial providers were not measured.
+ */
+export declare class WindowsCloudProbe implements CloudFileProbe {
+    private readonly run;
+    private syncRoots;
+    private disks;
+    constructor(options?: WindowsCloudProbeOptions);
+    /**
+     * The machine's non-physical roots, none of which reads a file's contents: the registered Cloud Files sync
+     * roots (registry), the mapped network and FUSE-compatible volume letters (`Win32_LogicalDisk`, one
+     * PowerShell process), and the share root of any search root that is a UNC path (`\\server\share\…`).
+     */
+    cloudRoots(searchRoots?: readonly string[]): Promise<readonly CloudRoot[]>;
+    datalessAmong(paths: readonly string[]): Promise<ReadonlySet<string>>;
+    providerOf(path: string): Promise<CloudProvider | undefined>;
+    canRelease(provider: CloudProvider | undefined): boolean;
+    evict(path: string): Promise<CloudEvictResult>;
+    /** The sync roots are read once per probe; a failure falls back to the path markers, never aborts a refresh. */
+    private loadSyncRoots;
+    /** The disk list is read once per probe in one PowerShell process; a failure contributes no roots. */
+    private loadDiskRoots;
+}
+/**
+ * How {@link runPowerShell} starts `powershell.exe`. The command line itself is a fixed, ASCII statement that
+ * switches the console to UTF-8 and only then evaluates the script, which it reads from the environment. An error in
+ * the script itself (a parse error included) is then written in UTF-8. A script passed on the command line is
+ * parsed before any statement can run, so a parse error came out in the console code page (CP949 on Korean Windows),
+ * which reads as mojibake.
+ */
+export declare function powerShellLaunch(script: string): {
+    readonly args: readonly string[];
+    readonly env: Readonly<Record<string, string>>;
+};
+/** The default {@link PowerShellRunner}. */
+export declare function runPowerShell(script: string, timeoutMs: number): Promise<PowerShellResult>;
+/** The probe for this platform, or `undefined` where AutoRAG has no cloud-aware handling yet. */
+export declare function defaultCloudProbe(): CloudFileProbe | undefined;

package/dist/filesystem/shared-exclusive-queue.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Orders async tasks like a read/write lock, in arrival order.
+ *
+ * `shared` tasks (searches) overlap each other. An `exclusive` task (index,
+ * stop) waits for every earlier task and holds back every later one, so a
+ * reindex never runs under a search and a search never reads a half-built
+ * index. A rejected task rejects its own caller only; the queue keeps going.
+ */
+export declare class SharedExclusiveQueue {
+    /** Shared tasks queued since the last exclusive one; the next exclusive task waits for all of them. */
+    private sharedSinceExclusive;
+    /** Settles when the last exclusive task has; every shared task waits for it. */
+    private lastExclusive;
+    shared<T>(task: () => Promise<T>): Promise<T>;
+    exclusive<T>(task: () => Promise<T>): Promise<T>;
+}

package/dist/fsearch/client.d.ts

@@ -144,7 +144,7 @@ export declare class FSearchClient {
     private readonly dbPath;
     private readonly pidPath;
     private binary;
-    private operation;
+    private readonly queue;
     constructor(options: FSearchClientOptions);
     isSupported(): boolean;
     /**
@@ -174,7 +174,8 @@ export declare class FSearchClient {
     private socketPath;
     private readWatchPid;
     private unsupportedFailure;
+    /** The probe is shared: searches run concurrently, and the first few must not each spawn `--version`. */
     private resolveBinary;
-    private serialize;
+    private probeBinary;
     private failure;
 }

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 export { AutoRAGAgent, type AutoRAGAgentOptions, type AutoRAGRefreshComponentStatus, type AutoRAGRefreshOptions, type AutoRAGRefreshProgress, type AutoRAGRefreshResult, type AutoRAGRefreshStatus, type AutoRAGWatchRefreshHandle, type AutoRAGWatchRefreshOptions, type AutoRefreshOptions, type RefreshMethod, } from "./agent/agent.js";
-export { type AutoRAGResultsDetails, reportSchema, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentsStreamEvent, type SearchDocumentWarning, } from "./agent/index.js";
+export { type AutoRAGResultsDetails, DECISION_MODEL_UNAVAILABLE_CODE, type DecisionModelFailureReason, DecisionModelUnavailableError, reportSchema, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentsStreamEvent, type SearchDocumentWarning, } from "./agent/index.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./agent/system-prompt.js";
 export * from "./cloud/index.js";
 export * from "./convert/index.js";