@autorag/librarian 2.13.0 → 2.13.2

package/dist/datasource/connector-skill.d.ts

@@ -1,7 +1,6 @@
 /**
  * Shared {@link DatasourceSkill} implementation for connector-backed
- * datasources (Slack, Discord, Notion, GitHub, Google Drive,
- * local mail exports, RSS/news).
+ * datasources (GitHub, cloud drives, RSS/news, Spotlight).
  *
  * Composition: a trusted {@link DatasourceConnector} fetches documents, a
  * {@link DatasourceChunkStore} persists and lexically indexes them, and this

package/dist/datasource/connector.d.ts

@@ -1,7 +1,6 @@
 /**
- * Connector contract shared by the built-in datasource skills (Slack, Discord,
- * Notion, GitHub, Google Drive, local mail exports,
- * RSS/news).
+ * Connector contract shared by the built-in datasource skills (GitHub,
+ * cloud drives, RSS/news, Spotlight).
  *
  * A *connector* is the trusted, server-configured bridge to one external
  * system. It fetches documents for indexing and NEVER throws — every failure

package/dist/datasource/index.d.ts

@@ -14,13 +14,12 @@ export { HISTER_SKILL_TYPE, HisterClient, type HisterClientOptions, HisterSkill,
 export { LarkClient, type LarkClientOptions, LarkSkill, type LarkSkillOptions } from "./skills/lark/index.js";
 export type { LazykatokFailureReason, LazykatokHit, LazykatokOptions, LazykatokSearchMode, LazykatokSearchOptions, LazykatokSearchResult, LazykatokSkillClient, LazykatokSkillOptions, } from "./skills/lazykatok/index.js";
 export { LazykatokBm25Method, LazykatokClient, LazykatokHybridMethod, LazykatokSkill, } from "./skills/lazykatok/index.js";
-export { MailExportConnector, type MailExportConnectorOptions, MailExportSkill, type MailExportSkillOptions, } from "./skills/mail-export/index.js";
 export { MAILCRAWL_MODES, MailcrawlClient, type MailcrawlFailure, type MailcrawlFailureReason, type MailcrawlIndexInfo, MailcrawlMethod, type MailcrawlOptions, type MailcrawlSearchHit, type MailcrawlSearchMode, type MailcrawlSearchResult, MailcrawlSkill, type MailcrawlSyncInfo, type MailcrawlSyncResult, } from "./skills/mailcrawl/index.js";
 export { NotcrawlClient, type NotcrawlOptions, NotionSkill, type NotionSkillOptions, } from "./skills/notion/index.js";
 export { ObsidianBm25Method, type ObsidianMethodOptions, ObsidianSemanticMethod, ObsidianSkill, type ObsidianSkillOptions, } from "./skills/obsidian/index.js";
 export { RssConnector, type RssConnectorOptions, type RssFeedConfig, RssSkill, type RssSkillOptions, } from "./skills/rss/index.js";
 export { SlackSkill, type SlackSkillOptions, SlacrawlClient, type SlacrawlOptions, } from "./skills/slack/index.js";
-export { SpotlightConnector, type SpotlightConnectorOptions, SpotlightSkill, type SpotlightSkillOptions, } from "./skills/spotlight/index.js";
+export { SpotlightConnector, type SpotlightConnectorFailure, type SpotlightConnectorOptions, type SpotlightConnectorResult, SpotlightSkill, type SpotlightSkillOptions, } from "./skills/spotlight/index.js";
 export { TelecrawlClient, type TelecrawlOptions, TelecrawlSkill, type TelecrawlSkillOptions, } from "./skills/telecrawl/index.js";
 export { WacrawlClient, type WacrawlOptions, WacrawlSkill, type WacrawlSkillOptions } from "./skills/wacrawl/index.js";
 export type { DatasourceDiagnostic, DatasourceDiagnosticCode, DatasourceIndexFail, DatasourceIndexOk, DatasourceIndexOptions, DatasourceIndexResult, DatasourceInstance, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription, } from "./types.js";

package/dist/datasource/sidecar/descriptors.d.ts

@@ -1,7 +1,7 @@
 /**
  * Declarative description of a crawler's read-only SQLite schema. The sidecar
  * engine (index.ts) is crawler-agnostic: everything that differs between
- * slacrawl, telecrawl, wacrawl and notcrawl is expressed here as SQL fragments.
+ * slacrawl, telecrawl, wacrawl and discrawl is expressed here as SQL fragments.
  *
  * Every SQL fragment is a trusted constant authored in this file. User input
  * (the search query, workspace id) is only ever bound as a parameter.
@@ -44,6 +44,16 @@ export interface CrawlerDescriptor {
     readonly fts: {
         readonly table: string;
         readonly joinOn: string;
+        /**
+         * Optional tokenizer-specific FTS tables the crawler maintains alongside the base index
+         * (discrawl's `message_fts_<lang>`). Each table that exists in the attached archive is
+         * searched and reciprocal-rank fused with the base table, so rows only the language
+         * tokenizer can match still reach the lexical candidate list. Absent tables are ignored.
+         */
+        readonly extraTables?: readonly {
+            readonly table: string;
+            readonly joinOn: string;
+        }[];
     };
     /** Extra result columns mapped into hit metadata: a SQL expression and the metadata key it fills. */
     readonly metadata: readonly {
@@ -59,10 +69,13 @@ export interface CrawlerDescriptor {
     /** Path-opaque label for the hierarchy: ["workspaces", ws, "channels", name] etc. */
     readonly hierarchy: (metadata: Readonly<Record<string, unknown>>) => readonly string[];
 }
+/** discrawl's FTS row refers to the stable Discord message id, independent of SQLite rowid. */
+export declare const discordDescriptor: CrawlerDescriptor;
 export declare const CRAWLER_DESCRIPTORS: {
     readonly slack: CrawlerDescriptor;
     readonly telegram: CrawlerDescriptor;
     readonly whatsapp: CrawlerDescriptor;
+    readonly discord: CrawlerDescriptor;
     readonly kakao: CrawlerDescriptor;
 };
 export type SidecarCrawlerId = keyof typeof CRAWLER_DESCRIPTORS;

package/dist/datasource/sidecar/index.d.ts

@@ -66,6 +66,8 @@ export declare class CrawlerSidecar {
     private opened;
     /** Identity of the attached crawler file; a crawler that replaces or deletes its archive forces a re-attach. */
     private attached;
+    /** Base FTS table plus any language-specific tables present in the attached archive. */
+    private ftsTables;
     /**
      * The last query's embedding and ranking. The semantic and hybrid methods of one search ask for the same
      * query, so they share one embedding call and one vector scan. Short-lived, and dropped on any write.
@@ -90,6 +92,8 @@ export declare class CrawlerSidecar {
     private whereParams;
     /** BM25-ranked lexical search. Natural-language queries are OR-ed term by term. */
     searchLexical(query: string, topK: number): SidecarHit[];
+    /** Descriptor-declared FTS tables that the attached archive actually defines. */
+    private presentExtraTables;
     /** Cosine search over embedded messages. Empty when there is no embedder or the query embedding fails. */
     searchSemantic(query: string, topK: number): Promise<SidecarHit[]>;
     /** Best `pool` stored vectors for `query`, reusing the previous ranking when it already covers `pool`. */

package/dist/datasource/sidecar/methods.d.ts

@@ -8,7 +8,8 @@ import type { CrawlerDescriptor } from "./descriptors.js";
 import type { CrawlerSidecar } from "./index.js";
 export type SidecarMode = "bm25" | "hybrid";
 export interface SidecarMethodOptions {
-    readonly sidecar: CrawlerSidecar;
+    /** Resolve lazily when the native CLI owns the archive location. */
+    readonly sidecar: CrawlerSidecar | ((signal?: AbortSignal) => Promise<CrawlerSidecar>);
     readonly descriptor: CrawlerDescriptor;
     readonly mode: SidecarMode;
     readonly datasourceId: string;
@@ -31,4 +32,6 @@ export declare class SidecarRetrievalMethod implements RetrievalMethod {
  * answers next and the crawler's CLI search last, so the datasource still searches when the embedder or
  * the sidecar index is down. A leader that answers with no hits is final.
  */
-export declare function sidecarFirstMethod(options: Omit<SidecarMethodOptions, "mode">, cli: RetrievalMethod): RetrievalMethod;
+export declare function sidecarFirstMethod(options: Omit<SidecarMethodOptions, "mode" | "sidecar"> & {
+    readonly sidecar: CrawlerSidecar;
+}, cli: RetrievalMethod): RetrievalMethod;

package/dist/datasource/skills/discrawl/attachments.d.ts

@@ -0,0 +1,139 @@
+import type { DiscrawlAttachmentFetchResult, DiscrawlAttachmentListResult, DiscrawlStatusResult } from "./types.js";
+/** Per-connection status file kept at the top of the attachments folder. */
+export declare const DISCORD_ATTACHMENT_STATUS_FILENAME = "download-status.json";
+/**
+ * Where a Discord connection's attachments are saved: `<AUTORAG_HOME>/attachments/discord/<connection>`.
+ * That is `~/.autorag/attachments/discord/<connection>` for every user on macOS, Linux, and Windows, unless
+ * `AUTORAG_HOME` moves the whole AutoRAG home.
+ */
+export declare function discordAttachmentsRoot(connection: string, env?: NodeJS.ProcessEnv): string;
+/**
+ * - `downloaded`: the saved file is still byte-identical to what Discord served.
+ * - `not-downloaded`: never saved.
+ * - `changed`: the saved file was edited. It stays as the user's own file; the original is fetched again.
+ * - `missing`: the saved file was deleted, moved, or renamed; the original is fetched again.
+ * - `failed`: the last download attempt failed (`error` says why).
+ * - `skipped`: discrawl refused the file (for example `too_large`) and it is not retried.
+ */
+export type DiscordAttachmentState = "downloaded" | "not-downloaded" | "changed" | "missing" | "failed" | "skipped";
+export interface DiscordAttachmentRecord {
+    readonly attachmentId: string;
+    readonly messageId: string;
+    readonly guildId: string;
+    readonly guildName?: string;
+    readonly channelId: string;
+    readonly channelName?: string;
+    readonly authorName?: string;
+    readonly createdAt?: string;
+    readonly filename: string;
+    readonly contentType?: string;
+    readonly size: number;
+    readonly state: DiscordAttachmentState;
+    /** True when the file is not saved intact and Discord's signed link still works. */
+    readonly downloadable: boolean;
+    /** When the signed link stops working (Discord signs attachment links for 24 hours). */
+    readonly linkExpiresAt?: string;
+    /** Saved file, relative to the attachments folder, with `/` separators. */
+    readonly path?: string;
+    readonly sha256?: string;
+    readonly bytes?: number;
+    readonly mtimeMs?: number;
+    readonly downloadedAt?: string;
+    readonly error?: string;
+}
+export interface DiscordAttachmentSummary {
+    readonly total: number;
+    readonly downloaded: number;
+    readonly notDownloaded: number;
+    readonly changed: number;
+    readonly missing: number;
+    readonly failed: number;
+    readonly skipped: number;
+    /** Not saved intact and still downloadable now. */
+    readonly downloadable: number;
+    /** Not saved intact and the link already expired: these can no longer be downloaded. */
+    readonly expired: number;
+}
+/** What one sync downloaded. */
+export interface DiscordAttachmentRun {
+    readonly attempted: number;
+    readonly fetched: number;
+    readonly failed: number;
+    readonly skipped: number;
+    readonly bytes: number;
+}
+export interface DiscordAttachmentSyncOk {
+    readonly ok: true;
+    readonly root: string;
+    readonly statusPath: string;
+    readonly summary: DiscordAttachmentSummary;
+    readonly run: DiscordAttachmentRun;
+    readonly records: readonly DiscordAttachmentRecord[];
+}
+export interface DiscordAttachmentSyncFail {
+    readonly ok: false;
+    readonly reason: "busy" | "discrawl-failed";
+    readonly message: string;
+}
+export type DiscordAttachmentSyncResult = DiscordAttachmentSyncOk | DiscordAttachmentSyncFail;
+export interface DiscordAttachmentSyncOptions {
+    /** Download every attachment that is not saved intact and whose link still works. */
+    readonly download: boolean;
+    readonly signal?: AbortSignal;
+}
+/** The discrawl calls the store needs; `DiscrawlClient` implements them. */
+export interface DiscordAttachmentClient {
+    status(signal?: AbortSignal): Promise<DiscrawlStatusResult>;
+    listAttachments(configPath: string, guildId?: string, signal?: AbortSignal): Promise<DiscrawlAttachmentListResult>;
+    fetchAttachments(configPath: string, messageId: string, signal?: AbortSignal): Promise<DiscrawlAttachmentFetchResult>;
+}
+/** What the Discord skill calls after each import; `DiscordAttachmentStore` implements it. */
+export interface DiscordAttachmentSyncer {
+    readonly root: string;
+    sync(options: DiscordAttachmentSyncOptions): Promise<DiscordAttachmentSyncResult>;
+}
+export interface DiscordAttachmentStoreOptions {
+    readonly client: DiscordAttachmentClient;
+    /** Attachments folder; see {@link discordAttachmentsRoot}. */
+    readonly root: string;
+    readonly guildId?: string;
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
+    readonly now?: () => number;
+}
+/**
+ * Saves Discord attachments into one folder and keeps `download-status.json` beside them.
+ *
+ * discrawl does the downloading; AutoRAG never talks to Discord itself. AutoRAG hands discrawl a private
+ * config whose `cache_dir` is `.discrawl/` inside the folder, asks it to fetch one message's attachments at a
+ * time, and then moves each file to `<server>/<channel>/<original name>`. A saved file is identified by its
+ * SHA-256: when the file at the recorded path is gone or its bytes differ, it no longer counts as the
+ * attachment. An edited file is left in place as the user's own and the original is saved again beside it
+ * under a free name, as long as Discord's signed link still works. Nothing in the folder is ever overwritten
+ * or deleted.
+ */
+export declare class DiscordAttachmentStore implements DiscordAttachmentSyncer {
+    readonly root: string;
+    private readonly client;
+    private readonly guildId;
+    private readonly channelIds;
+    private readonly channelNames;
+    private readonly now;
+    constructor(options: DiscordAttachmentStoreOptions);
+    get statusPath(): string;
+    sync(options: DiscordAttachmentSyncOptions): Promise<DiscordAttachmentSyncResult>;
+    private syncLocked;
+    /** discrawl keeps reading the operator's archive but stores downloads in this folder's private cache. */
+    private writeDiscrawlConfig;
+    private channelAllowed;
+    private isDownloadable;
+    /** Re-checks a saved file. Unchanged size and mtime is trusted; anything else is hashed. */
+    private verify;
+    private download;
+    /** Puts a downloaded file at `<server>/<channel>/<name>`, never over an existing file. */
+    private place;
+}
+/** Discord signs attachment links with `ex=<hex unix seconds>`. */
+export declare function signedLinkExpiry(url: string | undefined): string | undefined;
+/** The original Discord file name, made safe for macOS, Linux, and Windows. */
+export declare function safeFilename(name: string, attachmentId: string): string;

package/dist/datasource/skills/discrawl/client.d.ts

@@ -1,4 +1,4 @@
-import type { DiscrawlDoctorResult, DiscrawlEmbeddingConfigResult, DiscrawlEmbeddingRuntime, DiscrawlEmbedResult, DiscrawlMetadataResult, DiscrawlOptions, DiscrawlSearchMode, DiscrawlSearchOptions, DiscrawlSearchResult, DiscrawlStatusResult, DiscrawlSyncResult } from "./types.js";
+import type { DiscrawlAttachmentFetchResult, DiscrawlAttachmentListResult, DiscrawlDoctorResult, DiscrawlEmbeddingConfigResult, DiscrawlEmbeddingRuntime, DiscrawlEmbedResult, DiscrawlMetadataResult, DiscrawlOptions, DiscrawlSearchMode, DiscrawlSearchOptions, DiscrawlSearchResult, DiscrawlStatusResult, DiscrawlSyncResult } from "./types.js";
 /**
  * Thin external `discrawl` CLI wrapper. Every method spawns the `discrawl`
  * binary as a child process, parses JSON from stdout, and returns a
@@ -27,8 +27,18 @@ export declare class DiscrawlClient {
     sync(signal?: AbortSignal): Promise<DiscrawlSyncResult>;
     embed(limit?: number, rebuild?: boolean, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
     search(mode: DiscrawlSearchMode, query: string, options?: DiscrawlSearchOptions): Promise<DiscrawlSearchResult>;
+    /**
+     * Lists every archived attachment with discrawl's fetch bookkeeping. `configPath` is the
+     * attachments-only config that points discrawl's `cache_dir` at AutoRAG's attachments folder; it
+     * replaces the operator/managed config for this call only.
+     */
+    listAttachments(configPath: string, guildId?: string, signal?: AbortSignal): Promise<DiscrawlAttachmentListResult>;
+    /** Downloads the attachments of one message into the config's `cache_dir` while their signed links live. */
+    fetchAttachments(configPath: string, messageId: string, signal?: AbortSignal): Promise<DiscrawlAttachmentFetchResult>;
     private runJson;
     private run;
+    /** Runs with an explicit `--config` and no embedding runtime: attachment commands never embed. */
+    private runWithConfig;
 }
 export declare function discrawlWorkspace(options: DiscrawlOptions): string | undefined;
 export declare function nativeGatewayBaseUrl(baseUrl: string): string;

package/dist/datasource/skills/discrawl/index.d.ts

@@ -1,5 +1,6 @@
+export { DISCORD_ATTACHMENT_STATUS_FILENAME, type DiscordAttachmentClient, type DiscordAttachmentRecord, type DiscordAttachmentRun, type DiscordAttachmentState, DiscordAttachmentStore, type DiscordAttachmentStoreOptions, type DiscordAttachmentSummary, type DiscordAttachmentSyncer, type DiscordAttachmentSyncOptions, type DiscordAttachmentSyncResult, discordAttachmentsRoot, } from "./attachments.js";
 export { DiscrawlClient, discrawlWorkspace } from "./client.js";
 export { DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlMethodOptions, type DiscrawlSearchClient, DiscrawlSemanticMethod, } from "./methods.js";
 export { DISCRAWL_SOURCE_KIND, discrawlConfigPath, discrawlDatabasePath, discrawlDatasourceRoot, discrawlSourcePath, parseDiscrawlSourcePath, } from "./paths.js";
 export { DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions } from "./skill.js";
-export { DEFAULT_DISCRAWL_BINARY, DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_EMBEDDING_PROVIDER, DEFAULT_DISCRAWL_MAX_BUFFER_BYTES, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_TIMEOUT_MS, type DiscrawlDoctorInfo, type DiscrawlDoctorResult, type DiscrawlEmbeddingConfigResult, type DiscrawlEmbeddingRuntime, type DiscrawlEmbedInfo, type DiscrawlEmbedResult, type DiscrawlFailure, type DiscrawlFailureReason, type DiscrawlMessage, type DiscrawlMetadataInfo, type DiscrawlMetadataResult, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, type DiscrawlStatusInfo, type DiscrawlStatusResult, type DiscrawlSyncInfo, type DiscrawlSyncResult, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./types.js";
+export { DEFAULT_DISCRAWL_BINARY, DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_EMBEDDING_PROVIDER, DEFAULT_DISCRAWL_MAX_BUFFER_BYTES, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_TIMEOUT_MS, type DiscrawlAttachmentFetchResult, type DiscrawlAttachmentFetchStats, type DiscrawlAttachmentListResult, type DiscrawlAttachmentRow, type DiscrawlDoctorInfo, type DiscrawlDoctorResult, type DiscrawlEmbeddingConfigResult, type DiscrawlEmbeddingRuntime, type DiscrawlEmbedInfo, type DiscrawlEmbedResult, type DiscrawlFailure, type DiscrawlFailureReason, type DiscrawlMessage, type DiscrawlMetadataInfo, type DiscrawlMetadataResult, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, type DiscrawlStatusInfo, type DiscrawlStatusResult, type DiscrawlSyncInfo, type DiscrawlSyncResult, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./types.js";

package/dist/datasource/skills/discrawl/ranked-method.d.ts

@@ -0,0 +1,19 @@
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
+import type { DiscrawlSearchMode } from "./types.js";
+export interface DiscrawlRankedMethodOptions {
+    readonly mode: DiscrawlSearchMode;
+    readonly lexical: RetrievalMethod;
+    readonly nativeFts: RetrievalMethod;
+    readonly semantic: RetrievalMethod;
+    readonly workspaceId?: string;
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
+}
+/** One Discord ranking boundary: native FTS supplements BM25 rather than voting for the same messages again. */
+export declare class DiscrawlRankedMethod implements RetrievalMethod {
+    private readonly options;
+    constructor(options: DiscrawlRankedMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+    private matchesScope;
+}

package/dist/datasource/skills/discrawl/sidecar.d.ts

@@ -0,0 +1,21 @@
+import { type ResolveSidecarInput } from "../../sidecar/config.js";
+import { CrawlerSidecar, type CrawlerSidecarOptions } from "../../sidecar/index.js";
+import type { DiscrawlStatusResult } from "./types.js";
+export interface DiscrawlStatusClient {
+    status?(signal?: AbortSignal): Promise<DiscrawlStatusResult>;
+}
+export interface DiscrawlSidecarOptions extends Omit<CrawlerSidecarOptions, "crawlerDbPath" | "embedder"> {
+    /** Explicit archive path; omitted to discover it through the native client's status command. */
+    readonly crawlerDbPath?: string;
+}
+/** Discord reuses its native embeddings; its sidecar provides lexical ranking only. */
+export declare function resolveDiscrawlSidecarOptions(input: Omit<ResolveSidecarInput, "descriptor" | "defaultCrawlerDb">): DiscrawlSidecarOptions | undefined;
+/** Resolves the native archive on first search, without requiring indexing or a second vector store. */
+export declare class DiscrawlSidecar {
+    private readonly options;
+    private readonly client;
+    private sidecar;
+    constructor(options: DiscrawlSidecarOptions, client: DiscrawlStatusClient);
+    get(signal?: AbortSignal): Promise<CrawlerSidecar>;
+    close(): void;
+}

package/dist/datasource/skills/discrawl/skill.d.ts

@@ -1,8 +1,10 @@
 import type { RetrievalMethod } from "../../../retrieval/types.js";
 import type { DatasourceIndexOptions, DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import type { DiscordAttachmentSyncer } from "./attachments.js";
 import { type DiscrawlSearchClient } from "./methods.js";
+import { type DiscrawlSidecarOptions, type DiscrawlStatusClient } from "./sidecar.js";
 import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlSearchMode, DiscrawlSyncResult } from "./types.js";
-export interface DiscrawlSkillClient extends DiscrawlSearchClient {
+export interface DiscrawlSkillClient extends DiscrawlSearchClient, DiscrawlStatusClient {
     doctor(signal?: AbortSignal): Promise<DiscrawlDoctorResult>;
     sync(signal?: AbortSignal): Promise<DiscrawlSyncResult>;
     embed(limit?: number, rebuild?: boolean, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
@@ -22,12 +24,19 @@ export interface DiscrawlSkillOptions {
     readonly embedLimit?: number;
     readonly channelIds?: readonly string[];
     readonly channelNames?: readonly string[];
+    /** Optional lexical-only sidecar. Its archive path is discovered through native status when omitted. */
+    readonly sidecar?: DiscrawlSidecarOptions;
+    /**
+     * Saves attachments right after each successful import, while Discord's 24-hour signed links still work.
+     * Built by default (`connector.downloadAttachments: false` disables it); absent means attachments are not downloaded.
+     */
+    readonly attachments?: DiscordAttachmentSyncer;
 }
 /**
  * Discord datasource skill backed by the external `discrawl` CLI.
  *
  * Mirrors the lazykatok model: the CLI owns the archive, the FTS5 index, and the
- * vector index; AutoRAG only spawns it and maps results. AutoRAG never calls
+ * vector index; an optional read-only sidecar adds BM25 lexical ranking. AutoRAG never calls
  * the Discord API itself and never reads the Discord Desktop cache directly.
  *
  * `index()` runs the CLI's two-phase pipeline — `sync` (incremental, cursor
@@ -43,6 +52,9 @@ export declare class DiscrawlSkill implements DatasourceSkill {
     private readonly embeddingModel;
     private readonly defaultMode;
     private readonly embedLimit;
+    private readonly sidecar;
+    private readonly retrievalScope;
+    private readonly attachments;
     private lastIndexedAt;
     constructor(options: DiscrawlSkillOptions);
     describe(): DatasourceSkillDescriptor;
@@ -54,8 +66,12 @@ export declare class DiscrawlSkill implements DatasourceSkill {
      * beside it; an operator who pins `defaultMode` to `fts` or `semantic` gets exactly that mode.
      */
     retrievalMethods(): readonly RetrievalMethod[];
+    /** Release the optional sidecar without changing the native archive. */
+    close(): void;
     describeSources(): readonly SourceDescription[];
     skillManifest(): DatasourceSkillManifest;
+    /** Best effort: a failed or expired download never fails the index, and messages stay searchable. */
+    private downloadAttachments;
     private runEmbed;
     /**
      * English-only embedders map non-English text into one narrow similarity

package/dist/datasource/skills/discrawl/types.d.ts

@@ -101,6 +101,11 @@ export interface DiscrawlOptions {
      * issue #1413.
      */
     readonly defaultMode?: DiscrawlSearchMode;
+    /**
+     * Spawn timeout for `discrawl attachments fetch`. Defaults to the shared connector sync budget, because a
+     * burst of attachments downloads far longer than one wiretap import.
+     */
+    readonly attachmentTimeoutMs?: number;
     /** Environment overrides merged on top of `process.env` for the child. */
     readonly env?: Readonly<Record<string, string | undefined>>;
 }
@@ -203,6 +208,42 @@ export type DiscrawlMetadataResult = DiscrawlOk<DiscrawlMetadataInfo> | Discrawl
 export type DiscrawlSyncResult = DiscrawlOk<DiscrawlSyncInfo> | DiscrawlFailure;
 export type DiscrawlEmbedResult = DiscrawlOk<DiscrawlEmbedInfo> | DiscrawlFailure;
 export type DiscrawlSearchResult = DiscrawlSearchOk | DiscrawlFailure;
+/** One row of `discrawl --json attachments`: metadata plus discrawl's own fetch bookkeeping. */
+export interface DiscrawlAttachmentRow {
+    readonly attachmentId: string;
+    readonly messageId: string;
+    readonly guildId: string;
+    readonly guildName?: string;
+    readonly channelId: string;
+    readonly channelName?: string;
+    readonly authorName?: string;
+    readonly filename: string;
+    readonly contentType?: string;
+    readonly size: number;
+    readonly url?: string;
+    readonly proxyUrl?: string;
+    /** Path relative to `<cache_dir>/media`, set once discrawl stored the file. */
+    readonly mediaPath?: string;
+    readonly contentSha256?: string;
+    readonly contentSize?: number;
+    /** `fetched`, `failed`, a skip reason such as `too_large`, or empty when never attempted. */
+    readonly fetchStatus?: string;
+    readonly fetchError?: string;
+    /** When discrawl last attempted the download (RFC 3339). */
+    readonly fetchedAt?: string;
+    readonly createdAt?: string;
+}
+/** Result of `discrawl --json attachments fetch`. */
+export interface DiscrawlAttachmentFetchStats {
+    readonly attachments: number;
+    readonly fetched: number;
+    readonly reused: number;
+    readonly skipped: number;
+    readonly failed: number;
+    readonly bytes: number;
+}
+export type DiscrawlAttachmentListResult = DiscrawlOk<readonly DiscrawlAttachmentRow[]> | DiscrawlFailure;
+export type DiscrawlAttachmentFetchResult = DiscrawlOk<DiscrawlAttachmentFetchStats> | DiscrawlFailure;
 /**
  * Search-specific options. Reuses the shared {@link RetrievalOptions} so the
  * retrieval methods can pass their options straight through.

package/dist/datasource/skills/factory.d.ts

@@ -10,7 +10,7 @@
 import type { Embedder } from "../../embedding/embedder.js";
 import type { EmbeddingEndpointSource } from "../embedding-env.js";
 import type { DatasourceSkill } from "../types.js";
-import { type DiscrawlEmbeddingRuntime } from "./discrawl/index.js";
+import { DiscordAttachmentStore, type DiscrawlEmbeddingRuntime } from "./discrawl/index.js";
 import { type ObsidianMinSyncOptions } from "./obsidian/index.js";
 /** One configured datasource entry (the trusted `datasources.<name>` value). */
 export interface DatasourceSkillConfig {
@@ -50,6 +50,17 @@ export interface DatasourceEmbeddingOptions {
     /** MinSync binary, embedder and chunking shared with the local-file index; vault indexes run on it too. */
     readonly minSync?: ObsidianMinSyncOptions;
 }
+export interface DiscordAttachmentConnection {
+    readonly name: string;
+    /** Whether refreshes download automatically (`connector.downloadAttachments !== false`, default on). */
+    readonly autoDownload: boolean;
+    readonly store: DiscordAttachmentStore;
+}
+/**
+ * Every enabled Discord connection's attachments folder, built exactly as a refresh builds it, so
+ * `autorag attachments` and the refresh always agree on where files live.
+ */
+export declare function discordAttachmentConnections(config: DatasourcesConfig | undefined, workspaceRoot?: string): readonly DiscordAttachmentConnection[];
 /** Whether the skill's backend exists on `platform`. Names with no platform restriction are always supported. */
 export declare function datasourceSkillSupported(name: string, platform?: NodeJS.Platform): boolean;
 /** Skill names this factory can build on `platform`. */

package/dist/datasource/skills/hister/client.d.ts

@@ -69,11 +69,11 @@ export declare class HisterClient {
  * Wrap a free-text question as a Hister query confined to web pages.
  *
  * Hister ANDs bare terms, so a natural-language question would match almost nothing. Plain words become one
- * alternation group `(a|b|c)`. That pipe is Hister's only OR: the word `OR` is not an operator there, it is searched
- * as the literal English word, which matches most English pages. Hister's own syntax (quoted phrases,
- * `field:value`, wildcards, negation) passes through untouched, and a query using any of it is sent with its plain
- * words ANDed, exactly as written. A `sort:` directive is hoisted out of the group because Hister only honours it at
- * the top level.
+ * alternation group `(a|b|c)`, while uppercase `AND`/`OR`/`NOT` tokens in natural-language questions are stripped by
+ * {@link normalizeTerm} so Hister 0.21+'s boolean operator parser never rejects casual English phrasing. Hister's own
+ * syntax (quoted phrases, `field:value`, wildcards, negation) passes through untouched, and a query using any of it is
+ * sent with its plain words ANDed, exactly as written. A `sort:` directive is hoisted out of the group because Hister
+ * only honours it at the top level.
  */
 export declare function buildWebQuery(question: string, domain?: string): string | undefined;
 /** Hister snippets carry `<mark>` highlights and HTML entities; the model reads plain text. */

package/dist/datasource/skills/hister/setup.d.ts

@@ -41,6 +41,11 @@ export interface HisterServiceDeps {
         readonly semanticEnabled?: boolean;
         readonly reason?: string;
     }>;
+    /** Reload a running Hister server's config (`hister reload`, v0.21.0+). Preferred over restarting the service. */
+    readonly reload?: (baseUrl: string) => Promise<{
+        readonly ok: boolean;
+        readonly reason?: string;
+    }>;
     /** Restart the Hister service. Omitted means the operator must do it. */
     readonly restart?: () => Promise<{
         readonly ok: boolean;

package/dist/datasource/skills/mailcrawl/types.d.ts

@@ -13,7 +13,21 @@ export interface MailcrawlOptions {
     readonly backend?: string;
     readonly source?: string;
     readonly fixture?: string;
+    /** `source: "files"` (mailcrawl 0.3.0+): `.eml` files, `.mbox`/`.mbx` archives, Apple Mail bundles, or directories. */
+    readonly paths?: readonly string[];
     readonly himalayaConfig?: string;
+    /** Envelopes fetched per IMAP window (`--page-size`). */
+    readonly pageSize?: number;
+    /** Simultaneous message reads during Himalaya sync (`--concurrency`, mailcrawl 0.3.0+). */
+    readonly concurrency?: number;
+    /** Per-message read deadline in milliseconds (`--read-timeout`, mailcrawl 0.3.0+). */
+    readonly readTimeoutMs?: number;
+    /** Categories re-included over mailcrawl's default exclusions (`--include-category`, mailcrawl 0.3.0+). */
+    readonly includeCategories?: readonly string[];
+    /** Categories excluded from sync (`--exclude-category`, mailcrawl 0.3.0+). */
+    readonly excludeCategories?: readonly string[];
+    /** Force vector store rebuild on `index` (`--rebuild`, mailcrawl 0.3.0+). */
+    readonly rebuildIndex?: boolean;
     /**
      * Applies to every operation. `sync`/`index` additionally honor
      * `indexTimeoutMs`, because a 0.2.0 native index run is not interactive.
@@ -45,7 +59,11 @@ export interface MailcrawlSyncInfo {
     readonly deleted?: number;
     readonly unchanged?: number;
     readonly chunksAdded?: number;
+    readonly excluded?: number;
+    readonly files?: number;
+    readonly filesRead?: number;
     readonly archiveRevision?: string;
+    readonly account?: string;
     readonly messages: number;
 }
 export interface MailcrawlIndexInfo {

package/dist/datasource/skills/spotlight/connector.d.ts

@@ -1,23 +1,10 @@
-/**
- * macOS Spotlight connector (issue #1350).
- *
- * Runs the built-in `mdfind` CLI for trusted, server-configured Spotlight
- * queries and hydrates text content for chunking. No external installs —
- * Spotlight ships with macOS. **macOS only**: on any other platform the
- * connector reports `unavailable` before spawning anything.
- *
- * Full Disk Access: Spotlight returns hits for protected locations (Mail,
- * Messages, Safari data, parts of ~/Library) but reading those files fails
- * with EPERM/EACCES unless the process running AutoRAG (the terminal app or
- * host binary) has Full Disk Access under System Settings -> Privacy &
- * Security. When every content read is denied this maps to the `permission`
- * failure reason with opaque FDA guidance; partial denials surface as
- * count-only warnings. Full absolute paths are exposed in document metadata
- * (`path`) so results stay traceable back to the file on disk — privacy is
- * the operator's responsibility (run AutoRAG with a local LLM if paths must
- * not leave the machine).
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
+import type { ConnectorFetchFail, ConnectorFetchOk } from "../../connector.js";
+export interface SpotlightConnectorFailure extends ConnectorFetchFail {
+    readonly stderr?: string;
+    readonly stdout?: string;
+    readonly code?: number | null;
+}
+export type SpotlightConnectorResult = ConnectorFetchOk | SpotlightConnectorFailure;
 export interface ProcessRunResult {
     readonly ok: boolean;
     readonly stdout: string;
@@ -30,13 +17,13 @@ export interface ProcessRunResult {
 export type ProcessRunner = (command: string, args: readonly string[], timeoutMs: number) => Promise<ProcessRunResult>;
 export type FileReader = (path: string) => Promise<string>;
 export interface SpotlightConnectorOptions {
-    /** Trusted Spotlight query strings passed to `mdfind`. Required. */
+    /** Trusted Spotlight filters AND-ed with each user query. Required. */
     readonly queries?: readonly string[];
     /** Trusted directory restricting every query (`mdfind -onlyin`). */
     readonly onlyIn?: string;
     /** Hits kept per query after dedupe. Default 100. */
     readonly maxResultsPerQuery?: number;
-    /** Total documents hydrated per fetch. Default 300. */
+    /** Maximum documents hydrated by one live search. Default 300. */
     readonly maxDocuments?: number;
     /** Characters kept per document body. Default 100000. */
     readonly maxContentChars?: number;
@@ -51,11 +38,16 @@ export interface SpotlightConnectorOptions {
     /** Injectable file reader; defaults to utf8 `fs.readFile`. */
     readonly readFile?: FileReader;
 }
-export declare class SpotlightConnector implements DatasourceConnector {
+/** Executes live Spotlight queries. macOS owns indexing; no local snapshot is collected. */
+export declare class SpotlightConnector {
     private readonly options;
     private readonly run;
     private readonly readFile;
     private readonly platform;
     constructor(options?: SpotlightConnectorOptions);
-    fetch(): Promise<ConnectorFetchResult>;
+    /** Refresh-time health check only: never runs `mdfind` or hydrates files. */
+    probe(): Promise<SpotlightConnectorResult>;
+    /** Search the current macOS Spotlight index and hydrate only the requested hits. */
+    search(query: string, limit?: number, acceptDocument?: (docId: string) => boolean): Promise<SpotlightConnectorResult>;
+    private readinessFailure;
 }

package/dist/datasource/skills/spotlight/index.d.ts

@@ -1,2 +1,2 @@
-export { type FileReader, type ProcessRunner, type ProcessRunResult, SpotlightConnector, type SpotlightConnectorOptions, } from "./connector.js";
-export { SPOTLIGHT_SKILL_DEFINITION, SpotlightSkill, type SpotlightSkillOptions } from "./skill.js";
+export { type FileReader, type ProcessRunner, type ProcessRunResult, SpotlightConnector, type SpotlightConnectorFailure, type SpotlightConnectorOptions, type SpotlightConnectorResult, } from "./connector.js";
+export { SpotlightSkill, type SpotlightSkillOptions } from "./skill.js";