@autorag/librarian 2.0.0 → 2.2.0

package/dist/datasource/chunk-store.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Persistent chunk store shared by the connector-backed datasource skills.
+ *
+ * Documents fetched by a {@link DatasourceConnector} are split into bounded
+ * chunks, persisted as JSON under
+ * `<workspaceRoot>/.autorag/datasources/<skill>/<instance>/chunks.json`, and
+ * searched with a lightweight BM25-style lexical scorer. The store never
+ * throws from load/search; persistence failures degrade to in-memory-only
+ * operation.
+ */
+import type { ConnectorDocument } from "./connector.js";
+export interface StoredChunk {
+    readonly chunkId: string;
+    readonly docId: string;
+    /** Sanitized hierarchy segments under the instance root. */
+    readonly hierarchy: readonly string[];
+    readonly title?: string;
+    readonly content: string;
+    readonly publishedAt?: number;
+    readonly metadata: Readonly<Record<string, unknown>>;
+}
+export interface ScoredChunk {
+    readonly chunk: StoredChunk;
+    readonly score: number;
+}
+export interface DatasourceChunkStoreOptions {
+    readonly skillName: string;
+    readonly instanceId: string;
+    /** When set, chunks are persisted under `.autorag/datasources/…`. */
+    readonly workspaceRoot?: string;
+    /** Maximum characters per chunk. Default 2000. */
+    readonly maxChunkChars?: number;
+}
+export declare function datasourceChunkStorePath(workspaceRoot: string, skillName: string, instanceId: string): string;
+export declare class DatasourceChunkStore {
+    private readonly skillName;
+    private readonly instanceId;
+    private readonly storePath;
+    private readonly maxChunkChars;
+    private chunkList;
+    private loaded;
+    constructor(options: DatasourceChunkStoreOptions);
+    /**
+     * Replace the store contents with chunks derived from `documents`.
+     * Returns the resulting chunk count. Persistence failures are swallowed
+     * (in-memory contents stay authoritative for this process).
+     */
+    replaceDocuments(documents: readonly ConnectorDocument[]): number;
+    /** Load persisted chunks when present. Returns true when data was loaded. */
+    load(): boolean;
+    /** Ensure persisted chunks are loaded once for read paths. */
+    ensureLoaded(): void;
+    chunks(): readonly StoredChunk[];
+    /** Distinct hierarchy prefixes present in the store (for source listing). */
+    hierarchies(): readonly (readonly string[])[];
+    /**
+     * BM25-style lexical search over the stored chunks. Empty queries and
+     * empty stores return `[]`; this method never throws.
+     *
+     * Terms match document tokens exactly or as a prefix (min 2 chars), so
+     * agglutinative suffixes — Korean particles/endings (인증서 → 인증서를) and
+     * English inflections (index → indexing) — still rank. Prefix hits are
+     * slightly discounted against exact hits.
+     */
+    search(query: string, topK: number): readonly ScoredChunk[];
+    private persist;
+}

package/dist/datasource/connector-skill.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Shared {@link DatasourceSkill} implementation for connector-backed
+ * datasources (Slack, Discord, Notion, GitHub, Google Drive, Gmail/IMAP,
+ * local mail exports, RSS/news).
+ *
+ * Composition: a trusted {@link DatasourceConnector} fetches documents, a
+ * {@link DatasourceChunkStore} persists and lexically indexes them, and this
+ * class exposes the standard skill surface — descriptor, polling metadata,
+ * ok/fail indexing with traceable diagnostics, retrieval methods routed
+ * through the shared pipeline, source descriptions, and a
+ * progressive-disclosure agent-skill manifest.
+ *
+ * Invariants:
+ *  - Constructed only from trusted, server-supplied configuration.
+ *  - Sources are stable slash-hierarchical paths `/<skill>/<instance>/…`.
+ *  - `index()` never throws; failures surface as traceable diagnostics.
+ *  - Retrieval honors `scope` and `allowedScopes` narrowing; tool arguments
+ *    can never widen access (enforced upstream by DatasourceAccessContext).
+ *  - Results and metadata may carry real file paths or account identifiers;
+ *    privacy is the operator's responsibility (use a local LLM).
+ */
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
+import { DatasourceChunkStore } from "./chunk-store.js";
+import type { DatasourceConnector } from "./connector.js";
+import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+/** Static identity/behavior shared by all instances of one skill kind. */
+export interface ConnectorSkillDefinition {
+    /** Stable skill name and datasource id, e.g. `"slack"`. */
+    readonly skillName: string;
+    /** Skill kind, e.g. `"slack-workspace"`. */
+    readonly skillType: string;
+    /** One-line descriptor description. */
+    readonly description: string;
+    /** Capability tags such as `"chat"`, `"api"`, `"polling"`. */
+    readonly capabilities: readonly string[];
+    /** Default authorization tags when the server config supplies none. */
+    readonly defaultTags: readonly string[];
+    /** Content type reported on source descriptions, e.g. `"chat"`. */
+    readonly contentType: string;
+    /** Short "when to use" text for the system-prompt skills block. */
+    readonly manifestDescription: string;
+    /** Extra path-opaque manifest body lines (after the standard sections). */
+    readonly manifestNotes?: readonly string[];
+}
+export interface ConnectorSkillOptions {
+    readonly connector: DatasourceConnector;
+    readonly instanceId?: string;
+    readonly instances?: readonly string[];
+    /** Poll interval in ms. Default 15 minutes; `0` disables polling. */
+    readonly pollingIntervalMs?: number;
+    readonly tags?: readonly string[];
+    readonly lastIndexedAt?: number;
+    /** Workspace root for chunk persistence; omitted ⇒ in-memory only. */
+    readonly workspaceRoot?: string;
+    readonly maxChunkChars?: number;
+    /**
+     * Dedupe window in ms: when > 0, documents whose `docId` was already
+     * indexed within the window keep their previous timestamp identity and
+     * duplicates inside one fetch are dropped (RSS/news re-delivery).
+     */
+    readonly dedupeWindowMs?: number;
+}
+export declare class ConnectorDatasourceSkill implements DatasourceSkill {
+    protected readonly definition: ConnectorSkillDefinition;
+    protected readonly connector: DatasourceConnector;
+    protected readonly instanceId: string;
+    protected readonly instances: readonly string[];
+    protected readonly pollingIntervalMs: number;
+    protected readonly tags: readonly string[];
+    protected readonly store: DatasourceChunkStore;
+    protected readonly dedupeWindowMs: number;
+    private lastIndexedAt;
+    private lastPolledAt;
+    private lastError;
+    /** docId → last-seen epoch ms, for the dedupe window. */
+    private readonly seenDocs;
+    constructor(definition: ConnectorSkillDefinition, options: ConnectorSkillOptions);
+    describe(): DatasourceSkillDescriptor;
+    polling(): PollingMetadata;
+    index(): Promise<DatasourceIndexResult>;
+    retrievalMethods(): readonly RetrievalMethod[];
+    describeSources(): readonly SourceDescription[];
+    skillManifest(): DatasourceSkillManifest;
+    private applyDedupeWindow;
+    private fail;
+}
+interface ConnectorLexicalMethodOptions {
+    readonly skillName: string;
+    readonly skillType: string;
+    readonly instanceId: string;
+    readonly tags: readonly string[];
+    readonly store: DatasourceChunkStore;
+}
+/**
+ * Lexical retrieval over a connector skill's chunk store. Failures collapse
+ * to an empty result set; retrieval never throws. Sources are opaque
+ * `/<skill>/<instance>/chunks/<chunk-id>` paths and are matched against both
+ * the user scope and the trusted allowed scopes.
+ */
+export declare class ConnectorLexicalMethod implements RetrievalMethod {
+    private readonly options;
+    constructor(options: ConnectorLexicalMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+    private matchesScope;
+}
+export {};

package/dist/datasource/connector.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Connector contract shared by the built-in datasource skills (Slack, Discord,
+ * Notion, GitHub, Google Drive, Gmail/IMAP, local mail exports,
+ * RSS/news).
+ *
+ * A *connector* is the trusted, server-configured bridge to one external
+ * system. It fetches documents for indexing and NEVER throws — every failure
+ * is reported as a discriminated `ok: false` result with a coarse
+ * {@link ConnectorFailureReason} and a traceable diagnostic message.
+ * Connectors are constructed from server-supplied configuration only;
+ * model/tool arguments never reach a connector.
+ */
+import type { DatasourceDiagnosticCode } from "./types.js";
+/** Coarse failure classes a connector can report. */
+export type ConnectorFailureReason = "not-configured" | "auth" | "permission" | "rate-limited" | "unavailable" | "api-error" | "invalid-data" | "empty";
+/**
+ * One document fetched from the external system. `docId` and `hierarchy`
+ * segments are sanitized into opaque slash-safe segments before they become
+ * part of a source path.
+ */
+export interface ConnectorDocument {
+    /** Stable, unique id within the connector instance (pre-sanitization). */
+    readonly docId: string;
+    /**
+     * Optional hierarchy segments under the instance root, e.g.
+     * `["channels", "general"]` for a Slack channel or
+     * `["folders", "projects"]` for a vault folder.
+     */
+    readonly hierarchy?: readonly string[];
+    readonly title?: string;
+    readonly content: string;
+    /** Epoch milliseconds of the document's own timestamp when known. */
+    readonly publishedAt?: number;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface ConnectorFetchOk {
+    readonly ok: true;
+    readonly documents: readonly ConnectorDocument[];
+    /** Sanitized, path/PII-opaque warning strings. */
+    readonly warnings?: readonly string[];
+}
+export interface ConnectorFetchFail {
+    readonly ok: false;
+    readonly reason: ConnectorFailureReason;
+    /** Sanitized, path/PII-opaque failure message. */
+    readonly message?: string;
+}
+export type ConnectorFetchResult = ConnectorFetchOk | ConnectorFetchFail;
+/**
+ * The trusted bridge to one external system. Implementations never throw from
+ * `fetch` — they return an `ok: false` result instead.
+ */
+export interface DatasourceConnector {
+    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
+}
+/** Map a connector failure reason onto the datasource diagnostic union. */
+export declare function connectorFailureToDiagnosticCode(reason: ConnectorFailureReason): DatasourceDiagnosticCode;
+/**
+ * Bound free-form diagnostic text: trimmed and capped at 500 characters.
+ * Diagnostic messages are passed through verbatim (paths, account ids, and
+ * all) so failures stay traceable; privacy is the operator's responsibility
+ * (run AutoRAG with a local LLM if diagnostics must not leave the machine).
+ */
+export declare function boundDiagnosticText(value: string): string;
+/**
+ * Sanitize an arbitrary string into a slash-safe source-path segment.
+ * Unicode letters/digits (e.g. Hangul channel or folder names) are kept so
+ * scopes stay human-readable; separators, punctuation, and control chars
+ * collapse to `-`. When sanitization loses information, a short hash of the
+ * original value is appended so distinct inputs stay distinct.
+ */
+export declare function sanitizeIdSegment(value: string): string;

package/dist/datasource/crawler-client.d.ts

@@ -0,0 +1,9 @@
+import type { CrawlerCliOptions, CrawlerProfile, CrawlerSearchOptions, CrawlerSearchResult, CrawlerSyncResult } from "./crawler-types.js";
+export declare class CrawlerCliClient {
+    private readonly profile;
+    private readonly options;
+    constructor(profile: CrawlerProfile, options?: CrawlerCliOptions);
+    sync(signal?: AbortSignal): Promise<CrawlerSyncResult>;
+    search(query: string, options?: CrawlerSearchOptions): Promise<CrawlerSearchResult>;
+    private run;
+}

package/dist/datasource/crawler-skill.d.ts

@@ -0,0 +1,41 @@
+import type { RetrievalMethod } from "../retrieval/types.js";
+import type { CrawlerSearchOptions, CrawlerSearchResult, CrawlerSyncResult } from "./crawler-types.js";
+import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "./types.js";
+export interface CrawlerSkillClient {
+    sync(signal?: AbortSignal): Promise<CrawlerSyncResult>;
+    search(query: string, options?: CrawlerSearchOptions): Promise<CrawlerSearchResult>;
+}
+export interface CrawlerDatasourceDefinition {
+    readonly datasourceId: string;
+    readonly skillType: string;
+    readonly description: string;
+    readonly defaultTags: readonly string[];
+    readonly contentType: string;
+    readonly manifestDescription: string;
+    readonly backendName: string;
+}
+export interface CrawlerDatasourceSkillOptions {
+    readonly client: CrawlerSkillClient;
+    readonly instanceId?: string;
+    readonly instances?: readonly string[];
+    readonly pollingIntervalMs?: number;
+    readonly tags?: readonly string[];
+    readonly lastIndexedAt?: number;
+}
+export declare class CrawlerDatasourceSkill implements DatasourceSkill {
+    private readonly definition;
+    private readonly client;
+    private readonly instanceId;
+    private readonly instances;
+    private readonly pollingIntervalMs;
+    private readonly tags;
+    private lastIndexedAt;
+    constructor(definition: CrawlerDatasourceDefinition, options: CrawlerDatasourceSkillOptions);
+    describe(): DatasourceSkillDescriptor;
+    polling(): PollingMetadata;
+    index(): Promise<DatasourceIndexResult>;
+    retrievalMethods(): readonly RetrievalMethod[];
+    describeSources(): readonly SourceDescription[];
+    skillManifest(): DatasourceSkillManifest;
+    private fail;
+}

package/dist/datasource/crawler-types.d.ts

@@ -0,0 +1,53 @@
+import type { RetrievalOptions } from "../retrieval/types.js";
+export type CrawlerFailureReason = "binary-missing" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-output";
+export interface CrawlerFailure {
+    readonly ok: false;
+    readonly reason: CrawlerFailureReason;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number | null;
+}
+export interface CrawlerHit {
+    readonly id: string;
+    readonly content: string;
+    readonly score: number;
+    readonly title?: string;
+    readonly hierarchy?: readonly string[];
+    readonly publishedAt?: number;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface CrawlerSyncOk {
+    readonly ok: true;
+    readonly count: number;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+}
+export interface CrawlerSearchOk {
+    readonly ok: true;
+    readonly hits: readonly CrawlerHit[];
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+}
+export type CrawlerSyncResult = CrawlerSyncOk | CrawlerFailure;
+export type CrawlerSearchResult = CrawlerSearchOk | CrawlerFailure;
+export type CrawlerSearchOptions = RetrievalOptions;
+export interface CrawlerCliOptions {
+    readonly binaryPath?: string;
+    readonly databasePath?: string;
+    readonly sourcePath?: string;
+    readonly configPath?: string;
+    readonly syncSource?: string;
+    readonly timeoutMs?: number;
+    readonly maxBufferBytes?: number;
+    readonly env?: Readonly<Record<string, string | undefined>>;
+}
+export interface CrawlerProfile {
+    readonly binaryName: string;
+    readonly allowedEnvPrefixes: readonly string[];
+    readonly syncArgs: (options: CrawlerCliOptions) => readonly string[];
+    readonly searchArgs: (options: CrawlerCliOptions, query: string, topK: number) => readonly string[];
+    readonly parseSyncCount: (stdout: string) => number | undefined;
+    readonly parseHits: (stdout: string) => readonly CrawlerHit[] | undefined;
+}

package/dist/datasource/http.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Minimal HTTP helper shared by the API-backed datasource connectors.
+ *
+ * Every request is bounded by a timeout, honors an optional outer abort
+ * signal, and maps failures onto the coarse {@link ConnectorFailureReason}
+ * union with short, path/PII-opaque messages (never URLs, tokens, or bodies).
+ */
+import type { ConnectorFailureReason } from "./connector.js";
+export interface HttpRequestOptions {
+    readonly fetchImpl?: typeof fetch;
+    readonly timeoutMs?: number;
+    readonly signal?: AbortSignal;
+    readonly headers?: Readonly<Record<string, string>>;
+    readonly method?: string;
+    readonly body?: string;
+}
+export type HttpJsonResult = {
+    readonly ok: true;
+    readonly status: number;
+    readonly json: unknown;
+} | {
+    readonly ok: false;
+    readonly reason: ConnectorFailureReason;
+    readonly message: string;
+};
+export type HttpTextResult = {
+    readonly ok: true;
+    readonly status: number;
+    readonly text: string;
+} | {
+    readonly ok: false;
+    readonly reason: ConnectorFailureReason;
+    readonly message: string;
+};
+/** Map an HTTP status code onto a connector failure reason. */
+export declare function statusToReason(status: number): ConnectorFailureReason;
+/** GET/POST a URL and return the body text; never throws. */
+export declare function httpText(url: string, options?: HttpRequestOptions): Promise<HttpTextResult>;
+/** GET/POST a URL and parse the body as JSON; never throws. */
+export declare function httpJson(url: string, options?: HttpRequestOptions): Promise<HttpJsonResult>;
+/** Narrowing helpers for defensive JSON traversal. */
+export declare function asRecord(value: unknown): Record<string, unknown> | undefined;
+export declare function asArray(value: unknown): readonly unknown[];
+export declare function asString(value: unknown): string | undefined;
+export declare function asNumber(value: unknown): number | undefined;
+/** Parse a date-ish string to epoch ms, or undefined. */
+export declare function parseEpochMs(value: unknown): number | undefined;
+/** Resolve an auth token from explicit config or an environment variable name. */
+export declare function resolveToken(token: string | undefined, tokenEnv: string | undefined): string | undefined;

package/dist/datasource/index.d.ts

@@ -1,9 +1,25 @@
 export { DatasourceAccessContext, type DatasourceAccessContextOptions } from "./access-context.js";
+export { DatasourceChunkStore, type DatasourceChunkStoreOptions, datasourceChunkStorePath, type ScoredChunk, type StoredChunk, } from "./chunk-store.js";
+export { boundDiagnosticText, type ConnectorDocument, type ConnectorFailureReason, type ConnectorFetchFail, type ConnectorFetchOk, type ConnectorFetchResult, connectorFailureToDiagnosticCode, type DatasourceConnector, sanitizeIdSegment, } from "./connector.js";
+export { ConnectorDatasourceSkill, ConnectorLexicalMethod, type ConnectorSkillDefinition, type ConnectorSkillOptions, } from "./connector-skill.js";
 export { mapDatasourceDiagnostic, mapDatasourceDiagnostics } from "./diagnostics.js";
 export { type CronParseResult, isDue, parseCronExpr } from "./polling.js";
 export { DatasourceSkillRegistry, type RegisteredDatasourceSkill } from "./registry.js";
 export { DatasourceResultFilter, type ResultsByMethod } from "./result-filter.js";
 export { buildDatasourceChunkSource, buildDatasourceInstanceSource, DATASOURCE_CHUNKS_SEGMENT, datasourceSourceHasFragment, datasourceSourcePath, isDatasourceSource, matchesDatasourceScope, matchesVirtualPathScope, normalizeVirtualPath, } from "./scope.js";
+export { DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_SOURCE, DiscrawlClient, type DiscrawlFailureReason, DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, DiscrawlSemanticMethod, DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions, type DiscrawlSourceKind, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./skills/discrawl/index.js";
+export { BUILTIN_DATASOURCE_SKILL_NAMES, type BuildDatasourceSkillsResult, buildDatasourceSkills, type DatasourceSkillConfig, type DatasourcesConfig, } from "./skills/factory.js";
+export { GDriveConnector, type GDriveConnectorOptions, GDriveSkill, type GDriveSkillOptions, } from "./skills/gdrive/index.js";
+export { GitHubConnector, type GitHubConnectorOptions, GitHubSkill, type GitHubSkillOptions, } from "./skills/github/index.js";
+export { GmailConnector, type GmailConnectorOptions, GmailSkill, type GmailSkillOptions, } from "./skills/gmail/index.js";
 export type { KatokFailureReason, KatokHit, KatokOptions, KatokSearchMode, KatokSearchOptions, KatokSearchResult, KatokSkillClient, KatokSkillOptions, } from "./skills/katok/index.js";
 export { KatokBm25Method, KatokClient, KatokSemanticMethod, KatokSkill } from "./skills/katok/index.js";
+export { MailExportConnector, type MailExportConnectorOptions, MailExportSkill, type MailExportSkillOptions, } from "./skills/mail-export/index.js";
+export { NotcrawlClient, type NotcrawlOptions, NotionSkill, type NotionSkillOptions, } from "./skills/notion/index.js";
+export { ObsidianBm25Method, ObsidianSemanticMethod, ObsidianSkill, type ObsidianSkillClient, type ObsidianSkillOptions, QmdClient, type QmdOptions, type QmdSearchHit, type QmdSearchMode, type QmdSearchResult, } from "./skills/obsidian/index.js";
+export { RssConnector, type RssConnectorOptions, type RssFeedConfig, RssSkill, type RssSkillOptions, } from "./skills/rss/index.js";
+export { SlackSkill, type SlackSkillOptions, SlacrawlClient, type SlacrawlOptions, } from "./skills/slack/index.js";
+export { SpotlightConnector, type SpotlightConnectorOptions, SpotlightSkill, type SpotlightSkillOptions, } from "./skills/spotlight/index.js";
+export { TelecrawlClient, type TelecrawlOptions, TelecrawlSkill, type TelecrawlSkillOptions, } from "./skills/telecrawl/index.js";
+export { WacrawlClient, type WacrawlOptions, WacrawlSkill, type WacrawlSkillOptions } from "./skills/wacrawl/index.js";
 export type { DatasourceAccessible, DatasourceDiagnostic, DatasourceDiagnosticCode, DatasourceIndexFail, DatasourceIndexOk, DatasourceIndexResult, DatasourceInstance, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription, } from "./types.js";

package/dist/datasource/skills/discrawl/client.d.ts

@@ -0,0 +1,23 @@
+import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlOptions, DiscrawlSearchMode, DiscrawlSearchOptions, DiscrawlSearchResult, DiscrawlStatusResult, DiscrawlSyncResult } from "./types.js";
+/**
+ * Thin external `discrawl` CLI wrapper. Every method spawns the `discrawl`
+ * binary as a child process, parses JSON from stdout, and returns a
+ * discriminated ok/fail union. No method throws for expected failures (missing
+ * binary, rejected user token, CLI nonzero exit, timeout, oversized output, or
+ * invalid JSON). The client never opens the Discord archive database directly.
+ *
+ * `--json` is a *global* discrawl flag and must precede the subcommand;
+ * `discrawl search --json` is rejected by the CLI as an unknown flag.
+ */
+export declare class DiscrawlClient {
+    private readonly options;
+    constructor(options?: DiscrawlOptions);
+    doctor(signal?: AbortSignal): Promise<DiscrawlDoctorResult>;
+    status(signal?: AbortSignal): Promise<DiscrawlStatusResult>;
+    sync(signal?: AbortSignal): Promise<DiscrawlSyncResult>;
+    embed(limit?: number, signal?: AbortSignal): Promise<DiscrawlEmbedResult>;
+    search(mode: DiscrawlSearchMode, query: string, options?: DiscrawlSearchOptions): Promise<DiscrawlSearchResult>;
+    private runJson;
+    private run;
+}
+export declare function discrawlWorkspace(options: DiscrawlOptions): string | undefined;

package/dist/datasource/skills/discrawl/config.d.ts

@@ -0,0 +1,9 @@
+import type { DiscrawlOptions } from "./types.js";
+/**
+ * Creates or updates AutoRAG's workspace-local discrawl config.
+ *
+ * Only the fields AutoRAG owns are changed: the workspace database path and
+ * embedding enablement/provider/model. Existing operator settings such as
+ * guilds, source filters, sync policy, and search mode are preserved.
+ */
+export declare function ensureManagedDiscrawlConfig(configPath: string, databasePath: string, options: DiscrawlOptions): void;

package/dist/datasource/skills/discrawl/index.d.ts

@@ -0,0 +1,5 @@
+export { DiscrawlClient, discrawlWorkspace } from "./client.js";
+export { DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlMethodOptions, type DiscrawlSearchClient, DiscrawlSemanticMethod, } from "./methods.js";
+export { DISCRAWL_SOURCE_KIND, discrawlConfigPath, discrawlDatabasePath, discrawlDatasourceRoot, discrawlSourcePath, parseDiscrawlSourcePath, } from "./paths.js";
+export { DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions } from "./skill.js";
+export { DEFAULT_DISCRAWL_BINARY, DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_EMBEDDING_PROVIDER, DEFAULT_DISCRAWL_MAX_BUFFER_BYTES, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_SOURCE, DEFAULT_DISCRAWL_TIMEOUT_MS, type DiscrawlDoctorInfo, type DiscrawlDoctorResult, type DiscrawlEmbedInfo, type DiscrawlEmbedResult, type DiscrawlFailure, type DiscrawlFailureReason, type DiscrawlMessage, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, type DiscrawlSourceKind, type DiscrawlStatusInfo, type DiscrawlStatusResult, type DiscrawlSyncInfo, type DiscrawlSyncResult, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./types.js";

package/dist/datasource/skills/discrawl/methods.d.ts

@@ -0,0 +1,55 @@
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
+import type { DiscrawlSearchMode, DiscrawlSearchOptions, DiscrawlSearchResult } from "./types.js";
+/**
+ * Narrow client surface required by the Discord retrieval methods. The real
+ * {@link DiscrawlClient} satisfies this structurally; tests may stub it.
+ */
+export interface DiscrawlSearchClient {
+    search(mode: DiscrawlSearchMode, query: string, options?: DiscrawlSearchOptions): Promise<DiscrawlSearchResult>;
+}
+export interface DiscrawlMethodOptions {
+    readonly client: DiscrawlSearchClient;
+    readonly instanceId: string;
+    readonly tags?: readonly string[];
+}
+/**
+ * Lexical (SQLite FTS5) retrieval over a Discord archive via the external
+ * `discrawl` CLI. All client failures collapse to an empty result set;
+ * retrieval never throws.
+ */
+export declare class DiscrawlFtsMethod implements RetrievalMethod {
+    private readonly client;
+    private readonly instanceId;
+    private readonly tags;
+    constructor(options: DiscrawlMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+}
+/**
+ * Semantic (vector) retrieval over a Discord archive via the external
+ * `discrawl` CLI. Requires a configured embedding provider and a completed
+ * `discrawl embed` pass; when either is missing the CLI exits non-zero and
+ * this method returns no results rather than throwing.
+ */
+export declare class DiscrawlSemanticMethod implements RetrievalMethod {
+    private readonly client;
+    private readonly instanceId;
+    private readonly tags;
+    constructor(options: DiscrawlMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+}
+/**
+ * Hybrid retrieval (FTS5 + semantic, deduplicated by message id). This is the
+ * preferred Discord method: discrawl's FTS index welds words across newlines
+ * into one token, so semantic recall is what covers terms FTS cannot reach.
+ * See AutoRAG issue #1413.
+ */
+export declare class DiscrawlHybridMethod implements RetrievalMethod {
+    private readonly client;
+    private readonly instanceId;
+    private readonly tags;
+    constructor(options: DiscrawlMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+}

package/dist/datasource/skills/discrawl/paths.d.ts

@@ -0,0 +1,30 @@
+export declare const AUTORAG_DIRNAME = ".autorag";
+export declare const DATASOURCES_DIRNAME = "datasources";
+export declare const DISCRAWL_DATASOURCE_DIRNAME = "discrawl";
+/** Logical datasource kind for source identifiers. */
+export declare const DISCRAWL_SOURCE_KIND: "discord";
+/**
+ * Root directory for discrawl-managed state under a workspace. This is where
+ * the external `discrawl` CLI stores its SQLite archive, FTS index, and
+ * message vectors.
+ */
+export declare function discrawlDatasourceRoot(workspaceRoot: string): string;
+/** Config file discrawl reads for embedding provider/model settings. */
+export declare function discrawlConfigPath(workspaceRoot: string): string;
+/** SQLite archive maintained by the discrawl CLI. */
+export declare function discrawlDatabasePath(workspaceRoot: string): string;
+/**
+ * Builds the slash-hierarchical source identifier for a single archived
+ * message. Discord message ids are stable snowflakes, so these sources stay
+ * traceable across re-syncs.
+ */
+export declare function discrawlSourcePath(instanceId: string, messageId: string): string;
+/**
+ * Parses a discrawl source identifier back into its components. Returns
+ * `undefined` for malformed or non-discord sources so deny decisions stay
+ * explicit rather than undefined-as-deny.
+ */
+export declare function parseDiscrawlSourcePath(source: string): {
+    readonly instanceId: string;
+    readonly messageId: string;
+} | undefined;

package/dist/datasource/skills/discrawl/skill.d.ts

@@ -0,0 +1,58 @@
+import type { RetrievalMethod } from "../../../retrieval/types.js";
+import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import { type DiscrawlSearchClient } from "./methods.js";
+import type { DiscrawlDoctorResult, DiscrawlEmbedResult, DiscrawlSearchMode, DiscrawlSyncResult } from "./types.js";
+export interface DiscrawlSkillClient extends DiscrawlSearchClient {
+    doctor(): Promise<DiscrawlDoctorResult>;
+    sync(): Promise<DiscrawlSyncResult>;
+    embed(limit?: number): Promise<DiscrawlEmbedResult>;
+}
+export interface DiscrawlSkillOptions {
+    readonly client: DiscrawlSkillClient;
+    readonly instanceId?: string;
+    readonly instances?: readonly string[];
+    readonly pollingIntervalMs?: number;
+    readonly tags?: readonly string[];
+    readonly lastIndexedAt?: number;
+    readonly embeddingModel?: string;
+    readonly defaultMode?: DiscrawlSearchMode;
+    /** Max messages embedded per index pass. Undefined drains the whole backlog. */
+    readonly embedLimit?: number;
+}
+/**
+ * Discord datasource skill backed by the external `discrawl` CLI.
+ *
+ * Mirrors the katok model: the CLI owns the archive, the FTS5 index, and the
+ * vector index; AutoRAG only spawns it and maps results. AutoRAG never calls
+ * the Discord API itself and never reads the Discord Desktop cache directly.
+ *
+ * `index()` runs the CLI's two-phase pipeline — `sync` (incremental, cursor
+ * based) then `embed` (drains the embedding queue) — so a poll costs work
+ * proportional to what changed rather than to archive size.
+ */
+export declare class DiscrawlSkill implements DatasourceSkill {
+    private readonly client;
+    private readonly instanceId;
+    private readonly instances;
+    private readonly pollingIntervalMs;
+    private readonly tags;
+    private readonly embeddingModel;
+    private readonly defaultMode;
+    private readonly embedLimit;
+    private lastIndexedAt;
+    constructor(options: DiscrawlSkillOptions);
+    describe(): DatasourceSkillDescriptor;
+    polling(): PollingMetadata;
+    index(): Promise<DatasourceIndexResult>;
+    retrievalMethods(): readonly RetrievalMethod[];
+    describeSources(): readonly SourceDescription[];
+    skillManifest(): DatasourceSkillManifest;
+    private runEmbed;
+    /**
+     * English-only embedders map non-English text into one narrow similarity
+     * band, so semantic search degrades to noise without ever erroring. Surfacing
+     * it as a diagnostic keeps that failure visible. See AutoRAG issue #1414.
+     */
+    private englishOnlyModelDiagnostic;
+    private fail;
+}