@autorag/librarian 2.1.0 → 2.3.0

package/dist/datasource/skills/factory.d.ts

@@ -11,11 +11,18 @@ import type { DatasourceSkill } from "../types.js";
 /** One configured datasource entry (the trusted `datasources.<name>` value). */
 export interface DatasourceSkillConfig {
     readonly enabled?: boolean;
+    /** Built-in datasource template used when the config key is a connection alias. */
+    readonly type?: string;
     readonly instanceId?: string;
     readonly pollingIntervalMs?: number;
     readonly tags?: readonly string[];
     /** Connector-specific options (token env names, repos, feeds, paths, …). */
     readonly connector?: Record<string, unknown>;
+    /** Optional chat/channel allowlist; absent means all channels. */
+    readonly channels?: {
+        readonly ids?: readonly string[];
+        readonly names?: readonly string[];
+    };
 }
 /** The trusted `datasources` config section: skill name → config. */
 export type DatasourcesConfig = Readonly<Record<string, DatasourceSkillConfig | boolean>>;

package/dist/datasource/skills/gdrive/rclone-connector.d.ts

@@ -1,15 +1,10 @@
 /**
- * Rclone-backed drive connector (issue #1301, rclone path).
+ * Provider-neutral incremental rclone mirror.
  *
- * Uses the external `rclone` CLI (https://rclone.org) as the trusted bridge
- * to Google Drive — or any of rclone's 70+ storage backends — so OAuth
- * setup and token refresh live entirely in `rclone config`, not here
- * (katok/himalaya pattern). Google Docs/Sheets are transparently exported
- * as text via rclone's `--drive-export-formats`. Never throws; failures map
- * onto the coarse connector failure union with path/PII-opaque messages.
- *
- * Listing: `rclone lsjson <remote> --recursive --files-only`
- * Content: `rclone cat <remote>/<path>` (bounded, text formats only)
+ * `lsjson` inventories a remote, a workspace manifest computes its diff, and
+ * only added/changed indexable files are copied into a managed mirror. The
+ * manifest is published only after every changed file is available, so search
+ * can continue using the previous completed snapshot after an interrupted run.
  */
 import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
 export interface RcloneRunResult {
@@ -20,27 +15,22 @@ export interface RcloneRunResult {
 }
 export type RcloneRunner = (args: readonly string[], timeoutMs: number) => Promise<RcloneRunResult>;
 export interface RcloneConnectorOptions {
-    /** Path to the rclone binary. Default bare `rclone` PATH lookup. */
     readonly binaryPath?: string;
-    /**
-     * Rclone remote path, e.g. `gdrive:` or `gdrive:Team/Docs`. Required
-     * trusted configuration (`rclone config` defines the remote).
-     */
     readonly remote?: string;
-    /**
-     * Indexable file extensions (lowercase, with dot). Google Docs/Sheets
-     * surface as these after `--drive-export-formats txt,csv`.
-     */
+    readonly provider?: string;
+    readonly workspaceRoot?: string;
+    readonly instanceId?: string;
+    readonly skillName?: string;
     readonly extensions?: readonly string[];
-    /** Drive export formats forwarded to rclone. Default `txt,csv`. */
+    readonly include?: readonly string[];
+    readonly exclude?: readonly string[];
     readonly exportFormats?: string;
-    /** Max files whose contents are fetched per run. Default 200. */
     readonly maxDocuments?: number;
-    /** Files larger than this are skipped. Default 2 MiB. */
     readonly maxBytesPerFile?: number;
-    /** Per-spawn timeout. Default 60s (cloud listings can be slow). */
+    readonly concurrency?: number;
+    readonly bandwidthLimit?: string;
+    readonly dryRun?: boolean;
     readonly timeoutMs?: number;
-    /** Injectable process runner for tests. */
     readonly runner?: RcloneRunner;
 }
 export declare class RcloneConnector implements DatasourceConnector {
@@ -48,4 +38,6 @@ export declare class RcloneConnector implements DatasourceConnector {
     private readonly runner;
     constructor(options?: RcloneConnectorOptions);
     fetch(): Promise<ConnectorFetchResult>;
+    private fetchIncremental;
+    private fetchWithoutMirror;
 }

package/dist/datasource/skills/notion/index.d.ts

@@ -1,2 +1,15 @@
-export { NotionConnector, type NotionConnectorOptions } from "./connector.js";
-export { NOTION_SKILL_DEFINITION, NotionSkill, type NotionSkillOptions } from "./skill.js";
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface NotcrawlOptions extends CrawlerCliOptions {
+}
+export declare class NotcrawlClient extends CrawlerCliClient {
+    constructor(options?: NotcrawlOptions);
+}
+export interface NotionSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: NotcrawlOptions;
+}
+export declare class NotionSkill extends CrawlerDatasourceSkill {
+    constructor(options?: NotionSkillOptions);
+}

package/dist/datasource/skills/obsidian/client.d.ts

@@ -0,0 +1,15 @@
+import type { QmdEmbedResult, QmdEnsureResult, QmdOptions, QmdSearchMode, QmdSearchOptions, QmdSearchResult, QmdUpdateResult } from "./types.js";
+/**
+ * Thin external `qmd` CLI wrapper for Obsidian vault indexing/search.
+ * Every method spawns the `qmd` binary, never throws for expected failures,
+ * and isolates config/cache under the AutoRAG workspace.
+ */
+export declare class QmdClient {
+    private readonly options;
+    constructor(options?: QmdOptions);
+    ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
+    update(signal?: AbortSignal): Promise<QmdUpdateResult>;
+    embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
+    search(mode: QmdSearchMode, query: string, options?: QmdSearchOptions): Promise<QmdSearchResult>;
+    private run;
+}

package/dist/datasource/skills/obsidian/index.d.ts

@@ -1,2 +1,6 @@
-export { ObsidianConnector, type ObsidianConnectorOptions } from "./connector.js";
-export { OBSIDIAN_SKILL_DEFINITION, ObsidianSkill, type ObsidianSkillOptions } from "./skill.js";
+export { QmdClient } from "./client.js";
+export { ObsidianBm25Method, type ObsidianMethodOptions, type ObsidianSearchClient, ObsidianSemanticMethod, } from "./methods.js";
+export { obsidianDatasourceRoot, obsidianQmdCacheDir, obsidianQmdConfigDir, obsidianSourcePath, parseObsidianSourcePath, stripEdgeDashes, toQmdCollectionName, } from "./paths.js";
+export { ObsidianSkill, type ObsidianSkillClient, type ObsidianSkillOptions } from "./skill.js";
+export type { QmdEmbedResult, QmdEnsureResult, QmdFailureReason, QmdOptions, QmdSearchHit, QmdSearchMode, QmdSearchOptions, QmdSearchResult, QmdUpdateResult, } from "./types.js";
+export { DEFAULT_QMD_BINARY, DEFAULT_QMD_MAX_BUFFER_BYTES, DEFAULT_QMD_TIMEOUT_MS, } from "./types.js";

package/dist/datasource/skills/obsidian/methods.d.ts

@@ -0,0 +1,26 @@
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "../../../retrieval/types.js";
+import type { QmdSearchMode, QmdSearchOptions, QmdSearchResult } from "./types.js";
+export interface ObsidianSearchClient {
+    search(mode: QmdSearchMode, query: string, options?: QmdSearchOptions): Promise<QmdSearchResult>;
+}
+export interface ObsidianMethodOptions {
+    readonly client: ObsidianSearchClient;
+    readonly instanceId: string;
+    readonly tags?: readonly string[];
+}
+export declare class ObsidianBm25Method implements RetrievalMethod {
+    private readonly client;
+    private readonly instanceId;
+    private readonly tags;
+    constructor(options: ObsidianMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+}
+export declare class ObsidianSemanticMethod implements RetrievalMethod {
+    private readonly client;
+    private readonly instanceId;
+    private readonly tags;
+    constructor(options: ObsidianMethodOptions);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+}

package/dist/datasource/skills/obsidian/paths.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Path helpers for the Obsidian (`qmd`) datasource skill.
+ *
+ * All AutoRAG-managed state lives under
+ * `<workspaceRoot>/.autorag/datasources/obsidian/<instanceId>/`. The external
+ * `qmd` CLI is pointed at isolated config/cache dirs so multiple workspaces
+ * never share an index. Callers surface opaque slash-hierarchical sources of
+ * the form `/obsidian/<instance-id>/chunks/<chunk-id>`.
+ */
+export declare const AUTORAG_DIRNAME = ".autorag";
+export declare const DATASOURCES_DIRNAME = "datasources";
+export declare const OBSIDIAN_DATASOURCE_DIRNAME = "obsidian";
+export declare const OBSIDIAN_SOURCE_KIND: "obsidian";
+export declare function obsidianDatasourceRoot(workspaceRoot: string, instanceId: string): string;
+/** Directory passed to `QMD_CONFIG_DIR` (holds `index.yml`). */
+export declare function obsidianQmdConfigDir(workspaceRoot: string, instanceId: string): string;
+/** Directory passed to `XDG_CACHE_HOME` (holds `qmd/index.sqlite`). */
+export declare function obsidianQmdCacheDir(workspaceRoot: string, instanceId: string): string;
+export declare function obsidianSourcePath(instanceId: string, chunkId: string): string;
+export declare function parseObsidianSourcePath(source: string): {
+    readonly instanceId: string;
+    readonly chunkId: string;
+} | undefined;
+/** Strip leading and trailing ASCII dashes without a quadratic regex. */
+export declare function stripEdgeDashes(value: string): string;
+/** Sanitize an instance id into a qmd collection name. */
+export declare function toQmdCollectionName(instanceId: string): string;

package/dist/datasource/skills/obsidian/skill.d.ts

@@ -1,20 +1,44 @@
-/**
- * Obsidian vault datasource skill (issue #1314).
- *
- * Thin composition of ObsidianConnector with the shared
- * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
- * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
- * the shared pipeline, opaque `/obsidian/<instance>/…` sources, and a
- * progressive-disclosure agent-skill manifest.
- */
-import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import { ObsidianConnector, type ObsidianConnectorOptions } from "./connector.js";
-export declare const OBSIDIAN_SKILL_DEFINITION: ConnectorSkillDefinition;
-export interface ObsidianSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
-    /** Trusted connector configuration; a pre-built connector wins. */
-    readonly connector?: ObsidianConnector;
-    readonly connectorOptions?: ObsidianConnectorOptions;
+import type { RetrievalMethod } from "../../../retrieval/types.js";
+import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import { type ObsidianSearchClient } from "./methods.js";
+import type { QmdEmbedResult, QmdEnsureResult, QmdUpdateResult } from "./types.js";
+export interface ObsidianSkillClient extends ObsidianSearchClient {
+    ensureCollection(): Promise<QmdEnsureResult>;
+    update(): Promise<QmdUpdateResult>;
+    embed(): Promise<QmdEmbedResult>;
 }
-export declare class ObsidianSkill extends ConnectorDatasourceSkill {
+export interface ObsidianSkillOptions {
+    readonly datasourceId?: string;
+    readonly client?: ObsidianSkillClient;
+    readonly vaultPath?: string;
+    readonly binaryPath?: string;
+    readonly instanceId?: string;
+    readonly instances?: readonly string[];
+    readonly pollingIntervalMs?: number;
+    readonly tags?: readonly string[];
+    readonly lastIndexedAt?: number;
+    readonly workspaceRoot?: string;
+    readonly collectionName?: string;
+    readonly timeoutMs?: number;
+    readonly connectorOptions?: {
+        readonly vaultPath?: string;
+    };
+}
+export declare class ObsidianSkill implements DatasourceSkill {
+    private readonly client;
+    private readonly instanceId;
+    private readonly instances;
+    private readonly pollingIntervalMs;
+    private readonly tags;
+    private readonly vaultPath;
+    private lastIndexedAt;
+    private lastError;
     constructor(options?: ObsidianSkillOptions);
+    describe(): DatasourceSkillDescriptor;
+    polling(): PollingMetadata;
+    index(): Promise<DatasourceIndexResult>;
+    retrievalMethods(): readonly RetrievalMethod[];
+    describeSources(): readonly SourceDescription[];
+    skillManifest(): DatasourceSkillManifest;
+    private fail;
 }

package/dist/datasource/skills/obsidian/types.d.ts

@@ -0,0 +1,90 @@
+import type { RetrievalOptions } from "../../../retrieval/types.js";
+/**
+ * Search modes accepted by the external `qmd` CLI.
+ *
+ * - `search`  — BM25 / FTS5 lexical search
+ * - `vsearch` — dense vector semantic search
+ */
+export type QmdSearchMode = "search" | "vsearch";
+/**
+ * Configuration for {@link QmdClient}. The client spawns the `qmd` binary as a
+ * child process and isolates config/cache under the AutoRAG workspace.
+ */
+export interface QmdOptions {
+    /** Explicit path to the `qmd` binary. Defaults to a bare `qmd` PATH lookup. */
+    readonly binaryPath?: string;
+    /** Spawn timeout in milliseconds. Default 60_000 (embed can be slow). */
+    readonly timeoutMs?: number;
+    /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
+    readonly maxBufferBytes?: number;
+    /** Absolute path to the Obsidian vault. Required for ensure/update. */
+    readonly vaultPath?: string;
+    /** AutoRAG workspace root used to isolate qmd config/cache. */
+    readonly workspaceRoot?: string;
+    /** Instance id used for source paths and collection naming. Default `default`. */
+    readonly instanceId?: string;
+    /** Explicit collection name; defaults to a sanitized instance id. */
+    readonly collectionName?: string;
+    /** Environment overrides merged on top of a restricted process env. */
+    readonly env?: Readonly<Record<string, string | undefined>>;
+}
+export declare const DEFAULT_QMD_BINARY = "qmd";
+export declare const DEFAULT_QMD_TIMEOUT_MS = 60000;
+export declare const DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
+export type QmdFailureReason = "binary-missing" | "not-configured" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
+export interface QmdFailure {
+    readonly ok: false;
+    readonly reason: QmdFailureReason;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number | null;
+}
+export interface QmdOk<T> {
+    readonly ok: true;
+    readonly data: T;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+}
+export interface QmdUpdateInfo {
+    readonly indexed: number;
+    readonly updated: number;
+    readonly unchanged: number;
+    readonly removed: number;
+    readonly needsEmbedding?: boolean;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface QmdEmbedInfo {
+    readonly embedded: boolean;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface QmdEnsureInfo {
+    readonly collectionName: string;
+    readonly vaultPath: string;
+    readonly configDir: string;
+}
+/** A scored search hit from `qmd search` / `qmd vsearch`. */
+export interface QmdSearchHit {
+    readonly chunkId: string;
+    readonly score: number;
+    readonly content: string;
+    readonly title?: string;
+    readonly file?: string;
+    readonly docid?: string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface QmdSearchOk {
+    readonly ok: true;
+    readonly hits: readonly QmdSearchHit[];
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+    readonly data: {
+        readonly hits: readonly QmdSearchHit[];
+    };
+}
+export type QmdEnsureResult = QmdOk<QmdEnsureInfo> | QmdFailure;
+export type QmdUpdateResult = QmdOk<QmdUpdateInfo> | QmdFailure;
+export type QmdEmbedResult = QmdOk<QmdEmbedInfo> | QmdFailure;
+export type QmdSearchResult = QmdSearchOk | QmdFailure;
+export type QmdSearchOptions = RetrievalOptions;

package/dist/datasource/skills/slack/index.d.ts

@@ -1,2 +1,15 @@
-export { SlackConnector, type SlackConnectorOptions } from "./connector.js";
-export { SLACK_SKILL_DEFINITION, SlackSkill, type SlackSkillOptions } from "./skill.js";
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface SlacrawlOptions extends CrawlerCliOptions {
+}
+export declare class SlacrawlClient extends CrawlerCliClient {
+    constructor(options?: SlacrawlOptions);
+}
+export interface SlackSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: SlacrawlOptions;
+}
+export declare class SlackSkill extends CrawlerDatasourceSkill {
+    constructor(options?: SlackSkillOptions);
+}

package/dist/datasource/skills/telecrawl/index.d.ts

@@ -0,0 +1,15 @@
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface TelecrawlOptions extends CrawlerCliOptions {
+}
+export declare class TelecrawlClient extends CrawlerCliClient {
+    constructor(options?: TelecrawlOptions);
+}
+export interface TelecrawlSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: TelecrawlOptions;
+}
+export declare class TelecrawlSkill extends CrawlerDatasourceSkill {
+    constructor(options?: TelecrawlSkillOptions);
+}

package/dist/datasource/skills/wacrawl/index.d.ts

@@ -0,0 +1,15 @@
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface WacrawlOptions extends CrawlerCliOptions {
+}
+export declare class WacrawlClient extends CrawlerCliClient {
+    constructor(options?: WacrawlOptions);
+}
+export interface WacrawlSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: WacrawlOptions;
+}
+export declare class WacrawlSkill extends CrawlerDatasourceSkill {
+    constructor(options?: WacrawlSkillOptions);
+}

package/dist/dupey/cli.d.ts

@@ -0,0 +1,40 @@
+export interface DupeyScanFile {
+    readonly path: string;
+    readonly content_hash?: string;
+    readonly [key: string]: unknown;
+}
+export interface DupeyScanEdge {
+    readonly relation: string;
+    readonly a: string;
+    readonly b: string;
+    readonly [key: string]: unknown;
+}
+export interface DupeyScanFamily {
+    readonly id: number;
+    readonly relation: string;
+    readonly files: readonly string[];
+    readonly members: readonly Record<string, unknown>[];
+    readonly edges: readonly DupeyScanEdge[];
+    readonly pick?: Record<string, unknown>;
+    readonly [key: string]: unknown;
+}
+export interface DupeyScanResult {
+    readonly dir: string;
+    readonly threshold?: number;
+    readonly contains_threshold?: number;
+    readonly contains_min_jaccard?: number;
+    readonly files: readonly DupeyScanFile[];
+    readonly families: readonly DupeyScanFamily[];
+    readonly errors: readonly Record<string, unknown>[];
+    readonly [key: string]: unknown;
+}
+export interface DupeyCliOptions {
+    readonly executable?: string;
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+    readonly run?: (args: readonly string[]) => Promise<string>;
+}
+export declare class DupeyCliError extends Error {
+    constructor(message: string);
+}
+export declare function scanWithDupey(dir: string, options?: DupeyCliOptions): Promise<DupeyScanResult>;

package/dist/dupey/filter.d.ts

@@ -0,0 +1,8 @@
+import type { DupeyScanResult } from "./cli.js";
+export interface ExactDuplicateFilterResult {
+    readonly excluded: ReadonlySet<string>;
+    readonly keepers: ReadonlySet<string>;
+    readonly errors: readonly string[];
+}
+/** Selects the newest filesystem copy for each exact canonical-text hash. */
+export declare function selectExactDuplicateExclusions(root: string, scan: DupeyScanResult): Promise<ExactDuplicateFilterResult>;

package/dist/dupey/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./cli.js";
+export * from "./filter.js";

package/dist/index.d.ts

@@ -2,6 +2,7 @@ export { AutoRAGAgent, type AutoRAGAgentOptions, type AutoRAGRefreshComponentSta
 export { type AutoRAGResultsDetails, createEmitResultsTool, EMIT_AUTORAG_RESULTS_TOOL_NAME, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentWarning, } from "./agent/index.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./agent/system-prompt.js";
 export * from "./datasource/index.js";
+export * from "./dupey/index.js";
 export * from "./jikji/index.js";
 export * from "./manifest/index.js";
 export * from "./memory/index.js";