@autorag/librarian 2.2.0 → 2.3.0

package/dist/datasource/aliased-skill.d.ts

@@ -0,0 +1,28 @@
+import type { RetrievalMethod } from "../retrieval/types.js";
+import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, SourceDescription } from "./types.js";
+export interface AliasDatasourceSkillOptions {
+    readonly alias: string;
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
+}
+/**
+ * Gives any datasource connection an independent model-visible identity.
+ * Channel allowlists are enforced after the backend search so omitted lists
+ * retain the backend's default "all channels" behavior.
+ */
+export declare class AliasedDatasourceSkill implements DatasourceSkill {
+    private readonly skill;
+    private readonly alias;
+    private readonly originalId;
+    private readonly channelIds;
+    private readonly channelNames;
+    constructor(skill: DatasourceSkill, options: AliasDatasourceSkillOptions);
+    describe(): DatasourceSkillDescriptor;
+    polling(): import("./types.js").PollingMetadata;
+    index(): Promise<DatasourceIndexResult>;
+    retrievalMethods(): readonly RetrievalMethod[];
+    describeSources(): readonly SourceDescription[];
+    skillManifest(): DatasourceSkillManifest;
+    private matchesChannel;
+    private rewriteResult;
+}

package/dist/datasource/chunk-store.d.ts

@@ -46,6 +46,8 @@ export declare class DatasourceChunkStore {
      * (in-memory contents stay authoritative for this process).
      */
     replaceDocuments(documents: readonly ConnectorDocument[]): number;
+    /** Apply file-level additions/updates/deletions without rebuilding unchanged chunks. */
+    updateDocuments(documents: readonly ConnectorDocument[], deletedDocIds?: readonly string[]): number;
     /** Load persisted chunks when present. Returns true when data was loaded. */
     load(): boolean;
     /** Ensure persisted chunks are loaded once for read paths. */

package/dist/datasource/connector-skill.d.ts

@@ -33,6 +33,7 @@ export interface ConnectorSkillDefinition {
     readonly description: string;
     /** Capability tags such as `"chat"`, `"api"`, `"polling"`. */
     readonly capabilities: readonly string[];
+    readonly requiresExternalCli?: boolean;
     /** Default authorization tags when the server config supplies none. */
     readonly defaultTags: readonly string[];
     /** Content type reported on source descriptions, e.g. `"chat"`. */
@@ -44,6 +45,8 @@ export interface ConnectorSkillDefinition {
 }
 export interface ConnectorSkillOptions {
     readonly connector: DatasourceConnector;
+    /** Config alias used as the independent datasource/skill identity. */
+    readonly skillName?: string;
     readonly instanceId?: string;
     readonly instances?: readonly string[];
     /** Poll interval in ms. Default 15 minutes; `0` disables polling. */

package/dist/datasource/connector.d.ts

@@ -36,6 +36,10 @@ export interface ConnectorDocument {
 export interface ConnectorFetchOk {
     readonly ok: true;
     readonly documents: readonly ConnectorDocument[];
+    /** True when the completed datasource snapshot changed since the last fetch. */
+    readonly changed?: boolean;
+    /** Stable ids removed from an incremental datasource snapshot. */
+    readonly deletedDocIds?: readonly string[];
     /** Sanitized, path/PII-opaque warning strings. */
     readonly warnings?: readonly string[];
 }

package/dist/datasource/crawler-skill.d.ts

@@ -16,11 +16,15 @@ export interface CrawlerDatasourceDefinition {
 }
 export interface CrawlerDatasourceSkillOptions {
     readonly client: CrawlerSkillClient;
+    readonly datasourceId?: string;
     readonly instanceId?: string;
     readonly instances?: readonly string[];
     readonly pollingIntervalMs?: number;
     readonly tags?: readonly string[];
     readonly lastIndexedAt?: number;
+    /** Omitted/empty means search every chat/channel. */
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
 }
 export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     private readonly definition;
@@ -29,6 +33,8 @@ export declare class CrawlerDatasourceSkill implements DatasourceSkill {
     private readonly instances;
     private readonly pollingIntervalMs;
     private readonly tags;
+    private readonly channelIds;
+    private readonly channelNames;
     private lastIndexedAt;
     constructor(definition: CrawlerDatasourceDefinition, options: CrawlerDatasourceSkillOptions);
     describe(): DatasourceSkillDescriptor;

package/dist/datasource/index.d.ts

@@ -1,4 +1,5 @@
 export { DatasourceAccessContext, type DatasourceAccessContextOptions } from "./access-context.js";
+export { type AliasDatasourceSkillOptions, AliasedDatasourceSkill } from "./aliased-skill.js";
 export { DatasourceChunkStore, type DatasourceChunkStoreOptions, datasourceChunkStorePath, type ScoredChunk, type StoredChunk, } from "./chunk-store.js";
 export { boundDiagnosticText, type ConnectorDocument, type ConnectorFailureReason, type ConnectorFetchFail, type ConnectorFetchOk, type ConnectorFetchResult, connectorFailureToDiagnosticCode, type DatasourceConnector, sanitizeIdSegment, } from "./connector.js";
 export { ConnectorDatasourceSkill, ConnectorLexicalMethod, type ConnectorSkillDefinition, type ConnectorSkillOptions, } from "./connector-skill.js";
@@ -7,6 +8,7 @@ export { type CronParseResult, isDue, parseCronExpr } from "./polling.js";
 export { DatasourceSkillRegistry, type RegisteredDatasourceSkill } from "./registry.js";
 export { DatasourceResultFilter, type ResultsByMethod } from "./result-filter.js";
 export { buildDatasourceChunkSource, buildDatasourceInstanceSource, DATASOURCE_CHUNKS_SEGMENT, datasourceSourceHasFragment, datasourceSourcePath, isDatasourceSource, matchesDatasourceScope, matchesVirtualPathScope, normalizeVirtualPath, } from "./scope.js";
+export { CLOUD_DRIVE_SKILL_DEFINITION, CloudDriveSkill, type CloudDriveSkillOptions, } from "./skills/cloud-drive/index.js";
 export { DEFAULT_DISCRAWL_EMBEDDING_MODEL, DEFAULT_DISCRAWL_MODE, DEFAULT_DISCRAWL_SOURCE, DiscrawlClient, type DiscrawlFailureReason, DiscrawlFtsMethod, DiscrawlHybridMethod, type DiscrawlOptions, type DiscrawlSearchHit, type DiscrawlSearchMode, type DiscrawlSearchOptions, type DiscrawlSearchResult, DiscrawlSemanticMethod, DiscrawlSkill, type DiscrawlSkillClient, type DiscrawlSkillOptions, type DiscrawlSourceKind, ENGLISH_ONLY_EMBEDDING_MODELS, } from "./skills/discrawl/index.js";
 export { BUILTIN_DATASOURCE_SKILL_NAMES, type BuildDatasourceSkillsResult, buildDatasourceSkills, type DatasourceSkillConfig, type DatasourcesConfig, } from "./skills/factory.js";
 export { GDriveConnector, type GDriveConnectorOptions, GDriveSkill, type GDriveSkillOptions, } from "./skills/gdrive/index.js";

package/dist/datasource/skills/cloud-drive/index.d.ts

@@ -0,0 +1 @@
+export { CLOUD_DRIVE_SKILL_DEFINITION, CloudDriveSkill, type CloudDriveSkillOptions, } from "./skill.js";

package/dist/datasource/skills/cloud-drive/skill.d.ts

@@ -0,0 +1,14 @@
+import type { DatasourceConnector } from "../../connector.js";
+import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
+import { type RcloneConnectorOptions } from "../gdrive/rclone-connector.js";
+export declare const CLOUD_DRIVE_SKILL_DEFINITION: ConnectorSkillDefinition;
+export interface CloudDriveSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
+    /** Unique model-visible datasource name for this configured connection. */
+    readonly skillName?: string;
+    readonly provider?: "google-drive" | "onedrive" | "icloud" | string;
+    readonly connector?: DatasourceConnector;
+    readonly connectorOptions?: RcloneConnectorOptions;
+}
+export declare class CloudDriveSkill extends ConnectorDatasourceSkill {
+    constructor(options?: CloudDriveSkillOptions);
+}

package/dist/datasource/skills/discrawl/skill.d.ts

@@ -9,6 +9,7 @@ export interface DiscrawlSkillClient extends DiscrawlSearchClient {
 }
 export interface DiscrawlSkillOptions {
     readonly client: DiscrawlSkillClient;
+    readonly datasourceId?: string;
     readonly instanceId?: string;
     readonly instances?: readonly string[];
     readonly pollingIntervalMs?: number;
@@ -18,6 +19,8 @@ export interface DiscrawlSkillOptions {
     readonly defaultMode?: DiscrawlSearchMode;
     /** Max messages embedded per index pass. Undefined drains the whole backlog. */
     readonly embedLimit?: number;
+    readonly channelIds?: readonly string[];
+    readonly channelNames?: readonly string[];
 }
 /**
  * Discord datasource skill backed by the external `discrawl` CLI.

package/dist/datasource/skills/factory.d.ts

@@ -11,11 +11,18 @@ import type { DatasourceSkill } from "../types.js";
 /** One configured datasource entry (the trusted `datasources.<name>` value). */
 export interface DatasourceSkillConfig {
     readonly enabled?: boolean;
+    /** Built-in datasource template used when the config key is a connection alias. */
+    readonly type?: string;
     readonly instanceId?: string;
     readonly pollingIntervalMs?: number;
     readonly tags?: readonly string[];
     /** Connector-specific options (token env names, repos, feeds, paths, …). */
     readonly connector?: Record<string, unknown>;
+    /** Optional chat/channel allowlist; absent means all channels. */
+    readonly channels?: {
+        readonly ids?: readonly string[];
+        readonly names?: readonly string[];
+    };
 }
 /** The trusted `datasources` config section: skill name → config. */
 export type DatasourcesConfig = Readonly<Record<string, DatasourceSkillConfig | boolean>>;

package/dist/datasource/skills/gdrive/rclone-connector.d.ts

@@ -1,15 +1,10 @@
 /**
- * Rclone-backed drive connector (issue #1301, rclone path).
+ * Provider-neutral incremental rclone mirror.
  *
- * Uses the external `rclone` CLI (https://rclone.org) as the trusted bridge
- * to Google Drive — or any of rclone's 70+ storage backends — so OAuth
- * setup and token refresh live entirely in `rclone config`, not here
- * (katok/himalaya pattern). Google Docs/Sheets are transparently exported
- * as text via rclone's `--drive-export-formats`. Never throws; failures map
- * onto the coarse connector failure union with path/PII-opaque messages.
- *
- * Listing: `rclone lsjson <remote> --recursive --files-only`
- * Content: `rclone cat <remote>/<path>` (bounded, text formats only)
+ * `lsjson` inventories a remote, a workspace manifest computes its diff, and
+ * only added/changed indexable files are copied into a managed mirror. The
+ * manifest is published only after every changed file is available, so search
+ * can continue using the previous completed snapshot after an interrupted run.
  */
 import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
 export interface RcloneRunResult {
@@ -20,27 +15,22 @@ export interface RcloneRunResult {
 }
 export type RcloneRunner = (args: readonly string[], timeoutMs: number) => Promise<RcloneRunResult>;
 export interface RcloneConnectorOptions {
-    /** Path to the rclone binary. Default bare `rclone` PATH lookup. */
     readonly binaryPath?: string;
-    /**
-     * Rclone remote path, e.g. `gdrive:` or `gdrive:Team/Docs`. Required
-     * trusted configuration (`rclone config` defines the remote).
-     */
     readonly remote?: string;
-    /**
-     * Indexable file extensions (lowercase, with dot). Google Docs/Sheets
-     * surface as these after `--drive-export-formats txt,csv`.
-     */
+    readonly provider?: string;
+    readonly workspaceRoot?: string;
+    readonly instanceId?: string;
+    readonly skillName?: string;
     readonly extensions?: readonly string[];
-    /** Drive export formats forwarded to rclone. Default `txt,csv`. */
+    readonly include?: readonly string[];
+    readonly exclude?: readonly string[];
     readonly exportFormats?: string;
-    /** Max files whose contents are fetched per run. Default 200. */
     readonly maxDocuments?: number;
-    /** Files larger than this are skipped. Default 2 MiB. */
     readonly maxBytesPerFile?: number;
-    /** Per-spawn timeout. Default 60s (cloud listings can be slow). */
+    readonly concurrency?: number;
+    readonly bandwidthLimit?: string;
+    readonly dryRun?: boolean;
     readonly timeoutMs?: number;
-    /** Injectable process runner for tests. */
     readonly runner?: RcloneRunner;
 }
 export declare class RcloneConnector implements DatasourceConnector {
@@ -48,4 +38,6 @@ export declare class RcloneConnector implements DatasourceConnector {
     private readonly runner;
     constructor(options?: RcloneConnectorOptions);
     fetch(): Promise<ConnectorFetchResult>;
+    private fetchIncremental;
+    private fetchWithoutMirror;
 }

package/dist/datasource/skills/obsidian/skill.d.ts

@@ -8,6 +8,7 @@ export interface ObsidianSkillClient extends ObsidianSearchClient {
     embed(): Promise<QmdEmbedResult>;
 }
 export interface ObsidianSkillOptions {
+    readonly datasourceId?: string;
     readonly client?: ObsidianSkillClient;
     readonly vaultPath?: string;
     readonly binaryPath?: string;

package/dist/dupey/cli.d.ts

@@ -0,0 +1,40 @@
+export interface DupeyScanFile {
+    readonly path: string;
+    readonly content_hash?: string;
+    readonly [key: string]: unknown;
+}
+export interface DupeyScanEdge {
+    readonly relation: string;
+    readonly a: string;
+    readonly b: string;
+    readonly [key: string]: unknown;
+}
+export interface DupeyScanFamily {
+    readonly id: number;
+    readonly relation: string;
+    readonly files: readonly string[];
+    readonly members: readonly Record<string, unknown>[];
+    readonly edges: readonly DupeyScanEdge[];
+    readonly pick?: Record<string, unknown>;
+    readonly [key: string]: unknown;
+}
+export interface DupeyScanResult {
+    readonly dir: string;
+    readonly threshold?: number;
+    readonly contains_threshold?: number;
+    readonly contains_min_jaccard?: number;
+    readonly files: readonly DupeyScanFile[];
+    readonly families: readonly DupeyScanFamily[];
+    readonly errors: readonly Record<string, unknown>[];
+    readonly [key: string]: unknown;
+}
+export interface DupeyCliOptions {
+    readonly executable?: string;
+    readonly cwd?: string;
+    readonly timeoutMs?: number;
+    readonly run?: (args: readonly string[]) => Promise<string>;
+}
+export declare class DupeyCliError extends Error {
+    constructor(message: string);
+}
+export declare function scanWithDupey(dir: string, options?: DupeyCliOptions): Promise<DupeyScanResult>;

package/dist/dupey/filter.d.ts

@@ -0,0 +1,8 @@
+import type { DupeyScanResult } from "./cli.js";
+export interface ExactDuplicateFilterResult {
+    readonly excluded: ReadonlySet<string>;
+    readonly keepers: ReadonlySet<string>;
+    readonly errors: readonly string[];
+}
+/** Selects the newest filesystem copy for each exact canonical-text hash. */
+export declare function selectExactDuplicateExclusions(root: string, scan: DupeyScanResult): Promise<ExactDuplicateFilterResult>;

package/dist/dupey/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./cli.js";
+export * from "./filter.js";

package/dist/index.d.ts

@@ -2,6 +2,7 @@ export { AutoRAGAgent, type AutoRAGAgentOptions, type AutoRAGRefreshComponentSta
 export { type AutoRAGResultsDetails, createEmitResultsTool, EMIT_AUTORAG_RESULTS_TOOL_NAME, type SearchDocumentDiagnostic, type SearchDocumentDiagnosticCode, type SearchDocumentDiagnosticSeverity, type SearchDocumentsResponse, type SearchDocumentWarning, } from "./agent/index.js";
 export { buildSystemPrompt, type SystemPromptConfig } from "./agent/system-prompt.js";
 export * from "./datasource/index.js";
+export * from "./dupey/index.js";
 export * from "./jikji/index.js";
 export * from "./manifest/index.js";
 export * from "./memory/index.js";