@autorag/librarian 2.0.0 → 2.2.0

package/dist/datasource/skills/obsidian/skill.d.ts

@@ -0,0 +1,43 @@
+import type { RetrievalMethod } from "../../../retrieval/types.js";
+import type { DatasourceIndexResult, DatasourceSkill, DatasourceSkillDescriptor, DatasourceSkillManifest, PollingMetadata, SourceDescription } from "../../types.js";
+import { type ObsidianSearchClient } from "./methods.js";
+import type { QmdEmbedResult, QmdEnsureResult, QmdUpdateResult } from "./types.js";
+export interface ObsidianSkillClient extends ObsidianSearchClient {
+    ensureCollection(): Promise<QmdEnsureResult>;
+    update(): Promise<QmdUpdateResult>;
+    embed(): Promise<QmdEmbedResult>;
+}
+export interface ObsidianSkillOptions {
+    readonly client?: ObsidianSkillClient;
+    readonly vaultPath?: string;
+    readonly binaryPath?: string;
+    readonly instanceId?: string;
+    readonly instances?: readonly string[];
+    readonly pollingIntervalMs?: number;
+    readonly tags?: readonly string[];
+    readonly lastIndexedAt?: number;
+    readonly workspaceRoot?: string;
+    readonly collectionName?: string;
+    readonly timeoutMs?: number;
+    readonly connectorOptions?: {
+        readonly vaultPath?: string;
+    };
+}
+export declare class ObsidianSkill implements DatasourceSkill {
+    private readonly client;
+    private readonly instanceId;
+    private readonly instances;
+    private readonly pollingIntervalMs;
+    private readonly tags;
+    private readonly vaultPath;
+    private lastIndexedAt;
+    private lastError;
+    constructor(options?: ObsidianSkillOptions);
+    describe(): DatasourceSkillDescriptor;
+    polling(): PollingMetadata;
+    index(): Promise<DatasourceIndexResult>;
+    retrievalMethods(): readonly RetrievalMethod[];
+    describeSources(): readonly SourceDescription[];
+    skillManifest(): DatasourceSkillManifest;
+    private fail;
+}

package/dist/datasource/skills/obsidian/types.d.ts

@@ -0,0 +1,90 @@
+import type { RetrievalOptions } from "../../../retrieval/types.js";
+/**
+ * Search modes accepted by the external `qmd` CLI.
+ *
+ * - `search`  — BM25 / FTS5 lexical search
+ * - `vsearch` — dense vector semantic search
+ */
+export type QmdSearchMode = "search" | "vsearch";
+/**
+ * Configuration for {@link QmdClient}. The client spawns the `qmd` binary as a
+ * child process and isolates config/cache under the AutoRAG workspace.
+ */
+export interface QmdOptions {
+    /** Explicit path to the `qmd` binary. Defaults to a bare `qmd` PATH lookup. */
+    readonly binaryPath?: string;
+    /** Spawn timeout in milliseconds. Default 60_000 (embed can be slow). */
+    readonly timeoutMs?: number;
+    /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
+    readonly maxBufferBytes?: number;
+    /** Absolute path to the Obsidian vault. Required for ensure/update. */
+    readonly vaultPath?: string;
+    /** AutoRAG workspace root used to isolate qmd config/cache. */
+    readonly workspaceRoot?: string;
+    /** Instance id used for source paths and collection naming. Default `default`. */
+    readonly instanceId?: string;
+    /** Explicit collection name; defaults to a sanitized instance id. */
+    readonly collectionName?: string;
+    /** Environment overrides merged on top of a restricted process env. */
+    readonly env?: Readonly<Record<string, string | undefined>>;
+}
+export declare const DEFAULT_QMD_BINARY = "qmd";
+export declare const DEFAULT_QMD_TIMEOUT_MS = 60000;
+export declare const DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
+export type QmdFailureReason = "binary-missing" | "not-configured" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
+export interface QmdFailure {
+    readonly ok: false;
+    readonly reason: QmdFailureReason;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number | null;
+}
+export interface QmdOk<T> {
+    readonly ok: true;
+    readonly data: T;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+}
+export interface QmdUpdateInfo {
+    readonly indexed: number;
+    readonly updated: number;
+    readonly unchanged: number;
+    readonly removed: number;
+    readonly needsEmbedding?: boolean;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface QmdEmbedInfo {
+    readonly embedded: boolean;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface QmdEnsureInfo {
+    readonly collectionName: string;
+    readonly vaultPath: string;
+    readonly configDir: string;
+}
+/** A scored search hit from `qmd search` / `qmd vsearch`. */
+export interface QmdSearchHit {
+    readonly chunkId: string;
+    readonly score: number;
+    readonly content: string;
+    readonly title?: string;
+    readonly file?: string;
+    readonly docid?: string;
+    readonly metadata?: Readonly<Record<string, unknown>>;
+}
+export interface QmdSearchOk {
+    readonly ok: true;
+    readonly hits: readonly QmdSearchHit[];
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number;
+    readonly data: {
+        readonly hits: readonly QmdSearchHit[];
+    };
+}
+export type QmdEnsureResult = QmdOk<QmdEnsureInfo> | QmdFailure;
+export type QmdUpdateResult = QmdOk<QmdUpdateInfo> | QmdFailure;
+export type QmdEmbedResult = QmdOk<QmdEmbedInfo> | QmdFailure;
+export type QmdSearchResult = QmdSearchOk | QmdFailure;
+export type QmdSearchOptions = RetrievalOptions;

package/dist/datasource/skills/rss/connector.d.ts

@@ -0,0 +1,28 @@
+/**
+ * RSS/news connector (issue #1316).
+ *
+ * Fetches RSS 2.0 / Atom feeds from a trusted, server-configured feed list
+ * using the existing `fast-xml-parser` dependency. Feeds are referenced by
+ * index in warnings — never by URL — so diagnostics stay opaque. Per-feed
+ * failures degrade to warnings; only all feeds failing fails the fetch.
+ */
+import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
+export interface RssFeedConfig {
+    readonly url: string;
+    /** Optional category segment inserted into the hierarchy. */
+    readonly category?: string;
+}
+export interface RssConnectorOptions {
+    /** Feeds to poll. Required trusted configuration. */
+    readonly feeds?: readonly RssFeedConfig[];
+    readonly timeoutMs?: number;
+    readonly fetchImpl?: typeof fetch;
+    readonly maxDocuments?: number;
+    readonly maxItemsPerFeed?: number;
+}
+export declare class RssConnector implements DatasourceConnector {
+    private readonly options;
+    private readonly parser;
+    constructor(options?: RssConnectorOptions);
+    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
+}

package/dist/datasource/skills/rss/index.d.ts

@@ -0,0 +1,2 @@
+export { RssConnector, type RssConnectorOptions, type RssFeedConfig } from "./connector.js";
+export { RSS_SKILL_DEFINITION, RssSkill, type RssSkillOptions } from "./skill.js";

package/dist/datasource/skills/rss/skill.d.ts

@@ -0,0 +1,22 @@
+/**
+ * RSS/news datasource skill (issue #1316).
+ *
+ * Thin composition of RssConnector with the shared
+ * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
+ * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
+ * the shared pipeline, opaque `/rss/<instance>/…` sources, and a
+ * progressive-disclosure agent-skill manifest.
+ */
+import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
+import { RssConnector, type RssConnectorOptions } from "./connector.js";
+export declare const RSS_SKILL_DEFINITION: ConnectorSkillDefinition;
+export interface RssSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
+    /** Trusted connector configuration; a pre-built connector wins. */
+    readonly connector?: RssConnector;
+    readonly connectorOptions?: RssConnectorOptions;
+    /** Dedupe window for re-delivered feed items. Default 24 hours. */
+    readonly dedupeWindowMs?: number;
+}
+export declare class RssSkill extends ConnectorDatasourceSkill {
+    constructor(options?: RssSkillOptions);
+}

package/dist/datasource/skills/slack/index.d.ts

@@ -0,0 +1,15 @@
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface SlacrawlOptions extends CrawlerCliOptions {
+}
+export declare class SlacrawlClient extends CrawlerCliClient {
+    constructor(options?: SlacrawlOptions);
+}
+export interface SlackSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: SlacrawlOptions;
+}
+export declare class SlackSkill extends CrawlerDatasourceSkill {
+    constructor(options?: SlackSkillOptions);
+}

package/dist/datasource/skills/spotlight/connector.d.ts

@@ -0,0 +1,59 @@
+/**
+ * macOS Spotlight connector (issue #1350).
+ *
+ * Runs the built-in `mdfind` CLI for trusted, server-configured Spotlight
+ * queries and hydrates text content for chunking. No external installs —
+ * Spotlight ships with macOS. **macOS only**: on any other platform the
+ * connector reports `unavailable` before spawning anything.
+ *
+ * Full Disk Access: Spotlight returns hits for protected locations (Mail,
+ * Messages, Safari data, parts of ~/Library) but reading those files fails
+ * with EPERM/EACCES unless the process running AutoRAG (the terminal app or
+ * host binary) has Full Disk Access under System Settings -> Privacy &
+ * Security. When every content read is denied this maps to the `permission`
+ * failure reason with opaque FDA guidance; partial denials surface as
+ * count-only warnings. Full absolute paths are exposed in document metadata
+ * (`path`) so results stay traceable back to the file on disk — privacy is
+ * the operator's responsibility (run AutoRAG with a local LLM if paths must
+ * not leave the machine).
+ */
+import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
+export interface ProcessRunResult {
+    readonly ok: boolean;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly code: number | null;
+    /** Spawn error code, e.g. `"ENOENT"` when the binary is missing. */
+    readonly error?: string;
+}
+/** Process execution seam; injectable for tests. Never throws. */
+export type ProcessRunner = (command: string, args: readonly string[], timeoutMs: number) => Promise<ProcessRunResult>;
+export type FileReader = (path: string) => Promise<string>;
+export interface SpotlightConnectorOptions {
+    /** Trusted Spotlight query strings passed to `mdfind`. Required. */
+    readonly queries?: readonly string[];
+    /** Trusted directory restricting every query (`mdfind -onlyin`). */
+    readonly onlyIn?: string;
+    /** Hits kept per query after dedupe. Default 100. */
+    readonly maxResultsPerQuery?: number;
+    /** Total documents hydrated per fetch. Default 300. */
+    readonly maxDocuments?: number;
+    /** Files larger than this are skipped. Default 256 KiB. */
+    readonly maxBytesPerFile?: number;
+    /** Per-process timeout in ms. Default 15s. */
+    readonly timeoutMs?: number;
+    /** Injectable platform; defaults to `process.platform`. */
+    readonly platform?: NodeJS.Platform;
+    /** Injectable process runner; defaults to a bounded spawn wrapper. */
+    readonly run?: ProcessRunner;
+    /** Injectable file reader; defaults to utf8 `fs.readFile`. */
+    readonly readFile?: FileReader;
+}
+export declare class SpotlightConnector implements DatasourceConnector {
+    private readonly options;
+    private readonly run;
+    private readonly readFile;
+    private readonly platform;
+    constructor(options?: SpotlightConnectorOptions);
+    fetch(): Promise<ConnectorFetchResult>;
+}

package/dist/datasource/skills/spotlight/index.d.ts

@@ -0,0 +1,2 @@
+export { type FileReader, type ProcessRunner, type ProcessRunResult, SpotlightConnector, type SpotlightConnectorOptions, } from "./connector.js";
+export { SPOTLIGHT_SKILL_DEFINITION, SpotlightSkill, type SpotlightSkillOptions } from "./skill.js";

package/dist/datasource/skills/spotlight/skill.d.ts

@@ -0,0 +1,20 @@
+/**
+ * macOS Spotlight datasource skill (issue #1350).
+ *
+ * Thin composition of {@link SpotlightConnector} with the shared
+ * {@link ConnectorDatasourceSkill} base. The skill is **macOS only** and
+ * needs no external installs — it drives the built-in `mdfind` CLI. The
+ * manifest carries operator guidance for mdfind/mdls/mdutil and the Full
+ * Disk Access requirement.
+ */
+import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
+import { SpotlightConnector, type SpotlightConnectorOptions } from "./connector.js";
+export declare const SPOTLIGHT_SKILL_DEFINITION: ConnectorSkillDefinition;
+export interface SpotlightSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
+    /** Trusted connector configuration; a pre-built connector wins. */
+    readonly connector?: SpotlightConnector;
+    readonly connectorOptions?: SpotlightConnectorOptions;
+}
+export declare class SpotlightSkill extends ConnectorDatasourceSkill {
+    constructor(options?: SpotlightSkillOptions);
+}

package/dist/datasource/skills/telecrawl/index.d.ts

@@ -0,0 +1,15 @@
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface TelecrawlOptions extends CrawlerCliOptions {
+}
+export declare class TelecrawlClient extends CrawlerCliClient {
+    constructor(options?: TelecrawlOptions);
+}
+export interface TelecrawlSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: TelecrawlOptions;
+}
+export declare class TelecrawlSkill extends CrawlerDatasourceSkill {
+    constructor(options?: TelecrawlSkillOptions);
+}

package/dist/datasource/skills/wacrawl/index.d.ts

@@ -0,0 +1,15 @@
+import { CrawlerCliClient } from "../../crawler-client.js";
+import { CrawlerDatasourceSkill, type CrawlerDatasourceSkillOptions, type CrawlerSkillClient } from "../../crawler-skill.js";
+import type { CrawlerCliOptions } from "../../crawler-types.js";
+export interface WacrawlOptions extends CrawlerCliOptions {
+}
+export declare class WacrawlClient extends CrawlerCliClient {
+    constructor(options?: WacrawlOptions);
+}
+export interface WacrawlSkillOptions extends Omit<CrawlerDatasourceSkillOptions, "client"> {
+    readonly client?: CrawlerSkillClient;
+    readonly connectorOptions?: WacrawlOptions;
+}
+export declare class WacrawlSkill extends CrawlerDatasourceSkill {
+    constructor(options?: WacrawlSkillOptions);
+}