@autorag/librarian 2.4.1 → 2.5.0

package/skills/autorag-setup/SKILL.md

@@ -136,11 +136,21 @@ defaults to true; requires the Rust toolchain).
 
 Exact duplicate exclusion is enabled by default. AutoRAG invokes the external
 `dupey` CLI before parsed-mirror indexing, keeps the newest filesystem copy for
-each exact canonical-text hash, and excludes older copies from the mirror. Set
-`"excludeExactDuplicates": false` to index every copy. Missing dupey is
-non-fatal; refresh continues without this optimization.
-
-Optional MinSync embedder settings:
+each exact canonical-text hash, and excludes older copies from the mirror.
+Install dupey during setup when it is missing (`command -v dupey || cargo
+install dupey`) and tell the user the feature is available; when installation
+is impossible, refresh continues without this optimization and the user is
+told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
+index every copy.
+
+MinSync's default embedder is local EmbeddingGemma (768 dimensions, served
+locally via Ollama). The default needs no embedder flags and no API key, and
+no corpus text is sent to a remote embedding service. During setup, verify the
+local model is available (`ollama pull embeddinggemma` with `ollama serve`
+running) instead of silently falling back.
+
+Override the embedder only when intentionally using a different, for example
+remote, provider:
 
 ```bash
 autorag init \
@@ -152,18 +162,36 @@ autorag init \
 ```
 
 Only store the environment-variable name, never its value. Dimension and batch
-size must be positive integers.
-
-## Configure datasource skills when requested
-
-Prefer `autorag ui --no-open` to connect datasources. It writes the same trusted
-`datasources` / `datasourceAccess` fields as a hand-edited config, stores
-env-var names rather than secrets, and prints a loopback URL (`127.0.0.1`).
-Do not bind non-loopback hosts unless the user explicitly set `ui.allowRemote`.
+size must be positive integers, and the dimension must match the embedder
+(EmbeddingGemma is 768; text-embedding-3-small is 1536).
+
+## Probe and configure datasource skills (setup wizard)
+
+`autorag ui` is still in development — do not recommend it for datasource
+setup. Configure datasources directly in trusted config, wizard-style:
+
+1. Probe every datasource for setup feasibility before asking the user
+   anything: the backing CLI exists (`katok`, `discrawl`, `slacrawl`,
+   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`), its
+   local store or archive is present, and any credentials it needs are
+   available as environment variables or in the tool's own external
+   configuration.
+2. Auto-configure every datasource that probes feasible — write its trusted
+   `datasources` / `datasourceAccess` entries without asking. For example,
+   when Slack (`slacrawl`) and Discord (`discrawl`) are installed, set both up
+   automatically.
+3. Skip every datasource that probes infeasible (for example Notion or
+   Telegram when their CLIs are not installed) and always report the skipped
+   list to the user, with what is missing for each.
+4. Set up a skipped datasource only when the user explicitly asks for it:
+   install or authenticate the backing CLI first, then configure it.
+5. E-mail datasources (`mail-export`, `mailcrawl`) matter to most
+   users — always probe them and report their status, even when they end up
+   skipped.
 
 Datasource skills belong in trusted config and remain default-deny. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
-`clawgallery`, `notion`, `github`, `cloud-drive`, `gmail`, `mail-export`,
+`clawgallery`, `notion`, `github`, `cloud-drive`, `mail-export`,
 `mailcrawl`, `obsidian`, `rss`, and `spotlight`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
@@ -175,14 +203,13 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
     "github": { "connector": { "repos": ["owner/repo"], "tokenEnv": "GITHUB_TOKEN" } },
     "google-drive": { "type": "cloud-drive", "connector": { "provider": "google-drive", "remote": "gdrive:" } },
     "archive-drive": { "type": "cloud-drive", "connector": { "remote": "archive:" } },
-    "gmail": { "connector": { "tokenEnv": "GMAIL_ACCESS_TOKEN", "labelIds": ["INBOX"] } },
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
     "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
     "rss": { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
   },
   "datasourceAccess": {
-    "allowedTags": ["github", "cloud-drive", "gmail", "mailcrawl", "obsidian", "rss"],
-    "allowedScopes": ["/github/**", "/google-drive/**", "/archive-drive/**", "/gmail/**", "/mailcrawl/**", "/obsidian/**", "/rss/**"]
+    "allowedTags": ["github", "cloud-drive", "mailcrawl", "obsidian", "rss"],
+    "allowedScopes": ["/github/**", "/google-drive/**", "/archive-drive/**", "/mailcrawl/**", "/obsidian/**", "/rss/**"]
   }
 }
 ```
@@ -190,14 +217,13 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
 Tokens are environment-variable names, not raw secrets. CLI-backed connectors
 keep authentication in their external tool configuration.
 
-Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.1.4` or newer)
+Mailcrawl must be installed separately (`@nomadamas/mailcrawl@0.1.6` or newer)
 and configured through its own Himalaya account. AutoRAG runs its local `sync`
 and `index` lifecycle, then uses the mailcrawl CLI for BM25, semantic, or
 hybrid search. Do not use 0.1.3 or earlier: a no-op sync followed by `index`
 fails with `text array must be non-empty`.
-Use mailcrawl for Himalaya-backed IMAP/Maildir retrieval. The legacy
-`gmail` connector option `backend: "himalaya"` is no longer registered; migrate
-that configuration to an explicit `mailcrawl` datasource.
+Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
+datasource is removed.
 
 ## Verify and build indexes
 
@@ -228,10 +254,12 @@ autorag search "summarize the collection" --top-k 3 --json --debug
 ## Keep indexes fresh
 
 For continuous freshness, create or verify an OS-appropriate scheduled
-`autorag watch --once` job, typically every 15–30 minutes. Prefer cron or
-launchd on macOS, cron or a user systemd timer on Linux, and Task Scheduler on
-Windows. Use the same config as search, avoid overlapping runs, and keep logs
-outside source trees.
+`autorag watch --once` job, hourly by default (every 1 hour; shorten only when
+the user asks for fresher indexes). Prefer cron or launchd on macOS, cron or a
+user systemd timer on Linux, and Task Scheduler on Windows. Use the same
+config as search, avoid overlapping runs, and keep logs outside source trees.
+Once the schedule is installed, tell the user right away that hourly freshness
+is set up.
 
 ## Environment overrides
 
@@ -246,6 +274,9 @@ outside source trees.
 ## Completion condition
 
 Setup is complete only when the CLI is installed, roots are approved, a
-non-secret single-model config is written, `status` is acceptable, live `health`
+non-secret single-model config is written, every datasource has been probed
+and the auto-configured and skipped lists reported to the user, dupey is
+installed or its absence reported, `status` is acceptable, live `health`
 passes, `refresh` builds the requested indexes, one real structured search
-succeeds, and any requested ongoing schedule is installed or verified.
+succeeds, and any requested ongoing schedule is installed or verified with the
+user told it is active.

package/dist/datasource/skills/gmail/connector.d.ts

@@ -1,30 +0,0 @@
-/**
- * Gmail connector (issue #1304).
- *
- * Lists and fetches messages through the Gmail REST v1 API with a trusted,
- * server-configured OAuth access token. Per-message fetch failures degrade
- * to warnings. Never throws; messages stay path/PII-opaque (addresses appear
- * only inside indexed chunk content, never in diagnostics).
- */
-import type { ConnectorFetchResult, DatasourceConnector } from "../../connector.js";
-export interface GmailConnectorOptions {
-    /** Gmail REST base; override to point at a mock server in tests. */
-    readonly baseUrl?: string;
-    /** OAuth access token; explicit value wins over {@link tokenEnv}. */
-    readonly token?: string;
-    /** Env var name holding the access token. Default `GMAIL_ACCESS_TOKEN`. */
-    readonly tokenEnv?: string;
-    /** Restrict to these label ids (e.g. `["INBOX"]`). */
-    readonly labelIds?: readonly string[];
-    /** Gmail search query (the `q` parameter). */
-    readonly query?: string;
-    readonly timeoutMs?: number;
-    readonly fetchImpl?: typeof fetch;
-    readonly maxPages?: number;
-    readonly maxDocuments?: number;
-}
-export declare class GmailConnector implements DatasourceConnector {
-    private readonly options;
-    constructor(options?: GmailConnectorOptions);
-    fetch(signal?: AbortSignal): Promise<ConnectorFetchResult>;
-}

package/dist/datasource/skills/gmail/index.d.ts

@@ -1,2 +0,0 @@
-export { GmailConnector, type GmailConnectorOptions } from "./connector.js";
-export { GMAIL_SKILL_DEFINITION, GmailSkill, type GmailSkillOptions } from "./skill.js";

package/dist/datasource/skills/gmail/skill.d.ts

@@ -1,24 +0,0 @@
-/**
- * Gmail datasource skill (issue #1304).
- *
- * Thin composition of GmailConnector with the shared
- * {@link ConnectorDatasourceSkill} base: descriptor, polling metadata,
- * ok/fail indexing with path/PII-opaque diagnostics, lexical retrieval via
- * the shared pipeline, opaque `/gmail/<instance>/…` sources, and a
- * progressive-disclosure agent-skill manifest.
- */
-import type { DatasourceConnector } from "../../connector.js";
-import { ConnectorDatasourceSkill, type ConnectorSkillDefinition, type ConnectorSkillOptions } from "../../connector-skill.js";
-import { type GmailConnectorOptions } from "./connector.js";
-export declare const GMAIL_SKILL_DEFINITION: ConnectorSkillDefinition;
-export interface GmailSkillOptions extends Omit<ConnectorSkillOptions, "connector"> {
-    /**
-     * Trusted pre-built connector; wins over {@link connectorOptions}. Accepts
-     * any compatible mail-backed connector.
-     */
-    readonly connector?: DatasourceConnector;
-    readonly connectorOptions?: GmailConnectorOptions;
-}
-export declare class GmailSkill extends ConnectorDatasourceSkill {
-    constructor(options?: GmailSkillOptions);
-}