@autorag/librarian 2.8.0 → 2.10.0

package/skills/autorag-lite-setup/SKILL.md

@@ -14,7 +14,7 @@ Use `autorag-setup` instead when the model-backed librarian must be configured.
 ## Safety
 
 - Inspect only non-secret config metadata: `searchPaths`, `workspacePath`,
-  `memoryPath`, `minSync`, `jikji`, `datasources`, `datasourceAccess`.
+  `memoryPath`, `minSync`, `jikji`, `datasources`.
 - Never print, copy, migrate, compare, or persist credential values. Store only
   environment-variable names such as `tokenEnv` or `apiKeyEnv`.
 - Never move, rename, edit, or delete source documents. Lite commands write
@@ -98,7 +98,7 @@ integrated datasources add their own search tools; do not assume a fixed count.
 Run `autorag.status` and `autorag.datasources.list` through MCP, then refresh
 and search a known phrase from an approved document. Registration alone is
 not proof of a connected or searchable server. After config changes, restart
-the server so datasource tools and authorization are rebuilt.
+the server so datasource tools are rebuilt.
 
 `AUTORAG_MCP_READ_ONLY=1` omits refresh; build indexes with the CLI before
 connecting that mode. `AUTORAG_MCP_TOOLS` is an optional comma-separated exact
@@ -120,7 +120,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
    (such as `github`) require their credential environment variable
    (`GITHUB_TOKEN`).
 2. Auto-configure every datasource that probes feasible — write its trusted
-   `datasources` / `datasourceAccess` entries without asking. For example,
+   `datasources` entries without asking. For example,
    when Slack (`slacrawl`) and Discord (`discrawl`) are installed with local
    stores present, set both up automatically. Discord uses discrawl's local
    desktop wiretap archive; no Discord bot token is configured or needed.
@@ -133,15 +133,17 @@ setup. Configure datasources directly in trusted config, wizard-style:
    users — always probe them and report their status, even when they end up
    skipped.
 
-Datasource skills belong in trusted config and remain default-deny.
+Datasource skills belong in trusted config.
 
 Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
 `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
 `mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`) or connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
-`datasourceAccess.allowedTags` and `allowedScopes` narrow trusted access and
-can never grant it. Store only env-var names such as `tokenEnv` or
+`scope` narrows a query to a sub-path as ordinary filtering. Tags are
+descriptive metadata only, not search filters. Use MCP `datasourceIds` to
+select configured connections before retrieval; discover their IDs with
+`autorag.datasources.list`. Store only env-var names such as `tokenEnv` or
 `apiKeyEnv`, never credential values.
 
 ## Build and refresh indexes
@@ -167,7 +169,7 @@ AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite refresh --jso
   are missing or broken, run a full refresh or return to setup rather than
   silently degrading to lexical-only search.
 - MinSync's default embedder is in-process native Qwen3 embeddings
-  (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.5+): no
+  (`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.6): no
   embedder flags, no API key, and no external daemon (such as Ollama) are
   needed, and no corpus text leaves the machine. For gateway-profiled
   embeddings, prefetch with `autorag models prefetch --profile qwen3-embedding-0.6b`.
@@ -224,23 +226,24 @@ AUTORAG_CONFIG=/absolute/path/to/.autorag/config.json autorag lite index reset -
 Inspect MCP `isError`, `errorCode`, `diagnostics`, and `unsearched` before
 trusting results. `ok: true` with skipped surfaces is partial coverage, not
 proof that the whole corpus was searched. Preserve the underlying error text
-when reporting unavailable components such as `minsync-unavailable` or
-`retrieval-method-failed`. Repair the component rather than silently accepting
-degraded search.
+when reporting unavailable components such as `retrieval-method-failed`.
+MinSync is required: when its binary is missing the call fails with an
+`MinSync is required ...` error rather than returning partial results, so
+install it (`cargo install minsync`) instead of working around it. Repair the
+component rather than silently accepting degraded search.
 
 ## Normal MCP workflow and curation lifecycle
 
 MCP results carry source, method, and content directly. After a search, the
 optional curation lifecycle is also exposed through MCP: `autorag.report`
-records a curated answer, `autorag.evidence` returns the exact chunks behind
-numbered results, and `autorag.feedback` records which numbers were useful or
-not. These accept and return JSON matching the CLI's report/evidence/feedback
-contracts; discover the exact schemas with MCP `tools/list` rather than
-restating them here. Every bracketed `[n]` citation in a report `answer` must
-be a `results[].number`; an unmatched citation is removed from the persisted
-answer and returned as a `citation-without-result` diagnostic. The CLI
-`autorag report`, `autorag evidence`, and `autorag feedback` commands remain
-available for terminal maintenance.
+records a curated answer, and `autorag.evidence` returns the exact chunks behind
+numbered results. These accept and return JSON matching the CLI's
+report/evidence contracts; discover the exact schemas with MCP `tools/list`
+rather than restating them here. Every bracketed `[n]` citation in a report
+`answer` must be a `results[].number`; an unmatched citation is removed from
+the persisted answer and returned as a `citation-without-result` diagnostic.
+The CLI `autorag report` and `autorag evidence` commands remain available for
+terminal maintenance.
 
 ## Completion condition
 

package/skills/autorag-setup/SKILL.md

@@ -46,7 +46,7 @@ Check `--config`, `AUTORAG_CONFIG`, `$AUTORAG_HOME/config.json`, or
 - `model.provider`, `model.id`, `model.api`, `model.baseUrl`, `model.apiKeyEnv`
 - `bm25`, `minSync`, and `jikji`
 - `limits` (retrieval, baseline-prefetch, and model-facing candidate caps)
-- `datasources`, `datasourceAccess`, and `ui`
+- `datasources` and `ui`
 
 Preserve explicit user choices and a working config unless the user asks to
 replace them or health checks fail.
@@ -131,7 +131,7 @@ For a custom endpoint, add `api`, `baseUrl`, and `apiKeyEnv` to the single
   "searchPaths": ["/path/to/documents"],
   "model": {
     "provider": "openrouter",
-    "id": "anthropic/claude-sonnet-4",
+    "id": "anthropic/claude-sonnet-5.5",
     "api": "openai-completions",
     "baseUrl": "https://openrouter.ai/api/v1",
     "apiKeyEnv": "OPENROUTER_API_KEY"
@@ -184,7 +184,7 @@ told duplicate exclusion is off. Set `"excludeExactDuplicates": false` to
 index every copy.
 
 MinSync's default embedder is in-process native Qwen3 embeddings
-(`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.5+). The
+(`native:Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, MinSync 0.4.6). The
 default needs no embedder flags, no API key, and no external daemon (such as
 Ollama) — all embeddings run in-process locally and privately. For workspaces
 using the loopback llama-server gateway, prefetch the verified model with
@@ -200,16 +200,16 @@ remote, provider:
 
 ```bash
 autorag init \
-  --embedder-id "text-embedding-3-small" \
-  --embedder-base-url "https://api.openai.com/v1" \
-  --embedder-api-key-env "OPENAI_API_KEY" \
-  --embedder-dimension 1536 \
+  --embedder-id "voyageai/voyage-4-lite" \
+  --embedder-base-url "https://openrouter.ai/api/v1" \
+  --embedder-api-key-env "OPENROUTER_API_KEY" \
+  --embedder-dimension 1024 \
   --embedder-batch-size 64
 ```
 
 Only store the environment-variable name, never its value. Dimension and batch
 size must be positive integers, and the dimension must match the embedder
-(default Qwen3 is 1024; legacy EmbeddingGemma is 768; text-embedding-3-small is 1536).
+(default Qwen3 is 1024; legacy EmbeddingGemma is 768; `voyageai/voyage-4-lite` via OpenRouter is 1024).
 
 ### Jev routing and question decomposition (on by default)
 
@@ -218,10 +218,19 @@ fast and multi-part questions thorough.
 
 - **Jev** (`jev`, default `{ "backend": "openrouter" }`, model
   `typesafe/jev-1.13`) runs before the fast answer. It routes each question to
-  local search, web search, or a direct answer (general knowledge or small
-  talk skips retrieval entirely), and decides whether to decompose it. After
-  the fast answer it decides whether verification is needed, so a complete,
-  evidence-backed fast answer ends the run.
+  local search, web search, a direct answer (general knowledge or small
+  talk skips retrieval entirely), or the `config` branch, and decides whether
+  to decompose it. On
+  local search it also decides, per registered datasource, whether to search
+  it before the fast answer, using each datasource's `description` and where
+  similar past questions were answered (retrieval memory). When setting up a
+  datasource, always write a `description` from what it actually holds:
+  channels or rooms, people, topics, time range (for example `"Team Slack,
+  2024-2026: #release and #on-call channels, dependabot notifications"`).
+  Jev is told descriptions are short, non-exhaustive summaries, so list the
+  main content and do not try to list everything. After the fast answer it
+  decides whether verification is needed, so a complete, evidence-backed fast
+  answer ends the run.
 - **Question decomposition** (`queryDecomposition`, default model
   `openrouter/qwen/qwen3.7-flash`) splits a multi-part question into at most
   five search queries that run in parallel.
@@ -250,6 +259,14 @@ always verifies, so searches still work. A `query-route-fallback` diagnostic
   explicitly opts out, for example because questions must never leave the
   machine (Jev and decomposition send the question text to OpenRouter).
 
+When a user asks the running agent itself to change its settings (switch the
+default model, add a provider, check that a provider works), Jev's `config`
+branch loads this whole skill into that turn. The agent edits only the active
+config file (and `models.json` for a custom provider), verifies with
+`autorag health --json` and `autorag models list --available`, and reports each
+change as old → new in its final message. It never prints a
+credential value and never uses `init --force`.
+
 ### Retrieval and ingest caps
 
 `limits` bounds retrieval, baseline prefetch, and the candidate lists handed to
@@ -313,7 +330,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
    (such as `github`) require their credential environment variable
    (`GITHUB_TOKEN`).
 2. Auto-configure every datasource that probes feasible — write its trusted
-   `datasources` / `datasourceAccess` entries without asking. For example,
+   `datasources` entries without asking. For example,
    when Slack (`slacrawl`) and Discord (`discrawl`) are installed with local
    stores present, set both up automatically. Discord uses discrawl's local
    desktop wiretap archive; no Discord bot token is configured or needed.
@@ -326,13 +343,13 @@ setup. Configure datasources directly in trusted config, wizard-style:
    users — always probe them and report their status, even when they end up
    skipped.
 
-Datasource skills belong in trusted config and remain default-deny. Builtin
+Datasource skills belong in trusted config. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
 `clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
 `mailcrawl`, `obsidian`, `rss`, `spotlight`, and `lark`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
-`scope` and tags can narrow trusted access but cannot grant it.
+`scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.
 
 ```jsonc
 {
@@ -344,10 +361,6 @@ aliases with `"type": "<template>"`. Unknown names are skipped with an
     "mailcrawl": { "instanceId": "personal", "connector": { "account": "personal", "mailbox": "INBOX", "binaryPath": "mailcrawl" } },
     "obsidian": { "connector": { "vaultPath": "/path/to/vault" } },
     "rss": { "connector": { "feeds": [{ "url": "https://example.com/feed.xml" }] } }
-  },
-  "datasourceAccess": {
-    "allowedTags": ["github", "cloud-drive", "mailcrawl", "obsidian", "rss"],
-    "allowedScopes": ["/github/**", "/google-drive/**", "/archive-drive/**", "/mailcrawl/**", "/obsidian/**", "/rss/**"]
   }
 }
 ```
@@ -383,7 +396,7 @@ autorag search "summarize the collection" --top-k 3 --json --debug
   performs one live completion probe.
 - `health --skip-probes` is only for intentionally offline validation and does
   not prove live provider access.
-- `refresh` syncs parsed mirrors, MinSync, Jikji, authorized datasources, and
+- `refresh` syncs parsed mirrors, MinSync, Jikji, configured datasources, and
   on Windows the bundled Everything file-name index. `--method <csv>` may
   deliberately narrow it (`parsed,minsync,datasources,jikji,everything,all`).
 - Use `refresh --force` for a full resync only when incremental refresh is not
@@ -404,7 +417,7 @@ reload/reconnect, discover schemas with `tools/list`, and exercise
 `autorag.search` through MCP. Restart the MCP server after config changes.
 The MCP server returns model-free source chunks; the calling agent curates
 them. It does not invoke the configured librarian model. The same server also
-exposes `autorag.report`, `autorag.evidence`, and `autorag.feedback` for the
+exposes `autorag.report` and `autorag.evidence` for the
 curation lifecycle; the matching CLI commands remain a maintenance path.
 Discover the exact schemas with MCP `tools/list`. Keep the full `autorag` skill
 only when model-backed curated search is also wanted. For MCP-only setup, use

package/dist/agent/fast-answer-tool.d.ts

@@ -1,46 +0,0 @@
-import type { AgentTool } from "@earendil-works/pi-agent-core";
-import { Type } from "typebox";
-export declare const EMIT_FAST_ANSWER_TOOL_NAME = "emit_fast_answer";
-declare const fastAnswerSchema: Type.TObject<{
-    answer: Type.TString;
-    results: Type.TArray<Type.TObject<{
-        number: Type.TInteger;
-        title: Type.TString;
-        summary: Type.TString;
-        evidence: Type.TOptional<Type.TArray<Type.TObject<{
-            excerpt: Type.TString;
-            lineNumber: Type.TOptional<Type.TInteger>;
-        }>>>;
-        confidence: Type.TOptional<Type.TNumber>;
-    }>>;
-    sources: Type.TOptional<Type.TArray<Type.TObject<{
-        number: Type.TInteger;
-        source: Type.TString;
-    }>>>;
-}>;
-export interface AutoRAGFastAnswerResult {
-    readonly number: number;
-    readonly title: string;
-    readonly summary: string;
-    readonly evidence: readonly {
-        readonly excerpt: string;
-        readonly lineNumber?: number;
-    }[];
-    readonly confidence?: number;
-}
-export interface AutoRAGFastAnswerDetails {
-    readonly answer: string;
-    readonly results: readonly AutoRAGFastAnswerResult[];
-    readonly sources: readonly {
-        readonly number: number;
-        readonly source: string;
-    }[];
-}
-/**
- * Builds the non-terminating fast-answer tool used by the two-phase search
- * flow. The model calls this exactly once during the thinking-off fast phase
- * to deliver an immediate, complete first answer; the run then continues into
- * the verification phase, which ends with emit_autorag_results.
- */
-export declare function createEmitFastAnswerTool(capture: (details: AutoRAGFastAnswerDetails) => void): AgentTool<typeof fastAnswerSchema, AutoRAGFastAnswerDetails>;
-export {};

package/dist/cli/commands/feedback.d.ts

@@ -1,8 +0,0 @@
-import type { CommandContext } from "./types.js";
-/**
- * Run the `autorag feedback` command. Records per-result useful/not-useful
- * signals against an existing curated-results session. Returns 0 when at least
- * one feedback signal was applied, 2 for usage errors or an unknown session,
- * 1 for runtime/IO errors.
- */
-export declare function runFeedback(ctx: CommandContext): Promise<number>;

package/dist/datasource/access-context.d.ts

@@ -1,43 +0,0 @@
-/**
- * Trusted, server-bound access context for the datasource layer.
- *
- * Constructed from server-supplied allow-tags and allow-scopes ONLY — never
- * from model or tool arguments. Tags authorize datasource skills; scopes are
- * applied only to skills that advertise the `scoped` capability.
- */
-import type { DatasourceAccessible } from "./types.js";
-export interface DatasourceAccessContextOptions {
-    /** Trusted allow-tags. Undefined or empty ⇒ default-deny (isDenyAll). */
-    readonly allowedTags?: readonly string[];
-    /** Trusted allow-scopes as slash-hierarchical paths or globs. */
-    readonly allowedScopes?: readonly string[];
-}
-/**
- * Access context for gating datasource descriptors and sources.
- *
- * Non-datasource descriptors (those without a `datasourceId`) are NOT gated
- * by this context: {@link isAccessible} returns `true` for them so plain
- * retrieval methods (e.g. `posix`) pass through unchanged.
- */
-export declare class DatasourceAccessContext {
-    readonly allowedTags: readonly string[];
-    readonly allowedScopes: readonly string[];
-    private readonly denyAll;
-    constructor(options?: DatasourceAccessContextOptions);
-    /** True when the context denies everything (no trusted allow-tags). */
-    get isDenyAll(): boolean;
-    /**
-     * Whether a descriptor is accessible.
-     *
-     * Non-datasource descriptors (no `datasourceId`) pass through (`true`) —
-     * they are not gated by this context. Datasource descriptors require at
-     * least one tag intersecting the trusted allow-tags.
-     */
-    isAccessible(descriptor: DatasourceAccessible): boolean;
-    /**
-     * Scope-capable datasource results are filtered by trusted and user scopes.
-     * Datasources without the `scoped` capability bypass this predicate and
-     * are authorized only at the skill/tag level.
-     */
-    allowedSourcesPredicate(_userScope?: string, userScopes?: readonly string[]): (source: string) => boolean;
-}

package/dist/datasource/result-filter.d.ts

@@ -1,64 +0,0 @@
-/**
- * Result filter that applies datasource access gating to the shared retrieval
- * result stream.
- *
- * The filter sits *after* {@link ParallelRetriever} and *before* merging: it
- * takes the per-method result map and the {@link RetrievalMethod}s that
- * produced it, then for every datasource-backed method either drops all of
- * its results (when the trusted access context denies the descriptor) or
- * keeps only the sources the context allows for the current user scope when
- * the method advertises the `scoped` capability.
- *
- * Non-datasource methods (those whose descriptor has no `datasourceId`) pass
- * through completely untouched — they are gated by their own retrieval
- * pipeline, not by the datasource access context.
- *
- * Security invariants:
- *  - Deny is always an explicit `false` (an empty result array), never
- *    `undefined`-as-deny. {@link DatasourceAccessContext} itself returns
- *    explicit booleans from its predicate; this filter never passes
- *    `undefined` to `matchesVirtualPathScope` as a deny signal.
- *  - The filter never grants access. It only narrows results using the
- *    trusted, server-supplied {@link DatasourceAccessContext}; model/tool
- *    arguments (such as `userScope`) can only *further restrict* the trusted
- *    allow-scopes, never widen them.
- */
-import type { RetrievalMethod, RetrievalResult } from "../retrieval/types.js";
-import type { DatasourceAccessContext } from "./access-context.js";
-/** Per-method retrieval results keyed by method name. */
-export type ResultsByMethod = Map<string, RetrievalResult[]>;
-/**
- * Applies datasource access gating to a per-method result map.
- *
- * The instance form lets callers wire a {@link DatasourceSkillRegistry} for
- * future descriptor enrichment; the v1 filter only needs the
- * {@link RetrievalMethodDescriptor} carried by each method, which already
- * satisfies {@link DatasourceAccessible} structurally (`datasourceId?` +
- * `tags?`).
- */
-export declare class DatasourceResultFilter {
-    /**
-     * Filter `byMethod` according to datasource access rules.
-     *
-     * For each method present in both `byMethod` and `methods`:
-     *  - non-datasource methods (`descriptor.datasourceId` unset) are passed
-     *    through with their result array unchanged;
-     *  - datasource methods whose descriptor is denied by `ctx` are reduced to
-     *    an empty result array (`[]` — explicit deny, never `undefined`);
-     *  - datasource methods whose descriptor is allowed are filtered so only
-     *    sources matching `ctx.allowedSourcesPredicate(userScope)` survive.
-     *
-     * Result entries for methods not present in `methods` are passed through
-     * unchanged (the filter cannot classify them and must not fabricate a
-     * descriptor). A new {@link ResultsByMethod} is returned; the input map is
-     * not mutated.
-     *
-     * @param byMethod   Per-method results from the retriever.
-     * @param methods    The {@link RetrievalMethod}s that produced them.
-     * @param ctx        Trusted, server-supplied access context.
-     * @param userScope  Optional user/model-requested scope; when provided it
-     *   is intersected with the trusted allow-scopes and can only narrow
-     *   results. `undefined` means "no extra user restriction".
-     */
-    filter(byMethod: ResultsByMethod, methods: readonly RetrievalMethod[], ctx: DatasourceAccessContext, userScope?: string, userScopes?: readonly string[]): ResultsByMethod;
-}

package/dist/parser/ocr.d.ts

@@ -1,31 +0,0 @@
-import { type ParseInput, type ParseOutput, Parser } from "./types.js";
-export interface OcrEngineInput {
-    readonly bytes: Uint8Array;
-    readonly languages: readonly string[];
-    readonly timeoutMs: number;
-    readonly signal: AbortSignal;
-}
-export type OcrEngine = (input: OcrEngineInput) => Promise<string>;
-export type OcrOperation = {
-    readonly result: Promise<string>;
-    readonly cleanup: Promise<void>;
-};
-export interface OcrParserOptions {
-    readonly enabled: boolean;
-    readonly languages?: readonly string[];
-    readonly timeoutMs?: number;
-    readonly maxBytes?: number;
-    readonly engine?: OcrEngine;
-}
-export declare class ImageOcrParser extends Parser {
-    readonly name = "image-ocr";
-    readonly extensions: readonly [".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".webp"];
-    private readonly languages;
-    private readonly timeoutMs;
-    private readonly maxBytes;
-    private readonly engine;
-    constructor(options: OcrParserOptions);
-    parse(input: ParseInput): Promise<ParseOutput>;
-}
-export declare function createTesseractOcrOperation(input: OcrEngineInput): OcrOperation;
-export declare function withTimeout(operation: OcrOperation, timeoutMs: number, onTimeout: () => void): Promise<string>;