@autorag/librarian 2.11.0 → 2.13.0

package/dist/retrieval/chunk-location.d.ts

@@ -0,0 +1,28 @@
+/** Where a retrieved chunk sits inside its parsed document. Every field is measured, never model-written. */
+export interface ChunkLocation {
+    /** 1-based first line of the chunk in the parsed document. */
+    readonly lineStart: number;
+    /** 1-based last line of the chunk in the parsed document. */
+    readonly lineEnd: number;
+    /** Nearest article or heading line at or before the chunk's first line. */
+    readonly section?: string;
+}
+/**
+ * Find `chunk` in `document` by its non-blank lines (compared trimmed, so
+ * indentation and blank-line differences between the chunker's output and the
+ * file do not matter) and report the 1-based line range plus the section it
+ * sits in.
+ *
+ * Returns undefined unless the chunk matches exactly one place: a chunk the
+ * chunker rewrote (table flattening) or one that occurs twice has no single
+ * correct line, and reporting a guess would be worse than reporting nothing.
+ */
+export declare function locateChunk(document: string, chunk: string): ChunkLocation | undefined;
+/** One stable text form of a location, for tool output the model reads. Undefined when nothing is known. */
+export declare function describeLocation(location: {
+    readonly lineStart?: number;
+    readonly lineEnd?: number;
+    readonly section?: string;
+}): string | undefined;
+/** ` (lines 7-8, section "…")` for a retrieval result whose backend measured a location; empty otherwise. */
+export declare function locationSuffix(metadata: Readonly<Record<string, unknown>>): string;

package/dist/retrieval/fallback.d.ts

@@ -0,0 +1,24 @@
+import type { RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult } from "./types.js";
+/**
+ * One logical retrieval method that prefers its primary and degrades to a
+ * fallback only when the primary cannot run.
+ *
+ * Datasources used to register a hybrid method next to its lexical and semantic
+ * components and let the merger reconcile them. That ran the same backend
+ * several times and let the merger count a hybrid and its own component as
+ * independent corroboration. Registering one chain instead means the pipeline
+ * sees a single method per datasource: the hybrid when it works, a component
+ * only when it does not.
+ *
+ * The chain reports the primary's descriptor, so selection, diagnostics and
+ * surface grouping behave as if the primary were registered alone. A primary
+ * that answers with no hits is an answer, not a failure: the fallback is not
+ * consulted, because a lexical-only retry cannot find what a hybrid missed.
+ */
+export declare class FallbackRetrievalMethod implements RetrievalMethod {
+    private readonly primary;
+    private readonly fallbacks;
+    constructor(primary: RetrievalMethod, fallbacks: readonly RetrievalMethod[]);
+    describe(): RetrievalMethodDescriptor;
+    retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
+}

package/dist/retrieval/index.d.ts

@@ -1,5 +1,6 @@
 export { bm25Scores, tokenize } from "./bm25.js";
 export { RetrievalEngine, type RetrievalEngineOptions } from "./engine.js";
+export { FallbackRetrievalMethod } from "./fallback.js";
 export type { MergeOptions } from "./merger.js";
 export { ParallelRetriever, ResultMerger } from "./merger.js";
 export { RetrievalMethodRegistry } from "./registry.js";

package/dist/retrieval/rerank.d.ts

@@ -33,6 +33,12 @@ export declare const DEFAULT_DAZZI_RERANK_MODEL = "qwen/qwen3-reranker-8b";
 export declare const DEFAULT_RERANK_TIMEOUT_MS = 60000;
 /** Default number of merged chunks kept after reranking. */
 export declare const DEFAULT_RERANK_TOP_N = 25;
+/**
+ * Longest document, in UTF-16 units, sent to a reranker. The hosted Dazzi
+ * reranker rejects a document of 131,072 characters or more with HTTP 400 and
+ * fails the whole request; 131,000 stays under that with a small margin.
+ */
+export declare const RERANK_MAX_DOCUMENT_CHARS = 131000;
 /** Rerank providers {@link createReranker} can build. */
 export declare const SUPPORTED_RERANK_PROVIDERS: readonly string[];
 /** Default wire model for a rerank provider. */

package/dist/retrieval/types.d.ts

@@ -14,11 +14,27 @@ export interface RetrievalMethodDescriptor {
     datasourceId?: string;
     tags?: readonly string[];
 }
+/**
+ * A fallback chain answered with a fallback because its primary method failed. `reason` is the primary's
+ * error verbatim, so the degraded surface can be reported like any other skipped method.
+ */
+export interface RetrievalFallbackEvent {
+    /** Name of the primary method that failed. */
+    method: string;
+    /** Name of the fallback that answered instead. */
+    fallback: string;
+    reason: string;
+}
 export interface RetrievalOptions {
     topK?: number;
     scope?: string;
     filters?: Record<string, unknown>;
     signal?: AbortSignal;
+    /**
+     * Owned by the retrieval pipeline, not by callers: invoked when a method answered through a fallback.
+     * Methods that wrap another method must not forward it.
+     */
+    onFallback?: (event: RetrievalFallbackEvent) => void;
 }
 export interface RetrievalMethod {
     describe(): RetrievalMethodDescriptor;

package/internal-skills/organize/SKILL.md

@@ -0,0 +1,43 @@
+---
+name: organize
+description: Rules for acting on files and folders on disk — sort, move, copy, bulk-rename, rename by content, convert formats, create folder structures, or move to the trash. Plan every target path before changing anything, never overwrite or permanently delete, confirm with a listing, and report every change as old path → new path.
+license: Apache-2.0
+---
+
+# Organizing files and folders
+
+Use this skill when the user asks you to act on files or folders — tidy or sort
+a folder, move or copy files into folders, bulk or pattern rename, rename files
+according to what their content says, convert formats, create folder
+structures, archive, or move files to the trash. This is a file-organization
+task, not a document search: the answer is the change on disk, not information
+found in the files.
+
+## How to do it
+
+- Work with `bash` (`ls`, `mkdir`, `mv`, `cp`, and a bounded `find`). Read a
+  file (with `read` or `bash`) only when its content decides the new name or
+  folder, then carry the change out with `bash` or the file tools.
+- Use the converter tools for format conversions, the duplicate scan when the
+  request is about duplicates, and the file-name discovery tools to locate
+  files by name.
+- Do **not** search document contents, the user's datasources, or the web in
+  this turn: content retrieval and the web tools are not part of this task. The
+  turn prompt lists the exact tools to use and the ones to leave alone.
+
+## Rules
+
+- **Plan first.** List what you found, decide every target path, and check that
+  no two targets share a name and that no target already exists before you move
+  anything. Never overwrite an existing file.
+- **Touch only what the request names.** When the request does not name a
+  folder, ask which one instead of guessing, unless the conversation already
+  says.
+- **Never delete permanently** — no `rm`, `rmdir`, `unlink`, `find -delete`,
+  `shred`, or PowerShell `Remove-Item`. Move what must go to the OS trash
+  instead (macOS `/usr/bin/trash`, Linux `gio trash`, Windows Recycle Bin).
+- **Confirm afterwards.** List the destination folder to check the changes
+  really happened.
+- **Finish with a report**, as your final message and with no tool call: every
+  change as old path → new path (one per line), what you skipped and why, and
+  anything that still needs the user. Write in the language of the request.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.11.0",
+  "version": "2.13.0",
   "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {
@@ -18,6 +18,10 @@
     "./core": {
       "types": "./dist/core.d.ts",
       "default": "./dist/core.js"
+    },
+    "./profiles": {
+      "types": "./dist/profiles.d.ts",
+      "default": "./dist/profiles.js"
     }
   },
   "bin": {
@@ -27,6 +31,7 @@
   "files": [
     "dist",
     "skills",
+    "internal-skills",
     "README.md",
     "NOTICE",
     "licenses",
@@ -36,7 +41,7 @@
     "access": "public"
   },
   "scripts": {
-    "build": "bun build src/index.ts src/core.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
+    "build": "bun build src/index.ts src/core.ts src/profiles.ts src/cli/index.ts src/mcp/index.ts src/parser/parse-worker.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
     "prepublishOnly": "bun run typecheck && bun run build",
     "prepare": "bun run build",
     "test": "node scripts/run-test-files.mjs",
@@ -56,6 +61,7 @@
     "@earendil-works/pi-tui": "1.2.0",
     "@llblab/pi-telegram": "0.57.2",
     "@modelcontextprotocol/server": "^2.3.1",
+    "@parcel/watcher": "2.6.0",
     "chardet": "^2.2.0",
     "discord.js": "14.27.0",
     "fast-xml-parser": "^5.11.2",
@@ -65,6 +71,7 @@
     "kordoc": "^4.21.30",
     "mailparser": "^3.9.38",
     "pdfjs-dist": "^6.4.299",
+    "pi-ask-user": "0.17.0",
     "pi-discord-remote": "0.2.4",
     "puppeteer-core": "^25.13.0",
     "slides-grab": "^1.5.2",

package/skills/autorag/SKILL.md

@@ -45,6 +45,34 @@ near/contains families require human review. Exact duplicate exclusion during
 refresh is enabled by default and can be disabled with
 `"excludeExactDuplicates": false` in `config.json`.
 
+### Converting files
+
+The `convert_files` and `list_file_conversions` Agent tools run the
+[Filee](https://github.com/KnifeLemon/Filee) command line to convert local
+files offline: documents, spreadsheets, slides, HWP/HWPX, images, e-books,
+archives, audio, video, data, and subtitles. They work on any path the user
+names, not only the configured roots. This is the one exception to the
+"never write source files" rule above: a conversion writes a new file and
+never touches its source. An existing output name gets a number added unless
+the user asks to replace it, in which case pass `overwrite: true`. Never
+delete the source unless the user asked.
+
+Each converted file is checked with `dupey compare` against its source, using
+dupey's own thresholds, so the report says whether a later scan will group
+them: `exact` (same canonical text, for example DOCX to HWPX), `near` (same
+document in another format, for example Markdown to TXT), `different` (text
+differs, typically layout formats such as PDF), `unsupported` (dupey cannot
+read the format, for example HTML or images), or `unavailable` (dupey not
+installed). The command line is `filee.binaryPath` if set, else a `filee`/`filee-cli`
+on `PATH`, else the pinned [filee-cli](https://github.com/NomaDamas/filee-cli)
+release that `autorag setup` (or the first conversion) downloads into
+`<AUTORAG_HOME>/tools/filee-cli/`; `"filee": { "autoInstall": false }` forbids that
+download. The system prompt carries the conversion map (which format family
+converts to which, and the two-step chain when no direct route exists), so plan
+from it instead of probing. `"filee": { "enabled": false }` in `config.json` removes both
+tools; `filee.timeoutMs` (default 30 minutes) and `filee.dataDir` (Filee's
+own settings and engines, `FILEE_DATA_DIR`) tune them.
+
 ### Excluding local files from the index
 
 `"excludePaths"` in `config.json` lists files or folders (absolute, or relative
@@ -83,7 +111,10 @@ autorag search "what were the key findings in the Q3 report" --top-k 5 --json --
 only `answer` plus `results`. Every
 bracketed `[n]` citation in `answer` resolves to a `results[].number` of the
 same response; an unmatched citation is removed and reported as a
-`citation-without-result` diagnostic.
+`citation-without-result` diagnostic. Within one conversation (an interactive
+session, including a resumed one), a source keeps the `number` it was first
+shown with and a new source gets the next unused number, so `results[].number`
+can skip numbers and need not start at 1 on a later turn.
 
 To inspect the exact persisted evidence behind numbered results, use:
 
@@ -92,14 +123,21 @@ autorag evidence <sessionId> --result 1 --json
 ```
 
 The response includes the original source, retrieval method, stable evidence
-ID, raw excerpt/content, and any available `chunkIndex`, `lineNumber`,
-`retrievalResultId`, and metadata. Omit `--result` to inspect every result in
+ID, raw excerpt/content, and any available `chunkIndex`, `lineNumber` and
+`lineEnd` (the chunk's line range in the parsed document), `section` (the
+article or heading it sits in), `retrievalResultId`, and metadata. Omit `--result` to inspect every result in
 the session. Prefer this command whenever the caller wants detailed chunk text
 rather than only the curated summary.
 
 - `--scope` narrows datasource retrieval to a requested sub-path (ordinary
   per-query filtering).
 - `--json` is required for programmatic consumption.
+- Each search result carries `locations` (`lineStart`, `lineEnd`, `section`)
+  when the harness measured where its evidence sits in the parsed document. Quote
+  those, not line numbers or counts written in the answer prose; a
+  `line-reference-unverified` diagnostic (with `--debug`) flags prose line
+  numbers no cited evidence covers. Locations are absent for chunks the chunker
+  rewrote (such as flattened tables).
 - `--debug` is required for `sessionId` and diagnostics in search output.
 - `autorag evidence` is the detailed source/chunk inspection path.
 
@@ -116,12 +154,22 @@ default, OpenRouter), Jev first routes the question:
   datasources) is handled by the agent itself: it loads the setup skill, edits
   the active config, verifies it, and reports what changed. The running agent
   keeps its startup model; changes apply to the next `autorag` launch.
-- Private-data questions use local search; public current facts use web search.
+- A request to act on files or folders (move, bulk rename, rename by content,
+  convert, sort) is carried out in one tool turn with no retrieval and no
+  verification, and reports every change as old path → new path.
+- Anything else that needs no retrieval (commit, push, open a PR, run a build)
+  is carried out in one tool turn the same way.
+- Private-data questions use local search. Questions about public facts that may
+  be recent or have changed can use the public web: Jev picks the web in the
+  same check as the registered connections, one web search runs per search
+  query before the fast answer, and the fast phase may also call `web_search`
+  itself when the baseline lacks the answer.
 - A multi-part question is split into up to five parallel search queries.
 - On local search, Jev also picks which registered datasources (Slack,
-  Discord, KakaoTalk, email, ...) to search before the fast answer, from each
-  datasource's description and where similar past questions were answered;
-  their chunks are reranked together with local file evidence.
+  Discord, KakaoTalk, email, ...) to search before the fast answer — and
+  whether to search the public web — from each description and where similar
+  past questions were answered; their hits are reranked together with local
+  file evidence.
 
 After the fast answer, Jev ends the run if the answer is complete and
 evidence-backed. So `results` may come straight from the fast answer, with no
@@ -145,8 +193,42 @@ autorag index rebuild --yes --json
 autorag index reset --method parsed --yes --json
 autorag memory inspect --json
 autorag tui
+autorag automate status --json
+autorag automate install --json
+autorag automate uninstall --json
+autorag automate run invoice --event create:/path/to/file.pdf --json
+autorag automate daemon
 ```
 
+File-triggered automations live in `~/.autorag/automations.json` (`automations`
+array of `{ id, paths, events, include, exclude, run | agent, appendFiles, debounceMs, timeoutMs }`).
+Edit `automations.json` directly to add or change rules; matched `[{ type, path }]`
+events are piped on stdin to `run`. Use `autorag automate install` to register
+the background daemon with launchd, `systemd --user`, or Windows Task Scheduler.
+
+A rule runs exactly one of `run` (a command) or `agent` (an AI agent started in
+the daemon on each batch of changes):
+
+```json
+{ "id": "inbox-triage", "paths": ["~/Downloads"], "include": ["**/*.pdf"],
+  "agent": { "profile": "file-triage", "task": "Summarize each new PDF.",
+             "retrieval": { "searchAll": true, "datasources": ["kakao"] } } }
+```
+
+`agent.profile` names a profile defined in code: `file-triage` (reads, lists and
+searches files; changes nothing) or `file-organizer` (also `bash`, `edit`,
+`write`). A rule never carries a prompt or a tool list, so the rules file cannot
+grant an agent more than its profile has. `retrieval` optionally adds
+`search_all_documents` and one `search_datasource_<id>` per listed datasource.
+Retrieval memory is never connected to an automation agent.
+
+Every agent run gets a loop guard appended to its system prompt, naming the
+rule's folders, file types and events, and forbidding any change that would
+fire the rule again, editing the rules file, and running `autorag automate`. At
+most one agent run is in flight across all rules, and an agent rule is
+disabled after 6 runs within 60 seconds (command rules: 20). The run's final
+reply is the `stdout` of its log entry.
+
 Prefer a full refresh so parsed mirrors, MinSync, Jikji, configured
 datasources, and (on Windows) the bundled Everything file-name index stay
 aligned. `--method` accepts

package/skills/autorag-setup/SKILL.md

@@ -201,7 +201,7 @@ resolves in order:
 
 1. A configured remote endpoint (`minSync.embedder`), used for everything; no
    daemon is started and no local model is loaded.
-2. A running `minsync serve` daemon (loopback only; MinSync 0.4.11+).
+2. A running `minsync serve` daemon (loopback only; MinSync 0.4.12+).
    AutoRAG never starts it: run `minsync serve --idle-timeout 0` yourself, as
    a launchd/systemd service (see `docs/minsync-setup.md`), or press
    Ctrl+Shift+E in `autorag tui`. `autorag setup` only reports whether it is
@@ -239,9 +239,13 @@ fast and multi-part questions thorough.
 - **Jev** (`jev`, default `{ "backend": "dazzi" }` when signed in to Dazzi
   Models, else `{ "backend": "openrouter" }`; model `jev-1.13` on Dazzi,
   `typesafe/jev-1.13` on OpenRouter) runs before the fast answer. It routes each question to
-  local search, web search, a direct answer (general knowledge or small
-  talk skips retrieval entirely), or the `config` branch, and decides whether
-  to decompose it. On
+  local search (everything needing more information),
+  `organize` (moving, renaming, converting, or
+  sorting files and folders), `extra` (a knowledge answer, or any other action that needs no
+  retrieval, such as committing code: one turn with every tool), or the `config`
+  branch, and decides
+  whether to decompose it. A non-local branch only wins above a 0.75 confidence
+  floor; anything weaker stays on local search. On
   local search it also decides, per registered datasource, whether to search
   it before the fast answer, using each datasource's `description` and where
   similar past questions were answered (retrieval memory). When setting up a
@@ -540,6 +544,8 @@ is set up.
 - `AUTORAG_MEMORY_PATH`
 - `AUTORAG_MODEL_PROVIDER`
 - `AUTORAG_MODEL_ID`
+- `AUTORAG_PERMISSION_MODE` (`auto` default for the TUI, `full` for `autorag search` and a TUI with a chat bridge, `ask`, `full`; config key `permissionMode`; with `auto` Jev checks the agent's
+  file-changing calls and asks the user only when it will not approve, `ask` asks about every one, `full` never asks)
 
 ## Completion condition