@autorag/librarian 2.10.6 → 2.11.0

package/dist/minsync/workspace-lock.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Run a sync (anything that rewrites the workspace). The in-flight count is raised
+ * before this returns, so a query issued right after the call already sees the sync.
+ */
+export declare function runWorkspaceSync<T>(workspacePath: string, run: () => Promise<T>): Promise<T>;
+/** Run a query. It is refused while a sync is pending, because the index is mid-rewrite. */
+export declare function runWorkspaceQuery<T>(workspacePath: string, run: () => Promise<T>): Promise<T>;

package/dist/process/terminate-tree.d.ts

@@ -4,11 +4,10 @@ export declare const TREE_KILL_GRACE_MS = 500;
 /**
  * Terminate a spawned external CLI together with the descendants it created.
  *
- * CLIs such as qmd 2.8.3 are launchers: the process AutoRAG spawns starts a
- * runtime descendant that inherits the launcher's stdout/stderr pipes. Node
- * emits `close` only once every holder of those pipes has exited, so signalling
- * just the direct child leaves the caller waiting on a descendant that may live
- * for minutes — the hang reported in #1578.
+ * CLIs that are launchers (an npm shim, a wrapper script) start a runtime descendant that inherits
+ * the launcher's stdout/stderr pipes. Node emits `close` only once every holder of those pipes has
+ * exited, so signalling just the direct child leaves the caller waiting on a descendant that may
+ * live for minutes — the hang reported in #1578.
  *
  * Spawn the child with `detached: true` on POSIX so it leads its own process
  * group; the group survives the direct child and stays addressable as `-pid`.

package/licenses/hister-AGPL-3.0-notice.txt

@@ -0,0 +1,47 @@
+Hister — license notice for AutoRAG's `hister` datasource
+==========================================================
+
+Program:      Hister ("Your own search engine")
+Source:       https://github.com/asciimoo/hister
+Copyright:    asciimoo and Hister contributors (see the repository history)
+License:      GNU Affero General Public License, version 3 or (at your option)
+              any later version (AGPL-3.0-or-later)
+License text: https://www.gnu.org/licenses/agpl-3.0.txt
+              (also the LICENSE file in the Hister repository)
+
+How AutoRAG relates to Hister
+-----------------------------
+* AutoRAG does NOT contain, bundle, link, vendor, or redistribute any Hister
+  source code or binary. Nothing from the Hister repository is copied into
+  this package.
+* Users install Hister themselves, unmodified, from its own releases
+  (https://github.com/asciimoo/hister/releases, `brew install hister`, Docker,
+  or Nix), and run it as a separate process (`hister listen`).
+* AutoRAG talks to that process only over HTTP, using Hister's public JSON API
+  (`GET /search`, `GET /api/config`), the same API Hister's own `hister search`
+  command uses. Hister owns its index, its browser extension, its credentials,
+  and its optional embedding calls.
+* AutoRAG does not modify Hister, and it never hides that Hister is the backend:
+  the datasource manifest shown to the agent names Hister, its license, and its
+  source URL.
+* The `hister` datasource only reads web-page documents (`type:web`). It never
+  enables or uses Hister's local-directory indexing.
+
+Combining separate programs that communicate over a network is "mere
+aggregation" in the sense of the GNU licenses FAQ
+(https://www.gnu.org/licenses/gpl-faq.html#MereAggregation); AutoRAG therefore
+stays under its own license (Apache-2.0).
+
+If you run Hister as a network service for other people, AGPL-3.0 section 13
+applies to YOU as the operator of that Hister instance: you must offer those
+users the Corresponding Source of the Hister version you run (and of any
+modifications you made). Running an unmodified upstream release satisfies this
+by pointing users at https://github.com/asciimoo/hister.
+
+Obtaining the Hister source for a release you run
+-------------------------------------------------
+    git clone https://github.com/asciimoo/hister
+    git checkout <the version printed by `hister --version`>
+
+This file is a notice, not the license. The authoritative license terms are in
+the Hister repository and at the AGPL URL above.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.10.6",
-  "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
+  "version": "2.11.0",
+  "description": "Personal librarian agent for office work on the Pi framework, with pluggable retrieval methods",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",

package/skills/autorag/SKILL.md

@@ -15,10 +15,13 @@ search, calls MinSync, Jikji, datasource, and filesystem tools, reads
 sources, judges evidence, and curates the final answer. There is no subagent or
 separate model role.
 
-AutoRAG reads source documents and writes indexes only under the configured
+Indexing and refresh read source documents and write only under the configured
 workspace `.autorag/` directory and Jikji's central store (`<OS data dir>/jikji/`,
-plus a `.jikji_agent_map.md` guide in each indexed root). Never
-move, rename, edit, or delete source files.
+plus a `.jikji_agent_map.md` guide in each indexed root). The librarian may
+organize, create, or modify the user's own files in local sessions when the
+user asks; it never deletes permanently (OS trash only) and lists every change
+as old path → new path. Do not move, rename, edit, or delete source files on your own
+initiative while searching or diagnosing.
 
 ## Preflight
 

package/skills/autorag-doctor/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: autorag-doctor
-description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, qmd, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
+description: Diagnose and repair a broken or half-working AutoRAG install so every configured source is both indexed and searchable. Checks AutoRAG, MinSync, Jikji, the embedding backend, and every CLI-backed datasource (lazykatok, discrawl, slacrawl, wacrawl, telecrawl, notcrawl, mailcrawl, rclone, Spotlight), then fixes orphan locks, orphan processes, embedding dimension or identity mismatches, stale indexes, and missing setup. Use when search returns nothing or too little, refresh hangs or fails, a datasource disappeared from results, indexes look stale, the embedder is unavailable, or the user asks to check, diagnose, verify, or repair AutoRAG.
 license: Apache-2.0
 ---
 
@@ -15,9 +15,10 @@ Always finish with the status table in [Report](#report).
 
 ## Safety
 
-- Never delete, move, or edit source documents.
+- Diagnosis and repair never delete, move, or edit source documents; a doctor
+  run changes only AutoRAG-owned state.
 - Never delete a datasource's **native** store (`~/Library/Application Support/katok`, `~/.discrawl`,
-  `~/.mailcrawl`, `.qmd`, Telegram/WhatsApp/Notion archives). AutoRAG only
+  `~/.mailcrawl`, Telegram/WhatsApp/Notion archives). AutoRAG only
   reads them; rebuilding them is the owning CLI's job.
 - Only AutoRAG-owned state under `AUTORAG_HOME` and the workspace `.autorag`
   directory may be reset.
@@ -69,7 +70,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
 | WhatsApp | `wacrawl --json doctor` | `wacrawl import` |
 | Telegram | `telecrawl --json doctor`, `telecrawl --json status` | `telecrawl import` |
 | Notion | `notcrawl doctor`, `notcrawl status` | `notcrawl sync --source desktop` |
-| Obsidian / notes | `qmd status` | `qmd update && qmd embed` |
+| Obsidian / notes | `autorag status` (AutoRAG-owned index) | `autorag refresh --method datasources`; if corrupt, delete `.autorag/datasources/obsidian/<instance>` first |
 | Mail | `mailcrawl doctor`, `mailcrawl status` | `mailcrawl sync && mailcrawl index` |
 | Cloud drive | `rclone listremotes` | `autorag refresh --method datasources` |
 | Spotlight (macOS) | `mdutil -s /` | indexed by the OS; no AutoRAG sync |

package/skills/autorag-lite-setup/SKILL.md

@@ -17,9 +17,9 @@ Use `autorag-setup` instead when the model-backed librarian must be configured.
   `memoryPath`, `minSync`, `jikji`, `datasources`.
 - Never print, copy, migrate, compare, or persist credential values. Store only
   environment-variable names such as `tokenEnv` or `apiKeyEnv`.
-- Never move, rename, edit, or delete source documents. Lite commands write
-  indexes only under the configured workspace `.autorag/` directory and
-  Jikji's central store (`<OS data dir>/jikji/`).
+- Setup and Lite commands never move, rename, edit, or delete source documents.
+  They write indexes only under the configured workspace `.autorag/` directory
+  and Jikji's central store (`<OS data dir>/jikji/`).
 - Do not index system trees, app bundles, caches, credential stores,
   `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
   `.jikji`.
@@ -113,7 +113,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
 
 1. Probe every datasource for setup feasibility before asking the user
    anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
-   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`) and its
+   `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`) and its
    local store or archive is present. CLI-backed datasources own their own
    archive, index, and authentication, so environment credentials (such as bot
    tokens) are never required or checked for them. Non-CLI connectors
@@ -137,7 +137,7 @@ Datasource skills belong in trusted config.
 
 Config keys may be builtin template names (`kakao`, `whatsapp`, `telegram`,
 `slack`, `discord`, `clawgallery`, `notion`, `github`, `cloud-drive`,
-`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`) or connection
+`mail-export`, `mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, `hister`) or connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `scope` narrows a query to a sub-path as ordinary filtering. Tags are
@@ -146,6 +146,24 @@ select configured connections before retrieval; discover their IDs with
 `autorag.datasources.list`. Store only env-var names such as `tokenEnv` or
 `apiKeyEnv`, never credential values.
 
+### Browser history (Hister) and the shared embedding model
+
+`hister` needs a separately installed, always-running Hister server (AGPL-3.0, never
+bundled by AutoRAG). Run `autorag setup --format json` and read the `hister` object
+(`installed`, `running`, `embedding`); each problem carries the exact command to run:
+`brew install hister`, `brew services start hister`, then install the Hister browser
+extension in the user's browser.
+
+Hister embeds with its own `semantic_search` config, so it must use the same model as
+AutoRAG (the `minsync serve` daemon, `Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions) or
+its vectors stop lining up without any error. `autorag setup --sync-hister` copies the
+shared settings into Hister's `config.yml` (backup kept), restarts the service, waits
+for it, and runs `hister reindex` when needed; it is idempotent, and it refuses a
+remote or in-process backend. Re-run it after restarting the `minsync serve` daemon
+(its port changes). Verify with `hister doctor`: `server.index.embeddings` must be `ok`.
+Install a skip list in Hister's `rules.json` before importing past history, because the
+importer re-downloads URLs without the browser's cookies (docs/datasource-skills.md).
+
 ## Build and refresh indexes
 
 After connecting, call MCP `autorag.refresh` with `{}` for an incremental run.

package/skills/autorag-setup/SKILL.md

@@ -21,7 +21,8 @@ librarian, not the Lite MCP server.
 - Never print, copy, migrate, compare, or persist credential values. Store only
   environment-variable names such as `apiKeyEnv`.
 - Do not scan the whole filesystem or home directory without explicit approval.
-- Never move, rename, edit, or delete source documents.
+- Setup, indexing, and refresh never move, rename, edit, or delete source
+  documents. The librarian itself may organize files when the user asks it to.
 - Do not index system trees, app bundles, caches, credential stores,
   `node_modules`, `.git`, `dist`, `build`, `target`, `.cache`, `.autorag`, or
   `.jikji`.
@@ -361,7 +362,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
 
 1. Probe every datasource for setup feasibility before asking the user
    anything: the backing CLI exists (`lazykatok`, `discrawl`, `slacrawl`,
-   `wacrawl`, `telecrawl`, `notcrawl`, `qmd`, `mailcrawl`, `rclone`, `lark-cli`) and its
+   `wacrawl`, `telecrawl`, `notcrawl`, `mailcrawl`, `rclone`, `lark-cli`) and its
    local store or archive is present. CLI-backed datasources own their own
    archive, index, and authentication, so environment credentials (such as bot
    tokens) are never required or checked for them. Non-CLI connectors
@@ -384,7 +385,7 @@ setup. Configure datasources directly in trusted config, wizard-style:
 Datasource skills belong in trusted config. Builtin
 template names are `kakao`, `whatsapp`, `telegram`, `slack`, `discord`,
 `clawgallery`, `notion`, `github`, `github-gist`, `cloud-drive`, `mail-export`,
-`mailcrawl`, `obsidian`, `rss`, `spotlight`, and `lark`. Config keys may be connection
+`mailcrawl`, `obsidian`, `rss`, `spotlight`, `lark`, and `hister`. Config keys may be connection
 aliases with `"type": "<template>"`. Unknown names are skipped with an
 `unknown-datasource-skill` warning; they do not fail config resolution.
 `scope` narrows a query to a sub-path as ordinary filtering. Tags are descriptive metadata only, not search filters. MCP `datasourceIds` selects configured connections before retrieval; discover their IDs with `autorag.datasources.list`.
@@ -418,6 +419,62 @@ the next `index`. kakao needs `lazykatok` 0.3.3 or newer for the same reason.
 Use mailcrawl for Gmail, IMAP, and Maildir retrieval. The former Gmail REST
 datasource is removed.
 
+### Browser history (Hister) and the shared embedding model
+
+`hister` is the one datasource whose backing program AutoRAG does not install,
+start, or bundle (it is AGPL-3.0-or-later and stays a separate program). Setting
+it up is an agent task with these steps. Run each command; do not just describe it.
+
+1. **Probe first.** Add the datasource entry, then run `autorag setup --format json`
+   and read the `hister` object: `installed`, `running`, and `embedding`. Every
+   problem has a `remediation` string with the exact next command.
+2. **Install Hister** when `installed` is false: `brew install hister` (other
+   platforms: the release binaries at https://github.com/asciimoo/hister/releases).
+3. **Run it as a login item** when `running` is false: `brew services start hister`.
+   Hister must stay running: its browser extension only captures while it is up,
+   and a page visited while it is down is not re-sent. AutoRAG never starts it.
+4. **Install the browser extension** (Chrome Web Store "Hister", or Firefox
+   Add-ons). It must be done in the user's browser; tell the user if you cannot.
+5. **Share the embedding model** (below).
+6. **Before importing past history, install a skip list** in Hister's `rules.json`
+   (login, mail, documents, social and console domains, private hosts, token-bearing
+   URLs). `hister import browser history` re-downloads each URL *without* the
+   browser's cookies. See docs/datasource-skills.md, "Browser history via Hister".
+7. Add `"browser": { "type": "hister", "connector": { "semantic": true } }` under
+   `datasources`, then `autorag refresh` (only probes Hister) and
+   `autorag lite retrieve "<a question>" --scope '/browser/**' --json`.
+
+**Shared embedding model.** Hister keeps its own vector store and embeds with
+whatever its `config.yml` `semantic_search` block says. Pointing it at a different
+model does not fail: vectors simply stop lining up and results get worse. So Hister
+must use the same model as AutoRAG, which is the `minsync serve` daemon (model
+`Qwen/Qwen3-Embedding-0.6B`, 1024 dimensions, with Qwen3's query instruction).
+
+```bash
+autorag setup --format json            # reports hister.embedding.differs: the keys that disagree
+autorag setup --sync-hister            # writes them, restarts Hister, re-embeds pages if needed
+```
+
+- `--sync-hister` rewrites only `enable`, `embedding_endpoint`, `embedding_model`,
+  `dimensions`, `query_prefix`, `document_prefix` in Hister's config (a
+  `config.yml.bak.<time>` is kept next to it), then runs `brew services restart hister`,
+  waits for it to answer, and runs `hister reindex` when a change makes stored
+  vectors stale. Without the flag setup never writes Hister's config.
+- The daemon port is chosen when `minsync serve` starts. After restarting the daemon,
+  run `autorag setup --sync-hister` again. It is idempotent: when nothing differs
+  it changes nothing and restarts nothing.
+- It refuses, and says why, when the shared backend is a remote endpoint (browsing
+  text would leave the machine) or MinSync's in-process engine (Hister cannot reach
+  it; run `minsync serve --idle-timeout 0`, docs/minsync-setup.md).
+- If `restartRequired` or `reindexRequired` is true in the report, the step was
+  not done (no Homebrew service, or it failed): run `brew services restart hister`
+  and `hister reindex` yourself. Check with `hister doctor`: the line
+  `server.index.embeddings` must be `ok`.
+- A changed `query_prefix` alone needs a restart but no reindex. Never hand-edit the
+  `query_prefix` value across two lines: Hister's YAML parser folds the newline into
+  a space and the model's instruction is silently corrupted. `--sync-hister` writes
+  it correctly.
+
 ## Verify and build indexes
 
 Configuration alone is not a successful setup:

package/dist/datasource/skills/obsidian/client.d.ts

@@ -1,15 +0,0 @@
-import type { QmdEmbedResult, QmdEnsureResult, QmdOptions, QmdSearchMode, QmdSearchOptions, QmdSearchResult, QmdUpdateResult } from "./types.js";
-/**
- * Thin external `qmd` CLI wrapper for Obsidian vault indexing/search.
- * Every method spawns the `qmd` binary, never throws for expected failures,
- * and isolates config/cache under the AutoRAG workspace.
- */
-export declare class QmdClient {
-    private readonly options;
-    constructor(options?: QmdOptions);
-    ensureCollection(signal?: AbortSignal): Promise<QmdEnsureResult>;
-    update(signal?: AbortSignal): Promise<QmdUpdateResult>;
-    embed(signal?: AbortSignal): Promise<QmdEmbedResult>;
-    search(mode: QmdSearchMode, query: string, options?: QmdSearchOptions): Promise<QmdSearchResult>;
-    private run;
-}

package/dist/datasource/skills/obsidian/types.d.ts

@@ -1,92 +0,0 @@
-import type { RetrievalOptions } from "../../../retrieval/types.js";
-/**
- * Search modes accepted by the external `qmd` CLI.
- *
- * - `search`  — BM25 / FTS5 lexical search
- * - `vsearch` — dense vector semantic search
- */
-export type QmdSearchMode = "search" | "vsearch";
-/**
- * Configuration for {@link QmdClient}. The client spawns the `qmd` binary as a
- * child process and isolates config/cache under the AutoRAG workspace.
- */
-export interface QmdOptions {
-    /** Explicit path to the `qmd` binary. Defaults to a bare `qmd` PATH lookup. */
-    readonly binaryPath?: string;
-    /** Spawn timeout in milliseconds. Default 60_000 (embed can be slow). */
-    readonly timeoutMs?: number;
-    /** Max stdout/stderr bytes retained. Default 1_048_576 (1 MiB). */
-    readonly maxBufferBytes?: number;
-    /** Absolute path to the Obsidian vault. Required for ensure/update. */
-    readonly vaultPath?: string;
-    /** AutoRAG workspace root used to isolate qmd config/cache. */
-    readonly workspaceRoot?: string;
-    /** Instance id used for source paths and collection naming. Default `default`. */
-    readonly instanceId?: string;
-    /** Explicit collection name; defaults to a sanitized instance id. */
-    readonly collectionName?: string;
-    /** Environment overrides merged on top of a restricted process env. */
-    readonly env?: Readonly<Record<string, string | undefined>>;
-    /** Explicit operator-owned qmd config directory, never rewritten. */
-    readonly configPath?: string;
-}
-export declare const DEFAULT_QMD_BINARY = "qmd";
-export declare const DEFAULT_QMD_TIMEOUT_MS = 60000;
-export declare const DEFAULT_QMD_MAX_BUFFER_BYTES = 1048576;
-export type QmdFailureReason = "binary-missing" | "not-configured" | "nonzero-exit" | "spawn-error" | "timeout" | "aborted" | "stdout-too-large" | "stderr-too-large" | "invalid-json" | "invalid-shape";
-export interface QmdFailure {
-    readonly ok: false;
-    readonly reason: QmdFailureReason;
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number | null;
-}
-export interface QmdOk<T> {
-    readonly ok: true;
-    readonly data: T;
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number;
-}
-export interface QmdUpdateInfo {
-    readonly indexed: number;
-    readonly updated: number;
-    readonly unchanged: number;
-    readonly removed: number;
-    readonly needsEmbedding?: boolean;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-export interface QmdEmbedInfo {
-    readonly embedded: boolean;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-export interface QmdEnsureInfo {
-    readonly collectionName: string;
-    readonly vaultPath: string;
-    readonly configDir: string;
-}
-/** A scored search hit from `qmd search` / `qmd vsearch`. */
-export interface QmdSearchHit {
-    readonly chunkId: string;
-    readonly score: number;
-    readonly content: string;
-    readonly title?: string;
-    readonly file?: string;
-    readonly docid?: string;
-    readonly metadata?: Readonly<Record<string, unknown>>;
-}
-export interface QmdSearchOk {
-    readonly ok: true;
-    readonly hits: readonly QmdSearchHit[];
-    readonly stdout: string;
-    readonly stderr: string;
-    readonly code: number;
-    readonly data: {
-        readonly hits: readonly QmdSearchHit[];
-    };
-}
-export type QmdEnsureResult = QmdOk<QmdEnsureInfo> | QmdFailure;
-export type QmdUpdateResult = QmdOk<QmdUpdateInfo> | QmdFailure;
-export type QmdEmbedResult = QmdOk<QmdEmbedInfo> | QmdFailure;
-export type QmdSearchResult = QmdSearchOk | QmdFailure;
-export type QmdSearchOptions = RetrievalOptions;