@autorag/librarian 2.1.0 → 2.3.0

package/dist/memory/check-memory-tool.d.ts

@@ -8,6 +8,7 @@ export interface CheckMemoryDetails {
     signalCount: number;
     topMethod: string | null;
     insightCount: number;
+    contextHintCount: number;
 }
 export declare function createCheckMemoryTool(memory: RetrievalMemory): AgentTool<typeof checkMemorySchema, CheckMemoryDetails>;
 export {};

package/dist/memory/index.d.ts

@@ -1,5 +1,5 @@
 export type { CheckMemoryDetails } from "./check-memory-tool.js";
 export { createCheckMemoryTool } from "./check-memory-tool.js";
-export type { MemoryEntry, ResultFeedback, RetrievalInsight, RetrievalMemoryOptions, SearchAttempt } from "./memory.js";
-export { RetrievalMemory } from "./memory.js";
+export type { ContextValueHint, EvidenceContext, FeedbackIdInput, MemoryEntry, ResultFeedback, RetrievalContextHints, RetrievalInsight, RetrievalMemoryOptions, SearchAttempt, } from "./memory.js";
+export { normalizeSessionEvidenceRef, RetrievalMemory } from "./memory.js";
 export { renderMemoryContext } from "./renderer.js";

package/dist/memory/memory.d.ts

@@ -9,7 +9,16 @@ export interface SignalDefaults {
     readonly implicitCap: number;
     readonly decayHalfLifeMs?: number;
 }
-export interface EvidenceChunkRecord extends NormalizedEvidenceRef {
+export interface EvidenceContext {
+    readonly retrieverMix?: readonly string[];
+    readonly parserType?: string;
+    readonly documentType?: string;
+    readonly documentArea?: string;
+    readonly evidenceType?: string;
+    readonly evidenceLocation?: string;
+    readonly confidence?: number;
+}
+export interface EvidenceChunkRecord extends NormalizedEvidenceRef, EvidenceContext {
     readonly excerptHash: string;
     readonly firstSeenAt: number;
     readonly lastSeenAt: number;
@@ -24,6 +33,7 @@ export interface CuratedResultRecord {
     readonly summary: string;
     readonly resultHash: string;
     readonly evidenceIds: readonly string[];
+    readonly confidence?: number;
     readonly createdAt: number;
 }
 export interface FeedbackSignal {
@@ -53,6 +63,19 @@ export interface MethodHint {
     readonly confidence: number;
     readonly reason: string;
 }
+export interface ContextValueHint {
+    readonly value: string;
+    readonly score: number;
+    readonly confidence: number;
+}
+export interface RetrievalContextHints {
+    readonly documentAreas: readonly ContextValueHint[];
+    readonly documentTypes: readonly ContextValueHint[];
+    readonly evidenceTypes: readonly ContextValueHint[];
+    readonly evidenceLocations: readonly ContextValueHint[];
+    readonly parserTypes: readonly ContextValueHint[];
+    readonly retrieverMix: readonly ContextValueHint[];
+}
 export interface RetrievalInsight {
     readonly id: string;
     readonly clusterKey: string;
@@ -107,7 +130,7 @@ export interface ResultFeedback {
     source: string;
     useful: boolean;
 }
-export interface SessionEvidenceRef extends NormalizedEvidenceRef {
+export interface SessionEvidenceRef extends NormalizedEvidenceRef, EvidenceContext {
     readonly metadata?: Record<string, unknown>;
 }
 export interface SessionCuratedResultInput {
@@ -117,6 +140,7 @@ export interface SessionCuratedResultInput {
     readonly content: string;
     readonly method: string;
     readonly source: string;
+    readonly confidence?: number;
     readonly evidenceRefs: readonly SessionEvidenceRef[];
 }
 export interface SessionRecordInput {
@@ -132,6 +156,10 @@ export interface NumberedFeedbackInput {
         readonly useful: boolean;
     }[];
 }
+export interface FeedbackIdInput {
+    readonly feedbackId: string;
+    readonly useful: boolean;
+}
 export interface RetrievalMemoryOptions {
     storagePath: string;
     insightExtractor?: InsightExtractor;
@@ -150,8 +178,10 @@ export declare class RetrievalMemory {
     getSignalCount(): number;
     recordCuratedResultsSession(input: SessionRecordInput): void;
     recordNumberedFeedback(input: NumberedFeedbackInput): boolean;
+    recordFeedbackByIds(feedback: readonly FeedbackIdInput[]): boolean;
     recordWeakSignal(query: string, method: string, source: "followup" | "retry"): void;
-    getMethodHints(query: string): MethodHint[];
+    getMethodHints(_query: string): MethodHint[];
+    getContextHints(_query: string): RetrievalContextHints;
     getInsights(query: string): RetrievalInsight[];
     getMethodPriority(query: string): Array<{
         method: string;

package/dist/memory/renderer.d.ts

@@ -1,6 +1,7 @@
-import type { MethodHint, RetrievalInsight } from "./memory.js";
+import type { MethodHint, RetrievalContextHints, RetrievalInsight } from "./memory.js";
 export declare function renderMemoryContext(hints: readonly MethodHint[], opts?: {
     maxHints?: number;
     insights?: readonly RetrievalInsight[];
     maxInsights?: number;
+    contextHints?: RetrievalContextHints;
 }): string;

package/dist/mirror/sync.d.ts

@@ -10,8 +10,10 @@ export interface ParsedMirrorSyncOptions {
     readonly force?: boolean;
     /** Override max source size for parse attempts (bytes). */
     readonly maxSourceBytes?: number;
+    /** Absolute source paths omitted from parsed mirrors (for example exact duplicate copies). */
+    readonly excludeSourcePaths?: ReadonlySet<string>;
 }
-export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "deleted-mirror" | "stale-index";
+export type ParsedMirrorDiagnosticCode = "unsupported-file" | "parser-skipped" | "parser-failed" | "duplicate-excluded" | "deleted-mirror" | "stale-index";
 /** Path-opaque refresh diagnostic. `source` is an opaque virtual path, never a real fs path. */
 export interface ParsedMirrorDiagnostic {
     readonly code: ParsedMirrorDiagnosticCode;

package/dist/observability/run-log.d.ts

@@ -3,8 +3,7 @@ export type AutoRAGRunEvent = {
     timestamp: string;
     sessionId: string;
     queryLength: number;
-    orchestratorModel: string;
-    explorerModel: string;
+    model: string;
 } | {
     event: "search_completed";
     timestamp: string;
@@ -21,30 +20,6 @@ export type AutoRAGRunEvent = {
     sessionId: string;
     failureCount: number;
     errorTypes: string[];
-} | {
-    event: "dispatch_rejected";
-    schemaVersion: 1;
-    sessionId: string;
-    toolCallId: string | null;
-    sequence: number;
-    timestamp: string;
-    dispatchKind: "launch" | "admin" | "control" | "mutation" | "schedule" | "hybrid" | "unknown";
-    code: string;
-    field: string;
-    forceCorrectable: boolean;
-} | {
-    event: "dispatch_autofilled";
-    schemaVersion: 1;
-    sessionId: string;
-    toolCallId: string | null;
-    sequence: number;
-    timestamp: string;
-    dispatchKind: "launch";
-    fields: {
-        artifacts: boolean;
-        agentScope: boolean;
-        leafModelFillCount: number;
-    };
 };
 export declare class AutoRAGRunLogger {
     private readonly path;

package/dist/process/portable-spawn.d.ts

@@ -0,0 +1,10 @@
+export interface PortableSpawnCommand {
+    readonly command: string;
+    readonly args: readonly string[];
+}
+/**
+ * Windows does not execute Unix shebang files or extensionless scripts through
+ * child_process.spawn. Resolve test/dev script entrypoints through their
+ * interpreter while leaving native executables untouched.
+ */
+export declare function portableSpawnCommand(command: string, args: readonly string[], platform?: NodeJS.Platform): PortableSpawnCommand;

package/dist/retrieval/merger.d.ts

@@ -1,6 +1,7 @@
 import type { RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalWithDiagnostics } from "./types.js";
 export interface MergeOptions {
     topK: number;
+    /** Retained for API compatibility; exact-source aggregation is always enabled. */
     dedup: boolean;
 }
 export declare class ResultMerger {

package/dist/retrieval/scope.d.ts

@@ -1,4 +1,14 @@
+export interface RetrievalScopeBinding {
+    readonly virtualPrefix: string;
+    readonly physicalRoots: readonly string[];
+}
+export declare class RetrievalScopeError extends Error {
+    readonly code = "invalid-retrieval-scope";
+    constructor(virtualPrefixes: readonly string[]);
+}
 export declare function normalizeVirtualPath(value: string): string;
 export declare function normalizeVirtualPathScope(scope: string | undefined): string | undefined;
+export declare function resolveRetrievalScope(scope: string | undefined, bindings: readonly RetrievalScopeBinding[], platform?: NodeJS.Platform, passthroughVirtualPrefixes?: readonly string[]): string | undefined;
+export declare function buildRetrievalScopeBindings(workspaceRoot: string, searchPaths: readonly string[], configuredSearchPaths?: readonly string[]): readonly RetrievalScopeBinding[];
 export declare function matchesVirtualPathScope(virtualPath: string, scope: string | undefined): boolean;
 export declare function virtualPathScopeToRegExp(scope: string): RegExp;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autorag/librarian",
-  "version": "2.1.0",
+  "version": "2.3.0",
   "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
   "license": "MIT",
   "repository": {
@@ -28,40 +28,41 @@
     "access": "public"
   },
   "scripts": {
-    "build": "bun build src/index.ts src/cli/index.ts src/subagents/explorer-tools-extension.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
+    "build": "bun build src/index.ts src/cli/index.ts --outdir dist --target node --packages external && tsc -p tsconfig.build.json && node scripts/fixup-dts.mjs",
     "prepublishOnly": "bun run typecheck && bun run build",
     "prepare": "bun run build",
-    "test": "vitest run",
+    "test": "node scripts/run-test-files.mjs",
+    "test:windows": "node scripts/run-test-files.mjs",
     "check": "biome check --write .",
+    "lint": "biome check .",
     "typecheck": "tsc --noEmit"
   },
   "engines": {
     "node": ">=24.0.0"
   },
   "dependencies": {
-    "@earendil-works/pi-agent-core": "^0.80.10",
-    "@earendil-works/pi-ai": "^0.80.10",
-    "@earendil-works/pi-coding-agent": "0.80.10",
-    "@opendataloader/pdf": "^2.4.7",
+    "@earendil-works/pi-agent-core": "^0.84.2",
+    "@earendil-works/pi-ai": "^0.84.2",
+    "@earendil-works/pi-coding-agent": "0.84.2",
+    "@opendataloader/pdf": "^2.5.2",
     "@pngwasi/node-tantivy-binding": "^0.3.4",
-    "@rhwp/core": "0.7.19",
+    "@rhwp/core": "0.8.4",
     "chardet": "^2.2.0",
-    "fast-xml-parser": "^5.9.3",
-    "iconv-lite": "^0.7.2",
+    "fast-xml-parser": "^5.11.0",
+    "iconv-lite": "^0.7.3",
     "jszip": "^3.10.1",
-    "mailparser": "^3.9.12",
-    "pi-subagents": "0.35.1",
-    "slides-grab": "^1.3.1",
-    "smol-toml": "1.7.0",
+    "mailparser": "^3.9.15",
+    "slides-grab": "^1.5.0",
+    "smol-toml": "1.8.0",
     "tesseract.js": "^7.0.0",
-    "typebox": "^1.1.24",
-    "yaml": "^2.8.2"
+    "typebox": "^1.3.16",
+    "yaml": "^2.9.0"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.3.5",
+    "@biomejs/biome": "2.5.9",
     "@types/mailparser": "^3.4.6",
-    "@types/node": "^24.3.0",
-    "typescript": "^5.7.3",
-    "vitest": "^4.1.10"
+    "@types/node": "^26.2.0",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.11"
   }
 }

package/skills/autorag/SKILL.md

@@ -1,70 +1,82 @@
 ---
 name: autorag
-description: Use an already configured AutoRAG librarian agent to search, summarize, compare, and answer questions from local document collections. Use autorag-setup instead for provider/model discovery, first-time configuration, or selecting folders to index.
+description: Use an already configured AutoRAG librarian agent to search, summarize, compare, and answer questions from local document collections. Use autorag-setup for configuration or indexing changes.
 ---
 
 # AutoRAG Librarian Skill
 
 Use this skill when AutoRAG is already configured and the user asks to search,
 summarize, compare, or answer questions from local PDFs, wikis, notes, research
-papers, or knowledge bases. For first-time configuration, missing model/provider
-settings, subscription or API-provider detection, or changing indexed folders,
-use the `autorag-setup` skill instead.
+papers, or knowledge bases.
 
-AutoRAG is invoked through the `autorag` CLI. It is non-destructive: it reads
-source files and writes indexes under the configured workspace's `.autorag/`
-directory and, when enabled, Jikji's per-source `.jikji/` caches. Never move,
-rename, or delete source files.
+AutoRAG is the specialized librarian agent. One configured model plans the
+search, calls BM25, MinSync, Jikji, datasource, and filesystem tools, reads
+sources, judges evidence, and curates the final answer. There is no subagent or
+separate model role.
+
+AutoRAG reads source documents and writes indexes only under the configured
+workspace `.autorag/` directory and Jikji's per-source `.jikji/` caches. Never
+move, rename, edit, or delete source files.
 
 ## Preflight
 
-Confirm that `~/.autorag/config.json` exists (or an explicit `--config` /
-`AUTORAG_CONFIG` path). Inspect that non-secret config for usable
-`searchPaths`/`agents` without printing credentials. Then run:
+Confirm `~/.autorag/config.json`, an explicit `--config`, or `AUTORAG_CONFIG`
+exists. Inspect only non-secret `searchPaths` and `model` metadata, then run:
 
 ```bash
+autorag duplicates              # read-only duplicate-family review
+autorag duplicates --json       # machine-readable cleanup planning input
 autorag status
 autorag health
 ```
 
-`status` is model-free and path-opaque: it reports corpus freshness and index
-health, not absolute filesystem paths or role-model auth. `health` checks
-model/provider auth and explorer subagent setup: it resolves both role models,
-verifies credential presence, and (unless `--skip-probes`) probes a completion
-call per role. Use `health` to diagnose model, provider, auth, timeout, or
-subagent dispatch failures before searching. If configuration, authentication,
-role models, or indexes are missing or unhealthy, stop this workflow and use
-`autorag-setup`; do not guess private providers or model IDs.
+### Duplicate-file review
+
+Use `autorag duplicates` when the user asks to find duplicate files, choose
+likely latest copies, or reduce corpus/index space. The command and the
+`scan_duplicate_documents` Agent tool are read-only and never delete or move
+source files. Exact means dupey's canonical extracted-text hash matches;
+near/contains families require human review. Exact duplicate exclusion during
+refresh is enabled by default and can be disabled with
+`"excludeExactDuplicates": false` in `config.json`.
+
+`status` is model-free and path-opaque. `health` resolves the single model,
+checks credential presence, and normally probes one live completion. If the
+model, authentication, configuration, or indexes are unhealthy, use
+`autorag-setup` rather than guessing private provider details.
 
-## Searching
+BM25, MinSync, and Jikji should normally be healthy. If they are missing or
+stale, run a full `autorag refresh` or return to setup rather than silently
+degrading to lexical-only search.
+
+## Search
 
 ```bash
-autorag search "what were the key findings in the Q3 report" --top-k 5
+autorag search "what were the key findings in the Q3 report" --top-k 5 --json
 ```
 
-AutoRAG returns curated, numbered knowledge units grounded in sources plus a
-`sessionId` needed for feedback. Use:
+The response contains a `sessionId`, numbered curated results, source mapping,
+and an answer grounded in those results.
 
-- `--scope` to narrow to a configured virtual sub-path
-- `--tags tag1,tag2` only when trusted datasource access is already configured
-  server-side; tags narrow already-allowed results and never grant new access
-- `--json` for structured output (`sessionId`, numbered `results`, `answer`)
-- `--debug` only when diagnostics are needed
+- `--scope` narrows to a configured virtual sub-path.
+- `--tags` narrows already-authorized datasource results and never grants new
+  access.
+- `--json` is preferred for programmatic consumption.
+- `--debug` is for diagnostics only.
 
-Do not bypass AutoRAG with ad hoc raw search when the user explicitly requested
-the librarian agent. Search requires a resolvable orchestrator/explorer model
-pair from config, flags, env, or the authenticated local runtime. When `autorag
-search` fails for a model, provider, auth, timeout, or subagent reason, the
-error output includes a hint pointing to `autorag health` for diagnosis.
+Do not bypass the librarian with ad hoc raw search when the user requested
+AutoRAG. The search loop can use Jikji, BM25, MinSync, datasource retrieval, and
+direct source reading as appropriate. If search fails because of model,
+provider, auth, or timeout problems, diagnose with `autorag health`.
 
-Record feedback so retrieval memory learns which results were useful:
+Record feedback so retrieval memory can learn:
 
 ```bash
 autorag feedback <sessionId> --useful 1,3 --not-useful 2
 ```
 
-Supply at least one of `--useful` or `--not-useful`. Numbers refer to the
-numbered knowledge units from that session's search output.
+Supply at least one feedback list. Numbers refer to the returned knowledge
+units.
 
 ## Maintenance
 
@@ -72,7 +84,7 @@ numbered knowledge units from that session's search output.
 autorag status
 autorag health
 autorag refresh
-autorag refresh --method bm25,minsync
+autorag refresh --method bm25,minsync,jikji
 autorag watch --once
 autorag watch
 autorag refresh --force
@@ -81,69 +93,20 @@ autorag index reset --method bm25 --yes
 autorag memory inspect
 ```
 
-Use `refresh` after source documents change (parses sources and resyncs BM25 /
-MinSync / datasources / optional Jikji prepare). Prefer bounded refresh over
-reset. `watch --once` is the preferred single tick for scheduled jobs; long-running
-`watch` keeps an fs event loop open for interactive sessions.
-`--method <csv>` (e.g. `--method bm25,minsync,parsed`) restricts which methods
-refresh/index run; when omitted all methods run. BM25 and MinSync are enabled
-by default — no explicit configuration is needed for standard lexical + semantic
-retrieval. MinSync uses a pre-installed binary (`autoInstall: false`); configure
-`minSync.embedder` via `autorag init --embedder-*` flags for remote embedding.
-
-### Keep indexes fresh on a schedule (agent responsibility)
-
-There is **no always-on network service** in the CLI. Agents and operators must
-install an OS-appropriate periodic job that runs a model-free index tick:
-
-```bash
-# Preferred scheduled command (single non-daemon refresh)
-NODE_OPTIONS=--max-old-space-size=16384 autorag watch --once
-# Equivalent one-shot refresh
-NODE_OPTIONS=--max-old-space-size=16384 autorag refresh
-```
+Prefer a full refresh so parsed mirrors, BM25, MinSync, Jikji, and configured
+datasources stay aligned. Use `--method` only for deliberate narrowing.
+Scheduled maintenance should use non-daemon `autorag watch --once`, typically
+every 15–30 minutes, with the same config used by search and no overlapping
+runs.
 
-Schedule guidance by OS (typical interval: every 15–30 minutes; never more often
-than the corpus can finish refreshing):
-
-| OS | Preferred scheduler | Pattern |
-|---|---|---|
-| macOS | `cron` or `launchd` (`~/Library/LaunchAgents`) | `*/30 * * * * ... autorag watch --once` or a KeepAlive=false StartInterval plist |
-| Linux | `cron` / `systemd --user` timer | crontab `*/30 * * * *` or a oneshot service + timer |
-| Windows | Task Scheduler | repeating task every 30 minutes running `autorag watch --once` under the user profile |
-
-Rules for scheduled watch:
-
-1. Prefer `autorag watch --once` (or `autorag refresh`) over a permanent
-   long-running `autorag watch` daemon in user agents — daemons die on reboot
-   and are harder to supervise from skill workflows.
-2. Use the same config the search path uses (`~/.autorag/config.json` or an
-   explicit `--config` / `AUTORAG_CONFIG`).
-3. Raise Node heap for large home trees: `NODE_OPTIONS=--max-old-space-size=16384`.
-4. Redirect logs somewhere under the home/user temp tree, never into source
-   document folders.
-5. After install/setup, **create or verify** the scheduled job before claiming
-   ongoing indexing is covered. Re-check with `crontab -l`, `systemctl --user list-timers`,
-   or Task Scheduler inspection when the user asks about freshness.
-6. Do not schedule concurrent overlapping ticks; if a prior refresh is still
-   running, skip or wait (lock/log rather than stampeding MiniSync/BM25 writers).
-
-Destructive index commands:
-
-- `autorag index reset --yes` removes parsed, BM25, and MinSync directories under
-  workspace `.autorag` only. Add `--method bm25|minsync|parsed` to scope which
-  indexes are removed (e.g. `--method bm25` removes only the BM25 index).
-- `autorag index rebuild --yes` resets those indexes then forced-refreshes.
-  `--method` scopes both the reset and the rebuild refresh.
-
-Never run reset/rebuild against source documents. `memory inspect` is
-read-only and path-opaque.
+Reset and rebuild commands remove only selected workspace `.autorag` indexes.
+Never target source documents. `memory inspect` is read-only and path-opaque.
 
 ## Rules
 
 - Use only configured and approved search paths.
 - Never expose provider credentials or authentication payloads.
-- Never invent or reveal private provider names or model IDs.
-- Never treat a consumer subscription as API access without runtime evidence.
+- Never invent or reveal private provider names or model ids.
+- A Pi-usable subscription is valid; a subscription Pi cannot invoke is not.
 - Preserve real source mapping and numbered feedback identifiers.
-- Prefer `--json` when another agent must consume the result programmatically.
+- Prefer `--json` when another agent consumes the response.