@autorag/librarian 2.6.2 → 2.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/NOTICE +12 -0
  2. package/README.md +12 -3
  3. package/dist/agent/agent.d.ts +135 -6
  4. package/dist/agent/everything-search-tool.d.ts +35 -0
  5. package/dist/agent/fsearch-search-tool.d.ts +35 -0
  6. package/dist/agent/jikji-find-tool.d.ts +2 -0
  7. package/dist/agent/refresh-progress.d.ts +1 -1
  8. package/dist/agent/search-documents.d.ts +1 -1
  9. package/dist/cli/commands/refresh.d.ts +1 -1
  10. package/dist/cli/config.d.ts +51 -3
  11. package/dist/cli/index.js +2392 -527
  12. package/dist/core.js +2530 -693
  13. package/dist/datasource/connector-timeouts.d.ts +19 -0
  14. package/dist/datasource/crawler-types.d.ts +10 -0
  15. package/dist/datasource/skills/cloud-drive/rclone-connector.d.ts +2 -0
  16. package/dist/datasource/skills/lazykatok/client.d.ts +2 -0
  17. package/dist/datasource/skills/lazykatok/index.d.ts +1 -1
  18. package/dist/datasource/skills/lazykatok/types.d.ts +17 -1
  19. package/dist/datasource/skills/rss/connector.d.ts +2 -0
  20. package/dist/datasource/skills/spotlight/connector.d.ts +2 -0
  21. package/dist/dupey/filter.d.ts +7 -1
  22. package/dist/everything/bundle.d.ts +62 -0
  23. package/dist/everything/client.d.ts +123 -0
  24. package/dist/everything/index.d.ts +2 -0
  25. package/dist/fsearch/client.d.ts +173 -0
  26. package/dist/fsearch/index.d.ts +2 -0
  27. package/dist/fsearch/walk.d.ts +22 -0
  28. package/dist/index.d.ts +2 -0
  29. package/dist/index.js +2555 -693
  30. package/dist/minsync/method.d.ts +6 -0
  31. package/dist/retrieval/engine.d.ts +14 -0
  32. package/dist/retrieval/index.d.ts +1 -0
  33. package/dist/retrieval/rerank.d.ts +114 -0
  34. package/dist/retrieval/types.d.ts +1 -1
  35. package/licenses/es-MIT.txt +21 -0
  36. package/licenses/everything-MIT-and-PCRE-BSD.txt +51 -0
  37. package/package.json +4 -2
  38. package/skills/autorag/SKILL.md +4 -3
  39. package/skills/autorag-doctor/SKILL.md +2 -0
  40. package/skills/autorag-lite-setup/SKILL.md +3 -2
  41. package/skills/autorag-setup/SKILL.md +53 -3
  42. package/vendor/everything/ES-1.1.0.38.ARM64.zip +0 -0
  43. package/vendor/everything/ES-1.1.0.38.x64.zip +0 -0
  44. package/vendor/everything/Everything-1.4.1.1032.ARM64.en-US.zip +0 -0
  45. package/vendor/everything/Everything-1.4.1.1032.x64.en-US.zip +0 -0
  46. package/vendor/everything/manifest.json +30 -0
@@ -14,6 +14,10 @@ export interface MinSyncVectorMethodOptions {
14
14
  readonly maxChunkSize?: number;
15
15
  readonly mode?: MinSyncQueryMode;
16
16
  readonly runtime?: MinSyncRuntime;
17
+ /** Default `topK` used when the caller omits one. Default 50. */
18
+ readonly defaultTopK?: number;
19
+ /** Fetch cap applied when a scope narrows the query. Default 100. */
20
+ readonly scopedTopK?: number;
17
21
  }
18
22
  export type MinSyncHybridMethodOptions = Omit<MinSyncVectorMethodOptions, "mode">;
19
23
  export declare class MinSyncVectorMethod implements RetrievalMethod {
@@ -26,6 +30,8 @@ export declare class MinSyncVectorMethod implements RetrievalMethod {
26
30
  private readonly maxChunkSize;
27
31
  private readonly mode;
28
32
  private readonly runtime;
33
+ private readonly defaultTopK;
34
+ private readonly scopedTopK;
29
35
  private installFailed;
30
36
  constructor(options: MinSyncVectorMethodOptions);
31
37
  describe(): RetrievalMethodDescriptor;
@@ -15,6 +15,7 @@
15
15
  */
16
16
  import { DatasourceAccessContext, type DatasourceAccessContextOptions } from "../datasource/access-context.js";
17
17
  import { RetrievalMethodRegistry } from "./registry.js";
18
+ import type { Reranker } from "./rerank.js";
18
19
  import type { RetrievalDiagnostic, RetrievalMethod, RetrievalOptions, RetrievalResult, RetrievalUnsearchedSurface } from "./types.js";
19
20
  /** Options for constructing a standalone {@link RetrievalEngine}. */
20
21
  export interface RetrievalEngineOptions {
@@ -40,6 +41,13 @@ export interface RetrievalEngineOptions {
40
41
  * preservation — matching the existing {@link AutoRAGAgent} behavior.
41
42
  */
42
43
  readonly isMinSyncBinaryMissing?: () => boolean;
44
+ /**
45
+ * Optional post-merge reranker. When set, merged results are reordered by
46
+ * the reranker after dedup and score normalization. A reranker failure is
47
+ * reported as a `rerank-failed` diagnostic and the unranked order is kept —
48
+ * the engine never silently drops evidence because a reranker was down.
49
+ */
50
+ readonly reranker?: Reranker;
43
51
  }
44
52
  /**
45
53
  * Model-free retrieval pipeline.
@@ -71,6 +79,7 @@ export declare class RetrievalEngine {
71
79
  private readonly defaultTopK;
72
80
  private readonly defaultDedup;
73
81
  private readonly isMinSyncBinaryMissing;
82
+ private readonly reranker;
74
83
  constructor(options?: RetrievalEngineOptions);
75
84
  /** The underlying method registry. Intentionally public for tooling. */
76
85
  getMethodRegistry(): RetrievalMethodRegistry;
@@ -100,6 +109,11 @@ export declare class RetrievalEngine {
100
109
  diagnostics: RetrievalDiagnostic[];
101
110
  unsearched: RetrievalUnsearchedSurface[];
102
111
  }>;
112
+ /**
113
+ * Reorder merged results with the configured reranker. On failure the
114
+ * unranked order is returned and a `rerank-failed` diagnostic is appended.
115
+ */
116
+ private applyRerank;
103
117
  /**
104
118
  * Run retrieval and return results keyed per method (unmerged) with
105
119
  * diagnostics. Unlike {@link retrieve}, this does NOT merge or sort —
@@ -2,6 +2,7 @@ export { RetrievalEngine, type RetrievalEngineOptions } from "./engine.js";
2
2
  export type { MergeOptions } from "./merger.js";
3
3
  export { ParallelRetriever, ResultMerger } from "./merger.js";
4
4
  export { RetrievalMethodRegistry } from "./registry.js";
5
+ export { type CreateRerankerOptions, createReranker, DEFAULT_RERANK_API_KEY_ENV, DEFAULT_RERANK_MODEL, DEFAULT_RERANK_PROVIDER, OpenRouterReranker, type OpenRouterRerankerOptions, type Reranker, type RerankerDescriptor, type RerankOptions, } from "./rerank.js";
5
6
  export { matchesVirtualPathScope, normalizeVirtualPath, normalizeVirtualPathScope, virtualPathScopeToRegExp, } from "./scope.js";
6
7
  export { describeRetrievalError, groupUnsearchedSurfaces, MINSYNC_SURFACE, type RetrievalSkip, retrievalSurfaceFor, } from "./skip.js";
7
8
  export type { CuratedResult, NumberedResult, RetrievalDiagnostic, RetrievalDiagnosticCode, RetrievalMethod, RetrievalMethodDescriptor, RetrievalOptions, RetrievalResult, RetrievalUnsearchedSurface, RetrievalWithDiagnostics, } from "./types.js";
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Reranker seam — a post-retrieval reordering stage over merged evidence.
3
+ *
4
+ * The retrieval pipeline (registry → ParallelRetriever → DatasourceResultFilter
5
+ * → ResultMerger) produces a scored candidate list. A reranker reorders that
6
+ * list with a dedicated relevance model. The default implementation talks to
7
+ * OpenRouter's rerank router through the OpenRouter TypeScript SDK.
8
+ *
9
+ * Local-first is no longer a contract: a remote reranker is opt-in through
10
+ * trusted config, and a local reranker can implement this same interface
11
+ * without touching the pipeline. Nothing here silently falls back between
12
+ * providers — a configured reranker that cannot run is reported, not replaced.
13
+ */
14
+ import type { RetrievalResult } from "./types.js";
15
+ /** Default rerank model routed through OpenRouter. */
16
+ export declare const DEFAULT_RERANK_MODEL = "voyageai/rerank-3-lite";
17
+ /** Default rerank provider id. */
18
+ export declare const DEFAULT_RERANK_PROVIDER = "openrouter";
19
+ /** Default environment variable holding the OpenRouter API key. */
20
+ export declare const DEFAULT_RERANK_API_KEY_ENV = "OPENROUTER_API_KEY";
21
+ /** Static description of a reranker, used for diagnostics and inspection. */
22
+ export interface RerankerDescriptor {
23
+ /** Stable label (e.g. `openrouter`). */
24
+ readonly name: string;
25
+ /** Provider id (e.g. `openrouter`). */
26
+ readonly provider: string;
27
+ /** Wire model id (e.g. `cohere/rerank-4-fast`). */
28
+ readonly model: string;
29
+ /** Whether the reranker has everything it needs to run (credentials, config). */
30
+ readonly available: boolean;
31
+ /** Why the reranker is unavailable, when `available` is false. */
32
+ readonly reason?: string;
33
+ }
34
+ /** Per-call rerank options. */
35
+ export interface RerankOptions {
36
+ /** Return only the top N results. Omitted ⇒ every distinct result is returned, reordered. */
37
+ readonly topN?: number;
38
+ /** Abort the in-flight rerank request. */
39
+ readonly signal?: AbortSignal;
40
+ }
41
+ /** A post-retrieval reordering stage. */
42
+ export interface Reranker {
43
+ describe(): RerankerDescriptor;
44
+ /**
45
+ * Reorder `results` by relevance to `query`. Returns a new array; the input
46
+ * is not mutated. Implementations must preserve each result's `source`/`id`
47
+ * identity so evidence stays traceable.
48
+ */
49
+ rerank(query: string, results: readonly RetrievalResult[], options?: RerankOptions): Promise<RetrievalResult[]>;
50
+ }
51
+ /** Minimal rerank client shape, so tests can inject a fake without the SDK. */
52
+ export interface RerankClient {
53
+ rerank(request: {
54
+ requestBody: {
55
+ model: string;
56
+ query: string;
57
+ documents: string[];
58
+ topN?: number;
59
+ };
60
+ }): Promise<RerankResponse>;
61
+ }
62
+ /** Response body returned by OpenRouter's rerank router. */
63
+ export interface RerankResponse {
64
+ readonly results: ReadonlyArray<{
65
+ readonly index: number;
66
+ readonly relevanceScore: number;
67
+ }>;
68
+ readonly model?: string;
69
+ readonly provider?: string;
70
+ }
71
+ /** Options for {@link OpenRouterReranker}. */
72
+ export interface OpenRouterRerankerOptions {
73
+ /** Wire model id. @default DEFAULT_RERANK_MODEL */
74
+ readonly model?: string;
75
+ /** API key. When omitted, read from `apiKeyEnv` in the environment. */
76
+ readonly apiKey?: string;
77
+ /** Environment variable holding the API key. @default DEFAULT_RERANK_API_KEY_ENV */
78
+ readonly apiKeyEnv?: string;
79
+ /** Override the OpenRouter base URL (e.g. a self-hosted or gateway endpoint). */
80
+ readonly baseUrl?: string;
81
+ /** Per-request timeout in milliseconds. */
82
+ readonly timeoutMs?: number;
83
+ /** Environment to read the API key from. Defaults to `process.env`; a test seam. */
84
+ readonly env?: NodeJS.ProcessEnv;
85
+ /** Pre-built client, bypassing SDK construction. A test seam. */
86
+ readonly client?: RerankClient;
87
+ }
88
+ /** Options for {@link createReranker}: provider plus the OpenRouter reranker options. */
89
+ export interface CreateRerankerOptions extends OpenRouterRerankerOptions {
90
+ /** Provider id. @default DEFAULT_RERANK_PROVIDER */
91
+ readonly provider?: string;
92
+ }
93
+ /**
94
+ * Build the configured reranker. Returns `undefined` when reranking is disabled
95
+ * or absent, or when the provider is unsupported. Provider-agnostic by design:
96
+ * a local reranker implements {@link Reranker} and plugs in here without
97
+ * touching the retrieval pipeline.
98
+ */
99
+ export declare function createReranker(config: CreateRerankerOptions | false | undefined): Reranker | undefined;
100
+ /**
101
+ * Reranker backed by OpenRouter's rerank router.
102
+ *
103
+ * Documents are sent as their `content`; the original `RetrievalResult` for each
104
+ * returned index is preserved and re-scored with the provider's relevance score.
105
+ */
106
+ export declare class OpenRouterReranker implements Reranker {
107
+ private readonly model;
108
+ private readonly provider;
109
+ private readonly client;
110
+ private readonly unavailableReason;
111
+ constructor(options?: OpenRouterRerankerOptions);
112
+ describe(): RerankerDescriptor;
113
+ rerank(query: string, results: readonly RetrievalResult[], options?: RerankOptions): Promise<RetrievalResult[]>;
114
+ }
@@ -26,7 +26,7 @@ export interface RetrievalMethod {
26
26
  describe(): RetrievalMethodDescriptor;
27
27
  retrieve(query: string, options: RetrievalOptions): Promise<RetrievalResult[]>;
28
28
  }
29
- export type RetrievalDiagnosticCode = "retrieval-method-failed" | "minsync-unavailable";
29
+ export type RetrievalDiagnosticCode = "retrieval-method-failed" | "minsync-unavailable" | "rerank-failed";
30
30
  /**
31
31
  * A retrieval surface (local MinSync files or one datasource) that was not
32
32
  * searched for this query. Reported next to partial results so a caller that
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 voidtools
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,51 @@
1
+ Everything
2
+
3
+ Copyright (C) 2018 David Carpenter
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a
6
+ copy of this software and associated documentation files (the "Software"),
7
+ to deal in the Software without restriction, including without limitation
8
+ the rights to use, copy, modify, merge, publish, distribute, sublicense,
9
+ and/or sell copies of the Software, and to permit persons to whom the
10
+ Software is furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21
+ DEALINGS IN THE SOFTWARE.
22
+
23
+ Perl-Compatible Regular Expressions
24
+
25
+ Copyright (c) 1997-2012 University of Cambridge
26
+
27
+ Redistribution and use in source and binary forms, with or without
28
+ modification, are permitted provided that the following conditions are met:
29
+
30
+ * Redistributions of source code must retain the above copyright notice,
31
+ this list of conditions and the following disclaimer.
32
+
33
+ * Redistributions in binary form must reproduce the above copyright
34
+ notice, this list of conditions and the following disclaimer in the
35
+ documentation and/or other materials provided with the distribution.
36
+
37
+ * Neither the name of the University of Cambridge nor the names of its
38
+ contributors may be used to endorse or promote products derived from
39
+ this software without specific prior written permission.
40
+
41
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
42
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
45
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
46
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
47
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
48
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
49
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
50
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
51
+ POSSIBILITY OF SUCH DAMAGE.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@autorag/librarian",
3
- "version": "2.6.2",
3
+ "version": "2.6.3",
4
4
  "description": "Self-evolving librarian agent on Pi framework with pluggable retrieval methods",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -28,7 +28,8 @@
28
28
  "skills",
29
29
  "README.md",
30
30
  "NOTICE",
31
- "licenses"
31
+ "licenses",
32
+ "vendor"
32
33
  ],
33
34
  "publishConfig": {
34
35
  "access": "public"
@@ -53,6 +54,7 @@
53
54
  "@earendil-works/pi-ai": "^0.86.1",
54
55
  "@earendil-works/pi-coding-agent": "0.86.1",
55
56
  "@earendil-works/pi-tui": "0.86.1",
57
+ "@openrouter/sdk": "^1.4.21",
56
58
  "chardet": "^2.2.0",
57
59
  "fast-xml-parser": "^5.11.1",
58
60
  "iconv-lite": "^0.7.3",
@@ -124,9 +124,10 @@ autorag serve --force
124
124
  autorag p2p policy list --json
125
125
  ```
126
126
 
127
- Prefer a full refresh so parsed mirrors, MinSync, Jikji, and configured
128
- datasources stay aligned. `--method` accepts
129
- `parsed,minsync,datasources,jikji,all`. BM25 is a MinSync retrieval mode, not
127
+ Prefer a full refresh so parsed mirrors, MinSync, Jikji, configured
128
+ datasources, and (on Windows) the bundled Everything file-name index stay
129
+ aligned. `--method` accepts
130
+ `parsed,minsync,datasources,jikji,everything,all`. BM25 is a MinSync retrieval mode, not
130
131
  a `--method` name. Use `--method` only for deliberate narrowing. Scheduled
131
132
  maintenance should use non-daemon `autorag watch --once`, typically every
132
133
  1 hour, with the same config used by search and no overlapping runs.
@@ -58,6 +58,7 @@ Each CLI owns its archive, so ask the CLI, not AutoRAG. A datasource is only
58
58
  |---|---|---|
59
59
  | MinSync (local docs) | `minsync status`, `minsync check`, `minsync verify` | `autorag refresh --method minsync` |
60
60
  | Jikji (discovery) | `jikji doctor` | `autorag refresh --method jikji` |
61
+ | Everything (Windows file names) | `autorag status --json` → `components.everything` | `autorag refresh --method everything --json` |
61
62
  | KakaoTalk | `lazykatok doctor` | `lazykatok sync && lazykatok index` |
62
63
  | Discord | `discrawl --json metadata` | `discrawl sync` |
63
64
  | Slack | `slacrawl --json doctor` | `slacrawl sync` |
@@ -207,6 +208,7 @@ timer, or Task Scheduler).
207
208
  | `index-not-ready` | Index missing or never built | `autorag refresh --json` |
208
209
  | `minsync-unavailable` | MinSync binary missing or install failed | Check cargo, retry refresh |
209
210
  | `jikji-unavailable` | Jikji binary missing or install failed | Check cargo, retry refresh |
211
+ | `everything-index-failed` | Windows Everything instance could not start or index; message carries ES exit code and stderr | Fix the reported cause, `autorag refresh --method everything --json` |
210
212
  | `embedding-identity-mismatch` | Indexed vectors use a different embedder | `autorag index rebuild --method minsync` |
211
213
  | `embedder-unavailable` | Embedding gateway or runtime down | `autorag gateway status --format json` |
212
214
  | `lock-conflict` | Another runtime holds the lock | `autorag gateway stop` |
@@ -110,8 +110,9 @@ autorag lite refresh --force --json
110
110
  incremental refresh is not enough, such as after config changes to roots or
111
111
  index settings.
112
112
  - `--method <csv>` deliberately narrows the refresh. Valid values are
113
- `parsed`, `minsync`, `datasources`, `jikji`, and `all`. Omit the flag to run
114
- all methods. Unknown values are rejected.
113
+ `parsed`, `minsync`, `datasources`, `jikji`, `everything` (Windows only;
114
+ a no-op elsewhere), `fsearch` (macOS/Linux only; a no-op elsewhere), and
115
+ `all`. Omit the flag to run all methods. Unknown values are rejected.
115
116
  - MinSync and Jikji auto-install on first use by default. If they are missing
116
117
  or broken, run a full refresh or return to setup rather than silently
117
118
  degrading to lexical-only search.
@@ -40,6 +40,7 @@ Check `--config`, `AUTORAG_CONFIG`, `$AUTORAG_HOME/config.json`, or
40
40
  - `searchPaths`, `workspacePath`, and `memoryPath`
41
41
  - `model.provider`, `model.id`, `model.api`, `model.baseUrl`, `model.apiKeyEnv`
42
42
  - `bm25`, `minSync`, and `jikji`
43
+ - `limits` (retrieval, baseline-prefetch, and model-facing candidate caps)
43
44
  - `datasources`, `datasourceAccess`, and `ui`
44
45
 
45
46
  Preserve explicit user choices and a working config unless the user asks to
@@ -122,6 +123,13 @@ flags may be omitted. For a custom endpoint, add `api`, `baseUrl`, and
122
123
  }
123
124
  ```
124
125
 
126
+ When `provider`/`id` names a pi-ai catalog model, the catalog entry stays the
127
+ base: `baseUrl`, `api`, and any declared `reasoning`, `input`, `contextWindow`,
128
+ or `maxTokens` override only those fields, and the catalog's reasoning,
129
+ thinking, and compat settings are kept. Only an id outside the catalog (private
130
+ proxy, Ollama, LiteLLM) gets a generic text model with a 128k context window
131
+ unless those fields are declared.
132
+
125
133
  Use `--force` only when intentionally replacing an existing config. Legacy cwd
126
134
  `autorag.config.json` is a migration source only and is never deleted by init.
127
135
 
@@ -178,6 +186,48 @@ Only store the environment-variable name, never its value. Dimension and batch
178
186
  size must be positive integers, and the dimension must match the embedder
179
187
  (default Qwen3 is 1024; legacy EmbeddingGemma is 768; text-embedding-3-small is 1536).
180
188
 
189
+ ### Retrieval and ingest caps
190
+
191
+ `limits` bounds retrieval, baseline prefetch, and the candidate lists handed to
192
+ the model. Every field is optional — an omitted field keeps the shipped default,
193
+ so add the section only to tighten or widen a specific cap. Values must be
194
+ positive integers; unknown keys (and unknown `prefetch` keys) fail config
195
+ resolution.
196
+
197
+ | Field | Default | Controls |
198
+ |---|---|---|
199
+ | `mergedEvidenceCeiling` | 500 | `search_all_documents` / model-free merge ceiling when the model omits `topK` |
200
+ | `singleDatasourceTopK` | 50 | `search_datasource_*` merge default when the model omits `topK` |
201
+ | `minSyncTopK` | 50 | MinSync semantic default `topK` |
202
+ | `minSyncScopedQueryTopK` | 100 | MinSync fetch cap applied when a scope narrows the query |
203
+ | `toolDescriptionInstanceScopes` | 8 | Instance scopes listed in one datasource tool description |
204
+ | `prefetch.jikjiTopK` | 30 | Jikji find candidate count |
205
+ | `prefetch.minSyncTopK` | 100 | MinSync retrieve candidate count |
206
+ | `prefetch.jikjiPathLimit` | 100 | Max Jikji answer paths rendered into the baseline |
207
+ | `prefetch.sectionLimit` | 100 | Max results rendered per baseline section |
208
+
209
+ ```json
210
+ {
211
+ "limits": {
212
+ "mergedEvidenceCeiling": 1000,
213
+ "prefetch": { "jikjiTopK": 12, "sectionLimit": 30 }
214
+ }
215
+ }
216
+ ```
217
+
218
+ The baseline renders each prefetched result's full chunk content (not a
219
+ per-result excerpt) so the model can see where the hit came from. Bound baseline
220
+ size with `prefetch.sectionLimit` / `prefetch.minSyncTopK` /
221
+ `prefetch.jikjiPathLimit`, not with a per-result content truncation.
222
+
223
+ Ingest caps are trusted connector options under `datasources.<name>.connector`
224
+ and are never settable from model/tool arguments: `maxDocuments`,
225
+ `maxItemsPerFeed`, and `maxContentChars` (RSS); `maxDocuments`,
226
+ `maxContentChars`, `maxResultsPerQuery`, and `maxBytesPerFile` (Spotlight);
227
+ `maxDocuments`, `maxContentChars`, `maxBytesPerFile`, `concurrency`,
228
+ `bandwidthLimit`, and `dryRun` (cloud-drive/rclone). `maxContentChars` defaults
229
+ to 20000 for RSS and 100000 for Spotlight and cloud-drive.
230
+
181
231
  ## Probe and configure datasource skills (setup wizard)
182
232
 
183
233
  Use `autorag setup` to probe the local runtime, model profile, and known
@@ -269,9 +319,9 @@ autorag search "summarize the collection" --top-k 3 --json --debug
269
319
  performs one live completion probe.
270
320
  - `health --skip-probes` is only for intentionally offline validation and does
271
321
  not prove live provider access.
272
- - `refresh` syncs parsed mirrors, MinSync, Jikji, and authorized
273
- datasources. `--method <csv>` may deliberately narrow it
274
- (`parsed,minsync,datasources,jikji,all`).
322
+ - `refresh` syncs parsed mirrors, MinSync, Jikji, authorized datasources, and
323
+ on Windows the bundled Everything file-name index. `--method <csv>` may
324
+ deliberately narrow it (`parsed,minsync,datasources,jikji,everything,all`).
275
325
  - Use `refresh --force` for a full resync only when incremental refresh is not
276
326
  enough. Keep destructive reset/rebuild operations scoped to workspace
277
327
  `.autorag` indexes, never source documents.
@@ -0,0 +1,30 @@
1
+ {
2
+ "everythingVersion": "1.4.1.1032",
3
+ "esVersion": "1.1.0.38",
4
+ "licenseFiles": [
5
+ "licenses/everything-MIT-and-PCRE-BSD.txt",
6
+ "licenses/es-MIT.txt"
7
+ ],
8
+ "architectures": {
9
+ "x64": {
10
+ "everythingArchive": "Everything-1.4.1.1032.x64.en-US.zip",
11
+ "everythingArchiveSha256": "705e0b46c6cbf74bd6082a6ebf77afac526868db9d7578f5dc1447f459484611",
12
+ "everythingMember": "everything.exe",
13
+ "everythingSha256": "f191f756996a14a11e5445fa7103d302efd510cf2fbf920e6c0c8ed51d512e36",
14
+ "esArchive": "ES-1.1.0.38.x64.zip",
15
+ "esArchiveSha256": "5e0c70cbf4f694080c34aa7c6c745e606c16fe76a4b5423b93ebf9dc34274c99",
16
+ "esMember": "es.exe",
17
+ "esSha256": "f7378761cf6e01f51c4123a485e628d70e3fea147d341f473ce2820844e5cee5"
18
+ },
19
+ "arm64": {
20
+ "everythingArchive": "Everything-1.4.1.1032.ARM64.en-US.zip",
21
+ "everythingArchiveSha256": "5a7b0cd0372f5e076738e88db20da219fb784c832e61256b3d63f95cd9caeba1",
22
+ "everythingMember": "EverythingARM64.exe",
23
+ "everythingSha256": "462caf17906e56be11ce3d701a6ea51bb1733afae8a4610880f3a6985ee9ca5e",
24
+ "esArchive": "ES-1.1.0.38.ARM64.zip",
25
+ "esArchiveSha256": "86eddd4ae8925833e33be1d09e7faa2442110d00daf0e48a8e963cdeb7a141fe",
26
+ "esMember": "es.exe",
27
+ "esSha256": "7fc406610559f686a95a9bfc4472acffdebcc143b485ae9371b5ef4590e7ec31"
28
+ }
29
+ }
30
+ }