@autorag/librarian 2.10.0 → 2.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +195 -21
- package/NOTICE +16 -23
- package/README.md +399 -182
- package/dist/agent/agent.d.ts +163 -20
- package/dist/agent/context-token-guard.d.ts +23 -0
- package/dist/agent/evidence-ledger.d.ts +54 -5
- package/dist/agent/evidence-persistence.d.ts +24 -0
- package/dist/agent/jev-extension.d.ts +7 -4
- package/dist/agent/jikji-find-tool.d.ts +1 -0
- package/dist/agent/pi-session.d.ts +35 -2
- package/dist/agent/request-limits.d.ts +20 -0
- package/dist/agent/search-all-tool.d.ts +1 -0
- package/dist/agent/search-documents.d.ts +1 -1
- package/dist/agent/search-single-datasource-tool.d.ts +1 -0
- package/dist/agent/system-prompt.d.ts +0 -2
- package/dist/agent/tool-burst-guard.d.ts +26 -0
- package/dist/cli/config.d.ts +39 -23
- package/dist/cli/index.js +4195 -1478
- package/dist/cloud/config.d.ts +33 -9
- package/dist/cloud/credential.d.ts +17 -0
- package/dist/cloud/errors.d.ts +6 -6
- package/dist/cloud/index.d.ts +5 -4
- package/dist/cloud/jev-backend.d.ts +92 -0
- package/dist/cloud/models.d.ts +4 -4
- package/dist/cloud/oauth-login.d.ts +3 -3
- package/dist/cloud/provider.d.ts +16 -11
- package/dist/cloud/token.d.ts +2 -2
- package/dist/core.d.ts +2 -1
- package/dist/core.js +4200 -1506
- package/dist/datasource/aliased-skill.d.ts +2 -2
- package/dist/datasource/connector-skill.d.ts +2 -2
- package/dist/datasource/crawler-skill.d.ts +19 -2
- package/dist/datasource/described-skill.d.ts +2 -2
- package/dist/datasource/index.d.ts +1 -1
- package/dist/datasource/retry.d.ts +49 -0
- package/dist/datasource/sidecar/config.d.ts +26 -0
- package/dist/datasource/sidecar/descriptors.d.ts +67 -0
- package/dist/datasource/sidecar/fusion.d.ts +25 -0
- package/dist/datasource/sidecar/index.d.ts +131 -0
- package/dist/datasource/sidecar/methods.d.ts +27 -0
- package/dist/datasource/skills/clawgallery/skill.d.ts +4 -4
- package/dist/datasource/skills/discrawl/skill.d.ts +5 -5
- package/dist/datasource/skills/github-gist/skill.d.ts +2 -2
- package/dist/datasource/skills/lazykatok/skill.d.ts +5 -5
- package/dist/datasource/skills/mailcrawl/skill.d.ts +2 -2
- package/dist/datasource/skills/obsidian/skill.d.ts +5 -5
- package/dist/datasource/types.d.ts +16 -2
- package/dist/embedding-runtime/cache.d.ts +9 -0
- package/dist/embedding-runtime/index.d.ts +6 -6
- package/dist/embedding-runtime/manifest.d.ts +0 -12
- package/dist/embedding-runtime/types.d.ts +1 -1
- package/dist/index.d.ts +1 -2
- package/dist/index.js +4454 -1757
- package/dist/jikji/client.d.ts +13 -2
- package/dist/jikji/index.d.ts +1 -1
- package/dist/jikji/path-map.d.ts +9 -0
- package/dist/jikji/prepare-budget.d.ts +18 -0
- package/dist/jikji/serial-queue.d.ts +17 -0
- package/dist/jikji/types.d.ts +17 -0
- package/dist/mcp/index.js +4442 -1748
- package/dist/minsync/client.d.ts +6 -4
- package/dist/minsync/installer.d.ts +1 -1
- package/dist/minsync/method.d.ts +1 -1
- package/dist/minsync/process.d.ts +1 -0
- package/dist/minsync/types.d.ts +2 -2
- package/dist/mirror/atomic-write.d.ts +6 -0
- package/dist/mirror/index-store.d.ts +8 -1
- package/dist/mirror/index.d.ts +2 -2
- package/dist/mirror/sync.d.ts +57 -1
- package/dist/p2p/simplex-server.d.ts +0 -1
- package/dist/parser/defaults.d.ts +14 -4
- package/dist/parser/errors.d.ts +10 -0
- package/dist/parser/isolated.d.ts +77 -0
- package/dist/parser/kordoc.d.ts +1 -0
- package/dist/parser/ocr-engines.d.ts +6 -0
- package/dist/parser/parse-worker.d.ts +1 -0
- package/dist/parser/parse-worker.js +780 -0
- package/dist/parser/types.d.ts +5 -0
- package/dist/process/abort-reason.d.ts +2 -0
- package/dist/retrieval/index.d.ts +1 -1
- package/dist/retrieval/rerank.d.ts +28 -5
- package/licenses/embedding-assets.json +0 -35
- package/package.json +26 -29
- package/skills/autorag/SKILL.md +3 -2
- package/skills/autorag-doctor/SKILL.md +7 -1
- package/skills/autorag-lite-setup/SKILL.md +3 -3
- package/skills/autorag-setup/SKILL.md +43 -18
- package/dist/manifest/index.d.ts +0 -2
- package/dist/manifest/loader.d.ts +0 -3
- package/dist/manifest/types.d.ts +0 -8
- package/licenses/gemma-notice.txt +0 -24
- package/licenses/gemma-section-3.2-flow-down.txt +0 -9
- package/licenses/gemma-terms-of-use.html +0 -4750
- package/licenses/gemma-terms-of-use.txt +0 -158
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface ToolBurstGuardOptions {
|
|
2
|
+
/** Calls of one tool inside `windowMs` that count as a runaway loop. */
|
|
3
|
+
readonly maxCalls: number;
|
|
4
|
+
/** Sliding window, in milliseconds. */
|
|
5
|
+
readonly windowMs: number;
|
|
6
|
+
}
|
|
7
|
+
/** One tool called 60 times inside 60 seconds is a loop, not work. */
|
|
8
|
+
export declare const DEFAULT_TOOL_BURST_GUARD: ToolBurstGuardOptions;
|
|
9
|
+
/**
|
|
10
|
+
* Detects a runaway tool loop: the same tool called `maxCalls` times inside
|
|
11
|
+
* `windowMs`. It limits the rate of one tool, never a run's total length or
|
|
12
|
+
* duration, so a long indexing job or a many-step run is never cut short.
|
|
13
|
+
* It reports every call that is part of a burst; `reset()` clears it for the next run.
|
|
14
|
+
*/
|
|
15
|
+
export declare class ToolBurstGuard {
|
|
16
|
+
private readonly options;
|
|
17
|
+
private readonly calls;
|
|
18
|
+
constructor(options?: ToolBurstGuardOptions | false);
|
|
19
|
+
/**
|
|
20
|
+
* Record one call. Returns true while the tool is inside a burst: on the call that
|
|
21
|
+
* reaches `maxCalls` and on every further call in the window. An abort issued
|
|
22
|
+
* mid-tool does not always stop the next model turn, so the caller keeps aborting.
|
|
23
|
+
*/
|
|
24
|
+
record(tool: string, now?: number): boolean;
|
|
25
|
+
reset(): void;
|
|
26
|
+
}
|
package/dist/cli/config.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
2
|
import { ModelRuntime } from "@earendil-works/pi-coding-agent";
|
|
3
|
-
import type { AutoRAGAgentOptions, AutoRAGRetrievalLimits } from "../agent/agent.js";
|
|
3
|
+
import type { AutoRAGAgentOptions, AutoRAGRefreshLimits, AutoRAGRetrievalLimits } from "../agent/agent.js";
|
|
4
4
|
import type { JevBackendName } from "../agent/jev-extension.js";
|
|
5
5
|
import { type LoadLocalAutoRAGModelOptions } from "../agent/local-model.js";
|
|
6
6
|
import type { DecompositionModel } from "../agent/query-decomposition.js";
|
|
@@ -55,13 +55,14 @@ export interface WebSearchCliConfig {
|
|
|
55
55
|
}
|
|
56
56
|
/**
|
|
57
57
|
* Jev decision-tool config. Absent disables the tool; `enabled: false` disables
|
|
58
|
-
* it explicitly. Secrets never appear here:
|
|
59
|
-
*
|
|
60
|
-
* `OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`)
|
|
58
|
+
* it explicitly. Secrets never appear here: `jev-use` reads the backend
|
|
59
|
+
* credential from its own environment variable (`TYPESAFE_API_KEY`,
|
|
60
|
+
* `OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`), and the hosted Dazzi Models
|
|
61
|
+
* `dazzi` backend uses the credential stored by `/login`.
|
|
61
62
|
*/
|
|
62
63
|
export interface JevCliConfig {
|
|
63
64
|
enabled?: boolean;
|
|
64
|
-
/** Force one backend; omit to let the first credential present win. */
|
|
65
|
+
/** Force one backend (`openrouter`, `typesafe`, `vercel`, or `dazzi`); omit to let the first credential present win. */
|
|
65
66
|
backend?: JevBackendName;
|
|
66
67
|
/** Model id sent with every call, e.g. `jev-latest`. */
|
|
67
68
|
model?: string;
|
|
@@ -75,20 +76,22 @@ export interface QueryDecompositionConfig {
|
|
|
75
76
|
}
|
|
76
77
|
/**
|
|
77
78
|
* Post-merge reranking config. Routes merged evidence through a dedicated
|
|
78
|
-
* rerank model. `provider` is `openrouter`
|
|
79
|
-
*
|
|
80
|
-
*
|
|
79
|
+
* rerank model. `provider` is `openrouter` (default) or the hosted Dazzi Models
|
|
80
|
+
* plan (`dazzi`); each provider supplies its own default `model`. OpenRouter
|
|
81
|
+
* reads its key from `apiKeyEnv`; Dazzi uses the credential stored by `/login`.
|
|
82
|
+
* Secrets never appear here — only the environment-variable name that holds a
|
|
83
|
+
* provider API key.
|
|
81
84
|
*/
|
|
82
85
|
export interface RerankConfig {
|
|
83
86
|
/** `false` disables reranking. Missing means enabled when the block is present. */
|
|
84
87
|
enabled?: boolean;
|
|
85
|
-
/** Provider id. @default "openrouter" */
|
|
88
|
+
/** Provider id (`openrouter` or `dazzi`). @default "openrouter" */
|
|
86
89
|
provider?: string;
|
|
87
|
-
/** Wire model id. @default "voyageai/rerank-3-lite" */
|
|
90
|
+
/** Wire model id. @default "voyageai/rerank-3-lite" (`openrouter`) / "qwen/qwen3-reranker-8b" (`dazzi`) */
|
|
88
91
|
model?: string;
|
|
89
|
-
/** Environment variable holding the provider API key. @default "OPENROUTER_API_KEY" */
|
|
92
|
+
/** Environment variable holding the provider API key (`openrouter` only). @default "OPENROUTER_API_KEY" */
|
|
90
93
|
apiKeyEnv?: string;
|
|
91
|
-
/** Override the provider base URL (e.g. a gateway)
|
|
94
|
+
/** Override the provider base URL (e.g. a gateway); for `dazzi`, the API base including `/v1`. */
|
|
92
95
|
baseUrl?: string;
|
|
93
96
|
/** Return only the top N merged results. @default 25 (DEFAULT_RERANK_TOP_N) */
|
|
94
97
|
topN?: number;
|
|
@@ -110,7 +113,6 @@ export interface P2pConfig {
|
|
|
110
113
|
};
|
|
111
114
|
injectionClassifier?: boolean;
|
|
112
115
|
piiNer?: boolean;
|
|
113
|
-
searchTimeoutMs?: number;
|
|
114
116
|
newFilesPublic?: boolean;
|
|
115
117
|
}
|
|
116
118
|
export interface AgentModelConfig {
|
|
@@ -180,7 +182,7 @@ export interface CliConfig {
|
|
|
180
182
|
/**
|
|
181
183
|
* Question decomposition for the Jev query pipeline. `model` names the LLM
|
|
182
184
|
* that splits one question into at most five search queries; absent or `{}`
|
|
183
|
-
* uses {@link
|
|
185
|
+
* uses {@link defaultQueryDecompositionModel}, and `false` lets the
|
|
184
186
|
* agent's own model decompose.
|
|
185
187
|
*/
|
|
186
188
|
queryDecomposition?: QueryDecompositionConfig | false;
|
|
@@ -197,6 +199,8 @@ export interface CliConfig {
|
|
|
197
199
|
excludePaths?: string[];
|
|
198
200
|
/** Hard caps on retrieval, baseline prefetch, and model-facing candidate lists. */
|
|
199
201
|
limits?: AutoRAGRetrievalLimits;
|
|
202
|
+
/** Refresh limits (delete-ratio valve, walk caps) and `transientRetries` (default 3, EMFILE/EBUSY/EIO/ETIMEDOUT per file). */
|
|
203
|
+
refresh?: AutoRAGRefreshLimits;
|
|
200
204
|
/** Datasource skill configuration (skill name → config). */
|
|
201
205
|
datasources?: DatasourcesConfig;
|
|
202
206
|
/** P2P sharing configuration. Disabled by default. */
|
|
@@ -235,6 +239,8 @@ export declare function normalizeEmbedder(raw: unknown, path: string): MinSyncEm
|
|
|
235
239
|
export declare function normalizeIndexingConfig(raw: RawIndexingMethods): NormalizedIndexingConfig;
|
|
236
240
|
/** Validate the `limits` section and map it onto the agent option. */
|
|
237
241
|
export declare function normalizeLimitsConfig(raw: unknown): AutoRAGRetrievalLimits;
|
|
242
|
+
/** Validate the `refresh` section and map it onto the agent option. */
|
|
243
|
+
export declare function normalizeRefreshConfig(raw: unknown): AutoRAGRefreshLimits;
|
|
238
244
|
export declare function resolveConfig(input: ResolveConfigInput): CliConfig;
|
|
239
245
|
/**
|
|
240
246
|
* Resolve config without writing, migrating, or locking. Health and other
|
|
@@ -244,20 +250,30 @@ export declare function resolveConfig(input: ResolveConfigInput): CliConfig;
|
|
|
244
250
|
export declare function resolveConfigReadOnly(input: ResolveConfigInput): CliConfig;
|
|
245
251
|
/** Validate and normalize the `jev` config section. */
|
|
246
252
|
export declare function normalizeJevConfig(raw: unknown): JevCliConfig;
|
|
247
|
-
/** Jev backend used when the config names none (absent section or `{}`). */
|
|
248
|
-
export declare const DEFAULT_JEV_BACKEND: JevBackendName;
|
|
249
253
|
/**
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
* covering, language-preserving decompositions at ~0.9s p50, about 3.5x
|
|
253
|
-
* cheaper per call than the previous default (a 2025 Gemini flash-lite) at equal quality.
|
|
254
|
+
* Jev backend used when the config names none (absent section or `{}`): the
|
|
255
|
+
* hosted Dazzi Models plan when the user is signed in, else OpenRouter.
|
|
254
256
|
*/
|
|
255
|
-
export declare
|
|
257
|
+
export declare function defaultJevBackend(agentDir?: string): JevBackendName;
|
|
258
|
+
/** The OpenRouter fallback decomposition model, used when not signed in to Dazzi Models. */
|
|
259
|
+
export declare const OPENROUTER_QUERY_DECOMPOSITION_MODEL: AgentModelConfig;
|
|
260
|
+
/**
|
|
261
|
+
* Question-decomposition model used when the config names none: the hosted
|
|
262
|
+
* Dazzi Models plan when signed in, else OpenRouter. The OpenRouter pick came
|
|
263
|
+
* from a live bench (6 questions incl. Korean, 2 runs each): 12/12 valid,
|
|
264
|
+
* covering, language-preserving decompositions at ~0.9s p50.
|
|
265
|
+
*/
|
|
266
|
+
export declare function defaultQueryDecompositionModel(agentDir?: string): AgentModelConfig;
|
|
256
267
|
/** Validate the `queryDecomposition` config section (`false` = decompose with the session model). */
|
|
257
268
|
export declare function normalizeQueryDecompositionConfig(raw: unknown): QueryDecompositionConfig | false;
|
|
258
269
|
/** Normalize and validate the `rerank` config section, filling in defaults. */
|
|
259
270
|
export declare function normalizeRerankConfig(raw: unknown, path: string): RerankConfig | false;
|
|
260
|
-
|
|
271
|
+
/**
|
|
272
|
+
* Rerank block `autorag init` writes when none is given: the hosted Dazzi
|
|
273
|
+
* Models reranker when signed in, else OpenRouter's.
|
|
274
|
+
*/
|
|
275
|
+
export declare function defaultRerankConfig(agentDir?: string): RerankConfig;
|
|
276
|
+
export declare function buildAgentOptions(config: CliConfig, agentDir?: string): Omit<AutoRAGAgentOptions, "model">;
|
|
261
277
|
/**
|
|
262
278
|
* Extra resolution inputs beyond the local codex-runtime options. `agentDir`
|
|
263
279
|
* points at the pi agent home (`~/.pi/agent` by default) that owns
|
|
@@ -314,7 +330,7 @@ export declare function validateAgentConfigFile(configPath: string, options?: Re
|
|
|
314
330
|
/**
|
|
315
331
|
* Resolve the question-decomposition model and its credential through the same
|
|
316
332
|
* chain as the agent model. An absent section or `{}` uses
|
|
317
|
-
* {@link
|
|
333
|
+
* {@link defaultQueryDecompositionModel}; `queryDecomposition: false`
|
|
318
334
|
* returns undefined so the agent's own model decomposes.
|
|
319
335
|
*/
|
|
320
336
|
export declare function resolveQueryDecompositionModel(config: CliConfig, options?: ResolveAgentModelOptions): Promise<DecompositionModel | undefined>;
|