@autorag/librarian 2.10.0 → 2.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/LICENSE +195 -21
  2. package/NOTICE +16 -23
  3. package/README.md +399 -182
  4. package/dist/agent/agent.d.ts +163 -20
  5. package/dist/agent/context-token-guard.d.ts +23 -0
  6. package/dist/agent/evidence-ledger.d.ts +54 -5
  7. package/dist/agent/evidence-persistence.d.ts +24 -0
  8. package/dist/agent/jev-extension.d.ts +7 -4
  9. package/dist/agent/jikji-find-tool.d.ts +1 -0
  10. package/dist/agent/pi-session.d.ts +35 -2
  11. package/dist/agent/request-limits.d.ts +20 -0
  12. package/dist/agent/search-all-tool.d.ts +1 -0
  13. package/dist/agent/search-documents.d.ts +1 -1
  14. package/dist/agent/search-single-datasource-tool.d.ts +1 -0
  15. package/dist/agent/system-prompt.d.ts +0 -2
  16. package/dist/agent/tool-burst-guard.d.ts +26 -0
  17. package/dist/cli/config.d.ts +39 -23
  18. package/dist/cli/index.js +4195 -1478
  19. package/dist/cloud/config.d.ts +33 -9
  20. package/dist/cloud/credential.d.ts +17 -0
  21. package/dist/cloud/errors.d.ts +6 -6
  22. package/dist/cloud/index.d.ts +5 -4
  23. package/dist/cloud/jev-backend.d.ts +92 -0
  24. package/dist/cloud/models.d.ts +4 -4
  25. package/dist/cloud/oauth-login.d.ts +3 -3
  26. package/dist/cloud/provider.d.ts +16 -11
  27. package/dist/cloud/token.d.ts +2 -2
  28. package/dist/core.d.ts +2 -1
  29. package/dist/core.js +4200 -1506
  30. package/dist/datasource/aliased-skill.d.ts +2 -2
  31. package/dist/datasource/connector-skill.d.ts +2 -2
  32. package/dist/datasource/crawler-skill.d.ts +19 -2
  33. package/dist/datasource/described-skill.d.ts +2 -2
  34. package/dist/datasource/index.d.ts +1 -1
  35. package/dist/datasource/retry.d.ts +49 -0
  36. package/dist/datasource/sidecar/config.d.ts +26 -0
  37. package/dist/datasource/sidecar/descriptors.d.ts +67 -0
  38. package/dist/datasource/sidecar/fusion.d.ts +25 -0
  39. package/dist/datasource/sidecar/index.d.ts +131 -0
  40. package/dist/datasource/sidecar/methods.d.ts +27 -0
  41. package/dist/datasource/skills/clawgallery/skill.d.ts +4 -4
  42. package/dist/datasource/skills/discrawl/skill.d.ts +5 -5
  43. package/dist/datasource/skills/github-gist/skill.d.ts +2 -2
  44. package/dist/datasource/skills/lazykatok/skill.d.ts +5 -5
  45. package/dist/datasource/skills/mailcrawl/skill.d.ts +2 -2
  46. package/dist/datasource/skills/obsidian/skill.d.ts +5 -5
  47. package/dist/datasource/types.d.ts +16 -2
  48. package/dist/embedding-runtime/cache.d.ts +9 -0
  49. package/dist/embedding-runtime/index.d.ts +6 -6
  50. package/dist/embedding-runtime/manifest.d.ts +0 -12
  51. package/dist/embedding-runtime/types.d.ts +1 -1
  52. package/dist/index.d.ts +1 -2
  53. package/dist/index.js +4454 -1757
  54. package/dist/jikji/client.d.ts +13 -2
  55. package/dist/jikji/index.d.ts +1 -1
  56. package/dist/jikji/path-map.d.ts +9 -0
  57. package/dist/jikji/prepare-budget.d.ts +18 -0
  58. package/dist/jikji/serial-queue.d.ts +17 -0
  59. package/dist/jikji/types.d.ts +17 -0
  60. package/dist/mcp/index.js +4442 -1748
  61. package/dist/minsync/client.d.ts +6 -4
  62. package/dist/minsync/installer.d.ts +1 -1
  63. package/dist/minsync/method.d.ts +1 -1
  64. package/dist/minsync/process.d.ts +1 -0
  65. package/dist/minsync/types.d.ts +2 -2
  66. package/dist/mirror/atomic-write.d.ts +6 -0
  67. package/dist/mirror/index-store.d.ts +8 -1
  68. package/dist/mirror/index.d.ts +2 -2
  69. package/dist/mirror/sync.d.ts +57 -1
  70. package/dist/p2p/simplex-server.d.ts +0 -1
  71. package/dist/parser/defaults.d.ts +14 -4
  72. package/dist/parser/errors.d.ts +10 -0
  73. package/dist/parser/isolated.d.ts +77 -0
  74. package/dist/parser/kordoc.d.ts +1 -0
  75. package/dist/parser/ocr-engines.d.ts +6 -0
  76. package/dist/parser/parse-worker.d.ts +1 -0
  77. package/dist/parser/parse-worker.js +780 -0
  78. package/dist/parser/types.d.ts +5 -0
  79. package/dist/process/abort-reason.d.ts +2 -0
  80. package/dist/retrieval/index.d.ts +1 -1
  81. package/dist/retrieval/rerank.d.ts +28 -5
  82. package/licenses/embedding-assets.json +0 -35
  83. package/package.json +26 -29
  84. package/skills/autorag/SKILL.md +3 -2
  85. package/skills/autorag-doctor/SKILL.md +7 -1
  86. package/skills/autorag-lite-setup/SKILL.md +3 -3
  87. package/skills/autorag-setup/SKILL.md +43 -18
  88. package/dist/manifest/index.d.ts +0 -2
  89. package/dist/manifest/loader.d.ts +0 -3
  90. package/dist/manifest/types.d.ts +0 -8
  91. package/licenses/gemma-notice.txt +0 -24
  92. package/licenses/gemma-section-3.2-flow-down.txt +0 -9
  93. package/licenses/gemma-terms-of-use.html +0 -4750
  94. package/licenses/gemma-terms-of-use.txt +0 -158
@@ -0,0 +1,26 @@
1
+ export interface ToolBurstGuardOptions {
2
+ /** Calls of one tool inside `windowMs` that count as a runaway loop. */
3
+ readonly maxCalls: number;
4
+ /** Sliding window, in milliseconds. */
5
+ readonly windowMs: number;
6
+ }
7
+ /** One tool called 60 times inside 60 seconds is a loop, not work. */
8
+ export declare const DEFAULT_TOOL_BURST_GUARD: ToolBurstGuardOptions;
9
+ /**
10
+ * Detects a runaway tool loop: the same tool called `maxCalls` times inside
11
+ * `windowMs`. It limits the rate of one tool, never a run's total length or
12
+ * duration, so a long indexing job or a many-step run is never cut short.
13
+ * It reports every call that is part of a burst; `reset()` clears it for the next run.
14
+ */
15
+ export declare class ToolBurstGuard {
16
+ private readonly options;
17
+ private readonly calls;
18
+ constructor(options?: ToolBurstGuardOptions | false);
19
+ /**
20
+ * Record one call. Returns true while the tool is inside a burst: on the call that
21
+ * reaches `maxCalls` and on every further call in the window. An abort issued
22
+ * mid-tool does not always stop the next model turn, so the caller keeps aborting.
23
+ */
24
+ record(tool: string, now?: number): boolean;
25
+ reset(): void;
26
+ }
@@ -1,6 +1,6 @@
1
1
  import type { Api, Model } from "@earendil-works/pi-ai";
2
2
  import { ModelRuntime } from "@earendil-works/pi-coding-agent";
3
- import type { AutoRAGAgentOptions, AutoRAGRetrievalLimits } from "../agent/agent.js";
3
+ import type { AutoRAGAgentOptions, AutoRAGRefreshLimits, AutoRAGRetrievalLimits } from "../agent/agent.js";
4
4
  import type { JevBackendName } from "../agent/jev-extension.js";
5
5
  import { type LoadLocalAutoRAGModelOptions } from "../agent/local-model.js";
6
6
  import type { DecompositionModel } from "../agent/query-decomposition.js";
@@ -55,13 +55,14 @@ export interface WebSearchCliConfig {
55
55
  }
56
56
  /**
57
57
  * Jev decision-tool config. Absent disables the tool; `enabled: false` disables
58
- * it explicitly. Secrets never appear here: the `jev-use` engine reads the
59
- * backend credential from its own environment variable (`TYPESAFE_API_KEY`,
60
- * `OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`).
58
+ * it explicitly. Secrets never appear here: `jev-use` reads the backend
59
+ * credential from its own environment variable (`TYPESAFE_API_KEY`,
60
+ * `OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`), and the hosted Dazzi Models
61
+ * `dazzi` backend uses the credential stored by `/login`.
61
62
  */
62
63
  export interface JevCliConfig {
63
64
  enabled?: boolean;
64
- /** Force one backend; omit to let the first credential present win. */
65
+ /** Force one backend (`openrouter`, `typesafe`, `vercel`, or `dazzi`); omit to let the first credential present win. */
65
66
  backend?: JevBackendName;
66
67
  /** Model id sent with every call, e.g. `jev-latest`. */
67
68
  model?: string;
@@ -75,20 +76,22 @@ export interface QueryDecompositionConfig {
75
76
  }
76
77
  /**
77
78
  * Post-merge reranking config. Routes merged evidence through a dedicated
78
- * rerank model. `provider` is `openrouter` today; `model` is the OpenRouter
79
- * wire id (default `voyageai/rerank-3-lite`). Secrets never appear here — only
80
- * the environment-variable name that holds the provider API key.
79
+ * rerank model. `provider` is `openrouter` (default) or the hosted Dazzi Models
80
+ * plan (`dazzi`); each provider supplies its own default `model`. OpenRouter
81
+ * reads its key from `apiKeyEnv`; Dazzi uses the credential stored by `/login`.
82
+ * Secrets never appear here — only the environment-variable name that holds a
83
+ * provider API key.
81
84
  */
82
85
  export interface RerankConfig {
83
86
  /** `false` disables reranking. Missing means enabled when the block is present. */
84
87
  enabled?: boolean;
85
- /** Provider id. @default "openrouter" */
88
+ /** Provider id (`openrouter` or `dazzi`). @default "openrouter" */
86
89
  provider?: string;
87
- /** Wire model id. @default "voyageai/rerank-3-lite" */
90
+ /** Wire model id. @default "voyageai/rerank-3-lite" (`openrouter`) / "qwen/qwen3-reranker-8b" (`dazzi`) */
88
91
  model?: string;
89
- /** Environment variable holding the provider API key. @default "OPENROUTER_API_KEY" */
92
+ /** Environment variable holding the provider API key (`openrouter` only). @default "OPENROUTER_API_KEY" */
90
93
  apiKeyEnv?: string;
91
- /** Override the provider base URL (e.g. a gateway). */
94
+ /** Override the provider base URL (e.g. a gateway); for `dazzi`, the API base including `/v1`. */
92
95
  baseUrl?: string;
93
96
  /** Return only the top N merged results. @default 25 (DEFAULT_RERANK_TOP_N) */
94
97
  topN?: number;
@@ -110,7 +113,6 @@ export interface P2pConfig {
110
113
  };
111
114
  injectionClassifier?: boolean;
112
115
  piiNer?: boolean;
113
- searchTimeoutMs?: number;
114
116
  newFilesPublic?: boolean;
115
117
  }
116
118
  export interface AgentModelConfig {
@@ -180,7 +182,7 @@ export interface CliConfig {
180
182
  /**
181
183
  * Question decomposition for the Jev query pipeline. `model` names the LLM
182
184
  * that splits one question into at most five search queries; absent or `{}`
183
- * uses {@link DEFAULT_QUERY_DECOMPOSITION_MODEL}, and `false` lets the
185
+ * uses {@link defaultQueryDecompositionModel}, and `false` lets the
184
186
  * agent's own model decompose.
185
187
  */
186
188
  queryDecomposition?: QueryDecompositionConfig | false;
@@ -197,6 +199,8 @@ export interface CliConfig {
197
199
  excludePaths?: string[];
198
200
  /** Hard caps on retrieval, baseline prefetch, and model-facing candidate lists. */
199
201
  limits?: AutoRAGRetrievalLimits;
202
+ /** Refresh limits (delete-ratio valve, walk caps) and `transientRetries` (default 3, EMFILE/EBUSY/EIO/ETIMEDOUT per file). */
203
+ refresh?: AutoRAGRefreshLimits;
200
204
  /** Datasource skill configuration (skill name → config). */
201
205
  datasources?: DatasourcesConfig;
202
206
  /** P2P sharing configuration. Disabled by default. */
@@ -235,6 +239,8 @@ export declare function normalizeEmbedder(raw: unknown, path: string): MinSyncEm
235
239
  export declare function normalizeIndexingConfig(raw: RawIndexingMethods): NormalizedIndexingConfig;
236
240
  /** Validate the `limits` section and map it onto the agent option. */
237
241
  export declare function normalizeLimitsConfig(raw: unknown): AutoRAGRetrievalLimits;
242
+ /** Validate the `refresh` section and map it onto the agent option. */
243
+ export declare function normalizeRefreshConfig(raw: unknown): AutoRAGRefreshLimits;
238
244
  export declare function resolveConfig(input: ResolveConfigInput): CliConfig;
239
245
  /**
240
246
  * Resolve config without writing, migrating, or locking. Health and other
@@ -244,20 +250,30 @@ export declare function resolveConfig(input: ResolveConfigInput): CliConfig;
244
250
  export declare function resolveConfigReadOnly(input: ResolveConfigInput): CliConfig;
245
251
  /** Validate and normalize the `jev` config section. */
246
252
  export declare function normalizeJevConfig(raw: unknown): JevCliConfig;
247
- /** Jev backend used when the config names none (absent section or `{}`). */
248
- export declare const DEFAULT_JEV_BACKEND: JevBackendName;
249
253
  /**
250
- * Question-decomposition model used when the config names none. Picked from a
251
- * live OpenRouter bench (6 questions incl. Korean, 2 runs each): 12/12 valid,
252
- * covering, language-preserving decompositions at ~0.9s p50, about 3.5x
253
- * cheaper per call than the previous default (a 2025 Gemini flash-lite) at equal quality.
254
+ * Jev backend used when the config names none (absent section or `{}`): the
255
+ * hosted Dazzi Models plan when the user is signed in, else OpenRouter.
254
256
  */
255
- export declare const DEFAULT_QUERY_DECOMPOSITION_MODEL: AgentModelConfig;
257
+ export declare function defaultJevBackend(agentDir?: string): JevBackendName;
258
+ /** The OpenRouter fallback decomposition model, used when not signed in to Dazzi Models. */
259
+ export declare const OPENROUTER_QUERY_DECOMPOSITION_MODEL: AgentModelConfig;
260
+ /**
261
+ * Question-decomposition model used when the config names none: the hosted
262
+ * Dazzi Models plan when signed in, else OpenRouter. The OpenRouter pick came
263
+ * from a live bench (6 questions incl. Korean, 2 runs each): 12/12 valid,
264
+ * covering, language-preserving decompositions at ~0.9s p50.
265
+ */
266
+ export declare function defaultQueryDecompositionModel(agentDir?: string): AgentModelConfig;
256
267
  /** Validate the `queryDecomposition` config section (`false` = decompose with the session model). */
257
268
  export declare function normalizeQueryDecompositionConfig(raw: unknown): QueryDecompositionConfig | false;
258
269
  /** Normalize and validate the `rerank` config section, filling in defaults. */
259
270
  export declare function normalizeRerankConfig(raw: unknown, path: string): RerankConfig | false;
260
- export declare function buildAgentOptions(config: CliConfig): Omit<AutoRAGAgentOptions, "model">;
271
+ /**
272
+ * Rerank block `autorag init` writes when none is given: the hosted Dazzi
273
+ * Models reranker when signed in, else OpenRouter's.
274
+ */
275
+ export declare function defaultRerankConfig(agentDir?: string): RerankConfig;
276
+ export declare function buildAgentOptions(config: CliConfig, agentDir?: string): Omit<AutoRAGAgentOptions, "model">;
261
277
  /**
262
278
  * Extra resolution inputs beyond the local codex-runtime options. `agentDir`
263
279
  * points at the pi agent home (`~/.pi/agent` by default) that owns
@@ -314,7 +330,7 @@ export declare function validateAgentConfigFile(configPath: string, options?: Re
314
330
  /**
315
331
  * Resolve the question-decomposition model and its credential through the same
316
332
  * chain as the agent model. An absent section or `{}` uses
317
- * {@link DEFAULT_QUERY_DECOMPOSITION_MODEL}; `queryDecomposition: false`
333
+ * {@link defaultQueryDecompositionModel}; `queryDecomposition: false`
318
334
  * returns undefined so the agent's own model decomposes.
319
335
  */
320
336
  export declare function resolveQueryDecompositionModel(config: CliConfig, options?: ResolveAgentModelOptions): Promise<DecompositionModel | undefined>;