@autorag/librarian 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/NOTICE +10 -4
  2. package/README.md +122 -38
  3. package/dist/agent/agent.d.ts +210 -53
  4. package/dist/agent/answer-guidelines.d.ts +19 -0
  5. package/dist/agent/citations.d.ts +43 -0
  6. package/dist/agent/datasource-skill.d.ts +30 -16
  7. package/dist/agent/emit-results-tool.d.ts +37 -2
  8. package/dist/agent/evidence-judgment.d.ts +58 -0
  9. package/dist/agent/evidence-ledger.d.ts +59 -0
  10. package/dist/agent/evidence-origins.d.ts +49 -0
  11. package/dist/agent/fast-answer-tool.d.ts +16 -6
  12. package/dist/agent/index.d.ts +2 -1
  13. package/dist/agent/jev-extension.d.ts +68 -0
  14. package/dist/agent/jikji-find-tool.d.ts +2 -1
  15. package/dist/agent/pi-session.d.ts +28 -1
  16. package/dist/agent/query-decomposition.d.ts +25 -0
  17. package/dist/agent/query-peer-tool.d.ts +3 -0
  18. package/dist/agent/query-routing.d.ts +115 -0
  19. package/dist/agent/search-all-tool.d.ts +5 -5
  20. package/dist/agent/search-documents.d.ts +3 -11
  21. package/dist/agent/search-minsync-tool.d.ts +5 -5
  22. package/dist/agent/search-single-datasource-tool.d.ts +9 -8
  23. package/dist/agent/self-config.d.ts +40 -0
  24. package/dist/agent/system-prompt.d.ts +3 -3
  25. package/dist/agent/web-fetch-tool.d.ts +2 -1
  26. package/dist/agent/web-search-tool.d.ts +2 -1
  27. package/dist/cli/commands/evidence.d.ts +22 -0
  28. package/dist/cli/commands/init.d.ts +2 -1
  29. package/dist/cli/commands/memory.d.ts +1 -1
  30. package/dist/cli/commands/report.d.ts +1 -1
  31. package/dist/cli/commands/update-check.d.ts +8 -0
  32. package/dist/cli/config.d.ts +80 -7
  33. package/dist/cli/index.d.ts +1 -1
  34. package/dist/cli/index.js +3844 -2291
  35. package/dist/cli/output.d.ts +2 -6
  36. package/dist/cli/update-check.d.ts +34 -0
  37. package/dist/cli/version.d.ts +5 -0
  38. package/dist/config/home.d.ts +6 -0
  39. package/dist/core.d.ts +30 -6
  40. package/dist/core.js +7742 -6225
  41. package/dist/datasource/chunk-store.d.ts +2 -6
  42. package/dist/datasource/connector-skill.d.ts +2 -3
  43. package/dist/datasource/index.d.ts +2 -4
  44. package/dist/datasource/registry.d.ts +11 -13
  45. package/dist/datasource/scope.d.ts +19 -0
  46. package/dist/datasource/skills/github-gist/index.d.ts +1 -1
  47. package/dist/datasource/skills/github-gist/semantic.d.ts +3 -44
  48. package/dist/datasource/skills/github-gist/skill.d.ts +3 -3
  49. package/dist/datasource/types.d.ts +14 -28
  50. package/dist/embedding-runtime/gateway-embedder.d.ts +41 -0
  51. package/dist/everything/bundle.d.ts +15 -3
  52. package/dist/everything/client.d.ts +22 -5
  53. package/dist/everything/index.d.ts +1 -1
  54. package/dist/filesystem/name-search.d.ts +57 -0
  55. package/dist/fsearch/client.d.ts +7 -1
  56. package/dist/index.d.ts +2 -1
  57. package/dist/index.js +9139 -6376
  58. package/dist/jikji/client.d.ts +6 -1
  59. package/dist/language.d.ts +2 -1
  60. package/dist/mcp/index.d.ts +2 -0
  61. package/dist/mcp/index.js +30542 -0
  62. package/dist/mcp/server.d.ts +9 -0
  63. package/dist/memory/check-memory-tool.d.ts +8 -4
  64. package/dist/memory/context.d.ts +51 -0
  65. package/dist/memory/index.d.ts +8 -1
  66. package/dist/memory/judged-evidence.d.ts +32 -0
  67. package/dist/memory/memory.d.ts +36 -120
  68. package/dist/memory/renderer.d.ts +16 -7
  69. package/dist/memory/similar-queries.d.ts +40 -0
  70. package/dist/minsync/errors.d.ts +14 -0
  71. package/dist/minsync/index.d.ts +1 -0
  72. package/dist/minsync/method.d.ts +8 -5
  73. package/dist/mirror/index.d.ts +1 -1
  74. package/dist/mirror/paths.d.ts +6 -0
  75. package/dist/mirror/sync.d.ts +5 -2
  76. package/dist/parser/defaults.d.ts +16 -5
  77. package/dist/parser/index.d.ts +1 -1
  78. package/dist/parser/kordoc.d.ts +8 -1
  79. package/dist/parser/ocr-engines.d.ts +19 -0
  80. package/dist/retrieval/bm25.d.ts +19 -0
  81. package/dist/retrieval/engine.d.ts +33 -43
  82. package/dist/retrieval/index.d.ts +3 -1
  83. package/dist/retrieval/merger.d.ts +6 -0
  84. package/dist/retrieval/rerank.d.ts +4 -2
  85. package/dist/retrieval/selection.d.ts +89 -0
  86. package/dist/retrieval/types.d.ts +1 -3
  87. package/dist/web/search/format.d.ts +5 -2
  88. package/dist/web/search/index.d.ts +7 -1
  89. package/package.json +11 -7
  90. package/skills/autorag/SKILL.md +43 -23
  91. package/skills/autorag-doctor/SKILL.md +40 -26
  92. package/skills/autorag-lite-setup/SKILL.md +123 -64
  93. package/skills/autorag-setup/SKILL.md +107 -22
  94. package/dist/cli/commands/feedback.d.ts +0 -8
  95. package/dist/datasource/access-context.d.ts +0 -43
  96. package/dist/datasource/result-filter.d.ts +0 -64
  97. package/dist/parser/ocr.d.ts +0 -31
  98. package/skills/autorag-lite-search/SKILL.md +0 -245
@@ -1,13 +1,12 @@
1
- import type { Agent, AgentTool, Skill } from "@earendil-works/pi-agent-core";
1
+ import type { Agent, AgentTool } from "@earendil-works/pi-agent-core";
2
2
  import type { Api, Model } from "@earendil-works/pi-ai";
3
- import { type DatasourceAccessContextOptions } from "../datasource/access-context.js";
4
3
  import type { DatasourceIndexResult, DatasourceSkill } from "../datasource/types.js";
5
4
  import { type DupeyCliOptions } from "../dupey/index.js";
5
+ import { type Embedder } from "../embedding-runtime/gateway-embedder.js";
6
6
  import { type EverythingClientOptions, type EverythingSearchRequest, type EverythingSearchResult } from "../everything/index.js";
7
7
  import { type FSearchClientOptions, type FSearchFailureReason, type FSearchSearchRequest, type FSearchSearchResult } from "../fsearch/index.js";
8
8
  import { type JikjiFailureReason, type JikjiOptions } from "../jikji/index.js";
9
9
  import { type LanguageTag } from "../language.js";
10
- import type { ResultFeedback } from "../memory/memory.js";
11
10
  import { type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
12
11
  import { type ParsedMirrorSyncResult } from "../mirror/sync.js";
13
12
  import type { PolicyResolver } from "../p2p/policy-filter.js";
@@ -17,13 +16,17 @@ import type { PeerQueryResponse } from "../p2p/wire.js";
17
16
  import { type DefaultParserRegistryOptions } from "../parser/index.js";
18
17
  import { RetrievalEngine } from "../retrieval/engine.js";
19
18
  import { RetrievalMethodRegistry } from "../retrieval/registry.js";
19
+ import type { DatasourceCatalogEntry } from "../retrieval/selection.js";
20
20
  import type { CuratedResult, RetrievalDiagnostic, RetrievalOptions, RetrievalResult } from "../retrieval/types.js";
21
+ import { type DatasourceAgentSkill } from "./datasource-skill.js";
21
22
  import { type ScanDuplicateDocumentsDetails } from "./dupey-tool.js";
22
23
  import { type AutoRAGFastAnswerDetails } from "./fast-answer-tool.js";
24
+ import { type JevToolOptions } from "./jev-extension.js";
23
25
  import { type JikjiFindProviderResult } from "./jikji-find-tool.js";
24
26
  import { type AutoRAGPiInteractiveRuntime, type AutoRAGPiSession } from "./pi-session.js";
25
27
  import { type RefreshProgressCounts, type RefreshProgressPhase } from "./refresh-progress.js";
26
28
  import { type SearchDocumentDiagnostic, type SearchDocumentsResponse, type SearchDocumentsStreamEvent } from "./search-documents.js";
29
+ import { type SelfConfigOptions } from "./self-config.js";
27
30
  import { type WatcherFactory, type WatchRefreshHandle } from "./watch-refresh.js";
28
31
  import { type WebFetchToolOptions } from "./web-fetch-tool.js";
29
32
  import { type WebSearchToolOptions } from "./web-search-tool.js";
@@ -36,7 +39,7 @@ import { type WebSearchToolOptions } from "./web-search-tool.js";
36
39
  export interface AutoRAGRetrievalLimits {
37
40
  /** `search_all_documents` merge ceiling when the model omits `topK`. Default 500. */
38
41
  readonly mergedEvidenceCeiling?: number;
39
- /** `search_datasource_*` merge default when the model omits `topK`. Default 50. */
42
+ /** `search_datasource_*` merge default when the model omits `topK`. Default 20. */
40
43
  readonly singleDatasourceTopK?: number;
41
44
  /** MinSync semantic retrieval default `topK`. Default 50. */
42
45
  readonly minSyncTopK?: number;
@@ -160,12 +163,10 @@ export declare class RemoteSessionRejectedError extends Error {
160
163
  /** Thinking level applied per search phase. "off" requests no reasoning. */
161
164
  export type AutoRAGThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
162
165
  /**
163
- * Two-phase progressive answers: the fast phase delivers an immediate first
164
- * answer with {@link AutoRAGThinkingOptions.fast} thinking (default "off"),
165
- * then the verification phase re-checks and finalizes with
166
- * {@link AutoRAGThinkingOptions.final} thinking (default "high"). Pass
167
- * `false` on {@link AutoRAGAgentOptions.thinking} to keep the legacy
168
- * single-phase flow.
166
+ * Per-phase thinking for AutoRAG's two-phase search: the fast phase delivers
167
+ * an immediate first answer with {@link AutoRAGThinkingOptions.fast} thinking
168
+ * (default "off"), then the verification phase re-checks and finalizes with
169
+ * {@link AutoRAGThinkingOptions.final} thinking (default "high").
169
170
  */
170
171
  export interface AutoRAGThinkingOptions {
171
172
  /** Thinking level for the immediate first answer. Default "off". */
@@ -189,7 +190,7 @@ export interface AutoRAGAgentOptions {
189
190
  memoryPath?: string;
190
191
  workspacePath?: string;
191
192
  tools?: AgentTool[];
192
- minSync?: Omit<MinSyncVectorMethodOptions, "root"> | false;
193
+ minSync?: Omit<MinSyncVectorMethodOptions, "root">;
193
194
  jikji?: JikjiOptions | false;
194
195
  /**
195
196
  * Windows-only instant file/folder name search through the bundled
@@ -233,6 +234,43 @@ export interface AutoRAGAgentOptions {
233
234
  autoRefresh?: AutoRefreshOptions;
234
235
  parserOptions?: DefaultParserRegistryOptions;
235
236
  dupey?: DupeyCliOptions | false;
237
+ /**
238
+ * Jev (TypeSafe's judgment model: typed questions in, calibrated
239
+ * probabilities out). When set, the two-phase search asks Jev before the
240
+ * fast answer whether the question is answerable from intrinsic knowledge
241
+ * (direct answer) or needs local search, and whether to decompose it; after the fast answer, whether
242
+ * verification is needed. It also exposes the `jev` tool. Always omitted for
243
+ * remote P2P sessions.
244
+ *
245
+ * The CLI config enables this by default on OpenRouter (`buildAgentOptions`
246
+ * fills it in). On this programmatic option, absent or `false` keeps Jev
247
+ * off, so library callers never make paid network calls they did not ask for.
248
+ */
249
+ jev?: JevToolOptions | false;
250
+ /**
251
+ * Embedder used to find similar past questions in memory (semantic half of
252
+ * the hybrid search). Defaults to the local embedding gateway; `false` keeps
253
+ * memory search keyword-only.
254
+ */
255
+ memoryEmbedder?: Embedder | false;
256
+ /**
257
+ * Agent self-configuration. When set (and Jev is on), Jev gains a `config`
258
+ * branch for questions about AutoRAG's own settings: the turn skips
259
+ * `emit_fast_answer`, receives the full `autorag-setup` skill, edits
260
+ * `configPath` with the pi tools, and reports through
261
+ * `emit_autorag_results`. Always omitted for remote P2P sessions.
262
+ */
263
+ selfConfig?: SelfConfigOptions;
264
+ /**
265
+ * Question decomposition used by the Jev query pipeline. `model` (with its
266
+ * `apiKey`) is the LLM that splits one question into at most five search
267
+ * queries; omitted, the search session's own model decomposes. The CLI
268
+ * resolves its configured or default model (`openrouter/qwen/qwen3.7-flash`).
269
+ */
270
+ queryDecomposition?: {
271
+ readonly model?: Model<Api>;
272
+ readonly apiKey?: string;
273
+ };
236
274
  excludeExactDuplicates?: boolean;
237
275
  excludePaths?: readonly string[];
238
276
  /**
@@ -242,7 +280,6 @@ export interface AutoRAGAgentOptions {
242
280
  */
243
281
  limits?: AutoRAGRetrievalLimits;
244
282
  datasourceSkills?: readonly DatasourceSkill[];
245
- datasourceAccess?: DatasourceAccessContextOptions;
246
283
  /** Non-fatal diagnostics from config/agent construction (e.g. skipped unknown datasources). */
247
284
  startupDiagnostics?: readonly SearchDocumentDiagnostic[];
248
285
  /** Maximum time a model/tool search may run before it is aborted. */
@@ -256,14 +293,20 @@ export interface AutoRAGAgentOptions {
256
293
  peerQuery?: PeerQueryOptions | false;
257
294
  /** Restrict the agent to retrieval and result-emission tools for remote runs. */
258
295
  remoteSession?: boolean;
259
- /** Two-phase progressive answers with per-phase thinking control. Default enabled. */
260
- thinking?: AutoRAGThinkingOptions | false;
296
+ /** Per-phase thinking levels for the two-phase (fast → verification) search. */
297
+ thinking?: AutoRAGThinkingOptions;
261
298
  /** pi agent directory for auth/models/extensions. Defaults to ~/.pi/agent. */
262
299
  piAgentDir?: string;
263
300
  /** Optional persistent pi session directory. */
264
301
  piSessionDir?: string;
265
302
  /** Persist one pi session transcript per AutoRAG search. Defaults true. */
266
303
  persistPiSessions?: boolean;
304
+ /**
305
+ * Best-effort provider for an interactive-only startup notice (e.g. a newer
306
+ * AutoRAG release). Resolves to the notice text or `undefined`; never fails
307
+ * a launch.
308
+ */
309
+ updateNotice?: () => Promise<string | undefined>;
267
310
  }
268
311
  /** Post-merge reranking options. Mirrors the CLI `RerankConfig` (secrets via env). */
269
312
  export interface RerankAgentOptions {
@@ -319,16 +362,14 @@ export declare class AutoRAGAgent {
319
362
  private interactiveFastAnswerCallback;
320
363
  private modelNativeSearchAuth;
321
364
  private retrievalTrace;
365
+ /** Evidence every retrieval tool returned this run; emit tools resolve model-cited ids against it. */
366
+ private readonly evidenceLedger;
322
367
  private preliminaryCallback;
323
- /** Per-phase thinking levels; undefined marks the legacy single-phase flow. */
368
+ /** Per-phase thinking levels of the two-phase search. */
324
369
  private readonly fastThinkingLevel;
325
370
  private readonly finalThinkingLevel;
326
371
  private autoRefreshTimer;
327
372
  private refreshing;
328
- private jikjiPrepareInFlight;
329
- private jikjiReady;
330
- private minSyncPrepareInFlight;
331
- private minSyncReady;
332
373
  private refreshState;
333
374
  private readonly searchPaths;
334
375
  private readonly configuredSearchPaths;
@@ -341,17 +382,26 @@ export declare class AutoRAGAgent {
341
382
  private readonly merger;
342
383
  private readonly reranker;
343
384
  private readonly rerankTopN;
344
- private readonly datasourceFilter;
345
385
  private readonly minSyncMethod;
346
386
  private readonly jikjiClient;
347
387
  private readonly everythingClient;
348
388
  private readonly fsearchClient;
349
389
  private readonly datasourceSkills;
350
- private readonly datasourceAccessOptions;
351
390
  private readonly startupDiagnostics;
352
391
  private readonly datasourceAgentSkills;
353
392
  private readonly parserOptions;
354
393
  private readonly dupeyOptions;
394
+ /** pi extension registering the optional `jev` tool; undefined when disabled. */
395
+ private readonly jevExtension;
396
+ /** Jev judge shared by the `jev` tool and the query router; undefined when disabled. */
397
+ private readonly jevJudge;
398
+ /** Self-configuration, enabled only with Jev on a local session. */
399
+ private readonly selfConfig;
400
+ private readonly queryDecompositionModel;
401
+ /** Web search options for the agent's `web_search` / `web_fetch` tools; undefined when web tools are off. */
402
+ private readonly webSearchOptions;
403
+ /** Routing diagnostics for the in-flight search; reset per run. */
404
+ private routingDiagnostics;
355
405
  private readonly excludeExactDuplicates;
356
406
  private readonly excludePaths;
357
407
  private readonly baseSystemPromptConfig;
@@ -361,7 +411,19 @@ export declare class AutoRAGAgent {
361
411
  private readonly piAgentDir;
362
412
  private readonly piSessionDir;
363
413
  private readonly persistPiSessions;
414
+ private readonly updateNotice;
364
415
  private boundPiRuntime;
416
+ /** Embeds past questions for similar-question search; undefined keeps memory search keyword-only. */
417
+ private readonly memoryEmbedder;
418
+ private readonly memoryVectorPath;
419
+ /** Every retrieval result of the in-flight run with the query/method that surfaced it. */
420
+ private readonly evidenceOrigins;
421
+ /** Search-tool queries by tool call id, read back when the call ends. */
422
+ private readonly searchToolQueries;
423
+ /** Identifies this agent's conversation: everything judged in it is "current" memory. */
424
+ private readonly conversationId;
425
+ /** The memory context loaded for the in-flight run; cleared whenever a run starts. */
426
+ private memoryContextCache;
365
427
  /** True when this agent was constructed for an untrusted remote peer. */
366
428
  readonly remoteSession: boolean;
367
429
  private activeRetrievalOptions;
@@ -369,7 +431,42 @@ export declare class AutoRAGAgent {
369
431
  private readonly limits;
370
432
  constructor(options: AutoRAGAgentOptions);
371
433
  scanDuplicateDocuments(): Promise<ScanDuplicateDocumentsDetails>;
434
+ /**
435
+ * Reference memory for the current question: everything judged earlier in
436
+ * this conversation, up to 25 similar past questions, and matching
437
+ * long-term insights. Computed once per run (the datasource check and every
438
+ * model turn share it) so the similar-question search and its embedder call
439
+ * are not repeated and a fallback is reported once. Remote peers never read it.
440
+ */
441
+ private loadMemoryContextFor;
372
442
  private withMemoryContext;
443
+ /**
444
+ * Where evidence came from, or undefined when this run may not see it.
445
+ * Datasource chunks carry a virtual path rooted at the skill name
446
+ * (`/kakao/default/...`), but the model sometimes maps a datasource hit to a
447
+ * bare chunk id. The evidence method then still names the datasource
448
+ * (`search_datasource_kakao_work`, `datasource:kakao`, `kakao-work-lexical`):
449
+ * both sides collapse every non-alphanumeric to `_` exactly like generated
450
+ * tool names, and a datasource id that appears as a whole token attributes
451
+ * it; longer ids are checked first so `kakao-work` wins over `kakao`.
452
+ * Indexed files are absolute paths under a search path; web evidence is a URL.
453
+ * Memory is global (shared across workspaces and configs), so evidence from
454
+ * an unconfigured datasource, and evidence with no recognizable origin, is
455
+ * never attributed.
456
+ */
457
+ private evidenceAttribution;
458
+ /** Options shared by the prompt injection and the `check_memory` tool; rebuilt on every call because datasource configuration follows live settings. */
459
+ private memoryContextOptions;
460
+ /** Remember which query and method surfaced each retrieval result, so cited evidence can be traced back. */
461
+ private rememberEvidenceOrigins;
462
+ /**
463
+ * After the final answer: map every cited evidence back to the search query
464
+ * and method that found it, ask Jev (one batched call, evidence judged in
465
+ * parallel) whether it really supports the answer sentence it backs, and
466
+ * keep only what clears the threshold. Remote peers and runs without Jev
467
+ * store nothing; a failed judgment stores nothing and says why.
468
+ */
469
+ private recordJudgedEvidence;
373
470
  private resolveSessionModel;
374
471
  private createSearchSession;
375
472
  private configureSearchSession;
@@ -389,10 +486,6 @@ export declare class AutoRAGAgent {
389
486
  }): void;
390
487
  stopAutoRefresh(): void;
391
488
  private runAutoRefreshTick;
392
- submitFeedback(sessionId: string | undefined, satisfied: boolean): void;
393
- recordResultFeedback(feedback: ResultFeedback[]): void;
394
- recordFeedbackByNumbers(sessionId: string, usefulNumbers: number[], notUsefulNumbers?: number[]): void;
395
- recordFeedbackByIds(usefulFeedbackIds: readonly string[], notUsefulFeedbackIds?: readonly string[]): void;
396
489
  getResultRegistry(sessionId?: string): ReadonlyMap<number, CuratedResult>;
397
490
  searchDocuments(query: string, options?: RetrievalOptions): Promise<SearchDocumentsResponse>;
398
491
  /**
@@ -402,45 +495,103 @@ export declare class AutoRAGAgent {
402
495
  * stop by calling {@link abort}.
403
496
  */
404
497
  searchDocumentsStream(query: string, options?: RetrievalOptions): AsyncGenerator<SearchDocumentsStreamEvent, void, void>;
405
- private datasourceAccessContext;
406
498
  /**
407
- * Build the Pi agent-skill list for datasource skills authorized by the
408
- * trusted, server-bound access context. Only authorized skills become
409
- * model-visible; unauthorized skills are omitted entirely (default-deny).
499
+ * Build the Pi agent-skill list for every configured datasource skill. All
500
+ * configured skills are model-visible; a connection is removed from the
501
+ * model surface by removing it from the config, not by a permission layer.
410
502
  */
411
- private buildAuthorizedDatasourceSkills;
503
+ private buildDatasourceAgentSkills;
412
504
  /**
413
505
  * Per-connection tool specs for the generated `search_datasource_<id>`
414
- * tools, built from the same trusted config and access context as
415
- * {@link buildAuthorizedDatasourceSkills}. One spec per authorized
416
- * datasource skill; duplicate ids collapse to the first registration.
506
+ * tools, one per configured datasource skill; duplicate ids collapse to the
507
+ * first registration.
417
508
  */
418
509
  private singleDatasourceToolSpecs;
419
510
  /**
420
- * Resolve an authorized datasource agent skill by model-visible name for the
421
- * `load_datasource_skill` tool. Returns `undefined` for unknown or
422
- * unauthorized names — model/tool input can never widen authorization.
511
+ * Configured datasource descriptors for catalog/listing surfaces: every
512
+ * configured datasource is listed — including ones that expose no retrieval
513
+ * methods — and each entry carries only identity, capability tags, and
514
+ * source scope strings (never credentials, config paths, or raw instance
515
+ * metadata). Duplicate datasource ids collapse to the first registration.
423
516
  */
424
- loadDatasourceSkill(name: string): Skill | undefined;
517
+ listDatasources(): DatasourceCatalogEntry[];
518
+ /**
519
+ * Opaque source scope strings for one datasource. Sources containing a `#`
520
+ * fragment are invalid and skipped.
521
+ */
522
+ private datasourceSourceScopes;
523
+ /**
524
+ * Resolve a configured datasource agent skill by model-visible name for the
525
+ * `load_datasource_skill` tool. Returns `undefined` for unknown names.
526
+ */
527
+ loadDatasourceSkill(name: string): DatasourceAgentSkill | undefined;
425
528
  private indexDatasources;
426
529
  /** Path-opaque component diagnostics for the search response. */
427
530
  private collectComponentDiagnostics;
531
+ /**
532
+ * Jev check run after emit_fast_answer: does the answer need correction,
533
+ * clarification, or further research? "No" ends the run with the fast
534
+ * answer as the final answer. Without Jev, or when the check fails, the run
535
+ * always continues into verification.
536
+ */
537
+ private shouldFollowUp;
538
+ /**
539
+ * Jev query pipeline, run before the fast answer. Jev picks the branch
540
+ * (direct answer or local search) and whether the question
541
+ * needs decomposition; a "yes" splits it into at most five search queries
542
+ * with the configured decomposition model (default: the session model). On
543
+ * the local branch, Jev also judges, per registered datasource, whether it
544
+ * must be searched (in parallel with decomposition). Without Jev, and on any
545
+ * routing failure, this is today's single local search for the original
546
+ * question with no datasource search.
547
+ */
548
+ private planQuery;
549
+ /**
550
+ * Jev datasource check for the local branch: one `noul` per configured
551
+ * datasource that has retrieval methods, with where similar past questions
552
+ * were answered (from retrieval memory) in the state. Returns the ids to
553
+ * search with every query before the rerank; a failed check searches none.
554
+ */
555
+ private selectSearchDatasources;
556
+ /**
557
+ * Baseline local evidence for the fast answer. Jikji, MinSync, and every
558
+ * Jev-selected datasource run for every search query in parallel (MinSync
559
+ * itself queues per workspace), the per-query hits are interleaved and
560
+ * deduplicated into one pool, and that pool is reranked against the
561
+ * original question.
562
+ */
428
563
  private prefetchInitialRetrievalContext;
429
564
  /**
430
- * Prepare MinSync lazily without holding up the current search. Parsed
431
- * mirrors are built first because MinSync stages and indexes those mirrors.
565
+ * Rerank the pre-fast-answer baseline pool — Jikji answer paths, MinSync
566
+ * chunks, and Jev-selected datasource chunks — down to the configured
567
+ * `rerank.topN`. With decomposition the pool merges every search query's
568
+ * hits (interleaved), capped at the same per-source sizes as a single query
569
+ * so the rerank request does not grow with the query count. Returns
570
+ * `undefined` when reranking is disabled/unavailable or the pool is empty,
571
+ * so the caller keeps the unranked sections; a rerank failure never blocks
572
+ * the fast answer.
432
573
  */
433
- /** Prepare MinSync in the background; the returned promise is for tests. */
434
- scheduleMinSyncPrepareForTest(): Promise<MinSyncSyncResult | undefined> | undefined;
435
- private scheduleMinSyncPrepare;
574
+ private rerankPrefetchPool;
436
575
  /**
437
576
  * Fast-phase prompt for two-phase searches. The baseline retrieval context
438
577
  * is already gathered, so the model answers immediately with thinking off
439
578
  * via emit_fast_answer, without any further tool calls.
440
579
  */
441
580
  buildFastAnswerPrompt(query: string, options: RetrievalOptions, baseline: string): string;
581
+ /**
582
+ * Prompt for a question Jev routed to a direct answer: general knowledge or
583
+ * small talk. No retrieval ran and no verification phase follows, so the
584
+ * answer emitted here is final.
585
+ */
586
+ buildDirectAnswerPrompt(query: string): string;
442
587
  /** Discovery-tool hint naming only the registered discovery tools; empty when none. */
443
588
  private discoveryHint;
589
+ /**
590
+ * Verification-phase nudge toward the internet. Jev no longer routes to web
591
+ * search, so the agent decides after the fast answer: when local evidence is
592
+ * missing, thin, or possibly stale, it searches the web. Empty when web tools are off.
593
+ */
594
+ private webFallbackHint;
444
595
  /**
445
596
  * Verification-phase prompt for two-phase searches. The fast answer, when a
446
597
  * consumer already received it, is embedded verbatim so the model can diff
@@ -485,7 +636,7 @@ export declare class AutoRAGAgent {
485
636
  */
486
637
  private scanMirrorStaleness;
487
638
  private exactDuplicateExclusions;
488
- syncMinSync(force?: boolean): Promise<MinSyncSyncResult | undefined>;
639
+ syncMinSync(force?: boolean): Promise<MinSyncSyncResult>;
489
640
  /** Provider for the Windows-only everything_search tool. */
490
641
  searchEverything(request: EverythingSearchRequest): Promise<EverythingSearchResult>;
491
642
  /**
@@ -504,13 +655,13 @@ export declare class AutoRAGAgent {
504
655
  */
505
656
  stopFsearch(): Promise<void>;
506
657
  prepareJikji(): Promise<readonly AutoRAGJikjiPrepareResult[] | undefined>;
507
- private executeJikjiPrepare;
508
658
  /**
509
- * Start Jikji preparation without delaying the current search turn. A
510
- * subsequent turn can use the prepared index, while the child process keeps
511
- * running after emit_autorag_results has terminated the agent loop.
659
+ * Build or incrementally update every root's Jikji index. Called only by
660
+ * refresh/watch (and the explicit `prepareJikji()` API), never by a query:
661
+ * `jikji prepare` reuses unchanged documents, and roots run in parallel so
662
+ * one slow root does not serialize the rest.
512
663
  */
513
- private scheduleJikjiPrepare;
664
+ private executeJikjiPrepare;
514
665
  private sanitizeJikjiPrepareResult;
515
666
  /**
516
667
  * Provider method for the jikji_find tool. Runs JikjiClient.find over all
@@ -527,6 +678,14 @@ export declare class AutoRAGAgent {
527
678
  * Merge per-root answer packs into one. Concatenates answer_paths/candidates
528
679
  * preserving per-root order; dedupes by normalized path. Does NOT cross-root
529
680
  * rerank when any root has agentShouldNotRerank=true.
681
+ *
682
+ * Honours `excludePaths` at the retrieval boundary: Jikji indexes the source
683
+ * folders directly, bypassing the parsed mirror, so an excluded source still
684
+ * appears in the on-disk `.jikji_agent_map.md`. Dropping excluded paths here
685
+ * keeps them out of the agent-facing answer pack (the `jikji_find` tool and
686
+ * the baseline prefetch) while leaving the shared map artifact complete.
687
+ * Excluded paths remain reachable through direct file reads, as Jikji never
688
+ * blocks source verification.
530
689
  */
531
690
  private mergeAnswerPacks;
532
691
  /**
@@ -566,10 +725,9 @@ export declare class AutoRAGAgent {
566
725
  * Search one datasource connection only. Only the target connection's
567
726
  * retrieval methods are registered with the retriever, so no other
568
727
  * datasource CLI is spawned at all and only that connection's hits are
569
- * returned. Access is still gated by the trusted datasource context: an
570
- * unknown or unauthorized `datasourceId` yields an empty result set.
728
+ * returned. An unknown `datasourceId` yields an empty result set.
571
729
  *
572
- * This backs the generated `search_datasource_<id>` tools; every authorized
730
+ * This backs the generated `search_datasource_<id>` tools; every configured
573
731
  * connection has one. Cross-datasource fan-out is
574
732
  * {@link searchAllDocuments}, which spans every retrieval method.
575
733
  */
@@ -585,7 +743,7 @@ export declare class AutoRAGAgent {
585
743
  /**
586
744
  * The standalone retrieval engine for this agent's method pipeline.
587
745
  * Built on first access using the agent's registered methods and configured
588
- * datasource access context. Model-free — no agent state required.
746
+ * datasource catalog. Model-free — no agent state required.
589
747
  */
590
748
  private retrievalEngine;
591
749
  getRetrievalEngine(): RetrievalEngine;
@@ -596,7 +754,6 @@ export declare class AutoRAGAgent {
596
754
  * the merged order is preserved — a reranker outage never hides evidence.
597
755
  */
598
756
  private applyRerank;
599
- private rerankWithMemory;
600
757
  private remoteFilteredRetrievalMethod;
601
758
  private normalizeRetrievalOptions;
602
759
  private resolveRetrievalScope;
@@ -0,0 +1,19 @@
1
+ /**
2
+ * The single file-path exception to the answer "no file paths" rule
3
+ * (issue #1790). Every prompt and tool description that shapes `answer` uses
4
+ * this exact wording so a caller that can render images receives a markdown
5
+ * image embed whose path is a real result source, while the no-path rule
6
+ * otherwise still stands.
7
+ */
8
+ export declare const ANSWER_IMAGE_EMBED_RULE = "**Images**: when a retrieved result is itself an image file (png, jpg, jpeg, gif, webp, svg, bmp, avif, heic, tiff) and showing it answers the question, embed it in `answer` as a markdown image on its own line: `![short description](<absolute source path>)`, followed by its citation, e.g. `![Q3 revenue chart](</data/reports/q3 chart.png>) [2]`. Use only the real source path of a result in `results`/`mapping`. Never invent a path, use relative paths, or embed remote URLs. Apart from these image embeds, the no-file-paths rule stands.";
9
+ /**
10
+ * Delta (verification) answers must not re-embed an image the first answer
11
+ * already showed.
12
+ */
13
+ export declare const ANSWER_IMAGE_DELTA_RULE = "An image already embedded in the first answer must not be embedded again.";
14
+ /**
15
+ * One numbering space for citations (issue #1788): every `[n]` in `answer`
16
+ * must be a `results[].number` emitted in the same call. Retrieval candidate
17
+ * numbers and first-answer unit numbers are never citation numbers.
18
+ */
19
+ export declare const ANSWER_CITATION_RULE = "**Citation numbers**: every bracketed citation [n] in `answer` must be the `number` of a result you emit in `results` of the same call. Numbers shown in retrieval candidates or in an earlier first answer are NOT citation numbers \u2014 renumber against your own `results`, and add a result for any evidence you cite.";
@@ -0,0 +1,43 @@
1
+ /** Sorted, de-duplicated citation numbers in `answer` with no matching result. */
2
+ export declare function unresolvedCitations(answer: string, results: readonly {
3
+ readonly number: number;
4
+ }[]): number[];
5
+ /** Remove citation markers whose number has no matching result. */
6
+ export declare function stripUnresolvedCitations(answer: string, results: readonly {
7
+ readonly number: number;
8
+ }[]): {
9
+ readonly answer: string;
10
+ readonly unresolved: readonly number[];
11
+ };
12
+ export declare function formatCitationList(numbers: readonly number[]): string;
13
+ /**
14
+ * Throw a corrective error when two results share a number. A repeated number
15
+ * collapses into one registry entry and one feedback id, so the other result's
16
+ * evidence would be lost. Emit tools call this at tool time so the model sees
17
+ * the error and re-emits.
18
+ */
19
+ export declare function assertUniqueResultNumbers(label: string, results: readonly {
20
+ readonly number: number;
21
+ }[]): void;
22
+ /**
23
+ * Throw a corrective error unless `results` and `mapping` carry the same
24
+ * numbers, exactly one entry each. Duplicates are rejected even when both
25
+ * sides repeat them: a repeated number collapses into a single registry entry,
26
+ * so the other entry's evidence would be lost.
27
+ * Emit tools call this at tool time (issue #1807) so the model sees the error
28
+ * and re-emits; checking only after the run ended failed the whole search and
29
+ * discarded an already-delivered answer.
30
+ */
31
+ export declare function assertResultsMappingOneToOne(label: string, results: readonly {
32
+ readonly number: number;
33
+ }[], mapping: readonly {
34
+ readonly number: number;
35
+ }[]): void;
36
+ /**
37
+ * Throw a corrective error when `answer` cites numbers absent from `results`.
38
+ * Emit tools surface the message to the model as a tool error so it re-emits
39
+ * with one consistent numbering.
40
+ */
41
+ export declare function assertCitationsResolve(label: string, answer: string, results: readonly {
42
+ readonly number: number;
43
+ }[]): void;
@@ -1,42 +1,56 @@
1
- import type { AgentTool, Skill } from "@earendil-works/pi-agent-core";
1
+ import type { AgentTool } from "@earendil-works/pi-agent-core";
2
2
  import { Type } from "typebox";
3
3
  import type { DatasourceSkillManifest } from "../datasource/types.js";
4
4
  export declare const LOAD_DATASOURCE_SKILL_TOOL_NAME = "load_datasource_skill";
5
5
  /**
6
- * Opaque, path-free location used as the Pi `Skill.filePath` for a datasource
7
- * skill. Datasource skills have no real backing file, and AutoRAG must never
6
+ * A datasource skill as the agent sees it: name and description for the
7
+ * system-prompt listing, full `content` loaded on demand, and an opaque
8
+ * `datasource://` location. pi's own `Skill` is file-backed (`baseDir`,
9
+ * `sourceInfo`) and has no `content`, so datasource skills, which have no
10
+ * backing file, use this shape instead.
11
+ */
12
+ export interface DatasourceAgentSkill {
13
+ readonly name: string;
14
+ readonly description: string;
15
+ /** Full skill instructions, returned by `load_datasource_skill`. */
16
+ readonly content: string;
17
+ /** Opaque `datasource://<name>` location; never a real filesystem path. */
18
+ readonly filePath: string;
19
+ }
20
+ /**
21
+ * Opaque, path-free location used as the `filePath` of a datasource skill.
22
+ * Datasource skills have no real backing file, and AutoRAG must never
8
23
  * expose real filesystem paths, so we use a `datasource://<name>` scheme.
9
24
  */
10
25
  export declare function datasourceSkillLocation(name: string): string;
11
26
  /**
12
- * Map a datasource skill manifest onto the Pi agent-skill layer. The resulting
13
- * `Skill` is injected into the system prompt (name/description/location) for
27
+ * Map a datasource skill manifest onto the agent-skill shape. The resulting
28
+ * skill is listed in the system prompt (name/description/location) for
14
29
  * progressive disclosure and its `content` is loaded on demand.
15
30
  */
16
- export declare function toDatasourceAgentSkill(manifest: DatasourceSkillManifest): Skill;
31
+ export declare function toDatasourceAgentSkill(manifest: DatasourceSkillManifest): DatasourceAgentSkill;
17
32
  /**
18
- * Render the Pi `<available_skills>` progressive-disclosure block for the
19
- * authorized datasource skills using Pi's own formatter, so datasource skills
20
- * sit on the exact same layer as file-backed agent skills.
33
+ * Render the `<available_skills>` progressive-disclosure block for the
34
+ * configured datasource skills, in the same format pi uses for file-backed
35
+ * agent skills, so datasource skills sit on the same layer.
21
36
  */
22
- export declare function buildDatasourceSkillsPrompt(skills: readonly Skill[]): string;
37
+ export declare function buildDatasourceSkillsPrompt(skills: readonly DatasourceAgentSkill[]): string;
23
38
  /**
24
- * Format the full instructions for a loaded datasource skill, matching Pi's
39
+ * Format the full instructions for a loaded datasource skill, matching pi's
25
40
  * `<skill …>` invocation shape but with a path-opaque location (no real
26
41
  * filesystem reference, so no "references are relative to …" line).
27
42
  */
28
- export declare function formatDatasourceSkillInvocation(skill: Skill): string;
43
+ export declare function formatDatasourceSkillInvocation(skill: DatasourceAgentSkill): string;
29
44
  export interface LoadDatasourceSkillDetails {
30
45
  readonly skill: string;
31
46
  readonly loaded: boolean;
32
47
  }
33
48
  export interface DatasourceSkillProvider {
34
49
  /**
35
- * Resolve an authorized datasource skill by model-visible name. Returns
36
- * `undefined` when the name is unknown or not authorized by the trusted,
37
- * server-bound access context (default-deny).
50
+ * Resolve a configured datasource skill by model-visible name. Returns
51
+ * `undefined` when the name is unknown or not configured.
38
52
  */
39
- loadDatasourceSkill(name: string): Skill | undefined;
53
+ loadDatasourceSkill(name: string): DatasourceAgentSkill | undefined;
40
54
  }
41
55
  declare const loadDatasourceSkillSchema: Type.TObject<{
42
56
  name: Type.TString;
@@ -1,7 +1,35 @@
1
1
  import type { AgentTool } from "@earendil-works/pi-agent-core";
2
2
  import { Type } from "typebox";
3
+ import type { EvidenceLedger } from "./evidence-ledger.js";
3
4
  export declare const EMIT_AUTORAG_RESULTS_TOOL_NAME = "emit_autorag_results";
5
+ /**
6
+ * Model-facing schema of `emit_autorag_results`. The model cites evidence by
7
+ * the short ids retrieval tools print next to each result (`refs`); the
8
+ * harness resolves them to the recorded source, method, and chunk, so the
9
+ * model never retypes a path or a chunk.
10
+ */
4
11
  export declare const emitResultsSchema: Type.TObject<{
12
+ answer: Type.TString;
13
+ results: Type.TArray<Type.TObject<{
14
+ number: Type.TInteger;
15
+ title: Type.TString;
16
+ summary: Type.TString;
17
+ evidence: Type.TArray<Type.TObject<{
18
+ excerpt: Type.TString;
19
+ lineNumber: Type.TOptional<Type.TInteger>;
20
+ }>>;
21
+ confidence: Type.TNumber;
22
+ refs: Type.TArray<Type.TString>;
23
+ }>>;
24
+ warnings: Type.TOptional<Type.TArray<Type.TString>>;
25
+ }>;
26
+ /**
27
+ * Report schema with the explicit number -> source mapping. This is the
28
+ * persisted/typed shape and the input contract of `autorag report` and the MCP
29
+ * report tool, where an external agent curated the evidence and no harness
30
+ * ledger exists.
31
+ */
32
+ export declare const reportSchema: Type.TObject<{
5
33
  answer: Type.TString;
6
34
  results: Type.TArray<Type.TObject<{
7
35
  number: Type.TInteger;
@@ -79,10 +107,17 @@ export interface AutoRAGResultsDetails {
79
107
  readonly mapping: readonly AutoRAGMappingEntry[];
80
108
  readonly warnings: readonly string[];
81
109
  }
110
+ export interface EmitResultsToolOptions {
111
+ /** Evidence the run's retrieval tools returned; `refs` resolve against it. */
112
+ readonly ledger: EvidenceLedger;
113
+ /** False in remote sessions: only evidence a tool returned this run may be cited. */
114
+ readonly allowLocalFiles: boolean;
115
+ }
82
116
  /**
83
117
  * Builds the terminating structured-result tool. The model calls this exactly
84
118
  * once as its final action; the typed `details` plus `terminate: true` end the
85
119
  * Pi Agent run and hand the curated results back through `capture` — no
86
- * assistant-text parsing involved.
120
+ * assistant-text parsing involved. The number -> source mapping is built here
121
+ * from the harness-held evidence the model cited by id, never from model text.
87
122
  */
88
- export declare function createEmitResultsTool(capture: (details: AutoRAGResultsDetails) => void): AgentTool<typeof emitResultsSchema, AutoRAGResultsDetails>;
123
+ export declare function createEmitResultsTool(capture: (details: AutoRAGResultsDetails) => void, options: EmitResultsToolOptions): AgentTool<typeof emitResultsSchema, AutoRAGResultsDetails>;