@autorag/librarian 2.10.0 → 2.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/LICENSE +195 -21
  2. package/NOTICE +16 -23
  3. package/README.md +399 -182
  4. package/dist/agent/agent.d.ts +163 -20
  5. package/dist/agent/context-token-guard.d.ts +23 -0
  6. package/dist/agent/evidence-ledger.d.ts +54 -5
  7. package/dist/agent/evidence-persistence.d.ts +24 -0
  8. package/dist/agent/jev-extension.d.ts +7 -4
  9. package/dist/agent/jikji-find-tool.d.ts +1 -0
  10. package/dist/agent/pi-session.d.ts +35 -2
  11. package/dist/agent/request-limits.d.ts +20 -0
  12. package/dist/agent/search-all-tool.d.ts +1 -0
  13. package/dist/agent/search-documents.d.ts +1 -1
  14. package/dist/agent/search-single-datasource-tool.d.ts +1 -0
  15. package/dist/agent/system-prompt.d.ts +0 -2
  16. package/dist/agent/tool-burst-guard.d.ts +26 -0
  17. package/dist/cli/config.d.ts +39 -23
  18. package/dist/cli/index.js +4195 -1478
  19. package/dist/cloud/config.d.ts +33 -9
  20. package/dist/cloud/credential.d.ts +17 -0
  21. package/dist/cloud/errors.d.ts +6 -6
  22. package/dist/cloud/index.d.ts +5 -4
  23. package/dist/cloud/jev-backend.d.ts +92 -0
  24. package/dist/cloud/models.d.ts +4 -4
  25. package/dist/cloud/oauth-login.d.ts +3 -3
  26. package/dist/cloud/provider.d.ts +16 -11
  27. package/dist/cloud/token.d.ts +2 -2
  28. package/dist/core.d.ts +2 -1
  29. package/dist/core.js +4200 -1506
  30. package/dist/datasource/aliased-skill.d.ts +2 -2
  31. package/dist/datasource/connector-skill.d.ts +2 -2
  32. package/dist/datasource/crawler-skill.d.ts +19 -2
  33. package/dist/datasource/described-skill.d.ts +2 -2
  34. package/dist/datasource/index.d.ts +1 -1
  35. package/dist/datasource/retry.d.ts +49 -0
  36. package/dist/datasource/sidecar/config.d.ts +26 -0
  37. package/dist/datasource/sidecar/descriptors.d.ts +67 -0
  38. package/dist/datasource/sidecar/fusion.d.ts +25 -0
  39. package/dist/datasource/sidecar/index.d.ts +131 -0
  40. package/dist/datasource/sidecar/methods.d.ts +27 -0
  41. package/dist/datasource/skills/clawgallery/skill.d.ts +4 -4
  42. package/dist/datasource/skills/discrawl/skill.d.ts +5 -5
  43. package/dist/datasource/skills/github-gist/skill.d.ts +2 -2
  44. package/dist/datasource/skills/lazykatok/skill.d.ts +5 -5
  45. package/dist/datasource/skills/mailcrawl/skill.d.ts +2 -2
  46. package/dist/datasource/skills/obsidian/skill.d.ts +5 -5
  47. package/dist/datasource/types.d.ts +16 -2
  48. package/dist/embedding-runtime/cache.d.ts +9 -0
  49. package/dist/embedding-runtime/index.d.ts +6 -6
  50. package/dist/embedding-runtime/manifest.d.ts +0 -12
  51. package/dist/embedding-runtime/types.d.ts +1 -1
  52. package/dist/index.d.ts +1 -2
  53. package/dist/index.js +4454 -1757
  54. package/dist/jikji/client.d.ts +13 -2
  55. package/dist/jikji/index.d.ts +1 -1
  56. package/dist/jikji/path-map.d.ts +9 -0
  57. package/dist/jikji/prepare-budget.d.ts +18 -0
  58. package/dist/jikji/serial-queue.d.ts +17 -0
  59. package/dist/jikji/types.d.ts +17 -0
  60. package/dist/mcp/index.js +4442 -1748
  61. package/dist/minsync/client.d.ts +6 -4
  62. package/dist/minsync/installer.d.ts +1 -1
  63. package/dist/minsync/method.d.ts +1 -1
  64. package/dist/minsync/process.d.ts +1 -0
  65. package/dist/minsync/types.d.ts +2 -2
  66. package/dist/mirror/atomic-write.d.ts +6 -0
  67. package/dist/mirror/index-store.d.ts +8 -1
  68. package/dist/mirror/index.d.ts +2 -2
  69. package/dist/mirror/sync.d.ts +57 -1
  70. package/dist/p2p/simplex-server.d.ts +0 -1
  71. package/dist/parser/defaults.d.ts +14 -4
  72. package/dist/parser/errors.d.ts +10 -0
  73. package/dist/parser/isolated.d.ts +77 -0
  74. package/dist/parser/kordoc.d.ts +1 -0
  75. package/dist/parser/ocr-engines.d.ts +6 -0
  76. package/dist/parser/parse-worker.d.ts +1 -0
  77. package/dist/parser/parse-worker.js +780 -0
  78. package/dist/parser/types.d.ts +5 -0
  79. package/dist/process/abort-reason.d.ts +2 -0
  80. package/dist/retrieval/index.d.ts +1 -1
  81. package/dist/retrieval/rerank.d.ts +28 -5
  82. package/licenses/embedding-assets.json +0 -35
  83. package/package.json +26 -29
  84. package/skills/autorag/SKILL.md +3 -2
  85. package/skills/autorag-doctor/SKILL.md +7 -1
  86. package/skills/autorag-lite-setup/SKILL.md +3 -3
  87. package/skills/autorag-setup/SKILL.md +43 -18
  88. package/dist/manifest/index.d.ts +0 -2
  89. package/dist/manifest/loader.d.ts +0 -3
  90. package/dist/manifest/types.d.ts +0 -8
  91. package/licenses/gemma-notice.txt +0 -24
  92. package/licenses/gemma-section-3.2-flow-down.txt +0 -9
  93. package/licenses/gemma-terms-of-use.html +0 -4750
  94. package/licenses/gemma-terms-of-use.txt +0 -158
@@ -1,14 +1,15 @@
1
1
  import type { Agent, AgentTool } from "@earendil-works/pi-agent-core";
2
- import type { Api, Model } from "@earendil-works/pi-ai";
2
+ import type { Api, ImageContent, Model } from "@earendil-works/pi-ai";
3
+ import { type DatasourceRetryOptions } from "../datasource/retry.js";
3
4
  import type { DatasourceIndexResult, DatasourceSkill } from "../datasource/types.js";
4
5
  import { type DupeyCliOptions } from "../dupey/index.js";
5
6
  import { type Embedder } from "../embedding-runtime/gateway-embedder.js";
6
7
  import { type EverythingClientOptions, type EverythingSearchRequest, type EverythingSearchResult } from "../everything/index.js";
7
8
  import { type FSearchClientOptions, type FSearchFailureReason, type FSearchSearchRequest, type FSearchSearchResult } from "../fsearch/index.js";
8
- import { type JikjiFailureReason, type JikjiOptions } from "../jikji/index.js";
9
+ import { type JikjiFailureReason, type JikjiOptions, type JikjiPrepareOptions } from "../jikji/index.js";
9
10
  import { type LanguageTag } from "../language.js";
10
11
  import { type MinSyncSyncResult, type MinSyncVectorMethodOptions } from "../minsync/index.js";
11
- import { type ParsedMirrorSyncResult } from "../mirror/sync.js";
12
+ import { type ParsedMirrorSyncOptions, type ParsedMirrorSyncResult } from "../mirror/sync.js";
12
13
  import type { PolicyResolver } from "../p2p/policy-filter.js";
13
14
  import type { SimplexQueryState } from "../p2p/simplex-query-store.js";
14
15
  import { type SimplexTransport } from "../p2p/simplex-transport.js";
@@ -26,9 +27,28 @@ import { type AutoRAGPiInteractiveRuntime, type AutoRAGPiSession } from "./pi-se
26
27
  import { type RefreshProgressCounts, type RefreshProgressPhase } from "./refresh-progress.js";
27
28
  import { type SearchDocumentDiagnostic, type SearchDocumentsResponse, type SearchDocumentsStreamEvent } from "./search-documents.js";
28
29
  import { type SelfConfigOptions } from "./self-config.js";
30
+ import { type ToolBurstGuardOptions } from "./tool-burst-guard.js";
29
31
  import { type WatcherFactory, type WatchRefreshHandle } from "./watch-refresh.js";
30
32
  import { type WebFetchToolOptions } from "./web-fetch-tool.js";
31
33
  import { type WebSearchToolOptions } from "./web-search-tool.js";
34
+ /**
35
+ * Safety limits for the parsed-mirror refresh. Every field is optional; an
36
+ * omitted field keeps the shipped default.
37
+ */
38
+ export interface AutoRAGRefreshLimits {
39
+ /**
40
+ * Largest share (0–1) of indexed documents one refresh may delete because
41
+ * their source disappeared; a bigger deletion above 100 documents is
42
+ * suppressed unless the refresh is forced. Default 0.3.
43
+ */
44
+ readonly maxDeleteRatio?: number;
45
+ /** Directory entries examined per search root before the walk stops. Default 2,000,000. */
46
+ readonly maxScannedEntries?: number;
47
+ /** Deepest directory level walked below a search root. Default 64. */
48
+ readonly maxDepth?: number;
49
+ /** Retries for a transient per-file I/O error (EMFILE/EBUSY/EIO/ETIMEDOUT) before the file is skipped. Default 3. */
50
+ readonly transientRetries?: number;
51
+ }
32
52
  /**
33
53
  * Hard caps on retrieval, baseline prefetch, and the candidate lists handed to
34
54
  * the model. Every field is optional: an omitted field keeps the shipped
@@ -64,9 +84,40 @@ export interface AutoRefreshOptions {
64
84
  }
65
85
  /** Methods that `refresh` can selectively run. Defaults to all when omitted. */
66
86
  export type RefreshMethod = "parsed" | "minsync" | "datasources" | "jikji" | "everything" | "fsearch";
87
+ /**
88
+ * Progress of a refresh phase that can count its work. `parsed` counts source files
89
+ * (`currentVirtualPath` is the opaque virtual path of the file about to be processed);
90
+ * `datasources` counts configured datasource skills. Other phases do not report counts.
91
+ */
92
+ export interface AutoRAGRefreshProgress {
93
+ readonly phase: "parsed" | "datasources";
94
+ /** Items finished so far. Advances by one per item and ends equal to `total`. */
95
+ readonly processed: number;
96
+ readonly total: number;
97
+ readonly currentVirtualPath?: string;
98
+ }
67
99
  export interface AutoRAGRefreshOptions {
68
100
  /** Restrict refresh to specific methods. Defaults to all when undefined. */
69
101
  readonly methods?: readonly RefreshMethod[];
102
+ /**
103
+ * Retry sources previously skipped as parser-failed or source-unreadable once, ignoring their
104
+ * skip records. Unlike `force`, healthy unchanged mirrors are not re-parsed.
105
+ */
106
+ readonly retrySkipped?: boolean;
107
+ /**
108
+ * Cancels the refresh. It is checked between files and between phases, and it kills an in-flight parse
109
+ * worker and the MinSync, Jikji and datasource child processes. `refresh` then rejects with the signal's
110
+ * reason and records the run as failed; mirror entries written before the abort are kept.
111
+ */
112
+ readonly signal?: AbortSignal;
113
+ /** Called while the parse and datasource phases run. A throwing callback is ignored; it never fails the refresh. */
114
+ readonly onProgress?: (progress: AutoRAGRefreshProgress) => void;
115
+ /**
116
+ * Per-root `jikji prepare` controls: a hard `timeoutMs` wall per root and a
117
+ * `parseTimeout` (seconds, forwarded to `--parse-timeout`). Unset values fall
118
+ * back to the `jikji` client options. Cancel through the refresh `signal`.
119
+ */
120
+ readonly jikjiPrepare?: Omit<JikjiPrepareOptions, "signal">;
70
121
  }
71
122
  export interface AutoRAGMinSyncRefreshResult {
72
123
  readonly ok: boolean;
@@ -185,7 +236,6 @@ export interface AutoRAGAgentOptions {
185
236
  providerApiKeys?: Readonly<Record<string, string>>;
186
237
  searchPaths: string[];
187
238
  languages?: readonly LanguageTag[];
188
- manifestDir?: string;
189
239
  memoryPath?: string;
190
240
  workspacePath?: string;
191
241
  tools?: AgentTool[];
@@ -278,13 +328,23 @@ export interface AutoRAGAgentOptions {
278
328
  * {@link AutoRAGRetrievalLimits}. Values must be positive integers.
279
329
  */
280
330
  limits?: AutoRAGRetrievalLimits;
331
+ /** Parsed-mirror refresh limits and retry tuning. See {@link AutoRAGRefreshLimits}. */
332
+ refresh?: AutoRAGRefreshLimits;
281
333
  datasourceSkills?: readonly DatasourceSkill[];
334
+ /**
335
+ * Retry/backoff for datasource `index()` during refresh: 2 retries with
336
+ * jittered exponential backoff by default, applied only to failures the
337
+ * skill marks retryable. `maxRetries: 0` disables retrying.
338
+ */
339
+ datasourceRetry?: DatasourceRetryOptions;
282
340
  /** Non-fatal diagnostics from config/agent construction (e.g. skipped unknown datasources). */
283
341
  startupDiagnostics?: readonly SearchDocumentDiagnostic[];
284
- /** Maximum time a model/tool search may run before it is aborted. */
285
- searchTimeoutMs?: number;
286
- /** Maximum number of retrieval/tool executions allowed in one search. */
287
- maxSearchToolCalls?: number;
342
+ /**
343
+ * Runaway-loop guard: the run is aborted when one tool is called `maxCalls`
344
+ * times within `windowMs` (default 60 calls in 60 s). `false` disables it.
345
+ * There is no limit on a run's duration or its total number of tool calls.
346
+ */
347
+ toolCallBurstGuard?: ToolBurstGuardOptions | false;
288
348
  /**
289
349
  * Outbound SimpleX queries to trusted peer AutoRAG agents. Remote sessions
290
350
  * never receive this tool.
@@ -313,9 +373,9 @@ export interface RerankAgentOptions {
313
373
  provider?: string;
314
374
  /** Wire model id. @default "voyageai/rerank-3-lite" */
315
375
  model?: string;
316
- /** API key. Prefer `apiKeyEnv`; this is a programmatic seam. */
376
+ /** API key. Prefer `apiKeyEnv`; this is a programmatic seam. Not used by the `dazzi` provider. */
317
377
  apiKey?: string;
318
- /** Environment variable holding the provider API key. */
378
+ /** Environment variable holding the provider API key. Not used by the `dazzi` provider, which uses the `/login` credential. */
319
379
  apiKeyEnv?: string;
320
380
  /** Override the provider base URL. */
321
381
  baseUrl?: string;
@@ -356,9 +416,23 @@ export declare class AutoRAGAgent {
356
416
  private lastQuery;
357
417
  private lastSessionId;
358
418
  private activeRun;
419
+ /**
420
+ * Steering state of the running search. A TUI query runs detached from pi's
421
+ * input loop (pi reads the next input only after its input handler returns),
422
+ * so input typed meanwhile steers this run (Enter) or waits behind it
423
+ * (Alt+Enter), the way pi steers its own turns. {@link steer} feeds the same
424
+ * state for `searchDocuments()` / `searchDocumentsStream()` callers.
425
+ */
426
+ private readonly interactive;
427
+ /** Aborted by {@link abort} and by the run's timeout: gates publishing and the run's own requests. */
428
+ private activeRunAbort;
359
429
  private modelNativeSearchAuth;
360
430
  private retrievalTrace;
361
- /** Evidence every retrieval tool returned this run; the answer's inline `[eN]` citations resolve against it. */
431
+ /**
432
+ * Evidence the retrieval tools returned; the answer's inline `[eN]` citations resolve against it. A bound
433
+ * interactive session keeps it for the whole conversation and persists it with the transcript; any other
434
+ * run starts from an empty ledger.
435
+ */
362
436
  private readonly evidenceLedger;
363
437
  private preliminaryCallback;
364
438
  /** Per-phase thinking levels of the two-phase search. */
@@ -368,6 +442,8 @@ export declare class AutoRAGAgent {
368
442
  private refreshing;
369
443
  private refreshState;
370
444
  private readonly searchPaths;
445
+ /** Roots handed to Jikji: duplicates and roots nested in another root are dropped (see `outermostJikjiRoots`). */
446
+ private readonly jikjiRoots;
371
447
  private readonly configuredSearchPaths;
372
448
  readonly languages: readonly LanguageTag[];
373
449
  private retrievalScopeBindings;
@@ -383,6 +459,7 @@ export declare class AutoRAGAgent {
383
459
  private readonly everythingClient;
384
460
  private readonly fsearchClient;
385
461
  private readonly datasourceSkills;
462
+ private readonly datasourceRetry;
386
463
  private readonly startupDiagnostics;
387
464
  private readonly datasourceAgentSkills;
388
465
  private readonly parserOptions;
@@ -402,8 +479,6 @@ export declare class AutoRAGAgent {
402
479
  private readonly excludePaths;
403
480
  private readonly baseSystemPromptConfig;
404
481
  private readonly droppedCallerToolNames;
405
- private readonly searchTimeoutMs;
406
- private readonly maxSearchToolCalls;
407
482
  private readonly piAgentDir;
408
483
  private readonly piSessionDir;
409
484
  private readonly persistPiSessions;
@@ -423,8 +498,9 @@ export declare class AutoRAGAgent {
423
498
  /** True when this agent was constructed for an untrusted remote peer. */
424
499
  readonly remoteSession: boolean;
425
500
  private activeRetrievalOptions;
426
- private searchToolCallCount;
501
+ private readonly toolBurstGuard;
427
502
  private readonly limits;
503
+ private readonly refreshLimits;
428
504
  constructor(options: AutoRAGAgentOptions);
429
505
  scanDuplicateDocuments(): Promise<ScanDuplicateDocumentsDetails>;
430
506
  /**
@@ -470,8 +546,36 @@ export declare class AutoRAGAgent {
470
546
  private currentSystemPromptConfig;
471
547
  subscribe(listener: Parameters<Agent["subscribe"]>[0]): () => void;
472
548
  createPiInteractiveRuntime(): Promise<AutoRAGPiInteractiveRuntime>;
549
+ /**
550
+ * The session's custom tools, with retrieval and web reads interruptible by a
551
+ * steer: stopping them loses no side effect. bash and the file tools are pi
552
+ * built-ins and never pass through here.
553
+ */
554
+ private steerableTools;
555
+ /**
556
+ * Lets a steer cut a retrieval tool short, like an interruptible wait in pi:
557
+ * the tool's signal also aborts on the steer, and the call resolves at once
558
+ * with a skipped result telling the model to retry if the search still
559
+ * matters, so the steer reaches the model before the next request.
560
+ */
561
+ private interruptibleBySteer;
473
562
  /** Resolve the model's inline `[eN]` citations against this run's evidence and derive the structured results. */
474
563
  private deriveAnswer;
564
+ /**
565
+ * Routes one TUI input the way pi routes typed input for its own turns. With
566
+ * no search running it starts one. During a search, Enter steers it (the
567
+ * user is correcting a search that is going wrong) and Alt+Enter queues a
568
+ * follow-up question that runs once this search answers.
569
+ */
570
+ private routeInteractiveInput;
571
+ /**
572
+ * Esc and Alt+Enter during a search. pi handles both itself while a turn
573
+ * streams (abort, queue follow-up); between phases it ignores Esc and submits
574
+ * Alt+Enter like Enter, so AutoRAG acts on them there.
575
+ */
576
+ private handleInteractiveKey;
577
+ /** Runs one TUI query detached from pi's input loop, then any follow-ups queued behind it. */
578
+ private startInteractiveQuery;
475
579
  /**
476
580
  * Drives one TUI query through the two-phase search. The model's own
477
581
  * assistant messages (progress notes before a tool call, the fast answer,
@@ -480,7 +584,31 @@ export declare class AutoRAGAgent {
480
584
  * twice and put it in the model's context twice.
481
585
  */
482
586
  private runInteractivePiQuery;
587
+ /** Resolves once the TUI search and every follow-up queued behind it finished. */
588
+ waitForIdle(): Promise<void>;
589
+ /**
590
+ * Steers the running `searchDocuments()` / `searchDocumentsStream()` search,
591
+ * the way Enter steers a TUI search: `text` is a correction of the search in
592
+ * progress, not a new question. It reaches the search's next model request
593
+ * (also between phases, where no turn streams), cuts in-flight retrieval and
594
+ * the baseline prefetch short so it lands promptly, and forces the
595
+ * verification phase, whose prompt restates it. A steer that lands after
596
+ * verification's last request gets one more verification turn in the same
597
+ * search. The final answer stays the answer to the original question.
598
+ *
599
+ * Returns false, and does nothing, when no search is running, the search was
600
+ * stopped, or it is past its last model request; the caller should then ask
601
+ * the text as a new question.
602
+ */
603
+ steer(text: string, images?: readonly ImageContent[]): boolean;
604
+ /**
605
+ * Stops the running search as a whole: the streaming model turn, the
606
+ * baseline prefetch, in-flight retrieval, and every later phase. The search
607
+ * then resolves without a verified answer.
608
+ */
483
609
  abort(): void;
610
+ /** The next model request took the queued steers into the transcript; later retrieval runs uninterrupted until the next steer. */
611
+ private steersReachedModel;
484
612
  /**
485
613
  * Periodically re-runs the incremental {@link refresh} so parsed mirrors and
486
614
  * indexes stay current. Re-parsing is incremental (mtime/size) via the
@@ -584,6 +712,12 @@ export declare class AutoRAGAgent {
584
712
  * as plain text, without any further tool calls.
585
713
  */
586
714
  buildFastAnswerPrompt(query: string, options: RetrievalOptions, baseline: string): string;
715
+ /**
716
+ * True when the bound interactive session already holds an assistant turn:
717
+ * the query continues a conversation instead of opening one. Read from the
718
+ * transcript, so a fresh session (`/new`) starts over with the full pipeline.
719
+ */
720
+ private isConversationTurn;
587
721
  /**
588
722
  * Prompt for a question Jev routed to a direct answer: general knowledge or
589
723
  * small talk. No retrieval ran and no verification phase follows, so the
@@ -604,7 +738,7 @@ export declare class AutoRAGAgent {
604
738
  * against it; the model then verifies with thinking on and ends with the
605
739
  * final answer as a plain reply, returning only the delta.
606
740
  */
607
- buildRefinementPrompt(query: string, options: RetrievalOptions, fastAnswer: string | undefined, fastAnswerDelivered: boolean): string;
741
+ buildRefinementPrompt(query: string, options: RetrievalOptions, fastAnswer: string | undefined, fastAnswerDelivered: boolean, steers?: readonly string[]): string;
608
742
  buildSearchPrompt(query: string, options: RetrievalOptions, initialRetrievalContext?: string): string;
609
743
  refresh(force?: boolean, opts?: AutoRAGRefreshOptions): Promise<AutoRAGRefreshResult>;
610
744
  /**
@@ -633,7 +767,7 @@ export declare class AutoRAGAgent {
633
767
  */
634
768
  startWatchRefresh(options?: AutoRAGWatchRefreshOptions): AutoRAGWatchRefreshHandle;
635
769
  private defaultWatcherFactory;
636
- syncParsedMirrors(force?: boolean): Promise<ParsedMirrorSyncResult>;
770
+ syncParsedMirrors(force?: boolean, options?: Pick<ParsedMirrorSyncOptions, "signal" | "onProgress" | "retrySkipped">): Promise<ParsedMirrorSyncResult>;
637
771
  /**
638
772
  * Lightweight stat-only staleness scan used when `refresh` is called with
639
773
  * methods that exclude parsed mirrors (e.g. only `datasources` or `jikji`).
@@ -642,7 +776,7 @@ export declare class AutoRAGAgent {
642
776
  */
643
777
  private scanMirrorStaleness;
644
778
  private exactDuplicateExclusions;
645
- syncMinSync(force?: boolean): Promise<MinSyncSyncResult>;
779
+ syncMinSync(force?: boolean, signal?: AbortSignal): Promise<MinSyncSyncResult>;
646
780
  /** Provider for the Windows-only everything_search tool. */
647
781
  searchEverything(request: EverythingSearchRequest): Promise<EverythingSearchResult>;
648
782
  /**
@@ -660,12 +794,18 @@ export declare class AutoRAGAgent {
660
794
  * macOS/Linux.
661
795
  */
662
796
  stopFsearch(): Promise<void>;
663
- prepareJikji(): Promise<readonly AutoRAGJikjiPrepareResult[] | undefined>;
797
+ prepareJikji(options?: JikjiPrepareOptions): Promise<readonly AutoRAGJikjiPrepareResult[] | undefined>;
664
798
  /**
665
799
  * Build or incrementally update every root's Jikji index. Called only by
666
800
  * refresh/watch (and the explicit `prepareJikji()` API), never by a query:
667
- * `jikji prepare` reuses unchanged documents, and roots run in parallel so
668
- * one slow root does not serialize the rest.
801
+ * `jikji prepare` reuses unchanged documents. Roots are submitted together
802
+ * but `JikjiClient` runs them one at a time, because every root shares one
803
+ * central SQLite file that fails with "database is locked" under concurrent
804
+ * writers. Nested and duplicate roots are skipped: upstream rejects
805
+ * overlapping roots and the outer root's index already covers them. A failed
806
+ * or timed-out root is a degraded result (reported as a diagnostic), never a
807
+ * rejection, so the refresh still finishes. `jikji.parseTimeout` /
808
+ * `jikji.timeoutMs` / `jikji.prepareTimeoutMsPerFile` apply to every root.
669
809
  */
670
810
  private executeJikjiPrepare;
671
811
  private sanitizeJikjiPrepareResult;
@@ -679,6 +819,7 @@ export declare class AutoRAGAgent {
679
819
  findJikji(query: string, opts?: {
680
820
  readonly topK?: number;
681
821
  readonly first?: boolean;
822
+ readonly signal?: AbortSignal;
682
823
  }): Promise<JikjiFindProviderResult>;
683
824
  /**
684
825
  * Merge per-root answer packs into one. Concatenates answer_paths/candidates
@@ -723,6 +864,7 @@ export declare class AutoRAGAgent {
723
864
  searchAllDocuments(query: string, options?: {
724
865
  readonly topK?: number;
725
866
  readonly scope?: string;
867
+ readonly signal?: AbortSignal;
726
868
  }): Promise<{
727
869
  results: RetrievalResult[];
728
870
  diagnostics: RetrievalDiagnostic[];
@@ -740,6 +882,7 @@ export declare class AutoRAGAgent {
740
882
  searchSingleDatasourceDocuments(datasourceId: string, query: string, options?: {
741
883
  readonly topK?: number;
742
884
  readonly scope?: string;
885
+ readonly signal?: AbortSignal;
743
886
  }): Promise<{
744
887
  results: RetrievalResult[];
745
888
  diagnostics: RetrievalDiagnostic[];
@@ -0,0 +1,23 @@
1
+ import type { AgentMessage } from "@earendil-works/pi-agent-core";
2
+ import type { Model } from "@earendil-works/pi-ai";
3
+ type ContextTokenGuardModel = Model<any> | (() => Model<any> | undefined) | undefined;
4
+ /**
5
+ * Compose an optional context transform with the aggregate token guard. The
6
+ * guard runs on the final transcript immediately before each provider request,
7
+ * so it sees the system prompt, tool declarations, user query, memory context,
8
+ * and every prior tool result.
9
+ */
10
+ export declare function createContextTokenGuardedTransform(model: ContextTokenGuardModel, base?: (messages: AgentMessage[]) => Promise<AgentMessage[]>, fixedOverheadTokens?: () => number): (messages: AgentMessage[]) => Promise<AgentMessage[]>;
11
+ /**
12
+ * Keep complete retrieval candidates while the assembled request fits the
13
+ * model budget. Removing a per-chunk cut is only safe because this drops whole
14
+ * lower-ranked candidates (with an explicit marker) instead of hiding text.
15
+ */
16
+ export declare function guardAgentContextMessages(model: Model<any> | undefined, messages: AgentMessage[], useReportedUsage?: boolean, fixedOverheadTokens?: number): AgentMessage[];
17
+ /**
18
+ * Estimate the current transcript with pi-coding-agent's conservative chars/4
19
+ * counter. `useReportedUsage` adds the last assistant turn's provider-reported
20
+ * usage for the settled prefix; pass false when the messages were just edited.
21
+ */
22
+ export declare function estimateContextMessageTokens(messages: readonly AgentMessage[], useReportedUsage?: boolean): number;
23
+ export {};
@@ -1,3 +1,4 @@
1
+ import { type Static, Type } from "typebox";
1
2
  import type { RetrievalResult } from "../retrieval/types.js";
2
3
  import type { AutoRAGEvidenceRef } from "./results.js";
3
4
  export interface LedgerEvidenceInput {
@@ -13,28 +14,65 @@ export interface LedgerEvidenceInput {
13
14
  readonly evidenceType?: string;
14
15
  readonly evidenceLocation?: string;
15
16
  }
17
+ /** Custom session entry type under which a conversation persists its evidence ledger. */
18
+ export declare const EVIDENCE_ENTRY_TYPE = "autorag.evidence";
19
+ declare const ledgerEntrySchema: Type.TObject<{
20
+ id: Type.TString;
21
+ /** Source plus a digest of the full chunk: the same chunk retrieved again keeps its id. */
22
+ key: Type.TString;
23
+ method: Type.TString;
24
+ source: Type.TString;
25
+ content: Type.TString;
26
+ retrieverMix: Type.TArray<Type.TString>;
27
+ retrievalResultId: Type.TOptional<Type.TString>;
28
+ chunkIndex: Type.TOptional<Type.TNumber>;
29
+ lineNumber: Type.TOptional<Type.TNumber>;
30
+ parserType: Type.TOptional<Type.TString>;
31
+ documentType: Type.TOptional<Type.TString>;
32
+ documentArea: Type.TOptional<Type.TString>;
33
+ evidenceType: Type.TOptional<Type.TString>;
34
+ evidenceLocation: Type.TOptional<Type.TString>;
35
+ }>;
36
+ /** One recorded piece of evidence: what a retrieval tool returned, under the id it printed. */
37
+ export type LedgerEntry = Static<typeof ledgerEntrySchema>;
38
+ /**
39
+ * Entries a session persisted under {@link EVIDENCE_ENTRY_TYPE}. A malformed
40
+ * entry is skipped: its id then resolves to nothing and the citation is
41
+ * dropped with a diagnostic, never mis-resolved.
42
+ */
43
+ export declare function parseLedgerEntries(data: unknown): LedgerEntry[];
44
+ /** Highest evidence id ordinal among `entries` (`e12` is 12); 0 when there are none. */
45
+ export declare function highestOrdinal(entries: readonly LedgerEntry[]): number;
16
46
  export interface LookupEvidenceOptions {
17
47
  /** False in remote sessions: only evidence a tool returned this run may be cited. */
18
48
  readonly allowLocalFiles: boolean;
19
49
  }
20
50
  export declare function isRegularFile(path: string): boolean;
21
51
  /**
22
- * Per-run record of every piece of evidence a retrieval tool returned.
52
+ * Record of every piece of evidence the retrieval tools returned in one
53
+ * conversation.
23
54
  *
24
55
  * Tools hand the model a short id (`e1`, `e2`, ...) next to each result. The
25
56
  * model cites evidence by id; the harness resolves the id back to the exact
26
57
  * source, method, content, and chunk metadata it recorded. The model never
27
58
  * retypes a path or a chunk, so it cannot misquote or invent one, and the
28
59
  * stable evidence id stored in memory is always derived from real evidence.
60
+ *
61
+ * Ids count up for as long as the process lives and are never reissued, so an
62
+ * id the model saw in any earlier turn keeps naming the chunk it named. The
63
+ * hosting session persists what {@link takeUnsaved} returns and rebuilds the
64
+ * ledger with {@link restore} when a conversation is resumed.
29
65
  */
30
66
  export declare class EvidenceLedger {
31
67
  private entries;
32
68
  private readonly byKey;
33
69
  private readonly byId;
70
+ /** Entries recorded or changed since the last {@link takeUnsaved}. */
71
+ private readonly unsaved;
34
72
  /**
35
- * Ordinal of the last issued id. Survives {@link clear}: a hosted (TUI)
36
- * conversation keeps earlier runs' tool output in context, so restarting at
37
- * e1 would let a stale id silently resolve to a different chunk.
73
+ * Ordinal of the last issued id. Survives {@link clear} and {@link restore}:
74
+ * a conversation keeps earlier turns' tool output in context, so restarting
75
+ * at e1 would let a stale id silently resolve to a different chunk.
38
76
  */
39
77
  private lastOrdinal;
40
78
  /** Record evidence and return its id. The same chunk always gets the same id. */
@@ -49,6 +87,17 @@ export declare class EvidenceLedger {
49
87
  * citation instead of failing the run.
50
88
  */
51
89
  lookup(raw: string, options: LookupEvidenceOptions): AutoRAGEvidenceRef[];
52
- /** Forget every recorded entry. Ids are never reissued, so a previous run's id stays unresolvable. */
90
+ /** Forget every recorded entry. Ids are never reissued, so a previous conversation's id stays unresolvable. */
53
91
  clear(): void;
92
+ /** Entries recorded or changed since the last call, for the hosting session to persist. */
93
+ takeUnsaved(): LedgerEntry[];
94
+ /**
95
+ * Rebuild the ledger from persisted entries, such as the ones on the resumed
96
+ * conversation branch; a later entry with the same id replaces an earlier
97
+ * one. Ids keep counting past every restored id and past `ordinalFloor`, the
98
+ * highest id issued anywhere in the conversation (other branches included),
99
+ * so a restored ledger never reissues an id a transcript may still show.
100
+ */
101
+ restore(entries: readonly LedgerEntry[], ordinalFloor?: number): void;
54
102
  }
103
+ export {};
@@ -0,0 +1,24 @@
1
+ import type { ExtensionFactory, SessionEntry } from "@earendil-works/pi-coding-agent";
2
+ import { type EvidenceLedger } from "./evidence-ledger.js";
3
+ /** The part of pi's session manager the ledger is rebuilt from. */
4
+ interface SessionEntries {
5
+ getBranch(): SessionEntry[];
6
+ getEntries(): SessionEntry[];
7
+ }
8
+ /**
9
+ * Rebuild `ledger` for the conversation branch `session` is on. Evidence from
10
+ * other branches stays out of reach (their transcripts are not in context), but
11
+ * its ids still count: new evidence never reuses an id issued anywhere in the
12
+ * conversation.
13
+ */
14
+ export declare function restoreEvidenceLedger(ledger: EvidenceLedger, session: SessionEntries): void;
15
+ /**
16
+ * pi extension that keeps the conversation's evidence ledger in the session
17
+ * file, next to the transcript that shows the model its `[eN]` ids. Evidence is
18
+ * appended as the tools return it; switching, resuming, forking, or navigating
19
+ * the session tree rebuilds the ledger from the branch now in view. An id the
20
+ * model saw therefore resolves, and is never reissued, whatever turns, tool
21
+ * calls, resumes, or restarts happened in between.
22
+ */
23
+ export declare function createEvidencePersistence(ledger: EvidenceLedger): ExtensionFactory;
24
+ export {};
@@ -4,9 +4,10 @@ import { type JevBackend, type Judgment, type Question, type State } from "jev-u
4
4
  export declare const JEV_TOOL_NAME = "jev";
5
5
  /**
6
6
  * Jev backends the `jev-use` engine resolves from the environment, in
7
- * auto-select order: TypeSafe -> OpenRouter -> Vercel AI Gateway.
7
+ * auto-select order: TypeSafe -> OpenRouter -> Vercel AI Gateway. `dazzi` is
8
+ * the hosted Dazzi Models plan, built here rather than resolved by jev-use.
8
9
  */
9
- export type JevBackendName = "typesafe" | "openrouter" | "vercel";
10
+ export type JevBackendName = "typesafe" | "openrouter" | "vercel" | "dazzi";
10
11
  /**
11
12
  * OpenRouter's live Jev model id. `jev-use` 0.8.0 ships `typesafe/jev-latest`,
12
13
  * which OpenRouter rejects with HTTP 400 ("Model ... does not exist"), so the
@@ -39,6 +40,8 @@ export interface JevToolOptions {
39
40
  export interface JevJudgeOptions extends JevToolOptions {
40
41
  /** Environment consulted for the backend credential. Default `process.env`. */
41
42
  readonly env?: Record<string, string | undefined>;
43
+ /** Pi agent home whose `auth.json` holds the Dazzi `/login` credential. Default: Pi's agent dir. */
44
+ readonly agentDir?: string;
42
45
  }
43
46
  /**
44
47
  * One batched Jev judgment. Rejects only when no backend can be built (for
@@ -51,8 +54,8 @@ export type JevJudge = (state: State, questions: Question[], options?: {
51
54
  /**
52
55
  * Builds the Jev judge shared by the `jev` tool and the query router. The
53
56
  * `jev-use` client is created lazily on the first call, so no credential
54
- * resolves at construction time, and the OpenRouter model pin applies to
55
- * every caller.
57
+ * resolves at construction time, and the per-backend model default (the
58
+ * OpenRouter pin, or `jev-1.13` on the hosted plan) applies to every caller.
56
59
  */
57
60
  export declare function createJevJudge(options?: JevJudgeOptions): JevJudge;
58
61
  /**
@@ -44,6 +44,7 @@ export interface JikjiFindProvider {
44
44
  findJikji(query: string, opts?: {
45
45
  readonly topK?: number;
46
46
  readonly first?: boolean;
47
+ readonly signal?: AbortSignal;
47
48
  }): Promise<JikjiFindProviderResult>;
48
49
  }
49
50
  export interface JikjiFindDetails {
@@ -1,5 +1,5 @@
1
1
  import type { AgentMessage, AgentTool } from "@earendil-works/pi-agent-core";
2
- import type { Api, Model } from "@earendil-works/pi-ai";
2
+ import type { Api, ImageContent, Model } from "@earendil-works/pi-ai";
3
3
  import { type AgentSession, type AgentSessionRuntime, type ExtensionAPI, type ExtensionFactory, SettingsManager, type ToolDefinition, type TuiMode } from "@earendil-works/pi-coding-agent";
4
4
  export declare const PI_BUILTIN_TOOL_NAMES: readonly ["read", "bash", "edit", "write", "grep", "find", "ls"];
5
5
  /**
@@ -34,10 +34,43 @@ export interface AutoRAGPiSessionOptions {
34
34
  readonly remoteSession?: boolean;
35
35
  readonly contextTransform?: (messages: AgentMessage[]) => Promise<AgentMessage[]>;
36
36
  }
37
+ /**
38
+ * How AutoRAG takes one interactive input. `steer` while a pi turn streams
39
+ * returns the text pi should queue as steering (pi delivers it before the
40
+ * turn's next model request); every other outcome is fully handled by AutoRAG.
41
+ */
42
+ export type AutoRAGInputDisposition = {
43
+ readonly kind: "handled";
44
+ } | {
45
+ readonly kind: "steer";
46
+ readonly text: string;
47
+ };
48
+ export interface AutoRAGInteractiveInput {
49
+ readonly text: string;
50
+ readonly images: readonly ImageContent[] | undefined;
51
+ /** pi's delivery for input typed while a turn streams; undefined when no turn streams. */
52
+ readonly streamingBehavior: "steer" | "followUp" | undefined;
53
+ }
54
+ /** A TUI key AutoRAG acts on itself while a search runs, even when no pi turn streams. */
55
+ export type AutoRAGInteractiveKey = "interrupt" | "followUp";
37
56
  export interface AutoRAGPiInteractiveRuntimeOptions extends Omit<AutoRAGPiSessionOptions, "model"> {
38
57
  readonly model?: Model<Api>;
39
58
  readonly inactiveToolNames?: readonly string[];
40
- readonly onQuery: (query: string, pi: ExtensionAPI) => void | Promise<void>;
59
+ /**
60
+ * Routes one interactive input: a new query, a steer of the running search,
61
+ * or a follow-up queued behind it. Must not wait for a search to finish:
62
+ * pi's TUI reads the next input only after this returns.
63
+ */
64
+ readonly onInput: (input: AutoRAGInteractiveInput, pi: ExtensionAPI) => AutoRAGInputDisposition;
65
+ /** Whether a search is running: while it is, every interactive input belongs to it. */
66
+ readonly isSearchRunning?: () => boolean;
67
+ /**
68
+ * Called for pi's interrupt (Esc) and follow-up (Alt+Enter) keys while a
69
+ * search runs. pi acts on them only while a turn streams, but a search also
70
+ * spends time between turns (routing, prefetch, checks). Return true to
71
+ * consume the key; pi then never sees it.
72
+ */
73
+ readonly onKey?: (key: AutoRAGInteractiveKey) => boolean;
41
74
  /**
42
75
  * Optional best-effort provider for a startup update notice. Resolves to the
43
76
  * notice text, or `undefined` when there is nothing to announce. Called once
@@ -0,0 +1,20 @@
1
+ import type { AgentMessage } from "@earendil-works/pi-agent-core";
2
+ import type { ModelInputLimits } from "@earendil-works/pi-ai";
3
+ /** Stands in for an image dropped from a model request by {@link fitRequestLimits}. */
4
+ export declare const IMAGE_OMITTED_NOTE = "[Image omitted from this request to stay within the model provider's request limits. Read the file again if you still need to see it.]";
5
+ /**
6
+ * Fits a model request into the provider's declared `inputLimits`.
7
+ *
8
+ * pi keeps every image a tool returned in the transcript and resends all of
9
+ * them on each later turn, but it never enforces `maxRequestBytes` or
10
+ * `images.maxPerRequest`. A search that reads a handful of photos therefore
11
+ * grows past a provider's body limit and the request fails outright (the hosted
12
+ * AutoRAG API answers HTTP 413). Images are what grows the body, so the oldest
13
+ * ones are replaced with {@link IMAGE_OMITTED_NOTE} until the request fits; the
14
+ * newest images, which the model is looking at, go last. Text is never touched.
15
+ *
16
+ * The size is the serialized transcript, a close lower bound of the provider
17
+ * body, so `maxRequestBytes` should keep headroom below the hard limit.
18
+ * Returns undefined when the request already fits; never mutates `messages`.
19
+ */
20
+ export declare function fitRequestLimits(messages: readonly AgentMessage[], limits: ModelInputLimits | undefined): AgentMessage[] | undefined;
@@ -18,6 +18,7 @@ export interface SearchAllDocumentsProvider {
18
18
  searchAllDocuments(query: string, options?: {
19
19
  readonly topK?: number;
20
20
  readonly scope?: string;
21
+ readonly signal?: AbortSignal;
21
22
  }): Promise<SearchAllDocumentsResult>;
22
23
  }
23
24
  export interface SearchAllDocumentsDetails {
@@ -9,7 +9,7 @@ export type SearchDocumentDiagnosticSeverity = "info" | "warning" | "error";
9
9
  * (#21/#22) does not require another source-breaking type change; codes not yet
10
10
  * emitted are still valid members.
11
11
  */
12
- export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-staging-excluded" | "minsync-sync-failed" | "embedder-unavailable" | "embedding-identity-mismatch" | "parser-skipped" | "parser-failed" | "source-vanished" | "source-unreadable" | "parser-warning" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "pdf-extract-thin" | "retrieval-method-failed" | "rerank-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "everything-index-failed" | "fsearch-index-failed" | "fsearch-binary-missing" | "refresh-failed" | "refresh-interrupted" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "no-final-answer" | "model-request-failed" | "search-timeout" | "query-routed" | "query-route-fallback" | "query-decomposition-failed" | "follow-up-skipped" | "follow-up-check-fallback" | "datasources-selected" | "datasource-selection-fallback" | "citation-without-result" | "evidence-judged" | "evidence-judgment-fallback" | "memory-search-fallback" | "self-config-unavailable" | "self-config-rolled-back";
12
+ export type SearchDocumentDiagnosticCode = "empty-query" | "no-verified-results" | "unknown-warning" | "caller-tool-dropped" | "minsync-staging-excluded" | "minsync-sync-failed" | "minsync-lock-held" | "embedder-unavailable" | "embedding-identity-mismatch" | "parser-skipped" | "parser-failed" | "parser-crashed" | "source-vanished" | "source-unreadable" | "source-read-failed" | "mirror-write-failed" | "parser-warning" | "duplicate-excluded" | "unsupported-file" | "stale-index" | "deleted-mirror" | "search-root-unavailable" | "mass-delete-suppressed" | "walk-truncated" | "directory-unreadable" | "pdf-extract-thin" | "retrieval-method-failed" | "rerank-failed" | "jikji-unavailable" | "jikji-prepare-failed" | "jikji-find-failed" | "everything-index-failed" | "fsearch-index-failed" | "fsearch-binary-missing" | "refresh-failed" | "refresh-interrupted" | "watch-failed" | "watch-limited" | "unknown-datasource-skill" | "no-final-answer" | "model-request-failed" | "tool-call-burst" | "query-routed" | "query-route-fallback" | "query-decomposition-failed" | "follow-up-skipped" | "follow-up-check-fallback" | "datasources-selected" | "datasource-selection-fallback" | "citation-without-result" | "evidence-judged" | "evidence-judgment-fallback" | "memory-search-fallback" | "self-config-unavailable" | "self-config-rolled-back";
13
13
  export interface SearchDocumentDiagnostic {
14
14
  readonly code: SearchDocumentDiagnosticCode;
15
15
  readonly severity: SearchDocumentDiagnosticSeverity;
@@ -39,6 +39,7 @@ export interface SingleDatasourceSearchProvider {
39
39
  searchSingleDatasourceDocuments(datasourceId: string, query: string, options?: {
40
40
  readonly topK?: number;
41
41
  readonly scope?: string;
42
+ readonly signal?: AbortSignal;
42
43
  }): Promise<{
43
44
  readonly results: readonly RetrievalResult[];
44
45
  readonly diagnostics: readonly RetrievalDiagnostic[];
@@ -1,10 +1,8 @@
1
- import type { StoreManifest } from "../manifest/types.js";
2
1
  import type { DatasourceAgentSkill } from "./datasource-skill.js";
3
2
  export interface SystemPromptConfig {
4
3
  toolNames: string[];
5
4
  modelId?: string;
6
5
  memoryEntries?: readonly unknown[];
7
- manifests: StoreManifest[];
8
6
  jikjiIndexingEnabled?: boolean;
9
7
  datasourceSkills?: readonly DatasourceAgentSkill[];
10
8
  retrievedContentGuard?: boolean;