@thurstonsand/pi-librarian 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CONTEXT.md +10 -2
- package/README.md +25 -7
- package/RELEASE.md +20 -0
- package/docs/designs/02-continuable-runs.md +102 -0
- package/docs/designs/04-librarian-extra-tools.md +138 -0
- package/extensions/librarian/extra-tools.ts +130 -0
- package/extensions/librarian/prompt.ts +1 -1
- package/extensions/librarian/resource-loader.ts +40 -0
- package/extensions/librarian/run.ts +88 -45
- package/extensions/librarian/settings.ts +11 -3
- package/extensions/librarian/tools/checkout-repo.ts +3 -18
- package/extensions/librarian/tools/provide-results.ts +19 -6
- package/extensions/librarian/tools/read-github-file.ts +12 -16
- package/extensions/librarian/tools/search-repos.ts +1 -10
- package/extensions/librarian/view.ts +15 -1
- package/extensions/librarian.ts +25 -21
- package/package.json +1 -1
package/AGENTS.md
CHANGED
|
@@ -15,7 +15,7 @@ And this doesn't even touch on the advantages of giving the agent an efficient t
|
|
|
15
15
|
## Core principles
|
|
16
16
|
|
|
17
17
|
- Build on pi-native concepts, types, and extension APIs where available; read pi source when helpful
|
|
18
|
-
- The librarian
|
|
18
|
+
- The librarian does not include write/edit tools; clones live only in the checkout cache
|
|
19
19
|
- Prioritize ergonomics of the exposed interaction surface over internal implementation
|
|
20
20
|
|
|
21
21
|
See @DEV.md for code style and development commands.
|
package/CONTEXT.md
CHANGED
|
@@ -3,12 +3,15 @@
|
|
|
3
3
|
## Language
|
|
4
4
|
|
|
5
5
|
**Librarian**:
|
|
6
|
-
The research agent spawned by the `librarian` tool; runs in a nested
|
|
6
|
+
The research agent spawned by the `librarian` tool; runs in a nested agent session with its own toolset and system prompt.
|
|
7
7
|
|
|
8
8
|
**Librarian run**:
|
|
9
9
|
One invocation of the librarian: query in, findings out, with a recorded tool-call trace.
|
|
10
10
|
_Avoid_: session (reserved for pi sessions)
|
|
11
11
|
|
|
12
|
+
**Run id**:
|
|
13
|
+
The pi session uuid of a librarian run, so that it may be continued for follow up questions.
|
|
14
|
+
|
|
12
15
|
**Findings**:
|
|
13
16
|
The structured output of a librarian run, produced by `provide_results`: summary, locations, optional description.
|
|
14
17
|
|
|
@@ -16,7 +19,10 @@ The structured output of a librarian run, produced by `provide_results`: summary
|
|
|
16
19
|
The repository research tools this package registers: `search_repos`, `search_code`, `search_github_code`, `checkout_repo`, `read_github_file`, `provide_results`. Exclusive to librarian runs unless attached.
|
|
17
20
|
|
|
18
21
|
**Inherited tools**:
|
|
19
|
-
Pi built-ins granted to librarian runs (`read`, `grep`, `find`, `ls`, `bash`)
|
|
22
|
+
Pi built-ins granted to librarian runs (`read`, `grep`, `find`, `ls`, `bash`); `write`/`edit` are not included.
|
|
23
|
+
|
|
24
|
+
**Extra tools**:
|
|
25
|
+
Tools made available to the librarian as configured in `librarian.tools` settings, in addition to the **Repo tools** and **Inherited tools**.
|
|
20
26
|
|
|
21
27
|
**Attach / attached tools**:
|
|
22
28
|
Loading the repo tools into the main pi session via `/librarian`, making them directly usable by the main agent alongside the `librarian` tool.
|
|
@@ -26,3 +32,5 @@ _Avoid_: load, enable
|
|
|
26
32
|
|
|
27
33
|
- A **Librarian run** ends with exactly one **Findings**, containing zero or more **Locations**.
|
|
28
34
|
- **Attached tools** are the same **Repo tools** a **Librarian** uses, minus `provide_results`.
|
|
35
|
+
- **Extra tools** are only relevant inside librarian runs; **Attach** mode is unaffected because the main session manages its own set of tools.
|
|
36
|
+
- a **Librarian run** is persisted in its own session directory, identified by a **Run id**.
|
package/README.md
CHANGED
|
@@ -12,11 +12,19 @@ The `librarian` tool spawns a nested research agent with purpose-built tools:
|
|
|
12
12
|
- **`search_github_code`** — GitHub REST code search over public code and private repositories your configured GitHub auth can access.
|
|
13
13
|
- **`read_github_file`** — single-file API reads for quick peeks without cloning.
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
## GitHub auth for private repos
|
|
16
|
+
|
|
17
|
+
Public GitHub reads work without configuration. To let the librarian search and read private GitHub repositories, provide a token in one of these ways:
|
|
18
|
+
|
|
19
|
+
1. Set `GITHUB_TOKEN` or `GH_TOKEN` in the environment before starting pi.
|
|
20
|
+
2. Or authenticate the GitHub CLI so `gh auth token` returns a token:
|
|
21
|
+
|
|
22
|
+
The token is loaded once per pi session and passed to GitHub REST calls used by `search_repos`, `search_github_code`, `checkout_repo`, and `read_github_file`. For private repositories, use a token with read access to the target repos.
|
|
16
23
|
|
|
17
24
|
## Usage
|
|
18
25
|
|
|
19
26
|
- Ask pi a question involving other repos; it delegates to the `librarian` tool.
|
|
27
|
+
- Ask follow-up questions to earlier librarian runs.
|
|
20
28
|
- `/librarian` attaches the research tools directly to your session for manual lookups.
|
|
21
29
|
|
|
22
30
|
## Configuration
|
|
@@ -26,15 +34,25 @@ In pi's global `settings.json`:
|
|
|
26
34
|
```jsonc
|
|
27
35
|
{
|
|
28
36
|
"librarian": {
|
|
29
|
-
"model": "
|
|
30
|
-
"thinkingLevel": "
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"cacheDir": "/tmp/pi-librarian"
|
|
34
|
-
}
|
|
37
|
+
"model": "openai-codex/gpt-5.5",
|
|
38
|
+
"thinkingLevel": "off",
|
|
39
|
+
"tools": ["search_web", "fetch_web"],
|
|
40
|
+
"extensions": ["~/.pi/agent/extensions/parallel-web-tools"],
|
|
41
|
+
"cacheDir": "/tmp/pi-librarian",
|
|
42
|
+
},
|
|
35
43
|
}
|
|
36
44
|
```
|
|
37
45
|
|
|
46
|
+
| Setting | Recommended | Default |
|
|
47
|
+
| --------------- | --------------------------------------------- | ------------------------------ |
|
|
48
|
+
| `model` | `openai-codex/gpt-5.5` | current session model |
|
|
49
|
+
| `thinkingLevel` | `off` | current session thinking level |
|
|
50
|
+
| `tools` | names of extra tools to activate, when needed | `[]` |
|
|
51
|
+
| `extensions` | escape hatch paths for tools not loaded in pi | `[]` |
|
|
52
|
+
| `cacheDir` | `/tmp/pi-librarian` | `/tmp/pi-librarian` |
|
|
53
|
+
|
|
54
|
+
`librarian.tools` is the activation gate for extra tools. `librarian.extensions` only adds extension paths to the search space when a named tool is not already loaded in the main pi session; listing an extension path does not activate every tool in that bundle. Librarian runs exclude `write` and `edit`.
|
|
55
|
+
|
|
38
56
|
## Development
|
|
39
57
|
|
|
40
58
|
```bash
|
package/RELEASE.md
CHANGED
|
@@ -2,6 +2,26 @@
|
|
|
2
2
|
|
|
3
3
|
# Release notes
|
|
4
4
|
|
|
5
|
+
## 0.2.0
|
|
6
|
+
|
|
7
|
+
Adds continuable librarian runs and opt-in for extra tools.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Added continuation support for prior librarian runs by run id.
|
|
12
|
+
- Added `librarian.tools` for opting extra tools into librarian runs by name.
|
|
13
|
+
- Added startup warnings for unresolved extra tool names and failed escape-hatch extension loads.
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- Changed `librarian.extensions` into an escape hatch for resolving named tools not loaded by the main pi agent.
|
|
18
|
+
- Stripped extension hooks from extra tools loaded into nested librarian runs.
|
|
19
|
+
- Removed `librarian.disabledTools`; baseline librarian tools are fixed, while extra tools are additive by name.
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- Fixed tool error handling so pi marks librarian tool failures as errored executions.
|
|
24
|
+
|
|
5
25
|
## 0.1.1
|
|
6
26
|
|
|
7
27
|
Adds automated release and dependency maintenance infrastructure.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Continuable librarian runs
|
|
2
|
+
|
|
3
|
+
## Status
|
|
4
|
+
|
|
5
|
+
Draft
|
|
6
|
+
|
|
7
|
+
## Decision Summary
|
|
8
|
+
|
|
9
|
+
Persist librarian run transcripts as real pi session files in a librarian-owned directory, and add an optional `continue_from: <run uuid>` parameter to the `librarian` tool that reopens a prior run with its full research context. The run id is the durable transcript identifier: a continuation appends to the same run id rather than minting a second per-invocation id. Follow-up questions ("just one more thing about that flow you traced") reuse everything the run already read instead of starting cold. One machinery for both cheap follow-ups and full second research legs.
|
|
10
|
+
|
|
11
|
+
## Problem Statement / Background
|
|
12
|
+
|
|
13
|
+
A librarian run often answers 90% of a question, and the main session discovers the missing 10% only after reading the findings — "you traced the D1 prepared-statement flow; does the same apply to batch()?" Today every run is ephemeral (`SessionManager.inMemory`, disposed on completion), so the follow-up pays full price: re-clone context, re-grep, re-read, re-derive. The run's accumulated context — files read, structure understood, dead ends eliminated — is exactly the asset a follow-up needs.
|
|
14
|
+
|
|
15
|
+
pi already has the machinery: `SessionManager.create(cwd, sessionDir)` persists transcripts to any directory, `SessionManager.listAll(sessionDir)` lists sessions in an arbitrary directory, and `SessionManager.open(path)` reopens a specific file.
|
|
16
|
+
|
|
17
|
+
## Goals
|
|
18
|
+
|
|
19
|
+
- A follow-up to a prior run answers from that run's context when sufficient — potentially zero new tool calls — and researches further when not, with no special-casing between the two.
|
|
20
|
+
- The run uuid is discoverable by both the main agent (in the tool result content) and the human (in the rendered output), so continuation survives compaction boundaries: the user can supply the uuid if the agent lost it.
|
|
21
|
+
- Continuations work across pi restarts and from different main sessions.
|
|
22
|
+
|
|
23
|
+
## Non-Goals
|
|
24
|
+
|
|
25
|
+
- No integration with pi-sessions indexing/search in v1 (see Alternatives).
|
|
26
|
+
- No transcript eviction policy in v1 (mirrors the checkout-cache decision; `rm -rf /tmp/pi-librarian/sessions` is the manual escape for default settings).
|
|
27
|
+
- No lighter-weight result shape for cheap follow-ups — `provide_results` ceremony applies to every run.
|
|
28
|
+
|
|
29
|
+
## Exposed Shape
|
|
30
|
+
|
|
31
|
+
- `librarian` tool gains `continue_from?: string` — the uuid of a prior run. When present, the run resumes that transcript; `query` carries the follow-up question. `repos`/`owners` scope params remain usable.
|
|
32
|
+
- Every librarian tool result with an associated transcript ends with a `run: <uuid>` line (for the agent). On success this is appended after rendered findings; on error or abort it is appended after the error text.
|
|
33
|
+
- The final render shows the same uuid as a muted line above the footer (for the human), whether the run succeeded, errored, or aborted.
|
|
34
|
+
- `LibrarianRunDetails` carries `runId`. Its `trace` remains the live UI trace for the current librarian tool invocation, not the durable transcript history.
|
|
35
|
+
- Tool description documents: pass `continue_from` for follow-ups to earlier research; omit it for new topics.
|
|
36
|
+
|
|
37
|
+
## Design Decisions
|
|
38
|
+
|
|
39
|
+
### 1. Transcripts are pi session files in a librarian-owned directory
|
|
40
|
+
|
|
41
|
+
New runs use `SessionManager.create(cacheDir, <cacheDir>/sessions)` instead of `inMemory`. Continuations list `<cacheDir>/sessions` with `SessionManager.listAll(sessionDir)`, match the requested run id, and reopen via `SessionManager.open(path)`. The directory lives outside pi's default session store, so runs never appear in resume pickers and are invisible to pi-sessions indexing.
|
|
42
|
+
|
|
43
|
+
**Tradeoff**: research memory stays siloed from the pi-sessions "sessions are memory" thesis. Deliberate for v1 — see Alternatives.
|
|
44
|
+
|
|
45
|
+
### 2. Continuation rebuilds the runtime, the transcript provides the memory
|
|
46
|
+
|
|
47
|
+
Tools, system prompt, and model are never read from the transcript; each continuation re-registers the current repo tools, rebuilds the system prompt, and resolves the model fresh (configured → current session model — possibly a different model than the original run; pi handles resumed transcripts under a new model). Only the message history is resumed. This keeps continuation semantics identical to fresh runs — including `provide_results` enforcement and the reminder loop — and means prompt/tool improvements apply retroactively to old runs.
|
|
48
|
+
|
|
49
|
+
### 3. One machinery for cheap follow-ups and big second legs
|
|
50
|
+
|
|
51
|
+
No fast path. A follow-up that the prior context already answers naturally produces a zero-tool-call run ending in `provide_results`; a big second leg just runs longer. The system prompt gains a continuation expectation: runs may be resumed later; on resume, answer from prior context when it suffices, research further when it does not, cite evidence from the available transcript context, and always finish with `provide_results`.
|
|
52
|
+
|
|
53
|
+
### 4. uuid surfaced in two places, by design redundancy
|
|
54
|
+
|
|
55
|
+
The agent reads `run: <uuid>` from the result content; the human sees it in the render. If the agent loses the uuid across a compaction boundary, the user can paste it back. Inelegant, deliberately so — one fallback beats zero.
|
|
56
|
+
|
|
57
|
+
## Edge Cases & Failure Modes
|
|
58
|
+
|
|
59
|
+
- **Unknown/missing `continue_from` uuid**: structured error result telling the agent the run was not found and to start a fresh run (the sessions dir may have been cleaned, or an early-aborted run may never have flushed a session file). Fresh runs surface the session id pi gives us without post-run filesystem verification.
|
|
60
|
+
- **Concurrent continuations of the same run**: not locked; last write wins. Documented, not defended — a single-user tool.
|
|
61
|
+
- **Original run errored or was aborted**: if pi persisted the transcript, it is continuable; the follow-up prompt simply lands after the failure. Often the right move ("try again, but look at X").
|
|
62
|
+
- **Model changed between runs**: allowed; transcript resumes under the newly resolved model.
|
|
63
|
+
- **Aborted continuation**: same abort path as fresh runs; the transcript keeps whatever was persisted, and remains continuable.
|
|
64
|
+
|
|
65
|
+
## Alternatives
|
|
66
|
+
|
|
67
|
+
### Persist into the normal pi session store (pi-sessions integration)
|
|
68
|
+
|
|
69
|
+
- **Status:** Deferred
|
|
70
|
+
- **Open issue:** Landing runs in the regular store would let pi-sessions index them — past research searchable, askable, and linked into lineage. It also pollutes resume pickers with untitled subagent sessions and couples the packages.
|
|
71
|
+
- **Retained discussion:** The cleaner future shape is probably pi-sessions optionally indexing `<cacheDir>/sessions` as an additional source, keeping picker separation while gaining research memory.
|
|
72
|
+
- **Next step:** Revisit once continuations prove their worth in practice.
|
|
73
|
+
|
|
74
|
+
### In-memory session map for the pi process lifetime
|
|
75
|
+
|
|
76
|
+
- **Status:** Rejected
|
|
77
|
+
- **Decision or open issue:** Dies on restart and cannot serve a different main session; disk persistence costs nearly nothing given pi's existing machinery.
|
|
78
|
+
|
|
79
|
+
### Lighter result shape for one-line follow-ups
|
|
80
|
+
|
|
81
|
+
- **Status:** Rejected
|
|
82
|
+
- **Decision or open issue:** Two result shapes for the main agent to handle; `provide_results` on a zero-tool-call run is cheap enough.
|
|
83
|
+
|
|
84
|
+
## Implementation Plan
|
|
85
|
+
|
|
86
|
+
- [ ] Phase 1: Persistent run sessions + `continue_from`
|
|
87
|
+
- Goal: Runs persist to `<cacheDir>/sessions/`; `continue_from` resumes them end to end.
|
|
88
|
+
- Files: `extensions/librarian/run.ts`, `extensions/librarian.ts`, `extensions/librarian/prompt.ts`.
|
|
89
|
+
- Work: swap `SessionManager.inMemory` for `create(cacheDir, sessionsDir)`; capture `session.sessionManager.getSessionId()` into `LibrarianRunDetails.runId` and append `run: <uuid>` to all result content when a run id exists; add `continue_from` param, resolve uuid via `SessionManager.listAll(sessionsDir)`, `SessionManager.open`, structured not-found error; system prompt continuation expectation; tool description guidance.
|
|
90
|
+
- Validation: `npm run check` green and live smoke covers fresh run plus continuation.
|
|
91
|
+
|
|
92
|
+
- [ ] Phase 2: Render the run id
|
|
93
|
+
- Goal: Human-visible uuid in the final rendered state.
|
|
94
|
+
- Files: `extensions/librarian/view.ts`.
|
|
95
|
+
- Work: muted `run <uuid>` line above the footer in the final render (fresh and continued runs alike, including error and abort results).
|
|
96
|
+
- Validation: view unit test; visual check in TUI.
|
|
97
|
+
|
|
98
|
+
- [ ] Phase 3: Live smoke
|
|
99
|
+
- Goal: Continuation proven in a real pi session.
|
|
100
|
+
- Files: `SMOKE.md`.
|
|
101
|
+
- Work: run a deep dive; ask a follow-up via `continue_from` in the same session (expect few/zero tool calls); restart pi and continue the same run from a fresh session using the rendered uuid; record evidence.
|
|
102
|
+
- Validation: SMOKE.md entries with captured output.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# Librarian extra tools: name-based opt-in
|
|
2
|
+
|
|
3
|
+
## Status
|
|
4
|
+
|
|
5
|
+
Draft
|
|
6
|
+
|
|
7
|
+
## Decision Summary
|
|
8
|
+
|
|
9
|
+
Replace extension-path allowlisting as the user-facing control for librarian run composition with `librarian.tools`: a list of tool names, auto-resolved to their providing extensions through the main session's tool registry. Extension loading becomes an internal mechanism (plus a rarely-needed escape hatch), the named tools become the sole activation gate, and loaded bundles contribute tools only — their hooks are stripped. The tradeoff: extension code still loads to make a tool callable (pi has no cross-session tool proxying), so a bundle's other machinery is neutralized rather than never present.
|
|
10
|
+
|
|
11
|
+
## Problem Statement / Background
|
|
12
|
+
|
|
13
|
+
Design 01 (decision 3) made `librarian.extensions` — a list of extension paths — the way to grant librarian runs extra tools, with everything a loaded bundle registers activated wholesale. That model misplaces the abstraction: extensions are what users configure, but tools are what the run actually needs. The concrete case is BYO web search: the user has preferred web search/fetch tools provided by a pi extension and wants "use this web search tool in librarian" to be expressible as the tool, not as the path of the bundle it ships in, and without accepting whatever else that bundle registers.
|
|
14
|
+
|
|
15
|
+
Design 01 rejected name-based composition because `pi.getAllTools()` was believed metadata-only. It still is — `ToolInfo` carries no execute handle — but each `ToolInfo.sourceInfo.path` names the extension path pi loaded the tool from, which is exactly the form `additionalExtensionPaths` accepts. Name → path resolution is therefore a straight lookup against the main session, and the missing piece of design 01's rejected alternative exists.
|
|
16
|
+
|
|
17
|
+
## Goals
|
|
18
|
+
|
|
19
|
+
- "Make this tool available in librarian runs" is configured as the tool's name; extension paths are not the primary mental model.
|
|
20
|
+
- Librarian runs stay locked down by default: the baseline toolset (inherited built-ins + repo tools) is fixed and always on; extra tools are additive and explicit.
|
|
21
|
+
- A loaded bundle's unrequested surface (other tools, hooks, commands) does not leak into runs.
|
|
22
|
+
- Misconfiguration is loud at session start, not silently degrading every run.
|
|
23
|
+
|
|
24
|
+
## Non-Goals
|
|
25
|
+
|
|
26
|
+
- No skill opt-in. Skills stay hard-off in librarian runs until a concrete use case appears.
|
|
27
|
+
- No cross-session tool proxying; this design works within pi's load-code-to-get-tools constraint.
|
|
28
|
+
- No subtraction knob for the baseline: `librarian.disabledTools` is deleted, not replaced.
|
|
29
|
+
- No change to attach mode; the main session already has the user's tools.
|
|
30
|
+
|
|
31
|
+
## Exposed Shape
|
|
32
|
+
|
|
33
|
+
```jsonc
|
|
34
|
+
// pi global settings
|
|
35
|
+
{
|
|
36
|
+
"librarian": {
|
|
37
|
+
"tools": ["search_web", "fetch_web"], // extra tools, by name
|
|
38
|
+
"extensions": ["~/path/to/ext.ts"] // escape hatch: extra search space only
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
- **`librarian.tools`** — names of tools to activate inside librarian runs, in addition to the implied baseline (`read`, `grep`, `find`, `ls`, `bash`, plus the repo tools). This is the only thing that activates an extension tool, and may also explicitly opt in built-ins such as `write` or `edit`.
|
|
44
|
+
- **`librarian.extensions`** — demoted to an escape hatch: extension paths to load into runs when a named tool's extension is not loaded in the main session. Listing an extension here activates nothing by itself.
|
|
45
|
+
- **`librarian.disabledTools`** — removed.
|
|
46
|
+
- Unresolvable names produce a one-time warning at session start; runs proceed with whatever resolves.
|
|
47
|
+
|
|
48
|
+
## Design Decisions
|
|
49
|
+
|
|
50
|
+
### 1. `librarian.tools` is the sole activation gate
|
|
51
|
+
|
|
52
|
+
An extension tool is active in a librarian run iff its name appears in `librarian.tools`, regardless of how its code arrived (auto-resolution or the escape hatch). One rule, no bundle ever activates wholesale. This breaks the current `librarian.extensions` behavior deliberately: users who load a bundle must now also name the tools they want from it. The old model's "load path, get everything" was the mismatch this design exists to remove.
|
|
53
|
+
|
|
54
|
+
### 2. Name resolution rides the main session's registry
|
|
55
|
+
|
|
56
|
+
At `librarian` tool execution, the extension resolves each configured name via `pi.getAllTools()`: the matching `ToolInfo.sourceInfo.path` is the providing extension's path, collected (deduped) into `additionalExtensionPaths` for the nested run's resource loader. After session creation, `setActiveToolsByName` activates the baseline plus exactly the resolved names. Consequence: auto-resolution only sees tools loaded in the main session — which is the stated contract ("opt existing tools from your pi setup into librarian"), with the escape hatch covering the rest.
|
|
57
|
+
|
|
58
|
+
Names that resolve to pi-librarian's own registrations (the `librarian` tool, repo tools) are skipped with a warning to guard against recursive self-loading. Names that resolve to built-ins are allowed; no extension path is needed for them.
|
|
59
|
+
|
|
60
|
+
### 3. Loaded bundles contribute tools only — hooks stripped
|
|
61
|
+
|
|
62
|
+
The nested loader's `extensionsOverride` clears each loaded extension's `handlers` map before the runner sees it, so no bundle event handlers execute inside librarian runs. This is the lockdown design 01 wanted when it rejected full inheritance (UI companions, session machinery misbehaving in headless runs), now enforced structurally instead of by curating the allowlist.
|
|
63
|
+
|
|
64
|
+
**Tradeoff:** an extension whose tool depends on hook-based initialization (e.g. `session_start` state setup) breaks silently. Accepted for now; typical BYO tools are execute-only. An opt-out setting is added only when a real tool needs its hooks — not preemptively.
|
|
65
|
+
|
|
66
|
+
### 4. Validation warns once at session start; runs proceed
|
|
67
|
+
|
|
68
|
+
On `session_start` (after all main-session extensions are registered), configured names are checked: resolvable in the main session, or accounted for by a dry-load of the escape-hatch paths — the dry-load only happens when the escape hatch is configured and names remain unresolved. Misses produce a `ctx.ui.notify` warning naming the entry. Runs themselves do not fail on unresolved names; they proceed with what resolved. A failing run was rejected because a config typo shouldn't take down research entirely, but a silent miss was rejected too — hence loud-at-startup.
|
|
69
|
+
|
|
70
|
+
### 5. Name collisions follow pi's rule
|
|
71
|
+
|
|
72
|
+
If two extensions register the same tool name, pi's runner is first-registration-wins. Resolution inherits the main session's winner; the nested run inherits its own load order. No bespoke arbitration.
|
|
73
|
+
|
|
74
|
+
## Edge Cases & Failure Modes
|
|
75
|
+
|
|
76
|
+
- **Configured name matches nothing** (typo, extension removed): session-start warning naming the entry; runs proceed without it.
|
|
77
|
+
- **Name resolves to a synthetic path** (`<inline:N>` from in-process `extensionFactories`): deliberately not special-cased — the path flows through like any other, the nested loader fails to load it, and the failure surfaces as a generic loader error. In-process factories are rare enough that this is accepted as undefined behavior.
|
|
78
|
+
- **Name is a built-in** (`read`, `write`): activated by name without adding an extension path. Baseline built-ins are already active, but naming them is harmless.
|
|
79
|
+
- **Name is `librarian` or a repo tool**: skipped with warning; never loads pi-librarian into its own runs.
|
|
80
|
+
- **One extension provides several requested tools**: its path loads once; all requested names activate.
|
|
81
|
+
- **Escape-hatch extension fails to load in the nested run**: surfaced from the loader's error list into the run trace; remaining tools unaffected.
|
|
82
|
+
- **Tool exists but is deactivated in the main session**: still resolvable — `getAllTools()` returns configured tools regardless of active state, and naming it in `librarian.tools` expresses intent.
|
|
83
|
+
- **Hook-dependent tool breaks under stripping**: known accepted risk (decision 3); the fix is a future opt-out, not silent hook execution.
|
|
84
|
+
|
|
85
|
+
## Alternatives
|
|
86
|
+
|
|
87
|
+
### Keep extension paths as the loading surface, add a tool filter
|
|
88
|
+
|
|
89
|
+
- **Status:** Rejected
|
|
90
|
+
- **Decision:** Leaves the user managing paths — the exact problem stated. Auto-resolution makes the path bookkeeping mechanical, so the user-facing surface should be the intent (tool names).
|
|
91
|
+
|
|
92
|
+
### Escape-hatch extensions retain activate-all behavior
|
|
93
|
+
|
|
94
|
+
- **Status:** Rejected
|
|
95
|
+
- **Decision:** Two activation semantics for one run. A single gate (`librarian.tools`) is easier to reason about, and backwards compatibility was explicitly waived.
|
|
96
|
+
|
|
97
|
+
### Skill opt-in (`librarian.skills`)
|
|
98
|
+
|
|
99
|
+
- **Status:** Deferred
|
|
100
|
+
- **Open issue:** Mechanically straightforward (`skillsOverride` + name filtering against global skill discovery) but no concrete use case yet.
|
|
101
|
+
- **Next step:** Revisit when a specific skill would improve librarian research (e.g. a web-research skill).
|
|
102
|
+
|
|
103
|
+
### MCP tools via pi-mcp-adapter
|
|
104
|
+
|
|
105
|
+
- **Status:** Open
|
|
106
|
+
- **Open issue:** Not supported by this design, for two independent reasons. The adapter's tool executors depend on state initialized in its `session_start` handler, which hook stripping removes — and even with hooks kept, nested librarian sessions never emit `session_start` at all: `bindExtensions()` (which fires it) is called only by pi's modes (interactive/print/rpc), never by bare `createAgentSession`. MCP tools opted in by name would load, register, and then return "MCP not initialized" on every call.
|
|
107
|
+
- **Retained discussion:** This is the concrete instance of decision 3's hook-dependent-init risk, discovered by inspection rather than in production. Supporting it would take both a hook-keeping escape for the adapter's bundle and the librarian calling `session.bindExtensions({})` to emit `session_start` in nested runs — which also means MCP server processes spawning per run. The librarian would also have to emit `session_shutdown` before `dispose()` (pi's modes do this; `dispose()` alone does not), or every run leaks whatever the lifecycle hooks spawned.
|
|
108
|
+
- **Next step:** Revisit if an MCP-provided tool becomes genuinely wanted inside librarian runs.
|
|
109
|
+
|
|
110
|
+
### Fail runs on unresolved names
|
|
111
|
+
|
|
112
|
+
- **Status:** Rejected
|
|
113
|
+
- **Retained discussion:** Loud but disproportionate — a typo shouldn't block all research. Startup warning chosen; revisit if degraded runs still go unnoticed in practice.
|
|
114
|
+
|
|
115
|
+
### Keep `librarian.disabledTools` as a baseline subtraction knob
|
|
116
|
+
|
|
117
|
+
- **Status:** Rejected
|
|
118
|
+
- **Decision:** With extra tools opted in by name, disabling an opted-in tool is just removing it from the list; the only remaining use was dropping a built-in (e.g. `bash`), which nothing currently needs. Deleted for a single-knob model; trivially reintroducible.
|
|
119
|
+
|
|
120
|
+
## Implementation Plan
|
|
121
|
+
|
|
122
|
+
- [ ] Phase 1: Settings reshape + resolution + run wiring
|
|
123
|
+
- Goal: `librarian.tools` works end to end — named tools resolve, load, and activate in runs; old semantics gone.
|
|
124
|
+
- Files: `extensions/librarian/settings.ts`, new `extensions/librarian/extra-tools.ts` (resolution module), `extensions/librarian/run.ts`, `extensions/librarian.ts`, tests.
|
|
125
|
+
- Work: settings schema gains `tools`, drops `disabledTools`, keeps `extensions` (escape-hatch semantics); pure resolution function `(toolInfos, settings) → { extensionPaths, toolNames }` implementing the self-guard, built-in handling, and dedupe; `runLibrarian` accepts resolved paths + names, wires them into `additionalExtensionPaths`, strips hooks via `extensionsOverride`, and activates baseline + resolved names via `setActiveToolsByName`; entrypoint passes `pi.getAllTools()` output at execute time.
|
|
126
|
+
- Validation: unit tests for resolution (hit, miss, built-in, self, dedupe, escape-hatch pass-through) and run activation; `npm run check`.
|
|
127
|
+
|
|
128
|
+
- [ ] Phase 2: Session-start validation warnings
|
|
129
|
+
- Goal: Misconfiguration is loud once, at startup.
|
|
130
|
+
- Files: `extensions/librarian.ts`, `extensions/librarian/extra-tools.ts`, tests.
|
|
131
|
+
- Work: on `session_start`, collect extra-tool warnings against `pi.getAllTools()`; for names still unresolved with `librarian.extensions` configured, dry-load those paths through the same hook-stripping resource-loader factory used by runs to enumerate tools and surface loader errors; `ctx.ui.notify` per unresolved entry or loader failure with the specific reason.
|
|
132
|
+
- Validation: unit tests for warning selection (including the dry-load-only-when-needed rule); manual check that a typo'd name warns at pi startup; `npm run check`.
|
|
133
|
+
|
|
134
|
+
- [ ] Phase 3: Documentation + live smoke
|
|
135
|
+
- Goal: The new model is documented and proven against the motivating use case.
|
|
136
|
+
- Files: `README.md`, `AGENTS.md`/`CONTEXT.md` cross-references as needed, `SMOKE.md`.
|
|
137
|
+
- Work: document `librarian.tools` and the escape-hatch semantics of `librarian.extensions`; remove `disabledTools` references; run a live librarian query that uses a BYO web search tool opted in by name, and one with a deliberate typo showing the startup warning; record both in `SMOKE.md`.
|
|
138
|
+
- Validation: smoke evidence in `SMOKE.md`; `npm run check`.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import type { ToolInfo } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { createLibrarianResourceLoader } from "./resource-loader.ts";
|
|
3
|
+
import type { LibrarianSettings } from "./settings.ts";
|
|
4
|
+
import { LIBRARIAN_RUN_TOOL_NAMES } from "./tools/names.ts";
|
|
5
|
+
|
|
6
|
+
export const LIBRARIAN_BASELINE_TOOL_NAMES = ["read", "grep", "find", "ls", "bash"] as const;
|
|
7
|
+
|
|
8
|
+
const LIBRARIAN_SELF_TOOL_NAMES = new Set<string>(["librarian", ...LIBRARIAN_RUN_TOOL_NAMES]);
|
|
9
|
+
|
|
10
|
+
export type ExtraToolWarningReason = "self" | "unresolved" | "extensionLoad";
|
|
11
|
+
|
|
12
|
+
export interface ExtraToolWarning {
|
|
13
|
+
toolName: string;
|
|
14
|
+
reason: ExtraToolWarningReason;
|
|
15
|
+
message: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface ExtraToolsResolution {
|
|
19
|
+
extensionPaths: string[];
|
|
20
|
+
toolNames: string[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
interface ExtraToolsValidation {
|
|
24
|
+
resolution: ExtraToolsResolution;
|
|
25
|
+
warnings: ExtraToolWarning[];
|
|
26
|
+
unresolvedToolNames: string[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function resolveExtraToolsForValidation(
|
|
30
|
+
toolInfos: ToolInfo[],
|
|
31
|
+
settings: LibrarianSettings,
|
|
32
|
+
): ExtraToolsValidation {
|
|
33
|
+
const toolInfosByName = new Map(toolInfos.map((toolInfo) => [toolInfo.name, toolInfo]));
|
|
34
|
+
const extensionPaths = new Set(settings.extensions);
|
|
35
|
+
const toolNames = new Set<string>();
|
|
36
|
+
const warnings: ExtraToolWarning[] = [];
|
|
37
|
+
const unresolvedToolNames: string[] = [];
|
|
38
|
+
|
|
39
|
+
for (const toolName of settings.tools) {
|
|
40
|
+
if (LIBRARIAN_SELF_TOOL_NAMES.has(toolName)) {
|
|
41
|
+
warnings.push({
|
|
42
|
+
toolName,
|
|
43
|
+
reason: "self",
|
|
44
|
+
message: `librarian.tools includes ${toolName}, but librarian's own tools cannot be opted into librarian runs. Remove it from librarian.tools.`,
|
|
45
|
+
});
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const toolInfo = toolInfosByName.get(toolName);
|
|
50
|
+
if (!toolInfo) {
|
|
51
|
+
toolNames.add(toolName);
|
|
52
|
+
unresolvedToolNames.push(toolName);
|
|
53
|
+
warnings.push({
|
|
54
|
+
toolName,
|
|
55
|
+
reason: "unresolved",
|
|
56
|
+
message: `librarian.tools includes ${toolName}, but no loaded pi tool has that name.`,
|
|
57
|
+
});
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
toolNames.add(toolName);
|
|
62
|
+
if (toolInfo.sourceInfo.source !== "builtin") {
|
|
63
|
+
extensionPaths.add(toolInfo.sourceInfo.path);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
resolution: {
|
|
69
|
+
extensionPaths: [...extensionPaths],
|
|
70
|
+
toolNames: [...toolNames],
|
|
71
|
+
},
|
|
72
|
+
warnings,
|
|
73
|
+
unresolvedToolNames,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function resolveExtraTools(
|
|
78
|
+
toolInfos: ToolInfo[],
|
|
79
|
+
settings: LibrarianSettings,
|
|
80
|
+
): ExtraToolsResolution {
|
|
81
|
+
return resolveExtraToolsForValidation(toolInfos, settings).resolution;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export async function collectExtraToolWarnings(
|
|
85
|
+
toolInfos: ToolInfo[],
|
|
86
|
+
settings: LibrarianSettings,
|
|
87
|
+
): Promise<ExtraToolWarning[]> {
|
|
88
|
+
const validation = resolveExtraToolsForValidation(toolInfos, settings);
|
|
89
|
+
const nonUnresolvedWarnings = validation.warnings.filter(
|
|
90
|
+
(warning) => warning.reason !== "unresolved",
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
if (validation.unresolvedToolNames.length === 0 || settings.extensions.length === 0) {
|
|
94
|
+
return validation.warnings;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const resourceLoader = createLibrarianResourceLoader({
|
|
98
|
+
cacheDir: settings.cacheDir,
|
|
99
|
+
extensionPaths: settings.extensions,
|
|
100
|
+
systemPromptOverride: () => undefined,
|
|
101
|
+
});
|
|
102
|
+
await resourceLoader.reload();
|
|
103
|
+
|
|
104
|
+
const extensionsResult = resourceLoader.getExtensions();
|
|
105
|
+
const escapeHatchToolNames = new Set<string>();
|
|
106
|
+
for (const extension of extensionsResult.extensions) {
|
|
107
|
+
for (const toolName of extension.tools.keys()) {
|
|
108
|
+
escapeHatchToolNames.add(toolName);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const loaderWarnings = extensionsResult.errors.map((error) => ({
|
|
113
|
+
toolName: error.path,
|
|
114
|
+
reason: "extensionLoad" as const,
|
|
115
|
+
message: `librarian.extensions failed to load ${error.path}: ${error.error}`,
|
|
116
|
+
}));
|
|
117
|
+
|
|
118
|
+
const unresolvedWarnings = validation.unresolvedToolNames
|
|
119
|
+
.filter((toolName) => !escapeHatchToolNames.has(toolName))
|
|
120
|
+
.map((toolName) => ({
|
|
121
|
+
toolName,
|
|
122
|
+
reason: "unresolved" as const,
|
|
123
|
+
message:
|
|
124
|
+
loaderWarnings.length > 0
|
|
125
|
+
? `librarian.tools includes ${toolName}, but it was not found in the main session or successfully loaded escape-hatch tools. Fix librarian.extensions load errors or the tool name.`
|
|
126
|
+
: `librarian.tools includes ${toolName}, but no loaded pi tool or escape-hatch extension tool has that name.`,
|
|
127
|
+
}));
|
|
128
|
+
|
|
129
|
+
return [...nonUnresolvedWarnings, ...loaderWarnings, ...unresolvedWarnings];
|
|
130
|
+
}
|
|
@@ -21,7 +21,7 @@ Clones live under ${cacheDir}/repos/<owner>/<repo>. They are read-only research
|
|
|
21
21
|
|
|
22
22
|
## Citations
|
|
23
23
|
|
|
24
|
-
- Only cite files you actually read
|
|
24
|
+
- Only cite files you actually read within context.
|
|
25
25
|
- Cite as repo + file path + line range when you have one.
|
|
26
26
|
- If evidence is partial, say what is confirmed and what remains uncertain.
|
|
27
27
|
- If access fails (404/403, private repo), report that constraint plainly.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DefaultResourceLoader,
|
|
3
|
+
getAgentDir,
|
|
4
|
+
type LoadExtensionsResult,
|
|
5
|
+
} from "@earendil-works/pi-coding-agent";
|
|
6
|
+
|
|
7
|
+
export function stripExtensionHooks(result: LoadExtensionsResult): LoadExtensionsResult {
|
|
8
|
+
return {
|
|
9
|
+
...result,
|
|
10
|
+
extensions: result.extensions.map((extension) => ({
|
|
11
|
+
...extension,
|
|
12
|
+
handlers: new Map(),
|
|
13
|
+
})),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface LibrarianResourceLoaderOptions {
|
|
18
|
+
cacheDir: string;
|
|
19
|
+
extensionPaths: string[];
|
|
20
|
+
systemPromptOverride: (base: string | undefined) => string | undefined;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function createLibrarianResourceLoader(
|
|
24
|
+
options: LibrarianResourceLoaderOptions,
|
|
25
|
+
): DefaultResourceLoader {
|
|
26
|
+
return new DefaultResourceLoader({
|
|
27
|
+
cwd: options.cacheDir,
|
|
28
|
+
agentDir: getAgentDir(),
|
|
29
|
+
noExtensions: true,
|
|
30
|
+
...(options.extensionPaths.length > 0
|
|
31
|
+
? { additionalExtensionPaths: options.extensionPaths }
|
|
32
|
+
: {}),
|
|
33
|
+
noSkills: true,
|
|
34
|
+
noPromptTemplates: true,
|
|
35
|
+
noThemes: true,
|
|
36
|
+
extensionsOverride: stripExtensionHooks,
|
|
37
|
+
skillsOverride: () => ({ skills: [], diagnostics: [] }),
|
|
38
|
+
systemPromptOverride: options.systemPromptOverride,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
@@ -1,22 +1,21 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
2
3
|
import type { AgentToolResult, ThinkingLevel } from "@earendil-works/pi-agent-core";
|
|
3
4
|
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
4
5
|
import type { AgentSession, AgentSessionEvent } from "@earendil-works/pi-coding-agent";
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
DefaultResourceLoader,
|
|
8
|
-
getAgentDir,
|
|
9
|
-
SessionManager,
|
|
10
|
-
} from "@earendil-works/pi-coding-agent";
|
|
6
|
+
import { createAgentSession, SessionManager } from "@earendil-works/pi-coding-agent";
|
|
7
|
+
import { type ExtraToolsResolution, LIBRARIAN_BASELINE_TOOL_NAMES } from "./extra-tools.ts";
|
|
11
8
|
import type { GitHubClientProvider } from "./github.ts";
|
|
12
9
|
import {
|
|
13
10
|
buildLibrarianSystemPrompt,
|
|
14
11
|
buildLibrarianUserPrompt,
|
|
15
12
|
buildProvideResultsReminder,
|
|
16
13
|
} from "./prompt.ts";
|
|
14
|
+
import { createLibrarianResourceLoader } from "./resource-loader.ts";
|
|
17
15
|
import { renderFindingsMarkdown } from "./results.ts";
|
|
18
16
|
import type { LibrarianSettings } from "./settings.ts";
|
|
19
17
|
import { createCheckoutRepoTool } from "./tools/checkout-repo.ts";
|
|
18
|
+
import { LIBRARIAN_RUN_TOOL_NAMES } from "./tools/names.ts";
|
|
20
19
|
import type { Findings } from "./tools/provide-results.ts";
|
|
21
20
|
import { createProvideResultsTool } from "./tools/provide-results.ts";
|
|
22
21
|
import { createReadGitHubFileTool } from "./tools/read-github-file.ts";
|
|
@@ -26,9 +25,12 @@ import { createSearchReposTool } from "./tools/search-repos.ts";
|
|
|
26
25
|
import { asRepoToolDetails, summarizeToolResult } from "./trace.ts";
|
|
27
26
|
|
|
28
27
|
const MAX_PROVIDE_RESULTS_REMINDERS = 3;
|
|
29
|
-
const
|
|
28
|
+
const DEFAULT_EXCLUDED_TOOLS = ["write", "edit"];
|
|
29
|
+
|
|
30
|
+
class LibrarianRunError extends Error {}
|
|
30
31
|
|
|
31
32
|
export type LibrarianRunStatus = "running" | "done" | "error" | "aborted";
|
|
33
|
+
type FailedLibrarianRunStatus = "error" | "aborted";
|
|
32
34
|
|
|
33
35
|
export interface TraceCall {
|
|
34
36
|
id: string;
|
|
@@ -49,6 +51,7 @@ export interface LibrarianRunDetails {
|
|
|
49
51
|
findings?: Findings;
|
|
50
52
|
checkouts: Record<string, string>;
|
|
51
53
|
error?: string;
|
|
54
|
+
runId?: string;
|
|
52
55
|
startedAt: number;
|
|
53
56
|
endedAt?: number;
|
|
54
57
|
}
|
|
@@ -57,14 +60,30 @@ export interface LibrarianRunOptions {
|
|
|
57
60
|
query: string;
|
|
58
61
|
repos: string[];
|
|
59
62
|
owners: string[];
|
|
63
|
+
continueFrom: string | undefined;
|
|
60
64
|
model: Model<Api>;
|
|
61
65
|
thinkingLevel: ThinkingLevel;
|
|
62
66
|
settings: LibrarianSettings;
|
|
67
|
+
extraTools: ExtraToolsResolution;
|
|
63
68
|
githubClient: GitHubClientProvider;
|
|
64
69
|
signal: AbortSignal | undefined;
|
|
65
70
|
onUpdate: ((details: LibrarianRunDetails) => void) | undefined;
|
|
66
71
|
}
|
|
67
72
|
|
|
73
|
+
async function openContinuedSession(
|
|
74
|
+
runId: string,
|
|
75
|
+
sessionsDir: string,
|
|
76
|
+
cacheDir: string,
|
|
77
|
+
fail: (status: FailedLibrarianRunStatus, content: string) => never,
|
|
78
|
+
): Promise<SessionManager> {
|
|
79
|
+
const sessions = await SessionManager.listAll(sessionsDir);
|
|
80
|
+
const sessionFile = sessions.find((candidate) => candidate.id === runId)?.path;
|
|
81
|
+
if (!sessionFile) {
|
|
82
|
+
fail("error", `Librarian run not found: ${runId}`);
|
|
83
|
+
}
|
|
84
|
+
return SessionManager.open(sessionFile, sessionsDir, cacheDir);
|
|
85
|
+
}
|
|
86
|
+
|
|
68
87
|
export async function runLibrarian(
|
|
69
88
|
options: LibrarianRunOptions,
|
|
70
89
|
): Promise<AgentToolResult<LibrarianRunDetails>> {
|
|
@@ -88,26 +107,37 @@ export async function runLibrarian(
|
|
|
88
107
|
options.onUpdate?.({ ...details, trace: [...details.trace] });
|
|
89
108
|
};
|
|
90
109
|
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
content: string,
|
|
94
|
-
isError: boolean,
|
|
95
|
-
): AgentToolResult<LibrarianRunDetails> => {
|
|
110
|
+
const finalizeDetails = (status: LibrarianRunStatus, content: string): string => {
|
|
111
|
+
const resultContent = details.runId ? `${content}\n\nrun: ${details.runId}` : content;
|
|
96
112
|
details.status = status;
|
|
97
113
|
details.endedAt = Date.now();
|
|
98
|
-
if (
|
|
114
|
+
if (status !== "done") {
|
|
99
115
|
details.error = content.split("\n")[0] ?? content;
|
|
100
116
|
}
|
|
101
117
|
emit(true);
|
|
102
|
-
return
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
118
|
+
return resultContent;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const finish = (
|
|
122
|
+
status: LibrarianRunStatus,
|
|
123
|
+
content: string,
|
|
124
|
+
): AgentToolResult<LibrarianRunDetails> => ({
|
|
125
|
+
content: [{ type: "text", text: finalizeDetails(status, content) }],
|
|
126
|
+
details: { ...details, trace: [...details.trace] },
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
const fail = (status: FailedLibrarianRunStatus, content: string): never => {
|
|
130
|
+
throw new LibrarianRunError(finalizeDetails(status, content));
|
|
107
131
|
};
|
|
108
132
|
|
|
109
133
|
await fs.mkdir(options.settings.cacheDir, { recursive: true });
|
|
110
134
|
|
|
135
|
+
const sessionsDir = path.join(options.settings.cacheDir, "sessions");
|
|
136
|
+
const sessionManager = options.continueFrom
|
|
137
|
+
? await openContinuedSession(options.continueFrom, sessionsDir, options.settings.cacheDir, fail)
|
|
138
|
+
: SessionManager.create(options.settings.cacheDir, sessionsDir);
|
|
139
|
+
details.runId = sessionManager.getSessionId();
|
|
140
|
+
|
|
111
141
|
let findings: Findings | undefined;
|
|
112
142
|
const repoTools = [
|
|
113
143
|
createSearchReposTool(options.githubClient),
|
|
@@ -120,21 +150,27 @@ export async function runLibrarian(
|
|
|
120
150
|
}),
|
|
121
151
|
];
|
|
122
152
|
|
|
123
|
-
const resourceLoader =
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
noExtensions: true,
|
|
127
|
-
...(options.settings.extensions.length > 0
|
|
128
|
-
? { additionalExtensionPaths: options.settings.extensions }
|
|
129
|
-
: {}),
|
|
130
|
-
noSkills: true,
|
|
131
|
-
noPromptTemplates: true,
|
|
132
|
-
noThemes: true,
|
|
153
|
+
const resourceLoader = createLibrarianResourceLoader({
|
|
154
|
+
cacheDir: options.settings.cacheDir,
|
|
155
|
+
extensionPaths: options.extraTools.extensionPaths,
|
|
133
156
|
systemPromptOverride: () => buildLibrarianSystemPrompt(options.settings.cacheDir),
|
|
134
|
-
skillsOverride: () => ({ skills: [], diagnostics: [] }),
|
|
135
157
|
});
|
|
136
158
|
await resourceLoader.reload();
|
|
137
159
|
|
|
160
|
+
const extensionLoadErrors = resourceLoader.getExtensions().errors;
|
|
161
|
+
for (const [index, error] of extensionLoadErrors.entries()) {
|
|
162
|
+
const now = Date.now();
|
|
163
|
+
details.trace.push({
|
|
164
|
+
id: `extension-load-${index}`,
|
|
165
|
+
name: "load_extension",
|
|
166
|
+
args: { path: error.path },
|
|
167
|
+
startedAt: now,
|
|
168
|
+
endedAt: now,
|
|
169
|
+
isError: true,
|
|
170
|
+
resultSummary: error.error,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
|
|
138
174
|
emit(true);
|
|
139
175
|
|
|
140
176
|
let session: AgentSession | undefined;
|
|
@@ -145,20 +181,21 @@ export async function runLibrarian(
|
|
|
145
181
|
const created = await createAgentSession({
|
|
146
182
|
cwd: options.settings.cacheDir,
|
|
147
183
|
resourceLoader,
|
|
148
|
-
sessionManager
|
|
184
|
+
sessionManager,
|
|
149
185
|
model: options.model,
|
|
150
186
|
thinkingLevel: options.thinkingLevel,
|
|
151
187
|
customTools: repoTools,
|
|
152
|
-
excludeTools:
|
|
188
|
+
excludeTools: DEFAULT_EXCLUDED_TOOLS.filter(
|
|
189
|
+
(toolName) => !options.extraTools.toolNames.includes(toolName),
|
|
190
|
+
),
|
|
153
191
|
});
|
|
154
192
|
session = created.session;
|
|
155
193
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
.
|
|
160
|
-
|
|
161
|
-
session.setActiveToolsByName(allToolNames);
|
|
194
|
+
session.setActiveToolsByName([
|
|
195
|
+
...LIBRARIAN_BASELINE_TOOL_NAMES,
|
|
196
|
+
...LIBRARIAN_RUN_TOOL_NAMES,
|
|
197
|
+
...options.extraTools.toolNames,
|
|
198
|
+
]);
|
|
162
199
|
|
|
163
200
|
unsubscribe = session.subscribe((event: AgentSessionEvent) => {
|
|
164
201
|
switch (event.type) {
|
|
@@ -198,7 +235,7 @@ export async function runLibrarian(
|
|
|
198
235
|
});
|
|
199
236
|
|
|
200
237
|
if (options.signal?.aborted) {
|
|
201
|
-
|
|
238
|
+
fail("aborted", "Librarian run aborted before it started.");
|
|
202
239
|
}
|
|
203
240
|
|
|
204
241
|
const activeSession = session;
|
|
@@ -214,30 +251,34 @@ export async function runLibrarian(
|
|
|
214
251
|
let reminders = 0;
|
|
215
252
|
while (!findings && !options.signal?.aborted && reminders < MAX_PROVIDE_RESULTS_REMINDERS) {
|
|
216
253
|
reminders += 1;
|
|
217
|
-
await session.prompt(buildProvideResultsReminder(), {
|
|
254
|
+
await session.prompt(buildProvideResultsReminder(), {
|
|
255
|
+
expandPromptTemplates: false,
|
|
256
|
+
});
|
|
218
257
|
}
|
|
219
258
|
|
|
220
259
|
if (options.signal?.aborted) {
|
|
221
|
-
|
|
260
|
+
fail("aborted", "Librarian run aborted.");
|
|
222
261
|
}
|
|
223
262
|
|
|
224
263
|
if (findings) {
|
|
225
264
|
details.findings = findings;
|
|
226
|
-
return finish("done", renderFindingsMarkdown(findings, details.checkouts)
|
|
265
|
+
return finish("done", renderFindingsMarkdown(findings, details.checkouts));
|
|
227
266
|
}
|
|
228
267
|
|
|
229
268
|
const lastText = session.getLastAssistantText() ?? "";
|
|
230
|
-
|
|
269
|
+
fail(
|
|
231
270
|
"error",
|
|
232
271
|
`Librarian did not report structured findings after ${MAX_PROVIDE_RESULTS_REMINDERS} reminders.${lastText ? `\n\nRaw final message:\n${lastText}` : ""}`,
|
|
233
|
-
true,
|
|
234
272
|
);
|
|
235
273
|
} catch (error) {
|
|
274
|
+
if (error instanceof LibrarianRunError) {
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
236
277
|
if (options.signal?.aborted) {
|
|
237
|
-
|
|
278
|
+
fail("aborted", "Librarian run aborted.");
|
|
238
279
|
}
|
|
239
280
|
const message = error instanceof Error ? error.message : String(error);
|
|
240
|
-
|
|
281
|
+
fail("error", `Librarian run failed: ${message}`);
|
|
241
282
|
} finally {
|
|
242
283
|
if (onAbort) {
|
|
243
284
|
options.signal?.removeEventListener("abort", onAbort);
|
|
@@ -245,4 +286,6 @@ export async function runLibrarian(
|
|
|
245
286
|
unsubscribe?.();
|
|
246
287
|
session?.dispose();
|
|
247
288
|
}
|
|
289
|
+
|
|
290
|
+
throw new Error("Unreachable librarian run state.");
|
|
248
291
|
}
|
|
@@ -18,7 +18,7 @@ const LIBRARIAN_FILE_SETTINGS_SCHEMA = Type.Object({
|
|
|
18
18
|
model: Type.Optional(Type.String()),
|
|
19
19
|
thinkingLevel: Type.Optional(THINKING_LEVEL_SCHEMA),
|
|
20
20
|
extensions: Type.Optional(Type.Array(Type.String())),
|
|
21
|
-
|
|
21
|
+
tools: Type.Optional(Type.Array(Type.String())),
|
|
22
22
|
cacheDir: Type.Optional(Type.String()),
|
|
23
23
|
});
|
|
24
24
|
|
|
@@ -43,7 +43,7 @@ export interface LibrarianSettings {
|
|
|
43
43
|
model: ModelReference | undefined;
|
|
44
44
|
thinkingLevel: ThinkingLevel | undefined;
|
|
45
45
|
extensions: string[];
|
|
46
|
-
|
|
46
|
+
tools: string[];
|
|
47
47
|
cacheDir: string;
|
|
48
48
|
}
|
|
49
49
|
|
|
@@ -102,12 +102,20 @@ function normalizeExtensionPaths(values: string[] | undefined): string[] {
|
|
|
102
102
|
.map(expandHome);
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
function normalizeToolNames(values: string[] | undefined): string[] {
|
|
106
|
+
if (!values) {
|
|
107
|
+
return [];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return [...new Set(values.map((value) => value.trim()).filter((value) => value.length > 0))];
|
|
111
|
+
}
|
|
112
|
+
|
|
105
113
|
export function resolveLibrarianSettings(fileSettings: LibrarianFileSettings): LibrarianSettings {
|
|
106
114
|
return {
|
|
107
115
|
model: parseModelReference(fileSettings.model),
|
|
108
116
|
thinkingLevel: fileSettings.thinkingLevel,
|
|
109
117
|
extensions: normalizeExtensionPaths(fileSettings.extensions),
|
|
110
|
-
|
|
118
|
+
tools: normalizeToolNames(fileSettings.tools),
|
|
111
119
|
cacheDir: normalizeCacheDir(fileSettings.cacheDir),
|
|
112
120
|
};
|
|
113
121
|
}
|
|
@@ -64,24 +64,9 @@ export function createCheckoutRepoTool(cacheDir: string) {
|
|
|
64
64
|
: error instanceof Error
|
|
65
65
|
? error.message
|
|
66
66
|
: String(error);
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
type: "text",
|
|
71
|
-
text: `${message}\nIf the repo name is uncertain, resolve it with search_repos first.`,
|
|
72
|
-
},
|
|
73
|
-
],
|
|
74
|
-
details: {
|
|
75
|
-
kind: "checkout_repo",
|
|
76
|
-
repo: params.repo,
|
|
77
|
-
path: "",
|
|
78
|
-
headSha: "",
|
|
79
|
-
ref: params.ref ?? "",
|
|
80
|
-
fileCount: 0,
|
|
81
|
-
reusedClone: false,
|
|
82
|
-
},
|
|
83
|
-
isError: true,
|
|
84
|
-
};
|
|
67
|
+
throw new Error(
|
|
68
|
+
`${message}\nIf the repo name is uncertain, resolve it with search_repos first.`,
|
|
69
|
+
);
|
|
85
70
|
}
|
|
86
71
|
},
|
|
87
72
|
});
|
|
@@ -3,10 +3,16 @@ import { type Static, Type } from "typebox";
|
|
|
3
3
|
import { LIBRARIAN_TOOL_NAMES } from "./names.ts";
|
|
4
4
|
|
|
5
5
|
export const LocationSchema = Type.Object({
|
|
6
|
-
repo: Type.String({
|
|
7
|
-
|
|
6
|
+
repo: Type.String({
|
|
7
|
+
description: "Repository as owner/repo or a repository URL.",
|
|
8
|
+
}),
|
|
9
|
+
file: Type.String({
|
|
10
|
+
description: "Validated file path within the repository.",
|
|
11
|
+
}),
|
|
8
12
|
lines: Type.Optional(
|
|
9
|
-
Type.String({
|
|
13
|
+
Type.String({
|
|
14
|
+
description: 'Line range like "80-140" or a single line like "42".',
|
|
15
|
+
}),
|
|
10
16
|
),
|
|
11
17
|
note: Type.String({ description: "Relevance of the file." }),
|
|
12
18
|
});
|
|
@@ -16,7 +22,8 @@ export const FindingsSchema = Type.Object({
|
|
|
16
22
|
description: "1-3 sentence direct answer to the query. No preamble.",
|
|
17
23
|
}),
|
|
18
24
|
locations: Type.Array(LocationSchema, {
|
|
19
|
-
description:
|
|
25
|
+
description:
|
|
26
|
+
"Evidence source-file locations. Include only repository files you validated by reading.",
|
|
20
27
|
}),
|
|
21
28
|
description: Type.Optional(
|
|
22
29
|
Type.String({
|
|
@@ -50,9 +57,15 @@ export function createProvideResultsTool(onFindings: (findings: Findings) => voi
|
|
|
50
57
|
onFindings(params);
|
|
51
58
|
return {
|
|
52
59
|
content: [
|
|
53
|
-
{
|
|
60
|
+
{
|
|
61
|
+
type: "text",
|
|
62
|
+
text: "Findings recorded. You are done; do not call more tools.",
|
|
63
|
+
},
|
|
54
64
|
],
|
|
55
|
-
details: {
|
|
65
|
+
details: {
|
|
66
|
+
kind: "provide_results",
|
|
67
|
+
locationCount: params.locations.length,
|
|
68
|
+
},
|
|
56
69
|
terminate: true,
|
|
57
70
|
};
|
|
58
71
|
},
|
|
@@ -22,8 +22,10 @@ const ReadGitHubFileParams = Type.Object({
|
|
|
22
22
|
Type.String({ description: "Branch, tag, or commit sha. Default: the default branch." }),
|
|
23
23
|
),
|
|
24
24
|
range: Type.Optional(
|
|
25
|
-
Type.
|
|
25
|
+
Type.Array(Type.Number({ minimum: 1 }), {
|
|
26
26
|
description: "[startLine, endLine] (1-based, inclusive) for large files.",
|
|
27
|
+
minItems: 2,
|
|
28
|
+
maxItems: 2,
|
|
27
29
|
}),
|
|
28
30
|
),
|
|
29
31
|
});
|
|
@@ -42,6 +44,11 @@ export function createReadGitHubFileTool(githubClient: GitHubClientProvider) {
|
|
|
42
44
|
parameters: ReadGitHubFileParams,
|
|
43
45
|
|
|
44
46
|
async execute(_toolCallId, params) {
|
|
47
|
+
const range = params.range ? (params.range as [number, number]) : undefined;
|
|
48
|
+
if (range && range[1] < range[0]) {
|
|
49
|
+
throw new Error("range end must be greater than or equal to range start.");
|
|
50
|
+
}
|
|
51
|
+
|
|
45
52
|
try {
|
|
46
53
|
const github = await githubClient();
|
|
47
54
|
const contents = await github.readContents({
|
|
@@ -72,9 +79,9 @@ export function createReadGitHubFileTool(githubClient: GitHubClientProvider) {
|
|
|
72
79
|
let end = allLines.length;
|
|
73
80
|
let clipNote = "";
|
|
74
81
|
|
|
75
|
-
if (
|
|
76
|
-
start = Math.min(
|
|
77
|
-
end = Math.min(
|
|
82
|
+
if (range) {
|
|
83
|
+
start = Math.min(range[0], allLines.length);
|
|
84
|
+
end = Math.min(range[1], allLines.length);
|
|
78
85
|
} else if (allLines.length > MAX_LINES_WITHOUT_RANGE) {
|
|
79
86
|
end = MAX_LINES_WITHOUT_RANGE;
|
|
80
87
|
clipNote = `\n... clipped at line ${MAX_LINES_WITHOUT_RANGE} of ${allLines.length}; pass range to read further.`;
|
|
@@ -104,18 +111,7 @@ export function createReadGitHubFileTool(githubClient: GitHubClientProvider) {
|
|
|
104
111
|
: error instanceof Error
|
|
105
112
|
? error.message
|
|
106
113
|
: String(error);
|
|
107
|
-
|
|
108
|
-
content: [{ type: "text", text: message }],
|
|
109
|
-
details: {
|
|
110
|
-
kind: "read_github_file",
|
|
111
|
-
owner: params.owner,
|
|
112
|
-
repo: params.repo,
|
|
113
|
-
path: params.path,
|
|
114
|
-
lineCount: 0,
|
|
115
|
-
isDirectory: false,
|
|
116
|
-
},
|
|
117
|
-
isError: true,
|
|
118
|
-
};
|
|
114
|
+
throw new Error(message);
|
|
119
115
|
}
|
|
120
116
|
},
|
|
121
117
|
});
|
|
@@ -85,16 +85,7 @@ export function createSearchReposTool(githubClient: GitHubClientProvider) {
|
|
|
85
85
|
: error instanceof Error
|
|
86
86
|
? error.message
|
|
87
87
|
: String(error);
|
|
88
|
-
|
|
89
|
-
content: [{ type: "text", text: message }],
|
|
90
|
-
details: {
|
|
91
|
-
kind: "search_repos",
|
|
92
|
-
query: params.query,
|
|
93
|
-
resultCount: 0,
|
|
94
|
-
totalCount: 0,
|
|
95
|
-
},
|
|
96
|
-
isError: true,
|
|
97
|
-
};
|
|
88
|
+
throw new Error(message);
|
|
98
89
|
}
|
|
99
90
|
},
|
|
100
91
|
});
|
|
@@ -189,6 +189,17 @@ function renderFooter(details: LibrarianRunDetails, theme: Theme): string {
|
|
|
189
189
|
);
|
|
190
190
|
}
|
|
191
191
|
|
|
192
|
+
function isLibrarianRunDetails(value: unknown): value is LibrarianRunDetails {
|
|
193
|
+
return (
|
|
194
|
+
value !== null &&
|
|
195
|
+
typeof value === "object" &&
|
|
196
|
+
typeof (value as { status?: unknown }).status === "string" &&
|
|
197
|
+
typeof (value as { query?: unknown }).query === "string" &&
|
|
198
|
+
typeof (value as { modelLabel?: unknown }).modelLabel === "string" &&
|
|
199
|
+
Array.isArray((value as { trace?: unknown }).trace)
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
|
|
192
203
|
function renderTrace(
|
|
193
204
|
details: LibrarianRunDetails,
|
|
194
205
|
expanded: boolean,
|
|
@@ -245,7 +256,7 @@ export function renderLibrarianResult(
|
|
|
245
256
|
const container = new Container();
|
|
246
257
|
const details = result.details;
|
|
247
258
|
|
|
248
|
-
if (!details) {
|
|
259
|
+
if (!isLibrarianRunDetails(details)) {
|
|
249
260
|
const firstText = result.content.find((part) => part.type === "text");
|
|
250
261
|
container.addChild(
|
|
251
262
|
new Text(firstText && "text" in firstText ? firstText.text : "(no output)", 0, 0),
|
|
@@ -275,6 +286,9 @@ export function renderLibrarianResult(
|
|
|
275
286
|
}
|
|
276
287
|
|
|
277
288
|
container.addChild(new Spacer(1));
|
|
289
|
+
if (!running && details.runId) {
|
|
290
|
+
container.addChild(new Text(theme.fg("muted", `run ${details.runId}`), 0, 0));
|
|
291
|
+
}
|
|
278
292
|
container.addChild(new Text(renderFooter(details, theme), 0, 0));
|
|
279
293
|
return container;
|
|
280
294
|
}
|
package/extensions/librarian.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { ExtensionAPI, SessionStartEvent } from "@earendil-works/pi-coding-
|
|
|
2
2
|
import { Text } from "@earendil-works/pi-tui";
|
|
3
3
|
import { Type } from "typebox";
|
|
4
4
|
import { applyAttachState, readAttachState, setAttachState } from "./librarian/attach.ts";
|
|
5
|
+
import { collectExtraToolWarnings, resolveExtraTools } from "./librarian/extra-tools.ts";
|
|
5
6
|
import { createGitHubClientProvider } from "./librarian/github.ts";
|
|
6
7
|
import { resolveLibrarianModel } from "./librarian/model.ts";
|
|
7
8
|
import { type LibrarianRunDetails, runLibrarian, type TraceCall } from "./librarian/run.ts";
|
|
@@ -36,6 +37,12 @@ const LibrarianParams = Type.Object({
|
|
|
36
37
|
maxItems: 20,
|
|
37
38
|
}),
|
|
38
39
|
),
|
|
40
|
+
continue_from: Type.Optional(
|
|
41
|
+
Type.String({
|
|
42
|
+
description:
|
|
43
|
+
"Run id from an earlier librarian result. Pass this for follow-up questions for that prior run.",
|
|
44
|
+
}),
|
|
45
|
+
),
|
|
39
46
|
});
|
|
40
47
|
|
|
41
48
|
export default function librarianExtension(pi: ExtensionAPI): void {
|
|
@@ -84,41 +91,33 @@ export default function librarianExtension(pi: ExtensionAPI): void {
|
|
|
84
91
|
const thinkingLevel = settings.thinkingLevel ?? pi.getThinkingLevel();
|
|
85
92
|
const resolution = resolveLibrarianModel(ctx, settings.model, thinkingLevel);
|
|
86
93
|
if (!resolution) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
type: "text",
|
|
91
|
-
text: "No model available for the librarian. Configure librarian.model or select a session model.",
|
|
92
|
-
},
|
|
93
|
-
],
|
|
94
|
-
details: {
|
|
95
|
-
status: "error",
|
|
96
|
-
query: params.query,
|
|
97
|
-
modelLabel: "(none)",
|
|
98
|
-
thinkingLevel,
|
|
99
|
-
trace: [],
|
|
100
|
-
checkouts: {},
|
|
101
|
-
startedAt: Date.now(),
|
|
102
|
-
endedAt: Date.now(),
|
|
103
|
-
error: "No model available.",
|
|
104
|
-
} satisfies LibrarianRunDetails,
|
|
105
|
-
isError: true,
|
|
106
|
-
};
|
|
94
|
+
throw new Error(
|
|
95
|
+
"No model available for the librarian. Configure librarian.model or select a session model.",
|
|
96
|
+
);
|
|
107
97
|
}
|
|
108
98
|
|
|
99
|
+
const extraTools = resolveExtraTools(pi.getAllTools(), settings);
|
|
100
|
+
|
|
109
101
|
return runLibrarian({
|
|
110
102
|
query: params.query,
|
|
111
103
|
repos: params.repos ?? [],
|
|
112
104
|
owners: params.owners ?? [],
|
|
105
|
+
continueFrom: params.continue_from,
|
|
113
106
|
model: resolution.model,
|
|
114
107
|
thinkingLevel: resolution.thinkingLevel,
|
|
115
108
|
settings,
|
|
109
|
+
extraTools,
|
|
116
110
|
githubClient,
|
|
117
111
|
signal,
|
|
118
112
|
onUpdate: onUpdate
|
|
119
113
|
? (details) => {
|
|
120
114
|
onUpdate({
|
|
121
|
-
content: [
|
|
115
|
+
content: [
|
|
116
|
+
{
|
|
117
|
+
type: "text",
|
|
118
|
+
text: `Researching: ${shorten(params.query, 80)}`,
|
|
119
|
+
},
|
|
120
|
+
],
|
|
122
121
|
details,
|
|
123
122
|
});
|
|
124
123
|
}
|
|
@@ -177,5 +176,10 @@ export default function librarianExtension(pi: ExtensionAPI): void {
|
|
|
177
176
|
|
|
178
177
|
pi.on("session_start", async (_event: SessionStartEvent, ctx) => {
|
|
179
178
|
applyAttachState(pi, readAttachState(ctx));
|
|
179
|
+
|
|
180
|
+
const warnings = await collectExtraToolWarnings(pi.getAllTools(), settings);
|
|
181
|
+
for (const warning of warnings) {
|
|
182
|
+
ctx.ui.notify(warning.message, "warning");
|
|
183
|
+
}
|
|
180
184
|
});
|
|
181
185
|
}
|